class BBQ(object):
def __init__(self,t=180,count=8):
self.time = float(t)
self.count = int(count)
self.ThreadPool = ThreadPool(self.count)
'''
实际处理烧烤任务的函数
'''
def handle(self,task):
time.sleep(self.time)#模拟烧烤时间
try:
task[0] = True
except:
pass
return
'''
添加一个烧烤任务
task格式:[True/False],
True代表处理完成
False 代表等待处理
'''
def addTask(self,task):
self.ThreadPool.addTask(self.handle,task)
channels = list(db.channels.find())
client.disconnect() # probably not the most efficient, but the mongo client isn't thread safe
work_queue = Queue.Queue()
finish_events = []
tp = ThreadPool(multiprocessing.cpu_count(), queue_size=0, wait_timeout=1)
ii = ItemInserter(host, port, db_name, 'items', 'channels', work_queue, finish_events)
for channel in channels:
mod_date = None
try:
etag = channel['etag']
except KeyError:
etag = None
try:
mod_date = channel['last_modified']
except KeyError:
mod_date = None
e = threading.Event()
fp = ChannelFetcherParser(channel['url'], mod_date, etag, work_queue, e)
ii._finished.append(e)
tp.addTask(fp)
tp.addTask(ii)
work_queue.join()
tp.cleanUpThreads()
