class BBQ(object): def __init__(self,t=180,count=8): self.time = float(t) self.count = int(count) self.ThreadPool = ThreadPool(self.count) ''' 实际处理烧烤任务的函数 ''' def handle(self,task): time.sleep(self.time)#模拟烧烤时间 try: task[0] = True except: pass return ''' 添加一个烧烤任务 task格式:[True/False], True代表处理完成 False 代表等待处理 ''' def addTask(self,task): self.ThreadPool.addTask(self.handle,task)
channels = list(db.channels.find()) client.disconnect() # probably not the most efficient, but the mongo client isn't thread safe work_queue = Queue.Queue() finish_events = [] tp = ThreadPool(multiprocessing.cpu_count(), queue_size=0, wait_timeout=1) ii = ItemInserter(host, port, db_name, 'items', 'channels', work_queue, finish_events) for channel in channels: mod_date = None try: etag = channel['etag'] except KeyError: etag = None try: mod_date = channel['last_modified'] except KeyError: mod_date = None e = threading.Event() fp = ChannelFetcherParser(channel['url'], mod_date, etag, work_queue, e) ii._finished.append(e) tp.addTask(fp) tp.addTask(ii) work_queue.join() tp.cleanUpThreads()