await goods_item.save() if goods.add_num > 0: await Goods_Tmp.del_by('goods_id=?', goods.id) tmp = Goods_Tmp() tmp.goods_id = goods.id tmp.add_num = goods.add_num tmp.sell_num = goods.sell_num tmp.edit_time = datetime.datetime.now() await tmp.save() # 抓取top100 if __name__ == '__main__': start = datetime.datetime.now() loop = asyncio.get_event_loop() loop.run_until_complete(orm.create_pool(loop=loop, **configs.db)) # 头条榜单 top100 activity_id = 1559802426256 _ = tools.get_random_num(12) json = loop.run_until_complete(tools.get_activity_by_id(activity_id, _)) if json is None: sys.exit() if json.get('data') is None or len(json.get('data')) <= 0: sys.exit() if json.get('data').get('cards_list') is None or len(json.get('data').get('cards_list')) <= 0: sys.exit() material_ids = [] items = json.get('data').get('cards_list') for item in items: cardType = item.get('cardType')
def run(self): loop = asyncio.new_event_loop() loop.run_until_complete(orm.create_pool(loop=loop, **configs.db)) while True: # 判断栈是否为空 if self.queue.empty(): # 栈空 线程进入等待 self.event.wait(10) # if self.q_shops.empty() and self.queue.empty(): # self.event.set() # break # 线程唤醒后将flag设置为False if self.event.isSet(): self.event.clear() else: # 判断栈是否已满,为满则在向栈取数据后,则将Flag设置为True, # 唤醒前所有在等待的生产者线程 if self.queue.full(): # # 未满 向栈添加数据 # # self.lock.acquire() # data = self.queue.get() # # 请求数据库 # self.loop.run_until_complete(self.exec_data(data)) # self.queue.task_done() # # self.lock.release() # # 将Flag设置为True # self.event.set() tasks = [] for i in range(self.queue.qsize() - 50): item = self.queue.get() tasks.append( asyncio.ensure_future( test.exec_data(item, self.cids))) dones, pendings = loop.run_until_complete( asyncio.wait(tasks)) print("完成的任务数:%s" % len(dones)) self.queue.task_done() self.event.set() else: if self.queue.qsize() >= 100: tasks = [] for i in range(100): item = self.queue.get() tasks.append( asyncio.ensure_future( test.exec_data(item, self.cids))) dones, pendings = loop.run_until_complete( asyncio.wait(tasks)) print("完成的任务数:%s" % len(dones)) self.queue.task_done() else: # 未满 向栈添加数据 # self.lock.acquire() item = self.queue.get() tasks = [] tasks.append( asyncio.ensure_future( test.exec_data(item, self.cids))) dones, pendings = loop.run_until_complete( asyncio.wait(tasks)) print("完成的任务数:%s" % len(dones)) self.queue.task_done() # self.lock.release() print(self.name + "结束")