def main(): manager = Manager() lock = manager.Lock() partial_get_index = functools.partial(get_index, lock) pool = Pool() pool.map(partial_get_index, [i for i in range(START_PAGE, END_PAGE + 1)]) pool.close() pool.join() print('over')
try: # 执行sql语句 cursor.executemany(sql, dealdata) # 提交到数据库执行 db.commit() except pm.Warning as w: print(repr(w)) # 如果发生错误则回滚 db.rollback() if __name__ == '__main__': # pool = ThreadPool(4) # page = [x + 1 for x in range(18)] manager = Manager() lock = manager.Lock() # pool = ThreadPool(processes=8) for i in range(1, 20): # pool.apply_async(spider, (i, lock)) spider(i) print(list) for i in list: if i == 21: continue else: dealdata(i) # results = pool.map(spider ,page) # pool.close() # pool.join() # 关闭游标