def main():
manager = Manager()
lock = manager.Lock()
partial_get_index = functools.partial(get_index, lock)
pool = Pool()
pool.map(partial_get_index, [i for i in range(START_PAGE, END_PAGE + 1)])
pool.close()
pool.join()
print('over')
try:
# 执行sql语句
cursor.executemany(sql, dealdata)
# 提交到数据库执行
db.commit()
except pm.Warning as w:
print(repr(w))
# 如果发生错误则回滚
db.rollback()
if __name__ == '__main__':
# pool = ThreadPool(4)
# page = [x + 1 for x in range(18)]
manager = Manager()
lock = manager.Lock()
# pool = ThreadPool(processes=8)
for i in range(1, 20):
# pool.apply_async(spider, (i, lock))
spider(i)
print(list)
for i in list:
if i == 21:
continue
else:
dealdata(i)
# results = pool.map(spider ,page)
# pool.close()
# pool.join()
# 关闭游标
