def run(self): cx, cu = self.db_connection() pool = ThreadPool(size=20) pool.start() file_submission_id = open(FILE_SUBMISSION_ID) finished_submissions = [int(item[0]) for item in cu.execute("select submission_id from code")] all_submissions = [int(item) for item in file_submission_id.readlines()] for line in list(set(all_submissions).difference(set(finished_submissions))): sleep(0.2) pool.append_job(s.job, line) pool.join() pool.stop()
class Listener(): def __init__(self, redis_conn, channels): self.redis_conn = redis_conn self.pubsub = self.redis_conn.pubsub() self.pubsub.subscribe(channels) self.thread_pool = ThreadPool(size=10) def work(self, item): # 修改成你的代码逻辑 print item["channel"], item["data"] def run(self): self.thread_pool.start() for item in self.pubsub.listen(): self.thread_pool.append_job(self.work, item)
d = DBHandler() s = Spider() user_list = d.get_user_list() pool = ThreadPool(size=10) pool.start() def add_username(func, username, oj_username): data = func(oj_username) data["username"] = username return data for user in user_list: pool.append_job(add_username, s.bestcoder, user[0], user[1]) pool.append_job(add_username, s.codefoces, user[0], user[2]) pool.append_job(add_username, s.hduoj, user[0], user[3]) pool.join() pool.stop() while not pool.results.empty(): result = pool.results.get() username = result["username"] kwargs = {"bestcoder": {}, "codeforces": {}, "hduoj": {}} kwargs[result["website"]] = result d.save_user_info(username, **kwargs)
for i in range(0, len(item)): c = item[i].decode("gb2312") if i == 0: l.append(c) else: if c[0] == "&": l.append(0) else: l.append(1) rooms.append(l) with open( "data/" + campus + "." + building + "." + week + "." + week_day + ".json", "w") as f: f.write(json.dumps(rooms)) print "finish: week:" + week + " week_day:" + week_day return "success" if __name__ == "__main__": s = Spider() s.cookies = {"JSESSIONID": "8B7DA565F71772D37B04170241A757A8.TAB2;"} pool = ThreadPool(size=20) pool.start() for week in range(1, 21): for week_day in range(1, 8): print "start week:" + str(week) + " week_day:" + str(week_day) # 请自行确定info.py中的校区id和教学楼id是正确的 # 然后按照info.py中的数据修改校区和教学楼id pool.append_job(s.craw, "1709", "1783", str(week), str(week_day)) pool.join()
l = [] for i in range(0, len(item)): c = item[i].decode("gb2312") if i == 0: l.append(c) else: if c[0] == "&": l.append(0) else: l.append(1) rooms.append(l) f = open("data/" + campus + "." + building + "." + week + "." + week_day + ".json", "w") f.write(json.dumps(rooms)) f.close() print "finish: week:" + week + " week_day:" + week_day return "success" if __name__ == "__main__": s = Spider() s.cookies = {"JSESSIONID": "8B7DA565F71772D37B04170241A757A8.TAB2;"} pool = ThreadPool(size=20) pool.start() for week in range(1, 21): for week_day in range(1, 8): print "start week:" + str(week) + " week_day:" + str(week_day) # 请自行确定info.py中的校区id和教学楼id是正确的 # 然后按照info.py中的数据修改校区和教学楼id pool.append_job(s.craw, "1709", "1783", str(week), str(week_day)) pool.join()