def run(self):
cx, cu = self.db_connection()
pool = ThreadPool(size=20)
pool.start()
file_submission_id = open(FILE_SUBMISSION_ID)
finished_submissions = [int(item[0]) for item in cu.execute("select submission_id from code")]
all_submissions = [int(item) for item in file_submission_id.readlines()]
for line in list(set(all_submissions).difference(set(finished_submissions))):
sleep(0.2)
pool.append_job(s.job, line)
pool.join()
pool.stop()
class Listener():
def __init__(self, redis_conn, channels):
self.redis_conn = redis_conn
self.pubsub = self.redis_conn.pubsub()
self.pubsub.subscribe(channels)
self.thread_pool = ThreadPool(size=10)
def work(self, item):
# 修改成你的代码逻辑
print item["channel"], item["data"]
def run(self):
self.thread_pool.start()
for item in self.pubsub.listen():
self.thread_pool.append_job(self.work, item)
d = DBHandler()
s = Spider()
user_list = d.get_user_list()
pool = ThreadPool(size=10)
pool.start()
def add_username(func, username, oj_username):
data = func(oj_username)
data["username"] = username
return data
for user in user_list:
pool.append_job(add_username, s.bestcoder, user[0], user[1])
pool.append_job(add_username, s.codefoces, user[0], user[2])
pool.append_job(add_username, s.hduoj, user[0], user[3])
pool.join()
pool.stop()
while not pool.results.empty():
result = pool.results.get()
username = result["username"]
kwargs = {"bestcoder": {}, "codeforces": {}, "hduoj": {}}
kwargs[result["website"]] = result
d.save_user_info(username, **kwargs)
for i in range(0, len(item)):
c = item[i].decode("gb2312")
if i == 0:
l.append(c)
else:
if c[0] == "&":
l.append(0)
else:
l.append(1)
rooms.append(l)
with open(
"data/" + campus + "." + building + "." + week + "." +
week_day + ".json", "w") as f:
f.write(json.dumps(rooms))
print "finish: week:" + week + " week_day:" + week_day
return "success"
if __name__ == "__main__":
s = Spider()
s.cookies = {"JSESSIONID": "8B7DA565F71772D37B04170241A757A8.TAB2;"}
pool = ThreadPool(size=20)
pool.start()
for week in range(1, 21):
for week_day in range(1, 8):
print "start week:" + str(week) + " week_day:" + str(week_day)
# 请自行确定info.py中的校区id和教学楼id是正确的
# 然后按照info.py中的数据修改校区和教学楼id
pool.append_job(s.craw, "1709", "1783", str(week), str(week_day))
pool.join()
l = []
for i in range(0, len(item)):
c = item[i].decode("gb2312")
if i == 0:
l.append(c)
else:
if c[0] == "&":
l.append(0)
else:
l.append(1)
rooms.append(l)
f = open("data/" + campus + "." + building + "." + week + "." + week_day + ".json", "w")
f.write(json.dumps(rooms))
f.close()
print "finish: week:" + week + " week_day:" + week_day
return "success"
if __name__ == "__main__":
s = Spider()
s.cookies = {"JSESSIONID": "8B7DA565F71772D37B04170241A757A8.TAB2;"}
pool = ThreadPool(size=20)
pool.start()
for week in range(1, 21):
for week_day in range(1, 8):
print "start week:" + str(week) + " week_day:" + str(week_day)
# 请自行确定info.py中的校区id和教学楼id是正确的
# 然后按照info.py中的数据修改校区和教学楼id
pool.append_job(s.craw, "1709", "1783", str(week), str(week_day))
pool.join()
d = DBHandler()
s = Spider()
user_list = d.get_user_list()
pool = ThreadPool(size=10)
pool.start()
def add_username(func, username, oj_username):
data = func(oj_username)
data["username"] = username
return data
for user in user_list:
pool.append_job(add_username, s.bestcoder, user[0], user[1])
pool.append_job(add_username, s.codefoces, user[0], user[2])
pool.append_job(add_username, s.hduoj, user[0], user[3])
pool.join()
pool.stop()
while not pool.results.empty():
result = pool.results.get()
username = result["username"]
kwargs = {"bestcoder": {}, "codeforces": {}, "hduoj": {}}
kwargs[result["website"]] = result
d.save_user_info(username, **kwargs)
