def run(self): cx, cu = self.db_connection() pool = ThreadPool(size=20) pool.start() file_submission_id = open(FILE_SUBMISSION_ID) finished_submissions = [int(item[0]) for item in cu.execute("select submission_id from code")] all_submissions = [int(item) for item in file_submission_id.readlines()] for line in list(set(all_submissions).difference(set(finished_submissions))): sleep(0.2) pool.append_job(s.job, line) pool.join() pool.stop()
d = DBHandler() s = Spider() user_list = d.get_user_list() pool = ThreadPool(size=10) pool.start() def add_username(func, username, oj_username): data = func(oj_username) data["username"] = username return data for user in user_list: pool.append_job(add_username, s.bestcoder, user[0], user[1]) pool.append_job(add_username, s.codefoces, user[0], user[2]) pool.append_job(add_username, s.hduoj, user[0], user[3]) pool.join() pool.stop() while not pool.results.empty(): result = pool.results.get() username = result["username"] kwargs = {"bestcoder": {}, "codeforces": {}, "hduoj": {}} kwargs[result["website"]] = result d.save_user_info(username, **kwargs)
#Version : 1.0.0.1 #usage : # # ############################################################### #!/usr/bin/python # -*- coding: UTF-8 -*- import sys, os import time import threading from thread_pool import ThreadPool from log import log, set_log_path def call_cmd(num, num2): log(threading.current_thread().getName(), ":", time.time(), num, num2) os.system('cmd /c "echo ' + str(num) + ' & pause"') ############################# main ################################## if __name__ == '__main__': set_log_path("log.log") thread_pool = ThreadPool(call_cmd, 3) #thread_pool.start() for i in range(10): thread_pool.call(i, i) thread_pool.stop() log("")