def run(self):
     cx, cu = self.db_connection()
     pool = ThreadPool(size=20)
     pool.start()
     file_submission_id = open(FILE_SUBMISSION_ID)
     finished_submissions = [int(item[0]) for item in cu.execute("select submission_id from code")]
     all_submissions = [int(item) for item in file_submission_id.readlines()]
     for line in list(set(all_submissions).difference(set(finished_submissions))):
         sleep(0.2)
         pool.append_job(s.job, line)
     pool.join()
     pool.stop()
d = DBHandler()
s = Spider()

user_list = d.get_user_list()

pool = ThreadPool(size=10)
pool.start()


def add_username(func, username, oj_username):
    data = func(oj_username)
    data["username"] = username
    return data


for user in user_list:
    pool.append_job(add_username, s.bestcoder, user[0], user[1])
    pool.append_job(add_username, s.codefoces, user[0], user[2])
    pool.append_job(add_username, s.hduoj, user[0], user[3])
pool.join()
pool.stop()


while not pool.results.empty():
    result = pool.results.get()
    username = result["username"]
    kwargs = {"bestcoder": {}, "codeforces": {}, "hduoj": {}}
    kwargs[result["website"]] = result

    d.save_user_info(username, **kwargs)
Beispiel #3
0
d = DBHandler()
s = Spider()

user_list = d.get_user_list()

pool = ThreadPool(size=10)
pool.start()


def add_username(func, username, oj_username):
    data = func(oj_username)
    data["username"] = username
    return data


for user in user_list:
    pool.append_job(add_username, s.bestcoder, user[0], user[1])
    pool.append_job(add_username, s.codefoces, user[0], user[2])
    pool.append_job(add_username, s.hduoj, user[0], user[3])
pool.join()
pool.stop()

while not pool.results.empty():
    result = pool.results.get()
    username = result["username"]
    kwargs = {"bestcoder": {}, "codeforces": {}, "hduoj": {}}
    kwargs[result["website"]] = result

    d.save_user_info(username, **kwargs)
Beispiel #4
0
#Version : 1.0.0.1
#usage :
#
#
###############################################################

#!/usr/bin/python
# -*- coding: UTF-8 -*-

import sys, os
import time
import threading
from thread_pool import ThreadPool
from log import log, set_log_path


def call_cmd(num, num2):
    log(threading.current_thread().getName(), ":", time.time(), num, num2)
    os.system('cmd /c "echo ' + str(num) + ' & pause"')


############################# main ##################################
if __name__ == '__main__':
    set_log_path("log.log")
    thread_pool = ThreadPool(call_cmd, 3)
    #thread_pool.start()
    for i in range(10):
        thread_pool.call(i, i)

    thread_pool.stop()
    log("")