def stage_listings(batch_id, api_instance, api_parameters):
counter = 0
insert_list = []
for listing in api_instance.property_listings(**api_parameters):
row_fields = []
for field in Parameters.FIELD_LIST:
if hasattr(listing, field):
field_value = getattr(listing, field)
if isinstance(field_value, str):
field_value = field_value.replace('\'','^')
row_fields.append(field_value)
else:
row_fields.append('NULL')
insert_list.append(Parameters.SQL_STAGE_LISTING_VALUES.format(batch_id, *row_fields))
if len(insert_list)>Parameters.SQL_INSERT_BUFFER:
sql = Parameters.SQL_STAGE_LISTING_HEADER + ','.join(insert_list)
run_sql(sql)
insert_list = []
sql = Parameters.SQL_STAGE_LISTING_HEADER + ','.join(insert_list)
run_sql(sql)
def do_GET(self):
path = self.path
if path.startswith('/confessio'):
query = urlparse(path).query
q = dict(qc.split("=") for qc in query.split("&"))
sql = ' '.join([
'SELECT row, `column`, word.word, count(*), confessio.entry, word.id',
'FROM confessio LEFT JOIN word ON confessio.word = word.id',
'LEFT JOIN entry ON confessio.word = entry.word',
'WHERE confessio.section = %s GROUP BY row, `column`'
])
response = run_sql(sql, q['section'])
self.send_response(200)
self.end_headers()
self.wfile.write(json.dumps(response).encode('utf-8'))
elif path.startswith('/entry'):
query = urlparse(path).query
q = dict(qc.split("=") for qc in query.split("&"))
sql = ' '.join([
'SELECT id, property, forms, etymology, content, word',
'FROM entry LEFT JOIN definition ON entry.id = definition.eid',
'WHERE word = %s'
])
response = run_sql(sql, q['word'])
self.send_response(200)
self.end_headers()
self.wfile.write(json.dumps(response).encode('utf-8'))
else:
super().do_GET()
def update_user_contest(contest_id, user_id, use_total_time):
protect.dblock.acquire()
sql = "update django_web_user_contest set ac_num = ac_num + 1 , total_time = total_time + " + str(
use_total_time) + " where contest_id = " + str(
contest_id) + " and user_id = " + str(user_id)
db.run_sql(sql)
protect.dblock.release()
def update_punish_time(contest_id, user_id, problem_id):
protect.dblock.acquire()
sql = "update django_web_user_contest_problem set punish_time = punish_time + 1 where contest_id = " + str(
contest_id) + " and user_id = " + str(
user_id) + " and problem_id = " + str(problem_id)
db.run_sql(sql)
protect.dblock.release()
def Watcher():
global TaskCount
while True:
time.sleep(3)
try:
can = db.config.count_thread - TaskCount
if can == 0:
continue
Log('[I] => Have %d idlei thread ...' % can)
res = getList(can)
for item in res:
Mutex.acquire()
TaskCount += 1
Mutex.release()
uu = []
for i in range(0, len(db.config.poj_user)):
if db.config.poj_user[i][2] != True:
uu = db.config.poj_user[i]
db.config.poj_user[i][2] = True
break
db.run_sql(
"update Record set result='Waiting' where `id`='%s'" %
str(item[0]))
thread.start_new_thread(Worker, (item, uu[0], uu[1], i))
except Exception, e:
print e
def update_result(pi):
sql = "update solution set time = " + str(pi['take_time']) + ",memory = " + str(
pi['take_memory']) + ",result=" + str(pi['result']) + " where solution_id =" + str(pi['solution_id']) + ";"
run_sql(sql)
if str(pi['result']) == '1':
sql = "update problem set accepted =accepted +1 where problem_id =" + str(pi['problem_id'])
run_sql(sql)
def write_output(submission_id): '''将输出写入数据库，判题结束''' output_path = os.path.join(config.work_dir, str(submission_id), '%s.out' % str(submission_id)) output_file = open(output_path, 'r') output = output_file.read() sql = 'UPDATE ' + config.db_table_name + ' SET output = \'' + output + '\' status = \'' + status.DONE + '\' WHERE submission_id = ' + str(submission_id) + ';' db.run_sql(sql)
def update_sim(s_id, sim_s_id, sim):
# sql功能：存在该条记录时则更新该记录，不存在则插入;仅仅插入会导致rejudge时出错
sql = "INSERT INTO sdustoj.sim(s_id, sim_s_id, sim)" \
" VALUES (" + str(s_id) + ", " + str(sim_s_id) + ", " + str(sim) + \
") ON DUPLICATE KEY UPDATE sim_s_id = " + str(sim_s_id) + ", sim = " + str(sim) + ";"
# sql_for_dev = "UPDATE sdustoj.sim " \
# "SET sim_s_id = " + str(sim_s_id) + ", sim = " + str(sim) + " WHERE s_id = " + str(s_id) + ";"
run_sql(sql)
def update_result(result):
protect.dblock.acquire()
sql = "update django_web_submitproblem set take_time= " + str(
result["take_time"]) + " ,take_memory=" + str(
result["take_memory"]) + ", result=" + str(
result["result"]) + " where submit_id = " + str(
result["submit_id"])
db.run_sql(sql)
protect.dblock.release()
def update_user_contest_problem(contest_id, user_id, problem_id, use_time,
is_fac):
protect.dblock.acquire()
# sql = "insert into django_web_user_contest_problem(use_time,punish_time,contest_id,problem_id,user_id) values(%s,%s,%s,%s,%s)";
# data = (time_spent, punish_num, contest_id, problem_id, user_id)
# db.InsertData(sql, data)
sql = "update django_web_user_contest_problem set use_time = " + str(
use_time) + ",first_ac = " + str(
is_fac) + ",is_true = 1 where contest_id = " + str(
contest_id) + " and user_id = " + str(
user_id) + " and problem_id = " + str(problem_id)
db.run_sql(sql)
protect.dblock.release()
def add_images(files):
from util.files import strip_files, strtodatetime
stripped = strip_files(files)
for i, (f, s) in enumerate(zip(files, stripped)):
if not i % 1000:
print i
date, cam = s.lower().split(".")
date = strtodatetime(date)
sql = "insert ignore into image (filename, basename, cam_id, time) " \
"select '%s', '%s', id, '%s' from cam where name='%s'" % \
(f, s, date, cam)
run_sql(sql)
def add_images(files):
from util.files import strip_files, strtodatetime
stripped = strip_files(files)
for i, (f, s) in enumerate(zip(files, stripped)):
if not i%1000:
print i
date, cam = s.lower().split(".")
date = strtodatetime(date)
sql = "insert ignore into image (filename, basename, cam_id, time) " \
"select '%s', '%s', id, '%s' from cam where name='%s'" % \
(f, s, date, cam)
run_sql(sql)
def update_result(result, user, problem):
sql = "update Record set `result`='%s',`long`='%dMS',`memory`='%dK' where `id`='%s'" % (config.re_result_code[result['result']], result['take_time'], result['take_memory'], result['solution_id'])
run_sql(sql)
if result['result']== 1:
t_res = run_sql("select distinct oid from Record where `user`='%s' and result='Accepted'" % user)
t_res = map(lambda ptr:ptr[0], t_res)
run_sql("update Users set `plist`='%s',`ac`='%d' where `user`='%s'" % (' '.join(t_res), len(t_res), user))
logging.info('Solved problem list updated')
run_sql('update AJC_Problem set accepted=accepted+1,submissions=submissions+1 where id=%s' % problem)
else:
run_sql('update AJC_Problem set submissions=submissions+1 where id=%s' % problem)
def update_result(result, user, problem):
sql = "update Record set `result`='%s',`long`='%dMS',`memory`='%dK' where `id`='%s'" % (config.re_result_code[result['result']], result['take_time'], result['take_memory'], result['solution_id'])
run_sql(sql)
if result['result']== 1:
t_res = run_sql("select distinct oid from Record where `user`='%s' and result='Accepted'" % user)
t_res = map(lambda ptr:ptr[0], t_res)
run_sql("update Users set `plist`='%s',`ac`='%d' where `user`='%s'" % (' '.join(t_res), len(t_res), user))
logging.info('Solved problem list updated')
run_sql('update AJC_Problem set accepted=accepted+1,submissions=submissions+1 where id=%s' % problem)
else:
run_sql('update AJC_Problem set submissions=submissions+1 where id=%s' % problem)
def get_sim(solution_id, language, problem_id):
file_ext = {
"gcc": ".c",
"g++": ".cpp",
"java": ".java",
'ruby': ".rb",
"perl": ".pl",
"pascal": ".pas",
"go": ".go",
"lua": ".lua",
'python2': '.py',
'python3': '.py',
"haskell": ".hs"
}
# if language == "g++":
# language = "gcc"
file_name = str(solution_id) + file_ext[language]
src_path = os.path.join(config.sim_dir, str(problem_id), 'ac', file_name)
sim = subprocess.Popen('sim.sh %s %d' % (src_path, problem_id), shell=True, stdout=subprocess.PIPE)
result = ''
while sim.poll() is None:
line = sim.stdout.readline()
# print line
result = result + line
# print result
if result != '' and result is not None:
result_num = re.findall(r'\d+', result) # 提取出所有数字[sim_1, sim_s_id_1, sim_2, sim_s_id_2,...]
print result_num
if len(result_num) != 0: # 更改solution表的is_sim字段值
sql_issim = "UPDATE solution SET is_sim = 1 WHERE solution_id = " + str(solution_id)
dblock.acquire()
run_sql(sql_issim)
dblock.release()
sql = "INSERT INTO sdustoj.sim(s_id, sim, sim_s_id) VALUES "
# max_sim = -1
# max_sim_s_id = -1
for i in range(0, len(result_num), 2):
sql += "(" + str(solution_id) + "," + str(result_num[i]) + "," + str(result_num[i + 1]) + "), "
# if int(result_num[i]) > max_sim:
# max_sim = int(result_num[i])
# max_sim_s_id = int(result_num[i+1])
sql += "(-1,-1,-1) " # 作用：消除for循环中sql最后的逗号
sql += "ON DUPLICATE KEY UPDATE sim = VALUES(sim)"
print sql
dblock.acquire()
# update_sim(solution_id, max_sim_s_id, max_sim) # sim is written into DB
run_sql(sql)
dblock.release()
def get_input(submission_id):
'''从数据库获取input并写入data目录下对应的文件'''
sql= 'SELECT input FROM ' + config.db_table_name + ' WHERE submission_id = ' + str(submission_id) + ';'
data = db.run_sql(sql)
input_data = data[0][0]
file_name = str(submission_id) + '.in'
try:
data_path = os.path.join(config.data_dir, str(submission_id))
secure.low_level()
os.mkdir(data_path)
except OSError as e:
if str(e).find('exist') > 0: # 文件夹已经存在
pass
else:
return False
try:
real_path = os.path.join(config.data_dir, str(submission_id), file_name)
except KeyError as e:
return False
try:
log.log_info('Importing input, %d' % submission_id)
secure.low_level()
f = open(real_path, 'w')
try:
f.write(input_data)
except:
f.close()
log.log_error('Importing input failed, %d' % submission_id)
return False
f.close()
except OSError as e:
return False
return True
def get_ac_code_to_sim(solution_id, problem_id, language):
logging.info("--------------start get_ac_code_to_sim---------------")
file_ext = {
"gcc": ".c",
"g++": ".cpp",
"java": ".java",
'ruby': ".rb",
"perl": ".pl",
"pascal": ".pas",
"go": ".go",
"lua": ".lua",
'python2': '.py',
'python3': '.py',
"haskell": ".hs"
}
# <==> create the dir for solution_id to SIM
try:
sim_path = os.path.join(config.sim_dir, str(problem_id), "ac") # /usr/local/sdustoj/sim_dir/problem_id/ac/
low_level()
os.makedirs(sim_path)
logging.info("makedirs successful")
except OSError as e:
if str(e).find("exist") > 0: # 文件夹已经存在
pass
else:
logging.error(e)
return False
sql_get_code = "SELECT solution_id,source FROM sdustoj.source_code WHERE solution_id = " + str(solution_id)
feh = run_sql(sql_get_code) # get solution_id's code and other solutions' codes needed to compare
# logging.info("feh --%s" % feh)
if feh is not None:
try:
s_id = feh[0][0] # solution_id of every code
code = feh[0][1]
file_name = str(s_id) + file_ext[language] # file name of every code
logging.info("%s" % file_name)
try:
real_path = os.path.join(config.sim_dir, str(problem_id), "ac", file_name)
except OSError as e:
logging.error(e)
return False
try:
low_level()
f = codecs.open(real_path, 'w')
try:
f.write(code)
except:
logging.error("code of %s isn't written into file" % s_id)
f.close()
return False
f.close()
except OSError as e:
logging.error(e)
return False
except:
logging.error("cannot get codes of runid %s to SIM" % solution_id)
return False
return True
def get_result_code():
sql = "select id,status from django_web_result"
res = db.run_sql(sql)
global result_code
result_code = dict()
for id, status in res:
result_code[status] = id
def get_lans():
sql = "select id,lan,filename,buildcmd from django_web_lang"
res = db.run_sql(sql)
global lans
lans = dict()
for id, lan, filename, cmd in res:
lans[id] = (lan, filename, cmd)
def get_punish_time(contest_id):
protect.dblock.acquire()
sql = "select punish_time from django_web_contest where contest_id = " + str(
contest_id)
res = db.run_sql(sql)
protect.dblock.release()
return res[0][0]
def query(fields,
cam=None,
dates=None,
exp=None,
condition=None,
has_stars=None,
image_id=None,
star_id=None):
fields = _uniq(fields)
if condition is None:
condition = []
elif type(condition) == str:
condition = condition.split(" and ")
conditions, tables = _filter(cam, dates, exp, has_stars, image_id, star_id)
conditions = conditions + condition
tables += [field.split(".")[0] for field in fields]
tables += [_table_from_condition(cond) for cond in conditions]
tables = [t.strip() for t in tables]
table = _join_tables(tables)
fields = ", ".join(fields)
if conditions == []:
condition = "1"
else:
condition = " and ".join(conditions)
sql = "select %s from %s where %s" % (fields, table, condition)
return run_sql(sql)
def query(fields, cam=None, dates=None, exp=None, condition=None, has_stars=None, image_id=None, star_id=None):
fields = _uniq(fields)
if condition is None:
condition = []
elif type(condition) == str:
condition = condition.split(" and ")
conditions, tables = _filter(cam, dates, exp, has_stars, image_id, star_id)
conditions = conditions + condition
tables += [field.split(".")[0] for field in fields]
tables += [_table_from_condition(cond) for cond in conditions]
tables = [t.strip() for t in tables]
table = _join_tables(tables)
fields = ", ".join(fields)
if conditions == []:
condition = "1"
else:
condition = " and ".join(conditions)
sql = "select %s from %s where %s" % (fields, table, condition)
return run_sql(sql)
def get_problem_limit(problem_id):
protect.dblock.acquire()
sql = "select time_limit,mem_limit from django_web_problem where problem_id = " + str(
problem_id)
res = db.run_sql(sql)
protect.dblock.release()
return res[0]
def Worker(item, oj_user, oj_pass, index):
global TaskCount, TryResult
Log('[I] => Thread %d processing %s using %s ...' % (index, item[0], oj_user))
# Set cookies
cookie = ''
# Login
Log('[I] => Thread %d step %d ...' % (index, 1))
try:
t, cookie = CurlPOST("http://poj.org/login", {
"user_id1" : oj_user,
"password1" : oj_pass
}, cookie)
except Exception,e:
Log('[E] => Login failed !' + str(e))
db.run_sql("update Record set `rid`='NONE',`memory`='0K',`long`='0MS',`lang`='Unknown',`result`='System Error' where `id`='%s'" % item[0])
ExitThread(index, cookie)
thread.exit_thread()
def is_first_ac(contest_id, problem_id):
protect.dblock.acquire()
sql = "select count(*) from django_web_user_contest_problem where contest_id = " + str(
contest_id) + " and problem_id = " + str(
problem_id) + " and first_ac = 1"
res = db.run_sql(sql)
protect.dblock.release()
return (res[0][0] + 1) % 2
def get_phot_data(star, keys, clauses=None):
format = ", ".join(["%s"] * len(keys))
sql = ("SELECT %s from phot where id=%d" % (format, star)) % keys
if clauses:
sql += " and " + " and ".join(clauses)
return run_sql(sql)
def get_phot_data(star, keys, clauses=None):
format = ", ".join(["%s"] * len(keys))
sql = ("SELECT %s from phot where id=%d" % (format, star)) % keys
if clauses:
sql += " and " + " and ".join(clauses)
return run_sql(sql)
def get_code(solution_id, problem_id, pro_lang):
'''从数据库获取代码并写入work目录下对应的文件'''
file_name = {
"gcc": "main.c",
"g++": "main.cpp",
"java": "Main.java",
'ruby': "main.rb",
"perl": "main.pl",
"pascal": "main.pas",
"go": "main.go",
"lua": "main.lua",
'python2': 'main.py',
'python3': 'main.py',
"haskell": "main.hs"
}
select_code_sql = "select source from sdustoj.source_code where solution_id = "+str(solution_id)
feh = run_sql(select_code_sql)
if feh is not None:
try:
code = feh[0][0]
except:
logging.error("1 cannot get code of runid %s" % solution_id)
return False
else:
logging.error("2 cannot get code of runid %s" % solution_id)
return False
try:
work_path = os.path.join(config.work_dir, str(solution_id))
low_level()
os.makedirs(work_path)
except OSError as e:
if str(e).find("exist") > 0: # 文件夹已经存在
pass
else:
logging.error(e)
return False
try:
real_path = os.path.join(
config.work_dir,
str(solution_id),
file_name[pro_lang])
except KeyError as e:
logging.error(e)
return False
try:
low_level()
f = codecs.open(real_path, 'w')
try:
f.write(code)
except:
logging.error("%s not write code to file" % solution_id)
f.close()
return False
f.close()
except OSError as e:
logging.error(e)
return False
return True
def producer(q, dblock):
"""循环扫描数据库,将任务添加到队列"""
# print("start")
while True:
if q.full():
q.join()
dblock.acquire()
data = run_sql('select * from submission where message is NULL')
# print(data)
time.sleep(0.5) # 延时1秒,防止因速度太快不能获取代码
dblock.release()
for i in data:
id, language, content, message, user_id, question_id = i
dblock.acquire()
# print('1 put')
ret = get_code(id, question_id, language, content)
dblock.release()
# print(ret)
if ret is False:
# 防止因速度太快不能获取代码
# print('2 put')
time.sleep(1)
dblock.acquire()
ret = get_code(id, question_id, language, content)
dblock.release()
if ret is False:
dblock.acquire()
# 标记题目写入处理的结果信息
run_sql(
"update submission set message = 'unknown error' where id = %s"
% id)
dblock.release()
clean_work_dir(id)
continue
run_sql("update submission set message = 'success' where id = %s" %
id)
task = {
"id": id,
"user_id": user_id,
"question_id": question_id,
"content": content,
"language": language,
}
q.put(task)
time.sleep(0.5)
def is_not_accepted(user_id, problem_id, result_code, submit_id):
protect.dblock.acquire()
sql = "select count(*) from django_web_submitproblem where user_id = " + str(
user_id) + " and problem_id = " + str(
problem_id) + " and result = " + str(
result_code) + " and submit_id != " + str(submit_id)
res = db.run_sql(sql)
protect.dblock.release()
return res[0][0] == 0
def get_punish_num(user_id, contest_id, problem_id):
protect.dblock.acquire()
# sql = "select count(*) from django_web_submitproblem where user_id = " + str(user_id) + " and problem_id = " + str(
# problem_id) + " and result_id != " + str(result_code) + " and contest_id = " + str(contest_id)
sql = "select punish_time from django_web_user_contest_problem where user_id = " + str(
user_id) + " and contest_id = " + str(
contest_id) + " and problem_id = " + str(problem_id)
res = db.run_sql(sql)
protect.dblock.release()
return res[0][0]
def get_problem_limit(problem_id):
select_sql = "select `time`,`memory` from `AJC_Problem` where `id`='%s'" % problem_id
feh = run_sql(select_sql)
if feh is not None:
try:
time_t,memory_t = feh[0]
except:
logging.error("1 cannot get code of runid %s" % solution_id)
return False
return time_t,memory_t
def put_task_into_queue():
'''扫描数据库,将任务添加到队列'''
log.log_info('Scaning DB...')
while True:
init.queue.join() # 阻塞程序
sql = 'SELECT submission_id FROM ' + config.db_table_name + ' WHERE status = ' + status.QUEUING + ';'
data = db.run_sql(sql)
num = len(data)
if num > 0:
log.log_info('%d submission(s) coming...' % num)
for i in range(num):
time.sleep(1)
submission_id = data[i][0]
init.dblock.acquire()
ret = get_data.get_code(submission_id) and get_data.get_input(submission_id)
init.dblock.release()
if ret == False:
init.dblock.acquire()
update.update_status(submission_id, status.GET_DATA_ERROR)
init.dblock.release()
continue
inner_sql = 'SELECT language, time, memory FROM ' + config.db_table_name + ' WHERE submission_id = ' + str(submission_id) + ';'
#print inner_sql
inner_data = db.run_sql(inner_sql)
language_id = inner_data[i][0]
Time = inner_data[i][1]
memory = inner_data[i][2]
task = {
'submission_id': submission_id,
'language_id': language_id,
'time': Time,
'memory': memory
}
init.queue.put(task)
time.sleep(1)
def is_not_Accepted_Contest(user_id, contest_id, problem_id, result_code):
protect.dblock.acquire()
# sql = "select count(*) from django_web_submitproblem where user_id = " + str(user_id) + " and problem_id = " + str(
# problem_id) + " and result_id = " + str(result_code) + " and contest_id = " + str(contest_id)
sql = "select is_true from django_web_user_contest_problem where contest_id = " + str(
contest_id) + " and user_id = " + str(
user_id) + " and problem_id = " + str(problem_id)
res = db.run_sql(sql)
protect.dblock.release()
return res[0][0] == 0
def get_problem_limit(problem_id):
select_sql = "select `time`,`memory` from `AJC_Problem` where `id`='%s'" % problem_id
feh = run_sql(select_sql)
if feh is not None:
try:
time_t,memory_t = feh[0]
except:
logging.error("1 cannot get code of runid %s" % solution_id)
return False
return time_t,memory_t
def Worker(item, oj_user, oj_pass, index):
global TaskCount, TryResult
Log('[I] => Thread %d processing %s using %s ...' %
(index, item[0], oj_user))
# Set cookies
cookie = ''
# Login
Log('[I] => Thread %d step %d ...' % (index, 1))
try:
t, cookie = CurlPOST("http://poj.org/login", {
"user_id1": oj_user,
"password1": oj_pass
}, cookie)
except Exception, e:
Log('[E] => Login failed !' + str(e))
db.run_sql(
"update Record set `rid`='NONE',`memory`='0K',`long`='0MS',`lang`='Unknown',`result`='System Error' where `id`='%s'"
% item[0])
ExitThread(index, cookie)
thread.exit_thread()
def fetch_title(proxy: urllib3.ProxyManager, title_id) -> dict:
url = f"https://mangadex.org/api/v2/manga/{title_id}"
scrape_id = db.run_sql(
"INSERT INTO scrape (proxy, url) VALUES (?, ?)",
(proxy.proxy_url, url),
return_last_insert_rowid=True,
)
resp = proxy.request("GET", url)
assert resp.status in [200, 404], resp.data
db.run_sql(
"""
UPDATE scrape
SET resp_status = ?,
resp_body = ?,
ended_at = datetime('now')
WHERE id = ?;
""",
(resp.status, resp.data, scrape_id),
)
print("Saved title", title_id, "-", resp.status)
def Watcher():
global TaskCount
while True:
time.sleep(3)
try:
can = db.config.count_thread - TaskCount
if can == 0:
continue
Log('[I] => Have %d idlei thread ...' % can)
res = getList(can)
for item in res:
Mutex.acquire()
TaskCount += 1
Mutex.release()
uu = []
for i in range(0, len(db.config.poj_user)):
if db.config.poj_user[i][2] != True:
uu = db.config.poj_user[i]
db.config.poj_user[i][2] = True
break
db.run_sql("update Record set result='Waiting' where `id`='%s'" % str(item[0]))
thread.start_new_thread(Worker, (item, uu[0], uu[1], i))
except Exception,e:
print e
def put_task_into_queue():
while True:
q.join()
data = run_sql("select `id`,`tid`,`user`,`contest`,`lang`,`code` from Record where `rid`='__' and oj='AJC'")
time.sleep(0.2)
for i in data:
solution_id, problem_id, user_id, contest_id, pro_lang, code = i
put_code(solution_id, problem_id, pro_lang, code)
task = {
"solution_id": solution_id,
"problem_id": problem_id,
"contest_id": contest_id,
"user_id": user_id,
"pro_lang": pro_lang,
}
q.put(task)
update_solution_status(solution_id)
time.sleep(0.5)
def put_task_into_queue():
while True:
q.join()
data = run_sql("select `id`,`tid`,`user`,`contest`,`lang`,`code` from Record where `rid`='__' and oj='AJC'")
time.sleep(0.2)
for i in data:
solution_id, problem_id, user_id, contest_id, pro_lang, code = i
put_code(solution_id, problem_id, pro_lang, code)
task = {
"solution_id": solution_id,
"problem_id": problem_id,
"contest_id": contest_id,
"user_id": user_id,
"pro_lang": pro_lang,
}
q.put(task)
update_solution_status(solution_id)
time.sleep(0.5)
def put_task_into_queue():
'''循环扫描数据库,将任务添加到队列'''
while True:
protect.q.join() # 阻塞程序,直到队列里面的任务全部完成
sql = "select submit_id, problem_id, user_id, contest_id, `language` from django_web_submitproblem where result = 0"
data = db.run_sql(sql)
time.sleep(0.5) # 延时0.5秒,防止因速度太快不能获取代码
for i in data:
submit_id, problem_id, user_id, contest_id, language = i
protect.dblock.acquire()
ret = get_code.get_code(submit_id, problem_id, language)
protect.dblock.release()
if ret is False: # 生成文件失败 再次尝试
# 防止因速度太快不能获取代码
logging.error("首次生成文件失败 by %s" %
threading.current_thread().name)
time.sleep(0.5)
protect.dblock.acquire()
ret = get_code.get_code(submit_id, problem_id, language)
protect.dblock.release()
if ret is False: # 生成文件依旧失败
protect.dblock.acquire()
logging.error("再次生成文件失败 by %s " %
threading.current_thread().name)
update_solution_status(
submit_id, DBData.result_code["System Error"]) # 设置结果为系统错误
protect.dblock.release()
deal_data.clean_work_dir(submit_id) # 清理get_code函数所产生的文件夹及文件
continue
logging.info("生成文件成功 by %s " % threading.current_thread().name)
task = { # 生成判题任务
"submit_id": submit_id,
"problem_id": problem_id,
"contest_id": contest_id,
"user_id": user_id,
"language": language,
}
protect.q.put(task) # 将判题任务添加到判题队列中
protect.dblock.acquire()
update_solution_status(submit_id,
DBData.result_code["Judging"]) # 状态结果改成正在判题
protect.dblock.release()
time.sleep(0.5)
def get_code(submission_id):
'''从数据库获取代码并写入work目录下对应的文件'''
sql = 'SELECT code, language FROM ' + config.db_table_name + ' WHERE submission_id = ' + str(submission_id) + ';'
data = db.run_sql(sql)
if data != False:
code = data[0][0]
language_id = data[0][1]
file_name = language.get_filename(language_id)
else:
return False
try:
work_path = os.path.join(config.work_dir, str(submission_id))
secure.low_level()
os.mkdir(work_path)
except OSError as e:
if str(e).find('exist') > 0: # 文件夹已经存在
pass
else:
return False
try:
real_path = os.path.join(config.work_dir, str(submission_id), file_name)
except KeyError as e:
# print e
return False
try:
log.log_info('Importing code, %d' % submission_id)
secure.low_level()
f = open(real_path, 'w')
try:
f.write(code)
except:
f.close()
log.log_error('Importing code failed, %d' % submission_id)
return False
f.close()
except OSError as e:
return False
return True
def put_task_into_queue():
'''循环扫描数据库,将任务添加到队列'''
while True:
q.join() # 阻塞程序,直到队列里面的任务全部完成
sql = "select solution_id,problem_id,user_id,contest_id,language from solution where result = -1 or result =0 or result is null order by solution_id asc;"
data = run_sql(sql)
time.sleep(0.2) # 延时0.2秒,防止因速度太快不能获取代码
for i in data:
solution_id, problem_id, user_id, contest_id, pro_lang = i
language_list = ["gcc", "g++", "java", "ruby", "perl", "pascal", "go", "lua", "python2", "python3",
"haskell"]
pro_lang = language_list[pro_lang]
dblock.acquire()
ret = get_code(solution_id, problem_id, pro_lang)
dblock.release()
if ret == False:
# 防止因速度太快不能获取代码
time.sleep(0.5)
dblock.acquire()
ret = get_code(solution_id, problem_id, pro_lang)
dblock.release()
if ret == False:
dblock.acquire()
update_solution_status(solution_id, 11)
dblock.release()
clean_work_dir(solution_id)
continue
task = {
"solution_id": solution_id,
"problem_id": problem_id,
"contest_id": contest_id,
"user_id": user_id,
"pro_lang": pro_lang,
}
q.put(task)
dblock.acquire()
update_solution_status(solution_id, 12)
dblock.release()
time.sleep(0.5)
def put_task_into_queue():
'''循环扫描数据库,将任务添加到队列'''
while True:
q.join() # 阻塞程序,直到队列里面的任务全部完成
sql = "select solution_id,problem_id,user_id,contest_id,language from solution where result = -1 or result =0 or result is null order by solution_id asc;"
data = run_sql(sql)
time.sleep(0.2) # 延时0.2秒,防止因速度太快不能获取代码
for i in data:
solution_id, problem_id, user_id, contest_id, pro_lang = i
language_list = [ "gcc","g++","java", "ruby","perl","pascal","go","lua","python2","python3","haskell"]
pro_lang=language_list[pro_lang]
dblock.acquire()
ret = get_code(solution_id, problem_id, pro_lang)
dblock.release()
if ret == False:
# 防止因速度太快不能获取代码
time.sleep(0.5)
dblock.acquire()
ret = get_code(solution_id, problem_id, pro_lang)
dblock.release()
if ret == False:
dblock.acquire()
update_solution_status(solution_id, 11)
dblock.release()
clean_work_dir(solution_id)
continue
task = {
"solution_id": solution_id,
"problem_id": problem_id,
"contest_id": contest_id,
"user_id": user_id,
"pro_lang": pro_lang,
}
q.put(task)
dblock.acquire()
update_solution_status(solution_id,12)
dblock.release()
time.sleep(0.5)
def populate_cam_table():
for cam in ["sbc", "sky"]:
sql = "insert into cam set name='%s'" % cam
run_sql(sql)
def update_compile_info(submission_id, info): '''编译失败时，将编译错误信息写入cmpinfo中''' sql = 'UPDATE ' + config.db_table_name + ' SET cmpinfo = \'' + info + '\' status = \'' + status.COMPILATION_ERROR + '\' WHERE submission_id = ' + str(submission_id) + ';' db.run_sql(sql)
def update_compile_info(solution_id, result):
sql = "update Record set `compileinfo`='%s' where `id`='%s'" % (MySQLdb.escape_string(result), solution_id)
run_sql(sql)
def update_compile_info(solution_id, err):
err = MySQLdb.escape_string(err)
sql = "insert into compileinfo(solution_id,error) values (" + str(solution_id) + ",'" + str(err) + "')" \
"ON DUPLICATE KEY UPDATE error = '" + str(err) + "';"
run_sql(sql)
thread.exit_thread()
# Post code
Log('[I] => Thread %d step %d ...' % (index, 2))
time.sleep(1)
try:
CurlPOST(
"http://poj.org/submit", {
"problem_id": item[3],
"language": item[9],
"encoded": "1",
"source": item[14]
}, cookie)
except Exception, e:
Log('[E] => Submit failed !' + str(e))
db.run_sql(
"update Record set `rid`='NONE',`memory`='0K',`long`='0MS',`lang`='Unknown',`result`='System Error' where `id`='%s'"
% item[0])
ExitThread(index, cookie)
thread.exit_thread()
# Get RunID
Log('[I] => Thread %d step %d ...' % (index, 3))
time.sleep(1)
try:
html, t = CurlGET(
"http://poj.org/status?problem_id=%s&user_id=%s&result=&language=%s"
% (item[3], oj_user, item[9]), cookie)
except Exception, e:
Log('[E] => Get RunID failed !' + str(e))
db.run_sql(
"update Record set `rid`='NONE',`memory`='0K',`long`='0MS',`lang`='Unknown',`result`='System Error' where `id`='%s'"
% item[0])
def getList(len):
res = db.run_sql("select * from Record where `rid`='__' AND `oj`='POJ' AND `result`<>'Waiting' order by time DESC limit 0,%d" % len)
return res
def getList(len):
res = db.run_sql(
"select * from Record where `rid`='__' AND `oj`='POJ' AND `result`<>'Waiting' order by time DESC limit 0,%d"
% len)
return res
db.run_sql("update Record set `rid`='NONE',`memory`='0K',`long`='0MS',`lang`='Unknown',`result`='System Error' where `id`='%s'" % item[0])
ExitThread(index, cookie)
thread.exit_thread()
# Post code
Log('[I] => Thread %d step %d ...' % (index, 2))
time.sleep(1)
try:
CurlPOST("http://poj.org/submit", {
"problem_id" : item[3],
"language" : item[9],
"encoded" : "1",
"source" : item[14]
}, cookie)
except Exception,e:
Log('[E] => Submit failed !' + str(e))
db.run_sql("update Record set `rid`='NONE',`memory`='0K',`long`='0MS',`lang`='Unknown',`result`='System Error' where `id`='%s'" % item[0])
ExitThread(index, cookie)
thread.exit_thread()
# Get RunID
Log('[I] => Thread %d step %d ...' % (index, 3))
time.sleep(1)
try:
html, t = CurlGET("http://poj.org/status?problem_id=%s&user_id=%s&result=&language=%s" % (item[3], oj_user, item[9]), cookie)
except Exception,e:
Log('[E] => Get RunID failed !' + str(e))
db.run_sql("update Record set `rid`='NONE',`memory`='0K',`long`='0MS',`lang`='Unknown',`result`='System Error' where `id`='%s'" % item[0])
ExitThread(index, cookie)
thread.exit_thread()
# Get RunID List
match = re.findall(r'<td>(\d+)</td>', html)
RunID = -1
def update_status(submission_id, status): '''运行过程中，更新数据库status字段''' sql = 'UPDATE ' + config.db_table_name + ' SET status = \'' + str(status) + '\' WHERE submission_id = ' + str(submission_id) + ';' db.run_sql(sql)
import config from db import run_sql select_code_sql = "select status,status_id from status;" result = run_sql(select_code_sql) print result[0][0]
def update_solution_status(solution_id, value):
if value == None:
value = -1
sql = "update solution set result= " + str(value) + " where solution_id = " + str(solution_id) + ";"
run_sql(sql)
def update_users_attrib_ac(user_id):
sql = "UPDATE sdustoj.users SET accepted = accepted + 1 WHERE user_id = '" + str(user_id) + "';"
print sql
run_sql(sql)
def write_result(submission_id, result): '''运行结束，更新相关字段''' sql = 'UPDATE ' + config.db_table_name + ' SET time_used = ' + str(result['take_time']) + ' memory_used = \'' + str(result['take_memory']) + '\' WHERE submission_id = ' + str(submission_id) + ';' db.run_sql(sql)
def get_problem_limit(problem_id):
sql = "select time_limit,memory_limit from problem where problem_id = " + str(problem_id)
result = run_sql(sql)
return result[0][0], result[0][1]
def update_compile_info(solution_id,err):
sql = "insert into compileinfo(solution_id,error) values ("+str(solution_id)+",'"+str(err)+"');"
run_sql(sql)
