def run_server():
_parse_opt()
global _options, _use_ipv6, _port
conf.parse_file(_options.conf_file)
if not _options.no_daemon and conf.is_unix:
daemon.daemon_start()
daemon.pid_start()
try:
s = snc.socket(None, _port)
except snc.Error:
daemon.pid_end()
log.info("orzoj-server started, listening on {0}".format(_port))
threading.Thread(target=work.thread_work, name="work.thread_work").start()
while not control.test_termination_flag():
try:
(conn, addr) = s.accept(1)
except snc.ErrorTimeout:
continue
except snc.Error as e:
control.set_termination_flag()
break
log.info("connected by {0!r}".format(addr))
work.thread_new_judge_connection(conn).start()
s.close()
while threading.active_count() > 1:
log.debug("waiting for threads, current active count: {0}".format(
threading.active_count()))
for i in threading.enumerate():
log.debug("active thread: {0!r}".format(i.name))
time.sleep(1)
daemon.pid_end()
def run(self, var_dict, stdin=None, stdout=None, stderr=None):
"""run the limiter under variables defined in @var_dict
Note: @var_dict may be changed
execution result can be accessed via self.exe_status, self.exe_time (in microseconds),
self.exe_mem (in kb) and self.exe_extra_info
if @stdout and/or @stderr is SAVE_OUTPUT, stdout and/or stderr will be stored
in self.stdout and self.stderr
"""
self.stdout = None
self.stderr = None
if self._type == _LIMITER_FILE:
try:
ftmp = tempfile.mkstemp()
var_dict["FILENAME"] = ftmp[1]
except Exception as e:
log.error(
"[limiter {0!r}] failed to create temporary file: {1}".
format(self._name, e))
raise SysError("limiter communication error")
else:
var_dict["SOCKNAME"] = self._socket_name
try:
args = eval_arg_list(self._args, var_dict)
except Exception as e:
log.error(
"[limiter {0!r}] failed to evaluate argument: {1}".format(
self._name, e))
raise SysError("limiter configuration error")
log.debug("executing command: {0!r}".format(args))
try:
stdout_ = stdout
if stdout_ is SAVE_OUTPUT:
stdout_ = subprocess.PIPE
stderr_ = stderr
if stderr_ is SAVE_OUTPUT:
stderr_ = subprocess.PIPE
p = subprocess.Popen(args,
stdin=stdin,
stdout=stdout_,
stderr=stderr_)
except OSError as e:
log.error("error while calling Popen [errno {0}] "
"[filename {1!r}]: {2}".format(e.errno, e.filename,
e.strerror))
raise SysError("failed to execute limiter")
except Exception as e:
log.error("error while calling Popen: {0}".format(e))
raise SysError("failed to execute limiter")
if self._type == _LIMITER_SOCKET:
try:
s = self._socket
s.settimeout(1)
(conn, addr) = s.accept()
s.settimeout(None)
(self.exe_status, self.exe_time, self.exe_mem, info_len) = \
struct.unpack("IIII", conn.recv(16))
if info_len:
self.exe_extra_info = conn.recv(info_len)
else:
self.exe_extra_info = ''
except socket.timeout:
log.error("[limiter {0!r}] socket timed out".format(
self._name))
raise SysError("limiter socket error")
except Exception as e:
log.error(
"[limiter {0!r}] failed to retrieve data through socket: {1}"
.format(self._name, e))
raise SysError("limiter socket error")
if stdout is SAVE_OUTPUT or stderr is SAVE_OUTPUT:
(self.stdout, self.stderr) = p.communicate()
else:
p.wait()
log.debug('the command above now finished')
if self._type == _LIMITER_FILE:
try:
with open(ftmp[1], 'rb') as f:
(self.exe_status, self.exe_time, self.exe_mem, info_len) = \
struct.unpack("IIII", f.read(16))
if info_len:
self.exe_extra_info = f.read(info_len)
else:
self.exe_extra_info = ''
os.close(ftmp[0])
os.remove(ftmp[1])
except Exception as e:
log.error(
"[limiter {0!r}] failed to retrieve data through file: {1}"
.format(self._name, e))
raise SysError("limiter file error")
if self._type == _LIMITER_SOCKET:
try:
conn.close()
except Exception as e:
log.warning("failed to close socket connection: {0}".format(e))
def run(self):
judge = self._judge
def _write_msg(m):
msg.write_msg(self._snc, m)
def _write_str(s):
self._snc.write_str(s)
def _read_msg():
return msg.read_msg(self._snc)
def _read_str():
return self._snc.read_str()
def _read_uint32():
return self._snc.read_uint32()
def _check_msg(m):
if m != _read_msg():
log.warning("[judge {0!r}] message check error".format(
judge.id))
raise _internal_error
global _id_max_len, _judge_id_set, _judge_id_set_lock
try:
self._snc = snc.snc(self._sock, True)
_check_msg(msg.HELLO)
judge_id = _read_str()
if len(judge_id) > _id_max_len:
_write_msg(msg.ID_TOO_LONG)
raise _internal_error
with _judge_id_set_lock:
if judge_id in _judge_id_set:
_write_msg(msg.DUPLICATED_ID)
log.warning(
"another judge declares duplicated id {0!r}".format(
judge_id))
raise _internal_error
_judge_id_set.add(judge_id)
judge.id = judge_id
del judge_id
if _read_uint32() != msg.PROTOCOL_VERSION:
log.warning("[judge {0!r}] version check error".format(
judge.id))
_write_msg(msg.ERROR)
raise _internal_error
cnt = _read_uint32()
while cnt:
cnt -= 1
lang = _read_str()
judge.lang_supported.add(lang)
self._lang_id_set.add(_get_lang_id(lang))
_write_msg(msg.CONNECT_OK)
query_ans = dict()
for i in web.get_query_list():
_write_msg(msg.QUERY_INFO)
_write_str(i)
_check_msg(msg.ANS_QUERY)
query_ans[i] = _read_str()
web.register_new_judge(judge, query_ans)
self._web_registered = True
log.info("[judge {0!r}] successfully connected".format(judge.id))
while not control.test_termination_flag():
self._solve_task()
self._snc.close()
self._sock.close()
except snc.Error:
log.warning("[judge {0!r}] failed because of network error".format(
judge.id))
self._clean()
except _internal_error:
self._clean()
except web.Error:
log.warning(
"[judge {0!r}] failed because of error while communicating with website"
.format(judge.id))
_write_msg(msg.ERROR)
self._clean()
except sync_dir.Error:
log.warning(
"[judge {0!r}] failed to synchronize data directory".format(
judge.id))
self._clean()
except Exception as e:
log.warning("[judge {0!r}] error happens: {1}".format(judge.id, e))
log.debug(traceback.format_exc())
self._clean()
def recv(path, conn):
"""save the directory to @path via snc connection @conn,
return the speed in kb/s, or None if no file transferred"""
def _write_msg(m):
msg.write_msg(conn, m)
def _write_str(s):
conn.write_str(s)
def _write_uint32(v):
conn.write_uint32(v)
def _read_msg():
return msg.read_msg(conn)
def _read_str():
return conn.read_str()
def _read_uint32():
return conn.read_uint32()
def _check_msg(m):
while True:
m1 = _read_msg()
if m1 == msg.TELL_ONLINE:
continue
if m1 != m:
log.warning(
"message check error: expecting {0}, got {1}".format(
m, m1))
raise Error
return
try:
if os.path.isdir(path):
th_hash = _thread_get_file_list(path, False)
th_hash.start()
while th_hash.is_alive():
th_hash.join(msg.TELL_ONLINE_INTERVAL)
_write_msg(msg.TELL_ONLINE)
flist_local = th_hash.result
else:
if os.path.exists(path):
os.remove(path)
os.mkdir(path)
flist_local = dict()
if flist_local is None:
raise Error
flist_needed = list()
_check_msg(msg.SYNCDIR_BEGIN)
for i in range(_read_uint32()):
fname = _read_str()
checksum = _read_str()
try:
if checksum != flist_local[fname]:
os.remove(os.path.join(path, fname))
flist_needed.append(i)
del flist_local[fname]
except KeyError:
flist_needed.append(i)
for i in flist_local:
os.remove(os.path.join(path, i))
_write_msg(msg.SYNCDIR_FILELIST)
_write_uint32(len(flist_needed))
if len(flist_needed) == 0:
_write_msg(msg.SYNCDIR_DONE)
return None
for i in flist_needed:
_write_uint32(i)
_check_msg(msg.SYNCDIR_FTRANS)
try:
ftar = tempfile.mkstemp('orzoj')
speed = filetrans.recv(ftar[1], conn)
with open(ftar[1], 'r') as f:
th_extar = _thread_extract_tar(f, path)
th_extar.start()
while th_extar.is_alive():
th_extar.join(msg.TELL_ONLINE_INTERVAL)
_write_msg(msg.TELL_ONLINE)
if th_extar.error:
raise Error
_write_msg(msg.SYNCDIR_DONE)
return speed
finally:
os.close(ftar[0])
os.remove(ftar[1])
except Error as e:
raise e
except snc.Error:
log.warning("network error while synchronizing directory")
raise Error
except filetrans.OFTPError:
log.warning("failed to transfer file while synchronizing directory")
raise Error
except Exception as e:
log.error("failed to synchronize directory: {0}".format(e))
log.debug(traceback.format_exc())
raise Error
def send(path, conn):
"""send the directory at @path via snc connection @conn,
return the speed in kb/s, or None if no file transferred"""
def _write_msg(m):
msg.write_msg(conn, m)
def _write_str(s):
conn.write_str(s)
def _write_uint32(v):
conn.write_uint32(v)
def _read_msg():
return msg.read_msg(conn)
def _read_str():
return conn.read_str()
def _read_uint32():
return conn.read_uint32()
def _check_msg(m):
while True:
m1 = _read_msg()
if m1 == msg.TELL_ONLINE:
continue
if m1 != m:
log.warning(
"message check error: expecting {0}, got {1}".format(
m, m1))
raise Error
return
flist = _thread_get_file_list(path)
flist.start()
while flist.is_alive():
flist.join(msg.TELL_ONLINE_INTERVAL)
_write_msg(msg.TELL_ONLINE)
flist = flist.result
if flist is None:
raise Error
try:
_write_msg(msg.SYNCDIR_BEGIN)
_write_uint32(len(flist))
for i in flist:
_write_str(i[0])
_write_str(i[1])
_check_msg(msg.SYNCDIR_FILELIST)
nfile = _read_uint32()
if nfile == 0:
_check_msg(msg.SYNCDIR_DONE)
return None
flist_req = list()
while nfile:
nfile -= 1
flist_req.append(flist[_read_uint32()][0])
ftar = tempfile.mkstemp('orzoj')
try:
with open(ftar[1], 'wb') as f:
th_mktar = _thread_make_tar(f, path, flist_req)
th_mktar.start()
while th_mktar.is_alive():
th_mktar.join(msg.TELL_ONLINE_INTERVAL)
_write_msg(msg.TELL_ONLINE)
if th_mktar.error:
raise Error
_write_msg(msg.SYNCDIR_FTRANS)
speed = filetrans.send(ftar[1], conn)
_check_msg(msg.SYNCDIR_DONE)
return speed
finally:
os.close(ftar[0])
os.remove(ftar[1])
except Error as e:
raise e
except snc.Error:
log.warning("network error while synchronizing directory")
raise Error
except filetrans.OFTPError:
log.warning("failed to transfer file while synchronizing directory")
raise Error
except Exception as e:
log.error("failed to synchronize directory: {0}".format(e))
log.debug(traceback.format_exc())
raise Error
def connect(sock):
"""connect to orzoj-server via socket @sock
may raise Error"""
def _write_msg(m):
msg.write_msg(conn, m)
def _write_str(s):
conn.write_str(s)
def _write_uint32(v):
conn.write_uint32(v)
def _read_msg(timeout = 0):
return msg.read_msg(conn, timeout)
def _read_str():
return conn.read_str()
def _read_uint32():
return conn.read_uint32()
def _check_msg(m):
if m != _read_msg():
log.error("message check error.")
raise Error
try:
conn = snc.snc(sock)
_write_msg(msg.HELLO)
global _judge_id
_write_str(_judge_id)
_write_uint32(msg.PROTOCOL_VERSION)
_write_uint32(len(core.lang_dict))
for i in core.lang_dict:
_write_str(i)
m = _read_msg()
if m == msg.ERROR:
log.warning("failed to connect: orzoj-server says an error happens there`")
raise Error
if m == msg.DUPLICATED_ID:
log.error("failed to connect: duplicated id: {0!r}" . format(_judge_id))
raise Error
if m == msg.ID_TOO_LONG:
log.error("failed to connect: id {0!r} is too long for the orzoj-server" .
format(_judge_id))
raise Error
if m != msg.CONNECT_OK:
log.error("unexpected message from orzoj-server: {0}" .
format(m))
raise Error
log.info('connection established')
while not control.test_termination_flag():
m = _read_msg()
if m == msg.TELL_ONLINE:
continue
if m == msg.ERROR:
log.warning("failed to work: orzoj-server says an error happens there")
raise Error
if m == msg.QUERY_INFO:
global _info_dict
q = _read_str()
_write_msg(msg.ANS_QUERY)
try:
_write_str(_info_dict[q])
except KeyError:
_write_str("unknown")
continue
if m != msg.PREPARE_DATA:
log.error("unexpected message from orzoj-server: {0}" . format(m))
raise Error
pcode = _read_str()
log.info("received task for problem {0!r}" . format(pcode))
try:
speed = sync_dir.recv(pcode, conn)
if speed:
log.info("file transfer speed: {0!r}" . format(speed))
except sync_dir.Error:
log.error("failed to synchronize data for problem {0!r}" . format(pcode))
raise Error
try:
pconf = probconf.Prob_conf(pcode)
except Exception as e:
errmsg = "failed to parse problem configuration: {0}" . format(e)
_write_msg(msg.DATA_ERROR)
_write_str(errmsg)
log.error(errmsg)
log.debug(traceback.format_exc())
continue
_write_msg(msg.DATA_OK)
_write_uint32(len(pconf.case))
_check_msg(msg.START_JUDGE)
lang = _read_str()
src = _read_str()
input = _read_str()
output = _read_str()
core.lang_dict[lang].judge(conn, pcode, pconf, src, input, output)
except snc.Error as e:
log.error("failed to communicate with orzoj-server because of network error")
control.set_termination_flag()
raise Error
except core.Error:
control.set_termination_flag()
raise Error
def _read(data, maxlen=None):
"""if @maxlen is not None, data should be of dict type and is sent via GET method and without checksum
and the data read is returned;
otherwise @data will dumped by phpserialize and sent via POST method and the data read is returned
Note: @maxlen is not None iff now trying to login
"""
global _retry_cnt, _web_addr, _thread_req_id, _lock_thread_req_id, _passwd
def make_data():
"""return a tuple (checksum_base, data_sent)"""
thread_id = threading.current_thread().ident
with _lock_thread_req_id:
try:
req_id = _thread_req_id[thread_id]
except KeyError:
req_id = 0
_thread_req_id[thread_id] = req_id + 1
checksum_base = str(thread_id) + '$' + str(req_id) + '$' + _passwd
data_sent = urllib.urlencode({
"data":
phpserialize.dumps({
"thread_id": thread_id,
"data": data,
"checksum": _sha1sum(checksum_base + data)
})
})
return (checksum_base, data_sent)
if maxlen:
url = _web_addr + "?" + urllib.urlencode(data)
else:
data = phpserialize.dumps(data)
(checksum_base, data_sent) = make_data()
cnt = _retry_cnt
while cnt:
if control.test_termination_flag():
raise Error
cnt -= 1
try:
ret = None
if maxlen:
return urllib2.urlopen(url, None, _timeout).read(maxlen)
ret = urllib2.urlopen(_web_addr, data_sent, _timeout).read()
if ret == 'relogin':
if _lock_relogin.acquire(False):
log.warning("website requests relogin")
_login()
with _lock_thread_req_id:
_thread_req_id.clear()
_lock_relogin.release()
else:
_lock_relogin.acquire()
# wait until relogin finishes
_lock_relogin.release()
(checksum_base, data_sent) = make_data()
cnt = _retry_cnt
continue
ret = phpserialize.loads(ret)
ret_status = ret["status"]
ret_data = ret["data"]
if ret["checksum"] != _sha1sum(checksum_base + str(ret_status) +
ret_data):
raise _internal_error("website checksum error")
if int(ret_status):
raise _internal_error(
"website says an error happens there: {0}".format(
ret_data))
return phpserialize.loads(ret_data)
except Exception as e:
log.error(
"website communication error [left retries: {0}]: {1}".format(
cnt, e))
sys.stderr.write(
"orzoj-server: website communication error. See the log for details.\n"
)
log.debug("raw data from server: {0!r}".format(ret))
time.sleep(_retry_wait)
continue
raise Error
def judge(self, conn, pcode, pconf, src, input, output):
"""@pcode: problem code
@pconf: problem configuration (defined in probconf.py)
may raise Error or snc.Error"""
def _write_msg(m):
msg.write_msg(conn, m)
def _write_str(s):
conn.write_str(s)
def _write_uint32(v):
conn.write_uint32(v)
locked = False
global _lock_file_fd
if _lock_file_fd:
while True:
try:
fcntl.flock(_lock_file_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError as e:
if e.errno == errno.EACCES or e.errno == errno.EAGAIN:
_write_msg(msg.START_JUDGE_WAIT)
time.sleep(msg.TELL_ONLINE_INTERVAL)
continue
else:
log.error("failed to lock file: {0}".format(e))
_write_msg(msg.ERROR)
raise Error
except Exception as e:
log.error("failed to lock file: {0}".format(e))
_write_msg(msg.ERROR)
raise Error
locked = True
break # successfully locked
try:
_write_msg(msg.START_JUDGE_OK)
_clean_temp()
global _prog_path_abs, _cmd_vars
if self._compiler:
with open(_prog_path_abs + self._src_ext, "w") as f:
f.write(src)
_cmd_vars["MEMORY"] = 0
_cmd_vars["DATADIR"] = os.path.abspath(pcode)
th_tell_online = _thread_tell_online(conn)
th_tell_online.start()
if pconf.compiler and self._name in pconf.compiler:
(ok, info) = self._compiler.run_as_compiler(
_prog_path_abs, pconf.compiler[self._name])
else:
(ok, info) = self._compiler.run_as_compiler(_prog_path_abs)
th_tell_online.stop()
th_tell_online.join()
if not ok:
_write_msg(msg.COMPILE_FAIL)
_write_str(info)
return
_write_msg(msg.COMPILE_SUCCEED)
global _dir_temp_abs
os.chmod(
_prog_path_abs + self._exe_ext, stat.S_IRUSR | stat.S_IXUSR
| stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
global _prog_path
th_report_case = _thread_report_case_result(conn, len(pconf.case))
th_report_case.start()
for case in pconf.case:
try:
if pconf.extra_input:
for i in pconf.extra_input:
shutil.copy(_join_path(pcode, i), _dir_temp_abs)
stdin_path = _join_path(pcode, case.stdin)
if not input: # use stdin
prog_fin = open(stdin_path)
else:
tpath = _join_path(_dir_temp_abs, input)
shutil.copy(stdin_path, tpath)
os.chmod(tpath,
stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
prog_fin = limiter.get_null_dev(False)
if not output: # use stdout
prog_fout_path = _join_path(
_dir_temp_abs, "output.{0}".format(time.time()))
prog_fout = open(prog_fout_path, "w")
else:
prog_fout_path = _join_path(_dir_temp_abs, output)
prog_fout = limiter.get_null_dev()
except Exception as e:
log.error("failed to open data file: {0}".format(
stdin_path, e))
case_result = structures.case_result()
case_result.exe_status = structures.EXESTS_SYSTEM_ERROR
case_result.score = 0
case_result.full_score = 0
case_result.time = 0
case_result.memory = 0
case_result.extra_info = "failed to open data file"
else:
_cmd_vars["TIME"] = case.time
_cmd_vars["MEMORY"] = case.mem
umask_prev = os.umask(0)
case_result = self._executor.run(_prog_path,
stdin=prog_fin,
stdout=prog_fout)
case_result.full_score = case.score
os.umask(umask_prev)
if prog_fin:
prog_fin.close()
if prog_fout:
prog_fout.close()
if case_result.exe_status == structures.EXESTS_NORMAL:
if (not os.path.isfile(prog_fout_path)
) or os.path.islink(prog_fout_path):
(case_result.score, case_result.extra_info) = (
0, "output file not found")
else:
(case_result.score,
case_result.extra_info) = pconf.verify_func(
case.score, stdin_path,
_join_path(pcode, case.stdout),
prog_fout_path)
if case_result.score is None:
case_result.score = 0
case_result.exe_status = structures.EXESTS_SYSTEM_ERROR
if input:
try:
os.unlink(tpath)
except Exception as e:
log.warning(
"failed to remove program input file: {0}".
format(e))
try:
os.unlink(prog_fout_path)
except Exception as e:
log.warning(
"failed to remove program output file: {0}".format(
e))
th_report_case.add(case_result)
th_report_case.join()
th_report_case.check_error()
_write_msg(msg.REPORT_JUDGE_FINISH)
if locked:
fcntl.flock(_lock_file_fd, fcntl.LOCK_UN)
except Error:
if locked:
fcntl.flock(_lock_file_fd, fcntl.LOCK_UN)
raise
except snc.Error:
if locked:
fcntl.flock(_lock_file_fd, fcntl.LOCK_UN)
raise Error
except Exception as e:
if locked:
fcntl.flock(_lock_file_fd, fcntl.LOCK_UN)
log.error("[lang {0!r}] failed to judge: {1}".format(
self._name, e))
log.debug(traceback.format_exc())
_write_msg(msg.ERROR)
raise Error
