Exemple #1
0
def run_server():
    _parse_opt()
    global _options, _use_ipv6, _port
    conf.parse_file(_options.conf_file)

    if not _options.no_daemon and conf.is_unix:
        daemon.daemon_start()

    daemon.pid_start()

    try:
        s = snc.socket(None, _port)
    except snc.Error:
        daemon.pid_end()

    log.info("orzoj-server started, listening on {0}".format(_port))

    threading.Thread(target=work.thread_work, name="work.thread_work").start()

    while not control.test_termination_flag():
        try:
            (conn, addr) = s.accept(1)
        except snc.ErrorTimeout:
            continue
        except snc.Error as e:
            control.set_termination_flag()
            break

        log.info("connected by {0!r}".format(addr))
        work.thread_new_judge_connection(conn).start()

    s.close()
    while threading.active_count() > 1:
        log.debug("waiting for threads, current active count: {0}".format(
            threading.active_count()))
        for i in threading.enumerate():
            log.debug("active thread: {0!r}".format(i.name))
        time.sleep(1)

    daemon.pid_end()
    def run(self, var_dict, stdin=None, stdout=None, stderr=None):
        """run the limiter under variables defined in @var_dict
        Note: @var_dict may be changed
        
        execution result can be accessed via self.exe_status, self.exe_time (in microseconds),
        self.exe_mem (in kb) and self.exe_extra_info

        if @stdout and/or @stderr is SAVE_OUTPUT, stdout and/or stderr will be stored
        in self.stdout and self.stderr
        """

        self.stdout = None
        self.stderr = None

        if self._type == _LIMITER_FILE:
            try:
                ftmp = tempfile.mkstemp()
                var_dict["FILENAME"] = ftmp[1]
            except Exception as e:
                log.error(
                    "[limiter {0!r}] failed to create temporary file: {1}".
                    format(self._name, e))
                raise SysError("limiter communication error")
        else:
            var_dict["SOCKNAME"] = self._socket_name

        try:
            args = eval_arg_list(self._args, var_dict)
        except Exception as e:
            log.error(
                "[limiter {0!r}] failed to evaluate argument: {1}".format(
                    self._name, e))
            raise SysError("limiter configuration error")

        log.debug("executing command: {0!r}".format(args))

        try:
            stdout_ = stdout
            if stdout_ is SAVE_OUTPUT:
                stdout_ = subprocess.PIPE
            stderr_ = stderr
            if stderr_ is SAVE_OUTPUT:
                stderr_ = subprocess.PIPE

            p = subprocess.Popen(args,
                                 stdin=stdin,
                                 stdout=stdout_,
                                 stderr=stderr_)
        except OSError as e:
            log.error("error while calling Popen [errno {0}] "
                      "[filename {1!r}]: {2}".format(e.errno, e.filename,
                                                     e.strerror))
            raise SysError("failed to execute limiter")
        except Exception as e:
            log.error("error while calling Popen: {0}".format(e))
            raise SysError("failed to execute limiter")

        if self._type == _LIMITER_SOCKET:
            try:
                s = self._socket
                s.settimeout(1)
                (conn, addr) = s.accept()
                s.settimeout(None)
                (self.exe_status, self.exe_time, self.exe_mem, info_len) = \
                        struct.unpack("IIII", conn.recv(16))
                if info_len:
                    self.exe_extra_info = conn.recv(info_len)
                else:
                    self.exe_extra_info = ''
            except socket.timeout:
                log.error("[limiter {0!r}] socket timed out".format(
                    self._name))
                raise SysError("limiter socket error")
            except Exception as e:
                log.error(
                    "[limiter {0!r}] failed to retrieve data through socket: {1}"
                    .format(self._name, e))
                raise SysError("limiter socket error")

        if stdout is SAVE_OUTPUT or stderr is SAVE_OUTPUT:
            (self.stdout, self.stderr) = p.communicate()
        else:
            p.wait()

        log.debug('the command above now finished')

        if self._type == _LIMITER_FILE:
            try:
                with open(ftmp[1], 'rb') as f:
                    (self.exe_status, self.exe_time, self.exe_mem, info_len) = \
                            struct.unpack("IIII", f.read(16))
                    if info_len:
                        self.exe_extra_info = f.read(info_len)
                    else:
                        self.exe_extra_info = ''
                os.close(ftmp[0])
                os.remove(ftmp[1])
            except Exception as e:
                log.error(
                    "[limiter {0!r}] failed to retrieve data through file: {1}"
                    .format(self._name, e))
                raise SysError("limiter file error")

        if self._type == _LIMITER_SOCKET:
            try:
                conn.close()
            except Exception as e:
                log.warning("failed to close socket connection: {0}".format(e))
Exemple #3
0
    def run(self):
        judge = self._judge

        def _write_msg(m):
            msg.write_msg(self._snc, m)

        def _write_str(s):
            self._snc.write_str(s)

        def _read_msg():
            return msg.read_msg(self._snc)

        def _read_str():
            return self._snc.read_str()

        def _read_uint32():
            return self._snc.read_uint32()

        def _check_msg(m):
            if m != _read_msg():
                log.warning("[judge {0!r}] message check error".format(
                    judge.id))
                raise _internal_error

        global _id_max_len, _judge_id_set, _judge_id_set_lock

        try:
            self._snc = snc.snc(self._sock, True)
            _check_msg(msg.HELLO)
            judge_id = _read_str()

            if len(judge_id) > _id_max_len:
                _write_msg(msg.ID_TOO_LONG)
                raise _internal_error

            with _judge_id_set_lock:
                if judge_id in _judge_id_set:
                    _write_msg(msg.DUPLICATED_ID)
                    log.warning(
                        "another judge declares duplicated id {0!r}".format(
                            judge_id))
                    raise _internal_error

                _judge_id_set.add(judge_id)

            judge.id = judge_id
            del judge_id

            if _read_uint32() != msg.PROTOCOL_VERSION:
                log.warning("[judge {0!r}] version check error".format(
                    judge.id))
                _write_msg(msg.ERROR)
                raise _internal_error

            cnt = _read_uint32()
            while cnt:
                cnt -= 1
                lang = _read_str()
                judge.lang_supported.add(lang)
                self._lang_id_set.add(_get_lang_id(lang))

            _write_msg(msg.CONNECT_OK)

            query_ans = dict()
            for i in web.get_query_list():
                _write_msg(msg.QUERY_INFO)
                _write_str(i)
                _check_msg(msg.ANS_QUERY)
                query_ans[i] = _read_str()

            web.register_new_judge(judge, query_ans)
            self._web_registered = True

            log.info("[judge {0!r}] successfully connected".format(judge.id))

            while not control.test_termination_flag():
                self._solve_task()

            self._snc.close()
            self._sock.close()

        except snc.Error:
            log.warning("[judge {0!r}] failed because of network error".format(
                judge.id))
            self._clean()
        except _internal_error:
            self._clean()
        except web.Error:
            log.warning(
                "[judge {0!r}] failed because of error while communicating with website"
                .format(judge.id))
            _write_msg(msg.ERROR)
            self._clean()
        except sync_dir.Error:
            log.warning(
                "[judge {0!r}] failed to synchronize data directory".format(
                    judge.id))
            self._clean()
        except Exception as e:
            log.warning("[judge {0!r}] error happens: {1}".format(judge.id, e))
            log.debug(traceback.format_exc())
            self._clean()
Exemple #4
0
def recv(path, conn):
    """save the directory to @path via snc connection @conn,
    return the speed in kb/s, or None if no file transferred"""
    def _write_msg(m):
        msg.write_msg(conn, m)

    def _write_str(s):
        conn.write_str(s)

    def _write_uint32(v):
        conn.write_uint32(v)

    def _read_msg():
        return msg.read_msg(conn)

    def _read_str():
        return conn.read_str()

    def _read_uint32():
        return conn.read_uint32()

    def _check_msg(m):
        while True:
            m1 = _read_msg()
            if m1 == msg.TELL_ONLINE:
                continue
            if m1 != m:
                log.warning(
                    "message check error: expecting {0}, got {1}".format(
                        m, m1))
                raise Error
            return

    try:
        if os.path.isdir(path):
            th_hash = _thread_get_file_list(path, False)
            th_hash.start()
            while th_hash.is_alive():
                th_hash.join(msg.TELL_ONLINE_INTERVAL)
                _write_msg(msg.TELL_ONLINE)
            flist_local = th_hash.result
        else:
            if os.path.exists(path):
                os.remove(path)
            os.mkdir(path)
            flist_local = dict()
        if flist_local is None:
            raise Error

        flist_needed = list()
        _check_msg(msg.SYNCDIR_BEGIN)

        for i in range(_read_uint32()):
            fname = _read_str()
            checksum = _read_str()
            try:
                if checksum != flist_local[fname]:
                    os.remove(os.path.join(path, fname))
                    flist_needed.append(i)
                del flist_local[fname]
            except KeyError:
                flist_needed.append(i)

        for i in flist_local:
            os.remove(os.path.join(path, i))

        _write_msg(msg.SYNCDIR_FILELIST)
        _write_uint32(len(flist_needed))

        if len(flist_needed) == 0:
            _write_msg(msg.SYNCDIR_DONE)
            return None

        for i in flist_needed:
            _write_uint32(i)

        _check_msg(msg.SYNCDIR_FTRANS)

        try:
            ftar = tempfile.mkstemp('orzoj')
            speed = filetrans.recv(ftar[1], conn)
            with open(ftar[1], 'r') as f:
                th_extar = _thread_extract_tar(f, path)
                th_extar.start()
                while th_extar.is_alive():
                    th_extar.join(msg.TELL_ONLINE_INTERVAL)
                    _write_msg(msg.TELL_ONLINE)
                if th_extar.error:
                    raise Error
            _write_msg(msg.SYNCDIR_DONE)
            return speed

        finally:
            os.close(ftar[0])
            os.remove(ftar[1])

    except Error as e:
        raise e

    except snc.Error:
        log.warning("network error while synchronizing directory")
        raise Error

    except filetrans.OFTPError:
        log.warning("failed to transfer file while synchronizing directory")
        raise Error

    except Exception as e:
        log.error("failed to synchronize directory: {0}".format(e))
        log.debug(traceback.format_exc())
        raise Error
Exemple #5
0
def send(path, conn):
    """send the directory at @path via snc connection @conn,
    return the speed in kb/s, or None if no file transferred"""
    def _write_msg(m):
        msg.write_msg(conn, m)

    def _write_str(s):
        conn.write_str(s)

    def _write_uint32(v):
        conn.write_uint32(v)

    def _read_msg():
        return msg.read_msg(conn)

    def _read_str():
        return conn.read_str()

    def _read_uint32():
        return conn.read_uint32()

    def _check_msg(m):
        while True:
            m1 = _read_msg()
            if m1 == msg.TELL_ONLINE:
                continue
            if m1 != m:
                log.warning(
                    "message check error: expecting {0}, got {1}".format(
                        m, m1))
                raise Error
            return

    flist = _thread_get_file_list(path)
    flist.start()
    while flist.is_alive():
        flist.join(msg.TELL_ONLINE_INTERVAL)
        _write_msg(msg.TELL_ONLINE)
    flist = flist.result
    if flist is None:
        raise Error

    try:
        _write_msg(msg.SYNCDIR_BEGIN)
        _write_uint32(len(flist))
        for i in flist:
            _write_str(i[0])
            _write_str(i[1])

        _check_msg(msg.SYNCDIR_FILELIST)

        nfile = _read_uint32()
        if nfile == 0:
            _check_msg(msg.SYNCDIR_DONE)
            return None

        flist_req = list()
        while nfile:
            nfile -= 1
            flist_req.append(flist[_read_uint32()][0])

        ftar = tempfile.mkstemp('orzoj')

        try:
            with open(ftar[1], 'wb') as f:
                th_mktar = _thread_make_tar(f, path, flist_req)
                th_mktar.start()
                while th_mktar.is_alive():
                    th_mktar.join(msg.TELL_ONLINE_INTERVAL)
                    _write_msg(msg.TELL_ONLINE)
                if th_mktar.error:
                    raise Error

            _write_msg(msg.SYNCDIR_FTRANS)
            speed = filetrans.send(ftar[1], conn)
            _check_msg(msg.SYNCDIR_DONE)
            return speed

        finally:

            os.close(ftar[0])
            os.remove(ftar[1])

    except Error as e:
        raise e

    except snc.Error:
        log.warning("network error while synchronizing directory")
        raise Error

    except filetrans.OFTPError:
        log.warning("failed to transfer file while synchronizing directory")
        raise Error

    except Exception as e:
        log.error("failed to synchronize directory: {0}".format(e))
        log.debug(traceback.format_exc())
        raise Error
Exemple #6
0
def connect(sock):
    """connect to orzoj-server via socket @sock
    
    may raise Error"""

    def _write_msg(m):
        msg.write_msg(conn, m)

    def _write_str(s):
        conn.write_str(s)

    def _write_uint32(v):
        conn.write_uint32(v)

    def _read_msg(timeout = 0):
        return msg.read_msg(conn, timeout)

    def _read_str():
        return conn.read_str()

    def _read_uint32():
        return conn.read_uint32()

    def _check_msg(m):
        if m != _read_msg():
            log.error("message check error.")
            raise Error
    try:
        conn = snc.snc(sock)

        _write_msg(msg.HELLO)
        global _judge_id
        _write_str(_judge_id)
        _write_uint32(msg.PROTOCOL_VERSION)
        _write_uint32(len(core.lang_dict))
        for i in core.lang_dict:
            _write_str(i)

        m = _read_msg()
        if m == msg.ERROR:
            log.warning("failed to connect: orzoj-server says an error happens there`")
            raise Error

        if m == msg.DUPLICATED_ID:
            log.error("failed to connect: duplicated id: {0!r}" . format(_judge_id))
            raise Error

        if m == msg.ID_TOO_LONG:
            log.error("failed to connect: id {0!r} is too long for the orzoj-server" .
                    format(_judge_id))
            raise Error

        if m != msg.CONNECT_OK:
            log.error("unexpected message from orzoj-server: {0}" .
                    format(m))
            raise Error

        log.info('connection established')

        while not control.test_termination_flag():
            m = _read_msg()

            if m == msg.TELL_ONLINE:
                continue

            if m == msg.ERROR:
                log.warning("failed to work: orzoj-server says an error happens there")
                raise Error

            if m == msg.QUERY_INFO:
                global _info_dict
                q = _read_str()
                _write_msg(msg.ANS_QUERY)
                try:
                    _write_str(_info_dict[q])
                except KeyError:
                    _write_str("unknown")
                continue

            if m != msg.PREPARE_DATA:
                log.error("unexpected message from orzoj-server: {0}" .  format(m))
                raise Error

            pcode = _read_str()
            log.info("received task for problem {0!r}" . format(pcode))
            try:
                speed = sync_dir.recv(pcode, conn)
                if speed:
                    log.info("file transfer speed: {0!r}" . format(speed))

            except sync_dir.Error:
                log.error("failed to synchronize data for problem {0!r}" . format(pcode))
                raise Error

            try:
                pconf = probconf.Prob_conf(pcode)
            except Exception as e:
                errmsg = "failed to parse problem configuration: {0}" . format(e)
                _write_msg(msg.DATA_ERROR)
                _write_str(errmsg)
                log.error(errmsg)
                log.debug(traceback.format_exc())
                continue

            _write_msg(msg.DATA_OK)
            _write_uint32(len(pconf.case))

            _check_msg(msg.START_JUDGE)
            lang = _read_str()
            src = _read_str()
            input = _read_str()
            output = _read_str()

            core.lang_dict[lang].judge(conn, pcode, pconf, src, input, output)

    except snc.Error as e:
        log.error("failed to communicate with orzoj-server because of network error")
        control.set_termination_flag()
        raise Error

    except core.Error:
        control.set_termination_flag()
        raise Error
Exemple #7
0
def _read(data, maxlen=None):
    """if @maxlen is not None, data should be of dict type and is sent via GET method and without checksum
    and the data read is returned;
    otherwise @data will dumped by phpserialize and sent via POST method and the data read is returned

    Note: @maxlen is not None iff now trying to login
    """
    global _retry_cnt, _web_addr, _thread_req_id, _lock_thread_req_id, _passwd

    def make_data():
        """return a tuple (checksum_base, data_sent)"""
        thread_id = threading.current_thread().ident
        with _lock_thread_req_id:
            try:
                req_id = _thread_req_id[thread_id]
            except KeyError:
                req_id = 0
            _thread_req_id[thread_id] = req_id + 1
        checksum_base = str(thread_id) + '$' + str(req_id) + '$' + _passwd
        data_sent = urllib.urlencode({
            "data":
            phpserialize.dumps({
                "thread_id": thread_id,
                "data": data,
                "checksum": _sha1sum(checksum_base + data)
            })
        })

        return (checksum_base, data_sent)

    if maxlen:
        url = _web_addr + "?" + urllib.urlencode(data)
    else:
        data = phpserialize.dumps(data)
        (checksum_base, data_sent) = make_data()

    cnt = _retry_cnt

    while cnt:
        if control.test_termination_flag():
            raise Error
        cnt -= 1
        try:
            ret = None

            if maxlen:
                return urllib2.urlopen(url, None, _timeout).read(maxlen)

            ret = urllib2.urlopen(_web_addr, data_sent, _timeout).read()

            if ret == 'relogin':
                if _lock_relogin.acquire(False):
                    log.warning("website requests relogin")
                    _login()
                    with _lock_thread_req_id:
                        _thread_req_id.clear()
                    _lock_relogin.release()
                else:
                    _lock_relogin.acquire()
                    # wait until relogin finishes
                    _lock_relogin.release()
                (checksum_base, data_sent) = make_data()
                cnt = _retry_cnt
                continue

            ret = phpserialize.loads(ret)

            ret_status = ret["status"]
            ret_data = ret["data"]

            if ret["checksum"] != _sha1sum(checksum_base + str(ret_status) +
                                           ret_data):
                raise _internal_error("website checksum error")

            if int(ret_status):
                raise _internal_error(
                    "website says an error happens there: {0}".format(
                        ret_data))

            return phpserialize.loads(ret_data)

        except Exception as e:
            log.error(
                "website communication error [left retries: {0}]: {1}".format(
                    cnt, e))
            sys.stderr.write(
                "orzoj-server: website communication error. See the log for details.\n"
            )
            log.debug("raw data from server: {0!r}".format(ret))
            time.sleep(_retry_wait)
            continue

    raise Error
Exemple #8
0
    def judge(self, conn, pcode, pconf, src, input, output):
        """@pcode: problem code
        @pconf: problem configuration (defined in probconf.py)
        may raise Error or snc.Error"""
        def _write_msg(m):
            msg.write_msg(conn, m)

        def _write_str(s):
            conn.write_str(s)

        def _write_uint32(v):
            conn.write_uint32(v)

        locked = False

        global _lock_file_fd
        if _lock_file_fd:
            while True:
                try:
                    fcntl.flock(_lock_file_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
                except IOError as e:
                    if e.errno == errno.EACCES or e.errno == errno.EAGAIN:
                        _write_msg(msg.START_JUDGE_WAIT)
                        time.sleep(msg.TELL_ONLINE_INTERVAL)
                        continue
                    else:
                        log.error("failed to lock file: {0}".format(e))
                        _write_msg(msg.ERROR)
                        raise Error

                except Exception as e:
                    log.error("failed to lock file: {0}".format(e))
                    _write_msg(msg.ERROR)
                    raise Error

                locked = True
                break  # successfully locked

        try:
            _write_msg(msg.START_JUDGE_OK)

            _clean_temp()

            global _prog_path_abs, _cmd_vars

            if self._compiler:
                with open(_prog_path_abs + self._src_ext, "w") as f:
                    f.write(src)

                _cmd_vars["MEMORY"] = 0
                _cmd_vars["DATADIR"] = os.path.abspath(pcode)

                th_tell_online = _thread_tell_online(conn)
                th_tell_online.start()

                if pconf.compiler and self._name in pconf.compiler:
                    (ok, info) = self._compiler.run_as_compiler(
                        _prog_path_abs, pconf.compiler[self._name])
                else:
                    (ok, info) = self._compiler.run_as_compiler(_prog_path_abs)

                th_tell_online.stop()
                th_tell_online.join()

                if not ok:
                    _write_msg(msg.COMPILE_FAIL)
                    _write_str(info)
                    return

            _write_msg(msg.COMPILE_SUCCEED)

            global _dir_temp_abs

            os.chmod(
                _prog_path_abs + self._exe_ext, stat.S_IRUSR | stat.S_IXUSR
                | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
            global _prog_path

            th_report_case = _thread_report_case_result(conn, len(pconf.case))
            th_report_case.start()

            for case in pconf.case:

                try:
                    if pconf.extra_input:
                        for i in pconf.extra_input:
                            shutil.copy(_join_path(pcode, i), _dir_temp_abs)

                    stdin_path = _join_path(pcode, case.stdin)
                    if not input:  # use stdin
                        prog_fin = open(stdin_path)
                    else:
                        tpath = _join_path(_dir_temp_abs, input)
                        shutil.copy(stdin_path, tpath)
                        os.chmod(tpath,
                                 stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
                        prog_fin = limiter.get_null_dev(False)

                    if not output:  # use stdout
                        prog_fout_path = _join_path(
                            _dir_temp_abs, "output.{0}".format(time.time()))
                        prog_fout = open(prog_fout_path, "w")
                    else:
                        prog_fout_path = _join_path(_dir_temp_abs, output)
                        prog_fout = limiter.get_null_dev()
                except Exception as e:
                    log.error("failed to open data file: {0}".format(
                        stdin_path, e))
                    case_result = structures.case_result()
                    case_result.exe_status = structures.EXESTS_SYSTEM_ERROR
                    case_result.score = 0
                    case_result.full_score = 0
                    case_result.time = 0
                    case_result.memory = 0
                    case_result.extra_info = "failed to open data file"

                else:

                    _cmd_vars["TIME"] = case.time
                    _cmd_vars["MEMORY"] = case.mem

                    umask_prev = os.umask(0)
                    case_result = self._executor.run(_prog_path,
                                                     stdin=prog_fin,
                                                     stdout=prog_fout)
                    case_result.full_score = case.score
                    os.umask(umask_prev)

                    if prog_fin:
                        prog_fin.close()
                    if prog_fout:
                        prog_fout.close()

                    if case_result.exe_status == structures.EXESTS_NORMAL:
                        if (not os.path.isfile(prog_fout_path)
                            ) or os.path.islink(prog_fout_path):
                            (case_result.score, case_result.extra_info) = (
                                0, "output file not found")
                        else:
                            (case_result.score,
                             case_result.extra_info) = pconf.verify_func(
                                 case.score, stdin_path,
                                 _join_path(pcode, case.stdout),
                                 prog_fout_path)
                            if case_result.score is None:
                                case_result.score = 0
                                case_result.exe_status = structures.EXESTS_SYSTEM_ERROR

                    if input:
                        try:
                            os.unlink(tpath)
                        except Exception as e:
                            log.warning(
                                "failed to remove program input file: {0}".
                                format(e))
                    try:
                        os.unlink(prog_fout_path)
                    except Exception as e:
                        log.warning(
                            "failed to remove program output file: {0}".format(
                                e))
                th_report_case.add(case_result)

            th_report_case.join()
            th_report_case.check_error()
            _write_msg(msg.REPORT_JUDGE_FINISH)

            if locked:
                fcntl.flock(_lock_file_fd, fcntl.LOCK_UN)

        except Error:
            if locked:
                fcntl.flock(_lock_file_fd, fcntl.LOCK_UN)
            raise
        except snc.Error:
            if locked:
                fcntl.flock(_lock_file_fd, fcntl.LOCK_UN)
            raise Error
        except Exception as e:
            if locked:
                fcntl.flock(_lock_file_fd, fcntl.LOCK_UN)
            log.error("[lang {0!r}] failed to judge: {1}".format(
                self._name, e))
            log.debug(traceback.format_exc())
            _write_msg(msg.ERROR)
            raise Error