def judge_proc(need_monitor): from dmoj import judgeenv logfile = judgeenv.log_file try: logfile = logfile % env['id'] except TypeError: pass logging.basicConfig( filename=logfile, level=logging.INFO, format='%(levelname)s %(asctime)s %(process)d %(module)s %(message)s') setproctitle('DMOJ Judge: %s on %s' % (env['id'], make_host_port(judgeenv))) judge = ClassicJudge(judgeenv.server_host, judgeenv.server_port, secure=judgeenv.secure, no_cert_check=judgeenv.no_cert_check, cert_store=judgeenv.cert_store) if need_monitor: monitor = Monitor() monitor.callback = judge.update_problems else: monitor = DummyMonitor() if hasattr(signal, 'SIGUSR2'): def update_problem_signal(signum, frame): judge.update_problems() signal.signal(signal.SIGUSR2, update_problem_signal) if need_monitor and judgeenv.api_listen: judge_instance = judge class Handler(JudgeControlRequestHandler): judge = judge_instance api_server = HTTPServer(judgeenv.api_listen, Handler) thread = threading.Thread(target=api_server.serve_forever) thread.daemon = True thread.start() else: api_server = None print() with monitor, judge: try: judge.listen() except Exception: traceback.print_exc() finally: judge.murder() if api_server: api_server.shutdown()
def judge_proc(need_monitor): from dmoj import judgeenv logfile = judgeenv.log_file try: logfile = logfile % env['id'] except TypeError: pass logging.basicConfig(filename=logfile, level=logging.INFO, format='%(levelname)s %(asctime)s %(process)d %(module)s %(message)s') proctitle = 'DMOJ Judge: %s on %s' % (env['id'], make_host_port(judgeenv)) if six.PY2: setproctitle(utf8bytes(proctitle)) else: setproctitle(proctitle) judge = ClassicJudge(judgeenv.server_host, judgeenv.server_port, secure=judgeenv.secure, no_cert_check=judgeenv.no_cert_check, cert_store=judgeenv.cert_store) if need_monitor: monitor = Monitor() monitor.callback = judge.update_problems else: monitor = DummyMonitor() if hasattr(signal, 'SIGUSR2'): def update_problem_signal(signum, frame): judge.update_problems() signal.signal(signal.SIGUSR2, update_problem_signal) if need_monitor and judgeenv.api_listen: judge_instance = judge class Handler(JudgeControlRequestHandler): judge = judge_instance api_server = HTTPServer(judgeenv.api_listen, Handler) thread = threading.Thread(target=api_server.serve_forever) thread.daemon = True thread.start() else: api_server = None print() with monitor, judge: try: judge.listen() except Exception: traceback.print_exc() finally: judge.murder() if api_server: api_server.shutdown()
def judge_proc(need_monitor): from dmoj import judgeenv logfile = judgeenv.log_file try: logfile = logfile % env['id'] except TypeError: pass logging.basicConfig( filename=logfile, level=logging.INFO, format='%(levelname)s %(asctime)s %(module)s %(message)s') judge = ClassicJudge(judgeenv.server_host, judgeenv.server_port) if need_monitor: monitor = Monitor() monitor.callback = judge.update_problems else: monitor = DummyMonitor() if hasattr(signal, 'SIGUSR2'): def update_problem_signal(signum, frame): judge.update_problems() signal.signal(signal.SIGUSR2, update_problem_signal) if need_monitor and judgeenv.api_listen: from BaseHTTPServer import HTTPServer judge_instance = judge class Handler(JudgeControlRequestHandler): judge = judge_instance api_server = HTTPServer(judgeenv.api_listen, Handler) thread = threading.Thread(target=api_server.serve_forever) thread.daemon = True thread.start() else: api_server = None print with monitor, judge: try: judge.listen() except KeyboardInterrupt: pass except: traceback.print_exc() finally: judge.murder() if api_server: api_server.shutdown()
def __init__(self, judges): self.libc = self.__get_libc() self.prctl = self.libc.prctl self._try_respawn = True self.auth = {entry.id: entry.key for entry in judges} self.orig_signal = {} self.master_pid = os.getpid() self.pids = {} self.monitor_pid = None self.api_pid = None self.monitor = Monitor()
def __init__(self, judges): self.libc = self.__get_libc() self.prctl = self.libc.prctl self._try_respawn = True self.auth = {entry.id: entry.key for entry in judges} self.pids = {} self.orig_signal = {} self.monitor = Monitor() self.monitor.callback = lambda: self.signal_all(signal.SIGUSR2)
def judge_proc(need_monitor): from dmoj import judgeenv logfile = judgeenv.log_file try: logfile = logfile % env['id'] except TypeError: pass logging.basicConfig( filename=logfile, level=logging.INFO, format='%(levelname)s %(asctime)s %(module)s %(message)s') judge = ClassicJudge(judgeenv.server_host, judgeenv.server_port) if need_monitor: monitor = Monitor() monitor.callback = judge.update_problems else: monitor = DummyMonitor() if hasattr(signal, 'SIGUSR2'): def update_problem_signal(signum, frame): judge.update_problems() signal.signal(signal.SIGUSR2, update_problem_signal) print with monitor, judge: try: judge.listen() except KeyboardInterrupt: pass except: traceback.print_exc() finally: judge.murder()
def main(): # pragma: no cover unicode_stdout_stderr() if not sanity_check(): return 1 from dmoj import judgeenv, contrib, executors judgeenv.load_env() executors.load_executors() contrib.load_contrib_modules() print('Running live judge...') for warning in judgeenv.startup_warnings: print_ansi('#ansi[Warning: %s](yellow)' % warning) del judgeenv.startup_warnings logfile = judgeenv.log_file try: logfile = logfile % env['id'] except TypeError: pass logging.basicConfig( filename=logfile, level=logging.INFO, format='%(levelname)s %(asctime)s %(process)d %(module)s %(message)s') setproctitle('DMOJ Judge %s on %s' % (env['id'], make_host_port(judgeenv))) judge = ClassicJudge( judgeenv.server_host, judgeenv.server_port, secure=judgeenv.secure, no_cert_check=judgeenv.no_cert_check, cert_store=judgeenv.cert_store, ) monitor = Monitor() monitor.callback = judge.update_problems if hasattr(signal, 'SIGUSR2'): def update_problem_signal(signum, frame): judge.update_problems() signal.signal(signal.SIGUSR2, update_problem_signal) if judgeenv.api_listen: judge_instance = judge class Handler(JudgeControlRequestHandler): judge = judge_instance api_server = HTTPServer(judgeenv.api_listen, Handler) thread = threading.Thread(target=api_server.serve_forever) thread.daemon = True thread.start() else: api_server = None print() with monitor: try: judge.listen() except KeyboardInterrupt: pass except Exception: traceback.print_exc() finally: judge.murder() if api_server: api_server.shutdown()
class JudgeManager(object): signal_map = { k: v for v, k in sorted(signal.__dict__.items(), reverse=True) if v.startswith('SIG') and not v.startswith('SIG_') } def __init__(self, judges): self.libc = self.__get_libc() self.prctl = self.libc.prctl self._try_respawn = True self.auth = {entry.id: entry.key for entry in judges} self.orig_signal = {} self.master_pid = os.getpid() self.pids = {} self.monitor_pid = None self.api_pid = None self.monitor = Monitor() def __get_libc(self): from ctypes.util import find_library from ctypes import CDLL return CDLL(find_library('c')) def _forward_signal(self, sig, respawn=False): def handler(signum, frame): # SIGUSR2, the signal for file updates, may be triggered very quickly. # Due to processing delays, it may cause reentrancy issues when logging. # Band-aid fix is to avoid logging SIGUSR2. if signum not in (signal.SIGUSR2, ): logpm.info('Received signal (%s), forwarding...', self.signal_map.get(signum, signum)) if not respawn: logpm.info('Will no longer respawn judges.') self._try_respawn = False self.signal_all(signum) self.orig_signal[sig] = signal.signal(sig, handler) def _spawn_child(self, func, *args, **kwargs): sys.stdout.flush() sys.stderr.flush() ppid = os.getpid() # Pipe to signal signal handler initialization. pr, pw = os.pipe() try: pid = os.fork() except OSError: logpm.exception('Failed to spawn child process.') return if pid == 0: # In child. Scary business. self.prctl(PR_SET_PDEATHSIG, signal.SIGTERM) if ppid != os.getppid(): os.kill(os.getpid(), signal.SIGTERM) os._exit(2) sys.stdin.close() os.close(pr) for sig, handler in self.orig_signal.items(): signal.signal(sig, handler) os.close(pw) # How could we possibly return to top level? try: os._exit(func(*args, **kwargs) or 0) finally: os._exit( 1 ) # If that os._exit fails because ret is a truthy non-int, then this will ensure death. # In parent. os.close(pw) # Block until child initializes signals before we register this child to receive signals. while True: try: os.read(pr, 1) except OSError as e: if e.errno != errno.EINTR: raise else: break os.close(pr) return pid def _judge_proc(self, id): env['id'] = id env['key'] = self.auth[id] try: return judge_proc(False) except BaseException: return 1 finally: sys.stdout.flush() sys.stderr.flush() logging.shutdown() def _spawn_judge(self, id): pid = self._spawn_child(self._judge_proc, id) self.pids[pid] = id logpm.info('Judge %s is pid %d', id, pid) def _spawn_monitor(self): def monitor_proc(): setproctitle('DMOJ Judge: File monitor') signal.signal(signal.SIGUSR2, signal.SIG_IGN) event = threading.Event() stop = False def worker(): while True: event.wait() event.clear() if stop: return event.wait(1) if event.is_set(): continue os.kill(self.master_pid, signal.SIGUSR2) threading.Thread(target=worker).start() self.monitor.callback = event.set self.monitor.start() try: self.monitor.join() except KeyboardInterrupt: self.monitor.stop() stop = True event.set() self.monitor_pid = self._spawn_child(monitor_proc) logpm.info('Monitor is pid %d', self.monitor_pid) def _spawn_api(self): from dmoj import judgeenv master_pid = self.master_pid class Handler(JudgeControlRequestHandler): def update_problems(self): os.kill(master_pid, signal.SIGUSR2) server = HTTPServer(judgeenv.api_listen, Handler) def api_proc(): setproctitle('DMOJ Judge: API server') signal.signal(signal.SIGUSR2, signal.SIG_IGN) server.serve_forever() self.api_pid = self._spawn_child(api_proc) logpm.info('API server is pid %d', self.api_pid) def _spawn_all(self): from dmoj import judgeenv for id in self.auth: logpm.info('Spawning judge: %s', id) self._spawn_judge(id) if self.monitor.is_real: logpm.info('Spawning monitor') self._spawn_monitor() if judgeenv.api_listen is not None: logpm.info('Spawning API server') self._spawn_api() def _monitor(self): while self._try_respawn or self.pids: try: pid, status = os.wait() except (OSError, IOError) as e: if e.errno == errno.EINTR: continue raise if not os.WIFSIGNALED(status) and not os.WIFEXITED(status): continue if pid in self.pids: # A child just died. judge = self.pids[pid] del self.pids[pid] if self._try_respawn: logpm.warning( 'Judge died, respawning: %s (pid %d, 0x%08X)', judge, pid, status) self._spawn_judge(judge) else: logpm.info('Judge exited: %s (pid %d, 0x%08X)', judge, pid, status) elif pid == self.monitor_pid: if self._try_respawn: logpm.warning('Monitor died, respawning (0x%08X)', status) self._spawn_monitor() else: logpm.info('Monitor exited: (0x%08X)', status) elif pid == self.api_pid: if self._try_respawn: logpm.warning('API server died, respawning (0x%08X)', status) self._spawn_api() else: logpm.info('API server exited: (0x%08X)', status) else: logpm.error('I am not your father, %d (0x%08X)!', pid, status) def _respawn_judges(self, signum, frame): logpm.info('Received signal (%s), murderizing all children.', self.signal_map.get(signum, signum)) self.signal_all(signal.SIGTERM) def run(self): logpm.info('Starting process manager: %d.', os.getpid()) from dmoj import judgeenv setproctitle('DMOJ Judge: Process manager on %s' % (make_host_port(judgeenv), )) self._forward_signal(signal.SIGUSR2, respawn=True) self._forward_signal(signal.SIGINT) self._forward_signal(signal.SIGQUIT) self._forward_signal(signal.SIGTERM) signal.signal(signal.SIGHUP, self._respawn_judges) self._spawn_all() try: self._monitor() except KeyboardInterrupt: self._try_respawn = False self.signal_all(signal.SIGINT) self._monitor() logpm.info('Exited gracefully: %d.', os.getpid()) def signal_all(self, signum): for pid in chain(self.pids, [self.monitor_pid, self.api_pid]): if pid is None: continue try: os.kill(pid, signum) except OSError as e: if e.errno != errno.ESRCH: raise
class JudgeManager(object): signal_map = { k: v for v, k in sorted(signal.__dict__.items(), reverse=True) if v.startswith('SIG') and not v.startswith('SIG_') } def __init__(self, judges): self.libc = self.__get_libc() self.prctl = self.libc.prctl self._try_respawn = True self.auth = {entry.id: entry.key for entry in judges} self.orig_signal = {} self.master_pid = os.getpid() self.pids = {} self.monitor_pid = None self.api_pid = None self.monitor = Monitor() self.monitor.callback = lambda: os.kill(self.master_pid, signal.SIGUSR2 ) def __get_libc(self): from ctypes.util import find_library from ctypes import CDLL return CDLL(find_library('c')) def _forward_signal(self, sig, respawn=False): def handler(signum, frame): print >> sys.stderr, 'judgepm: Received signal (%s), forwarding...' % self.signal_map.get( signum, signum) if not respawn: print >> sys.stderr, 'judgepm: Will no longer respawn judges.' self._try_respawn = False self.signal_all(signum) self.orig_signal[sig] = signal.signal(sig, handler) def _spawn_child(self, func, *args, **kwargs): sys.stdout.flush() sys.stderr.flush() ppid = os.getpid() try: pid = os.fork() except OSError: print >> sys.stderr, 'judgepm: Failed to spawn judge:', id return if pid == 0: # In child. Scary business. self.prctl(PR_SET_PDEATHSIG, signal.SIGTERM) if ppid != os.getppid(): os.kill(os.getpid(), signal.SIGTERM) os._exit(2) sys.stdin.close() for sig, handler in self.orig_signal.iteritems(): signal.signal(sig, handler) # How could we possibly return to top level? try: os._exit(func(*args, **kwargs) or 0) finally: os._exit( 1 ) # If that os._exit fails because ret is a truthy non-int, then this will ensure death. return pid def _judge_proc(self, id): env['id'] = id env['key'] = self.auth[id] try: return judge_proc(False) except BaseException: return 1 finally: sys.stdout.flush() sys.stderr.flush() logging.shutdown() def _spawn_judge(self, id): pid = self._spawn_child(self._judge_proc, id) self.pids[pid] = id def _spawn_monitor(self): def monitor_proc(): signal.signal(signal.SIGUSR2, signal.SIG_IGN) self.monitor.start() try: self.monitor.join() except KeyboardInterrupt: self.monitor.stop() self.monitor_pid = self._spawn_child(monitor_proc) def _spawn_api(self): from dmoj import judgeenv from BaseHTTPServer import HTTPServer master_pid = self.master_pid class Handler(JudgeControlRequestHandler): def update_problems(self): os.kill(master_pid, signal.SIGUSR2) server = HTTPServer(judgeenv.api_listen, Handler) def api_proc(): signal.signal(signal.SIGUSR2, signal.SIG_IGN) server.serve_forever() self.api_pid = self._spawn_child(api_proc) def _spawn_all(self): from dmoj import judgeenv for id in self.auth: print >> sys.stderr, 'judgepm: Spawning judge:', id self._spawn_judge(id) if self.monitor.is_real: print >> sys.stderr, 'judgepm: Spawning monitor' self._spawn_monitor() if judgeenv.api_listen is not None: print >> sys.stderr, 'judgepm: Spawning API server' self._spawn_api() def _monitor(self): while self._try_respawn or self.pids: try: pid, status = os.wait() except (OSError, IOError) as e: if e.errno == errno.EINTR: continue raise if not os.WIFSIGNALED(status) and not os.WIFEXITED(status): continue if pid in self.pids: # A child just died. judge = self.pids[pid] del self.pids[pid] if self._try_respawn: print >> sys.stderr, 'judgepm: Judge died, respawning: %s (0x%08X)' % ( judge, status) self._spawn_judge(judge) else: print >> sys.stderr, 'judgepm: Judge exited: %s (0x%08X)' % ( judge, status) elif pid == self.monitor_pid: if self._try_respawn: print >> sys.stderr, 'judgepm: Monitor died, respawning (0x%08X)' % status self._spawn_monitor() else: print >> sys.stderr, 'judgepm: Monitor exited: (0x%08X)' % status elif pid == self.api_pid: if self._try_respawn: print >> sys.stderr, 'judgepm: API server died, respawning (0x%08X)' % status self._spawn_api() else: print >> sys.stderr, 'judgepm: API server exited: (0x%08X)' % status else: print >> sys.stderr, 'judgepm: I am not your father, %d (0x%08X)!' % ( pid, status) def run(self): print >> sys.stderr, 'judgepm: Starting process manager: %d.' % os.getpid( ) self._forward_signal(signal.SIGUSR2, respawn=True) self._forward_signal(signal.SIGINT) self._forward_signal(signal.SIGHUP) self._forward_signal(signal.SIGQUIT) self._forward_signal(signal.SIGTERM) self._spawn_all() try: self._monitor() except KeyboardInterrupt: self._try_respawn = False self.signal_all(signal.SIGINT) self._monitor() print >> sys.stderr, 'judgepm: Exited gracefully: %d.' % os.getpid() def signal_all(self, signum): for pid in chain(self.pids, [self.monitor_pid, self.api_pid]): if pid is None: continue try: os.kill(pid, signum) except OSError as e: if e.errno != errno.ESRCH: raise
class JudgeManager(object): signal_map = {k: v for v, k in sorted(signal.__dict__.items(), reverse=True) if v.startswith('SIG') and not v.startswith('SIG_')} def __init__(self, judges): self.libc = self.__get_libc() self.prctl = self.libc.prctl self._try_respawn = True self.auth = {entry.id: entry.key for entry in judges} self.orig_signal = {} self.master_pid = os.getpid() self.pids = {} self.monitor_pid = None self.api_pid = None self.monitor = Monitor() def __get_libc(self): from ctypes.util import find_library from ctypes import CDLL return CDLL(find_library('c')) def _forward_signal(self, sig, respawn=False): def handler(signum, frame): # SIGUSR2, the signal for file updates, may be triggered very quickly. # Due to processing delays, it may cause reentrancy issues when logging. # Band-aid fix is to avoid logging SIGUSR2. if signum not in (signal.SIGUSR2,): logpm.info('Received signal (%s), forwarding...', self.signal_map.get(signum, signum)) if not respawn: logpm.info('Will no longer respawn judges.') self._try_respawn = False self.signal_all(signum) self.orig_signal[sig] = signal.signal(sig, handler) def _spawn_child(self, func, *args, **kwargs): sys.stdout.flush() sys.stderr.flush() ppid = os.getpid() # Pipe to signal signal handler initialization. pr, pw = os.pipe() try: pid = os.fork() except OSError: logpm.exception('Failed to spawn child process.') return if pid == 0: # In child. Scary business. self.prctl(PR_SET_PDEATHSIG, signal.SIGTERM) if ppid != os.getppid(): os.kill(os.getpid(), signal.SIGTERM) os._exit(2) sys.stdin.close() os.close(pr) for sig, handler in self.orig_signal.items(): signal.signal(sig, handler) os.close(pw) # How could we possibly return to top level? try: os._exit(func(*args, **kwargs) or 0) finally: os._exit(1) # If that os._exit fails because ret is a truthy non-int, then this will ensure death. # In parent. os.close(pw) # Block until child initializes signals before we register this child to receive signals. while True: try: os.read(pr, 1) except OSError as e: if e.errno != errno.EINTR: raise else: break os.close(pr) return pid def _judge_proc(self, id): env['id'] = id env['key'] = self.auth[id] try: return judge_proc(False) except BaseException: return 1 finally: sys.stdout.flush() sys.stderr.flush() logging.shutdown() def _spawn_judge(self, id): pid = self._spawn_child(self._judge_proc, id) self.pids[pid] = id logpm.info('Judge %s is pid %d', id, pid) def _spawn_monitor(self): def monitor_proc(): setproctitle('DMOJ Judge: File monitor') signal.signal(signal.SIGUSR2, signal.SIG_IGN) event = threading.Event() stop = False def worker(): while True: event.wait() event.clear() if stop: return event.wait(1) if event.is_set(): continue os.kill(self.master_pid, signal.SIGUSR2) threading.Thread(target=worker).start() self.monitor.callback = event.set self.monitor.start() try: self.monitor.join() except KeyboardInterrupt: self.monitor.stop() stop = True event.set() self.monitor_pid = self._spawn_child(monitor_proc) logpm.info('Monitor is pid %d', self.monitor_pid) def _spawn_api(self): from dmoj import judgeenv master_pid = self.master_pid class Handler(JudgeControlRequestHandler): def update_problems(self): os.kill(master_pid, signal.SIGUSR2) server = HTTPServer(judgeenv.api_listen, Handler) def api_proc(): setproctitle('DMOJ Judge: API server') signal.signal(signal.SIGUSR2, signal.SIG_IGN) server.serve_forever() self.api_pid = self._spawn_child(api_proc) logpm.info('API server is pid %d', self.api_pid) def _spawn_all(self): from dmoj import judgeenv for id in self.auth: logpm.info('Spawning judge: %s', id) self._spawn_judge(id) if self.monitor.is_real: logpm.info('Spawning monitor') self._spawn_monitor() if judgeenv.api_listen is not None: logpm.info('Spawning API server') self._spawn_api() def _monitor(self): while self._try_respawn or self.pids: try: pid, status = os.wait() except (OSError, IOError) as e: if e.errno == errno.EINTR: continue raise if not os.WIFSIGNALED(status) and not os.WIFEXITED(status): continue if pid in self.pids: # A child just died. judge = self.pids[pid] del self.pids[pid] if self._try_respawn: logpm.warning('Judge died, respawning: %s (pid %d, 0x%08X)', judge, pid, status) self._spawn_judge(judge) else: logpm.info('Judge exited: %s (pid %d, 0x%08X)', judge, pid, status) elif pid == self.monitor_pid: if self._try_respawn: logpm.warning('Monitor died, respawning (0x%08X)', status) self._spawn_monitor() else: logpm.info('Monitor exited: (0x%08X)', status) elif pid == self.api_pid: if self._try_respawn: logpm.warning('API server died, respawning (0x%08X)', status) self._spawn_api() else: logpm.info('API server exited: (0x%08X)', status) else: logpm.error('I am not your father, %d (0x%08X)!', pid, status) def _respawn_judges(self, signum, frame): logpm.info('Received signal (%s), murderizing all children.', self.signal_map.get(signum, signum)) self.signal_all(signal.SIGTERM) def run(self): logpm.info('Starting process manager: %d.', os.getpid()) from dmoj import judgeenv setproctitle('DMOJ Judge: Process manager on %s' % (make_host_port(judgeenv),)) self._forward_signal(signal.SIGUSR2, respawn=True) self._forward_signal(signal.SIGINT) self._forward_signal(signal.SIGQUIT) self._forward_signal(signal.SIGTERM) signal.signal(signal.SIGHUP, self._respawn_judges) self._spawn_all() try: self._monitor() except KeyboardInterrupt: self._try_respawn = False self.signal_all(signal.SIGINT) self._monitor() logpm.info('Exited gracefully: %d.', os.getpid()) def signal_all(self, signum): for pid in chain(self.pids, [self.monitor_pid, self.api_pid]): if pid is None: continue try: os.kill(pid, signum) except OSError as e: if e.errno != errno.ESRCH: raise