Esempio n. 1
0
def judge_proc(need_monitor):
    from dmoj import judgeenv

    logfile = judgeenv.log_file

    try:
        logfile = logfile % env['id']
    except TypeError:
        pass

    logging.basicConfig(
        filename=logfile,
        level=logging.INFO,
        format='%(levelname)s %(asctime)s %(process)d %(module)s %(message)s')

    setproctitle('DMOJ Judge: %s on %s' %
                 (env['id'], make_host_port(judgeenv)))

    judge = ClassicJudge(judgeenv.server_host,
                         judgeenv.server_port,
                         secure=judgeenv.secure,
                         no_cert_check=judgeenv.no_cert_check,
                         cert_store=judgeenv.cert_store)
    if need_monitor:
        monitor = Monitor()
        monitor.callback = judge.update_problems
    else:
        monitor = DummyMonitor()

    if hasattr(signal, 'SIGUSR2'):

        def update_problem_signal(signum, frame):
            judge.update_problems()

        signal.signal(signal.SIGUSR2, update_problem_signal)

    if need_monitor and judgeenv.api_listen:
        judge_instance = judge

        class Handler(JudgeControlRequestHandler):
            judge = judge_instance

        api_server = HTTPServer(judgeenv.api_listen, Handler)
        thread = threading.Thread(target=api_server.serve_forever)
        thread.daemon = True
        thread.start()
    else:
        api_server = None

    print()
    with monitor, judge:
        try:
            judge.listen()
        except Exception:
            traceback.print_exc()
        finally:
            judge.murder()
            if api_server:
                api_server.shutdown()
Esempio n. 2
0
File: judge.py Progetto: DMOJ/judge
def judge_proc(need_monitor):
    from dmoj import judgeenv

    logfile = judgeenv.log_file

    try:
        logfile = logfile % env['id']
    except TypeError:
        pass

    logging.basicConfig(filename=logfile, level=logging.INFO,
                        format='%(levelname)s %(asctime)s %(process)d %(module)s %(message)s')

    proctitle = 'DMOJ Judge: %s on %s' % (env['id'], make_host_port(judgeenv))
    if six.PY2:
        setproctitle(utf8bytes(proctitle))
    else:
        setproctitle(proctitle)

    judge = ClassicJudge(judgeenv.server_host, judgeenv.server_port,
                         secure=judgeenv.secure, no_cert_check=judgeenv.no_cert_check,
                         cert_store=judgeenv.cert_store)
    if need_monitor:
        monitor = Monitor()
        monitor.callback = judge.update_problems
    else:
        monitor = DummyMonitor()

    if hasattr(signal, 'SIGUSR2'):
        def update_problem_signal(signum, frame):
            judge.update_problems()

        signal.signal(signal.SIGUSR2, update_problem_signal)

    if need_monitor and judgeenv.api_listen:
        judge_instance = judge

        class Handler(JudgeControlRequestHandler):
            judge = judge_instance

        api_server = HTTPServer(judgeenv.api_listen, Handler)
        thread = threading.Thread(target=api_server.serve_forever)
        thread.daemon = True
        thread.start()
    else:
        api_server = None

    print()
    with monitor, judge:
        try:
            judge.listen()
        except Exception:
            traceback.print_exc()
        finally:
            judge.murder()
            if api_server:
                api_server.shutdown()
Esempio n. 3
0
def judge_proc(need_monitor):
    from dmoj import judgeenv

    logfile = judgeenv.log_file

    try:
        logfile = logfile % env['id']
    except TypeError:
        pass

    logging.basicConfig(
        filename=logfile,
        level=logging.INFO,
        format='%(levelname)s %(asctime)s %(module)s %(message)s')

    judge = ClassicJudge(judgeenv.server_host, judgeenv.server_port)
    if need_monitor:
        monitor = Monitor()
        monitor.callback = judge.update_problems
    else:
        monitor = DummyMonitor()

    if hasattr(signal, 'SIGUSR2'):

        def update_problem_signal(signum, frame):
            judge.update_problems()

        signal.signal(signal.SIGUSR2, update_problem_signal)

    if need_monitor and judgeenv.api_listen:
        from BaseHTTPServer import HTTPServer
        judge_instance = judge

        class Handler(JudgeControlRequestHandler):
            judge = judge_instance

        api_server = HTTPServer(judgeenv.api_listen, Handler)
        thread = threading.Thread(target=api_server.serve_forever)
        thread.daemon = True
        thread.start()
    else:
        api_server = None

    print
    with monitor, judge:
        try:
            judge.listen()
        except KeyboardInterrupt:
            pass
        except:
            traceback.print_exc()
        finally:
            judge.murder()
            if api_server:
                api_server.shutdown()
Esempio n. 4
0
    def __init__(self, judges):
        self.libc = self.__get_libc()
        self.prctl = self.libc.prctl

        self._try_respawn = True
        self.auth = {entry.id: entry.key for entry in judges}
        self.orig_signal = {}

        self.master_pid = os.getpid()
        self.pids = {}
        self.monitor_pid = None
        self.api_pid = None

        self.monitor = Monitor()
Esempio n. 5
0
    def __init__(self, judges):
        self.libc = self.__get_libc()
        self.prctl = self.libc.prctl

        self._try_respawn = True
        self.auth = {entry.id: entry.key for entry in judges}
        self.pids = {}
        self.orig_signal = {}
        self.monitor = Monitor()
        self.monitor.callback = lambda: self.signal_all(signal.SIGUSR2)
Esempio n. 6
0
def judge_proc(need_monitor):
    from dmoj import judgeenv

    logfile = judgeenv.log_file

    try:
        logfile = logfile % env['id']
    except TypeError:
        pass

    logging.basicConfig(
        filename=logfile,
        level=logging.INFO,
        format='%(levelname)s %(asctime)s %(module)s %(message)s')

    judge = ClassicJudge(judgeenv.server_host, judgeenv.server_port)
    if need_monitor:
        monitor = Monitor()
        monitor.callback = judge.update_problems
    else:
        monitor = DummyMonitor()

    if hasattr(signal, 'SIGUSR2'):

        def update_problem_signal(signum, frame):
            judge.update_problems()

        signal.signal(signal.SIGUSR2, update_problem_signal)

    print
    with monitor, judge:
        try:
            judge.listen()
        except KeyboardInterrupt:
            pass
        except:
            traceback.print_exc()
        finally:
            judge.murder()
Esempio n. 7
0
File: judge.py Progetto: DMOJ/judge
    def __init__(self, judges):
        self.libc = self.__get_libc()
        self.prctl = self.libc.prctl

        self._try_respawn = True
        self.auth = {entry.id: entry.key for entry in judges}
        self.orig_signal = {}

        self.master_pid = os.getpid()
        self.pids = {}
        self.monitor_pid = None
        self.api_pid = None

        self.monitor = Monitor()
Esempio n. 8
0
def main():  # pragma: no cover
    unicode_stdout_stderr()

    if not sanity_check():
        return 1

    from dmoj import judgeenv, contrib, executors

    judgeenv.load_env()

    executors.load_executors()
    contrib.load_contrib_modules()

    print('Running live judge...')

    for warning in judgeenv.startup_warnings:
        print_ansi('#ansi[Warning: %s](yellow)' % warning)
    del judgeenv.startup_warnings

    logfile = judgeenv.log_file

    try:
        logfile = logfile % env['id']
    except TypeError:
        pass

    logging.basicConfig(
        filename=logfile,
        level=logging.INFO,
        format='%(levelname)s %(asctime)s %(process)d %(module)s %(message)s')

    setproctitle('DMOJ Judge %s on %s' % (env['id'], make_host_port(judgeenv)))

    judge = ClassicJudge(
        judgeenv.server_host,
        judgeenv.server_port,
        secure=judgeenv.secure,
        no_cert_check=judgeenv.no_cert_check,
        cert_store=judgeenv.cert_store,
    )
    monitor = Monitor()
    monitor.callback = judge.update_problems

    if hasattr(signal, 'SIGUSR2'):

        def update_problem_signal(signum, frame):
            judge.update_problems()

        signal.signal(signal.SIGUSR2, update_problem_signal)

    if judgeenv.api_listen:
        judge_instance = judge

        class Handler(JudgeControlRequestHandler):
            judge = judge_instance

        api_server = HTTPServer(judgeenv.api_listen, Handler)
        thread = threading.Thread(target=api_server.serve_forever)
        thread.daemon = True
        thread.start()
    else:
        api_server = None

    print()
    with monitor:
        try:
            judge.listen()
        except KeyboardInterrupt:
            pass
        except Exception:
            traceback.print_exc()
        finally:
            judge.murder()
            if api_server:
                api_server.shutdown()
Esempio n. 9
0
class JudgeManager(object):
    signal_map = {
        k: v
        for v, k in sorted(signal.__dict__.items(), reverse=True)
        if v.startswith('SIG') and not v.startswith('SIG_')
    }

    def __init__(self, judges):
        self.libc = self.__get_libc()
        self.prctl = self.libc.prctl

        self._try_respawn = True
        self.auth = {entry.id: entry.key for entry in judges}
        self.orig_signal = {}

        self.master_pid = os.getpid()
        self.pids = {}
        self.monitor_pid = None
        self.api_pid = None

        self.monitor = Monitor()

    def __get_libc(self):
        from ctypes.util import find_library
        from ctypes import CDLL
        return CDLL(find_library('c'))

    def _forward_signal(self, sig, respawn=False):
        def handler(signum, frame):
            # SIGUSR2, the signal for file updates, may be triggered very quickly.
            # Due to processing delays, it may cause reentrancy issues when logging.
            # Band-aid fix is to avoid logging SIGUSR2.
            if signum not in (signal.SIGUSR2, ):
                logpm.info('Received signal (%s), forwarding...',
                           self.signal_map.get(signum, signum))
                if not respawn:
                    logpm.info('Will no longer respawn judges.')
                    self._try_respawn = False
            self.signal_all(signum)

        self.orig_signal[sig] = signal.signal(sig, handler)

    def _spawn_child(self, func, *args, **kwargs):
        sys.stdout.flush()
        sys.stderr.flush()
        ppid = os.getpid()

        # Pipe to signal signal handler initialization.
        pr, pw = os.pipe()
        try:
            pid = os.fork()
        except OSError:
            logpm.exception('Failed to spawn child process.')
            return
        if pid == 0:
            # In child. Scary business.
            self.prctl(PR_SET_PDEATHSIG, signal.SIGTERM)
            if ppid != os.getppid():
                os.kill(os.getpid(), signal.SIGTERM)
                os._exit(2)
            sys.stdin.close()
            os.close(pr)

            for sig, handler in self.orig_signal.items():
                signal.signal(sig, handler)
            os.close(pw)

            # How could we possibly return to top level?
            try:
                os._exit(func(*args, **kwargs) or 0)
            finally:
                os._exit(
                    1
                )  # If that os._exit fails because ret is a truthy non-int, then this will ensure death.

        # In parent.
        os.close(pw)

        # Block until child initializes signals before we register this child to receive signals.
        while True:
            try:
                os.read(pr, 1)
            except OSError as e:
                if e.errno != errno.EINTR:
                    raise
            else:
                break
        os.close(pr)
        return pid

    def _judge_proc(self, id):
        env['id'] = id
        env['key'] = self.auth[id]
        try:
            return judge_proc(False)
        except BaseException:
            return 1
        finally:
            sys.stdout.flush()
            sys.stderr.flush()
            logging.shutdown()

    def _spawn_judge(self, id):
        pid = self._spawn_child(self._judge_proc, id)
        self.pids[pid] = id
        logpm.info('Judge %s is pid %d', id, pid)

    def _spawn_monitor(self):
        def monitor_proc():
            setproctitle('DMOJ Judge: File monitor')
            signal.signal(signal.SIGUSR2, signal.SIG_IGN)

            event = threading.Event()
            stop = False

            def worker():
                while True:
                    event.wait()
                    event.clear()
                    if stop:
                        return
                    event.wait(1)
                    if event.is_set():
                        continue
                    os.kill(self.master_pid, signal.SIGUSR2)

            threading.Thread(target=worker).start()
            self.monitor.callback = event.set
            self.monitor.start()
            try:
                self.monitor.join()
            except KeyboardInterrupt:
                self.monitor.stop()
                stop = True
                event.set()

        self.monitor_pid = self._spawn_child(monitor_proc)
        logpm.info('Monitor is pid %d', self.monitor_pid)

    def _spawn_api(self):
        from dmoj import judgeenv

        master_pid = self.master_pid

        class Handler(JudgeControlRequestHandler):
            def update_problems(self):
                os.kill(master_pid, signal.SIGUSR2)

        server = HTTPServer(judgeenv.api_listen, Handler)

        def api_proc():
            setproctitle('DMOJ Judge: API server')
            signal.signal(signal.SIGUSR2, signal.SIG_IGN)
            server.serve_forever()

        self.api_pid = self._spawn_child(api_proc)
        logpm.info('API server is pid %d', self.api_pid)

    def _spawn_all(self):
        from dmoj import judgeenv

        for id in self.auth:
            logpm.info('Spawning judge: %s', id)
            self._spawn_judge(id)
        if self.monitor.is_real:
            logpm.info('Spawning monitor')
            self._spawn_monitor()
        if judgeenv.api_listen is not None:
            logpm.info('Spawning API server')
            self._spawn_api()

    def _monitor(self):
        while self._try_respawn or self.pids:
            try:
                pid, status = os.wait()
            except (OSError, IOError) as e:
                if e.errno == errno.EINTR:
                    continue
                raise
            if not os.WIFSIGNALED(status) and not os.WIFEXITED(status):
                continue
            if pid in self.pids:
                # A child just died.
                judge = self.pids[pid]
                del self.pids[pid]
                if self._try_respawn:
                    logpm.warning(
                        'Judge died, respawning: %s (pid %d, 0x%08X)', judge,
                        pid, status)
                    self._spawn_judge(judge)
                else:
                    logpm.info('Judge exited: %s (pid %d, 0x%08X)', judge, pid,
                               status)
            elif pid == self.monitor_pid:
                if self._try_respawn:
                    logpm.warning('Monitor died, respawning (0x%08X)', status)
                    self._spawn_monitor()
                else:
                    logpm.info('Monitor exited: (0x%08X)', status)
            elif pid == self.api_pid:
                if self._try_respawn:
                    logpm.warning('API server died, respawning (0x%08X)',
                                  status)
                    self._spawn_api()
                else:
                    logpm.info('API server exited: (0x%08X)', status)
            else:
                logpm.error('I am not your father, %d (0x%08X)!', pid, status)

    def _respawn_judges(self, signum, frame):
        logpm.info('Received signal (%s), murderizing all children.',
                   self.signal_map.get(signum, signum))
        self.signal_all(signal.SIGTERM)

    def run(self):
        logpm.info('Starting process manager: %d.', os.getpid())

        from dmoj import judgeenv
        setproctitle('DMOJ Judge: Process manager on %s' %
                     (make_host_port(judgeenv), ))

        self._forward_signal(signal.SIGUSR2, respawn=True)
        self._forward_signal(signal.SIGINT)
        self._forward_signal(signal.SIGQUIT)
        self._forward_signal(signal.SIGTERM)
        signal.signal(signal.SIGHUP, self._respawn_judges)

        self._spawn_all()
        try:
            self._monitor()
        except KeyboardInterrupt:
            self._try_respawn = False
            self.signal_all(signal.SIGINT)
            self._monitor()

        logpm.info('Exited gracefully: %d.', os.getpid())

    def signal_all(self, signum):
        for pid in chain(self.pids, [self.monitor_pid, self.api_pid]):
            if pid is None:
                continue
            try:
                os.kill(pid, signum)
            except OSError as e:
                if e.errno != errno.ESRCH:
                    raise
Esempio n. 10
0
class JudgeManager(object):
    signal_map = {
        k: v
        for v, k in sorted(signal.__dict__.items(), reverse=True)
        if v.startswith('SIG') and not v.startswith('SIG_')
    }

    def __init__(self, judges):
        self.libc = self.__get_libc()
        self.prctl = self.libc.prctl

        self._try_respawn = True
        self.auth = {entry.id: entry.key for entry in judges}
        self.orig_signal = {}

        self.master_pid = os.getpid()
        self.pids = {}
        self.monitor_pid = None
        self.api_pid = None

        self.monitor = Monitor()
        self.monitor.callback = lambda: os.kill(self.master_pid, signal.SIGUSR2
                                                )

    def __get_libc(self):
        from ctypes.util import find_library
        from ctypes import CDLL
        return CDLL(find_library('c'))

    def _forward_signal(self, sig, respawn=False):
        def handler(signum, frame):
            print >> sys.stderr, 'judgepm: Received signal (%s), forwarding...' % self.signal_map.get(
                signum, signum)
            if not respawn:
                print >> sys.stderr, 'judgepm: Will no longer respawn judges.'
                self._try_respawn = False
            self.signal_all(signum)

        self.orig_signal[sig] = signal.signal(sig, handler)

    def _spawn_child(self, func, *args, **kwargs):
        sys.stdout.flush()
        sys.stderr.flush()
        ppid = os.getpid()
        try:
            pid = os.fork()
        except OSError:
            print >> sys.stderr, 'judgepm: Failed to spawn judge:', id
            return
        if pid == 0:
            # In child. Scary business.
            self.prctl(PR_SET_PDEATHSIG, signal.SIGTERM)
            if ppid != os.getppid():
                os.kill(os.getpid(), signal.SIGTERM)
                os._exit(2)
            sys.stdin.close()

            for sig, handler in self.orig_signal.iteritems():
                signal.signal(sig, handler)

            # How could we possibly return to top level?
            try:
                os._exit(func(*args, **kwargs) or 0)
            finally:
                os._exit(
                    1
                )  # If that os._exit fails because ret is a truthy non-int, then this will ensure death.
        return pid

    def _judge_proc(self, id):
        env['id'] = id
        env['key'] = self.auth[id]
        try:
            return judge_proc(False)
        except BaseException:
            return 1
        finally:
            sys.stdout.flush()
            sys.stderr.flush()
            logging.shutdown()

    def _spawn_judge(self, id):
        pid = self._spawn_child(self._judge_proc, id)
        self.pids[pid] = id

    def _spawn_monitor(self):
        def monitor_proc():
            signal.signal(signal.SIGUSR2, signal.SIG_IGN)

            self.monitor.start()
            try:
                self.monitor.join()
            except KeyboardInterrupt:
                self.monitor.stop()

        self.monitor_pid = self._spawn_child(monitor_proc)

    def _spawn_api(self):
        from dmoj import judgeenv
        from BaseHTTPServer import HTTPServer

        master_pid = self.master_pid

        class Handler(JudgeControlRequestHandler):
            def update_problems(self):
                os.kill(master_pid, signal.SIGUSR2)

        server = HTTPServer(judgeenv.api_listen, Handler)

        def api_proc():
            signal.signal(signal.SIGUSR2, signal.SIG_IGN)
            server.serve_forever()

        self.api_pid = self._spawn_child(api_proc)

    def _spawn_all(self):
        from dmoj import judgeenv

        for id in self.auth:
            print >> sys.stderr, 'judgepm: Spawning judge:', id
            self._spawn_judge(id)
        if self.monitor.is_real:
            print >> sys.stderr, 'judgepm: Spawning monitor'
            self._spawn_monitor()
        if judgeenv.api_listen is not None:
            print >> sys.stderr, 'judgepm: Spawning API server'
            self._spawn_api()

    def _monitor(self):
        while self._try_respawn or self.pids:
            try:
                pid, status = os.wait()
            except (OSError, IOError) as e:
                if e.errno == errno.EINTR:
                    continue
                raise
            if not os.WIFSIGNALED(status) and not os.WIFEXITED(status):
                continue
            if pid in self.pids:
                # A child just died.
                judge = self.pids[pid]
                del self.pids[pid]
                if self._try_respawn:
                    print >> sys.stderr, 'judgepm: Judge died, respawning: %s (0x%08X)' % (
                        judge, status)
                    self._spawn_judge(judge)
                else:
                    print >> sys.stderr, 'judgepm: Judge exited: %s (0x%08X)' % (
                        judge, status)
            elif pid == self.monitor_pid:
                if self._try_respawn:
                    print >> sys.stderr, 'judgepm: Monitor died, respawning (0x%08X)' % status
                    self._spawn_monitor()
                else:
                    print >> sys.stderr, 'judgepm: Monitor exited: (0x%08X)' % status
            elif pid == self.api_pid:
                if self._try_respawn:
                    print >> sys.stderr, 'judgepm: API server died, respawning (0x%08X)' % status
                    self._spawn_api()
                else:
                    print >> sys.stderr, 'judgepm: API server exited: (0x%08X)' % status
            else:
                print >> sys.stderr, 'judgepm: I am not your father, %d (0x%08X)!' % (
                    pid, status)

    def run(self):
        print >> sys.stderr, 'judgepm: Starting process manager: %d.' % os.getpid(
        )

        self._forward_signal(signal.SIGUSR2, respawn=True)
        self._forward_signal(signal.SIGINT)
        self._forward_signal(signal.SIGHUP)
        self._forward_signal(signal.SIGQUIT)
        self._forward_signal(signal.SIGTERM)

        self._spawn_all()
        try:
            self._monitor()
        except KeyboardInterrupt:
            self._try_respawn = False
            self.signal_all(signal.SIGINT)
            self._monitor()
        print >> sys.stderr, 'judgepm: Exited gracefully: %d.' % os.getpid()

    def signal_all(self, signum):
        for pid in chain(self.pids, [self.monitor_pid, self.api_pid]):
            if pid is None:
                continue
            try:
                os.kill(pid, signum)
            except OSError as e:
                if e.errno != errno.ESRCH:
                    raise
Esempio n. 11
0
File: judge.py Progetto: DMOJ/judge
class JudgeManager(object):
    signal_map = {k: v for v, k in sorted(signal.__dict__.items(), reverse=True)
                  if v.startswith('SIG') and not v.startswith('SIG_')}

    def __init__(self, judges):
        self.libc = self.__get_libc()
        self.prctl = self.libc.prctl

        self._try_respawn = True
        self.auth = {entry.id: entry.key for entry in judges}
        self.orig_signal = {}

        self.master_pid = os.getpid()
        self.pids = {}
        self.monitor_pid = None
        self.api_pid = None

        self.monitor = Monitor()

    def __get_libc(self):
        from ctypes.util import find_library
        from ctypes import CDLL
        return CDLL(find_library('c'))

    def _forward_signal(self, sig, respawn=False):
        def handler(signum, frame):
            # SIGUSR2, the signal for file updates, may be triggered very quickly.
            # Due to processing delays, it may cause reentrancy issues when logging.
            # Band-aid fix is to avoid logging SIGUSR2.
            if signum not in (signal.SIGUSR2,):
                logpm.info('Received signal (%s), forwarding...', self.signal_map.get(signum, signum))
                if not respawn:
                    logpm.info('Will no longer respawn judges.')
                    self._try_respawn = False
            self.signal_all(signum)

        self.orig_signal[sig] = signal.signal(sig, handler)

    def _spawn_child(self, func, *args, **kwargs):
        sys.stdout.flush()
        sys.stderr.flush()
        ppid = os.getpid()

        # Pipe to signal signal handler initialization.
        pr, pw = os.pipe()
        try:
            pid = os.fork()
        except OSError:
            logpm.exception('Failed to spawn child process.')
            return
        if pid == 0:
            # In child. Scary business.
            self.prctl(PR_SET_PDEATHSIG, signal.SIGTERM)
            if ppid != os.getppid():
                os.kill(os.getpid(), signal.SIGTERM)
                os._exit(2)
            sys.stdin.close()
            os.close(pr)

            for sig, handler in self.orig_signal.items():
                signal.signal(sig, handler)
            os.close(pw)

            # How could we possibly return to top level?
            try:
                os._exit(func(*args, **kwargs) or 0)
            finally:
                os._exit(1)  # If that os._exit fails because ret is a truthy non-int, then this will ensure death.

        # In parent.
        os.close(pw)

        # Block until child initializes signals before we register this child to receive signals.
        while True:
            try:
                os.read(pr, 1)
            except OSError as e:
                if e.errno != errno.EINTR:
                    raise
            else:
                break
        os.close(pr)
        return pid

    def _judge_proc(self, id):
        env['id'] = id
        env['key'] = self.auth[id]
        try:
            return judge_proc(False)
        except BaseException:
            return 1
        finally:
            sys.stdout.flush()
            sys.stderr.flush()
            logging.shutdown()

    def _spawn_judge(self, id):
        pid = self._spawn_child(self._judge_proc, id)
        self.pids[pid] = id
        logpm.info('Judge %s is pid %d', id, pid)

    def _spawn_monitor(self):
        def monitor_proc():
            setproctitle('DMOJ Judge: File monitor')
            signal.signal(signal.SIGUSR2, signal.SIG_IGN)

            event = threading.Event()
            stop = False

            def worker():
                while True:
                    event.wait()
                    event.clear()
                    if stop:
                        return
                    event.wait(1)
                    if event.is_set():
                        continue
                    os.kill(self.master_pid, signal.SIGUSR2)

            threading.Thread(target=worker).start()
            self.monitor.callback = event.set
            self.monitor.start()
            try:
                self.monitor.join()
            except KeyboardInterrupt:
                self.monitor.stop()
                stop = True
                event.set()

        self.monitor_pid = self._spawn_child(monitor_proc)
        logpm.info('Monitor is pid %d', self.monitor_pid)

    def _spawn_api(self):
        from dmoj import judgeenv

        master_pid = self.master_pid

        class Handler(JudgeControlRequestHandler):
            def update_problems(self):
                os.kill(master_pid, signal.SIGUSR2)

        server = HTTPServer(judgeenv.api_listen, Handler)

        def api_proc():
            setproctitle('DMOJ Judge: API server')
            signal.signal(signal.SIGUSR2, signal.SIG_IGN)
            server.serve_forever()

        self.api_pid = self._spawn_child(api_proc)
        logpm.info('API server is pid %d', self.api_pid)

    def _spawn_all(self):
        from dmoj import judgeenv

        for id in self.auth:
            logpm.info('Spawning judge: %s', id)
            self._spawn_judge(id)
        if self.monitor.is_real:
            logpm.info('Spawning monitor')
            self._spawn_monitor()
        if judgeenv.api_listen is not None:
            logpm.info('Spawning API server')
            self._spawn_api()

    def _monitor(self):
        while self._try_respawn or self.pids:
            try:
                pid, status = os.wait()
            except (OSError, IOError) as e:
                if e.errno == errno.EINTR:
                    continue
                raise
            if not os.WIFSIGNALED(status) and not os.WIFEXITED(status):
                continue
            if pid in self.pids:
                # A child just died.
                judge = self.pids[pid]
                del self.pids[pid]
                if self._try_respawn:
                    logpm.warning('Judge died, respawning: %s (pid %d, 0x%08X)', judge, pid, status)
                    self._spawn_judge(judge)
                else:
                    logpm.info('Judge exited: %s (pid %d, 0x%08X)', judge, pid, status)
            elif pid == self.monitor_pid:
                if self._try_respawn:
                    logpm.warning('Monitor died, respawning (0x%08X)', status)
                    self._spawn_monitor()
                else:
                    logpm.info('Monitor exited: (0x%08X)', status)
            elif pid == self.api_pid:
                if self._try_respawn:
                    logpm.warning('API server died, respawning (0x%08X)', status)
                    self._spawn_api()
                else:
                    logpm.info('API server exited: (0x%08X)', status)
            else:
                logpm.error('I am not your father, %d (0x%08X)!', pid, status)

    def _respawn_judges(self, signum, frame):
        logpm.info('Received signal (%s), murderizing all children.', self.signal_map.get(signum, signum))
        self.signal_all(signal.SIGTERM)

    def run(self):
        logpm.info('Starting process manager: %d.', os.getpid())

        from dmoj import judgeenv
        setproctitle('DMOJ Judge: Process manager on %s' % (make_host_port(judgeenv),))

        self._forward_signal(signal.SIGUSR2, respawn=True)
        self._forward_signal(signal.SIGINT)
        self._forward_signal(signal.SIGQUIT)
        self._forward_signal(signal.SIGTERM)
        signal.signal(signal.SIGHUP, self._respawn_judges)

        self._spawn_all()
        try:
            self._monitor()
        except KeyboardInterrupt:
            self._try_respawn = False
            self.signal_all(signal.SIGINT)
            self._monitor()

        logpm.info('Exited gracefully: %d.', os.getpid())

    def signal_all(self, signum):
        for pid in chain(self.pids, [self.monitor_pid, self.api_pid]):
            if pid is None:
                continue
            try:
                os.kill(pid, signum)
            except OSError as e:
                if e.errno != errno.ESRCH:
                    raise