def _sweeper_loop(self):
"""Regularly check the database for unscored results.
Try to sweep the database once every SWEEPER_TIMEOUT seconds
but make sure that no two sweeps run simultaneously. That is,
start a new sweep SWEEPER_TIMEOUT seconds after the previous
one started or when the previous one finished, whatever comes
last.
The search_jobs_not_done RPC method can interfere with this
regularity, as it tries to run a sweeper as soon as possible:
immediately, if no sweeper is running, or as soon as the
current one terminates.
Any error during the sweep is sent to the logger and then
suppressed, because the loop must go on.
"""
while True:
self._sweeper_start = monotonic_time()
self._sweeper_event.clear()
try:
self._sweep()
except Exception:
logger.error("Unexpected error when searching for unscored "
"submissions.", exc_info=True)
self._sweeper_event.wait(max(self._sweeper_start +
self.SWEEPER_TIMEOUT -
monotonic_time(), 0))
def _sweeper_loop(self):
"""Regularly check for missed operations.
Run the sweep once every _sweeper_timeout seconds but make
sure that no two sweeps run simultaneously. That is, start a
new sweep _sweeper_timeout seconds after the previous one
started or when the previous one finished, whatever comes
last.
The search_operations_not_done RPC method can interfere with
this regularity, as it tries to run a sweeper as soon as
possible: immediately, if no sweeper is running, or as soon as
the current one terminates.
Any error during the sweep is sent to the logger and then
suppressed, because the loop must go on.
"""
while True:
self._sweeper_start = monotonic_time()
self._sweeper_event.clear()
try:
self._sweep()
except Exception:
logger.error(
"Unexpected error when searching for missed "
"operations.",
exc_info=True)
self._sweeper_event.wait(
max(
self._sweeper_start + self._sweeper_timeout -
monotonic_time(), 0))
def _sweeper_loop(self):
"""Regularly check for missed operations.
Run the sweep once every _sweeper_timeout() seconds but make
sure that no two sweeps run simultaneously. That is, start a
new sweep _sweeper_timeout() seconds after the previous one
started or when the previous one finished, whatever comes
last.
The search_operations_not_done RPC method can interfere with
this regularity, as it tries to run a sweeper as soon as
possible: immediately, if no sweeper is running, or as soon as
the current one terminates.
Any error during the sweep is sent to the logger and then
suppressed, because the loop must go on.
"""
# If the timeout is None, it means the subclass does not want
# a sweeper.
if self._sweeper_timeout() is None:
return
while True:
self._sweeper_start = monotonic_time()
self._sweeper_event.clear()
try:
self._sweep()
except Exception:
logger.error("Unexpected error when searching for missed "
"operations.", exc_info=True)
self._sweeper_event.wait(max(self._sweeper_start +
self._sweeper_timeout() -
monotonic_time(), 0))
def wait_or_kill(self):
"""Wait for the program to terminate, or kill it after 5s."""
if self.instance.poll() is None:
# We try one more time to kill gracefully using Ctrl-C.
logger.info("Interrupting %s and waiting...", self.coord)
self.instance.send_signal(signal.SIGINT)
# FIXME on py3 this becomes self.instance.wait(timeout=5)
t = monotonic_time()
while monotonic_time() - t < 5:
if self.instance.poll() is not None:
logger.info("Terminated %s.", self.coord)
break
time.sleep(0.1)
else:
self.kill()
def wait_or_kill(self):
"""Wait for the program to terminate, or kill it after 5s."""
if self.instance.poll() is None:
# We try one more time to kill gracefully using Ctrl-C.
logger.info("Interrupting %s and waiting...", self.coord)
self.instance.send_signal(signal.SIGINT)
# FIXME on py3 this becomes self.instance.wait(timeout=5)
t = monotonic_time()
while monotonic_time() - t < 5:
if self.instance.poll() is not None:
logger.info("Terminated %s.", self.coord)
break
time.sleep(0.1)
else:
self.kill()
def __init__(self, size, flush_latency_seconds, callback):
# Elements contained in the dict that force a flush.
self.size = size
# How much time we wait for other key-values before flushing.
self.flush_latency_seconds = flush_latency_seconds
# Function to flush the data to.
self.callback = callback
# This contains all the key-values received and not yet
# flushed.
self.d = dict()
# This contains all the key-values that are currently being flushed
self.fd = dict()
# The greenlet that checks if the dict should be flushed or not
# TODO: do something if the FlushingDict is deleted
self.flush_greenlet = gevent.spawn(self._check_flush)
# This lock ensures that if a key-value arrives while flush is
# executing, it is not inserted in the dict until flush
# terminates.
self.d_lock = RLock()
# Time when an item was last inserted in the dict
self.last_insert = monotonic_time()
def __init__(self, size, flush_latency_seconds, callback):
# Elements contained in the dict that force a flush.
self.size = size
# How much time we wait for other key-values before flushing.
self.flush_latency_seconds = flush_latency_seconds
# Function to flush the data to.
self.callback = callback
# This contains all the key-values received and not yet
# flushed.
self.d = dict()
# This contains all the key-values that are currently being flushed
self.fd = dict()
# The greenlet that checks if the dict should be flushed or not
# TODO: do something if the FlushingDict is deleted
self.flush_greenlet = gevent.spawn(self._check_flush)
# This lock ensures that if a key-value arrives while flush is
# executing, it is not inserted in the dict until flush
# terminates.
self.d_lock = RLock()
# Time when an item was last inserted in the dict
self.last_insert = monotonic_time()
def _check_flush(self):
while True:
while True:
with self.d_lock:
since_last_insert = monotonic_time() - self.last_insert
if len(self.d) != 0 and (
len(self.d) >= self.size or
since_last_insert > self.flush_latency_seconds):
break
gevent.sleep(0.05)
self.flush()
def _check_flush(self):
while True:
while True:
with self.d_lock:
since_last_insert = monotonic_time() - self.last_insert
if len(self.d) != 0 and (
len(self.d) >= self.size
or since_last_insert > self.flush_latency_seconds):
break
gevent.sleep(0.05)
self.flush()
def repeater(func, period):
"""Repeatedly call the given function.
Continuosly calls the given function, over and over. For a call to
be issued the previous one needs to have returned, and at least the
given number of seconds needs to have passed. Raised exceptions are
caught, logged and then suppressed.
func (function): the function to call.
period (float): the desired interval between successive calls.
"""
while True:
call = monotonic_time()
try:
func()
except Exception:
logger.error("Unexpected error.", exc_info=True)
gevent.sleep(max(call + period - monotonic_time(), 0))
def repeater(func, period):
"""Repeatedly call the given function.
Continuosly calls the given function, over and over. For a call to
be issued the previous one needs to have returned, and at least the
given number of seconds needs to have passed. Raised exceptions are
caught, logged and then suppressed.
func (function): the function to call.
period (float): the desired interval between successive calls.
"""
while True:
call = monotonic_time()
try:
func()
except Exception:
logger.error("Unexpected error.", exc_info=True)
gevent.sleep(max(call + period - monotonic_time(), 0))
def get_execution_wall_clock_time(self):
"""Return the total time from the start of the sandbox to the
conclusion of the task.
return (float): total time the sandbox was alive.
"""
if self.exec_time:
return self.exec_time
if self.popen_time:
self.exec_time = monotonic_time() - self.popen_time
return self.exec_time
return None
def get_execution_wall_clock_time(self):
"""Return the total time from the start of the sandbox to the
conclusion of the task.
return (float): total time the sandbox was alive.
"""
if self.exec_time:
return self.exec_time
if self.popen_time:
self.exec_time = monotonic_time() - self.popen_time
return self.exec_time
return None
def stop(self):
"""Quit gracefully. Or not: if the quit RPC does not work, kill."""
if self.service_name != "RankingWebServer":
# Try to terminate gracefully (RWS does not have a way to do it).
logger.info("Asking %s/%s to terminate...", self.service_name,
self.shard)
rs = RemoteService(self.cms_config, self.service_name, self.shard)
rs.call("quit", {"reason": "from test harness"})
# If it didn't understand, use bad manners.
self._check()
if self.healthy:
logger.info("Interrupting %s/%s.", self.service_name, self.shard)
self.instance.send_signal(signal.SIGINT)
# FIXME on py3 this becomes self.instance.wait(timeout=5)
t = monotonic_time()
while monotonic_time() - t < 5:
if self.instance.poll() is not None:
break
time.sleep(0.1)
else:
logger.info("Killing %s/%s.", self.service_name, self.shard)
self.instance.kill()
def stop(self):
"""Quit gracefully. Or not: if the quit RPC does not work, kill."""
if self.service_name != "RankingWebServer":
# Try to terminate gracefully (RWS does not have a way to do it).
logger.info("Asking %s/%s to terminate...",
self.service_name, self.shard)
rs = RemoteService(self.cms_config, self.service_name, self.shard)
rs.call("quit", {"reason": "from test harness"})
# If it didn't understand, use bad manners.
self._check()
if self.healthy:
logger.info("Interrupting %s/%s.", self.service_name, self.shard)
self.instance.send_signal(signal.SIGINT)
# FIXME on py3 this becomes self.instance.wait(timeout=5)
t = monotonic_time()
while monotonic_time() - t < 5:
if self.instance.poll() is not None:
break
time.sleep(0.1)
else:
logger.info("Killing %s/%s.", self.service_name, self.shard)
self.instance.kill()
def add_timeout(self, func, plus, seconds, immediately=False):
"""Registers a function to be called every x seconds.
func (function): the function to call.
plus (object): additional data to pass to the function.
seconds (float): the function will be called every seconds
seconds.
immediately (bool): if True, func will be called also at the
beginning.
"""
next_timeout = monotonic_time()
if not immediately:
next_timeout += seconds
heapq.heappush(self._timeouts, (next_timeout, seconds, func, plus))
# Wake up the run() cycle
self.event.set()
def _trigger(self, maximum=2.0):
"""Call the timeouts that have expired and find interval to
next timeout (capped to maximum second).
maximum (float): seconds to cap to the value.
return (float): seconds to next timeout.
"""
current = monotonic_time()
# Try to connect to disconnected services.
self._reconnect()
# Check if some scheduled function needs to be called.
while self._timeouts != []:
timeout_data = self._timeouts[0]
next_timeout, _, _, _ = timeout_data
if current > next_timeout:
heapq.heappop(self._timeouts)
# The helper function checks the return value and, if
# needed, enqueues the next timeout call
def helper(timeout_data):
next_timeout, seconds, func, plus = timeout_data
if plus is None:
ret = func()
else:
ret = func(plus)
if ret:
heapq.heappush(self._timeouts,
(next_timeout + seconds,
seconds, func, plus))
gevent.spawn(helper, timeout_data)
else:
break
# Compute time to next timeout call
next_timeout = maximum
if self._timeouts != []:
next_timeout = min(next_timeout, self._timeouts[0][0] - current)
return max(0.0, next_timeout)
def execute_without_std(self, command, wait=False):
"""Execute the given command in the sandbox using
subprocess.Popen and discarding standard input, output and
error. More specifically, the standard input gets closed just
after the execution has started; standard output and error are
read until the end, in a way that prevents the execution from
being blocked because of insufficient buffering.
command ([string]): executable filename and arguments of the
command.
return (bool): True if the sandbox didn't report errors
(caused by the sandbox itself), False otherwise
"""
def preexec_fn(self):
"""Set limits for the child process.
"""
if self.chdir:
os.chdir(self.chdir)
# TODO - We're not checking that setrlimit() returns
# successfully (they may try to set to higher limits than
# allowed to); anyway, this is just for testing
# environment, not for real contests, so who cares.
if self.timeout:
rlimit_cpu = self.timeout
if self.extra_timeout:
rlimit_cpu += self.extra_timeout
rlimit_cpu = int(rlimit_cpu) + 1
resource.setrlimit(resource.RLIMIT_CPU,
(rlimit_cpu, rlimit_cpu))
if self.address_space:
rlimit_data = int(self.address_space * 1024)
resource.setrlimit(resource.RLIMIT_DATA,
(rlimit_data, rlimit_data))
if self.stack_space:
rlimit_stack = int(self.stack_space * 1024)
resource.setrlimit(resource.RLIMIT_STACK,
(rlimit_stack, rlimit_stack))
# TODO - Doesn't work as expected
#resource.setrlimit(resource.RLIMIT_NPROC, (1, 1))
# Setup std*** redirection
if self.stdin_file:
stdin_fd = os.open(os.path.join(self.path, self.stdin_file),
os.O_RDONLY)
else:
stdin_fd = subprocess.PIPE
if self.stdout_file:
stdout_fd = os.open(os.path.join(self.path, self.stdout_file),
os.O_WRONLY | os.O_TRUNC | os.O_CREAT,
stat.S_IRUSR | stat.S_IRGRP |
stat.S_IROTH | stat.S_IWUSR)
else:
stdout_fd = subprocess.PIPE
if self.stderr_file:
stderr_fd = os.open(os.path.join(self.path, self.stderr_file),
os.O_WRONLY | os.O_TRUNC | os.O_CREAT,
stat.S_IRUSR | stat.S_IRGRP |
stat.S_IROTH | stat.S_IWUSR)
else:
stderr_fd = subprocess.PIPE
# Note down execution time
self.popen_time = monotonic_time()
# Actually call the Popen
self.popen = self._popen(command,
stdin=stdin_fd,
stdout=stdout_fd,
stderr=stderr_fd,
preexec_fn=partial(preexec_fn, self),
close_fds=True)
# Close file descriptors passed to the child
if self.stdin_file:
os.close(stdin_fd)
if self.stdout_file:
os.close(stdout_fd)
if self.stderr_file:
os.close(stderr_fd)
if self.wallclock_timeout:
# Kill the process after the wall clock time passed
def timed_killer(timeout, popen):
gevent.sleep(timeout)
# TODO - Here we risk to kill some other process that gets
# the same PID in the meantime; I don't know how to
# properly solve this problem
try:
popen.kill()
except OSError:
# The process had died by itself
pass
# Setup the killer
full_wallclock_timeout = self.wallclock_timeout
if self.extra_timeout:
full_wallclock_timeout += self.extra_timeout
gevent.spawn(timed_killer, full_wallclock_timeout, self.popen)
# If the caller wants us to wait for completion, we also avoid
# std*** to interfere with command. Otherwise we let the
# caller handle these issues.
if wait:
return self.translate_box_exitcode(
wait_without_std([self.popen])[0])
else:
return self.popen
def add(self, key, value):
logger.debug("Adding item %s", key)
with self.d_lock:
self.d[key] = value
self.last_insert = monotonic_time()
def add(self, key, value):
logger.debug("Adding item %s", key)
with self.d_lock:
self.d[key] = value
self.last_insert = monotonic_time()
def execute_without_std(self, command, wait=False):
"""Execute the given command in the sandbox using
subprocess.Popen and discarding standard input, output and
error. More specifically, the standard input gets closed just
after the execution has started; standard output and error are
read until the end, in a way that prevents the execution from
being blocked because of insufficient buffering.
command ([string]): executable filename and arguments of the
command.
return (bool): True if the sandbox didn't report errors
(caused by the sandbox itself), False otherwise
"""
def preexec_fn(self):
"""Set limits for the child process.
"""
if self.chdir:
os.chdir(self.chdir)
# TODO - We're not checking that setrlimit() returns
# successfully (they may try to set to higher limits than
# allowed to); anyway, this is just for testing
# environment, not for real contests, so who cares.
if self.timeout:
rlimit_cpu = self.timeout
if self.extra_timeout:
rlimit_cpu += self.extra_timeout
rlimit_cpu = int(rlimit_cpu) + 1
resource.setrlimit(resource.RLIMIT_CPU,
(rlimit_cpu, rlimit_cpu))
if self.address_space:
rlimit_data = int(self.address_space * 1024)
resource.setrlimit(resource.RLIMIT_DATA,
(rlimit_data, rlimit_data))
if self.stack_space:
rlimit_stack = int(self.stack_space * 1024)
resource.setrlimit(resource.RLIMIT_STACK,
(rlimit_stack, rlimit_stack))
# TODO - Doesn't work as expected
#resource.setrlimit(resource.RLIMIT_NPROC, (1, 1))
# Setup std*** redirection
if self.stdin_file:
stdin_fd = os.open(os.path.join(self.path, self.stdin_file),
os.O_RDONLY)
else:
stdin_fd = subprocess.PIPE
if self.stdout_file:
stdout_fd = os.open(os.path.join(self.path, self.stdout_file),
os.O_WRONLY | os.O_TRUNC | os.O_CREAT,
stat.S_IRUSR | stat.S_IRGRP |
stat.S_IROTH | stat.S_IWUSR)
else:
stdout_fd = subprocess.PIPE
if self.stderr_file:
stderr_fd = os.open(os.path.join(self.path, self.stderr_file),
os.O_WRONLY | os.O_TRUNC | os.O_CREAT,
stat.S_IRUSR | stat.S_IRGRP |
stat.S_IROTH | stat.S_IWUSR)
else:
stderr_fd = subprocess.PIPE
# Note down execution time
self.popen_time = monotonic_time()
# Actually call the Popen
self.popen = self._popen(command,
stdin=stdin_fd,
stdout=stdout_fd,
stderr=stderr_fd,
preexec_fn=partial(preexec_fn, self),
close_fds=True)
# Close file descriptors passed to the child
if self.stdin_file:
os.close(stdin_fd)
if self.stdout_file:
os.close(stdout_fd)
if self.stderr_file:
os.close(stderr_fd)
if self.wallclock_timeout:
# Kill the process after the wall clock time passed
def timed_killer(timeout, popen):
gevent.sleep(timeout)
# TODO - Here we risk to kill some other process that gets
# the same PID in the meantime; I don't know how to
# properly solve this problem
try:
popen.kill()
except OSError:
# The process had died by itself
pass
# Setup the killer
full_wallclock_timeout = self.wallclock_timeout
if self.extra_timeout:
full_wallclock_timeout += self.extra_timeout
gevent.spawn(timed_killer, full_wallclock_timeout, self.popen)
# If the caller wants us to wait for completion, we also avoid
# std*** to interfere with command. Otherwise we let the
# caller handle these issues.
if wait:
return self.translate_box_exitcode(
wait_without_std([self.popen])[0])
else:
return self.popen
