def results(self):
for target_info in self.targets:
(target, otype, _) = target_info
target_results = list()
for (site_name, site_conf) in self.sites.items():
if otype.lower() not in map(lambda x: x.lower(),
site_conf["otypes"]):
continue
site_conf["target"] = target
site_conf["verbose"] = self.args.verbose
scraper = Site.from_conf(site_conf) # , verbose=self.verbose)
try:
with stopit.SignalTimeout(15, swallow_exc=False):
run_results = list()
for r in scraper.run():
if "value" not in r:
r = {"value": r, "pretty_name": None}
run_results.append(
Result(r["value"], r["pretty_name"]))
except stopit.TimeoutException:
target_results.append(
ErrorResult(target_info, site_conf, "Timeout"))
except Exception as e:
target_results.append(
ErrorResult(target_info, site_conf, e))
else:
target_results.append(SiteResults(site_conf, run_results))
yield ResultSet(target_info, target_results)
def timeout(func, *args, seconds=None, **kwargs):
if seconds is None:
return func(*args, **kwargs)
try:
start = time.time()
with stopit.SignalTimeout(seconds) as to_ctx_mgr:
result = func(*args, **kwargs)
duration = time.time() - start
# OK, let's check what happened
if to_ctx_mgr.state == to_ctx_mgr.EXECUTED:
# All's fine, everything was executed
return result
elif to_ctx_mgr.state == to_ctx_mgr.TIMED_OUT:
# Timeout occurred while executing the block
raise TimeoutError("Timed out after {0} seconds.".format(
round(duration)))
else:
traceback.print_tb()
raise RuntimeError("Something weird happened.")
except stopit.TimeoutException as e:
raise TimeoutError("Timed out after {0} seconds.".format(e.seconds))
duration = time.time() - seconds
def test_solver_on_env(env: MapfEnv, env_name: str,
solver_describer: SolverDescriber):
info = {}
start = time.time()
# Try to solve with a time limit
with stopit.SignalTimeout(TEST_SINGLE_SCENARIO_TIMEOUT,
swallow_exc=False) as timeout_ctx:
try:
policy = solver_describer.func(env, info)
except stopit.utils.TimeoutException:
print(
f'solver {solver_describer.description} got timeout on {env_name}',
end=' ')
assert False
solve_time = round(time.time() - start, 2)
reward, clashed, _ = evaluate_policy(policy, 100, 1000)
print(
f'env:{env_name}, reward:{reward}, time: {solve_time}, solver:{solver_describer.description}',
end=' ')
assert not clashed
# Assert the reward was not -1000
assert reward >= 999 * env.reward_of_living + env.reward_of_goal
def results(self):
creds = None
if self.args.auth and os.path.isfile(self.args.auth):
with open(self.args.auth) as auth_f:
creds = utils.safe_load(auth_f.read())
proxies = {}
if self.args.http_proxy:
proxies["http"] = self.args.http_proxy
proxies["https"] = self.args.http_proxy
else:
if "HTTP_PROXY" in os.environ:
proxies["http"] = os.environ["HTTP_PROXY"]
elif "http_proxy" in os.environ:
proxies["http"] = os.environ["http_proxy"]
if "HTTPS_PROXY" in os.environ:
proxies["https"] = os.environ["HTTPS_PROXY"]
elif "https_proxy" in os.environ:
proxies["https"] = os.environ["https_proxy"]
if "http" in proxies:
print("HTTP Proxy: {http}".format(**proxies), file=sys.stderr)
if "https" in proxies:
print("HTTPS Proxy: {https}".format(**proxies), file=sys.stderr)
for target_info in self.targets:
(target, otype, _) = target_info
target_results = list()
for (site_name, site_conf) in self.sites.items():
if otype.lower() not in map(lambda x: x.lower(),
site_conf["otypes"]):
continue
site_conf["target"] = target
site_conf["verbose"] = self.args.verbose
scraper = Site.from_conf(
site_conf, creds=creds,
proxies=proxies) # , verbose=self.verbose)
try:
with stopit.SignalTimeout(15, swallow_exc=False):
run_results = list()
for r in scraper.run():
if "value" not in r:
r = {"value": r, "pretty_name": None}
run_results.append(
Result(r["value"], r["pretty_name"]))
except stopit.TimeoutException:
target_results.append(
ErrorResult(target_info, site_conf, "Timeout"))
except Exception as e:
target_results.append(
ErrorResult(target_info, site_conf, e))
else:
target_results.append(SiteResults(site_conf, run_results))
yield ResultSet(target_info, target_results)
def solve_single_instance(log_func, insert_to_db_func,
instance: InstanceMetaData):
instance_data = {
'type': 'instance_data',
'map': instance.map,
'scen_id': instance.scen_id,
'fail_prob': instance.fail_prob,
'n_agents': instance.n_agents,
'solver': instance.solver
}
configuration_string = '_'.join(
[f'{key}:{value}' for key, value in instance_data.items()])
log_func(DEBUG, f'starting solving instance {configuration_string}')
# Create mapf env, some of the benchmarks from movingAI might have bugs so be careful
try:
env = create_mapf_env(instance.map, instance.scen_id,
instance.n_agents, instance.fail_prob / 2,
instance.fail_prob / 2, -1000, -1, -1)
except Exception as ex:
log_func(ERROR, f'{configuration_string} is invalid')
instance_data.update({
'solver_data': {},
'end_reason':
'invalid',
'error':
''.join(traceback.TracebackException.from_exception(ex).format())
})
insert_to_db_func(instance_data)
return
# Run the solver
instance_data.update({'solver_data': {}})
with stopit.SignalTimeout(SINGLE_SCENARIO_TIMEOUT,
swallow_exc=False) as timeout_ctx:
try:
start = time.time()
policy = instance.plan_func(env, instance_data['solver_data'])
if policy is not None:
# policy might be None if the problem is too big for the solver
reward, clashed, all_rewards = evaluate_policy(
policy, 100, MAX_STEPS)
instance_data['average_reward'] = reward
instance_data['reward_std'] = np.std(all_rewards)
instance_data['clashed'] = clashed
except stopit.utils.TimeoutException:
instance_data['end_reason'] = 'timeout'
log_func(DEBUG, f'{configuration_string} got timeout')
end = time.time()
instance_data['total_time'] = round(end - start, 2)
if 'end_reason' not in instance_data:
instance_data['end_reason'] = 'done'
log_func(DEBUG, f'inserting {configuration_string} to DB')
# Insert stats about this instance to the DB
insert_to_db_func(instance_data)
def test_build_exceptions_not_suppressed(app):
with patch("application.sitebuilder.build_service.do_it") as do_it_patch:
do_it_patch.side_effect = GeneralTestException("build error")
request_build()
with pytest.raises(GeneralTestException) as e, stopit.SignalTimeout(2):
build_site(app)
assert str(e.value) == "build error"
def __enter__(self):
if self.n is None:
# timeout disabled
return None
if sys.platform.startswith("win"):
# Windows does not support signal-based timeout
self._obj = stopit.ThreadingTimeout(self.n)
else:
self._obj = stopit.SignalTimeout(self.n)
return self._obj.__enter__()
def benchmark_planners_on_env(env, env_str, solver_describers):
results_df = pd.DataFrame(
columns=['env', 'solver', 'time', 'avg_reward', 'clashed'])
for solver_describer in solver_describers:
solver_str = solver_describer.short_description
solve_func = solver_describer.func
# Fill default values in case of a timeout (we will not be able to evaluate the policy)
solved = True
reward, clashed = -1000, False
# Prepare for running
print(f'Running {solver_str} on {env_str}')
# Run with time limit
with stopit.SignalTimeout(SINGLE_SCENARIO_TIMEOUT,
swallow_exc=False) as timeout_ctx:
try:
start = time.time()
info = {}
policy = solve_func(env, info)
except stopit.utils.TimeoutException:
print(f'{solver_str} on {env_str} got timeout')
solved = False
# Evaluate policy is solved
if solved:
print(f'evaluating policy calculated by {solver_str} on {env_str}')
reward, clashed, _ = evaluate_policy(policy, 1000, 1000)
# Measure time
total_time = time.time() - start
# Collect results
row = {
'env': env_str,
'solver': solver_str,
'time': total_time,
'avg_reward': reward,
'clashed': clashed
}
row.update(dict(solver_describer.extra_info(info)._asdict()))
# Insert new row to results data frame
results_df = results_df.append(row, ignore_index=True)
# print the result
print(f'-----{env_str}------')
print(results_df)
return results_df
def _timed_execution(self):
self.job.started()
if self.job.limit_time is not None:
try:
with stopit.SignalTimeout(self.job.limit_time,
swallow_exc=False):
self.work.run(self.job)
except stopit.TimeoutException:
print "[Worker] Job execution timeout!"
else:
self.work.run(self.job)
self.job.completed()
self.job.produced_output = True
self.job.save()
def restore_weird_stuff():
"""Restore weird performance of ID-MA-RTDP on sanity envs from the heuristics experiment"""
print('start restoring')
env = create_mapf_env('sanity-2-32', 1, 3, 0.1, 0.1, -1000, -1, -1)
solver = long_id_ma_rtdp_sum_pvi_describer.func
with stopit.SignalTimeout(SINGLE_SCENARIO_TIMEOUT,
swallow_exc=False) as timeout_ctx:
try:
info = {}
policy = solver(env, info)
except stopit.utils.TimeoutException:
print('got timeout!!!')
import ipdb
ipdb.set_trace()
print('OMG')
def _wait_for_ssh(self):
LOG.debug("Waiting for SSH to become available from worker")
not_reachable = True
try:
# ThreadingTimeout does not work with PyPy, using signals instead
with stopit.SignalTimeout(self._ssh_timeout, swallow_exc=False):
while not_reachable:
try:
connection = socket.create_connection(
("127.0.0.1", self._ssh_port))
not_reachable = False
connection.close()
except socket.error as e:
LOG.debug("Unable to connect just yet, sleeping")
time.sleep(1)
except stopit.TimeoutException:
LOG.error("SSH did not become available within %s seconds.",
self._ssh_timeout)
raise EnvironmentError("SSH did not become available")
def vm(self):
print("[*] Spawning up VM to run jobs within")
drive = "file={0._disk},media=disk,if=virtio".format(self)
netdev = (
"user,id=fakenet0,net=172.16.6.0/24,restrict={0._restrict_net},"
"hostfwd=tcp:127.0.0.1:{0._ssh_port}-:22,").format(self)
kvm_command = [
"kvm", "-name", self._vm_name, "-sandbox", self._sandbox,
"-machine", "pc-i440fx-1.7,accel=kvm,usb=off", "-cpu",
"SandyBridge", "-drive", drive, "-m", "2G", "-netdev", netdev,
"-net", "nic,netdev=fakenet0,model=virtio", "-daemonize", "-vnc",
"none"
]
try:
kvm_process = subprocess.Popen(kvm_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self.kvm_process = kvm_process
except OSError as e:
print("[!] Is KVM installed? Popen raised %s" % e)
raise EnvironmentError("Unable to start VM, KVM process failed %s",
e)
stdout = None
stderr = None
try:
stdout, stderr = kvm_process.communicate()
except Exception as e:
print("[!] VM did not start within %s seconds, killing it" %
self._kvm_timeout)
print("[!] stdout: %s" % stdout)
print("[!] stderr: %s" % stderr)
kvm_process.terminate()
kvm_process.kill()
print("[!] 5 seconds grace period before forcefully killing VM")
time.sleep(5)
kvm_process.terminate()
kvm_process.kill()
raise EnvironmentError("KVM start did not boot up properly")
print("[*] Waiting for SSH to become available from worker")
not_reachable = True
try:
# ThreadingTimeout does not work with PyPy, using signals instead
with stopit.SignalTimeout(self._ssh_timeout, swallow_exc=False):
while not_reachable:
try:
connection = socket.create_connection(
("127.0.0.1", self._ssh_port))
not_reachable = False
connection.close()
except socket.error as e:
print("[!] Unable to connect just yet, sleeping")
time.sleep(1)
except stopit.TimeoutException:
print("[!] SSH did not become available within %s seconds." %
self._ssh_timeout)
print("[!] stdout: %s" % stdout)
print("[!] stderr: %s" % stderr)
raise EnvironmentError("SSH did not become available")
print("[*] Connecting to the VM via SSH")
self.ssh = paramiko.client.SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.client.AutoAddPolicy())
try:
self.ssh.connect("127.0.0.1",
port=self._ssh_port,
username=self._ssh_username,
key_filename=self._ssh_keyfile,
timeout=self._ssh_timeout)
# also raises BadHostKeyException, should be taken care of via AutoAddPolicy()
# also raises AuthenticationException, should never occur because keys are provisioned
except socket.error as e:
print("[!] TCP error connecting to SSH on VM.")
print("[!] stdout: %s" % stdout)
print("[!] stderr: %s" % stderr)
raise e
except paramiko.SSHException as e:
print("[!] SSH error trying to connect to VM.")
print("[!] stdout: %s" % stdout)
print("[!] stderr: %s" % stderr)
raise e
print("[+] Setting up route to database etc.")
try:
ssh_exec_command(self.ssh, "ip r add default via 172.16.6.2")
except paramiko.SSHException as e:
print("[!] Unable to setup routes on host: %s" % e)
raise e
print("[+] All OK")
return True
