def route(*uris):
for uri in uris:
if any((
uri.startswith('magnet:?xt=urn:btih:'),
uri.endswith('.torrent'),
'://www.youtube.com/watch' in uri,
# v.reddit
)):
local['~/Code/stream.sh'](uri)
elif any((
uri.startswith('https://i.imgur.com/'),
uri.startswith('https://i.redd.it/'),
)):
gwenview[uri] & BG(stderr=sys.stderr)
elif any((
uri.startswith('https://imgur.com/a/'),
uri.startswith('https://imgur.com/gallery/'),
)):
r = get(uri)
soup = BeautifulSoup(r.text)
for img in soup.find_all(attrs={'class': 'post-image-container'}):
route(f"https://i.imgur.com/{img.attrs['id']}.jpg")
elif sys.stdin.isatty():
if uri.startswith('https://www.reddit.com/r/'):
rtv[uri] & FG
else:
lynx['-accept_all_cookies', uri] & FG
else:
with local.env(GTK_USE_PORTAL=1):
firefox[uri] & BG(stderr=sys.stderr)
def launch_redis(ip):
logfile = 'logs/redis_{}.log'.format(ip)
f = (redis['--bind', ip, '--port', REDIS_PORT] > logfile) & BG(-9)
try:
yield RedisNode(ip, REDIS_PORT)
finally:
f.proc.kill()
f.wait()
def bg(a, *commands):
result = local[a][commands] & BG(retcode=None)
result.wait()
bg_return_code = result.returncode
if bg_return_code != 0:
print(f"Failed to execute in background, error code: {bg_return_code}")
return False
else:
return True
def launch_redis(ip):
logfile = 'logs/redis_{}.log'.format(ip)
f = (redis['--requirePass', 'testpass', '--masterauth', 'testpass',
'--bind', ip, '--port', REDIS_PORT] > logfile) & BG(-9)
try:
#print("Redis future: {}".format(str(f)))
#print("Redis started, pid: {}".format(f.proc.pid))
yield RedisNode(ip, REDIS_PORT)
finally:
f.proc.kill()
f.wait()
def launch(self, seeds_list):
config_filename = self.write_config(seeds_list)
with launch_redis(self.ip):
logfile = 'logs/dynomite_{}.log'.format(self.ip)
dynomite_future = dynomite['-o', logfile, '-c', config_filename,
'-v6'] & BG(-9)
try:
yield DynoNode(self.ip)
finally:
dynomite_future.proc.kill()
dynomite_future.wait()
def launch(self, seeds_list):
config_filename = self.write_config(seeds_list)
with launch_redis(self.ip):
logfile = 'logs/dynomite_{}.log'.format(self.ip)
# Add '-v', '11' for maximum verbosity in logs
# Dynomite exits with status 1 on SIGINT, this is what we expect.
dynomite_future = dynomite['-v', '11', '-o', logfile, '-c',
config_filename, '-v6'] & BG(1)
try:
yield DynoNode(self.ip)
finally:
dynomite_future.proc.send_signal(SIGINT)
return_code = dynomite_future.wait()
def run_atari_qdqn(batch_size=32,
learning_rate=1e-4,
actor_count=1,
tf_thread_count=8,
target_update_frequency=1000,
env='PongNoFrameskip-v4'):
print("Starting Atari training")
f = python3["-m", "baselines.qdqn.experiments.train_atari",
"--batch_size={}".format(batch_size),
"--learning_rate={}".format(learning_rate),
"--actor_count={}".format(actor_count),
"--tf_thread_count={}".format(tf_thread_count),
"--target_update_frequency={}".format(target_update_frequency),
"--num_iterations={}".format(int(5e7)),
"--env_name={}".format(env)] & BG(stdout=sys.stdout,
stderr=sys.stderr)
#taskset -cp $i $pid
wait_for(f)
def run_cartpole_qdqn(batch_size=32,
learning_rate=1e-4,
actor_count=1,
tf_thread_count=8,
target_update_frequency=500,
num_iterations=None,
env='ppaquette/BasicDoom-v0'):
print("Starting cartpole training with QDQN")
f = python3[
"-m", "baselines.qdqn.experiments.cartpole",
"--batch_size={}".format(batch_size),
"--learning_rate={}".format(learning_rate),
"--actor_count={}".format(actor_count),
"--tf_thread_count={}".format(tf_thread_count),
# "--target_update_frequency={}".format(target_update_frequency),
"--num_iterations={}".format(num_iterations),
"--cleanup={}".format(True), "--env_name={}".format(env)] & BG(
stdout=sys.stdout, stderr=sys.stderr)
#taskset -cp $i $pid
wait_for(f)
def align_and_sort(args):
"""Main function."""
ref = _get_reference(args)
fai_indices = local.path(ref + ".fai")
if not fai_indices:
print("Indexing fasta with samtools", sys.stderr)
samtools_index = samtools("faidx", ref) & BG
samtools_index.wait()
bowtie2_index = ref.with_suffix(".1.bt2")
if not bowtie2_index.exists():
print("Indexing fasta with bowtie2")
prefix = ref.with_suffix("")
build = local["bowtie2-build"]
p = build[ref, prefix] & BG
p.wait()
align_pipe = (bowtie2.__getitem__(args) | samtools["view", "-bhS"]
| samtools["sort"])
p = align_pipe & BG(stdout=sys.stdout, stderr=sys.stderr)
p.wait()
def run_doom_dqn(batch_size=32,
learning_rate=1e-4,
train_frequency=4,
worker_count=1,
tf_thread_count=8,
target_update_frequency=500,
num_iterations=None,
env='ppaquette/BasicDoom-v0'):
if num_iterations is None:
num_iterations = 10000
print("Starting doom training with DQN")
f = python3["-m", "baselines.multi_deepq.experiments.doom",
"--batch_size={}".format(batch_size),
"--train_frequency={}".format(train_frequency),
"--learning_rate={}".format(learning_rate),
"--worker_count={}".format(worker_count),
"--target_update_frequency={}".format(target_update_frequency),
"--num_iterations={}".format(num_iterations),
"--tf_thread_count={}".format(tf_thread_count),
"--env_name={}".format(env)] & BG(stdout=sys.stdout,
stderr=sys.stderr)
#taskset -cp $i $pid
wait_for(f)
def test_proxy_from_dhcp_wpad(self):
# set up mock dbus with dhcp settings
wifi1 = self.dbusmock.AddWiFiDevice('mock_WiFi1', 'wlan0',
DeviceState.ACTIVATED)
ap1 = self.dbusmock.AddAccessPoint(
wifi1, 'Mock_AP1', 'The_SSID', '00:23:F8:7E:12:BB',
InfrastructureMode.NM_802_11_MODE_INFRA, 2425, 5400, 82,
NM80211ApSecurityFlags.NM_802_11_AP_SEC_KEY_MGMT_PSK)
con1 = self.dbusmock.AddWiFiConnection(wifi1, 'Mock_Con1', 'The_SSID', '')
active_con1 = self.dbusmock.AddActiveConnection(
[wifi1], con1, ap1, 'Mock_Active1',
NMActiveConnectionState.NM_ACTIVE_CONNECTION_STATE_ACTIVATED)
self.dbusmock.AddObject(
'/org/freedesktop/NetworkManager/DHCP4Config/1',
'org.freedesktop.NetworkManager.DHCP4Config',
{
'Options': {
'wpad': 'http://localhost:8080/wpad.dat',
},
},
[],
)
conn_obj = dbus.Interface(
self.dbus_con.get_object("org.freedesktop.NetworkManager", active_con1),
dbusmock.MOCK_IFACE,
)
conn_obj.AddProperty(
'org.freedesktop.NetworkManager.Connection.Active',
'Dhcp4Config',
'/org/freedesktop/NetworkManager/DHCP4Config/1',
)
# for inspecting the resulting dbus objects
# with plumbum.local.env(DBUS_SYSTEM_BUS_ADDRESS=os.environ['DBUS_SYSTEM_BUS_ADDRESS']):
# gdbus["introspect", "--system", "--recurse", "-d", "org.freedesktop.NetworkManager", "-o", "/"] & FG
# server for wpad.dat
with plumbum.local.cwd(testdir / "wpadserver"):
static_server = python["-m", "http.server", 8080] & BG
# mock upstream proxies
fake_proxy_1 = (serve_once[23130] < testdir / "fake-proxy-1-response") & BG
fake_proxy_2 = (serve_once[23131] < testdir / "fake-proxy-2-response") & BG
# proxy getting its config from DHCP
with plumbum.local.env(DBUS_SYSTEM_BUS_ADDRESS=os.environ['DBUS_SYSTEM_BUS_ADDRESS']):
proxy_to_test = pac4cli["-p", "23129"] & BG(stdout=sys.stdout, stderr=sys.stderr)
sleep(3)
try:
with plumbum.local.env(http_proxy="localhost:23129"):
self.assertEqual(
curl("http://www.booking.com"),
# when changing this string, don't forget to change
# the Content-Length header as well.
"Hello from fake proxy no 1!",
)
self.assertEqual(
curl("http://www.google.com"),
# (same)
"Hello from fake proxy no 2!",
)
finally:
proxy_to_test.proc.kill()
fake_proxy_2.proc.kill()
fake_proxy_1.proc.kill()
static_server.proc.kill()
def bg_content(a, *cmds):
result = local[a][cmds] & BG(retcode=None)
result.wait()
return result