def __init__(self, path):
self.path = process_path(path)
self.objects = FileSystemObjectStore(path)
self.map_idx = 0
self.reduce_idx = 0
self.op_idx = 0
self.n_signals = 0
def process_omniglot(data_dir, quiet):
try:
omniglot_dir = process_path(os.path.join(data_dir, 'omniglot'))
if _validate_omniglot(omniglot_dir):
print("Omniglot data seems to be present already.")
return
else:
try:
shutil.rmtree(omniglot_dir)
except FileNotFoundError:
pass
os.makedirs(omniglot_dir, exist_ok=False)
with cd(omniglot_dir):
subprocess.run("git clone https://github.com/brendenlake/omniglot --depth=1".split(), check=True)
with cd('omniglot/python'):
zip_ref = zipfile.ZipFile('images_evaluation.zip', 'r')
zip_ref.extractall('.')
zip_ref.close()
zip_ref = zipfile.ZipFile('images_background.zip', 'r')
zip_ref.extractall('.')
zip_ref.close()
subprocess.run('mv omniglot/python/images_background/* .', shell=True, check=True)
subprocess.run('mv omniglot/python/images_evaluation/* .', shell=True, check=True)
print("Done setting up Omniglot data.")
finally:
try:
shutil.rmtree(os.path.join(omniglot_dir, 'omniglot'))
except FileNotFoundError:
pass
def _search_plot_cmd(
path, y_field, x_field, groupby, spread_measure,
style, do_legend=False, **axes_kwargs):
path = process_path(path)
print("Plotting searches stored at {}.".format(path))
search = HyperSearch(path)
with plt.style.context(style):
ax = plt.axes(xlabel=x_field, ylabel=y_field, **axes_kwargs)
dist = search.objects.load_object('metadata', 'dist')
dist = Config(dist)
df = search.extract_summary_data()
groups = sorted(df.groupby(groupby))
colours = plt.rcParams['axes.prop_cycle'].by_key()['color']
legend = []
for i, (k, _df) in enumerate(groups):
values = list(_df.groupby(x_field))
x = [v[0] for v in values]
ys = [v[1][y_field] for v in values]
y = [_y.mean() for _y in ys]
if spread_measure == 'std_dev':
y_upper = y_lower = [_y.std() for _y in ys]
elif spread_measure == 'conf_int':
conf_int = [confidence_interval(_y.values, 0.95) for _y in ys]
y_lower = y - np.array([ci[0] for ci in conf_int])
y_upper = np.array([ci[1] for ci in conf_int]) - y
elif spread_measure == 'std_err':
y_upper = y_lower = [standard_error(_y.values) for _y in ys]
else:
raise Exception("NotImplemented")
yerr = np.vstack((y_lower, y_upper))
c = colours[i % len(colours)]
ax.semilogx(x, y, c=c, basex=2)
handle = ax.errorbar(x, y, yerr=yerr, c=c)
label = "{} = {}".format(groupby, k)
legend.append((handle, label))
if do_legend:
handles, labels = zip(*legend)
ax.legend(
handles, labels, loc='center left',
bbox_to_anchor=(1.05, 0.5), ncol=1)
# plt.subplots_adjust(
# left=0.09, bottom=0.13, right=0.7, top=0.93, wspace=0.05, hspace=0.18)
filename = "value_plot.pdf"
print("Saving plot as {}".format(filename))
plt.savefig(filename)
def __init__(
self, name, input_zip, pattern, scratch, local_scratch_prefix='/tmp/dps/hyper/',
n_nodes=1, tasks_per_node=12, cpus_per_task=1, mem_per_cpu="", gpu_set="",
wall_time="1hour", cleanup_time="1min", slack_time="1min",
add_date=True, dry_run=0, kind="slurm", env_vars=None, output_to_files=True, n_retries=0,
copy_venv="", step_time_limit=None, ignore_gpu=False, ssh_options=None,
loud_output=True, rsync_verbosity=0, copy_locally=True):
args = locals().copy()
del args['self']
print("\nParallelSession args:")
print(args)
launch_venv = os.getenv('VIRTUAL_ENV')
if launch_venv:
launch_venv = os.path.split(launch_venv)[1]
if ssh_options is None:
ssh_options = (
"-oPasswordAuthentication=no "
"-oStrictHostKeyChecking=no "
"-oConnectTimeout=5 "
"-oServerAliveInterval=2"
)
if kind == "pbs":
local_scratch_prefix = "\\$RAMDISK"
assert kind in "slurm slurm-local".split()
# Create directory to run the job from - should be on scratch.
scratch = os.path.abspath(os.path.expandvars(scratch))
es = ExperimentStore(scratch, prefix="run")
job_dir = es.new_experiment(name, 0, add_date=add_date, force_fresh=1)
job_dir.record_environment()
with open(job_dir.path_for('run_kwargs.json'), 'w') as f:
json.dump(args, f, default=str, indent=4, sort_keys=True)
del f
del args
job_path = job_dir.path
job_dir.make_directory('experiments')
input_zip_stem = path_stem(input_zip)
input_zip = shutil.copy(input_zip, job_dir.path_for("orig.zip"))
input_zip_abs = process_path(input_zip)
input_zip_base = os.path.basename(input_zip)
archive_root = zip_root(input_zip)
self.copy_files(
job_dir, input_zip, archive_root,
["README.md", "sampled_configs.txt", "config.json", "config.pkl"])
# storage local to each node, from the perspective of that node
local_scratch = os.path.join(local_scratch_prefix, os.path.basename(job_path))
output_to_files = "--output-to-files" if output_to_files else ""
env = os.environ.copy()
env_vars = env_vars or {}
env.update({e: str(v) for e, v in env_vars.items()})
env_vars = ' '.join('--env ' + k for k in env_vars)
rsync_verbosity = "" if not rsync_verbosity else "-" + "v" * rsync_verbosity
ro_job = ReadOnlyJob(input_zip)
indices_to_run = sorted([op.idx for op in ro_job.ready_incomplete_ops(sort=False)])
del ro_job
n_jobs_to_run = len(indices_to_run)
if n_jobs_to_run == 0:
print("All jobs are finished! Exiting.")
return
dirty_hosts = set()
n_tasks_per_step = n_nodes * tasks_per_node
n_steps = int(np.ceil(n_jobs_to_run / n_tasks_per_step))
node_file = " --sshloginfile nodefile.txt "
wall_time_seconds, total_seconds_per_step, parallel_seconds_per_step, python_seconds_per_step = \
self.compute_time_limits(wall_time, cleanup_time, slack_time, step_time_limit, n_steps)
self.__dict__.update(locals())
self.print_time_limits()
def __init__(
self, name, input_zip, pattern, scratch, local_scratch_prefix='/tmp/dps/hyper/', ppn=12, cpp=1,
pmem=None, wall_time="1hour", cleanup_time="1min", slack_time="1min", add_date=True, dry_run=0,
parallel_exe=None, kind="parallel", host_pool=None, load_avg_threshold=8., min_hosts=None,
max_hosts=1, env_vars=None, output_to_files=True, n_retries=0, gpu_set="", copy_venv="",
python_startup=False, step_time_limit=None, ignore_gpu=False, ssh_options=None, loud_output=True,
rsync_verbosity=0):
args = locals().copy()
del args['self']
print("\nParallelSession args:")
print(args)
launch_venv = os.getenv('VIRTUAL_ENV')
if launch_venv:
launch_venv = os.path.split(launch_venv)[1]
if not parallel_exe:
parallel_exe = "$HOME/.local/bin/parallel"
if ssh_options is None:
ssh_options = (
"-oPasswordAuthentication=no "
"-oStrictHostKeyChecking=no "
"-oConnectTimeout=5 "
"-oServerAliveInterval=2"
)
if kind == "pbs":
local_scratch_prefix = "\\$RAMDISK"
assert kind in "parallel pbs slurm slurm-local".split()
hpc = kind != "parallel"
# Create directory to run the job from - should be on scratch.
scratch = os.path.abspath(os.path.expandvars(scratch))
es = ExperimentStore(scratch, prefix="run_search")
job_dir = es.new_experiment(name, 0, add_date=add_date, force_fresh=1)
job_dir.record_environment()
with open(job_dir.path_for('run_kwargs.json'), 'w') as f:
json.dump(args, f, default=str, indent=4, sort_keys=True)
del f
del args
job_path = job_dir.path
job_dir.make_directory('experiments')
input_zip_stem = path_stem(input_zip)
input_zip = shutil.copy(input_zip, job_dir.path_for("orig.zip"))
input_zip_abs = process_path(input_zip)
input_zip_base = os.path.basename(input_zip)
archive_root = zip_root(input_zip)
self.copy_files(
job_dir, input_zip, archive_root,
["README.md", "sampled_configs.txt", "config.json", "config.pkl"])
# storage local to each node, from the perspective of that node
local_scratch = os.path.join(local_scratch_prefix, os.path.basename(job_path))
output_to_files = "--output-to-files" if output_to_files else ""
env = os.environ.copy()
env_vars = env_vars or {}
env.update({e: str(v) for e, v in env_vars.items()})
env_vars = ' '.join('--env ' + k for k in env_vars)
rsync_verbosity = "" if not rsync_verbosity else "-" + "v" * rsync_verbosity
ro_job = ReadOnlyJob(input_zip)
indices_to_run = sorted([op.idx for op in ro_job.ready_incomplete_ops(sort=False)])
del ro_job
n_jobs_to_run = len(indices_to_run)
if n_jobs_to_run == 0:
print("All jobs are finished! Exiting.")
return
dirty_hosts = set()
if hpc:
host_pool = []
n_nodes = max_hosts
n_procs = n_nodes * ppn
n_steps = int(np.ceil(n_jobs_to_run / n_procs))
else:
self.__dict__.update(locals())
host_pool = host_pool or DEFAULT_HOST_POOL
if isinstance(host_pool, str):
host_pool = host_pool.split()
# Get an estimate of the number of hosts we'll have available.
with cd(job_path):
hosts, n_procs = self.recruit_hosts(
hpc, min_hosts, max_hosts, host_pool,
ppn, max_procs=np.inf)
n_nodes = len(hosts)
if n_jobs_to_run < n_procs:
n_steps = 1
n_nodes = int(np.ceil(n_jobs_to_run / ppn))
n_procs = n_nodes * ppn
hosts = hosts[:n_nodes]
else:
n_steps = int(np.ceil(n_jobs_to_run / n_procs))
node_file = " --sshloginfile nodefile.txt "
wall_time_seconds, total_seconds_per_step, parallel_seconds_per_step, python_seconds_per_step = \
self.compute_time_limits(wall_time, cleanup_time, slack_time, step_time_limit, n_steps)
self.__dict__.update(locals())
self.print_time_limits()
import argparse
import numpy as np
import pandas as pd
from collections import defaultdict, OrderedDict
import matplotlib.pyplot as plt
from dps import cfg
from dps.hyper import HyperSearch
from dps.train import FrozenTrainingLoopData
from dps.utils import (
process_path, Config, sha_cache, set_clear_cache,
confidence_interval, standard_error, grid_subplots
)
data_dir = "/media/data/Dropbox/experiment_data/active/aaai_2020/"
cache_dir = process_path('/home/eric/.cache/dps_plots')
plot_dir = '/media/data/Dropbox/writeups/spatially_invariant_air/aaai_2020/figures'
plot_paths = Config()
plot_paths[''] = ''
verbose_cache = True
def std_dev(ys):
y_upper = y_lower = [_y.std() for _y in ys]
return y_upper, y_lower
def ci95(ys):
conf_int = [confidence_interval(_y, 0.95) for _y in ys]