def __init__(self, path): self.path = process_path(path) self.objects = FileSystemObjectStore(path) self.map_idx = 0 self.reduce_idx = 0 self.op_idx = 0 self.n_signals = 0
def process_omniglot(data_dir, quiet): try: omniglot_dir = process_path(os.path.join(data_dir, 'omniglot')) if _validate_omniglot(omniglot_dir): print("Omniglot data seems to be present already.") return else: try: shutil.rmtree(omniglot_dir) except FileNotFoundError: pass os.makedirs(omniglot_dir, exist_ok=False) with cd(omniglot_dir): subprocess.run("git clone https://github.com/brendenlake/omniglot --depth=1".split(), check=True) with cd('omniglot/python'): zip_ref = zipfile.ZipFile('images_evaluation.zip', 'r') zip_ref.extractall('.') zip_ref.close() zip_ref = zipfile.ZipFile('images_background.zip', 'r') zip_ref.extractall('.') zip_ref.close() subprocess.run('mv omniglot/python/images_background/* .', shell=True, check=True) subprocess.run('mv omniglot/python/images_evaluation/* .', shell=True, check=True) print("Done setting up Omniglot data.") finally: try: shutil.rmtree(os.path.join(omniglot_dir, 'omniglot')) except FileNotFoundError: pass
def _search_plot_cmd( path, y_field, x_field, groupby, spread_measure, style, do_legend=False, **axes_kwargs): path = process_path(path) print("Plotting searches stored at {}.".format(path)) search = HyperSearch(path) with plt.style.context(style): ax = plt.axes(xlabel=x_field, ylabel=y_field, **axes_kwargs) dist = search.objects.load_object('metadata', 'dist') dist = Config(dist) df = search.extract_summary_data() groups = sorted(df.groupby(groupby)) colours = plt.rcParams['axes.prop_cycle'].by_key()['color'] legend = [] for i, (k, _df) in enumerate(groups): values = list(_df.groupby(x_field)) x = [v[0] for v in values] ys = [v[1][y_field] for v in values] y = [_y.mean() for _y in ys] if spread_measure == 'std_dev': y_upper = y_lower = [_y.std() for _y in ys] elif spread_measure == 'conf_int': conf_int = [confidence_interval(_y.values, 0.95) for _y in ys] y_lower = y - np.array([ci[0] for ci in conf_int]) y_upper = np.array([ci[1] for ci in conf_int]) - y elif spread_measure == 'std_err': y_upper = y_lower = [standard_error(_y.values) for _y in ys] else: raise Exception("NotImplemented") yerr = np.vstack((y_lower, y_upper)) c = colours[i % len(colours)] ax.semilogx(x, y, c=c, basex=2) handle = ax.errorbar(x, y, yerr=yerr, c=c) label = "{} = {}".format(groupby, k) legend.append((handle, label)) if do_legend: handles, labels = zip(*legend) ax.legend( handles, labels, loc='center left', bbox_to_anchor=(1.05, 0.5), ncol=1) # plt.subplots_adjust( # left=0.09, bottom=0.13, right=0.7, top=0.93, wspace=0.05, hspace=0.18) filename = "value_plot.pdf" print("Saving plot as {}".format(filename)) plt.savefig(filename)
def __init__( self, name, input_zip, pattern, scratch, local_scratch_prefix='/tmp/dps/hyper/', n_nodes=1, tasks_per_node=12, cpus_per_task=1, mem_per_cpu="", gpu_set="", wall_time="1hour", cleanup_time="1min", slack_time="1min", add_date=True, dry_run=0, kind="slurm", env_vars=None, output_to_files=True, n_retries=0, copy_venv="", step_time_limit=None, ignore_gpu=False, ssh_options=None, loud_output=True, rsync_verbosity=0, copy_locally=True): args = locals().copy() del args['self'] print("\nParallelSession args:") print(args) launch_venv = os.getenv('VIRTUAL_ENV') if launch_venv: launch_venv = os.path.split(launch_venv)[1] if ssh_options is None: ssh_options = ( "-oPasswordAuthentication=no " "-oStrictHostKeyChecking=no " "-oConnectTimeout=5 " "-oServerAliveInterval=2" ) if kind == "pbs": local_scratch_prefix = "\\$RAMDISK" assert kind in "slurm slurm-local".split() # Create directory to run the job from - should be on scratch. scratch = os.path.abspath(os.path.expandvars(scratch)) es = ExperimentStore(scratch, prefix="run") job_dir = es.new_experiment(name, 0, add_date=add_date, force_fresh=1) job_dir.record_environment() with open(job_dir.path_for('run_kwargs.json'), 'w') as f: json.dump(args, f, default=str, indent=4, sort_keys=True) del f del args job_path = job_dir.path job_dir.make_directory('experiments') input_zip_stem = path_stem(input_zip) input_zip = shutil.copy(input_zip, job_dir.path_for("orig.zip")) input_zip_abs = process_path(input_zip) input_zip_base = os.path.basename(input_zip) archive_root = zip_root(input_zip) self.copy_files( job_dir, input_zip, archive_root, ["README.md", "sampled_configs.txt", "config.json", "config.pkl"]) # storage local to each node, from the perspective of that node local_scratch = os.path.join(local_scratch_prefix, os.path.basename(job_path)) output_to_files = "--output-to-files" if output_to_files else "" env = os.environ.copy() env_vars = env_vars or {} env.update({e: str(v) for e, v in env_vars.items()}) env_vars = ' '.join('--env ' + k for k in env_vars) rsync_verbosity = "" if not rsync_verbosity else "-" + "v" * rsync_verbosity ro_job = ReadOnlyJob(input_zip) indices_to_run = sorted([op.idx for op in ro_job.ready_incomplete_ops(sort=False)]) del ro_job n_jobs_to_run = len(indices_to_run) if n_jobs_to_run == 0: print("All jobs are finished! Exiting.") return dirty_hosts = set() n_tasks_per_step = n_nodes * tasks_per_node n_steps = int(np.ceil(n_jobs_to_run / n_tasks_per_step)) node_file = " --sshloginfile nodefile.txt " wall_time_seconds, total_seconds_per_step, parallel_seconds_per_step, python_seconds_per_step = \ self.compute_time_limits(wall_time, cleanup_time, slack_time, step_time_limit, n_steps) self.__dict__.update(locals()) self.print_time_limits()
def __init__( self, name, input_zip, pattern, scratch, local_scratch_prefix='/tmp/dps/hyper/', ppn=12, cpp=1, pmem=None, wall_time="1hour", cleanup_time="1min", slack_time="1min", add_date=True, dry_run=0, parallel_exe=None, kind="parallel", host_pool=None, load_avg_threshold=8., min_hosts=None, max_hosts=1, env_vars=None, output_to_files=True, n_retries=0, gpu_set="", copy_venv="", python_startup=False, step_time_limit=None, ignore_gpu=False, ssh_options=None, loud_output=True, rsync_verbosity=0): args = locals().copy() del args['self'] print("\nParallelSession args:") print(args) launch_venv = os.getenv('VIRTUAL_ENV') if launch_venv: launch_venv = os.path.split(launch_venv)[1] if not parallel_exe: parallel_exe = "$HOME/.local/bin/parallel" if ssh_options is None: ssh_options = ( "-oPasswordAuthentication=no " "-oStrictHostKeyChecking=no " "-oConnectTimeout=5 " "-oServerAliveInterval=2" ) if kind == "pbs": local_scratch_prefix = "\\$RAMDISK" assert kind in "parallel pbs slurm slurm-local".split() hpc = kind != "parallel" # Create directory to run the job from - should be on scratch. scratch = os.path.abspath(os.path.expandvars(scratch)) es = ExperimentStore(scratch, prefix="run_search") job_dir = es.new_experiment(name, 0, add_date=add_date, force_fresh=1) job_dir.record_environment() with open(job_dir.path_for('run_kwargs.json'), 'w') as f: json.dump(args, f, default=str, indent=4, sort_keys=True) del f del args job_path = job_dir.path job_dir.make_directory('experiments') input_zip_stem = path_stem(input_zip) input_zip = shutil.copy(input_zip, job_dir.path_for("orig.zip")) input_zip_abs = process_path(input_zip) input_zip_base = os.path.basename(input_zip) archive_root = zip_root(input_zip) self.copy_files( job_dir, input_zip, archive_root, ["README.md", "sampled_configs.txt", "config.json", "config.pkl"]) # storage local to each node, from the perspective of that node local_scratch = os.path.join(local_scratch_prefix, os.path.basename(job_path)) output_to_files = "--output-to-files" if output_to_files else "" env = os.environ.copy() env_vars = env_vars or {} env.update({e: str(v) for e, v in env_vars.items()}) env_vars = ' '.join('--env ' + k for k in env_vars) rsync_verbosity = "" if not rsync_verbosity else "-" + "v" * rsync_verbosity ro_job = ReadOnlyJob(input_zip) indices_to_run = sorted([op.idx for op in ro_job.ready_incomplete_ops(sort=False)]) del ro_job n_jobs_to_run = len(indices_to_run) if n_jobs_to_run == 0: print("All jobs are finished! Exiting.") return dirty_hosts = set() if hpc: host_pool = [] n_nodes = max_hosts n_procs = n_nodes * ppn n_steps = int(np.ceil(n_jobs_to_run / n_procs)) else: self.__dict__.update(locals()) host_pool = host_pool or DEFAULT_HOST_POOL if isinstance(host_pool, str): host_pool = host_pool.split() # Get an estimate of the number of hosts we'll have available. with cd(job_path): hosts, n_procs = self.recruit_hosts( hpc, min_hosts, max_hosts, host_pool, ppn, max_procs=np.inf) n_nodes = len(hosts) if n_jobs_to_run < n_procs: n_steps = 1 n_nodes = int(np.ceil(n_jobs_to_run / ppn)) n_procs = n_nodes * ppn hosts = hosts[:n_nodes] else: n_steps = int(np.ceil(n_jobs_to_run / n_procs)) node_file = " --sshloginfile nodefile.txt " wall_time_seconds, total_seconds_per_step, parallel_seconds_per_step, python_seconds_per_step = \ self.compute_time_limits(wall_time, cleanup_time, slack_time, step_time_limit, n_steps) self.__dict__.update(locals()) self.print_time_limits()
import argparse import numpy as np import pandas as pd from collections import defaultdict, OrderedDict import matplotlib.pyplot as plt from dps import cfg from dps.hyper import HyperSearch from dps.train import FrozenTrainingLoopData from dps.utils import ( process_path, Config, sha_cache, set_clear_cache, confidence_interval, standard_error, grid_subplots ) data_dir = "/media/data/Dropbox/experiment_data/active/aaai_2020/" cache_dir = process_path('/home/eric/.cache/dps_plots') plot_dir = '/media/data/Dropbox/writeups/spatially_invariant_air/aaai_2020/figures' plot_paths = Config() plot_paths[''] = '' verbose_cache = True def std_dev(ys): y_upper = y_lower = [_y.std() for _y in ys] return y_upper, y_lower def ci95(ys): conf_int = [confidence_interval(_y, 0.95) for _y in ys]