def create_sample(num_classes, num_samples, exp_name): # create sample script logger = log.SearchLogger('logs', exp_name, delete_if_exists=True, create_parent_folders=True) result_fp = 'logs/%s/sample_info.json' % exp_name out_fp = 'logs/%s/sample_slurm.out' % exp_name jobname = 'sampling_architecture_for_%s_experiment' % exp_name script = [ '#!/bin/bash', '#SBATCH --nodes=1', '#SBATCH --partition=GPU-shared', '#SBATCH --gres=gpu:p100:1', '#SBATCH --ntasks-per-node=1', '#SBATCH --time=00:30:00', '#SBATCH --job-name=%s' % jobname, '#SBATCH --output=%s' % out_fp, 'module load keras/2.2.0_tf1.7_py3_gpu ', 'source activate', 'export PYTHONPATH=${PYTHONPATH}:/pylon5/ci4s8dp/maxle18/deep_architect/', 'python examples/tutorials/multiworker/searcher.py --num_classes %d --num_samples %d --exp_name %s --result_fp %s' % (num_classes, num_samples, exp_name, result_fp) ] script_fp = 'logs/%s/sample.sh' % exp_name ut.write_textfile(script_fp, script) check_output(["chmod", "+x", script_fp]) # submit batch job and obtain id job_id = check_output(['sbatch', script_fp]).split()[3] while True: # perhaps need to have a time guard here if (done(job_id) and valid_filepath(result_fp)): return ut.read_jsonfile(result_fp) return dict()
def create_and_submit_job(i, exp_name, config_fp, evaluation_fp): jobname = "%s_arch%d" % (exp_name, i) out_fp = "%s/slurm.out" % evaluation_fp result_fp = "%s/result.json" % evaluation_fp script = [ '#!/bin/bash', '#SBATCH --nodes=1', '#SBATCH --partition=GPU-shared', '#SBATCH --gres=gpu:p100:1', '#SBATCH --ntasks-per-node=1', '#SBATCH --time=00:15:00', '#SBATCH --job-name=%s' % jobname, '#SBATCH --output=%s' % out_fp, 'python evaluator.py --config %s > %s' % (config_fp, result_fp) ] script_fp = "%s/run%d.sh" % (evaluation_fp, i) ut.write_textfile(script_fp, script) check_output(["chmod", "+x", script_fp]) # submit bash job and obtain id out = check_output(['sbatch', script_fp]) return (out.split()[3], result_fp)
if s == ".. code:: python": inside_block = True # if it terminates with a block. likely unusual. if inside_block: for b_line in block_lst[block_idx]: out_lines.append((' ' * indent) + s) elif d["in_doc_filepath"].endswith('.md'): for line in lines: s = line.rstrip() if s == '```python': inside_block = True out_lines.append(s) elif s == '```': if inside_block: out_lines.extend(block_lst[block_idx]) block_idx += 1 inside_block = False out_lines.append(s) else: if not inside_block: out_lines.append(s) else: raise ValueError( "File %s is not supported. Supported formats .md and .rst" % d["in_doc_filepath"]) ut.write_textfile(d["out_filepath"], out_lines)
def calibration_table(time_sequence_lst, value_sequence_lst, maximizing=True, max_num_ranks=8, rank_increase_factor=2, rank_multiplicative_increases=True, start_time=1e-3, num_time_instants=16, time_increase_factor=2, time_multiplicative_increases=True, time_label=None, value_label=None, show=True, table_filepath=None): """Gets the rank of different models at different time steps. The true rank of a model is determined by the best performance that it achieves in the sequence. A good amount of computation is suffient to order models close to their true ordering. Entries in the table are true rank of the model. This functionality is useful to inform what amount of computation is sufficient to perform architecture search on this problem. """ assert len(time_sequence_lst) == len(value_sequence_lst) if maximizing: value_lst = [np.max(seq) for seq in value_sequence_lst] else: value_lst = [np.min(seq) for seq in value_sequence_lst] (sorted_time_sequence_lst, sorted_value_sequence_lst) = sort_sequences( [time_sequence_lst, value_sequence_lst], value_lst, maximizing) num_sequences = len(time_sequence_lst) indices = generate_indices(num_sequences, max_num_ranks, rank_increase_factor, rank_multiplicative_increases) if time_multiplicative_increases: time_instants = [ start_time * (time_increase_factor**i) for i in range(num_time_instants) ] else: time_instants = [ start_time + i * time_increase_factor for i in range(num_time_instants) ] rows = [] for t in time_instants: values_at_t = [] for i in range(num_sequences): v = get_value_at_time(t, sorted_time_sequence_lst[i], sorted_value_sequence_lst[i], maximizing=maximizing) values_at_t.append(v) ranks = argsort(values_at_t, [lambda x: x], increasing=not maximizing) row = [ranks[idx] for idx in indices] rows.append(row) # constructing the table. table_lines = ["Total number of sequences: %d" % num_sequences] if time_label is not None or value_label is not None: lst = [] if time_label is not None: lst.append('time=%s' % time_label) if value_label is not None: lst.append('value=%s' % value_label) line = ", ".join(lst) table_lines.append(line) line = " ".join(['time / rank\t'] + ["%5d" % idx for idx in indices]) table_lines.append(line) for i, t in enumerate(time_instants): line = " ".join(["%2.2e\t" % t] + ["%5d" % rank for rank in rows[i]]) table_lines.append(line) if table_filepath is not None: ut.write_textfile(table_filepath, table_lines) if show: print "\n".join(table_lines)