def create_sample(num_classes, num_samples, exp_name):
# create sample script
logger = log.SearchLogger('logs',
exp_name,
delete_if_exists=True,
create_parent_folders=True)
result_fp = 'logs/%s/sample_info.json' % exp_name
out_fp = 'logs/%s/sample_slurm.out' % exp_name
jobname = 'sampling_architecture_for_%s_experiment' % exp_name
script = [
'#!/bin/bash', '#SBATCH --nodes=1', '#SBATCH --partition=GPU-shared',
'#SBATCH --gres=gpu:p100:1', '#SBATCH --ntasks-per-node=1',
'#SBATCH --time=00:30:00',
'#SBATCH --job-name=%s' % jobname,
'#SBATCH --output=%s' % out_fp,
'module load keras/2.2.0_tf1.7_py3_gpu ', 'source activate',
'export PYTHONPATH=${PYTHONPATH}:/pylon5/ci4s8dp/maxle18/deep_architect/',
'python examples/tutorials/multiworker/searcher.py --num_classes %d --num_samples %d --exp_name %s --result_fp %s'
% (num_classes, num_samples, exp_name, result_fp)
]
script_fp = 'logs/%s/sample.sh' % exp_name
ut.write_textfile(script_fp, script)
check_output(["chmod", "+x", script_fp])
# submit batch job and obtain id
job_id = check_output(['sbatch', script_fp]).split()[3]
while True: # perhaps need to have a time guard here
if (done(job_id) and valid_filepath(result_fp)):
return ut.read_jsonfile(result_fp)
return dict()
def create_and_submit_job(i, exp_name, config_fp, evaluation_fp):
jobname = "%s_arch%d" % (exp_name, i)
out_fp = "%s/slurm.out" % evaluation_fp
result_fp = "%s/result.json" % evaluation_fp
script = [
'#!/bin/bash', '#SBATCH --nodes=1', '#SBATCH --partition=GPU-shared',
'#SBATCH --gres=gpu:p100:1', '#SBATCH --ntasks-per-node=1',
'#SBATCH --time=00:15:00',
'#SBATCH --job-name=%s' % jobname,
'#SBATCH --output=%s' % out_fp,
'python evaluator.py --config %s > %s' % (config_fp, result_fp)
]
script_fp = "%s/run%d.sh" % (evaluation_fp, i)
ut.write_textfile(script_fp, script)
check_output(["chmod", "+x", script_fp])
# submit bash job and obtain id
out = check_output(['sbatch', script_fp])
return (out.split()[3], result_fp)
if s == ".. code:: python":
inside_block = True
# if it terminates with a block. likely unusual.
if inside_block:
for b_line in block_lst[block_idx]:
out_lines.append((' ' * indent) + s)
elif d["in_doc_filepath"].endswith('.md'):
for line in lines:
s = line.rstrip()
if s == '```python':
inside_block = True
out_lines.append(s)
elif s == '```':
if inside_block:
out_lines.extend(block_lst[block_idx])
block_idx += 1
inside_block = False
out_lines.append(s)
else:
if not inside_block:
out_lines.append(s)
else:
raise ValueError(
"File %s is not supported. Supported formats .md and .rst" %
d["in_doc_filepath"])
ut.write_textfile(d["out_filepath"], out_lines)
def calibration_table(time_sequence_lst,
value_sequence_lst,
maximizing=True,
max_num_ranks=8,
rank_increase_factor=2,
rank_multiplicative_increases=True,
start_time=1e-3,
num_time_instants=16,
time_increase_factor=2,
time_multiplicative_increases=True,
time_label=None,
value_label=None,
show=True,
table_filepath=None):
"""Gets the rank of different models at different time steps.
The true rank of a model is determined by the best performance that it
achieves in the sequence. A good amount of computation is suffient to
order models close to their true ordering. Entries in the table are
true rank of the model.
This functionality is useful to inform what amount of computation is
sufficient to perform architecture search on this problem.
"""
assert len(time_sequence_lst) == len(value_sequence_lst)
if maximizing:
value_lst = [np.max(seq) for seq in value_sequence_lst]
else:
value_lst = [np.min(seq) for seq in value_sequence_lst]
(sorted_time_sequence_lst, sorted_value_sequence_lst) = sort_sequences(
[time_sequence_lst, value_sequence_lst], value_lst, maximizing)
num_sequences = len(time_sequence_lst)
indices = generate_indices(num_sequences, max_num_ranks,
rank_increase_factor,
rank_multiplicative_increases)
if time_multiplicative_increases:
time_instants = [
start_time * (time_increase_factor**i)
for i in range(num_time_instants)
]
else:
time_instants = [
start_time + i * time_increase_factor
for i in range(num_time_instants)
]
rows = []
for t in time_instants:
values_at_t = []
for i in range(num_sequences):
v = get_value_at_time(t,
sorted_time_sequence_lst[i],
sorted_value_sequence_lst[i],
maximizing=maximizing)
values_at_t.append(v)
ranks = argsort(values_at_t, [lambda x: x], increasing=not maximizing)
row = [ranks[idx] for idx in indices]
rows.append(row)
# constructing the table.
table_lines = ["Total number of sequences: %d" % num_sequences]
if time_label is not None or value_label is not None:
lst = []
if time_label is not None:
lst.append('time=%s' % time_label)
if value_label is not None:
lst.append('value=%s' % value_label)
line = ", ".join(lst)
table_lines.append(line)
line = " ".join(['time / rank\t'] + ["%5d" % idx for idx in indices])
table_lines.append(line)
for i, t in enumerate(time_instants):
line = " ".join(["%2.2e\t" % t] + ["%5d" % rank for rank in rows[i]])
table_lines.append(line)
if table_filepath is not None:
ut.write_textfile(table_filepath, table_lines)
if show:
print "\n".join(table_lines)