def read_search_folder(search_folderpath):
"""Reads all the standard JSON log files associated to a search experiment.
See also :func:`deep_architect.search_logging.read_evaluation_folder` for the function
that reads a single evaluation folder. The list of dictionaries is ordered
in increasing order of evaluation id.
Args:
search_folderpath (str): Path to the search folder used for logging.
Returns:
list[dict[str,dict[str,object]]]:
List of nested dictionaries with the logged information. Each
dictionary in the list corresponds to an evaluation.
"""
assert ut.folder_exists(search_folderpath)
all_evaluations_folderpath = ut.join_paths(
[search_folderpath, 'evaluations'])
eval_id = 0
log_lst = []
while True:
evaluation_folderpath = ut.join_paths(
[all_evaluations_folderpath,
'x%d' % eval_id])
if ut.folder_exists(evaluation_folderpath):
name_to_log = read_evaluation_folder(evaluation_folderpath)
log_lst.append(name_to_log)
eval_id += 1
else:
break
return log_lst
def draw_graph_evolution(outputs,
hyperp_value_lst,
out_folderpath,
graph_name='graph',
draw_hyperparameters=True,
draw_io_labels=True,
draw_module_hyperparameter_info=True):
def draw_fn(i):
return draw_graph(
outputs,
draw_hyperparameters=draw_hyperparameters,
draw_io_labels=draw_io_labels,
draw_module_hyperparameter_info=draw_module_hyperparameter_info,
out_folderpath=out_folderpath,
graph_name=graph_name + '-%d' % i,
print_to_screen=False)
draw_fn(0)
h_iter = co.unassigned_independent_hyperparameter_iterator(outputs)
for i, v in enumerate(hyperp_value_lst):
h = next(h_iter)
h.assign_value(v)
draw_fn(i + 1)
filepath_lst = [
ut.join_paths([out_folderpath, graph_name + '-%d.pdf' % i])
for i in range(len(hyperp_value_lst) + 1)
]
out_filepath = ut.join_paths([out_folderpath, graph_name + '.pdf'])
cmd = " ".join(["pdftk"] + filepath_lst + ["cat", "output", out_filepath])
ut.run_bash_command(cmd)
for fp in filepath_lst:
ut.delete_file(fp)
def __init__(self,
folderpath,
search_name,
evaluation_id,
abort_if_exists=False,
abort_if_notexists=False):
self.evaluation_folderpath = get_evaluation_folderpath(
folderpath, search_name, evaluation_id)
self.evaluation_data_folderpath = get_evaluation_data_folderpath(
folderpath, search_name, evaluation_id)
assert (not abort_if_exists) or (not ut.folder_exists(
self.evaluation_folderpath))
assert (not abort_if_notexists) or ut.folder_exists(
self.evaluation_folderpath)
ut.create_folder(self.evaluation_folderpath,
abort_if_exists=abort_if_exists,
create_parent_folders=True)
ut.create_folder(self.evaluation_data_folderpath,
abort_if_exists=abort_if_exists,
create_parent_folders=True)
self.config_filepath = ut.join_paths(
[self.evaluation_folderpath, 'config.json'])
self.results_filepath = ut.join_paths(
[self.evaluation_folderpath, 'results.json'])
def save_state(self, folderpath):
state = {
'num_evals': len(self.vecs_lst),
'vals_lst': self.vals_lst,
}
ut.write_jsonfile(state,
ut.join_paths([folderpath, 'hash_model_state.json']))
for i, vecs in enumerate(self.vecs_lst):
sp.save_npz(ut.join_paths([folderpath, str(i) + '.npz']), vecs)
def load_state(self, folderpath):
state = ut.read_jsonfile(
ut.join_paths([folderpath, 'hash_model_state.json']))
self.vals_lst = state['vals_lst']
num_evals = state['num_evals']
for i in range(num_evals):
self.vecs_lst.append(
sp.load_npz(ut.join_paths([folderpath,
str(i) + '.npz'])))
if num_evals > 0:
self._refit()
def load_state(self, folderpath):
filepath = ut.join_paths([folderpath, 'evolution_searcher.json'])
state = ut.read_jsonfile(filepath)
self.P = state["P"]
self.S = state["S"]
self.regularized = state['regularized']
self.population = deque(state['population'])
self.initializing = state['initializing']
def save_state(self, folderpath):
filepath = ut.join_paths([folderpath, 'evolution_searcher.json'])
state = {
"P": self.P,
"S": self.S,
"population": list(self.population),
"regularized": self.regularized,
"initializing": self.initializing,
}
ut.write_jsonfile(state, filepath)
def load_state(self, folder_name):
filepath = join_paths([folder_name, 'evolution_searcher.json'])
if not file_exists(filepath):
raise RuntimeError("Load file does not exist")
state = read_jsonfile(filepath)
self.P = state["P"]
self.S = state["S"]
self.regularized = state['regularized']
self.population = deque(state['population'])
self.initializing = state['initializing']
def main():
num_classes = 10
(Xtrain, ytrain, Xval, yval, Xtest, ytest) = load_mnist('data/mnist')
train_dataset = InMemoryDataset(Xtrain, ytrain, True)
val_dataset = InMemoryDataset(Xval, yval, False)
test_dataset = InMemoryDataset(Xtest, ytest, False)
evaluator = SimpleClassifierEvaluator(
train_dataset,
val_dataset,
num_classes,
ut.join_paths(['temp', 'benchmarks', cfg['search_name']]),
max_eval_time_in_minutes=cfg['max_eval_time_in_minutes'],
log_output_to_terminal=True,
test_dataset=test_dataset)
for rep_i in range(cfg['num_repetitions']):
for search_space_name in cfg['search_space_name_lst']:
for searcher_name in cfg['searcher_name_lst']:
folderpath = ut.join_paths([
cfg['logs_folderpath'], cfg['search_name'],
search_space_name, searcher_name
])
sl.create_search_folderpath(
folderpath,
'rep%d' % rep_i,
abort_if_exists=True,
delete_if_exists=cfg["delete_if_exists"],
create_parent_folders=True)
search_space_fn = local_ss.name_to_search_space_fn[
search_space_name](num_classes)
searcher = local_se.name_to_get_searcher_fn[searcher_name](
search_space_fn)
run_searcher(
searcher, evaluator, cfg['num_samples'] -
search_logger.get_current_evaluation_id(), search_logger)
def start_worker(comm,
evaluator,
search_space_factory,
folderpath,
search_name,
resume=True,
save_every=1):
# set the available gpu for process
print('WORKER %d' % comm.get_rank())
step = 0
sl.create_search_folderpath(folderpath, search_name)
search_data_folder = sl.get_search_data_folderpath(folderpath, search_name)
save_filepath = ut.join_paths(
(search_data_folder, 'worker' + str(comm.get_rank()) + '.json'))
if resume:
evaluator.load_state(search_data_folder)
state = ut.read_jsonfile(save_filepath)
step = state['step']
while (True):
arch = comm.receive_architecture_in_worker()
# if a kill signal is received
if arch is None:
break
vs, evaluation_id, searcher_eval_token = arch
inputs, outputs = search_space_factory.get_search_space()
se.specify(outputs, vs)
results = evaluator.eval(inputs, outputs)
step += 1
if step % save_every == 0:
evaluator.save_state(search_data_folder)
state = {'step': step}
ut.write_jsonfile(state, save_filepath)
comm.publish_results_to_master(results, evaluation_id,
searcher_eval_token)
def read_evaluation_folder(evaluation_folderpath):
"""Reads all the standard JSON log files associated to a single evaluation.
See also :func:`deep_architect.search_logging.read_search_folder` for the function
that reads all the evaluations in a search folder.
Args:
evaluation_folderpath (str): Path to the folder containing the standard
JSON logs.
Returns:
dict[str,dict[str,object]]:
Nested dictionaries with the logged information. The first
dictionary has keys corresponding to the names of the standard
log files.
"""
assert ut.folder_exists(evaluation_folderpath)
name_to_log = {}
for name in ['config', 'results']:
log_filepath = ut.join_paths([evaluation_folderpath, name + '.json'])
name_to_log[name] = ut.read_jsonfile(log_filepath)
return name_to_log
def start_searcher(comm,
searcher,
resume_if_exists,
folderpath,
search_name,
searcher_load_path,
num_samples=-1,
num_epochs=-1,
save_every=1):
assert num_samples != -1 or num_epochs != -1
print('SEARCHER')
sl.create_search_folderpath(folderpath, search_name)
search_data_folder = sl.get_search_data_folderpath(folderpath, search_name)
save_filepath = ut.join_paths((search_data_folder, searcher_load_path))
models_sampled = 0
epochs = 0
finished = 0
killed = 0
best_accuracy = 0.
# Load previous searcher
if resume_if_exists:
searcher.load(search_data_folder)
state = ut.read_jsonfile(save_filepath)
epochs = state['epochs']
killed = state['killed']
models_sampled = state['models_finished']
finished = state['models_finished']
while (finished < models_sampled or killed < comm.num_workers):
# Search end conditions
cont = num_samples == -1 or models_sampled < num_samples
cont = cont and (num_epochs == -1 or epochs < num_epochs)
if cont:
# See whether workers are ready to consume architectures
if comm.is_ready_to_publish_architecture():
eval_logger = sl.EvaluationLogger(folderpath, search_name,
models_sampled)
_, _, vs, searcher_eval_token = searcher.sample()
eval_logger.log_config(vs, searcher_eval_token)
comm.publish_architecture_to_worker(vs, models_sampled,
searcher_eval_token)
models_sampled += 1
else:
if comm.is_ready_to_publish_architecture():
comm.kill_worker()
killed += 1
# See which workers have finished evaluation
for worker in range(comm.num_workers):
msg = comm.receive_results_in_master(worker)
if msg is not None:
results, model_id, searcher_eval_token = msg
eval_logger = sl.EvaluationLogger(folderpath, search_name,
model_id)
eval_logger.log_results(results)
if 'epoch' in results:
epochs = max(epochs, results['epoch'])
searcher.update(results['validation_accuracy'],
searcher_eval_token)
best_accuracy = max(best_accuracy,
results['validation_accuracy'])
finished += 1
if finished % save_every == 0:
print('Models sampled: %d Best Accuracy: %f' %
(finished, best_accuracy))
best_accuracy = 0.
searcher.save_state(search_data_folder)
state = {
'models_finished': finished,
'epochs': epochs,
'killed': killed
}
ut.write_jsonfile(state, save_filepath)
def main():
configs = ut.read_jsonfile(
"./examples/tensorflow/full_benchmarks/experiment_config.json")
parser = argparse.ArgumentParser("MPI Job for architecture search")
parser.add_argument('--config',
'-c',
action='store',
dest='config_name',
default='normal')
# Other arguments
parser.add_argument('--display-output',
'-o',
action='store_true',
dest='display_output',
default=False)
parser.add_argument('--resume',
'-r',
action='store_true',
dest='resume',
default=False)
options = parser.parse_args()
config = configs[options.config_name]
num_procs = config['num_procs'] if 'num_procs' in config else 0
comm = get_communicator(config['communicator'], num_procs)
if len(gpu_utils.get_gpu_information()) != 0:
#https://github.com/tensorflow/tensorflow/issues/1888
gpu_utils.set_visible_gpus(
[comm.get_rank() % gpu_utils.get_total_num_gpus()])
if 'eager' in config and config['eager']:
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)
tf.enable_eager_execution()
datasets = {
'cifar10': lambda: (load_cifar10('data/cifar10/', one_hot=False), 10),
'mnist': lambda: (load_mnist('data/mnist/'), 10),
}
(Xtrain, ytrain, Xval, yval, Xtest,
ytest), num_classes = datasets[config['dataset']]()
search_space_factory = name_to_search_space_factory_fn[
config['search_space']](num_classes)
save_every = 1 if 'save_every' not in config else config['save_every']
if comm.get_rank() == 0:
searcher = name_to_searcher_fn[config['searcher']](
search_space_factory.get_search_space)
num_samples = -1 if 'samples' not in config else config['samples']
num_epochs = -1 if 'epochs' not in config else config['epochs']
start_searcher(comm,
searcher,
options.resume,
config['search_folder'],
config['search_name'],
config['searcher_file_name'],
num_samples=num_samples,
num_epochs=num_epochs,
save_every=save_every)
else:
train_d_advataset = InMemoryDataset(Xtrain, ytrain, True)
val_dataset = InMemoryDataset(Xval, yval, False)
test_dataset = InMemoryDataset(Xtest, ytest, False)
search_path = sl.get_search_folderpath(config['search_folder'],
config['search_name'])
ut.create_folder(ut.join_paths([search_path, 'scratch_data']),
create_parent_folders=True)
scratch_folder = ut.join_paths(
[search_path, 'scratch_data', 'eval_' + str(comm.get_rank())])
ut.create_folder(scratch_folder)
evaluators = {
'simple_classification':
lambda: SimpleClassifierEvaluator(
train_dataset,
val_dataset,
num_classes,
'./temp' + str(comm.get_rank()),
max_num_training_epochs=config['eval_epochs'],
log_output_to_terminal=options.display_output,
test_dataset=test_dataset),
}
assert not config['evaluator'].startswith('enas') or hasattr(
search_space_factory, 'weight_sharer')
evaluator = evaluators[config['evaluator']]()
start_worker(comm,
evaluator,
search_space_factory,
config['search_folder'],
config['search_name'],
resume=options.resume,
save_every=save_every)
def eval(self, inputs, outputs):
tf.reset_default_graph()
model_dir = ut.join_paths(
[self.base_dir, 'eval' + str(self.num_archs)])
ut.create_folder(model_dir, abort_if_exists=False)
def model_fn(features, labels, mode, params):
feature_columns = list(get_feature_columns().values())
images = tf.feature_column.input_layer(
features=features, feature_columns=feature_columns)
images = tf.reshape(images,
shape=(-1, IMAGE_HEIGHT, IMAGE_WIDTH,
IMAGE_DEPTH))
set_recompile(outputs, True)
gc.collect()
htfe.set_is_training(outputs, mode == tf.estimator.ModeKeys.TRAIN)
co.forward({inputs['in']: images})
logits = outputs['out'].val
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
predicted_classes = tf.argmax(logits, 1)
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'class_ids': predicted_classes[:, tf.newaxis],
'probabilities': tf.nn.softmax(logits),
'logits': logits,
}
return tf.estimator.EstimatorSpec(mode,
predictions=predictions)
# define loss and optimizer
train_vars = tf.trainable_variables()
with tf.variable_scope('l2'):
l2_loss = tf.add_n([
tf.nn.l2_loss(v) for v in train_vars if 'kernel' in v.name
]) * self.weight_decay
unreg_loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=labels))
loss = unreg_loss + l2_loss
# Compute evaluation metrics.
accuracy = tf.metrics.accuracy(labels=tf.argmax(labels, 1),
predictions=predicted_classes,
name='acc_op')
metrics = {'accuracy': accuracy}
if mode == tf.estimator.ModeKeys.EVAL:
loss = tf.Print(loss, [
accuracy, l2_loss, unreg_loss, loss,
tf.argmax(labels, 1), predicted_classes
],
summarize=10)
return tf.estimator.EstimatorSpec(mode,
loss=loss,
eval_metric_ops=metrics)
# Create training op.
assert mode == tf.estimator.ModeKeys.TRAIN
step = tf.train.get_or_create_global_step()
learning_rate = self.get_learning_rate(step)
optimizer = tf.train.RMSPropOptimizer(learning_rate,
.9,
momentum=.9,
epsilon=1.0)
loss = tf.Print(loss, [
accuracy, l2_loss, unreg_loss, loss, learning_rate,
tf.argmax(labels, 1), predicted_classes
],
summarize=10)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(
loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode,
loss=loss,
train_op=train_op)
# NUM_GPUS = 2
# strategy = tf.contrib.distribute.MirroredStrategy(num_gpus=NUM_GPUS)
# config = tf.estimator.RunConfig(train_distribute=strategy)
gpu_ops = tf.GPUOptions(allow_growth=True)
config = tf.ConfigProto(gpu_options=gpu_ops)
run_config = tf.estimator.RunConfig(model_dir=model_dir,
session_config=config)
estimator = tf.estimator.Estimator(model_fn=model_fn,
config=run_config,
params={})
seqs = ut.SequenceTracker(abort_if_different_lengths=True)
best_val_acc = -np.inf
stop_counter = self.stop_patience
timer_manager = ut.TimerManager()
timer_manager.create_timer('eval')
# getting the gpu_id based on the environment.
if gpu_utils.is_environment_variable_defined('CUDA_VISIBLE_DEVICES'):
s = gpu_utils.get_environment_variable('CUDA_VISIBLE_DEVICES')
s_lst = s.split(',')
if len(s_lst) == 1 and len(s_lst[0]) > 0:
gpu_id = int(s_lst[0])
else:
gpu_id = None
else:
gpus = gpu_utils.get_gpu_information()
if len(gpus) == 1:
gpu_id = 0
else:
gpu_id = None
for epoch in range(self.max_num_training_epochs):
train_fn = lambda: input_fn(self.X_train, self.y_train, train=True)
val_fn = lambda: input_fn(self.X_val, self.y_val, train=False)
print('\n\nTraining')
estimator.train(input_fn=train_fn)
print('\n\nEvaluating')
eval_results = estimator.evaluate(input_fn=val_fn)
# early stopping
val_acc = eval_results['accuracy']
# Display logs per epoch step
if self.log_output_to_terminal and epoch % self.display_step == 0:
print(
"time:", "%7.1f" %
timer_manager.get_time_since_event('eval', 'start'),
"epoch:", '%04d' % (epoch + 1), "validation loss:",
"{:.9f}".format(eval_results['loss']),
"validation_accuracy:", "%.5f" % val_acc)
d = {
'validation_accuracy':
val_acc,
'validation_loss':
eval_results['loss'],
'epoch_number':
epoch + 1,
'time_in_minutes':
timer_manager.get_time_since_event('eval',
'start',
units='minutes'),
}
seqs.append(d)
# update the patience counters.
if best_val_acc < val_acc:
best_val_acc = val_acc
# reinitialize all the counters.
stop_counter = self.stop_patience
else:
stop_counter -= 1
if stop_counter == 0:
break
print("Optimization Finished!")
timer_manager.tick_timer('eval')
eval_results = estimator.evaluate(
input_fn=lambda: input_fn(self.X_val, self.y_val))
val_acc = eval_results['accuracy']
t_infer = (
timer_manager.get_time_since_last_tick('eval', 'miliseconds') /
self.X_val.shape[0])
print("Validation accuracy: %f" % val_acc)
seqs_dict = seqs.get_dict()
results = {
'validation_accuracy': val_acc,
'num_parameters': float(htf.get_num_trainable_parameters()),
'inference_time_per_example_in_miliseconds': t_infer,
'num_training_epochs': seqs_dict['epoch_number'],
'sequences': seqs_dict
}
if 'gpu_utilization_in_percent' in seqs_dict:
results['average_gpu_utilization_in_percent'] = np.mean(
seqs_dict['gpu_utilization_in_percent'])
results['average_gpu_memory_utilization_in_gigabytes'] = np.mean(
seqs_dict['gpu_memory_utilization_in_gigabytes'])
if self.X_test != None and self.y_test != None:
test_results = estimator.evaluate(
input_fn=lambda: input_fn(self.X_test, self.y_test))
test_acc = test_results['accuracy']
print("Test accuracy: %f" % test_acc)
results['test_accuracy'] = test_acc
results['training_time_in_hours'] = timer_manager.get_time_since_event(
'eval', 'start', units='hours')
self.num_archs += 1
return results
def get_all_evaluations_folderpath(folderpath, search_name):
return ut.join_paths(
[get_search_folderpath(folderpath, search_name), 'evaluations'])
def load_state(self, folderpath):
self.mcts.load_state(folderpath)
self.surr_model.load_state(folderpath)
state = ut.load_jsonfile(ut.join_paths([folderpath, "state.json"]))
self.cnt = state["cnt"]
def process_config_and_args():
parser = argparse.ArgumentParser("MPI Job for architecture search")
parser.add_argument('--config',
'-c',
action='store',
dest='config_name',
default='normal')
parser.add_argument(
'--config-file',
action='store',
dest='config_file',
default=
'/deep_architect/kubernetes/mongo_communicator/train_best_config.json')
parser.add_argument('--bucket',
'-b',
action='store',
dest='bucket',
default=BUCKET_NAME)
# Other arguments
parser.add_argument('--resume',
'-r',
action='store_true',
dest='resume',
default=False)
parser.add_argument('--mongo-host',
'-m',
action='store',
dest='mongo_host',
default='127.0.0.1')
parser.add_argument('--mongo-port',
'-p',
action='store',
dest='mongo_port',
default=27017)
parser.add_argument('--repetition', default=0)
options = parser.parse_args()
configs = ut.read_jsonfile(options.config_file)
config = configs[options.config_name]
config['bucket'] = options.bucket
comm = MongoCommunicator(host=options.mongo_host,
port=options.mongo_port,
data_refresher=True)
# SET UP GOOGLE STORE FOLDER
config['search_name'] = config['search_name'] + '_' + str(
options.repetition)
search_logger = sl.SearchLogger(config['search_folder'],
config['search_name'])
search_data_folder = search_logger.get_search_data_folderpath()
config['save_filepath'] = ut.join_paths(
(search_data_folder, config['searcher_file_name']))
config['eval_path'] = sl.get_all_evaluations_folderpath(
config['search_folder'], config['search_name'])
config['full_search_folder'] = sl.get_search_folderpath(
config['search_folder'], config['search_name'])
config['results_file'] = os.path.join(
config['results_prefix'] + '_' + str(options.repetition),
config['results_file'])
state = {'finished': 0, 'best_accuracy': 0.0}
if options.resume:
try:
download_folder(search_data_folder, config['full_search_folder'],
config['bucket'])
searcher.load_state(search_data_folder)
if ut.file_exists(config['save_filepath']):
old_state = ut.read_jsonfile(config['save_filepath'])
state['finished'] = old_state['finished']
state['best_accuracy'] = old_state['best_accuracy']
except:
pass
return comm, search_logger, state, config
def get_search_data_folderpath(folderpath, search_name):
return ut.join_paths(
[get_search_folderpath(folderpath, search_name), "search_data"])
def get_search_folderpath(folderpath, search_name):
return ut.join_paths([folderpath, search_name])
def is_search_log_folder(folderpath):
return ut.folder_exists(ut.join_paths([folderpath, 'evaluations', 'x0']))
def save_state(self, folder_name):
state = self.get_searcher_state_token()
write_jsonfile(state,
join_paths([folder_name, 'evolution_searcher.json']))
def get_evaluation_folderpath(folderpath, search_name, evaluation_id):
return ut.join_paths([
get_all_evaluations_folderpath(folderpath, search_name),
'x%d' % evaluation_id
])
def get_evaluation_data_folderpath(folderpath, search_name, evaluation_id):
return ut.join_paths([
get_evaluation_folderpath(folderpath, search_name, evaluation_id),
"eval_data"
])
def save_state(self, folderpath):
self.mcts.save_state(folderpath)
self.surr_model.save_state(folderpath)
ut.write_jsonfile({"cnt": self.cnt},
ut.join_paths([folderpath, "state.json"]))
ds = []
for i, log in enumerate(log_lst):
d = log['results']
# d.pop('sequences')
# d['num_training_epochs'] = len(d['num_training_epochs'])
d['evaluation_id'] = i
ds.append(d)
return ds
### NOTE: this needs to be changed with the paths for log folders that you
# wish to consider.
logs_folderpath = "logs"
# path_lst = ['logs/test_cifar10_short', 'logs/test_cifar10_medium', 'logs/test']
path_lst = [
ut.join_paths([logs_folderpath, p])
for p in ['cifar10_medium', "cifar10_short"]
]
print(path_lst)
path_to_log = {p: process_logs(sl.read_search_folder(p)) for p in path_lst}
keys = path_to_log.values()[0][0].keys()
print keys
ds = path_to_log.values()[0]
### Components to layout the HTML page.
def full_column(contents):
return html.Div(contents, className="row")
def one_half_one_half_column(left_contents, right_contents):
def save_state(self, folderpath):
ut.write_jsonfile(
{
'mcts_root_node': MCTSTreeNode.serialize(self.mcts_root_node),
}, ut.join_paths([folderpath, 'mcts_searcher_state.json']))
def load_state(self, folderpath):
state = ut.read_jsonfile(
ut.join_paths([folderpath, 'mcts_searcher_state.json']))
self.mcts_root_node = MCTSTreeNode.deserialize(state['mcts_root_node'])
data_dir, num_classes = datasets[config['dataset']]
search_space_factory = name_to_search_space_factory_fn[config['search_space']](
num_classes)
save_every = 1 if 'save_every' not in config else config['save_every']
searcher = name_to_searcher_fn[config['searcher']](
search_space_factory.get_search_space)
num_epochs = -1 if 'epochs' not in config else config['epochs']
num_samples = -1 if 'samples' not in config else config['samples']
eval_epochs = config['eval_epochs']
# SET UP GOOGLE STORE FOLDER
search_logger = sl.SearchLogger(config['search_folder'], config['search_name'])
# sl.create_search_folderpath(config['search_folder'], config['search_name'])
search_data_folder = search_logger.get_search_data_folderpath()
save_filepath = ut.join_paths(
(search_data_folder, config['searcher_file_name']))
eval_path = sl.get_all_evaluations_folderpath(config['search_folder'],
config['search_name'])
search_folder = sl.get_search_folderpath(config['search_folder'],
config['search_name'])
models_sampled = 0
epochs = 0
finished = 0
killed = 0
best_accuracy = 0.
# Load previous searcher
if options.resume:
download_folder(search_folder)
searcher.load(search_data_folder)
state = ut.read_jsonfile(save_filepath)
def process_config_and_args():
parser = argparse.ArgumentParser("MPI Job for architecture search")
parser.add_argument('--config',
'-c',
action='store',
dest='config_name',
default='normal')
parser.add_argument(
'--config-file',
action='store',
dest='config_file',
default=
'/deep_architect/examples/contrib/kubernetes/experiment_config.json')
parser.add_argument('--bucket',
'-b',
action='store',
dest='bucket',
default=BUCKET_NAME)
# Other arguments
parser.add_argument('--resume',
'-r',
action='store_true',
dest='resume',
default=False)
parser.add_argument('--mongo-host',
'-m',
action='store',
dest='mongo_host',
default='127.0.0.1')
parser.add_argument('--mongo-port',
'-p',
action='store',
dest='mongo_port',
default=27017)
parser.add_argument('--log',
choices=['debug', 'info', 'warning', 'error'],
default='info')
parser.add_argument('--repetition', default=0)
options = parser.parse_args()
numeric_level = getattr(logging, options.log.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % options.log)
logging.getLogger().setLevel(numeric_level)
configs = ut.read_jsonfile(options.config_file)
config = configs[options.config_name]
config['bucket'] = options.bucket
comm = MongoCommunicator(host=options.mongo_host,
port=options.mongo_port,
data_refresher=True,
refresh_period=10)
datasets = {
'cifar10': ('data/cifar10/', 10),
}
_, num_classes = datasets[config['dataset']]
search_space_factory = name_to_search_space_factory_fn[
config['search_space']](num_classes)
config['save_every'] = 1 if 'save_every' not in config else config[
'save_every']
searcher = name_to_searcher_fn[config['searcher']](
search_space_factory.get_search_space)
config['num_epochs'] = -1 if 'epochs' not in config else config['epochs']
config[
'num_samples'] = -1 if 'samples' not in config else config['samples']
# SET UP GOOGLE STORE FOLDER
config['search_name'] = config['search_name'] + '_' + str(
options.repetition)
search_logger = sl.SearchLogger(config['search_folder'],
config['search_name'])
search_data_folder = search_logger.get_search_data_folderpath()
config['save_filepath'] = ut.join_paths(
(search_data_folder, config['searcher_file_name']))
config['eval_path'] = sl.get_all_evaluations_folderpath(
config['search_folder'], config['search_name'])
config['full_search_folder'] = sl.get_search_folderpath(
config['search_folder'], config['search_name'])
config['eval_hparams'] = {} if 'eval_hparams' not in config else config[
'eval_hparams']
state = {
'epochs': 0,
'models_sampled': 0,
'finished': 0,
'best_accuracy': 0.0
}
if options.resume:
try:
download_folder(search_data_folder, config['full_search_folder'],
config['bucket'])
searcher.load_state(search_data_folder)
if ut.file_exists(config['save_filepath']):
old_state = ut.read_jsonfile(config['save_filepath'])
state['epochs'] = old_state['epochs']
state['models_sampled'] = old_state['models_sampled']
state['finished'] = old_state['finished']
state['best_accuracy'] = old_state['best_accuracy']
except:
pass
return comm, search_logger, searcher, state, config
