def main():
num_classes = 10
num_samples = 3 # number of architecture to sample
best_val_acc, best_architecture = 0., -1
# donwload and normalize data, using test as val for simplicity
X_train, y_train, X_val, y_val, _, _ = load_mnist('data/mnist',
normalize_range=True)
# defining evaluator
evaluator = SimpleClassifierEvaluator((X_train, y_train), (X_val, y_val),
num_classes,
max_num_training_epochs=5,
log_output_to_terminal=True)
searcher = se.RandomSearcher(get_search_space(num_classes))
for i in xrange(num_samples):
print("Sampling architecture %d" % i)
M.renew_collection()
inputs, outputs, _, searcher_eval_token = searcher.sample()
val_acc = evaluator.evaluate(
inputs,
outputs)['val_acc'] # evaluate and return validation accuracy
print("Finished evaluating architecture %d, validation accuracy is %f" %
(i, val_acc))
if val_acc > best_val_acc:
best_val_acc = val_acc
best_architecture = i
searcher.update(val_acc, searcher_eval_token)
print("Best validation accuracy is %f with architecture %d" %
(best_val_acc, best_architecture))
def main():
num_classes = 10
num_samples = 4
num_training_epochs = 2
# NOTE: change to True for graph visualization
show_graph = False
# load data
(X_train, y_train, X_val, y_val, X_test, y_test) = load_mnist('data/mnist',
flatten=True)
train_dataset = InMemoryDataset(X_train, y_train, True)
val_dataset = InMemoryDataset(X_val, y_val, False)
test_dataset = InMemoryDataset(X_test, y_test, False)
# defining evaluator and searcher
evaluator = SimpleClassifierEvaluator(
train_dataset,
val_dataset,
num_classes,
num_training_epochs=num_training_epochs,
log_output_to_terminal=True)
ssf = mo.SearchSpaceFactory(lambda: dnn_net(num_classes))
searcher = se.RandomSearcher(ssf.get_search_space)
for i in range(num_samples):
inputs, outputs, _, searcher_eval_token = searcher.sample()
if show_graph:
# try setting draw_module_hyperparameter_info=False and
# draw_hyperparameters=True for a different visualization.
vi.draw_graph(outputs,
draw_module_hyperparameter_info=False,
draw_hyperparameters=True)
results = evaluator.evaluate(inputs, outputs)
# updating the searcher. no-op for the random searcher.
searcher.update(results['validation_accuracy'], searcher_eval_token)
def main():
# Loading the config file.
cfg = ut.get_config()
num_classes = 10
num_samples = cfg['num_samples']
# Loading the data.
(Xtrain, ytrain, Xval, yval, Xtest, ytest) = load_mnist('data/mnist')
train_dataset = InMemoryDataset(Xtrain, ytrain, True)
val_dataset = InMemoryDataset(Xval, yval, False)
test_dataset = InMemoryDataset(Xtest, ytest, False)
# Creating up the evaluator.
evaluator = SimpleClassifierEvaluator(
train_dataset,
val_dataset,
num_classes,
'./temp',
max_eval_time_in_minutes=cfg['max_eval_time_in_minutes'],
log_output_to_terminal=True,
test_dataset=test_dataset)
# Creating the search space.
search_space_fn = lambda: css_dnn.dnn_net(num_classes)
search_space_factory = mo.SearchSpaceFactory(search_space_fn)
sl.create_search_folderpath(
cfg["folderpath"],
cfg["search_name"],
delete_if_exists=cfg['delete_if_exists'],
abort_if_exists=False,
create_parent_folders=True)
# Creating the searcher.
searcher = RandomSearcher(search_space_factory.get_search_space)
# Search loop.
for evaluation_id in range(num_samples):
eval_logger = sl.EvaluationLogger(cfg["folderpath"], cfg["search_name"],
evaluation_id)
if not eval_logger.config_exists():
inputs, outputs, hyperp_value_lst, eval_token = searcher.sample()
results = evaluator.eval(inputs, outputs)
# Logging results (including graph).
eval_logger.log_config(hyperp_value_lst, eval_token)
eval_logger.log_results(results)
vi.draw_graph(
outputs,
draw_module_hyperparameter_info=True,
print_to_screen=False,
out_folderpath=eval_logger.get_evaluation_data_folderpath())
# Updating the searcher given the results of logging.
searcher.update(results['validation_accuracy'], eval_token)
def main():
num_classes = 10
num_samples = 8
(Xtrain, ytrain, Xval, yval, Xtest, ytest) = load_mnist('data/mnist')
train_dataset = InMemoryDataset(Xtrain, ytrain, True)
val_dataset = InMemoryDataset(Xval, yval, False)
test_dataset = InMemoryDataset(Xtest, ytest, False)
evaluator = SimpleClassifierEvaluator(
train_dataset,
val_dataset,
num_classes,
'./temp',
max_eval_time_in_minutes=1.0,
log_output_to_terminal=True)
search_space_fn = lambda: css_dnn.dnn_net(num_classes)
searcher = RandomSearcher(search_space_fn)
for _ in range(num_samples):
inputs, outputs, searcher_eval_token, _ = searcher.sample()
val_acc = evaluator.eval(inputs, outputs)['validation_accuracy']
searcher.update(val_acc, searcher_eval_token)
def main():
num_classes = 10
(Xtrain, ytrain, Xval, yval, Xtest, ytest) = load_mnist('data/mnist')
train_dataset = InMemoryDataset(Xtrain, ytrain, True)
val_dataset = InMemoryDataset(Xval, yval, False)
test_dataset = InMemoryDataset(Xtest, ytest, False)
evaluator = SimpleClassifierEvaluator(
train_dataset,
val_dataset,
num_classes,
ut.join_paths(['temp', 'benchmarks', cfg['search_name']]),
max_eval_time_in_minutes=cfg['max_eval_time_in_minutes'],
log_output_to_terminal=True,
test_dataset=test_dataset)
for rep_i in range(cfg['num_repetitions']):
for search_space_name in cfg['search_space_name_lst']:
for searcher_name in cfg['searcher_name_lst']:
folderpath = ut.join_paths([
cfg['logs_folderpath'], cfg['search_name'],
search_space_name, searcher_name
])
sl.create_search_folderpath(
folderpath,
'rep%d' % rep_i,
abort_if_exists=True,
delete_if_exists=cfg["delete_if_exists"],
create_parent_folders=True)
search_space_fn = local_ss.name_to_search_space_fn[
search_space_name](num_classes)
searcher = local_se.name_to_get_searcher_fn[searcher_name](
search_space_fn)
run_searcher(
searcher, evaluator, cfg['num_samples'] -
search_logger.get_current_evaluation_id(), search_logger)
def main():
num_classes = 10
num_samples = 3
num_training_epochs = 2
batch_size = 256
# NOTE: change to True for graph visualization
show_graph = False
# load and normalize data
(X_train, y_train, X_val, y_val, X_test,
y_test) = load_mnist(flatten=True, one_hot=False)
train_dataset = InMemoryDataset(X_train, y_train, True)
val_dataset = InMemoryDataset(X_val, y_val, False)
test_dataset = InMemoryDataset(X_test, y_test, False)
# defining evaluator and searcher
evaluator = SimpleClassifierEvaluator(
train_dataset,
val_dataset,
num_classes,
num_training_epochs=num_training_epochs,
batch_size=batch_size,
log_output_to_terminal=True)
search_space_fn = lambda: dnn_net(num_classes)
searcher = se.RandomSearcher(search_space_fn)
for i in range(num_samples):
inputs, outputs, _, searcher_eval_token = searcher.sample()
if show_graph:
# try setting draw_module_hyperparameter_info=False and
# draw_hyperparameters=True for a different visualization.
vi.draw_graph(outputs,
draw_module_hyperparameter_info=False,
draw_hyperparameters=True)
results = evaluator.evaluate(inputs, outputs)
searcher.update(results['validation_accuracy'], searcher_eval_token)
def main():
cmd = ut.CommandLineArgs()
cmd.add('config_filepath', 'str')
cmd.add('worker_id', 'int')
cmd.add('num_workers', 'int')
out = cmd.parse()
cfg = ut.read_jsonfile(out['config_filepath'])
# Loading the data.
(Xtrain, ytrain, Xval, yval, Xtest, ytest) = load_mnist('data/mnist')
train_dataset = InMemoryDataset(Xtrain, ytrain, True)
val_dataset = InMemoryDataset(Xval, yval, False)
test_dataset = InMemoryDataset(Xtest, ytest, False)
# Creating up the evaluator.
evaluator = SimpleClassifierEvaluator(
train_dataset,
val_dataset,
ss.num_classes,
'./temp/worker%d' % out["worker_id"],
max_eval_time_in_minutes=cfg['max_eval_time_in_minutes'],
log_output_to_terminal=True,
test_dataset=test_dataset)
for evaluation_id in range(out["worker_id"], cfg["num_samples"],
out["num_workers"]):
logger = sl.EvaluationLogger(cfg["folderpath"],
cfg["search_name"],
evaluation_id,
abort_if_notexists=True)
if not logger.results_exist():
eval_cfg = logger.read_config()
inputs, outputs = ss.search_space_fn()
specify(outputs, eval_cfg["hyperp_value_lst"])
results = evaluator.eval(inputs, outputs)
logger.log_results(results)
default='mpi')
parser.add_argument('--num-procs', '-n', type=int, default=2)
args = parser.parse_args()
# First, create the communicator. This communicator is used by by to master to
# send candidate architectures to the workers to evaluate, and by the workers
# to send back the results for the architectures they evaluated. Currently,
# the communicator can be MPI based or file based (file based requires the
# processes to share a filesystem).
comm = get_communicator(args.comm, num_procs=args.num_procs)
# This is the number of total models to be evaluated in search
num_total_models = 25
# Now we set up the datasets and the search space factory.
X_train, y_train, X_val, y_val, _, _ = load_mnist('data/mnist',
normalize_range=True)
train_dataset = InMemoryDataset(X_train, y_train, True)
val_dataset = InMemoryDataset(X_val, y_val, False)
ssf = mo.SearchSpaceFactory(lambda: dnn.dnn_net(10))
# Each process should have a unique rank. The process with rank 0 will act as the
# master process that is in charge of the searcher. Every other process acts
# as a worker that evaluates architectures sent to them.
if comm.get_rank() == 0:
searcher = RandomSearcher(ssf.get_search_space)
models_sampled = 0
killed = 0
finished = 0
# This process keeps going as long as we have not received results for all sampled
def main():
configs = ut.read_jsonfile(
"./examples/tensorflow/full_benchmarks/experiment_config.json")
parser = argparse.ArgumentParser("MPI Job for architecture search")
parser.add_argument('--config',
'-c',
action='store',
dest='config_name',
default='normal')
# Other arguments
parser.add_argument('--display-output',
'-o',
action='store_true',
dest='display_output',
default=False)
parser.add_argument('--resume',
'-r',
action='store_true',
dest='resume',
default=False)
options = parser.parse_args()
config = configs[options.config_name]
num_procs = config['num_procs'] if 'num_procs' in config else 0
comm = get_communicator(config['communicator'], num_procs)
if len(gpu_utils.get_gpu_information()) != 0:
#https://github.com/tensorflow/tensorflow/issues/1888
gpu_utils.set_visible_gpus(
[comm.get_rank() % gpu_utils.get_total_num_gpus()])
if 'eager' in config and config['eager']:
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)
tf.enable_eager_execution()
datasets = {
'cifar10': lambda: (load_cifar10('data/cifar10/', one_hot=False), 10),
'mnist': lambda: (load_mnist('data/mnist/'), 10),
}
(Xtrain, ytrain, Xval, yval, Xtest,
ytest), num_classes = datasets[config['dataset']]()
search_space_factory = name_to_search_space_factory_fn[
config['search_space']](num_classes)
save_every = 1 if 'save_every' not in config else config['save_every']
if comm.get_rank() == 0:
searcher = name_to_searcher_fn[config['searcher']](
search_space_factory.get_search_space)
num_samples = -1 if 'samples' not in config else config['samples']
num_epochs = -1 if 'epochs' not in config else config['epochs']
start_searcher(comm,
searcher,
options.resume,
config['search_folder'],
config['search_name'],
config['searcher_file_name'],
num_samples=num_samples,
num_epochs=num_epochs,
save_every=save_every)
else:
train_d_advataset = InMemoryDataset(Xtrain, ytrain, True)
val_dataset = InMemoryDataset(Xval, yval, False)
test_dataset = InMemoryDataset(Xtest, ytest, False)
search_path = sl.get_search_folderpath(config['search_folder'],
config['search_name'])
ut.create_folder(ut.join_paths([search_path, 'scratch_data']),
create_parent_folders=True)
scratch_folder = ut.join_paths(
[search_path, 'scratch_data', 'eval_' + str(comm.get_rank())])
ut.create_folder(scratch_folder)
evaluators = {
'simple_classification':
lambda: SimpleClassifierEvaluator(
train_dataset,
val_dataset,
num_classes,
'./temp' + str(comm.get_rank()),
max_num_training_epochs=config['eval_epochs'],
log_output_to_terminal=options.display_output,
test_dataset=test_dataset),
}
assert not config['evaluator'].startswith('enas') or hasattr(
search_space_factory, 'weight_sharer')
evaluator = evaluators[config['evaluator']]()
start_worker(comm,
evaluator,
search_space_factory,
config['search_folder'],
config['search_name'],
resume=options.resume,
save_every=save_every)