def run_meta_neuralnet(search_space, dicts,
k=10,
verbose=1,
num_ensemble=5,
epochs=10000,
lr=0.00001,
loss='scaled',
explore_type='its',
explore_factor=0.5):
# data: list of arch dictionary objects
# trains a meta neural network
# returns list of k arch dictionary objects - the k best predicted
results = []
meta_neuralnet = MetaNeuralnet()
data = search_space.encode_data(dicts)
xtrain = np.array([d[1] for d in data])
ytrain = np.array([d[2] for d in data])
candidates = search_space.get_candidates(data,
acq_opt_type='mutation_random',
encode_paths=True,
allow_isomorphisms=True,
deterministic_loss=None)
xcandidates = np.array([c[1] for c in candidates])
candidates_specs = [c[0] for c in candidates]
predictions = []
# train an ensemble of neural networks
train_error = 0
for _ in range(num_ensemble):
meta_neuralnet = MetaNeuralnet()
train_error += meta_neuralnet.fit(xtrain, ytrain,
loss=loss,
epochs=epochs,
lr=lr)
predictions.append(np.squeeze(meta_neuralnet.predict(xcandidates)))
train_error /= num_ensemble
if verbose:
print('Meta neural net train error: {}'.format(train_error))
sorted_indices = acq_fn(predictions, explore_type)
top_k_candidates = [candidates_specs[i] for i in sorted_indices[:k]]
candidates_dict = []
for candidate in top_k_candidates:
d = {}
d['spec'] = candidate
candidates_dict.append(d)
return candidates_dict
def bananas(search_space, metann_params,
num_init=10,
k=10,
total_queries=150,
num_ensemble=5,
acq_opt_type='mutation',
explore_type='its',
encode_paths=True,
allow_isomorphisms=False,
deterministic=True,
verbose=1):
"""
Bayesian optimization with a neural network model
"""
data = search_space.generate_random_dataset(num=num_init,
encode_paths=encode_paths,
allow_isomorphisms=allow_isomorphisms,
deterministic_loss=deterministic)
query = num_init + k
while query <= total_queries:
xtrain = np.array([d[1] for d in data])
ytrain = np.array([d[2] for d in data])
candidates = search_space.get_candidates(data,
acq_opt_type=acq_opt_type,
encode_paths=encode_paths,
allow_isomorphisms=allow_isomorphisms,
deterministic_loss=deterministic)
xcandidates = np.array([c[1] for c in candidates])
predictions = []
# train an ensemble of neural networks
train_error = 0
for _ in range(num_ensemble):
meta_neuralnet = MetaNeuralnet()
train_error += meta_neuralnet.fit(xtrain, ytrain, **metann_params)
# predict the validation loss of the candidate architectures
predictions.append(np.squeeze(meta_neuralnet.predict(xcandidates)))
# clear the tensorflow graph
tf.reset_default_graph()
train_error /= num_ensemble
if verbose:
print('Query {}, Meta neural net train error: {}'.format(query, train_error))
# compute the acquisition function for all the candidate architectures
sorted_indices = acq_fn(predictions, explore_type)
# add the k arches with the minimum acquisition function values
for i in sorted_indices[:k]:
archtuple = search_space.query_arch(candidates[i][0],
encode_paths=encode_paths,
deterministic=deterministic)
data.append(archtuple)
if verbose:
top_5_loss = sorted([d[2] for d in data])[:min(5, len(data))]
print('Query {}, top 5 val losses {}'.format(query, top_5_loss))
query += k
return data
def dngo_search(search_space,
num_init=10,
k=10,
loss='val_loss',
total_queries=150,
encoding_type='path',
cutoff=40,
acq_opt_type='mutation',
explore_type='ucb',
deterministic=True,
verbose=True):
import torch
from pybnn import DNGO
from pybnn.util.normalization import zero_mean_unit_var_normalization, zero_mean_unit_var_denormalization
from acquisition_functions import acq_fn
def fn(arch):
return search_space.query_arch(arch, deterministic=deterministic)[loss]
# set up initial data
data = search_space.generate_random_dataset(num=num_init,
encoding_type=encoding_type,
cutoff=cutoff,
deterministic_loss=deterministic)
query = num_init + k
while query <= total_queries:
# set up data
x = np.array([d['encoding'] for d in data])
y = np.array([d[loss] for d in data])
# get a set of candidate architectures
candidates = search_space.get_candidates(data,
acq_opt_type=acq_opt_type,
encoding_type=encoding_type,
cutoff=cutoff,
deterministic_loss=deterministic)
xcandidates = np.array([d['encoding'] for d in candidates])
# train the model
model = DNGO(do_mcmc=False)
model.train(x, y, do_optimize=True)
predictions = model.predict(xcandidates)
candidate_indices = acq_fn(np.array(predictions), explore_type)
# add the k arches with the minimum acquisition function values
for i in candidate_indices[:k]:
arch_dict = search_space.query_arch(candidates[i]['spec'],
encoding_type=encoding_type,
cutoff=cutoff,
deterministic=deterministic)
data.append(arch_dict)
if verbose:
top_5_loss = sorted([(d[loss], d['epochs']) for d in data], key=lambda d: d[0])[:min(5, len(data))]
print('dngo, query {}, top 5 val losses (val, test, epoch): {}'.format(query, top_5_loss))
recent_10_loss = [(d[loss], d['epochs']) for d in data[-10:]]
print('dngo, query {}, most recent 10 (val, test, epoch): {}'.format(query, recent_10_loss))
query += k
return data
def bananas(search_space,
metann_params,
num_init=10,
k=10,
loss='val_loss',
total_queries=150,
num_ensemble=5,
acq_opt_type='mutation',
num_arches_to_mutate=1,
explore_type='its',
encoding_type='trunc_path',
cutoff=40,
deterministic=True,
verbose=1):
"""
Bayesian optimization with a neural network model
"""
from acquisition_functions import acq_fn
from meta_neural_net import MetaNeuralnet
data = search_space.generate_random_dataset(num=num_init,
encoding_type=encoding_type,
cutoff=cutoff,
deterministic_loss=deterministic)
query = num_init + k
while query <= total_queries:
xtrain = np.array([d['encoding'] for d in data])
ytrain = np.array([d[loss] for d in data])
if (query == num_init + k) and verbose:
print('bananas xtrain shape', xtrain.shape)
print('bananas ytrain shape', ytrain.shape)
# get a set of candidate architectures
candidates = search_space.get_candidates(data,
acq_opt_type=acq_opt_type,
encoding_type=encoding_type,
cutoff=cutoff,
num_arches_to_mutate=num_arches_to_mutate,
loss=loss,
deterministic_loss=deterministic)
xcandidates = np.array([c['encoding'] for c in candidates])
candidate_predictions = []
# train an ensemble of neural networks
train_error = 0
for _ in range(num_ensemble):
meta_neuralnet = MetaNeuralnet()
train_error += meta_neuralnet.fit(xtrain, ytrain, **metann_params)
# predict the validation loss of the candidate architectures
candidate_predictions.append(np.squeeze(meta_neuralnet.predict(xcandidates)))
# clear the tensorflow graph
tf.reset_default_graph()
tf.keras.backend.clear_session()
train_error /= num_ensemble
if verbose:
print('query {}, Meta neural net train error: {}'.format(query, train_error))
# compute the acquisition function for all the candidate architectures
candidate_indices = acq_fn(candidate_predictions, explore_type)
# add the k arches with the minimum acquisition function values
for i in candidate_indices[:k]:
arch_dict = search_space.query_arch(candidates[i]['spec'],
encoding_type=encoding_type,
cutoff=cutoff,
deterministic=deterministic)
data.append(arch_dict)
if verbose:
top_5_loss = sorted([(d[loss], d['epochs']) for d in data], key=lambda d: d[0])[:min(5, len(data))]
print('bananas, query {}, top 5 losses (loss, test, epoch): {}'.format(query, top_5_loss))
recent_10_loss = [(d[loss], d['epochs']) for d in data[-10:]]
print('bananas, query {}, most recent 10 (loss, test, epoch): {}'.format(query, recent_10_loss))
query += k
return data
def bananas(search_space,
metann_params,
num_init=DEFAULT_NUM_INIT,
k=DEFAULT_K,
loss=DEFAULT_LOSS,
total_queries=DEFAULT_TOTAL_QUERIES,
num_ensemble=5,
acq_opt_type='mutation',
num_arches_to_mutate=1,
explore_type='its',
predictor_encoding='trunc_path',
cutoff=0,
mutate_encoding='adj',
random_encoding='adj',
deterministic=True,
verbose=1):
"""
Bayesian optimization with a neural network model
"""
data = search_space.generate_random_dataset(
num=num_init,
predictor_encoding=predictor_encoding,
random_encoding=random_encoding,
deterministic_loss=deterministic,
cutoff=cutoff)
query = num_init + k
while query <= total_queries:
xtrain = np.array([d['encoding'] for d in data])
ytrain = np.array([d[loss] for d in data])
if verbose and query == num_init + k:
print('xtrain shape', xtrain.shape)
print('ytrain shape', ytrain.shape)
# get a set of candidate architectures
candidates = search_space.get_candidates(
data,
acq_opt_type=acq_opt_type,
predictor_encoding=predictor_encoding,
mutate_encoding=mutate_encoding,
num_arches_to_mutate=num_arches_to_mutate,
loss=loss,
deterministic_loss=deterministic,
cutoff=cutoff)
xcandidates = np.array([c['encoding'] for c in candidates])
candidate_predictions = []
# train an ensemble of neural networks
train_error = 0
for _ in range(num_ensemble):
meta_neuralnet = MetaNeuralnet()
train_error += meta_neuralnet.fit(xtrain, ytrain, **metann_params)
# predict the validation loss of the candidate architectures
candidate_predictions.append(
np.squeeze(meta_neuralnet.predict(xcandidates)))
tf.reset_default_graph()
tf.keras.backend.clear_session()
train_error /= num_ensemble
if verbose:
print('query {}, Meta neural net train error: {}'.format(
query, train_error))
# compute the acquisition function for all the candidate architectures
candidate_indices = acq_fn(candidate_predictions, explore_type)
# add the k arches with the minimum acquisition function values
for i in candidate_indices[:k]:
arch_dict = search_space.query_arch(
candidates[i]['spec'],
predictor_encoding=predictor_encoding,
deterministic=deterministic,
cutoff=cutoff)
data.append(arch_dict)
if verbose:
top_5_loss = sorted([(d[loss], d['epochs']) for d in data],
key=lambda d: d[0])[:min(5, len(data))]
print('bananas, query {}, top 5 losses (loss, test, epoch): {}'.
format(query, top_5_loss))
# we just finished performing k queries
query += k
return data
