def run_meta_neuralnet(search_space, dicts, k=10, verbose=1, num_ensemble=5, epochs=10000, lr=0.00001, loss='scaled', explore_type='its', explore_factor=0.5): # data: list of arch dictionary objects # trains a meta neural network # returns list of k arch dictionary objects - the k best predicted results = [] meta_neuralnet = MetaNeuralnet() data = search_space.encode_data(dicts) xtrain = np.array([d[1] for d in data]) ytrain = np.array([d[2] for d in data]) candidates = search_space.get_candidates(data, acq_opt_type='mutation_random', encode_paths=True, allow_isomorphisms=True, deterministic_loss=None) xcandidates = np.array([c[1] for c in candidates]) candidates_specs = [c[0] for c in candidates] predictions = [] # train an ensemble of neural networks train_error = 0 for _ in range(num_ensemble): meta_neuralnet = MetaNeuralnet() train_error += meta_neuralnet.fit(xtrain, ytrain, loss=loss, epochs=epochs, lr=lr) predictions.append(np.squeeze(meta_neuralnet.predict(xcandidates))) train_error /= num_ensemble if verbose: print('Meta neural net train error: {}'.format(train_error)) sorted_indices = acq_fn(predictions, explore_type) top_k_candidates = [candidates_specs[i] for i in sorted_indices[:k]] candidates_dict = [] for candidate in top_k_candidates: d = {} d['spec'] = candidate candidates_dict.append(d) return candidates_dict
def bananas(search_space, metann_params, num_init=10, k=10, total_queries=150, num_ensemble=5, acq_opt_type='mutation', explore_type='its', encode_paths=True, allow_isomorphisms=False, deterministic=True, verbose=1): """ Bayesian optimization with a neural network model """ data = search_space.generate_random_dataset(num=num_init, encode_paths=encode_paths, allow_isomorphisms=allow_isomorphisms, deterministic_loss=deterministic) query = num_init + k while query <= total_queries: xtrain = np.array([d[1] for d in data]) ytrain = np.array([d[2] for d in data]) candidates = search_space.get_candidates(data, acq_opt_type=acq_opt_type, encode_paths=encode_paths, allow_isomorphisms=allow_isomorphisms, deterministic_loss=deterministic) xcandidates = np.array([c[1] for c in candidates]) predictions = [] # train an ensemble of neural networks train_error = 0 for _ in range(num_ensemble): meta_neuralnet = MetaNeuralnet() train_error += meta_neuralnet.fit(xtrain, ytrain, **metann_params) # predict the validation loss of the candidate architectures predictions.append(np.squeeze(meta_neuralnet.predict(xcandidates))) # clear the tensorflow graph tf.reset_default_graph() train_error /= num_ensemble if verbose: print('Query {}, Meta neural net train error: {}'.format(query, train_error)) # compute the acquisition function for all the candidate architectures sorted_indices = acq_fn(predictions, explore_type) # add the k arches with the minimum acquisition function values for i in sorted_indices[:k]: archtuple = search_space.query_arch(candidates[i][0], encode_paths=encode_paths, deterministic=deterministic) data.append(archtuple) if verbose: top_5_loss = sorted([d[2] for d in data])[:min(5, len(data))] print('Query {}, top 5 val losses {}'.format(query, top_5_loss)) query += k return data
def dngo_search(search_space, num_init=10, k=10, loss='val_loss', total_queries=150, encoding_type='path', cutoff=40, acq_opt_type='mutation', explore_type='ucb', deterministic=True, verbose=True): import torch from pybnn import DNGO from pybnn.util.normalization import zero_mean_unit_var_normalization, zero_mean_unit_var_denormalization from acquisition_functions import acq_fn def fn(arch): return search_space.query_arch(arch, deterministic=deterministic)[loss] # set up initial data data = search_space.generate_random_dataset(num=num_init, encoding_type=encoding_type, cutoff=cutoff, deterministic_loss=deterministic) query = num_init + k while query <= total_queries: # set up data x = np.array([d['encoding'] for d in data]) y = np.array([d[loss] for d in data]) # get a set of candidate architectures candidates = search_space.get_candidates(data, acq_opt_type=acq_opt_type, encoding_type=encoding_type, cutoff=cutoff, deterministic_loss=deterministic) xcandidates = np.array([d['encoding'] for d in candidates]) # train the model model = DNGO(do_mcmc=False) model.train(x, y, do_optimize=True) predictions = model.predict(xcandidates) candidate_indices = acq_fn(np.array(predictions), explore_type) # add the k arches with the minimum acquisition function values for i in candidate_indices[:k]: arch_dict = search_space.query_arch(candidates[i]['spec'], encoding_type=encoding_type, cutoff=cutoff, deterministic=deterministic) data.append(arch_dict) if verbose: top_5_loss = sorted([(d[loss], d['epochs']) for d in data], key=lambda d: d[0])[:min(5, len(data))] print('dngo, query {}, top 5 val losses (val, test, epoch): {}'.format(query, top_5_loss)) recent_10_loss = [(d[loss], d['epochs']) for d in data[-10:]] print('dngo, query {}, most recent 10 (val, test, epoch): {}'.format(query, recent_10_loss)) query += k return data
def bananas(search_space, metann_params, num_init=10, k=10, loss='val_loss', total_queries=150, num_ensemble=5, acq_opt_type='mutation', num_arches_to_mutate=1, explore_type='its', encoding_type='trunc_path', cutoff=40, deterministic=True, verbose=1): """ Bayesian optimization with a neural network model """ from acquisition_functions import acq_fn from meta_neural_net import MetaNeuralnet data = search_space.generate_random_dataset(num=num_init, encoding_type=encoding_type, cutoff=cutoff, deterministic_loss=deterministic) query = num_init + k while query <= total_queries: xtrain = np.array([d['encoding'] for d in data]) ytrain = np.array([d[loss] for d in data]) if (query == num_init + k) and verbose: print('bananas xtrain shape', xtrain.shape) print('bananas ytrain shape', ytrain.shape) # get a set of candidate architectures candidates = search_space.get_candidates(data, acq_opt_type=acq_opt_type, encoding_type=encoding_type, cutoff=cutoff, num_arches_to_mutate=num_arches_to_mutate, loss=loss, deterministic_loss=deterministic) xcandidates = np.array([c['encoding'] for c in candidates]) candidate_predictions = [] # train an ensemble of neural networks train_error = 0 for _ in range(num_ensemble): meta_neuralnet = MetaNeuralnet() train_error += meta_neuralnet.fit(xtrain, ytrain, **metann_params) # predict the validation loss of the candidate architectures candidate_predictions.append(np.squeeze(meta_neuralnet.predict(xcandidates))) # clear the tensorflow graph tf.reset_default_graph() tf.keras.backend.clear_session() train_error /= num_ensemble if verbose: print('query {}, Meta neural net train error: {}'.format(query, train_error)) # compute the acquisition function for all the candidate architectures candidate_indices = acq_fn(candidate_predictions, explore_type) # add the k arches with the minimum acquisition function values for i in candidate_indices[:k]: arch_dict = search_space.query_arch(candidates[i]['spec'], encoding_type=encoding_type, cutoff=cutoff, deterministic=deterministic) data.append(arch_dict) if verbose: top_5_loss = sorted([(d[loss], d['epochs']) for d in data], key=lambda d: d[0])[:min(5, len(data))] print('bananas, query {}, top 5 losses (loss, test, epoch): {}'.format(query, top_5_loss)) recent_10_loss = [(d[loss], d['epochs']) for d in data[-10:]] print('bananas, query {}, most recent 10 (loss, test, epoch): {}'.format(query, recent_10_loss)) query += k return data
def bananas(search_space, metann_params, num_init=DEFAULT_NUM_INIT, k=DEFAULT_K, loss=DEFAULT_LOSS, total_queries=DEFAULT_TOTAL_QUERIES, num_ensemble=5, acq_opt_type='mutation', num_arches_to_mutate=1, explore_type='its', predictor_encoding='trunc_path', cutoff=0, mutate_encoding='adj', random_encoding='adj', deterministic=True, verbose=1): """ Bayesian optimization with a neural network model """ data = search_space.generate_random_dataset( num=num_init, predictor_encoding=predictor_encoding, random_encoding=random_encoding, deterministic_loss=deterministic, cutoff=cutoff) query = num_init + k while query <= total_queries: xtrain = np.array([d['encoding'] for d in data]) ytrain = np.array([d[loss] for d in data]) if verbose and query == num_init + k: print('xtrain shape', xtrain.shape) print('ytrain shape', ytrain.shape) # get a set of candidate architectures candidates = search_space.get_candidates( data, acq_opt_type=acq_opt_type, predictor_encoding=predictor_encoding, mutate_encoding=mutate_encoding, num_arches_to_mutate=num_arches_to_mutate, loss=loss, deterministic_loss=deterministic, cutoff=cutoff) xcandidates = np.array([c['encoding'] for c in candidates]) candidate_predictions = [] # train an ensemble of neural networks train_error = 0 for _ in range(num_ensemble): meta_neuralnet = MetaNeuralnet() train_error += meta_neuralnet.fit(xtrain, ytrain, **metann_params) # predict the validation loss of the candidate architectures candidate_predictions.append( np.squeeze(meta_neuralnet.predict(xcandidates))) tf.reset_default_graph() tf.keras.backend.clear_session() train_error /= num_ensemble if verbose: print('query {}, Meta neural net train error: {}'.format( query, train_error)) # compute the acquisition function for all the candidate architectures candidate_indices = acq_fn(candidate_predictions, explore_type) # add the k arches with the minimum acquisition function values for i in candidate_indices[:k]: arch_dict = search_space.query_arch( candidates[i]['spec'], predictor_encoding=predictor_encoding, deterministic=deterministic, cutoff=cutoff) data.append(arch_dict) if verbose: top_5_loss = sorted([(d[loss], d['epochs']) for d in data], key=lambda d: d[0])[:min(5, len(data))] print('bananas, query {}, top 5 losses (loss, test, epoch): {}'. format(query, top_5_loss)) # we just finished performing k queries query += k return data