def run_np(n, lr, train_data, test_data):
arch_data = [d[0] for d in train_data]
arch_graph = [[d[0]['matrix'], d[0]['ops']] for d in train_data]
arch_data_edge_idx_list = []
arch_data_node_f_list = []
for arch in arch_graph:
edge_index, node_f = nasbench2graph2(arch)
arch_data_edge_idx_list.append(edge_index)
arch_data_node_f_list.append(node_f)
val_accuracy = np.array([d[2] for d in train_data])
candiate_edge_list = []
candiate_node_list = []
candidate_arch_graph = [[d[0]['matrix'], d[0]['ops']] for d in test_data]
for cand in candidate_arch_graph:
edge_index, node_f = nasbench2graph2(cand)
candiate_edge_list.append(edge_index)
candiate_node_list.append(node_f)
val_test_accuracy = np.array([d[2] for d in test_data])
train_mean_errors = []
val_test_errors = []
agent = NasBenchGinPredictorTrainer('nnp',
lr=lr,
device=device,
epochs=300,
train_images=len(arch_data),
batch_size=16,
rate=20)
agent.fit(arch_data_edge_idx_list,
arch_data_node_f_list,
val_accuracy,
logger=None)
mean_train = agent.pred(arch_data_edge_idx_list, arch_data_node_f_list)
mean_test_pred = agent.pred(candiate_edge_list, candiate_node_list)
mean_train = mean_train.cpu().numpy()
mean_test_pred = mean_test_pred.cpu().numpy()
train_error_mean = np.mean(np.abs(mean_train - val_accuracy))
train_mean_errors.append(train_error_mean)
val_mean_error = np.mean(np.abs(val_test_accuracy - mean_test_pred))
val_test_errors.append(val_mean_error)
train_mean_error = np.round(np.mean(train_mean_errors, axis=0), 3)
val_mean_error = np.round(np.mean(val_test_errors, axis=0), 3)
print('Neural predictor training size: {}'.format(n))
print('Train mean error: {}, Test mean error: {}'.format(
train_mean_error, val_mean_error))
title = 'Neural predictor, training set size {}'.format(n)
file_all = 'neural_predictor_samples_%d_epoch_%d.png' % (n, 300)
# plot_meta_neuralnet(val_accuracy, mean_train, val_test_accuracy, mean_test_pred, title=title, file_name=file_all)
return train_mean_error, val_mean_error
def gin_predictor_search_open(search_space,
algo_info,
logger,
gpus,
save_dir,
verbose=True,
dataset='cifar10',
seed=111222333):
"""
regularized evolution
"""
total_queries = algo_info['total_queries']
num_init = algo_info['num_init']
k_num = algo_info['k']
epochs = algo_info['epochs']
batch_size = algo_info['batch_size']
lr = algo_info['lr']
encode_path = algo_info['encode_path']
candidate_nums = algo_info['candidate_nums']
macro_graph_dict = {}
model_keys = []
init_nasbench_macro_cifar10(save_dir)
# set seed
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
cudnn.benchmark = False
cudnn.enabled = True
cudnn.deterministic = True
torch.cuda.manual_seed_all(seed)
data_dict = search_space.generate_random_dataset(num=num_init,
encode_paths=encode_path)
data_dict_keys = [
convert_to_genotype(d[0], verbose=False) for d in data_dict
]
data_dict_keys = [
sha256(str(k).encode('utf-8')).hexdigest() for k in data_dict_keys
]
model_keys.extend(data_dict_keys)
for i, d in enumerate(data_dict):
macro_graph_dict[data_dict_keys[i]] = list(d)
darts_neural_dict = search_space.assemble_cifar10_neural_net(data_dict)
data = async_macro_model_train(model_data=darts_neural_dict,
gpus=gpus,
save_dir=save_dir,
dataset=dataset)
for k, v in data.items():
if k not in macro_graph_dict:
raise ValueError(
'model trained acc key should in macro_graph_dict')
macro_graph_dict[k].extend(v)
query = num_init + len(data_dict_keys)
while query <= total_queries:
train_data = search_space.assemble_graph(macro_graph_dict, model_keys)
val_losses = np.array([macro_graph_dict[k][2] for k in model_keys])
arch_data_edge_idx_list = []
arch_data_node_f_list = []
for arch in train_data:
edge_index, node_f = nasbench2graph2(arch)
arch_data_edge_idx_list.append(edge_index)
arch_data_node_f_list.append(node_f)
candidate_graph_dict = {}
candidates = search_space.get_candidates(
macro_graph_dict,
model_keys,
num=candidate_nums,
encode_paths=encode_path,
)
candidate_dict_keys = [
convert_to_genotype(d[0], verbose=False) for d in candidates
]
candidate_dict_keys = [
sha256(str(k).encode('utf-8')).hexdigest()
for k in candidate_dict_keys
]
for i, d in enumerate(candidates):
candidate_graph_dict[candidate_dict_keys[i]] = list(d)
xcandidates = search_space.assemble_graph(candidate_graph_dict,
candidate_dict_keys)
candiate_edge_list = []
candiate_node_list = []
for cand in xcandidates:
edge_index, node_f = nasbench2graph2(cand)
candiate_edge_list.append(edge_index)
candiate_node_list.append(node_f)
meta_neuralnet = NasBenchGinPredictorTrainer(
lr=lr,
epochs=epochs,
train_images=len(arch_data_edge_idx_list),
batch_size=batch_size,
input_dim=11,
agent_type='gin_gaussian',
rate=15.)
meta_neuralnet.fit(arch_data_edge_idx_list,
arch_data_node_f_list,
val_losses,
logger=logger)
pred_train = meta_neuralnet.pred(arch_data_edge_idx_list,
arch_data_node_f_list)
acc_pred = meta_neuralnet.pred(candiate_edge_list, candiate_node_list)
candidate_np = acc_pred.cpu().numpy()
sorted_indices = np.argsort(candidate_np)
temp_candidate_train_arch = []
for j in sorted_indices[:k_num]:
model_keys.append(candidate_dict_keys[j])
macro_graph_dict[candidate_dict_keys[j]] = candidate_graph_dict[
candidate_dict_keys[j]]
temp_candidate_train_arch.append(
candidate_graph_dict[candidate_dict_keys[j]])
darts_candidate_neural_dict = search_space.assemble_cifar10_neural_net(
temp_candidate_train_arch)
darts_candidate_acc = async_macro_model_train(
model_data=darts_candidate_neural_dict,
gpus=gpus,
save_dir=save_dir,
dataset=dataset)
for k, v in darts_candidate_acc.items():
if k not in macro_graph_dict:
raise ValueError(
'model trained acc key should in macro_graph_dict')
macro_graph_dict[k].extend(v)
train_error = np.mean(np.abs(pred_train.cpu().numpy() - val_losses))
val_losses = np.array([macro_graph_dict[k][2] for k in model_keys])
if verbose:
top_5_loss = sorted(val_losses)[:min(5, len(val_losses))]
logger.info('Query {}, training mean loss is {}'.format(
query, train_error))
logger.info('Query {}, top 5 val losses {}'.format(
query, top_5_loss))
query += len(temp_candidate_train_arch)
return macro_graph_dict, model_keys
def gin_predictor_nasbench_201(search_space,
num_init=10,
k=10,
total_queries=150,
acq_opt_type='mutation',
allow_isomorphisms=False,
verbose=1,
agent=None,
logger=None,
gpu='0',
lr=0.01,
candidate_nums=100,
epochs=1000,
rate=10,
algo_name=None):
"""
Bayesian optimization with a neural network model
"""
device = torch.device('cuda:%d' % gpu)
data = search_space.generate_random_dataset(
num=num_init,
allow_isomorphisms=allow_isomorphisms,
deterministic_loss=True)
query = num_init + k
search_agent = agent
if len(data) <= 10:
batch_size = 10
else:
batch_size = 16
while query <= total_queries:
arch_data = [d[0] for d in data]
agent = NasBenchGinPredictorTrainer(search_agent,
lr=lr,
device=device,
epochs=epochs,
train_images=len(data),
batch_size=batch_size,
input_dim=8,
rate=rate,
algo_name=algo_name)
val_accuracy = np.array([d[4] for d in data])
arch_data_edge_idx_list = []
arch_data_node_f_list = []
for arch in arch_data:
edge_index, node_f = nasbench2graph_201(arch)
arch_data_edge_idx_list.append(edge_index)
arch_data_node_f_list.append(node_f)
candidates = search_space.get_candidates(
data, num=candidate_nums, allow_isomorphisms=allow_isomorphisms)
candiate_edge_list = []
candiate_node_list = []
for cand in candidates:
edge_index, node_f = nasbench2graph_201(cand[0])
candiate_edge_list.append(edge_index)
candiate_node_list.append(node_f)
agent.fit(arch_data_edge_idx_list,
arch_data_node_f_list,
val_accuracy,
logger=None)
acc_train = agent.pred(arch_data_edge_idx_list, arch_data_node_f_list)
acc_pred = agent.pred(candiate_edge_list, candiate_node_list)
candidate_np = acc_pred.cpu().numpy()
sorted_indices = np.argsort(candidate_np)
for i in sorted_indices[:k]:
archtuple = candidates[i]
data.append(archtuple)
if verbose:
top_5_loss = sorted([d[4] for d in data])[:min(5, len(data))]
logger.info('Query {}, training mean loss is {}'.format(
query,
np.mean(np.abs(acc_train.cpu().numpy() - val_accuracy))))
logger.info('Query {}, top 5 val losses {}'.format(
query, top_5_loss))
query += k
return data
def gin_unsupervised_predictor(search_space,
model_dir=None,
num_init=10,
k=10,
total_queries=150,
acq_opt_type='mutation',
allow_isomorphisms=False,
verbose=1,
agent=None,
logger=None,
gpu='0',
lr=0.01,
candidate_nums=100,
epochs=1000,
predictor_type=None,
algo_name=None,
benchmark=None,
rate=10):
assert benchmark is not None, 'The benchmark have to be nasbench_101 or nasbench_201.'
device = torch.device('cuda:%d' % gpu)
data = search_space.generate_random_dataset(
num=num_init,
allow_isomorphisms=allow_isomorphisms,
deterministic_loss=True)
query = num_init + k
search_agent = agent
input_dim = 6 if benchmark == 'nasbench_101' else 8
if len(data) <= 10:
batch_size = 10
else:
batch_size = 16
while query <= total_queries:
arch_data = [d[0] for d in data]
agent = NasBenchGinPredictorTrainer(search_agent,
lr=lr,
device=device,
epochs=epochs,
train_images=len(data),
batch_size=batch_size,
input_dim=input_dim,
model_dir=model_dir,
predictor_type=predictor_type,
logger=logger,
algo_name=algo_name,
rate=rate)
val_accuracy = np.array([d[4] for d in data])
arch_data_edge_idx_list = []
arch_data_node_f_list = []
for arch in arch_data:
edge_index, node_f = nasbench2graph_101(
arch) if benchmark == 'nasbench_101' else nasbench2graph_201(
arch)
arch_data_edge_idx_list.append(edge_index)
arch_data_node_f_list.append(node_f)
candidates = search_space.get_candidates(
data, num=candidate_nums, allow_isomorphisms=allow_isomorphisms)
candiate_edge_list = []
candiate_node_list = []
for cand in candidates:
edge_index, node_f = nasbench2graph_101(
cand[0]
) if benchmark == 'nasbench_101' else nasbench2graph_201(cand[0])
candiate_edge_list.append(edge_index)
candiate_node_list.append(node_f)
agent.fit(arch_data_edge_idx_list,
arch_data_node_f_list,
val_accuracy,
logger=None)
acc_train = agent.pred(arch_data_edge_idx_list, arch_data_node_f_list)
acc_pred = agent.pred(candiate_edge_list, candiate_node_list)
candidate_np = acc_pred.cpu().numpy()
sorted_indices = np.argsort(candidate_np)
for i in sorted_indices[:k]:
if benchmark == 'nasbench_101':
archtuple = search_space.query_arch(matrix=candidates[i][1],
ops=candidates[i][2])
elif benchmark == 'nasbench_201':
archtuple = candidates[i]
else:
raise NotImplementedError()
data.append(archtuple)
if verbose:
top_5_loss = sorted([d[4] for d in data])[:min(5, len(data))]
logger.info('Query {}, training mean loss is {}'.format(
query,
np.mean(np.abs(acc_train.cpu().numpy() - val_accuracy))))
logger.info('Query {}, top 5 val losses {}'.format(
query, top_5_loss))
query += k
# data = [list(dd) for dd in data]
# for i, d in enumerate(data):
# data[i][4] = (1 - d[4]) * 100
# data[i][5] = (1 - d[5]) * 100
return data