def run_np(n, lr, train_data, test_data): arch_data = [d[0] for d in train_data] arch_graph = [[d[0]['matrix'], d[0]['ops']] for d in train_data] arch_data_edge_idx_list = [] arch_data_node_f_list = [] for arch in arch_graph: edge_index, node_f = nasbench2graph2(arch) arch_data_edge_idx_list.append(edge_index) arch_data_node_f_list.append(node_f) val_accuracy = np.array([d[2] for d in train_data]) candiate_edge_list = [] candiate_node_list = [] candidate_arch_graph = [[d[0]['matrix'], d[0]['ops']] for d in test_data] for cand in candidate_arch_graph: edge_index, node_f = nasbench2graph2(cand) candiate_edge_list.append(edge_index) candiate_node_list.append(node_f) val_test_accuracy = np.array([d[2] for d in test_data]) train_mean_errors = [] val_test_errors = [] agent = NasBenchGinPredictorTrainer('nnp', lr=lr, device=device, epochs=300, train_images=len(arch_data), batch_size=16, rate=20) agent.fit(arch_data_edge_idx_list, arch_data_node_f_list, val_accuracy, logger=None) mean_train = agent.pred(arch_data_edge_idx_list, arch_data_node_f_list) mean_test_pred = agent.pred(candiate_edge_list, candiate_node_list) mean_train = mean_train.cpu().numpy() mean_test_pred = mean_test_pred.cpu().numpy() train_error_mean = np.mean(np.abs(mean_train - val_accuracy)) train_mean_errors.append(train_error_mean) val_mean_error = np.mean(np.abs(val_test_accuracy - mean_test_pred)) val_test_errors.append(val_mean_error) train_mean_error = np.round(np.mean(train_mean_errors, axis=0), 3) val_mean_error = np.round(np.mean(val_test_errors, axis=0), 3) print('Neural predictor training size: {}'.format(n)) print('Train mean error: {}, Test mean error: {}'.format( train_mean_error, val_mean_error)) title = 'Neural predictor, training set size {}'.format(n) file_all = 'neural_predictor_samples_%d_epoch_%d.png' % (n, 300) # plot_meta_neuralnet(val_accuracy, mean_train, val_test_accuracy, mean_test_pred, title=title, file_name=file_all) return train_mean_error, val_mean_error
def gin_predictor_search_open(search_space, algo_info, logger, gpus, save_dir, verbose=True, dataset='cifar10', seed=111222333): """ regularized evolution """ total_queries = algo_info['total_queries'] num_init = algo_info['num_init'] k_num = algo_info['k'] epochs = algo_info['epochs'] batch_size = algo_info['batch_size'] lr = algo_info['lr'] encode_path = algo_info['encode_path'] candidate_nums = algo_info['candidate_nums'] macro_graph_dict = {} model_keys = [] init_nasbench_macro_cifar10(save_dir) # set seed np.random.seed(seed) random.seed(seed) torch.manual_seed(seed) cudnn.benchmark = False cudnn.enabled = True cudnn.deterministic = True torch.cuda.manual_seed_all(seed) data_dict = search_space.generate_random_dataset(num=num_init, encode_paths=encode_path) data_dict_keys = [ convert_to_genotype(d[0], verbose=False) for d in data_dict ] data_dict_keys = [ sha256(str(k).encode('utf-8')).hexdigest() for k in data_dict_keys ] model_keys.extend(data_dict_keys) for i, d in enumerate(data_dict): macro_graph_dict[data_dict_keys[i]] = list(d) darts_neural_dict = search_space.assemble_cifar10_neural_net(data_dict) data = async_macro_model_train(model_data=darts_neural_dict, gpus=gpus, save_dir=save_dir, dataset=dataset) for k, v in data.items(): if k not in macro_graph_dict: raise ValueError( 'model trained acc key should in macro_graph_dict') macro_graph_dict[k].extend(v) query = num_init + len(data_dict_keys) while query <= total_queries: train_data = search_space.assemble_graph(macro_graph_dict, model_keys) val_losses = np.array([macro_graph_dict[k][2] for k in model_keys]) arch_data_edge_idx_list = [] arch_data_node_f_list = [] for arch in train_data: edge_index, node_f = nasbench2graph2(arch) arch_data_edge_idx_list.append(edge_index) arch_data_node_f_list.append(node_f) candidate_graph_dict = {} candidates = search_space.get_candidates( macro_graph_dict, model_keys, num=candidate_nums, encode_paths=encode_path, ) candidate_dict_keys = [ convert_to_genotype(d[0], verbose=False) for d in candidates ] candidate_dict_keys = [ sha256(str(k).encode('utf-8')).hexdigest() for k in candidate_dict_keys ] for i, d in enumerate(candidates): candidate_graph_dict[candidate_dict_keys[i]] = list(d) xcandidates = search_space.assemble_graph(candidate_graph_dict, candidate_dict_keys) candiate_edge_list = [] candiate_node_list = [] for cand in xcandidates: edge_index, node_f = nasbench2graph2(cand) candiate_edge_list.append(edge_index) candiate_node_list.append(node_f) meta_neuralnet = NasBenchGinPredictorTrainer( lr=lr, epochs=epochs, train_images=len(arch_data_edge_idx_list), batch_size=batch_size, input_dim=11, agent_type='gin_gaussian', rate=15.) meta_neuralnet.fit(arch_data_edge_idx_list, arch_data_node_f_list, val_losses, logger=logger) pred_train = meta_neuralnet.pred(arch_data_edge_idx_list, arch_data_node_f_list) acc_pred = meta_neuralnet.pred(candiate_edge_list, candiate_node_list) candidate_np = acc_pred.cpu().numpy() sorted_indices = np.argsort(candidate_np) temp_candidate_train_arch = [] for j in sorted_indices[:k_num]: model_keys.append(candidate_dict_keys[j]) macro_graph_dict[candidate_dict_keys[j]] = candidate_graph_dict[ candidate_dict_keys[j]] temp_candidate_train_arch.append( candidate_graph_dict[candidate_dict_keys[j]]) darts_candidate_neural_dict = search_space.assemble_cifar10_neural_net( temp_candidate_train_arch) darts_candidate_acc = async_macro_model_train( model_data=darts_candidate_neural_dict, gpus=gpus, save_dir=save_dir, dataset=dataset) for k, v in darts_candidate_acc.items(): if k not in macro_graph_dict: raise ValueError( 'model trained acc key should in macro_graph_dict') macro_graph_dict[k].extend(v) train_error = np.mean(np.abs(pred_train.cpu().numpy() - val_losses)) val_losses = np.array([macro_graph_dict[k][2] for k in model_keys]) if verbose: top_5_loss = sorted(val_losses)[:min(5, len(val_losses))] logger.info('Query {}, training mean loss is {}'.format( query, train_error)) logger.info('Query {}, top 5 val losses {}'.format( query, top_5_loss)) query += len(temp_candidate_train_arch) return macro_graph_dict, model_keys
def gin_predictor_nasbench_201(search_space, num_init=10, k=10, total_queries=150, acq_opt_type='mutation', allow_isomorphisms=False, verbose=1, agent=None, logger=None, gpu='0', lr=0.01, candidate_nums=100, epochs=1000, rate=10, algo_name=None): """ Bayesian optimization with a neural network model """ device = torch.device('cuda:%d' % gpu) data = search_space.generate_random_dataset( num=num_init, allow_isomorphisms=allow_isomorphisms, deterministic_loss=True) query = num_init + k search_agent = agent if len(data) <= 10: batch_size = 10 else: batch_size = 16 while query <= total_queries: arch_data = [d[0] for d in data] agent = NasBenchGinPredictorTrainer(search_agent, lr=lr, device=device, epochs=epochs, train_images=len(data), batch_size=batch_size, input_dim=8, rate=rate, algo_name=algo_name) val_accuracy = np.array([d[4] for d in data]) arch_data_edge_idx_list = [] arch_data_node_f_list = [] for arch in arch_data: edge_index, node_f = nasbench2graph_201(arch) arch_data_edge_idx_list.append(edge_index) arch_data_node_f_list.append(node_f) candidates = search_space.get_candidates( data, num=candidate_nums, allow_isomorphisms=allow_isomorphisms) candiate_edge_list = [] candiate_node_list = [] for cand in candidates: edge_index, node_f = nasbench2graph_201(cand[0]) candiate_edge_list.append(edge_index) candiate_node_list.append(node_f) agent.fit(arch_data_edge_idx_list, arch_data_node_f_list, val_accuracy, logger=None) acc_train = agent.pred(arch_data_edge_idx_list, arch_data_node_f_list) acc_pred = agent.pred(candiate_edge_list, candiate_node_list) candidate_np = acc_pred.cpu().numpy() sorted_indices = np.argsort(candidate_np) for i in sorted_indices[:k]: archtuple = candidates[i] data.append(archtuple) if verbose: top_5_loss = sorted([d[4] for d in data])[:min(5, len(data))] logger.info('Query {}, training mean loss is {}'.format( query, np.mean(np.abs(acc_train.cpu().numpy() - val_accuracy)))) logger.info('Query {}, top 5 val losses {}'.format( query, top_5_loss)) query += k return data
def gin_unsupervised_predictor(search_space, model_dir=None, num_init=10, k=10, total_queries=150, acq_opt_type='mutation', allow_isomorphisms=False, verbose=1, agent=None, logger=None, gpu='0', lr=0.01, candidate_nums=100, epochs=1000, predictor_type=None, algo_name=None, benchmark=None, rate=10): assert benchmark is not None, 'The benchmark have to be nasbench_101 or nasbench_201.' device = torch.device('cuda:%d' % gpu) data = search_space.generate_random_dataset( num=num_init, allow_isomorphisms=allow_isomorphisms, deterministic_loss=True) query = num_init + k search_agent = agent input_dim = 6 if benchmark == 'nasbench_101' else 8 if len(data) <= 10: batch_size = 10 else: batch_size = 16 while query <= total_queries: arch_data = [d[0] for d in data] agent = NasBenchGinPredictorTrainer(search_agent, lr=lr, device=device, epochs=epochs, train_images=len(data), batch_size=batch_size, input_dim=input_dim, model_dir=model_dir, predictor_type=predictor_type, logger=logger, algo_name=algo_name, rate=rate) val_accuracy = np.array([d[4] for d in data]) arch_data_edge_idx_list = [] arch_data_node_f_list = [] for arch in arch_data: edge_index, node_f = nasbench2graph_101( arch) if benchmark == 'nasbench_101' else nasbench2graph_201( arch) arch_data_edge_idx_list.append(edge_index) arch_data_node_f_list.append(node_f) candidates = search_space.get_candidates( data, num=candidate_nums, allow_isomorphisms=allow_isomorphisms) candiate_edge_list = [] candiate_node_list = [] for cand in candidates: edge_index, node_f = nasbench2graph_101( cand[0] ) if benchmark == 'nasbench_101' else nasbench2graph_201(cand[0]) candiate_edge_list.append(edge_index) candiate_node_list.append(node_f) agent.fit(arch_data_edge_idx_list, arch_data_node_f_list, val_accuracy, logger=None) acc_train = agent.pred(arch_data_edge_idx_list, arch_data_node_f_list) acc_pred = agent.pred(candiate_edge_list, candiate_node_list) candidate_np = acc_pred.cpu().numpy() sorted_indices = np.argsort(candidate_np) for i in sorted_indices[:k]: if benchmark == 'nasbench_101': archtuple = search_space.query_arch(matrix=candidates[i][1], ops=candidates[i][2]) elif benchmark == 'nasbench_201': archtuple = candidates[i] else: raise NotImplementedError() data.append(archtuple) if verbose: top_5_loss = sorted([d[4] for d in data])[:min(5, len(data))] logger.info('Query {}, training mean loss is {}'.format( query, np.mean(np.abs(acc_train.cpu().numpy() - val_accuracy)))) logger.info('Query {}, top 5 val losses {}'.format( query, top_5_loss)) query += k # data = [list(dd) for dd in data] # for i, d in enumerate(data): # data[i][4] = (1 - d[4]) * 100 # data[i][5] = (1 - d[5]) * 100 return data