def analysis(): search_space_1 = SearchSpace1() search_space_1.sample(with_loose_ends=False) # Load NASBench nasbench = NasbenchWrapper('nasbench_analysis/nasbench_data/108_e/nasbench_full.tfrecord') test_error = [] valid_error = [] search_space_1 = SearchSpace1() search_space_1.sample_with_loose_ends() for i in range(10000): adjacency_matrix, node_list = search_space_1.sample() adjacency_list = adjacency_matrix.astype(np.int).tolist() node_list = [INPUT, *node_list, OUTPUT] model_spec = api.ModelSpec(matrix=adjacency_list, ops=node_list) nasbench.query(model_spec) for adjacency_matrix, ops, model_spec in search_space_1.generate_search_space_without_loose_ends(): # Query NASBench data = nasbench.query(model_spec) for item in data: test_error.append(1 - item['test_accuracy']) valid_error.append(1 - item['validation_accuracy']) print('Number of architectures', len(test_error) / len(data)) plt.figure() plt.title( 'Distribution of test error in search space (no. architectures {})'.format( int(len(test_error) / len(data)))) plt.hist(test_error, bins=800, density=True) ax = plt.gca() ax.set_xscale('log') ax.set_yscale('log') plt.xlabel('Test error') plt.grid(True, which="both", ls="-", alpha=0.5) plt.tight_layout() plt.xlim(0, 0.3) plt.savefig('nasbench_analysis/search_spaces/export/search_space_1/test_error_distribution.pdf', dpi=600) plt.show() plt.figure() plt.title('Distribution of validation error in search space (no. architectures {})'.format( int(len(valid_error) / len(data)))) plt.hist(valid_error, bins=800, density=True) ax = plt.gca() ax.set_xscale('log') ax.set_yscale('log') plt.xlabel('Validation error') plt.grid(True, which="both", ls="-", alpha=0.5) plt.tight_layout() plt.xlim(0, 0.3) plt.savefig('nasbench_analysis/search_spaces/export/search_space_1/valid_error_distribution.pdf', dpi=600) plt.show() print('test_error', min(test_error), 'valid_error', min(valid_error))
def eval_one_shot_model(config, model): nasbench = NasbenchWrapper( dataset_file= '/home/darts_weight_sharing_analysis/cnn/bohb/src/nasbench_analysis/nasbench_data/108_e/nasbench_only108.tfrecord' ) model_list = pickle.load(open(model, 'rb')) alphas_mixed_op = model_list[0] chosen_node_ops = softmax(alphas_mixed_op, axis=-1).argmax(-1) node_list = [PRIMITIVES[i] for i in chosen_node_ops] alphas_output = model_list[1] alphas_inputs = model_list[2:] if config['search_space'] == '1': search_space = SearchSpace1() num_inputs = list(search_space.num_parents_per_node.values())[3:-1] parents_node_3, parents_node_4 = \ [get_top_k(softmax(alpha, axis=1), num_input) for num_input, alpha in zip(num_inputs, alphas_inputs)] output_parents = get_top_k(softmax(alphas_output), num_inputs[-1]) parents = { '0': [], '1': [0], '2': [0, 1], '3': parents_node_3, '4': parents_node_4, '5': output_parents } node_list = [INPUT, *node_list, CONV1X1, OUTPUT] elif config['search_space'] == '2': search_space = SearchSpace2() num_inputs = list(search_space.num_parents_per_node.values())[2:] parents_node_2, parents_node_3, parents_node_4 = \ [get_top_k(softmax(alpha, axis=1), num_input) for num_input, alpha in zip(num_inputs[:-1], alphas_inputs)] output_parents = get_top_k(softmax(alphas_output), num_inputs[-1]) parents = { '0': [], '1': [0], '2': parents_node_2, '3': parents_node_3, '4': parents_node_4, '5': output_parents } node_list = [INPUT, *node_list, CONV1X1, OUTPUT] elif config['search_space'] == '3': search_space = SearchSpace3() num_inputs = list(search_space.num_parents_per_node.values())[2:] parents_node_2, parents_node_3, parents_node_4, parents_node_5 = \ [get_top_k(softmax(alpha, axis=1), num_input) for num_input, alpha in zip(num_inputs[:-1], alphas_inputs)] output_parents = get_top_k(softmax(alphas_output), num_inputs[-1]) parents = { '0': [], '1': [0], '2': parents_node_2, '3': parents_node_3, '4': parents_node_4, '5': parents_node_5, '6': output_parents } node_list = [INPUT, *node_list, OUTPUT] else: raise ValueError('Unknown search space') adjacency_matrix = search_space.create_nasbench_adjacency_matrix(parents) # Convert the adjacency matrix in format for nasbench adjacency_list = adjacency_matrix.astype(np.int).tolist() model_spec = api.ModelSpec(matrix=adjacency_list, ops=node_list) # Query nasbench data = nasbench.query(model_spec) valid_error, test_error, runtime, params = [], [], [], [] for item in data: test_error.append(1 - item['test_accuracy']) valid_error.append(1 - item['validation_accuracy']) runtime.append(item['training_time']) params.append(item['trainable_parameters']) return test_error, valid_error, runtime, params
def analysis(): search_space_1 = SearchSpace(num_parents_per_node={ '0': 0, '1': 1, '2': 2, '3': 2, '4': 2, '5': 2 }, search_space_number=1, num_intermediate_nodes=4) # Load NASBench nasbench = NasbenchWrapper( '/home/siemsj/projects/darts_weight_sharing_analysis/nasbench_full.tfrecord' ) test_error = [] valid_error = [] search_space_creator = search_space_1.create_search_space( with_loose_ends=False, upscale=False) for adjacency_matrix, ops, model_spec in search_space_creator: # Query NASBench data = nasbench.query(model_spec) for item in data: test_error.append(1 - item['test_accuracy']) valid_error.append(1 - item['validation_accuracy']) print('Number of architectures', len(test_error) / len(data)) plt.figure() plt.title( 'Distribution of test error in search space (no. architectures {})'. format(int(len(test_error) / len(data)))) plt.hist(test_error, bins=800, density=True) ax = plt.gca() ax.set_xscale('log') ax.set_yscale('log') plt.xlabel('Test error') plt.grid(True, which="both", ls="-", alpha=0.5) plt.tight_layout() plt.xlim(0, 0.3) plt.savefig( 'nasbench_analysis/search_spaces/export/search_space_1/test_error_distribution.pdf', dpi=600) plt.show() plt.figure() plt.title( 'Distribution of validation error in search space (no. architectures {})' .format(int(len(valid_error) / len(data)))) plt.hist(valid_error, bins=800, density=True) ax = plt.gca() ax.set_xscale('log') ax.set_yscale('log') plt.xlabel('Validation error') plt.grid(True, which="both", ls="-", alpha=0.5) plt.tight_layout() plt.xlim(0, 0.3) plt.savefig( 'nasbench_analysis/search_spaces/export/search_space_1/valid_error_distribution.pdf', dpi=600) plt.show() print('test_error', min(test_error), 'valid_error', min(valid_error))