def test_prune_islands(self): """Tests isolated components are pruned.""" model1 = model_spec.ModelSpec( np.array([[0, 1, 0, 0], [0, 0, 0, 1], [0, 0, 0, 0], [0, 0, 0, 0]]), [1, 2, 3, 4]) assert model1.valid_spec assert np.array_equal(model1.matrix, np.array([[0, 1, 0], [0, 0, 1], [0, 0, 0]])) assert model1.ops == [1, 2, 4] model2 = model_spec.ModelSpec( np.array([[0, 1, 0, 0, 0], [0, 0, 0, 0, 1], [0, 0, 0, 1, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]), [1, 2, 3, 4, 5]) assert model2.valid_spec assert np.array_equal(model2.matrix, np.array([[0, 1, 0], [0, 0, 1], [0, 0, 0]])) assert model2.ops == [1, 2, 5]
def test_prune_dangling(self): """Tests dangling vertices are pruned.""" model1 = model_spec.ModelSpec( np.array([[0, 1, 1, 0], [0, 0, 0, 0], [0, 0, 0, 1], [0, 0, 0, 0]]), [1, 2, 3, 4]) assert model1.valid_spec assert np.array_equal(model1.matrix, np.array([[0, 1, 0], [0, 0, 1], [0, 0, 0]])) assert model1.ops == [1, 3, 4] model2 = model_spec.ModelSpec( np.array([[0, 0, 1, 0], [0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 0, 0]]), [1, 2, 3, 4]) assert model2.valid_spec assert np.array_equal(model2.matrix, np.array([[0, 1, 0], [0, 0, 1], [0, 0, 0]])) assert model2.ops == [1, 3, 4]
def test_prune_noop(self): """Tests graphs which require no pruning.""" model1 = model_spec.ModelSpec( np.array([[0, 1, 0], [0, 0, 1], [0, 0, 0]]), [0, 0, 0]) assert model1.valid_spec assert np.array_equal(model1.original_matrix, model1.matrix) assert model1.original_ops == model1.original_ops model2 = model_spec.ModelSpec( np.array([[0, 1, 1], [0, 0, 1], [0, 0, 0]]), [0, 0, 0]) assert model2.valid_spec assert np.array_equal(model2.original_matrix, model2.matrix) assert model2.original_ops == model2.ops model3 = model_spec.ModelSpec( np.array([[0, 1, 1, 0], [0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 0, 0]]), [0, 0, 0, 0]) assert model3.valid_spec assert np.array_equal(model3.original_matrix, model3.matrix) assert model3.original_ops == model3.ops
def test_prune_disconnected(self): """Tests graphs where with no input to output path are marked invalid.""" model1 = model_spec.ModelSpec(np.array([[0, 0], [0, 0]]), [0, 0]) assert not model1.valid_spec model2 = model_spec.ModelSpec( np.array([[0, 1, 0, 0], [0, 0, 0, 0], [0, 0, 0, 1], [0, 0, 0, 0]]), [1, 2, 3, 4]) assert not model2.valid_spec model3 = model_spec.ModelSpec( np.array([[0, 0, 0, 0], [0, 0, 1, 0], [0, 0, 0, 0], [0, 0, 0, 0]]), [1, 2, 3, 4]) assert not model3.valid_spec
def create_inception_resnet_spec(config): """Construct an Inception-ResNet like spec. This spec is very similar to the InceptionV2 module with an added residual connection except that there is an extra projection in front of the max pool. The overall network filter counts and module counts do not match the actual source model. Args: config: config dict created by config.py. Returns: ModelSpec object. """ spec = model_spec.ModelSpec( np.array([[0, 1, 1, 1, 0, 1, 1], [0, 0, 0, 0, 0, 0, 1], [0, 0, 0, 0, 0, 0, 1], [0, 0, 0, 0, 1, 0, 0], [0, 0, 0, 0, 0, 0, 1], [0, 0, 0, 0, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0]]), [ 'input', 'conv1x1-bn-relu', 'conv3x3-bn-relu', 'conv3x3-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3', 'output' ]) config['num_stacks'] = 3 config['num_modules_per_stack'] = 3 config['stem_filter_size'] = 128 return spec
def main(*args, **kwargs): nasbench = nasbench_api.NASBench(FLAGS.path_to_nasbench) module = nasbench.fixed_statistics[FLAGS.hash_key] spec = model_spec.ModelSpec(module['module_adjacency'], module['module_operations']) config = nasbench_config.build_config() for flag in FLAGS.flags_by_module_dict()[args[0][0]]: config[flag.name] = flag.value config['use_tpu'] = False config['use_KD'] = False config['intermediate_evaluations'] = ['1.0'] trainset_multipier = FLAGS.trainset_part_percentage / 100.0 config['num_train'] = int(config['num_train'] * trainset_multipier) config['num_train_eval'] = int(config['num_train_eval'] * trainset_multipier) config['num_augment'] = int(config['num_augment'] * trainset_multipier) logging.info("Prepare KD dataset") dataset_files = FLAGS.train_data_files + [ FLAGS.valid_data_file, FLAGS.test_data_file, FLAGS.sample_data_file ] prepare_kd_dataset(spec, config, FLAGS.save_path, dataset_files, FLAGS.new_dataset_path, FLAGS.trainset_part_percentage)
def main(): config = _config.build_config() config['use_tpu'] = False config['train_data_files'] = ['train_1.tfrecords', 'train_2.tfrecords', 'train_3.tfrecords', 'train_4.tfrecords'] config['valid_data_file'] = ['validation.tfrecords'] config['test_data_file'] = ['test.tfrecords'] config['sample_data_file'] = ['sample.tfrecords'] config['train_epochs'] = 1 config['use_KD'] = False matrix=[[0, 1, 1, 1, 0, 1, 0], # input layer [0, 0, 0, 0, 0, 0, 1], # 1x1 conv [0, 0, 0, 0, 0, 0, 1], # 3x3 conv [0, 0, 0, 0, 1, 0, 0], # 5x5 conv (replaced by two 3x3's) [0, 0, 0, 0, 0, 0, 1], # 5x5 conv (replaced by two 3x3's) [0, 0, 0, 0, 0, 0, 1], # 3x3 max-pool [0, 0, 0, 0, 0, 0, 0]] # output layer # Operations at the vertices of the module, matches order of matrix labels=['input', 'conv1x1-bn-relu', 'conv3x3-bn-relu', 'conv3x3-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3', 'output'] matrix = np.array(matrix) labels = np.array(labels) spec = model_spec.ModelSpec(matrix, labels) model_dir = '../data/tmp' meta = evaluate.train_and_evaluate(spec, config, model_dir) output_file = os.path.join(model_dir, RESULTS_FILE) with tf.gfile.Open(output_file, 'w') as f: json.dump(meta, f, cls=NumpyEncoder) print('OK') print(spec.__dict__)
def _evaluate_work_unit(self, index): """Runs the evaluation of the model at the specified index. The index records the current index of the work unit being evaluated. Each worker will only compute the work units with index modulo total_workers equal to the worker_id. Args: index: int index into total work units. """ if self.remainders: assert self.ordered_keys is None model_id = self.remainders[index][0] model_repeat = self.remainders[index][1] else: model_id = self.ordered_keys[index % self.num_models] model_repeat = index // self.num_models + 1 matrix, labels = self.models[model_id] matrix = np.array(matrix) # Re-label to config['available_ops'] labels = (['input'] + [self.config['available_ops'][lab] for lab in labels[1:-1]] + ['output']) spec = model_spec.ModelSpec(matrix, labels) assert spec.valid_spec assert np.sum(spec.matrix) <= self.config['max_edges'] # Split the directory into 16^2 roughly equal subdirectories model_dir = os.path.join(self.output_dir, model_id[:2], model_id, 'repeat_%d' % model_repeat) # evaluate the model try: meta = evaluate.train_and_evaluate(spec, self.config, model_dir) except evaluate.AbortError: # After hitting the retry limit, the job will continue to the next work # unit. These failed jobs may need to be re-run at a later point. return # Write data to model_dir output_file = os.path.join(model_dir, RESULTS_FILE) with tf.io.gfile.GFile(output_file, 'w') as f: json.dump(meta, f, cls=NumpyEncoder) # Delete some files to reclaim space self._clean_model_dir(model_dir)
def create_resnet50_spec(config): """Construct a ResNet-50-like spec. The main difference is that there is an extra projection layer before the conv1x1 whereas the original ResNet doesn't have this. This increases the parameter count of this version slightly. Args: config: config dict created by config.py. Returns: ModelSpec object. """ spec = model_spec.ModelSpec(np.array([[0, 1, 1], [0, 0, 1], [0, 0, 0]]), ['input', 'bottleneck3x3', 'output']) config['num_stacks'] = 3 config['num_modules_per_stack'] = 6 config['stem_filter_size'] = 128 return spec
def main(*args, **kwargs): nasbench = nasbench_api.NASBench(FLAGS.path_to_nasbench) module = nasbench.fixed_statistics[FLAGS.hash_key] spec = model_spec.ModelSpec(module['module_adjacency'], module['module_operations']) config = nasbench_config.build_config() for flag in FLAGS.flags_by_module_dict()[args[0][0]]: config[flag.name] = flag.value config['use_tpu'] = False config['use_KD'] = False config['intermediate_evaluations'] = ['1.0'] trainset_multipier = FLAGS.trainset_part_percentage / 100.0 config['num_train'] = int(config['num_train'] * trainset_multipier) config['num_train_eval'] = int(config['num_train_eval'] * trainset_multipier) config['num_augment'] = int(config['num_augment'] * trainset_multipier) logging.info("Train and evaluate with config\n{}\n and spec\n{}".format(config, spec)) train(spec, config, FLAGS.save_path)
def create_best_nasbench_spec(config): """Construct the best spec in the NASBench dataset w.r.t. mean test accuracy. Args: config: config dict created by config.py. Returns: ModelSpec object. """ spec = model_spec.ModelSpec( np.array([[0, 1, 1, 0, 0, 1, 1], [0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 1, 0, 0, 0], [0, 0, 0, 0, 1, 0, 0], [0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 0, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0]]), [ 'input', 'conv1x1-bn-relu', 'conv3x3-bn-relu', 'maxpool3x3', 'conv3x3-bn-relu', 'conv3x3-bn-relu', 'output' ]) config['num_stacks'] = 3 config['num_modules_per_stack'] = 3 config['stem_filter_size'] = 128 return spec
def keytotuple(key): cur_network_data = nb.get_metrics_from_hash(key) #print(cur_network_data[0]) #print(cur_network_data[0].keys()) model = model_spec.ModelSpec(cur_network_data[0]['module_adjacency'], cur_network_data[0]['module_operations']) model_fn = model_builder.build_model_fn(model, cfg, 60000) if os.path.exists('empty'): shutil.rmtree('empty') run_cfg = tf.contrib.tpu.RunConfig( model_dir='empty', keep_checkpoint_max=3, # Keeps ckpt at start, halfway, and end save_checkpoints_secs=2**30) #tpu_config=tf.contrib.tpu.TPUConfig( # iterations_per_loop=cfg['tpu_iterations_per_loop'], # num_shards=cfg['tpu_num_shards'])) #estimator = tf.contrib.tpu.TPUEstimator(model_fn, config=run_cfg, # train_batch_size=cfg['batch_size'], # eval_batch_size=cfg['batch_size'], # predict_batch_size=cfg['batch_size'], # use_tpu=False)#, params=cfg) estimator = tf.estimator.Estimator(model_fn, config=run_cfg, params=cfg) print(estimator) #dummy_input = np.zeros((1, 224, 224, 3)) #dummy_label = np.zeros((1, 100)) #dummy_label[0] = 1 input_train = cifar.CIFARInput('train', cfg) print(cfg['batch_size']) #input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": dummy_input}, y=dummy_label, shuffle=True) #estimator.train(input_fn) #estimator.train(input_fn=input_train.input_fn, # max_steps=1) #print(tf.get_default_graph().as_graph_def()) with tf.Graph().as_default() as g: features = tf.placeholder(tf.float32, [cfg['batch_size'], 32, 32, 3]) labels = tf.placeholder(tf.int32, [cfg['batch_size']]) _ = model_fn(features, labels, mode=tf.estimator.ModeKeys.TRAIN, params=cfg) with tf.Session() as sess: run_meta = tf.RunMetadata() opts = tf.profiler.ProfileOptionBuilder.float_operation() flops = tf.profiler.profile(g, run_meta=run_meta, cmd='op', options=opts) n_flops = flops.total_float_ops print(n_flops) #print(sess.graph.as_graph_def()) training_time_sum = 0.0 acc_sum = 0.0 params = cur_network_data[0]['trainable_parameters'] count = 0 for item in cur_network_data[1][108]: count += 1 training_time_sum += item['final_training_time'] acc_sum += item['final_test_accuracy'] training_time = training_time_sum / count acc = acc_sum / count return (params, training_time, acc, n_flops)