def __init__(self, spec, config, model_dir): """Initialize evaluator. See train_and_evaluate docstring.""" self.input_train = cifar.CIFARInput('train', config) self.input_train_eval = cifar.CIFARInput('train_eval', config) self.input_valid = cifar.CIFARInput('valid', config) self.input_test = cifar.CIFARInput('test', config) self.input_sample = cifar.CIFARInput('sample', config) self.estimator = _create_estimator(spec, config, model_dir, self.input_train.num_images, self.input_sample.num_images) self.spec = spec self.config = config self.model_dir = model_dir
def __init__(self, spec, config, model_dir): """Initialize evaluator. See train_and_evaluate docstring.""" self.input_train = cifar.CIFARInput('train', config) self.input_train_eval = cifar.CIFARInput('train_eval', config) self.input_valid = cifar.CIFARInput('valid', config) self.input_test = cifar.CIFARInput('test', config) self.input_sample = cifar.CIFARInput('sample', config) self.estimator = _create_estimator(spec, config, model_dir, self.input_train.num_images, self.input_sample.num_images) self.spec = spec self.config = config self.model_dir = model_dir if self.config['max_samples'] > 30: self.max_samples = self.config['max_samples'] else: self.max_samples = 30 if self.config['number_of_steps'] > 0: self.number_of_steps = self.config['number_of_steps'] else: self.number_of_steps = 1
def _augment_and_evaluate_impl(spec, config, model_dir, epochs_per_eval=5): """Augment and evaluate implementation, see augment_and_evaluate docstring.""" input_augment, input_test = [ cifar.CIFARInput(m, config) for m in ['augment', 'test']] estimator = _create_estimator(spec, config, model_dir, input_augment.num_images) if config['train_seconds'] > 0.0: timing = training_time.limit(config['train_seconds']) else: timing = training_time.limit(None) steps_per_epoch = input_augment.num_images / config['batch_size'] # float ckpt = tf.train.latest_checkpoint(model_dir) if not ckpt: current_step = 0 else: current_step = int(ckpt.split('-')[-1]) max_steps = int(config['train_epochs'] * steps_per_epoch) while current_step < max_steps: next_step = current_step + int(epochs_per_eval * steps_per_epoch) next_step = min(next_step, max_steps) estimator.train( input_fn=input_augment.input_fn, max_steps=next_step, hooks=[timing.train_hook], saving_listeners=[timing.saving_listener]) current_step = next_step test_accuracy = _evaluate(estimator, input_test, config) metadata = { 'trainable_params': _get_param_count(model_dir), 'test_accuracy': test_accuracy, } return metadata
def keytotuple(key): cur_network_data = nb.get_metrics_from_hash(key) #print(cur_network_data[0]) #print(cur_network_data[0].keys()) model = model_spec.ModelSpec(cur_network_data[0]['module_adjacency'], cur_network_data[0]['module_operations']) model_fn = model_builder.build_model_fn(model, cfg, 60000) if os.path.exists('empty'): shutil.rmtree('empty') run_cfg = tf.contrib.tpu.RunConfig( model_dir='empty', keep_checkpoint_max=3, # Keeps ckpt at start, halfway, and end save_checkpoints_secs=2**30) #tpu_config=tf.contrib.tpu.TPUConfig( # iterations_per_loop=cfg['tpu_iterations_per_loop'], # num_shards=cfg['tpu_num_shards'])) #estimator = tf.contrib.tpu.TPUEstimator(model_fn, config=run_cfg, # train_batch_size=cfg['batch_size'], # eval_batch_size=cfg['batch_size'], # predict_batch_size=cfg['batch_size'], # use_tpu=False)#, params=cfg) estimator = tf.estimator.Estimator(model_fn, config=run_cfg, params=cfg) print(estimator) #dummy_input = np.zeros((1, 224, 224, 3)) #dummy_label = np.zeros((1, 100)) #dummy_label[0] = 1 input_train = cifar.CIFARInput('train', cfg) print(cfg['batch_size']) #input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": dummy_input}, y=dummy_label, shuffle=True) #estimator.train(input_fn) #estimator.train(input_fn=input_train.input_fn, # max_steps=1) #print(tf.get_default_graph().as_graph_def()) with tf.Graph().as_default() as g: features = tf.placeholder(tf.float32, [cfg['batch_size'], 32, 32, 3]) labels = tf.placeholder(tf.int32, [cfg['batch_size']]) _ = model_fn(features, labels, mode=tf.estimator.ModeKeys.TRAIN, params=cfg) with tf.Session() as sess: run_meta = tf.RunMetadata() opts = tf.profiler.ProfileOptionBuilder.float_operation() flops = tf.profiler.profile(g, run_meta=run_meta, cmd='op', options=opts) n_flops = flops.total_float_ops print(n_flops) #print(sess.graph.as_graph_def()) training_time_sum = 0.0 acc_sum = 0.0 params = cur_network_data[0]['trainable_parameters'] count = 0 for item in cur_network_data[1][108]: count += 1 training_time_sum += item['final_training_time'] acc_sum += item['final_test_accuracy'] training_time = training_time_sum / count acc = acc_sum / count return (params, training_time, acc, n_flops)