def evaluate_model(config): """ Train the model using the passed in config """ ########################################################### # Generate the model ########################################################### outputs = create_generator(config, input_utils.get_data_shape(config.dataset)) ########################################################### # Setup the evaluation metrics and summaries ########################################################### # Generate the canvases that lead to the final output image summaries = [] summaries.extend(layers.summarize_collection(graph_utils.GraphKeys.RNN_OUTPUTS)) with tf.name_scope('canvases'): for step, canvas in enumerate(outputs): canvas = input_utils.reshape_images(canvas, config.dataset) tiled_images = image_utils.tile_images(canvas) summaries.append(tf.summary.image('step{0}'.format(step), tiled_images)) summary_op = tf.summary.merge(summaries, name='summaries') ########################################################### # Begin evaluation ########################################################### checkpoint_path = FLAGS.checkpoint_path if tf.gfile.IsDirectory(checkpoint_path): checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) eval_ops = tf.group(*outputs) hooks = [ training.SummaryAtEndHook(FLAGS.log_dir, summary_op), training.StopAfterNEvalsHook(FLAGS.count)] training.evaluate_once(checkpoint_path, hooks=hooks, eval_ops=eval_ops)
def evaluate(train_dir, eval_dir, config, dataset_fn, num_batches, master=''): """Evaluate the model repeatedly.""" tf.gfile.MakeDirs(eval_dir) _trial_summary( config.hparams, config.eval_examples_path or config.tfds_name, eval_dir) with tf.Graph().as_default(): model = config.model model.build(config.hparams, config.data_converter.output_depth, is_training=False) eval_op = model.eval( **_get_input_tensors(dataset_fn().take(num_batches), config)) hooks = [ contrib_training.StopAfterNEvalsHook(num_batches), contrib_training.SummaryAtEndHook(eval_dir) ] contrib_training.evaluate_repeatedly( train_dir, eval_ops=eval_op, hooks=hooks, eval_interval_secs=60, master=master)
def run_eval(build_graph_fn, train_dir, eval_dir, num_batches, timeout_secs=300): """Runs the training loop. Args: build_graph_fn: A function that builds the graph ops. train_dir: The path to the directory where checkpoints will be loaded from for evaluation. eval_dir: The path to the directory where the evaluation summary events will be written to. num_batches: The number of full batches to use for each evaluation step. timeout_secs: The number of seconds after which to stop waiting for a new checkpoint. Raises: ValueError: If `num_batches` is less than or equal to 0. """ if num_batches <= 0: raise ValueError( '`num_batches` must be greater than 0. Check that the batch size is ' 'no larger than the number of records in the eval set.') with tf.Graph().as_default(): build_graph_fn() global_step = tf.train.get_or_create_global_step() loss = tf.get_collection('loss')[0] perplexity = tf.get_collection('metrics/perplexity')[0] accuracy = tf.get_collection('metrics/accuracy')[0] eval_ops = tf.get_collection('eval_ops') logging_dict = { 'Global Step': global_step, 'Loss': loss, 'Perplexity': perplexity, 'Accuracy': accuracy } hooks = [ EvalLoggingTensorHook(logging_dict, every_n_iter=num_batches), contrib_training.StopAfterNEvalsHook(num_batches), contrib_training.SummaryAtEndHook(eval_dir), ] contrib_training.evaluate_repeatedly(train_dir, eval_ops=eval_ops, hooks=hooks, eval_interval_secs=60, timeout=timeout_secs)
def Evaluacion(build_graph_fn, train_dir, eval_dir, num_batches, timeout_secs=300): tf.compat.v1.logging.set_verbosity('INFO') if num_batches <= 0: raise ValueError( '`num_batches` must be greater than 0. Check that the batch size is ' 'no larger than the number of records in the eval set.') with tf.Graph().as_default(): # Creamos un modelo igual que el del entrenamiento build_graph_fn() # Define the summaries to write: global_step = tf.train.get_or_create_global_step() loss = tf.get_collection('loss')[0] perplexity = tf.get_collection('metrics/perplexity')[0] accuracy = tf.get_collection('metrics/accuracy')[0] eval_ops = tf.get_collection('eval_ops') logging_dict = { 'Global Step': global_step, 'Loss': loss, 'Perplexity': perplexity, 'Accuracy': accuracy } hooks = [ EvalLoggingTensorHook(logging_dict, every_n_iter=num_batches), contrib_training.StopAfterNEvalsHook(num_batches), contrib_training.SummaryAtEndHook(eval_dir), ] # names_to_values = contrib_training.evaluate_once( # checkpoint_path=train_dir, # eval_ops=eval_ops, # final_ops=logging_dict, # hooks=hooks, # config=None) # for name in names_to_values: # print('Metric %s has value %f.' % (name, names_to_values[name])) contrib_training.evaluate_repeatedly(train_dir, eval_ops=eval_ops, hooks=hooks, eval_interval_secs=2, timeout=timeout_secs)
def evaluate_model(config): """ Train the model using the passed in config """ ########################################################### # Create the input pipeline ########################################################### with tf.name_scope('input_pipeline'): dataset = input_utils.get_dataset(config.datadir, config.dataset, config.datasubset, num_folds=config.fold_count, fold=config.fold, holdout=True) init_op, init_feed_dict, image = input_utils.get_data( config.dataset, dataset, config.batch_size, num_epochs=config.num_epochs, num_readers=config.num_readers) images = tf.train.batch([image], config.batch_size, num_threads=config.num_preprocessing_threads, capacity=5 * config.batch_size) ########################################################### # Generate the model ########################################################### outputs = create_model(config, images, dataset) ########################################################### # Setup the evaluation metrics and summaries ########################################################### summaries = [] metrics_map = {} for loss in tf.losses.get_losses(): metrics_map[loss.op.name] = metrics.streaming_mean(loss) for metric in tf.get_collection(graph_utils.GraphKeys.METRICS): metrics_map[metric.op.name] = metrics.streaming_mean(metric) total_loss = tf.losses.get_total_loss() metrics_map[total_loss.op.name] = metrics.streaming_mean(total_loss) names_to_values, names_to_updates = metrics.aggregate_metric_map( metrics_map) # Create summaries of the metrics and print them to the screen for name, value in names_to_values.iteritems(): summary = tf.summary.scalar(name, value, collections=[]) summaries.append(tf.Print(summary, [value], name)) summaries.extend(layers.summarize_collection(tf.GraphKeys.MODEL_VARIABLES)) summaries.extend(layers.summarize_collection( graph_utils.GraphKeys.METRICS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.RNN_OUTPUTS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS)) images = input_utils.reshape_images(images, config.dataset) tiled_images = image_utils.tile_images(images) summaries.append(tf.summary.image('input_batch', tiled_images)) # Generate the canvases that lead to the final output image with tf.name_scope('canvases'): for step, canvas in enumerate(outputs): canvas = input_utils.reshape_images(canvas, config.dataset) tiled_images = image_utils.tile_images(canvas) summaries.append( tf.summary.image('step{0}'.format(step), tiled_images)) summary_op = tf.summary.merge(summaries, name='summaries') ########################################################### # Begin evaluation ########################################################### checkpoint_path = FLAGS.checkpoint_path eval_ops = tf.group(*names_to_updates.values()) hooks = [ training.SummaryAtEndHook(log_dir=FLAGS.log_dir, summary_op=summary_op), training.StopAfterNEvalsHook( math.ceil(dataset.num_samples / float(config.batch_size))) ] eval_kwargs = {} eval_fn = training.evaluate_repeatedly if FLAGS.once: if tf.gfile.IsDirectory(checkpoint_path): checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) eval_fn = training.evaluate_once else: assert tf.gfile.IsDirectory(checkpoint_path), ( 'checkpoint path must be a directory when using loop evaluation') eval_fn(checkpoint_path, hooks=hooks, eval_ops=eval_ops, **eval_kwargs)
def evaluate_model(config): """ Train the model using the passed in config """ ########################################################### # Create the input pipeline ########################################################### with tf.name_scope('input_pipeline'): dataset = input_utils.get_dataset(config.datadir, config.dataset, config.datasubset) init_op, init_feed_dict, image, label = input_utils.get_data( config.dataset, dataset, config.batch_size, num_epochs=config.num_epochs, num_readers=config.num_readers) images, labels = tf.train.batch( [image, label], config.batch_size, num_threads=config.num_preprocessing_threads, capacity=5 * config.batch_size) ########################################################### # Generate the model ########################################################### outputs = create_model(config, images, dataset) tfprof.model_analyzer.print_model_analysis(tf.get_default_graph()) ########################################################### # Setup the evaluation metrics and summaries ########################################################### summaries = [] metrics_map = {} for metric in tf.get_collection(graph_utils.GraphKeys.METRICS): metrics_map[metric.op.name] = metrics.streaming_mean(metric) predictions = tf.argmax(outputs, 1) metrics_map['accuracy'] = metrics.streaming_accuracy(predictions, labels) metrics_map['recall_5'] = metrics.streaming_sparse_recall_at_k( outputs, tf.expand_dims(labels, 1), 5) names_to_values, names_to_updates = metrics.aggregate_metric_map( metrics_map) # Create summaries of the metrics and print them to the screen for name, value in names_to_values.iteritems(): summary = tf.summary.scalar(name, value, collections=[]) summaries.append(tf.Print(summary, [value], name)) summaries.extend(layers.summarize_collection( graph_utils.GraphKeys.METRICS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.QUANTIZED_VARIABLES)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS)) tiled_images = image_utils.tile_images(images) summaries.append(tf.summary.image('input_batch', tiled_images)) summary_op = tf.summary.merge(summaries, name='summaries') ########################################################### # Begin evaluation ########################################################### checkpoint_path = FLAGS.checkpoint_path eval_ops = tf.group(*names_to_updates.values()) scaffold = tf.train.Scaffold(init_op, init_feed_dict) hooks = [ training.SummaryAtEndHook(FLAGS.log_dir, summary_op), training.StopAfterNEvalsHook( math.ceil(dataset.num_samples / float(config.batch_size))) ] eval_kwargs = {} eval_fn = training.evaluate_repeatedly if FLAGS.once: if tf.gfile.IsDirectory(checkpoint_path): checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) eval_fn = training.evaluate_once else: assert tf.gfile.IsDirectory(checkpoint_path), ( 'checkpoint path must be a directory when using loop evaluation') # On Tensorflow master fd87896 fixes this, but for now just set a very large number eval_kwargs['max_number_of_evaluations'] = sys.maxint eval_fn(checkpoint_path, scaffold=scaffold, hooks=hooks, eval_ops=eval_ops, **eval_kwargs)