def build_test_metrics(self): dense_labels = dense_labels_fn(tf.shape(self.prediction), self.timestamps, self.mmsis) thresholded_prediction = tf.to_int32(self.prediction > 0.5) valid = tf.to_int32(tf.not_equal(dense_labels, -1)) ones = tf.to_int32(dense_labels > 0.5) weights = tf.to_float(valid) prediction = self.prediction unclear = tf.to_int32((self.prediction > 0.333) & (self.prediction < 0.666)) if eval_window: b, e = eval_window prediction = prediction[:, b:e] dense_labels = dense_labels[:, b:e] thresholded_prediction = thresholded_prediction[:, b:e] valid = valid[:, b:e] ones = ones[:, b:e] weights = weights[:, b:e] unclear = unclear[:, b:e] recall = slim.metrics.streaming_recall(thresholded_prediction, ones, weights=weights) precision = slim.metrics.streaming_precision( thresholded_prediction, ones, weights=weights) raw_metrics = { 'Test-MSE': slim.metrics.streaming_mean_squared_error( prediction, tf.to_float(ones), weights=weights), 'Test-accuracy': slim.metrics.streaming_accuracy(thresholded_prediction, ones, weights=weights), 'Test-precision': precision, 'Test-recall': recall, 'Test-F1-score': f1(recall, precision), 'Test-prediction-fraction': slim.metrics.streaming_accuracy(thresholded_prediction, valid, weights=weights), 'Test-unclear-fraction': slim.metrics.streaming_accuracy(unclear, valid, weights=weights), 'Test-label-fraction': slim.metrics.streaming_accuracy(ones, valid, weights=weights) } return metrics.aggregate_metric_map({ "{}/{}".format(self.name, k): v for (k, v) in raw_metrics.items() })
def build_test_metrics(self): raw_loss = self.masked_mean_error(self.prediction, self.mmsis) return metrics.aggregate_metric_map({ '%s/Test-error' % self.name: metrics.streaming_mean(raw_loss) })
def eval(self, input_sequence, output_sequence, sequence_length, control_sequence=None): """Evaluate on the given sequences, returning metric update ops. Args: input_sequence: The sequence to be fed to the encoder. output_sequence: The sequence expected from the decoder. sequence_length: The length of the given sequences (which must be identical). control_sequence: (Optional) sequence on which to condition the decoder. Returns: metric_update_ops: tf.metrics update ops. """ metric_map, scalars_to_summarize = self._compute_model_loss( input_sequence, output_sequence, sequence_length, control_sequence) for n, t in scalars_to_summarize.items(): metric_map[n] = tf.metrics.mean(t) metrics_to_values, metrics_to_updates = ( contrib_metrics.aggregate_metric_map(metric_map)) for metric_name, metric_value in metrics_to_values.items(): tf.summary.scalar(metric_name, metric_value) return list(metrics_to_updates.values())
def main(_): assert FLAGS.train_dir, "--train_dir is required." if tf.gfile.Exists(FLAGS.summaries_dir): tf.gfile.DeleteRecursively(FLAGS.summaries_dir) tf.gfile.MakeDirs(FLAGS.summaries_dir) config = configuration.Config() dataset_eval = loader.get_split(FLAGS.split_name, dataset_dir=FLAGS.data_dir) if FLAGS.preprocess_abs: preprocess_fn = tf.abs else: preprocess_fn = None # whther it is a 2d input is_2D = common.is_2D(FLAGS.model) series, labels, labels_one_hot = loader.load_batch( dataset_eval, batch_size=config.batch_size, is_2D=is_2D, preprocess_fn=preprocess_fn) # Build lazy model model = common.convert_name_to_instance(FLAGS.model, config, 'eval') endpoints = model.build(inputs=series, is_training=False) predictions = tf.to_int64(tf.argmax(endpoints.logits, 1)) slim.get_or_create_global_step() # Choose the metrics to compute: names_to_values, names_to_updates = metrics.aggregate_metric_map({ 'accuracy': metrics.streaming_accuracy(predictions, labels), 'precision': metrics.streaming_precision(predictions, labels), 'recall': metrics.streaming_recall(predictions, labels), }) # Create the summary ops such that they also print out to std output: summary_ops = [] for metric_name, metric_value in names_to_values.iteritems(): op = tf.summary.scalar(metric_name, metric_value) op = tf.Print(op, [metric_value], metric_name) summary_ops.append(op) slim.evaluation.evaluation_loop( master='', checkpoint_dir=FLAGS.train_dir, logdir=FLAGS.summaries_dir, eval_op=names_to_updates.values(), num_evals=min(FLAGS.num_batches, dataset_eval.num_samples), eval_interval_secs=FLAGS.eval_interval_secs, max_number_of_evaluations=FLAGS.num_of_steps, summary_op=tf.summary.merge(summary_ops), session_config=config.session_config, )
def metric_def(self): self.training_metrics = { "learning_rate": self.learning_rate, "adam_beta_1": self.adam_beta_1, "adam_beta_2": self.adam_beta_2, "ema_decay": self.ema_decay, "cons_coefficient": self.cons_coefficient, "train/error/1": self.mean_error_1, "train/error/ema": self.mean_error_ema, "train/class_cost/1": self.mean_class_cost_1, "train/class_cost/ema": self.mean_class_cost_ema, "train/cons_cost/mt": self.mean_cons_cost_mt, "train/total_cost/mt": self.mean_total_cost_mt, } with tf.variable_scope("validation_metrics") as metrics_scope: self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map( { "eval/error/1": streaming_mean(self.errors_1), "eval/error/ema": streaming_mean(self.errors_ema), "eval/class_cost/1": streaming_mean(self.class_costs_1), "eval/class_cost/ema": streaming_mean(self.class_costs_ema), }) metric_variables = slim.get_local_variables( scope=metrics_scope.name) self.metric_init_op = tf.variables_initializer(metric_variables) self.result_formatter = string_utils.DictFormatter( order=[ "eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt" ], default_format='{name}: {value:>10.6f}', separator=", ") self.result_formatter.add_format('error', '{name}: {value:>6.1%}')
def evaluate(): with tf.Graph().as_default(): config = tf.ConfigProto(device_count={'GPU': 0}) images, labels = utils.load_batch(shards=VAL_SHARDS, batch_size=FLAGS.batch_size, train=False, crop=False, flip=False) predictions = alexnet.AlexNet(images, batch_size=FLAGS.batch_size).model prediction = tf.to_int64(tf.argmax(predictions, 1)) # Returns index of largest mse_op = metrics.streaming_mean_squared_error(prediction, labels) rmse_op = metrics.streaming_root_mean_squared_error(prediction, labels) accuracy_op = metrics.streaming_accuracy(prediction, labels) precision_op = metrics.streaming_precision(prediction, labels) metrics_to_values, metrics_to_updates = metrics.aggregate_metric_map({ 'mse': mse_op, 'rmse': rmse_op, 'accuracy': accuracy_op, 'precision': precision_op, }) for metric_name, metric_value in metrics_to_values.items(): tf.summary.scalar(metric_name, metric_value) slim.evaluation.evaluation_loop('', FLAGS.trainlog_dir, FLAGS.evallog_dir, num_evals=FLAGS.num_evals, eval_op=list( metrics_to_updates.values()), eval_interval_secs=5, session_config=config) '''checkpoint_list = [FLAGS.trainlog_momentum_dir,
def build_test_metrics(self): raw_labels = multihot_labels(self.mmsis) mask = tf.to_float(tf.equal(tf.reduce_sum(raw_labels, 1), 1)) labels = tf.to_int32(tf.argmax(raw_labels, 1)) predictions = tf.to_int32(tf.argmax(self.prediction, 1)) metrics_map = { '%s/Test-accuracy' % self.name: metrics.streaming_accuracy(predictions, labels, weights=mask) } if self.metrics == 'all': for i, cls in enumerate(self.classes): cls_name = cls.replace(' ', '-') trues = tf.to_int32(tf.equal(labels, i)) preds = tf.to_int32(tf.equal(predictions, i)) recall = metrics.streaming_recall(preds, trues, weights=mask) precision = metrics.streaming_precision(preds, trues, weights=mask) metrics_map["%s/Class-%s-Precision" % (self.name, cls_name)] = recall metrics_map["%s/Class-%s-Recall" % (self.name, cls_name)] = precision metrics_map["%s/Class-%s-F1-Score" % (self.name, cls_name)] = f1( recall, precision) metrics_map["%s/Class-%s-ROC-AUC" % (self.name, cls_name)] = metrics.streaming_auc( self.prediction[:, i], trues, weights=mask) return metrics.aggregate_metric_map(metrics_map)
def __init__(self, result_dir): self.checkpoint_dir = os.path.join(result_dir, 'checkpoints') self.summary_dir = os.path.join(result_dir, 'summaries') os.makedirs(self.checkpoint_dir) os.makedirs(self.summary_dir) with tf.name_scope("placeholders"): self.images = tf.placeholder(dtype=tf.float32, shape=(None, 32, 32, 3), name='images') self.labels = tf.placeholder(dtype=tf.int32, shape=(None, ), name='labels') self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training') self.global_step = tf.Variable(0, trainable=False, name='global_step') tf.add_to_collection("init_in_init", self.global_step) self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS) for var in self.hyper.variables.values(): tf.add_to_collection("init_in_init", var) with tf.name_scope("ramps"): sigmoid_rampup_value = sigmoid_rampup(self.global_step, self.hyper['rampup_length']) sigmoid_rampdown_value = sigmoid_rampdown( self.global_step, self.hyper['rampdown_length'], self.hyper['training_length']) self.learning_rate = tf.multiply(sigmoid_rampup_value * sigmoid_rampdown_value, self.hyper['max_learning_rate'], name='learning_rate') self.adam_beta_1 = tf.add( sigmoid_rampdown_value * self.hyper['adam_beta_1_before_rampdown'], (1 - sigmoid_rampdown_value) * self.hyper['adam_beta_1_after_rampdown'], name='adam_beta_1') self.cons_coefficient = tf.multiply( sigmoid_rampup_value, self.hyper['max_consistency_coefficient'], name='consistency_coefficient') step_rampup_value = step_rampup(self.global_step, self.hyper['rampup_length']) self.adam_beta_2 = tf.add( (1 - step_rampup_value) * self.hyper['adam_beta_2_during_rampup'], step_rampup_value * self.hyper['adam_beta_2_after_rampup'], name='adam_beta_2') self.ema_decay = tf.add( (1 - step_rampup_value) * self.hyper['ema_decay_during_rampup'], step_rampup_value * self.hyper['ema_decay_after_rampup'], name='ema_decay') self.logits_1, self.logits_2, self.logits_ema = inference( self.images, is_training=self.is_training, ema_decay=self.ema_decay, normalize_input=self.hyper['normalize_input'], flip_horizontally=self.hyper['flip_horizontally']) with tf.name_scope("objectives"): self.mean_error_1, self.errors_1 = errors(self.logits_1, self.labels) self.mean_error_ema, self.errors_ema = errors( self.logits_ema, self.labels) self.mean_class_cost_1, self.class_costs_1 = classification_costs( self.logits_1, self.labels) self.mean_class_cost_ema, self.class_costs_ema = classification_costs( self.logits_ema, self.labels) labeled_consistency = self.hyper['apply_consistency_to_labeled'] consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency) self.mean_cons_cost_pi, self.cons_costs_pi = consistency_costs( self.logits_1, self.logits_2, self.cons_coefficient, consistency_mask) self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs( self.logits_1, self.logits_ema, self.cons_coefficient, consistency_mask) self.mean_total_cost_pi, self.total_costs_pi = total_costs( self.class_costs_1, self.cons_costs_pi) self.mean_total_cost_mt, self.total_costs_mt = total_costs( self.class_costs_1, self.cons_costs_mt) self.cost_to_be_minimized = tf.cond( self.hyper['ema_consistency'], lambda: self.mean_total_cost_mt, lambda: self.mean_total_cost_pi) with tf.name_scope("train_step"): update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): self.train_step_op = nn.adam_optimizer( self.cost_to_be_minimized, self.global_step, learning_rate=self.learning_rate, beta1=self.adam_beta_1, beta2=self.adam_beta_2, epsilon=self.hyper['adam_epsilon']) self.training_control = training_control(self.global_step, self.hyper['print_span'], self.hyper['evaluation_span'], self.hyper['training_length']) self.training_metrics = { "learning_rate": self.learning_rate, "adam_beta_1": self.adam_beta_1, "adam_beta_2": self.adam_beta_2, "ema_decay": self.ema_decay, "cons_coefficient": self.cons_coefficient, "train/error/1": self.mean_error_1, "train/error/ema": self.mean_error_ema, "train/class_cost/1": self.mean_class_cost_1, "train/class_cost/ema": self.mean_class_cost_ema, "train/cons_cost/pi": self.mean_cons_cost_pi, "train/cons_cost/mt": self.mean_cons_cost_mt, "train/total_cost/pi": self.mean_total_cost_pi, "train/total_cost/mt": self.mean_total_cost_mt, } with tf.variable_scope("validation_metrics") as metrics_scope: self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map( { "eval/error/1": streaming_mean(self.errors_1), "eval/error/ema": streaming_mean(self.errors_ema), # Note that the evaluation costs are not directly comparable # to the training costs. Training batches contain unlabeled # samples but the evaluation batches do not. Because # classification cost is zero for unlabeled samples, the # training costs are smaller than evaluation costs when # doing semi-supervised learning. "eval/class_cost/1": streaming_mean(self.class_costs_1), "eval/class_cost/ema": streaming_mean(self.class_costs_ema), "eval/cons_cost/pi": streaming_mean(self.cons_costs_pi), "eval/cons_cost/mt": streaming_mean(self.cons_costs_mt), "eval/total_cost/pi": streaming_mean(self.total_costs_pi), "eval/total_cost/mt": streaming_mean(self.total_costs_mt) }) metric_variables = slim.get_local_variables( scope=metrics_scope.name) self.metric_init_op = tf.variables_initializer(metric_variables) with tf.name_scope("initializers"): init_init_variables = tf.get_collection("init_in_init") train_init_variables = [ var for var in tf.global_variables() if var not in init_init_variables ] self.init_init_op = tf.variables_initializer(init_init_variables) self.train_init_op = tf.variables_initializer(train_init_variables) self.saver = tf.train.Saver() self.session = tf.Session() self.run(self.init_init_op)
def evaluate_model(config): """ Train the model using the passed in config """ ########################################################### # Create the input pipeline ########################################################### with tf.name_scope('input_pipeline'): dataset = input_utils.get_dataset(config.datadir, config.dataset, config.datasubset) init_op, init_feed_dict, image, label = input_utils.get_data( config.dataset, dataset, config.batch_size, num_epochs=config.num_epochs, num_readers=config.num_readers) images, labels = tf.train.batch( [image, label], config.batch_size, num_threads=config.num_preprocessing_threads, capacity=5 * config.batch_size) ########################################################### # Generate the model ########################################################### outputs = create_model(config, images, dataset) tfprof.model_analyzer.print_model_analysis(tf.get_default_graph()) ########################################################### # Setup the evaluation metrics and summaries ########################################################### summaries = [] metrics_map = {} for metric in tf.get_collection(graph_utils.GraphKeys.METRICS): metrics_map[metric.op.name] = metrics.streaming_mean(metric) predictions = tf.argmax(outputs, 1) metrics_map['accuracy'] = metrics.streaming_accuracy(predictions, labels) metrics_map['recall_5'] = metrics.streaming_sparse_recall_at_k( outputs, tf.expand_dims(labels, 1), 5) names_to_values, names_to_updates = metrics.aggregate_metric_map( metrics_map) # Create summaries of the metrics and print them to the screen for name, value in names_to_values.iteritems(): summary = tf.summary.scalar(name, value, collections=[]) summaries.append(tf.Print(summary, [value], name)) summaries.extend(layers.summarize_collection( graph_utils.GraphKeys.METRICS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.QUANTIZED_VARIABLES)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS)) tiled_images = image_utils.tile_images(images) summaries.append(tf.summary.image('input_batch', tiled_images)) summary_op = tf.summary.merge(summaries, name='summaries') ########################################################### # Begin evaluation ########################################################### checkpoint_path = FLAGS.checkpoint_path eval_ops = tf.group(*names_to_updates.values()) scaffold = tf.train.Scaffold(init_op, init_feed_dict) hooks = [ training.SummaryAtEndHook(FLAGS.log_dir, summary_op), training.StopAfterNEvalsHook( math.ceil(dataset.num_samples / float(config.batch_size))) ] eval_kwargs = {} eval_fn = training.evaluate_repeatedly if FLAGS.once: if tf.gfile.IsDirectory(checkpoint_path): checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) eval_fn = training.evaluate_once else: assert tf.gfile.IsDirectory(checkpoint_path), ( 'checkpoint path must be a directory when using loop evaluation') # On Tensorflow master fd87896 fixes this, but for now just set a very large number eval_kwargs['max_number_of_evaluations'] = sys.maxint eval_fn(checkpoint_path, scaffold=scaffold, hooks=hooks, eval_ops=eval_ops, **eval_kwargs)
def __init__(self, run_context=None): if run_context is not None: self.training_log = run_context.create_train_log('training') self.validation_log = run_context.create_train_log('validation') self.checkpoint_path = os.path.join(run_context.transient_dir, 'checkpoint') self.tensorboard_path = os.path.join(run_context.result_dir, 'tensorboard') with tf.name_scope("placeholders"): self.images = tf.placeholder(dtype=tf.float32, shape=(None, 32, 32, 3), name='images') self.labels = tf.placeholder(dtype=tf.int32, shape=(None, ), name='labels') self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training') self.global_step = tf.Variable(0, trainable=False, name='global_step') tf.add_to_collection("init_in_init", self.global_step) self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS) for var in self.hyper.variables.values(): tf.add_to_collection("init_in_init", var) with tf.name_scope("ramps"): sigmoid_rampup_value = sigmoid_rampup(self.global_step, self.hyper['rampup_length']) sigmoid_rampdown_value = sigmoid_rampdown( self.global_step, self.hyper['rampdown_length'], self.hyper['training_length']) self.learning_rate = tf.multiply(sigmoid_rampup_value * sigmoid_rampdown_value, self.hyper['max_learning_rate'], name='learning_rate') self.adam_beta_1 = tf.add( sigmoid_rampdown_value * self.hyper['adam_beta_1_before_rampdown'], (1 - sigmoid_rampdown_value) * self.hyper['adam_beta_1_after_rampdown'], name='adam_beta_1') self.cons_coefficient = tf.multiply( sigmoid_rampup_value, self.hyper['max_consistency_cost'], name='consistency_coefficient') step_rampup_value = step_rampup(self.global_step, self.hyper['rampup_length']) self.adam_beta_2 = tf.add( (1 - step_rampup_value) * self.hyper['adam_beta_2_during_rampup'], step_rampup_value * self.hyper['adam_beta_2_after_rampup'], name='adam_beta_2') self.ema_decay = tf.add( (1 - step_rampup_value) * self.hyper['ema_decay_during_rampup'], step_rampup_value * self.hyper['ema_decay_after_rampup'], name='ema_decay') ((self.class_logits_1, self.cons_logits_1), (self.class_logits_2, self.cons_logits_2), (self.class_logits_ema, self.cons_logits_ema)) = inference( self.images, is_training=self.is_training, ema_decay=self.ema_decay, input_noise=self.hyper['input_noise'], student_dropout_probability=self. hyper['student_dropout_probability'], teacher_dropout_probability=self. hyper['teacher_dropout_probability'], normalize_input=self.hyper['normalize_input'], flip_horizontally=self.hyper['flip_horizontally'], translate=self.hyper['translate'], num_logits=self.hyper['num_logits']) with tf.name_scope("objectives"): self.mean_error_1, self.errors_1 = errors(self.class_logits_1, self.labels) self.mean_error_ema, self.errors_ema = errors( self.class_logits_ema, self.labels) self.mean_class_cost_1, self.class_costs_1 = classification_costs( self.class_logits_1, self.labels) self.mean_class_cost_ema, self.class_costs_ema = classification_costs( self.class_logits_ema, self.labels) labeled_consistency = self.hyper['apply_consistency_to_labeled'] consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency) self.mean_cons_cost_pi, self.cons_costs_pi = consistency_costs( self.cons_logits_1, self.class_logits_2, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust']) self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs( self.cons_logits_1, self.class_logits_ema, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust']) def l2_norms(matrix): l2s = tf.reduce_sum(matrix**2, axis=1) mean_l2 = tf.reduce_mean(l2s) return mean_l2, l2s self.mean_res_l2_1, self.res_l2s_1 = l2_norms(self.class_logits_1 - self.cons_logits_1) self.mean_res_l2_ema, self.res_l2s_ema = l2_norms( self.class_logits_ema - self.cons_logits_ema) self.res_costs_1 = self.hyper[ 'logit_distance_cost'] * self.res_l2s_1 self.mean_res_cost_1 = tf.reduce_mean(self.res_costs_1) self.res_costs_ema = self.hyper[ 'logit_distance_cost'] * self.res_l2s_ema self.mean_res_cost_ema = tf.reduce_mean(self.res_costs_ema) self.mean_total_cost_pi, self.total_costs_pi = total_costs( self.class_costs_1, self.cons_costs_pi, self.res_costs_1) self.mean_total_cost_mt, self.total_costs_mt = total_costs( self.class_costs_1, self.cons_costs_mt, self.res_costs_1) assert_shape(self.total_costs_pi, [3]) assert_shape(self.total_costs_mt, [3]) self.cost_to_be_minimized = tf.cond( self.hyper['ema_consistency'], lambda: self.mean_total_cost_mt, lambda: self.mean_total_cost_pi) with tf.name_scope("train_step"): update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): self.train_step_op = nn.adam_optimizer( self.cost_to_be_minimized, self.global_step, learning_rate=self.learning_rate, beta1=self.adam_beta_1, beta2=self.adam_beta_2, epsilon=self.hyper['adam_epsilon']) self.training_control = training_control(self.global_step, self.hyper['print_span'], self.hyper['evaluation_span'], self.hyper['training_length']) self.training_metrics = { "learning_rate": self.learning_rate, "adam_beta_1": self.adam_beta_1, "adam_beta_2": self.adam_beta_2, "ema_decay": self.ema_decay, "cons_coefficient": self.cons_coefficient, "train/error/1": self.mean_error_1, "train/error/ema": self.mean_error_ema, "train/class_cost/1": self.mean_class_cost_1, "train/class_cost/ema": self.mean_class_cost_ema, "train/cons_cost/pi": self.mean_cons_cost_pi, "train/cons_cost/mt": self.mean_cons_cost_mt, "train/res_cost/1": self.mean_res_cost_1, "train/res_cost/ema": self.mean_res_cost_ema, "train/total_cost/pi": self.mean_total_cost_pi, "train/total_cost/mt": self.mean_total_cost_mt, } with tf.variable_scope("validation_metrics") as metrics_scope: self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map( { "eval/error/1": streaming_mean(self.errors_1), "eval/error/ema": streaming_mean(self.errors_ema), "eval/class_cost/1": streaming_mean(self.class_costs_1), "eval/class_cost/ema": streaming_mean(self.class_costs_ema), "eval/res_cost/1": streaming_mean(self.res_costs_1), "eval/res_cost/ema": streaming_mean(self.res_costs_ema), }) metric_variables = slim.get_local_variables( scope=metrics_scope.name) self.metric_init_op = tf.variables_initializer(metric_variables) self.result_formatter = string_utils.DictFormatter( order=[ "eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt" ], default_format='{name}: {value:>10.6f}', separator=", ") self.result_formatter.add_format('error', '{name}: {value:>6.1%}') with tf.name_scope("initializers"): init_init_variables = tf.get_collection("init_in_init") train_init_variables = [ var for var in tf.global_variables() if var not in init_init_variables ] self.init_init_op = tf.variables_initializer(init_init_variables) self.train_init_op = tf.variables_initializer(train_init_variables) self.saver = tf.train.Saver() self.session = tf.Session() self.run(self.init_init_op)
def evaluate_model(config): """ Train the model using the passed in config """ ########################################################### # Create the input pipeline ########################################################### with tf.name_scope('input_pipeline'): dataset = input_utils.get_dataset(config.datadir, config.dataset, config.datasubset, num_folds=config.fold_count, fold=config.fold, holdout=True) init_op, init_feed_dict, image = input_utils.get_data( config.dataset, dataset, config.batch_size, num_epochs=config.num_epochs, num_readers=config.num_readers) images = tf.train.batch([image], config.batch_size, num_threads=config.num_preprocessing_threads, capacity=5 * config.batch_size) ########################################################### # Generate the model ########################################################### outputs = create_model(config, images, dataset) ########################################################### # Setup the evaluation metrics and summaries ########################################################### summaries = [] metrics_map = {} for loss in tf.losses.get_losses(): metrics_map[loss.op.name] = metrics.streaming_mean(loss) for metric in tf.get_collection(graph_utils.GraphKeys.METRICS): metrics_map[metric.op.name] = metrics.streaming_mean(metric) total_loss = tf.losses.get_total_loss() metrics_map[total_loss.op.name] = metrics.streaming_mean(total_loss) names_to_values, names_to_updates = metrics.aggregate_metric_map( metrics_map) # Create summaries of the metrics and print them to the screen for name, value in names_to_values.iteritems(): summary = tf.summary.scalar(name, value, collections=[]) summaries.append(tf.Print(summary, [value], name)) summaries.extend(layers.summarize_collection(tf.GraphKeys.MODEL_VARIABLES)) summaries.extend(layers.summarize_collection( graph_utils.GraphKeys.METRICS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.RNN_OUTPUTS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS)) images = input_utils.reshape_images(images, config.dataset) tiled_images = image_utils.tile_images(images) summaries.append(tf.summary.image('input_batch', tiled_images)) # Generate the canvases that lead to the final output image with tf.name_scope('canvases'): for step, canvas in enumerate(outputs): canvas = input_utils.reshape_images(canvas, config.dataset) tiled_images = image_utils.tile_images(canvas) summaries.append( tf.summary.image('step{0}'.format(step), tiled_images)) summary_op = tf.summary.merge(summaries, name='summaries') ########################################################### # Begin evaluation ########################################################### checkpoint_path = FLAGS.checkpoint_path eval_ops = tf.group(*names_to_updates.values()) hooks = [ training.SummaryAtEndHook(log_dir=FLAGS.log_dir, summary_op=summary_op), training.StopAfterNEvalsHook( math.ceil(dataset.num_samples / float(config.batch_size))) ] eval_kwargs = {} eval_fn = training.evaluate_repeatedly if FLAGS.once: if tf.gfile.IsDirectory(checkpoint_path): checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) eval_fn = training.evaluate_once else: assert tf.gfile.IsDirectory(checkpoint_path), ( 'checkpoint path must be a directory when using loop evaluation') eval_fn(checkpoint_path, hooks=hooks, eval_ops=eval_ops, **eval_kwargs)
def __init__( self, config, output_dir="./output", use_rnn=False, testing=False, use_best=False, ): self.config = config self.output_dir = output_dir self.checkpoint_path = os.path.join(self.output_dir, "checkpoint") self.best_ckpt_path = os.path.join(self.output_dir, "best_ckpt") self.weights_path = os.path.join(self.output_dir, "weights") self.log_dir = os.path.join(self.output_dir, "log") self.use_rnn = use_rnn # Placeholder with tf.variable_scope("placeholders") as scope: self.signals = tf.placeholder(dtype=tf.float32, shape=(None, self.config["input_size"], 1, 1), name='signals') self.labels = tf.placeholder(dtype=tf.int32, shape=(None, ), name='labels') self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training') if self.use_rnn: self.loss_weights = tf.placeholder(dtype=tf.float32, shape=(None, ), name='loss_weights') self.seq_lengths = tf.placeholder(dtype=tf.int32, shape=(None, ), name='seq_lengths') # Monitor global step update self.global_step = tf.Variable(0, trainable=False, name='global_step') # Monitor the number of epochs passed self.global_epoch = tf.Variable(0, trainable=False, name='global_epoch') # Build a network that receives inputs from placeholders net = self.build_cnn() if self.use_rnn: # Check whether the corresponding config is given if "n_rnn_layers" not in self.config: raise Exception("Invalid config.") # Append the RNN if needed net = self.append_rnn(net) # Softmax linear net = nn.fc("softmax_linear", net, self.config["n_classes"], bias=0.0) # Outputs self.logits = net self.preds = tf.argmax(self.logits, axis=1) # Cross-entropy loss self.loss_per_sample = tf.nn.sparse_softmax_cross_entropy_with_logits( labels=self.labels, logits=self.logits, name="loss_ce_per_sample") with tf.name_scope("loss_ce_mean") as scope: if self.use_rnn: # Weight by sequence loss_w_seq = tf.multiply(self.loss_weights, self.loss_per_sample) # Weight by class sample_weights = tf.reduce_sum( tf.multiply( tf.one_hot(indices=self.labels, depth=self.config["n_classes"]), np.asarray(self.config["class_weights"], dtype=np.float32)), 1) loss_w_class = tf.multiply(loss_w_seq, sample_weights) # Computer average loss scaled with the sequence length self.loss_ce = tf.reduce_sum(loss_w_class) / tf.reduce_sum( self.loss_weights) else: self.loss_ce = tf.reduce_mean(self.loss_per_sample) # Regularization loss self.reg_losses = self.regularization_loss() # Total loss self.loss = self.loss_ce + self.reg_losses # Metrics (used when we want to compute a metric from the output from minibatches) with tf.variable_scope("stream_metrics") as scope: self.metric_value_op, self.metric_update_op = contrib_metrics.aggregate_metric_map( { "loss": tf.metrics.mean(values=self.loss), "accuracy": tf.metrics.accuracy(labels=self.labels, predictions=self.preds), "precision": tf.metrics.precision(labels=self.labels, predictions=self.preds), "recall": tf.metrics.recall(labels=self.labels, predictions=self.preds), }) # Manually create reset operations of local vars metric_vars = contrib_slim.get_local_variables(scope=scope.name) self.metric_init_op = tf.variables_initializer(metric_vars) # Training outputs self.train_outputs = { "global_step": self.global_step, "train/loss": self.loss, "train/preds": self.preds, "train/stream_metrics": self.metric_update_op, } if self.use_rnn: self.train_outputs.update({ "train/init_state": self.init_state, "train/final_state": self.final_state, }) # Test outputs self.test_outputs = { "global_step": self.global_step, "test/loss": self.loss, "test/preds": self.preds, } if self.use_rnn: self.test_outputs.update({ "test/init_state": self.init_state, "test/final_state": self.final_state, }) # Tensoflow config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) if not testing: self.train_writer = tf.summary.FileWriter( os.path.join(self.log_dir, "train")) self.train_writer.add_graph(self.sess.graph) logger.info("Saved tensorboard graph to {}".format( self.train_writer.get_logdir())) # Optimizer if not testing: # self.lr = tf.train.exponential_decay( # learning_rate=self.config["learning_rate_decay"], # global_step=self.global_step, # decay_steps=self.config["decay_steps"], # decay_rate=self.config["decay_rate"], # staircase=False, # name="learning_rate" # ) self.lr = tf.constant(self.config["learning_rate"], dtype=tf.float32) with tf.variable_scope("optimizer") as scope: update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): # Pretraining if not self.use_rnn: self.train_step_op, self.grad_op = nn.adam_optimizer( loss=self.loss, training_variables=tf.trainable_variables(), global_step=self.global_step, # learning_rate=self.config["learning_rate"], learning_rate=self.lr, beta1=self.config["adam_beta_1"], beta2=self.config["adam_beta_2"], epsilon=self.config["adam_epsilon"], ) # Fine-tuning else: # Use different learning rates for CNN and RNN self.train_step_op, self.grad_op = nn.adam_optimizer_clip( loss=self.loss, training_variables=tf.trainable_variables(), global_step=self.global_step, # learning_rate=self.config["learning_rate"], learning_rate=self.lr, beta1=self.config["adam_beta_1"], beta2=self.config["adam_beta_2"], epsilon=self.config["adam_epsilon"], clip_value=self.config["clip_grad_value"], ) # Initializer with tf.variable_scope("initializer") as scope: # tf.trainable_variables() or tf.global_variables() self.init_global_op = tf.variables_initializer( tf.global_variables()) self.init_local_op = tf.variables_initializer(tf.local_variables()) # Saver for storing variables self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=1) self.best_saver = tf.train.Saver(tf.global_variables(), max_to_keep=1) # Initialize variables self.run([self.init_global_op, self.init_local_op]) # Restore variables (if possible) is_restore = False if use_best: if os.path.exists(self.best_ckpt_path): if os.path.isfile( os.path.join(self.best_ckpt_path, "checkpoint")): # Restore the last checkpoint latest_checkpoint = tf.train.latest_checkpoint( self.best_ckpt_path) self.saver.restore(self.sess, latest_checkpoint) logger.info("Best model restored from {}".format( latest_checkpoint)) is_restore = True else: if os.path.exists(self.checkpoint_path): if os.path.isfile( os.path.join(self.checkpoint_path, "checkpoint")): # Restore the last checkpoint latest_checkpoint = tf.train.latest_checkpoint( self.checkpoint_path) self.saver.restore(self.sess, latest_checkpoint) logger.info( "Model restored from {}".format(latest_checkpoint)) is_restore = True if not is_restore: logger.info("Model started from random weights")
def __init__(self, run_context=None): if run_context is not None: self.training_log = run_context.create_train_log('training') self.validation_log = run_context.create_train_log('validation') self.checkpoint_path = os.path.join(run_context.transient_dir, 'checkpoint') self.tensorboard_path = os.path.join(run_context.result_dir, 'tensorboard') with tf.name_scope("placeholders"): self.images = tf.placeholder(dtype=tf.float32, shape=(None, 32, 32, 3), name='images') self.labels = tf.placeholder(dtype=tf.int32, shape=(None,), name='labels') self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training') self.global_step = tf.Variable(0, trainable=False, name='global_step') tf.add_to_collection("init_in_init", self.global_step) self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS) for var in self.hyper.variables.values(): tf.add_to_collection("init_in_init", var) with tf.name_scope("ramps"): sigmoid_rampup_value = sigmoid_rampup(self.global_step, self.hyper['rampup_length']) sigmoid_rampdown_value = sigmoid_rampdown(self.global_step, self.hyper['rampdown_length'], self.hyper['training_length']) self.learning_rate = tf.multiply(sigmoid_rampup_value * sigmoid_rampdown_value, self.hyper['max_learning_rate'], name='learning_rate') self.adam_beta_1 = tf.add(sigmoid_rampdown_value * self.hyper['adam_beta_1_before_rampdown'], (1 - sigmoid_rampdown_value) * self.hyper['adam_beta_1_after_rampdown'], name='adam_beta_1') self.cons_coefficient = tf.multiply(sigmoid_rampup_value, self.hyper['max_consistency_cost'], name='consistency_coefficient') step_rampup_value = step_rampup(self.global_step, self.hyper['rampup_length']) self.adam_beta_2 = tf.add((1 - step_rampup_value) * self.hyper['adam_beta_2_during_rampup'], step_rampup_value * self.hyper['adam_beta_2_after_rampup'], name='adam_beta_2') self.ema_decay = tf.add((1 - step_rampup_value) * self.hyper['ema_decay_during_rampup'], step_rampup_value * self.hyper['ema_decay_after_rampup'], name='ema_decay') ( (self.class_logits_1, self.cons_logits_1), (self.class_logits_2, self.cons_logits_2), (self.class_logits_ema, self.cons_logits_ema) ) = inference( self.images, is_training=self.is_training, ema_decay=self.ema_decay, input_noise=self.hyper['input_noise'], student_dropout_probability=self.hyper['student_dropout_probability'], teacher_dropout_probability=self.hyper['teacher_dropout_probability'], normalize_input=self.hyper['normalize_input'], flip_horizontally=self.hyper['flip_horizontally'], translate=self.hyper['translate'], num_logits=self.hyper['num_logits']) with tf.name_scope("objectives"): self.mean_error_1, self.errors_1 = errors(self.class_logits_1, self.labels) self.mean_error_ema, self.errors_ema = errors(self.class_logits_ema, self.labels) self.mean_class_cost_1, self.class_costs_1 = classification_costs( self.class_logits_1, self.labels) self.mean_class_cost_ema, self.class_costs_ema = classification_costs( self.class_logits_ema, self.labels) labeled_consistency = self.hyper['apply_consistency_to_labeled'] consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency) self.mean_cons_cost_pi, self.cons_costs_pi = consistency_costs( self.cons_logits_1, self.class_logits_2, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust']) self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs( self.cons_logits_1, self.class_logits_ema, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust']) def l2_norms(matrix): l2s = tf.reduce_sum(matrix ** 2, axis=1) mean_l2 = tf.reduce_mean(l2s) return mean_l2, l2s self.mean_res_l2_1, self.res_l2s_1 = l2_norms(self.class_logits_1 - self.cons_logits_1) self.mean_res_l2_ema, self.res_l2s_ema = l2_norms(self.class_logits_ema - self.cons_logits_ema) self.res_costs_1 = self.hyper['logit_distance_cost'] * self.res_l2s_1 self.mean_res_cost_1 = tf.reduce_mean(self.res_costs_1) self.res_costs_ema = self.hyper['logit_distance_cost'] * self.res_l2s_ema self.mean_res_cost_ema = tf.reduce_mean(self.res_costs_ema) self.mean_total_cost_pi, self.total_costs_pi = total_costs( self.class_costs_1, self.cons_costs_pi, self.res_costs_1) self.mean_total_cost_mt, self.total_costs_mt = total_costs( self.class_costs_1, self.cons_costs_mt, self.res_costs_1) assert_shape(self.total_costs_pi, [3]) assert_shape(self.total_costs_mt, [3]) self.cost_to_be_minimized = tf.cond(self.hyper['ema_consistency'], lambda: self.mean_total_cost_mt, lambda: self.mean_total_cost_pi) with tf.name_scope("train_step"): update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): self.train_step_op = nn.adam_optimizer(self.cost_to_be_minimized, self.global_step, learning_rate=self.learning_rate, beta1=self.adam_beta_1, beta2=self.adam_beta_2, epsilon=self.hyper['adam_epsilon']) self.training_control = training_control(self.global_step, self.hyper['print_span'], self.hyper['evaluation_span'], self.hyper['training_length']) self.training_metrics = { "learning_rate": self.learning_rate, "adam_beta_1": self.adam_beta_1, "adam_beta_2": self.adam_beta_2, "ema_decay": self.ema_decay, "cons_coefficient": self.cons_coefficient, "train/error/1": self.mean_error_1, "train/error/ema": self.mean_error_ema, "train/class_cost/1": self.mean_class_cost_1, "train/class_cost/ema": self.mean_class_cost_ema, "train/cons_cost/pi": self.mean_cons_cost_pi, "train/cons_cost/mt": self.mean_cons_cost_mt, "train/res_cost/1": self.mean_res_cost_1, "train/res_cost/ema": self.mean_res_cost_ema, "train/total_cost/pi": self.mean_total_cost_pi, "train/total_cost/mt": self.mean_total_cost_mt, } with tf.variable_scope("validation_metrics") as metrics_scope: self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map({ "eval/error/1": streaming_mean(self.errors_1), "eval/error/ema": streaming_mean(self.errors_ema), "eval/class_cost/1": streaming_mean(self.class_costs_1), "eval/class_cost/ema": streaming_mean(self.class_costs_ema), "eval/res_cost/1": streaming_mean(self.res_costs_1), "eval/res_cost/ema": streaming_mean(self.res_costs_ema), }) metric_variables = slim.get_local_variables(scope=metrics_scope.name) self.metric_init_op = tf.variables_initializer(metric_variables) self.result_formatter = string_utils.DictFormatter( order=["eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"], default_format='{name}: {value:>10.6f}', separator=", ") self.result_formatter.add_format('error', '{name}: {value:>6.1%}') with tf.name_scope("initializers"): init_init_variables = tf.get_collection("init_in_init") train_init_variables = [ var for var in tf.global_variables() if var not in init_init_variables ] self.init_init_op = tf.variables_initializer(init_init_variables) self.train_init_op = tf.variables_initializer(train_init_variables) self.saver = tf.train.Saver() self.session = tf.Session() self.run(self.init_init_op)
def main(unused_argv): tf.logging.set_verbosity(tf.logging.INFO) dataset = data_generator.Dataset( # 获取验证集图片数据 dataset_name=FLAGS.dataset, # 数据集名称 cityscapes 默认为 pascal_voc_seg split_name=FLAGS.eval_split, # 指定带有val的tfrecorder数据集 默认为“val” dataset_dir=FLAGS.dataset_dir, # 数据集目录 tfrecoder文件的数据集目录 batch_size=FLAGS.eval_batch_size, # 每个batch包含的image数量 默认为1 crop_size=[int(sz) for sz in FLAGS.eval_crop_size], # 评估时crop_size 默认为513,513 min_resize_value=FLAGS.min_resize_value, # 默认为None max_resize_value=FLAGS.max_resize_value, # 默认为None resize_factor=FLAGS.resize_factor, # 默认为None model_variant=FLAGS.model_variant, # 模型的变体 本次训练为 xception_65 num_readers=2, # 并行读取图片的数量 is_training=False, # 不训练 should_shuffle=False, # 不将输入的数据随机打乱 should_repeat=False) # 不一直重复 tf.gfile.MakeDirs(FLAGS.eval_logdir) # 创建评估目录 tf.logging.info('Evaluating on %s set', FLAGS.eval_split) with tf.Graph().as_default(): samples = dataset.get_one_shot_iterator().get_next() # 获取一次迭代的验证集数据 ''' samples: {'image_name': <tf.Tensor 'IteratorGetNext:2' shape=(?,) dtype=string>, 'width': <tf.Tensor 'IteratorGetNext:5' shape=(?,) dtype=int64>, 'image': <tf.Tensor 'IteratorGetNext:1' shape=(?, 1024, 2048, 3) dtype=float32>, 'height': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=int64>, 'label': <tf.Tensor 'IteratorGetNext:3' shape=(?, 1024, 2048, 1) dtype=int32>, 'original_image': <tf.Tensor 'IteratorGetNext:4' shape=(?, ?, ?, 3) dtype=uint8>} ''' model_options = common.ModelOptions( # 模型参数 outputs_to_num_classes={ common.OUTPUT_TYPE: dataset.num_of_classes }, # {semantic: 19} crop_size=[int(sz) for sz in FLAGS.eval_crop_size], # 1024,2048 atrous_rates=FLAGS.atrous_rates, # 6,12,18 output_stride=FLAGS.output_stride) # 16 # Set shape in order for tf.contrib.tfprof.model_analyzer to work properly. samples[common.IMAGE].set_shape( # 设置形状 [ FLAGS.eval_batch_size, # 默认为1 int(FLAGS.eval_crop_size[0]), int(FLAGS.eval_crop_size[1]), 3 ]) if tuple(FLAGS.eval_scales) == (1.0, ): # 默认 评估尺度为1 tf.logging.info('Performing single-scale test.') predictions = model.predict_labels( samples[common.IMAGE], model_options, # 进行每个像素点预测 image_pyramid=FLAGS.image_pyramid) ''' predictions: {'semantic': <tf.Tensor 'ArgMax:0' shape=(1, 1024, 2048) dtype=int64>, 'semantic_prob': <tf.Tensor 'Softmax:0' shape=(1, 1024, 2048, 19) dtype=float32>} ''' else: tf.logging.info('Performing multi-scale test.') if FLAGS.quantize_delay_step >= 0: raise ValueError( 'Quantize mode is not supported with multi-scale test.') predictions = model.predict_labels_multi_scale( samples[common.IMAGE], model_options=model_options, eval_scales=FLAGS.eval_scales, add_flipped_images=FLAGS.add_flipped_images) predictions = predictions[common.OUTPUT_TYPE] predictions = tf.reshape(predictions, shape=[-1]) # 预测标签 labels = tf.reshape(samples[common.LABEL], shape=[-1]) # 真实标签 weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label)) # 各标签权重 # Set ignore_label regions to label 0, because metrics.mean_iou requires # range of labels = [0, dataset.num_classes). Note the ignore_label regions # are not evaluated since the corresponding regions contain weights = 0. labels = tf.where(tf.equal(labels, dataset.ignore_label), tf.zeros_like(labels), labels) predictions_tag = 'miou' # MIoU predictions_tag1 = 'accuracy_pixel' # 像素精度 for eval_scale in FLAGS.eval_scales: # 默认为单尺度[1.0] predictions_tag += '_' + str(eval_scale) predictions_tag1 += '_' + str(eval_scale) if FLAGS.add_flipped_images: # 默认为False 不设置左右翻转来评估模型 predictions_tag += '_flipped' predictions_tag1 += '_flipped' # Define the evaluation metric. metric_map = {} num_classes = dataset.num_of_classes # 19 metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou( labels=labels, predictions=predictions, num_classes=num_classes, weights=weights) ''' metric_map: {'eval/miou_1.0_overall': (<tf.Tensor 'mean_iou/Select_1:0' shape=() dtype=float32>, <tf.Tensor 'mean_iou/AssignAdd:0' shape=(19, 19) dtype=float64_ref>)} ''' metric_map['eval/%s_overall_accuracy_' % predictions_tag] = tf.metrics.accuracy( labels=labels, predictions=predictions, weights=weights) # IoU for each class. ''' tf.one_hot(indices, depth, on_value=None, off_value=None, axis=None, dtype=None, name=None) Returns a one-hot tensor. ndices表示输入的多个数值,通常是矩阵形式;depth表示输出的尺寸。 ''' one_hot_predictions = tf.one_hot(predictions, num_classes) one_hot_predictions = tf.reshape(one_hot_predictions, [-1, num_classes]) # 预测输出的one_hot one_hot_labels = tf.one_hot(labels, num_classes) one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes]) # 真实label的one_hot for c in range(num_classes): predictions_tag_c = '%s_class_%d' % (predictions_tag, c ) # miou_1.0_class_c predictions_tag_c1 = '%s_class_%d' % (predictions_tag1, c) tp, tp_op = tf.metrics.true_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fp, fp_op = tf.metrics.false_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fn, fn_op = tf.metrics.false_negatives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) tn, tn_op = tf.metrics.true_negatives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op) iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp), tf.constant(np.NaN)) ap = tf.where(tf.greater(tp + fn, 0.0), (tp + tn) / (tp + tn + fn + fp), tf.constant(np.NaN)) metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op) metric_map['eval/%s' % predictions_tag_c1] = (ap, tp_fp_fn_op) (metrics_to_values, metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map) ''' (metrics_to_values, metrics_to_updates): ({'eval/miou_1.0_class_5': <tf.Tensor 'Select_6:0' shape=() dtype=float32>, 'eval/miou_1.0_class_18': <tf.Tensor 'Select_19:0' shape=() dtype=float32>, 'eval/miou_1.0_class_13': <tf.Tensor 'Select_14:0' shape=() dtype=float32>, 'eval/miou_1.0_class_1': <tf.Tensor 'Select_2:0' shape=() dtype=float32>, 'eval/miou_1.0_overall': <tf.Tensor 'mean_iou/Select_1:0' shape=() dtype=float32>, 'eval/miou_1.0_class_17': <tf.Tensor 'Select_18:0' shape=() dtype=float32>, 'eval/miou_1.0_class_8': <tf.Tensor 'Select_9:0' shape=() dtype=float32>, 'eval/miou_1.0_class_2': <tf.Tensor 'Select_3:0' shape=() dtype=float32>, 'eval/miou_1.0_class_0': <tf.Tensor 'Select_1:0' shape=() dtype=float32>, 'eval/miou_1.0_class_3': <tf.Tensor 'Select_4:0' shape=() dtype=float32>, 'eval/miou_1.0_class_14': <tf.Tensor 'Select_15:0' shape=() dtype=float32>, 'eval/miou_1.0_class_11': <tf.Tensor 'Select_12:0' shape=() dtype=float32>, 'eval/miou_1.0_class_6': <tf.Tensor 'Select_7:0' shape=() dtype=float32>, 'eval/miou_1.0_class_15': <tf.Tensor 'Select_16:0' shape=() dtype=float32>, 'eval/miou_1.0_class_4': <tf.Tensor 'Select_5:0' shape=() dtype=float32>, 'eval/miou_1.0_class_9': <tf.Tensor 'Select_10:0' shape=() dtype=float32>, 'eval/miou_1.0_class_16': <tf.Tensor 'Select_17:0' shape=() dtype=float32>, 'eval/miou_1.0_class_7': <tf.Tensor 'Select_8:0' shape=() dtype=float32>, 'eval/miou_1.0_class_10': <tf.Tensor 'Select_11:0' shape=() dtype=float32>, 'eval/miou_1.0_class_12': <tf.Tensor 'Select_13:0' shape=() dtype=float32>}, {'eval/miou_1.0_class_5': <tf.Operation 'group_deps_5' type=NoOp>, 'eval/miou_1.0_class_18': <tf.Operation 'group_deps_18' type=NoOp>, 'eval/miou_1.0_class_13': <tf.Operation 'group_deps_13' type=NoOp>, 'eval/miou_1.0_class_1': <tf.Operation 'group_deps_1' type=NoOp>, 'eval/miou_1.0_overall': <tf.Tensor 'mean_iou/AssignAdd:0' shape=(19, 19) dtype=float64_ref>, 'eval/miou_1.0_class_17': <tf.Operation 'group_deps_17' type=NoOp>, 'eval/miou_1.0_class_8': <tf.Operation 'group_deps_8' type=NoOp>, 'eval/miou_1.0_class_2': <tf.Operation 'group_deps_2' type=NoOp>, 'eval/miou_1.0_class_0': <tf.Operation 'group_deps' type=NoOp>, 'eval/miou_1.0_class_3': <tf.Operation 'group_deps_3' type=NoOp>, 'eval/miou_1.0_class_14': <tf.Operation 'group_deps_14' type=NoOp>, 'eval/miou_1.0_class_11': <tf.Operation 'group_deps_11' type=NoOp>, 'eval/miou_1.0_class_6': <tf.Operation 'group_deps_6' type=NoOp>, 'eval/miou_1.0_class_15': <tf.Operation 'group_deps_15' type=NoOp>, 'eval/miou_1.0_class_4': <tf.Operation 'group_deps_4' type=NoOp>, 'eval/miou_1.0_class_9': <tf.Operation 'group_deps_9' type=NoOp>, 'eval/miou_1.0_class_16': <tf.Operation 'group_deps_16' type=NoOp>, 'eval/miou_1.0_class_7': <tf.Operation 'group_deps_7' type=NoOp>, 'eval/miou_1.0_class_10': <tf.Operation 'group_deps_10' type=NoOp>, 'eval/miou_1.0_class_12': <tf.Operation 'group_deps_12' type=NoOp>}) ''' ''' tf.Print(input, data, message=None, first_n=None, summarize=None, name=None) 最低要求两个输入,input和data,input是需要打印的变量的名字,data要求是一个list,里面包含要打印的内容。 ''' summary_ops = [] for metric_name, metric_value in six.iteritems(metrics_to_values): op = tf.summary.scalar(metric_name, metric_value) # 显示标量信息 op = tf.Print(op, [metric_value], metric_name) summary_ops.append(op) summary_op = tf.summary.merge(summary_ops) summary_hook = contrib_training.SummaryAtEndHook( log_dir=FLAGS.eval_logdir, summary_op=summary_op) hooks = [summary_hook] num_eval_iters = None if FLAGS.max_number_of_evaluations > 0: # 为0 暂不考虑 num_eval_iters = FLAGS.max_number_of_evaluations if FLAGS.quantize_delay_step >= 0: # -1 暂不考虑 contrib_quantize.create_eval_graph() contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer. TRAINABLE_VARS_PARAMS_STAT_OPTIONS) contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS) contrib_training.evaluate_repeatedly( checkpoint_dir=FLAGS.checkpoint_dir, master=FLAGS.master, eval_ops=list(metrics_to_updates.values()), max_number_of_evaluations=num_eval_iters, hooks=hooks, eval_interval_secs=FLAGS.eval_interval_secs)
def __init__(self, run_context=None): self.name = "Tweet Data Class" if run_context is not None: self.training_log = run_context.create_train_log('training') self.validation_log = run_context.create_train_log('validation') self.checkpoint_path = os.path.join(run_context.transient_dir, 'checkpoint') self.tensorboard_path = os.path.join(run_context.result_dir, 'tensorboard') with tf.name_scope("placeholders"): self.tweets = tf.placeholder(dtype=tf.float32, shape=(None, 500), name='tweets') self.labels = tf.placeholder(dtype=tf.int32, shape=(None,), name='labels') self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training') self.global_step = tf.Variable(0, trainable=False, name='global_step') tf.add_to_collection("init_in_init", self.global_step) self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS) for var in self.hyper.variables.values(): tf.add_to_collection("init_in_init", var) with tf.name_scope("ramps"): # Ramp-up and ramp-down has been removed for simplicity # self.learning_rate = tf.constant(self.hyper['max_learning_rate'], dtype = tf.float32) # self.adam_beta_1 = tf.constant(self.hyper['adam_beta_1_after_rampdown'], dtype = tf.float32) # self.cons_coefficient = tf.constant(self.hyper['max_consistency_cost'], dtype = tf.float32) # self.adam_beta_2 = tf.constant(self.hyper['adam_beta_2_after_rampup'], dtype = tf.float32) # self.ema_decay = tf.constant(self.hyper['ema_decay_after_rampup'], dtype = tf.float32) # self.learning_rate =self.DEFAULT_HYPERPARAMS['max_learning_rate'] # self.adam_beta_1 = self.DEFAULT_HYPERPARAMS['adam_beta_1_after_rampdown'] # self.cons_coefficient = self.DEFAULT_HYPERPARAMS['max_consistency_cost'] # self.adam_beta_2 = self.DEFAULT_HYPERPARAMS['adam_beta_2_after_rampup'] # self.ema_decay = self.DEFAULT_HYPERPARAMS['ema_decay_after_rampup'] sigmoid_rampup_value = sigmoid_rampup(self.global_step, self.hyper['rampup_length']) sigmoid_rampdown_value = sigmoid_rampdown(self.global_step, self.hyper['rampdown_length'], self.hyper['training_length']) self.learning_rate = tf.multiply(sigmoid_rampup_value * sigmoid_rampdown_value, self.hyper['max_learning_rate'], name='learning_rate') self.adam_beta_1 = tf.add(sigmoid_rampdown_value * self.hyper['adam_beta_1_before_rampdown'], (1 - sigmoid_rampdown_value) * self.hyper['adam_beta_1_after_rampdown'], name='adam_beta_1') self.cons_coefficient = tf.multiply(sigmoid_rampup_value, self.hyper['max_consistency_cost'], name='consistency_coefficient') step_rampup_value = step_rampup(self.global_step, self.hyper['rampup_length']) self.adam_beta_2 = tf.add((1 - step_rampup_value) * self.hyper['adam_beta_2_during_rampup'], step_rampup_value * self.hyper['adam_beta_2_after_rampup'], name='adam_beta_2') self.ema_decay = tf.add((1 - step_rampup_value) * self.hyper['ema_decay_during_rampup'], step_rampup_value * self.hyper['ema_decay_after_rampup'], name='ema_decay') # below is where the interesting stuff happens, mostly. # Inference is a function which creates the towers and sets up the different logits for the two models ( (self.class_logits_1, self.cons_logits_1), (self.class_logits_2, self.cons_logits_2), (self.class_logits_ema, self.cons_logits_ema) ) = inference( self.tweets, is_training=self.is_training, ema_decay=self.ema_decay, input_noise=self.hyper['input_noise'], hidden_dims = self.DEFAULT_HYPERPARAMS['hidden_dims'], student_dropout_probability=self.hyper['student_dropout_probability'], teacher_dropout_probability=self.hyper['teacher_dropout_probability'], num_logits=self.hyper['num_logits']) with tf.name_scope("objectives"): # something weird is done with errors for unlabeled examples. # I think errors are only calculated for labeled, but you don't calculate it for unlabeled, so it is NaN for unlabeled self.mean_error_1, self.errors_1 = errors(self.class_logits_1, self.labels) self.mean_error_ema, self.errors_ema = errors(self.class_logits_ema, self.labels) # where we calculate classification costs. # the cost_1 should be for student and ema is for teacher self.mean_class_cost_1, self.class_costs_1 = classification_costs( self.class_logits_1, self.labels) self.mean_class_cost_ema, self.class_costs_ema = classification_costs( self.class_logits_ema, self.labels) labeled_consistency = self.hyper['apply_consistency_to_labeled'] consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency) self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs( self.cons_logits_1, self.class_logits_ema, self.cons_coefficient, consistency_mask) def l2_norms(matrix): l2s = tf.reduce_sum(matrix ** 2, axis=1) mean_l2 = tf.reduce_mean(l2s) return mean_l2, l2s self.mean_res_l2_1, self.res_l2s_1 = l2_norms(self.class_logits_1 - self.cons_logits_1) self.mean_res_l2_ema, self.res_l2s_ema = l2_norms(self.class_logits_ema - self.cons_logits_ema) # mean total cost is what you are optimizng. self.mean_total_cost_mt, self.total_costs_mt = total_costs( self.class_costs_1, self.cons_costs_mt) assert_shape(self.total_costs_mt, [2]) self.cost_to_be_minimized = self.mean_total_cost_mt with tf.name_scope("train_step"): update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): self.train_step_op = adam_optimizer(self.cost_to_be_minimized, self.global_step, learning_rate=self.learning_rate, beta1=self.adam_beta_1, beta2=self.adam_beta_2, epsilon=self.hyper['adam_epsilon']) # TODO do we really need this? self.training_control = training_control(self.global_step, self.hyper['print_span'], self.hyper['evaluation_span'], self.hyper['training_length']) self.training_metrics = { # NOTE these should not need training, since we don't do ramp-up and ramp-down "learning_rate": self.learning_rate, "adam_beta_1": self.adam_beta_1, "adam_beta_2": self.adam_beta_2, "ema_decay": self.ema_decay, "cons_coefficient": self.cons_coefficient, "train/error/1": self.mean_error_1, "train/error/ema": self.mean_error_ema, "train/class_cost/1": self.mean_class_cost_1, "train/class_cost/ema": self.mean_class_cost_ema, "train/cons_cost/mt": self.mean_cons_cost_mt, "train/total_cost/mt": self.mean_total_cost_mt, } # TODO not sure what streaming mean does? with tf.variable_scope("validation_metrics") as metrics_scope: self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map({ "eval/error/1": streaming_mean(self.errors_1), "eval/error/ema": streaming_mean(self.errors_ema), "eval/class_cost/1": streaming_mean(self.class_costs_1), "eval/class_cost/ema": streaming_mean(self.class_costs_ema), }) metric_variables = slim.get_local_variables(scope=metrics_scope.name) self.metric_init_op = tf.variables_initializer(metric_variables) # TODO string utils just formats dictionary results in a nice way for logging, not needed? self.result_formatter = string_utils.DictFormatter( order=["eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"], default_format='{name}: {value:>10.6f}', separator=", ") self.result_formatter.add_format('error', '{name}: {value:>6.1%}') with tf.name_scope("initializers"): init_init_variables = tf.get_collection("init_in_init") train_init_variables = [ var for var in tf.global_variables() if var not in init_init_variables ] self.init_init_op = tf.variables_initializer(init_init_variables) print("Train init variables:") for var in train_init_variables: print(var) self.train_init_op = tf.variables_initializer(train_init_variables) self.saver = tf.train.Saver() self.session = tf.Session() self.run(self.init_init_op)
def build(): """Builds the Tensorflow graph.""" inputs, labels, lengths = None, None, None if mode in ('train', 'eval'): if isinstance(no_event_label, numbers.Number): label_shape = [] else: label_shape = [len(no_event_label)] inputs, labels, lengths = magenta.common.get_padded_batch( sequence_example_file_paths, hparams.batch_size, input_size, label_shape=label_shape, shuffle=mode == 'train') elif mode == 'generate': inputs = tf.placeholder(tf.float32, [hparams.batch_size, None, input_size]) if isinstance(encoder_decoder, magenta.music.OneHotIndexEventSequenceEncoderDecoder): expanded_inputs = tf.one_hot( tf.cast(tf.squeeze(inputs, axis=-1), tf.int64), encoder_decoder.input_depth) else: expanded_inputs = inputs dropout_keep_prob = 1.0 if mode == 'generate' else hparams.dropout_keep_prob if hparams.use_cudnn: outputs, initial_state, final_state = make_cudnn( expanded_inputs, hparams.rnn_layer_sizes, hparams.batch_size, mode, dropout_keep_prob=dropout_keep_prob, residual_connections=hparams.residual_connections) else: cell = make_rnn_cell( hparams.rnn_layer_sizes, dropout_keep_prob=dropout_keep_prob, attn_length=hparams.attn_length, residual_connections=hparams.residual_connections) initial_state = cell.zero_state(hparams.batch_size, tf.float32) outputs, final_state = tf.nn.dynamic_rnn( cell, inputs, sequence_length=lengths, initial_state=initial_state, swap_memory=True) outputs_flat = magenta.common.flatten_maybe_padded_sequences( outputs, lengths) if isinstance(num_classes, numbers.Number): num_logits = num_classes else: num_logits = sum(num_classes) logits_flat = contrib_layers.linear(outputs_flat, num_logits) if mode in ('train', 'eval'): labels_flat = magenta.common.flatten_maybe_padded_sequences( labels, lengths) if isinstance(num_classes, numbers.Number): softmax_cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits( labels=labels_flat, logits=logits_flat) predictions_flat = tf.argmax(logits_flat, axis=1) else: logits_offsets = np.cumsum([0] + num_classes) softmax_cross_entropy = [] predictions = [] for i in range(len(num_classes)): softmax_cross_entropy.append( tf.nn.sparse_softmax_cross_entropy_with_logits( labels=labels_flat[:, i], logits=logits_flat[ :, logits_offsets[i]:logits_offsets[i + 1]])) predictions.append( tf.argmax(logits_flat[ :, logits_offsets[i]:logits_offsets[i + 1]], axis=1)) predictions_flat = tf.stack(predictions, 1) correct_predictions = tf.to_float( tf.equal(labels_flat, predictions_flat)) event_positions = tf.to_float(tf.not_equal(labels_flat, no_event_label)) no_event_positions = tf.to_float(tf.equal(labels_flat, no_event_label)) # Compute the total number of time steps across all sequences in the # batch. For some models this will be different from the number of RNN # steps. def batch_labels_to_num_steps(batch_labels, lengths): num_steps = 0 for labels, length in zip(batch_labels, lengths): num_steps += encoder_decoder.labels_to_num_steps(labels[:length]) return np.float32(num_steps) num_steps = tf.py_func( batch_labels_to_num_steps, [labels, lengths], tf.float32) if mode == 'train': loss = tf.reduce_mean(softmax_cross_entropy) perplexity = tf.exp(loss) accuracy = tf.reduce_mean(correct_predictions) event_accuracy = ( tf.reduce_sum(correct_predictions * event_positions) / tf.reduce_sum(event_positions)) no_event_accuracy = ( tf.reduce_sum(correct_predictions * no_event_positions) / tf.reduce_sum(no_event_positions)) loss_per_step = tf.reduce_sum(softmax_cross_entropy) / num_steps perplexity_per_step = tf.exp(loss_per_step) optimizer = tf.train.AdamOptimizer(learning_rate=hparams.learning_rate) train_op = contrib_slim.learning.create_train_op( loss, optimizer, clip_gradient_norm=hparams.clip_norm) tf.add_to_collection('train_op', train_op) vars_to_summarize = { 'loss': loss, 'metrics/perplexity': perplexity, 'metrics/accuracy': accuracy, 'metrics/event_accuracy': event_accuracy, 'metrics/no_event_accuracy': no_event_accuracy, 'metrics/loss_per_step': loss_per_step, 'metrics/perplexity_per_step': perplexity_per_step, } elif mode == 'eval': vars_to_summarize, update_ops = contrib_metrics.aggregate_metric_map({ 'loss': tf.metrics.mean(softmax_cross_entropy), 'metrics/accuracy': tf.metrics.accuracy(labels_flat, predictions_flat), 'metrics/per_class_accuracy': tf.metrics.mean_per_class_accuracy(labels_flat, predictions_flat, num_classes), 'metrics/event_accuracy': tf.metrics.recall(event_positions, correct_predictions), 'metrics/no_event_accuracy': tf.metrics.recall(no_event_positions, correct_predictions), 'metrics/loss_per_step': tf.metrics.mean( tf.reduce_sum(softmax_cross_entropy) / num_steps, weights=num_steps), }) for updates_op in update_ops.values(): tf.add_to_collection('eval_ops', updates_op) # Perplexity is just exp(loss) and doesn't need its own update op. vars_to_summarize['metrics/perplexity'] = tf.exp( vars_to_summarize['loss']) vars_to_summarize['metrics/perplexity_per_step'] = tf.exp( vars_to_summarize['metrics/loss_per_step']) for var_name, var_value in six.iteritems(vars_to_summarize): tf.summary.scalar(var_name, var_value) tf.add_to_collection(var_name, var_value) elif mode == 'generate': temperature = tf.placeholder(tf.float32, []) if isinstance(num_classes, numbers.Number): softmax_flat = tf.nn.softmax( tf.div(logits_flat, tf.fill([num_classes], temperature))) softmax = tf.reshape( softmax_flat, [hparams.batch_size, -1, num_classes]) else: logits_offsets = np.cumsum([0] + num_classes) softmax = [] for i in range(len(num_classes)): sm = tf.nn.softmax( tf.div( logits_flat[:, logits_offsets[i]:logits_offsets[i + 1]], tf.fill([num_classes[i]], temperature))) sm = tf.reshape(sm, [hparams.batch_size, -1, num_classes[i]]) softmax.append(sm) tf.add_to_collection('inputs', inputs) tf.add_to_collection('temperature', temperature) tf.add_to_collection('softmax', softmax) # Flatten state tuples for metagraph compatibility. for state in tf_nest.flatten(initial_state): tf.add_to_collection('initial_state', state) for state in tf_nest.flatten(final_state): tf.add_to_collection('final_state', state)
def __init__(self, run_context=None, hyper_dict={}): #inilization of hyper for i in hyper_dict: assert i in self.hyper, "Wrong hyper dict '{}'!".format(i) self.hyper[i] = hyper_dict[i] # inilize bg noise input if self.hyper['bg_noise']: self.bg_noise_input = tf.convert_to_tensor(self.hyper['bg_noise_input'],dtype=tf.float32) else: self.bg_noise_input = tf.convert_to_tensor(np.zeros((32,32)),dtype=tf.float32) # inilization model print('{} is initliazed!'.format(self.hyper['cnn'])) self.cnn = getattr(model,self.hyper['cnn']) if run_context is not None: self.training_log = run_context.create_train_log('training') self.validation_log = run_context.create_train_log('validation') self.checkpoint_path = os.path.join(run_context.transient_dir, 'checkpoint') self.tensorboard_path = os.path.join(run_context.result_dir, 'tensorboard') with tf.name_scope("placeholders"): self.images = tf.placeholder(dtype=tf.float32, shape=(None,) + self.hyper['input_dim'], name='images') self.labels = tf.placeholder(dtype=tf.int32, shape=(None,) + self.hyper['label_dim'], name='labels') self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training') self.global_step = tf.Variable(0, trainable=False, name='global_step') tf.add_to_collection("init_in_init", self.global_step) with tf.name_scope("ramps"): self.learning_rate, self.cons_coefficient, \ self.adam_beta_1, self.adam_beta_2, \ self.ema_decay = ramp_value(self.global_step,self.hyper) ( self.class_logits_1, self.class_logits_ema ) = self.inference( self.images, is_training=self.is_training, ema_decay=self.ema_decay, input_noise=self.hyper['input_noise'], student_dropout_probability=self.hyper['student_dropout_probability'], teacher_dropout_probability=self.hyper['teacher_dropout_probability'], normalize_input=self.hyper['normalize_input'], flip_horizontally=self.hyper['flip_horizontally'], translate=self.hyper['translate']) with tf.name_scope("objectives"): self.mean_error_1, self.errors_1 = errors(self.class_logits_1, self.labels,sig = self.hyper['sig']) self.mean_error_ema, self.errors_ema = errors(self.class_logits_ema, self.labels,sig = self.hyper['sig']) self.mean_class_cost_1, self.class_costs_1 = classification_costs( self.class_logits_1, self.labels,sig = self.hyper['sig']) self.mean_class_cost_ema, self.class_costs_ema = classification_costs( self.class_logits_ema, self.labels,sig = self.hyper['sig']) labeled_consistency = self.hyper['apply_consistency_to_labeled'] consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency) self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs( self.class_logits_1, self.class_logits_ema, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust'],sig = 'softmax') self.mean_total_cost_mt, self.total_costs_mt = total_costs( self.class_costs_1, self.cons_costs_mt) self.cost_to_be_minimized = self.mean_total_cost_mt with tf.name_scope("train_step"): update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): if self.hyper['optimizer']=='adam': self.train_step_op = nn.adam_optimizer(self.cost_to_be_minimized, self.global_step, learning_rate=self.learning_rate, beta1=self.adam_beta_1, beta2=self.adam_beta_2, epsilon=self.hyper['adam_epsilon']) elif self.hyper['optimizer']=='sgd': self.train_step_op = nn.sgd_optimizer(self.cost_to_be_minimized, self.global_step, learning_rate=self.hyper['max_learning_rate']) else: assert False, 'Wrong optimizer input!' self.training_metrics = { "learning_rate": self.learning_rate, "adam_beta_1": self.adam_beta_1, "adam_beta_2": self.adam_beta_2, "ema_decay": self.ema_decay, "cons_coefficient": self.cons_coefficient, "train/error/1": self.mean_error_1, "train/error/ema": self.mean_error_ema, "train/class_cost/1": self.mean_class_cost_1, "train/class_cost/ema": self.mean_class_cost_ema, "train/cons_cost/mt": self.mean_cons_cost_mt, "train/total_cost/mt": self.mean_total_cost_mt, } with tf.variable_scope("validation_metrics") as metrics_scope: self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map({ "eval/error/1": streaming_mean(self.errors_1), "eval/error/ema": streaming_mean(self.errors_ema), "eval/class_cost/1": streaming_mean(self.class_costs_1), "eval/class_cost/ema": streaming_mean(self.class_costs_ema), }) metric_variables = slim.get_local_variables(scope=metrics_scope.name) self.metric_init_op = tf.variables_initializer(metric_variables) self.result_formatter = string_utils.DictFormatter( order=["eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"], default_format='{name}: {value:>10.6f}', separator=", ") self.result_formatter.add_format('error', '{name}: {value:>6.1%}') with tf.name_scope("initializers"): init_init_variables = tf.get_collection("init_in_init") train_init_variables = [ var for var in tf.global_variables() if var not in init_init_variables ] self.init_init_op = tf.variables_initializer(init_init_variables) self.train_init_op = tf.variables_initializer(train_init_variables) self.saver = tf.train.Saver() config = tf.ConfigProto() config.gpu_options.allow_growth = True self.session = tf.Session(config=config) self.run(self.init_init_op) self.save_tensorboard_graph()
def main(unused_argv): tf.logging.set_verbosity(tf.logging.INFO) dataset = data_generator.Dataset( dataset_name=FLAGS.dataset, split_name=FLAGS.eval_split, dataset_dir=FLAGS.dataset_dir, batch_size=FLAGS.eval_batch_size, crop_size=[int(sz) for sz in FLAGS.eval_crop_size], min_resize_value=FLAGS.min_resize_value, max_resize_value=FLAGS.max_resize_value, resize_factor=FLAGS.resize_factor, model_variant=FLAGS.model_variant, num_readers=2, is_training=False, should_shuffle=False, should_repeat=False, with_cls=True, cls_only=False, output_valid=True) tf.gfile.MakeDirs(FLAGS.eval_logdir) tf.logging.info('Evaluating on %s set', FLAGS.eval_split) with tf.Graph().as_default(): samples = dataset.get_one_shot_iterator().get_next() model_options = common.ModelOptions( outputs_to_num_classes={ common.OUTPUT_TYPE: dataset.num_of_classes }, crop_size=[int(sz) for sz in FLAGS.eval_crop_size], atrous_rates=FLAGS.atrous_rates, output_stride=FLAGS.output_stride) # Set shape in order for tf.contrib.tfprof.model_analyzer to work properly. samples[common.IMAGE].set_shape([ FLAGS.eval_batch_size, int(FLAGS.eval_crop_size[0]), int(FLAGS.eval_crop_size[1]), 3 ]) if tuple(FLAGS.eval_scales) == (1.0, ): tf.logging.info('Performing single-scale test.') predictions = model.predict_labels( samples[common.IMAGE], model_options, image_pyramid=FLAGS.image_pyramid) else: tf.logging.info('Performing multi-scale test.') raise NotImplementedError('Multi-scale is not supported yet!') metric_map = {} ## Extract cls logits if FLAGS.weakly: _, end_points = feature_extractor.extract_features( samples[common.IMAGE], output_stride=model_options.output_stride, multi_grid=model_options.multi_grid, model_variant=model_options.model_variant, depth_multiplier=model_options.depth_multiplier, divisible_by=model_options.divisible_by, reuse=tf.AUTO_REUSE, is_training=False, preprocessed_images_dtype=model_options. preprocessed_images_dtype, global_pool=True, num_classes=dataset.num_of_classes - 1) # ResNet beta version has an additional suffix in FLAGS.model_variant, but # it shares the same variable names with original version. Add a special # handling here for beta version ResNet. logits = end_points['{}/logits'.format( FLAGS.model_variant).replace('_beta', '')] logits = tf.reshape(logits, [-1, dataset.num_of_classes - 1]) cls_pred = tf.sigmoid(logits) # Multi-label classification evaluation cls_label = samples['cls_label'] cls_pred = tf.cast(tf.greater_equal(cls_pred, 0.5), tf.int32) ## For classification metric_map['eval/cls_overall'] = tf.metrics.accuracy( labels=cls_label, predictions=cls_pred) metric_map['eval/cls_precision'] = tf.metrics.precision( labels=cls_label, predictions=cls_pred) metric_map['eval/cls_recall'] = tf.metrics.recall( labels=cls_label, predictions=cls_pred) ## For segmentation branch eval predictions = predictions[common.OUTPUT_TYPE] predictions = tf.reshape(predictions, shape=[-1]) labels = tf.reshape(samples[common.LABEL], shape=[-1]) weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label)) # Set ignore_label regions to label 0, because metrics.mean_iou requires # range of labels = [0, dataset.num_classes). Note the ignore_label regions # are not evaluated since the corresponding regions contain weights = 0. labels = tf.where(tf.equal(labels, dataset.ignore_label), tf.zeros_like(labels), labels) predictions_tag = 'miou' # Define the evaluation metric. num_classes = dataset.num_of_classes ## For segmentation metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou( labels=labels, predictions=predictions, num_classes=num_classes, weights=weights) # IoU for each class. one_hot_predictions = tf.one_hot(predictions, num_classes) one_hot_predictions = tf.reshape(one_hot_predictions, [-1, num_classes]) one_hot_labels = tf.one_hot(labels, num_classes) one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes]) for c in range(num_classes): predictions_tag_c = '%s_class_%d' % (predictions_tag, c) tp, tp_op = tf.metrics.true_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fp, fp_op = tf.metrics.false_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fn, fn_op = tf.metrics.false_negatives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op) iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp), tf.constant(np.NaN)) metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op) (metrics_to_values, metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map) summary_ops = [] for metric_name, metric_value in six.iteritems(metrics_to_values): op = tf.summary.scalar(metric_name, metric_value) op = tf.Print(op, [metric_value], metric_name) summary_ops.append(op) summary_op = tf.summary.merge(summary_ops) summary_hook = contrib_training.SummaryAtEndHook( log_dir=FLAGS.eval_logdir, summary_op=summary_op) hooks = [summary_hook] num_eval_iters = None if FLAGS.max_number_of_evaluations > 0: num_eval_iters = FLAGS.max_number_of_evaluations if FLAGS.quantize_delay_step >= 0: contrib_quantize.create_eval_graph() contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer. TRAINABLE_VARS_PARAMS_STAT_OPTIONS) contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS) contrib_training.evaluate_repeatedly( checkpoint_dir=FLAGS.checkpoint_dir, master=FLAGS.master, eval_ops=list(metrics_to_updates.values()), max_number_of_evaluations=num_eval_iters, hooks=hooks, eval_interval_secs=FLAGS.eval_interval_secs)
def main(unused_argv): tf.logging.set_verbosity(tf.logging.INFO) dataset = data_generator.Dataset( dataset_name=FLAGS.dataset, split_name=FLAGS.eval_split, dataset_dir=FLAGS.dataset_dir, batch_size=FLAGS.eval_batch_size, crop_size=[int(sz) for sz in FLAGS.eval_crop_size], min_resize_value=FLAGS.min_resize_value, max_resize_value=FLAGS.max_resize_value, resize_factor=FLAGS.resize_factor, model_variant=FLAGS.model_variant, num_readers=2, is_training=False, should_shuffle=False, should_repeat=False) tf.gfile.MakeDirs(FLAGS.eval_logdir) tf.logging.info('Evaluating on %s set', FLAGS.eval_split) with tf.Graph().as_default(): samples = dataset.get_one_shot_iterator().get_next() model_options = common.ModelOptions( model_name=FLAGS.model_name, outputs_to_num_classes={ common.OUTPUT_TYPE: dataset.num_of_classes }, crop_size=[int(sz) for sz in FLAGS.eval_crop_size], atrous_rates=FLAGS.atrous_rates, output_stride=FLAGS.output_stride) # Set shape in order for tf.contrib.tfprof.model_analyzer to work properly. samples[common.IMAGE].set_shape([ FLAGS.eval_batch_size, int(FLAGS.eval_crop_size[0]), int(FLAGS.eval_crop_size[1]), 3 ]) if tuple(FLAGS.eval_scales) == (1.0, ): tf.logging.info('Performing single-scale test.') predictions = model.predict_labels( samples[common.IMAGE], model_options, image_pyramid=FLAGS.image_pyramid) else: tf.logging.info('Performing multi-scale test.') if FLAGS.quantize_delay_step >= 0: raise ValueError( 'Quantize mode is not supported with multi-scale test.') predictions = model.predict_labels_multi_scale( samples[common.IMAGE], model_options=model_options, eval_scales=FLAGS.eval_scales, add_flipped_images=FLAGS.add_flipped_images) predictions = predictions[common.OUTPUT_TYPE] predictions = tf.reshape(predictions, shape=[-1]) labels = tf.reshape(samples[common.LABEL], shape=[-1]) weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label)) # Set ignore_label regions to label 0, because metrics.mean_iou requires # range of labels = [0, dataset.num_classes). Note the ignore_label regions # are not evaluated since the corresponding regions contain weights = 0. labels = tf.where(tf.equal(labels, dataset.ignore_label), tf.zeros_like(labels), labels) predictions_tag = 'miou' for eval_scale in FLAGS.eval_scales: predictions_tag += '_' + str(eval_scale) if FLAGS.add_flipped_images: predictions_tag += '_flipped' # Define the evaluation metric. metric_map = {} num_classes = dataset.num_of_classes metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou( labels=labels, predictions=predictions, num_classes=num_classes, weights=weights) # IoU for each class. one_hot_predictions = tf.one_hot(predictions, num_classes) one_hot_predictions = tf.reshape(one_hot_predictions, [-1, num_classes]) one_hot_labels = tf.one_hot(labels, num_classes) one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes]) for c in range(num_classes): predictions_tag_c = '%s_class_%d' % (predictions_tag, c) tp, tp_op = tf.metrics.true_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fp, fp_op = tf.metrics.false_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fn, fn_op = tf.metrics.false_negatives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op) iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp), tf.constant(np.NaN)) metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op) (metrics_to_values, metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map) summary_ops = [] for metric_name, metric_value in six.iteritems(metrics_to_values): op = tf.summary.scalar(metric_name, metric_value) op = tf.Print(op, [metric_value], metric_name) summary_ops.append(op) summary_op = tf.summary.merge(summary_ops) summary_hook = contrib_training.SummaryAtEndHook( log_dir=FLAGS.eval_logdir, summary_op=summary_op) hooks = [summary_hook] num_eval_iters = None if FLAGS.max_number_of_evaluations > 0: num_eval_iters = FLAGS.max_number_of_evaluations if FLAGS.quantize_delay_step >= 0: contrib_quantize.create_eval_graph() contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer. TRAINABLE_VARS_PARAMS_STAT_OPTIONS) contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS) contrib_training.evaluate_repeatedly( checkpoint_dir=FLAGS.checkpoint_dir, master=FLAGS.master, eval_ops=list(metrics_to_updates.values()), max_number_of_evaluations=num_eval_iters, hooks=hooks, eval_interval_secs=FLAGS.eval_interval_secs)
def build(): """Builds the Tensorflow graph.""" inputs, lengths = None, None if mode in ('train', 'eval'): inputs, _, lengths = magenta.common.get_padded_batch( sequence_example_file_paths, hparams.batch_size, input_size, shuffle=mode == 'train') elif mode == 'generate': inputs = tf.placeholder(tf.float32, [hparams.batch_size, None, input_size]) cell = events_rnn_graph.make_rnn_cell( hparams.rnn_layer_sizes, dropout_keep_prob=hparams.dropout_keep_prob if mode == 'train' else 1.0, attn_length=hparams.attn_length, residual_connections=hparams.residual_connections) rnn_nade = RnnNade(cell, num_dims=input_size, num_hidden=hparams.nade_hidden_units) if mode in ('train', 'eval'): log_probs, cond_probs = rnn_nade.log_prob(inputs, lengths) inputs_flat = tf.to_float( magenta.common.flatten_maybe_padded_sequences(inputs, lengths)) predictions_flat = tf.to_float(tf.greater_equal(cond_probs, .5)) if mode == 'train': loss = tf.reduce_mean(-log_probs) perplexity = tf.reduce_mean(tf.exp(log_probs)) correct_predictions = tf.to_float( tf.equal(inputs_flat, predictions_flat)) accuracy = tf.reduce_mean(correct_predictions) precision = (tf.reduce_sum(inputs_flat * predictions_flat) / tf.reduce_sum(predictions_flat)) recall = (tf.reduce_sum(inputs_flat * predictions_flat) / tf.reduce_sum(inputs_flat)) optimizer = tf.train.AdamOptimizer( learning_rate=hparams.learning_rate) train_op = contrib_slim.learning.create_train_op( loss, optimizer, clip_gradient_norm=hparams.clip_norm) tf.add_to_collection('train_op', train_op) vars_to_summarize = { 'loss': loss, 'metrics/perplexity': perplexity, 'metrics/accuracy': accuracy, 'metrics/precision': precision, 'metrics/recall': recall, } elif mode == 'eval': vars_to_summarize, update_ops = contrib_metrics.aggregate_metric_map( { 'loss': tf.metrics.mean(-log_probs), 'metrics/perplexity': tf.metrics.mean(tf.exp(log_probs)), 'metrics/accuracy': tf.metrics.accuracy(inputs_flat, predictions_flat), 'metrics/precision': tf.metrics.precision(inputs_flat, predictions_flat), 'metrics/recall': tf.metrics.recall(inputs_flat, predictions_flat), }) for updates_op in update_ops.values(): tf.add_to_collection('eval_ops', updates_op) precision = vars_to_summarize['metrics/precision'] recall = vars_to_summarize['metrics/precision'] f1_score = tf.where( tf.greater(precision + recall, 0), 2 * ((precision * recall) / (precision + recall)), 0) vars_to_summarize['metrics/f1_score'] = f1_score for var_name, var_value in vars_to_summarize.items(): tf.summary.scalar(var_name, var_value) tf.add_to_collection(var_name, var_value) elif mode == 'generate': initial_state = rnn_nade.zero_state(hparams.batch_size) final_state = rnn_nade.steps(inputs, initial_state) samples, log_prob = rnn_nade.sample_single(initial_state) tf.add_to_collection('inputs', inputs) tf.add_to_collection('sample', samples) tf.add_to_collection('log_prob', log_prob) # Flatten state tuples for metagraph compatibility. for state in tf_nest.flatten(initial_state): tf.add_to_collection('initial_state', state) for state in tf_nest.flatten(final_state): tf.add_to_collection('final_state', state)