def testWeight1(self): # Generate summaries for showing PR curves in TensorBoard. with tf.Session() as sess: summary.op( tag='tag_bar', labels=tf.constant([True, False, False, True, True, False]), predictions=tf.constant([0.81, 0.8, 0.61, 0.6, 0.4, 0.2]), num_thresholds=10) merged_summary_op = tf.summary.merge_all() foo_directory = os.path.join(self.logdir, 'foo') writer = tf.summary.FileWriter(foo_directory, sess.graph) writer.add_summary(sess.run(merged_summary_op), 1) writer.close() # Create a multiplexer for reading the data we just wrote. multiplexer = event_multiplexer.EventMultiplexer() multiplexer.AddRunsFromDirectory(self.logdir) multiplexer.Reload() # Verify that the metadata was correctly written. accumulator = multiplexer.GetAccumulator('foo') tag_content_dict = accumulator.PluginTagToContent('pr_curve') self.assertListEqual(['tag_bar/tag_bar'], list(tag_content_dict.keys())) # Parse the data within the JSON string and set the proto's fields. plugin_data = pr_curve_pb2.PrCurvePluginData() json_format.Parse(tag_content_dict['tag_bar/tag_bar'], plugin_data) self.assertEqual(10, plugin_data.num_thresholds) # Test the summary contents. tensor_events = accumulator.Tensors('tag_bar/tag_bar') self.assertEqual(1, len(tensor_events)) tensor_event = tensor_events[0] self.assertEqual(1, tensor_event.step) tensor_nd_array = tf.make_ndarray(tensor_event.tensor_proto) np.testing.assert_allclose( [ # True positives. [3.0, 3.0, 3.0, 3.0, 2.0, 2.0, 1.0, 1.0, 0.0, 0.0], # False positives. [3.0, 3.0, 2.0, 2.0, 2.0, 2.0, 1.0, 1.0, 0.0, 0.0], # True negatives. [0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 3.0, 3.0], # False negatives. [0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 2.0, 2.0, 3.0, 3.0], # Precision. [0.5, 0.5, 0.6, 0.6, 0.5, 0.5, 0.5, 0.5, 1.0, 1.0], # Recall. [1.0, 1.0, 1.0, 1.0, 2 / 3, 2 / 3, 1 / 3, 1 / 3, 0.0, 0.0], ], tensor_nd_array)
def test_matches_op_with_updates(self): predictions = tf.constant([0.2, 0.4, 0.5, 0.6, 0.8], dtype=tf.float32) labels = tf.constant([False, True, True, False, True], dtype=tf.bool) pr_curve, update_op = summary.streaming_op( name="pr_curve", predictions=predictions, labels=labels, num_thresholds=10, ) complete_predictions = tf.tile(predictions, [3]) complete_labels = tf.tile(labels, [3]) expected_pr_curve = summary.op( name="pr_curve", predictions=complete_predictions, labels=complete_labels, num_thresholds=10, ) with self.test_session() as sess: sess.run(tf.compat.v1.local_variables_initializer()) sess.run([update_op]) sess.run([update_op]) sess.run([update_op]) proto = self.pb_via_op(pr_curve) expected_proto = self.pb_via_op(expected_pr_curve) # Need to detect and fix the automatic _1 appended to second namespace. self.assertEqual(proto.value[0].tag, "pr_curve/pr_curves") self.assertEqual(expected_proto.value[0].tag, "pr_curve_1/pr_curves") expected_proto.value[0].tag = "pr_curve/pr_curves" self.assertProtoEquals(expected_proto, proto)
def compute_and_check_summary_pb(self, name, labels, predictions, num_thresholds, weights=None, display_name=None, description=None, feed_dict=None): """Use both `op` and `pb` to get a summary, asserting equality. Returns: a `Summary` protocol buffer """ labels_tensor = tf.constant(labels) predictions_tensor = tf.constant(predictions) weights_tensor = None if weights is None else tf.constant(weights) op = summary.op(name=name, labels=labels_tensor, predictions=predictions_tensor, num_thresholds=num_thresholds, weights=weights_tensor, display_name=display_name, description=description) pb = self.normalize_summary_pb( summary.pb(name=name, labels=labels, predictions=predictions, num_thresholds=num_thresholds, weights=weights, display_name=display_name, description=description)) pb_via_op = self.normalize_summary_pb( self.pb_via_op(op, feed_dict=feed_dict)) self.assertProtoEquals(pb, pb_via_op) return pb
def test_add_pr_curve_plugin(live_mock_server, test_settings): tf.compat.v1.disable_v2_behavior() wandb.init(sync_tensorboard=True, settings=test_settings) summ_op = pr_curve_plugin_summary.op( name="test_pr", labels=tf.constant([True, False, True]), predictions=tf.constant([0.7, 0.2, 0.3]), num_thresholds=5, ) merged_summary_op = tf.compat.v1.summary.merge([summ_op]) sess = tf.compat.v1.Session() writer = tf.compat.v1.summary.FileWriter(wandb.run.dir, sess.graph) merged_summary = sess.run(merged_summary_op) writer.add_summary(merged_summary, 0) writer.close() wandb.finish() server_ctx = live_mock_server.get_ctx() assert ("test_pr/pr_curves" in server_ctx["config"][2]["_wandb"]["value"]["visualize"].keys()) assert (server_ctx["config"][2]["_wandb"]["value"]["visualize"] ["test_pr/pr_curves"] == PR_CURVE_PANEL_CONFIG) wandb.tensorboard.unpatch()
def testMatchesOp(self): predictions = tf.constant([0.2, 0.4, 0.5, 0.6, 0.8], dtype=tf.float32) labels = tf.constant([False, True, True, False, True], dtype=tf.bool) pr_curve, update_op = summary.streaming_op(tag='pr_curve', predictions=predictions, labels=labels, num_thresholds=10) expected_pr_curve = summary.op(tag='pr_curve', predictions=predictions, labels=labels, num_thresholds=10) with self.test_session() as sess: sess.run(tf.local_variables_initializer()) sess.run([update_op]) proto = self.pb_via_op(pr_curve) expected_proto = self.pb_via_op(expected_pr_curve) # Need to detect and fix the automatic _1 appended to second namespace. self.assertEqual(proto.value[0].tag, 'pr_curve/pr_curves') self.assertEqual(expected_proto.value[0].tag, 'pr_curve_1/pr_curves') expected_proto.value[0].tag = 'pr_curve/pr_curves' self.assertProtoEquals(expected_proto, proto)
def get_train_hooks(self, labels, logits, alpha=None): nclass = self.config['data']['task']['classes']['num'] metric_tensor = { "batch_accuracy": metrics_lib.accuracy(logits, labels), 'global_step': tf.train.get_or_create_global_step(), } if nclass > 100: logging.info('Too many classes, disable confusion matrix in train: %d' % (nclass)) else: metric_tensor['batch_confusion'] = \ metrics_lib.confusion_matrix(logits, labels, nclass) tf.summary.scalar('batch_accuracy', metric_tensor['batch_accuracy']) if alpha: metric_tensor.update({"alignment": alpha}) # plot PR curve true_label_bool = tf.cast(labels, tf.bool) softmax = tf.nn.softmax(logits) pr_summary.op( name='pr_curve_train_batch', labels=true_label_bool, predictions=softmax[:, -1], num_thresholds=16, weights=None) train_hooks = [ tf.train.StepCounterHook( every_n_steps=100, every_n_secs=None, output_dir=None, summary_writer=None), tf.train.FinalOpsHook( final_ops=[tf.train.get_or_create_global_step()], final_ops_feed_dict=None), tf.train.LoggingTensorHook( tensors=metric_tensor, every_n_iter=100, every_n_secs=None, at_end=False, formatter=None), ] return train_hooks
def set_model(self, model): super(VisualizeTensorBoard, self).set_model(model) if self.pr_curve: predictions = self.model._feed_outputs[0] labels = tf.cast(self.model._feed_targets[0], tf.bool) self.pr_summary = pr_summary.op( name='pr_curve', predictions=predictions, labels=labels, display_name='Precision-Recall Curve')
def set_model(self, model): super(PRTensorBoard, self).set_model(model) if self.pr_curve: # Get the prediction and label tensor placeholders. predictions = self.model._feed_outputs[0] labels = tf.cast(self.model._feed_targets[0], tf.bool) # Create the PR summary OP. self.pr_summary = pr_summary.op(name='pr_curve', predictions=predictions, labels=labels, display_name='Precision-Recall Curve')
def log_pr_curve(self, tag, pred_probability, labels, step=0, N = 10000): labels = labels.astype('int32') # If values are images, then image dimensions must be flattened with the batch dimension if len(pred_probability.shape) > 2: Nb = pred_probability.shape[0] Nc = pred_probability.shape[1] Ns = np.prod(pred_probability.shape[2:]) pred_probability = np.reshape(pred_probability, [Nb, Nc, Ns]) pred_probability = np.transpose(pred_probability, [0, 2, 1]) pred_probability = np.reshape(pred_probability, [Nb * Ns, Nc]) labels = labels.flatten() # Remove ignore-labels keep = np.logical_not(labels == -100) pred_probability = pred_probability[keep, :] labels = labels[keep] #Get size of array Nb = pred_probability.shape[0] Nc = pred_probability.shape[1] # Decimate if Nb > N: inds = np.linspace(0, Nb-1, N).astype('int64') labels = labels[inds] pred_probability = pred_probability[inds,:] Nb = pred_probability.shape[0] # Make one-hot bool array of true labels tmp = np.zeros([Nb, Nc]) for i in range(Nb): if labels[i] >= 0: tmp[i, labels[i]] = 1 labels = tmp.astype('bool') #Add summary to dict if tag not in self.summaries.keys(): self.summaries[tag] = pr_summary.op(name=tag, labels=self.placeholder_pr_curve_labels, predictions=self.placeholder_pr_curve_predictions, num_thresholds=21) #Make summary merged_summary_op = tf.summary.merge([self.summaries[tag]]) with self._tf_session() as sess: merged_summary = sess.run(merged_summary_op, feed_dict={self.placeholder_pr_curve_labels: labels, self.placeholder_pr_curve_predictions: pred_probability}) #Write self.writer.add_summary(merged_summary, step)
def set_model(self, model): super().set_model(model) if self.layout_summary: self.writer.add_summary(self.layout_summary) if self.pr_curve: for class_idx in range(self.num_classes): predictions = self.model._feed_outputs[0][..., class_idx] labels = tf.cast(self.model._feed_targets[0][..., class_idx], tf.bool) summary_op = pr_summary.op( name='pr_curve_class_' + str(class_idx), predictions=predictions, labels=labels, display_name='Precision-Recall (Class ' + str(class_idx) + ')') self.pr_summary.append(summary_op) if self.tfpn: predictions = self.model._feed_outputs[0] labels = tf.cast(self.model._feed_targets[0], tf.bool) _, precision = tf.metrics.precision(labels, predictions) _, recall = tf.metrics.recall(labels, predictions) _, tp = tf.metrics.true_positives(labels, predictions) _, fn = tf.metrics.false_negatives(labels, predictions) _, fp = tf.metrics.false_positives(labels, predictions) _, f1 = tf.contrib.metrics.f1_score(labels, predictions) # TODO this recalcs tp,fn,fp. TODO utilise existing results self.precision_summary = scalar_summary.op(name='precision', data=precision) self.recall_summary = scalar_summary.op(name='recall', data=recall) self.f1_summary = scalar_summary.op(name='f1', data=f1) self.fp_summary = scalar_summary.op(name='fp', data=fp) self.tp_summary = scalar_summary.op(name='tp', data=tp) self.fn_summary = scalar_summary.op(name='fn', data=fn) self.merged = tf.summary.merge_all()
def start_runs(logdir, steps, run_name, thresholds, mask_every_other_prediction=False): """Generate a PR curve with precision and recall evenly weighted. Arguments: logdir: The directory into which to store all the runs' data. steps: The number of steps to run for. run_name: The name of the run. thresholds: The number of thresholds to use for PR curves. mask_every_other_prediction: Whether to mask every other prediction by alternating weights between 0 and 1. """ tf.compat.v1.reset_default_graph() tf.compat.v1.set_random_seed(42) # Create a normal distribution layer used to generate true color labels. distribution = tf.compat.v1.distributions.Normal(loc=0., scale=142.) # Sample the distribution to generate colors. Lets generate different numbers # of each color. The first dimension is the count of examples. # The calls to sample() are given fixed random seed values that are "magic" # in that they correspond to the default seeds for those ops when the PR # curve test (which depends on this code) was written. We've pinned these # instead of continuing to use the defaults since the defaults are based on # node IDs from the sequence of nodes added to the graph, which can silently # change when this code or any TF op implementations it uses are modified. # TODO(nickfelt): redo the PR curve test to avoid reliance on random seeds. # Generate reds. number_of_reds = 100 true_reds = tf.clip_by_value( tf.concat([ 255 - tf.abs(distribution.sample([number_of_reds, 1], seed=11)), tf.abs(distribution.sample([number_of_reds, 2], seed=34)) ], axis=1), 0, 255) # Generate greens. number_of_greens = 200 true_greens = tf.clip_by_value( tf.concat([ tf.abs(distribution.sample([number_of_greens, 1], seed=61)), 255 - tf.abs(distribution.sample([number_of_greens, 1], seed=82)), tf.abs(distribution.sample([number_of_greens, 1], seed=105)) ], axis=1), 0, 255) # Generate blues. number_of_blues = 150 true_blues = tf.clip_by_value( tf.concat([ tf.abs(distribution.sample([number_of_blues, 2], seed=132)), 255 - tf.abs(distribution.sample([number_of_blues, 1], seed=153)) ], axis=1), 0, 255) # Assign each color a vector of 3 booleans based on its true label. labels = tf.concat([ tf.tile(tf.constant([[True, False, False]]), (number_of_reds, 1)), tf.tile(tf.constant([[False, True, False]]), (number_of_greens, 1)), tf.tile(tf.constant([[False, False, True]]), (number_of_blues, 1)), ], axis=0) # We introduce 3 normal distributions. They are used to predict whether a # color falls under a certain class (based on distances from corners of the # color triangle). The distributions vary per color. We have the distributions # narrow over time. initial_standard_deviations = [v + FLAGS.steps for v in (158, 200, 242)] iteration = tf.compat.v1.placeholder(tf.int32, shape=[]) red_predictor = tf.compat.v1.distributions.Normal( loc=0., scale=tf.cast(initial_standard_deviations[0] - iteration, dtype=tf.float32)) green_predictor = tf.compat.v1.distributions.Normal( loc=0., scale=tf.cast(initial_standard_deviations[1] - iteration, dtype=tf.float32)) blue_predictor = tf.compat.v1.distributions.Normal( loc=0., scale=tf.cast(initial_standard_deviations[2] - iteration, dtype=tf.float32)) # Make predictions (assign 3 probabilities to each color based on each color's # distance to each of the 3 corners). We seek double the area in the right # tail of the normal distribution. examples = tf.concat([true_reds, true_greens, true_blues], axis=0) probabilities_colors_are_red = (1 - red_predictor.cdf( tf.norm(tensor=examples - tf.constant([255., 0, 0]), axis=1))) * 2 probabilities_colors_are_green = (1 - green_predictor.cdf( tf.norm(tensor=examples - tf.constant([0, 255., 0]), axis=1))) * 2 probabilities_colors_are_blue = (1 - blue_predictor.cdf( tf.norm(tensor=examples - tf.constant([0, 0, 255.]), axis=1))) * 2 predictions = (probabilities_colors_are_red, probabilities_colors_are_green, probabilities_colors_are_blue) # This is the crucial piece. We write data required for generating PR curves. # We create 1 summary per class because we create 1 PR curve per class. for i, color in enumerate(('red', 'green', 'blue')): description = ( 'The probabilities used to create this PR curve are ' 'generated from a normal distribution. Its standard ' 'deviation is initially %0.0f and decreases over time.' % initial_standard_deviations[i]) weights = None if mask_every_other_prediction: # Assign a weight of 0 to every even-indexed prediction. Odd-indexed # predictions are assigned a default weight of 1. consecutive_indices = tf.reshape( tf.range(tf.size(input=predictions[i])), tf.shape(input=predictions[i])) weights = tf.cast(consecutive_indices % 2, dtype=tf.float32) summary.op(name=color, labels=labels[:, i], predictions=predictions[i], num_thresholds=thresholds, weights=weights, display_name='classifying %s' % color, description=description) merged_summary_op = tf.compat.v1.summary.merge_all() events_directory = os.path.join(logdir, run_name) sess = tf.compat.v1.Session() writer = tf.compat.v1.summary.FileWriter(events_directory, sess.graph) for step in xrange(steps): feed_dict = { iteration: step, } merged_summary = sess.run(merged_summary_op, feed_dict=feed_dict) writer.add_summary(merged_summary, step) writer.close()
def main(experiment_name, list_experiments=False, load_and_evaluate_ckpt=None, config_file=None, ckpt_file=None, gpu_device='/gpu:0'): """Create a tensorflow worker to run experiments in your DB.""" if list_experiments: exps = db.list_experiments() print '_' * 30 print 'Initialized experiments:' print '_' * 30 for l in exps: print l.values()[0] print '_' * 30 if len(exps) == 0: print 'No experiments found.' else: print 'You can add to the DB with: '\ 'python prepare_experiments.py --experiment=%s' % \ exps[0].values()[0] return if experiment_name is None: print 'No experiment specified. Pulling one out of the DB.' experiment_name = db.get_experiment_name() # Prepare to run the model config = Config() condition_label = '%s_%s' % (experiment_name, py_utils.get_dt_stamp()) experiment_label = '%s' % (experiment_name) log = logger.get(os.path.join(config.log_dir, condition_label)) experiment_dict = experiments.experiments()[experiment_name]() config = add_to_config(d=experiment_dict, config=config) # Globals config.load_and_evaluate_ckpt = load_and_evaluate_ckpt config, exp_params = process_DB_exps( experiment_name=experiment_name, log=log, config=config) # Update config w/ DB params config = np.load(config_file).item() dataset_module = py_utils.import_module(model_dir=config.dataset_info, dataset=config.dataset) dataset_module = dataset_module.data_processing() # hardcoded class name train_data, train_means_image, train_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=dataset_module.folds.keys()[1], # TODO: SEARCH FOR INDEX. log=log) val_data, val_means_image, val_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=dataset_module.folds.keys()[0], log=log) # Initialize output folders dir_list = { 'checkpoints': os.path.join(config.checkpoints, condition_label), 'summaries': os.path.join(config.summaries, condition_label), 'condition_evaluations': os.path.join(config.condition_evaluations, condition_label), 'experiment_evaluations': os.path.join( # DEPRECIATED config.experiment_evaluations, experiment_label), 'visualization': os.path.join(config.visualizations, condition_label), 'weights': os.path.join(config.condition_evaluations, condition_label, 'weights') } [py_utils.make_dir(v) for v in dir_list.values()] # Prepare data loaders on the cpu if all(isinstance(i, list) for i in config.data_augmentations): if config.data_augmentations: config.data_augmentations = py_utils.flatten_list( config.data_augmentations, log) config.epochs = 1 config.shuffle = False with tf.device('/cpu:0'): train_images, train_labels = data_loader.inputs( dataset=train_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_train, resize_output=config.resize_output) if hasattr(config, 'val_augmentations'): val_augmentations = config.val_augmentations else: val_augmentations = config.data_augmentations val_images, val_labels = data_loader.inputs( dataset=val_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=['resize_and_crop'], num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_val, resize_output=config.resize_output) log.info('Created tfrecord dataloader tensors.') # Load model specification struct_name = config.model_struct.split(os.path.sep)[-1] try: model_dict = py_utils.import_module( dataset=struct_name, model_dir=os.path.join('models', 'structs', experiment_name).replace(os.path.sep, '.')) except IOError: print 'Could not find the model structure: %s in folder %s' % ( struct_name, experiment_name) # Inject model_dict with hyperparameters if requested model_dict.layer_structure = hp_opt_utils.inject_model_with_hps( layer_structure=model_dict.layer_structure, exp_params=exp_params) # Prepare model on GPU with tf.device(gpu_device): with tf.variable_scope('cnn') as scope: # Normalize labels if needed if 'normalize_labels' in exp_params.keys(): if exp_params['normalize_labels'] == 'zscore': train_labels -= train_means_label['mean'] train_labels /= train_means_label['std'] log.info('Z-scoring labels.') elif exp_params['normalize_labels'] == 'mean': train_labels -= train_means_label['mean'] log.info('Mean-centering labels.') # Training model if len(dataset_module.output_size) == 2: log.warning('Found > 1 dimension for your output size.' 'Converting to a scalar.') dataset_module.output_size = np.prod( dataset_module.output_size) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None model = model_utils.model_class( mean=train_means_image, training=True, output_size=dataset_module.output_size) train_scores, model_summary = model.build( data=train_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') eval_graph = tf.Graph() with eval_graph.as_default(): with eval_graph.gradient_override_map({'selu': 'GradLRP'}): train_grad_images = tf.gradients( train_scores[0] * tf.cast(train_labels, tf.float32), train_images)[0] log.info('Built training model.') log.debug(json.dumps(model_summary, indent=4), verbose=0) print_model_architecture(model_summary) # Check the shapes of labels and scores if not isinstance(train_scores, list): if len(train_scores.get_shape()) != len( train_labels.get_shape()): train_shape = train_scores.get_shape().as_list() label_shape = train_labels.get_shape().as_list() if len(train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_labels = tf.expand_dims(train_labels, axis=-1) elif len(train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_scores = tf.expand_dims(train_scores, axis=-1) # Prepare the loss function train_loss, _ = loss_utils.loss_interpreter( logits=train_scores, # TODO labels=train_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(train_loss, list): for lidx, tl in enumerate(train_loss): tf.summary.scalar('training_loss_%s' % lidx, tl) train_loss = tf.add_n(train_loss) else: tf.summary.scalar('training_loss', train_loss) # Add weight decay if requested if len(model.regularizations) > 0: train_loss = loss_utils.wd_loss( regularizations=model.regularizations, loss=train_loss, wd_penalty=config.regularization_strength) train_op = loss_utils.optimizer_interpreter( loss=train_loss, lr=config.lr, optimizer=config.optimizer, constraints=config.optimizer_constraints, model=model) log.info('Built training loss function.') # Add a score for the training set train_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO: Attach to exp cnfg pred=train_scores, # TODO labels=train_labels) # Add aux scores if requested train_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: train_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=train_scores, labels=train_labels)[0] # TODO: Fix for multiloss # Prepare remaining tensorboard summaries if len(train_images.get_shape()) == 4: tf_fun.image_summaries(train_images, tag='Training images') if len(train_labels.get_shape()) > 2: tf_fun.image_summaries(train_labels, tag='Training_targets') tf_fun.image_summaries(train_scores, tag='Training_predictions') if isinstance(train_accuracy, list): for tidx, ta in enumerate(train_accuracy): tf.summary.scalar('training_accuracy_%s' % tidx, ta) else: tf.summary.scalar('training_accuracy', train_accuracy) if config.pr_curve: if isinstance(train_scores, list): for pidx, train_score in enumerate(train_scores): train_label = train_labels[:, pidx] pr_summary.op( tag='training_pr_%s' % pidx, predictions=tf.cast( tf.argmax(train_score, axis=-1), tf.float32), labels=tf.cast(train_label, tf.bool), display_name='training_precision_recall_%s' % pidx) else: pr_summary.op(tag='training_pr', predictions=tf.cast( tf.argmax(train_scores, axis=-1), tf.float32), labels=tf.cast(train_labels, tf.bool), display_name='training_precision_recall') log.info('Added training summaries.') # Validation model scope.reuse_variables() val_model = model_utils.model_class( mean=train_means_image, # Normalize with train data training=False, # False, output_size=dataset_module.output_size) val_scores, _ = val_model.build( # Ignore summary data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') eval_graph = tf.Graph() with eval_graph.as_default(): with eval_graph.gradient_override_map({'selu': 'GradLRP'}): val_grad_images = tf.gradients( val_scores[0] * tf.cast(val_labels, tf.float32), val_images)[0] log.info('Built validation model.') # Check the shapes of labels and scores if not isinstance(train_scores, list): if len(val_scores.get_shape()) != len(val_labels.get_shape()): val_shape = val_scores.get_shape().as_list() val_label_shape = val_labels.get_shape().as_list() if len(val_shape) == 2 and len( val_label_shape) == 1 and val_shape[-1] == 1: val_labels = tf.expand_dims(val_labels, axis=-1) if len(val_shape) == 2 and len( val_label_shape) == 1 and val_shape[-1] == 1: val_scores = tf.expand_dims(val_scores, axis=-1) val_loss, _ = loss_utils.loss_interpreter( logits=val_scores, labels=val_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(val_loss, list): for lidx, tl in enumerate(val_loss): tf.summary.scalar('validation_loss_%s' % lidx, tl) val_loss = tf.add_n(val_loss) else: tf.summary.scalar('validation_loss', val_loss) # Add a score for the validation set val_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO pred=val_scores, labels=val_labels) # Add aux scores if requested val_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: val_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=val_scores, labels=val_labels)[0] # TODO: Fix for multiloss # Prepare tensorboard summaries if len(val_images.get_shape()) == 4: tf_fun.image_summaries(val_images, tag='Validation') if len(val_labels.get_shape()) > 2: tf_fun.image_summaries(val_labels, tag='Validation_targets') tf_fun.image_summaries(val_scores, tag='Validation_predictions') if isinstance(val_accuracy, list): for vidx, va in enumerate(val_accuracy): tf.summary.scalar('validation_accuracy_%s' % vidx, va) else: tf.summary.scalar('validation_accuracy', val_accuracy) if config.pr_curve: if isinstance(val_scores, list): for pidx, val_score in enumerate(val_scores): val_label = val_labels[:, pidx] pr_summary.op( tag='validation_pr_%s' % pidx, predictions=tf.cast(tf.argmax(val_score, axis=-1), tf.float32), labels=tf.cast(val_label, tf.bool), display_name='validation_precision_recall_%s' % pidx) else: pr_summary.op(tag='validation_pr', predictions=tf.cast( tf.argmax(val_scores, axis=-1), tf.float32), labels=tf.cast(val_labels, tf.bool), display_name='validation_precision_recall') log.info('Added validation summaries.') # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) # Set up exemplar threading coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # Create dictionaries of important training and validation information train_dict = { 'train_loss': train_loss, 'train_images': train_images, 'train_labels': train_labels, 'train_op': train_op, 'train_scores': train_scores, 'train_grad_images': train_grad_images } val_dict = { 'val_loss': val_loss, 'val_images': val_images, 'val_labels': val_labels, 'val_scores': val_scores, 'val_grad_images': val_grad_images } if isinstance(train_accuracy, list): for tidx, (ta, va) in enumerate(zip(train_accuracy, val_accuracy)): train_dict['train_accuracy_%s' % tidx] = ta val_dict['val_accuracy_%s' % tidx] = va else: train_dict['train_accuracy_0'] = train_accuracy val_dict['val_accuracy_0'] = val_accuracy if load_and_evaluate_ckpt is not None: # Remove the train operation and add a ckpt pointer del train_dict['train_op'] if hasattr(dataset_module, 'aux_score'): # Attach auxillary scores to tensor dicts for m in dataset_module.aux_scores: train_dict['train_aux_%s' % m] = train_aux[m] val_dict['val_aux_%s' % m] = val_aux[m] # Start training loop checkpoint_dir = dir_list['checkpoints'] step = 0 train_losses, train_accs, train_aux, timesteps = {}, {}, {}, {} val_scores, val_aux, val_labels, val_grads = {}, {}, {}, {} train_images, val_images = {}, {} train_scores, train_labels = {}, {} train_aux_check = np.any(['aux_score' in k for k in train_dict.keys()]) val_aux_check = np.any(['aux_score' in k for k in val_dict.keys()]) # Restore model saver.restore(sess, ckpt_file) # Start evaluation try: while not coord.should_stop(): start_time = time.time() train_vars = sess.run(train_dict.values()) it_train_dict = { k: v for k, v in zip(train_dict.keys(), train_vars) } duration = time.time() - start_time train_losses[step] = it_train_dict['train_loss'] train_accs[step] = it_train_dict['train_accuracy_0'] train_images[step] = it_train_dict['train_images'] train_labels[step] = it_train_dict['train_labels'] train_scores[step] = it_train_dict['train_scores'] timesteps[step] = duration if train_aux_check: # Loop through to find aux scores it_train_aux = { itk: itv for itk, itv in it_train_dict.iteritems() if 'aux_score' in itk } train_aux[step] = it_train_aux assert not np.isnan(it_train_dict['train_loss']).any( ), 'Model diverged with loss = NaN' if step % config.validation_iters == 0: it_val_scores, it_val_labels, it_val_aux, it_val_grads, it_val_ims = [], [], [], [], [] for num_vals in range(config.num_validation_evals): # Validation accuracy as the average of n batches val_vars = sess.run(val_dict.values()) it_val_dict = { k: v for k, v in zip(val_dict.keys(), val_vars) } it_val_labels += [it_val_dict['val_labels']] it_val_scores += [it_val_dict['val_scores']] it_val_grads += [it_val_dict['val_grad_images']] it_val_ims += [it_val_dict['val_images']] if val_aux_check: iva = { itk: itv for itk, itv in it_val_dict.iteritems() if 'aux_score' in itk } it_val_aux += [iva] val_scores[step] = it_val_scores val_labels[step] = it_val_labels val_aux[step] = it_val_aux val_images[step] = it_val_grads val_grads[step] = it_val_ims # End iteration step += 1 except tf.errors.OutOfRangeError: print 'Done with evaluation for %d epochs, %d steps.' % (config.epochs, step) print 'Saved to: %s' % checkpoint_dir finally: coord.request_stop() coord.join(threads) sess.close() import ipdb ipdb.set_trace() np.savez( 'val_imgs_grads', val_images=val_images, # it_val_dict['val_images'], val_grads=val_grads, # it_val_dict['val_grad_images'], val_labels=val_labels, # it_val_dict['val_labels'], val_scores=val_scores) # it_val_dict['val_scores'][0])
def main( experiment_name, list_experiments=False, load_and_evaluate_ckpt=None, placeholder_data=None, grad_images=False, gpu_device='/gpu:0'): """Create a tensorflow worker to run experiments in your DB.""" if list_experiments: exps = db.list_experiments() print '_' * 30 print 'Initialized experiments:' print '_' * 30 for l in exps: print l.values()[0] print '_' * 30 if len(exps) == 0: print 'No experiments found.' else: print 'You can add to the DB with: '\ 'python prepare_experiments.py --experiment=%s' % \ exps[0].values()[0] return if experiment_name is None: print 'No experiment specified. Pulling one out of the DB.' experiment_name = db.get_experiment_name() # Prepare to run the model config = Config() condition_label = '%s_%s' % (experiment_name, py_utils.get_dt_stamp()) experiment_label = '%s' % (experiment_name) log = logger.get(os.path.join(config.log_dir, condition_label)) assert experiment_name is not None, 'Empty experiment name.' experiment_dict = experiments.experiments()[experiment_name]() config = add_to_config(d=experiment_dict, config=config) # Globals config.load_and_evaluate_ckpt = load_and_evaluate_ckpt if load_and_evaluate_ckpt is not None: # Remove the train operation and add a ckpt pointer from ops import evaluation config, exp_params = process_DB_exps( experiment_name=experiment_name, log=log, config=config) # Update config w/ DB params dataset_module = py_utils.import_module( model_dir=config.dataset_info, dataset=config.dataset) dataset_module = dataset_module.data_processing() # hardcoded class name train_key = [k for k in dataset_module.folds.keys() if 'train' in k] if not len(train_key): train_key = 'train' else: train_key = train_key[0] train_data, train_means_image, train_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=train_key, log=log) val_key = [k for k in dataset_module.folds.keys() if 'val' in k] if not len(val_key): val_key = 'train' else: val_key = val_key[0] val_data, val_means_image, val_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=val_key, log=log) # Initialize output folders dir_list = { 'checkpoints': os.path.join( config.checkpoints, condition_label), 'summaries': os.path.join( config.summaries, condition_label), 'condition_evaluations': os.path.join( config.condition_evaluations, condition_label), 'experiment_evaluations': os.path.join( # DEPRECIATED config.experiment_evaluations, experiment_label), 'visualization': os.path.join( config.visualizations, condition_label), 'weights': os.path.join( config.condition_evaluations, condition_label, 'weights') } [py_utils.make_dir(v) for v in dir_list.values()] # Prepare data loaders on the cpu if all(isinstance(i, list) for i in config.data_augmentations): if config.data_augmentations: config.data_augmentations = py_utils.flatten_list( config.data_augmentations, log) if load_and_evaluate_ckpt is not None: config.epochs = 1 config.train_shuffle = False config.val_shuffle = False with tf.device('/cpu:0'): if placeholder_data: placeholder_shape = placeholder_data['train_image_shape'] placeholder_dtype = placeholder_data['train_image_dtype'] original_train_images = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='train_images') placeholder_shape = placeholder_data['train_label_shape'] placeholder_dtype = placeholder_data['train_label_dtype'] original_train_labels = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='train_labels') placeholder_shape = placeholder_data['val_image_shape'] placeholder_dtype = placeholder_data['val_image_dtype'] original_val_images = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='val_images') placeholder_shape = placeholder_data['val_label_shape'] placeholder_dtype = placeholder_data['val_label_dtype'] original_val_labels = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='val_labels') # Apply augmentations ( train_images, train_labels ) = data_loader.placeholder_image_augmentations( images=original_train_images, model_input_image_size=dataset_module.model_input_image_size, labels=original_train_labels, data_augmentations=config.data_augmentations, batch_size=config.batch_size) ( val_images, val_labels ) = data_loader.placeholder_image_augmentations( images=original_val_images, model_input_image_size=dataset_module.model_input_image_size, labels=original_val_labels, data_augmentations=config.data_augmentations, batch_size=config.batch_size) # Store in the placeholder dict placeholder_data['train_images'] = original_train_images placeholder_data['train_labels'] = original_train_labels placeholder_data['val_images'] = original_val_images placeholder_data['val_labels'] = original_val_labels else: train_images, train_labels = data_loader.inputs( dataset=train_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_train, resize_output=config.resize_output) if hasattr(config, 'val_augmentations'): val_augmentations = config.val_augmentations else: val_augmentations = config.data_augmentations val_images, val_labels = data_loader.inputs( dataset=val_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=val_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_val, resize_output=config.resize_output) log.info('Created tfrecord dataloader tensors.') # Load model specification struct_name = config.model_struct.split(os.path.sep)[-1] try: model_dict = py_utils.import_module( dataset=struct_name, model_dir=os.path.join( 'models', 'structs', experiment_name).replace(os.path.sep, '.') ) except IOError: print 'Could not find the model structure: %s in folder %s' % ( struct_name, experiment_name) # Inject model_dict with hyperparameters if requested model_dict.layer_structure = hp_opt_utils.inject_model_with_hps( layer_structure=model_dict.layer_structure, exp_params=exp_params) # Prepare variables for the models if len(dataset_module.output_size) == 2: log.warning( 'Found > 1 dimension for your output size.' 'Converting to a scalar.') dataset_module.output_size = np.prod( dataset_module.output_size) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None # Correct number of output neurons if needed if config.dataloader_override and\ 'weights' in output_structure[-1].keys(): output_neurons = output_structure[-1]['weights'][0] size_check = output_neurons != dataset_module.output_size fc_check = output_structure[-1]['layers'][0] == 'fc' if size_check and fc_check: output_structure[-1]['weights'][0] = dataset_module.output_size log.warning('Adjusted output neurons from %s to %s.' % ( output_neurons, dataset_module.output_size)) # Prepare model on GPU if not hasattr(dataset_module, 'input_normalization'): dataset_module.input_normalization = None with tf.device(gpu_device): with tf.variable_scope('cnn') as scope: # Training model model = model_utils.model_class( mean=train_means_image, training=True, output_size=dataset_module.output_size, input_normalization=dataset_module.input_normalization) train_scores, model_summary, _ = model.build( data=train_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') if grad_images: oh_dims = int(train_scores.get_shape()[-1]) target_scores = tf.one_hot(train_labels, oh_dims) * train_scores train_gradients = tf.gradients(target_scores, train_images)[0] log.info('Built training model.') log.debug( json.dumps(model_summary, indent=4), verbose=0) print_model_architecture(model_summary) # Normalize labels on GPU if needed if 'normalize_labels' in exp_params.keys(): if exp_params['normalize_labels'] == 'zscore': train_labels -= train_means_label['mean'] train_labels /= train_means_label['std'] val_labels -= train_means_label['mean'] val_labels /= train_means_label['std'] log.info('Z-scoring labels.') elif exp_params['normalize_labels'] == 'mean': train_labels -= train_means_label['mean'] val_labels -= val_means_label['mean'] log.info('Mean-centering labels.') # Check the shapes of labels and scores if not isinstance(train_scores, list): if len( train_scores.get_shape()) != len( train_labels.get_shape()): train_shape = train_scores.get_shape().as_list() label_shape = train_labels.get_shape().as_list() val_shape = val_scores.get_shape().as_list() val_label_shape = val_labels.get_shape().as_list() if len( train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_labels = tf.expand_dims(train_labels, axis=-1) val_labels = tf.expand_dims(val_labels, axis=-1) elif len( train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_scores = tf.expand_dims(train_scores, axis=-1) val_scores = tf.expand_dims(val_scores, axis=-1) # Prepare the loss function train_loss, _ = loss_utils.loss_interpreter( logits=train_scores, # TODO labels=train_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(train_loss, list): for lidx, tl in enumerate(train_loss): tf.summary.scalar('training_loss_%s' % lidx, tl) train_loss = tf.add_n(train_loss) else: tf.summary.scalar('training_loss', train_loss) # Add weight decay if requested if len(model.regularizations) > 0: train_loss = loss_utils.wd_loss( regularizations=model.regularizations, loss=train_loss, wd_penalty=config.regularization_strength) assert config.lr is not None, 'No learning rate.' # TODO: Make a QC function if config.lr > 1: old_lr = config.lr config.lr = loss_utils.create_lr_schedule( train_batch=config.batch_size, num_training=config.lr) config.optimizer = 'momentum' log.info('Forcing momentum classifier.') else: old_lr = None train_op = loss_utils.optimizer_interpreter( loss=train_loss, lr=config.lr, optimizer=config.optimizer, constraints=config.optimizer_constraints, model=model) log.info('Built training loss function.') # Add a score for the training set train_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO: Attach to exp cnfg pred=train_scores, # TODO labels=train_labels) # Add aux scores if requested train_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: train_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=train_scores, labels=train_labels) # [0] # TODO: Fix for multiloss # Prepare remaining tensorboard summaries if config.tensorboard_images: if len(train_images.get_shape()) == 4: tf_fun.image_summaries(train_images, tag='Training images') if (np.asarray( train_labels.get_shape().as_list()) > 1).sum() > 2: tf_fun.image_summaries( train_labels, tag='Training_targets') tf_fun.image_summaries( train_scores, tag='Training_predictions') if isinstance(train_accuracy, list): for tidx, ta in enumerate(train_accuracy): tf.summary.scalar('training_accuracy_%s' % tidx, ta) else: tf.summary.scalar('training_accuracy', train_accuracy) if config.pr_curve: if isinstance(train_scores, list): for pidx, train_score in enumerate(train_scores): train_label = train_labels[:, pidx] pr_summary.op( tag='training_pr_%s' % pidx, predictions=tf.cast( tf.argmax( train_score, axis=-1), tf.float32), labels=tf.cast(train_label, tf.bool), display_name='training_precision_recall_%s' % pidx) else: pr_summary.op( tag='training_pr', predictions=tf.cast( tf.argmax( train_scores, axis=-1), tf.float32), labels=tf.cast(train_labels, tf.bool), display_name='training_precision_recall') log.info('Added training summaries.') with tf.variable_scope('cnn', tf.AUTO_REUSE) as scope: # Validation model scope.reuse_variables() val_model = model_utils.model_class( mean=train_means_image, # Normalize with train data training=False, output_size=dataset_module.output_size, input_normalization=dataset_module.input_normalization) val_scores, _, _ = val_model.build( # Ignore summary data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') if grad_images: oh_dims = int(val_scores.get_shape()[-1]) target_scores = tf.one_hot(val_labels, oh_dims) * val_scores val_gradients = tf.gradients(target_scores, val_images)[0] log.info('Built validation model.') # Check the shapes of labels and scores val_loss, _ = loss_utils.loss_interpreter( logits=val_scores, labels=val_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(val_loss, list): for lidx, tl in enumerate(val_loss): tf.summary.scalar('validation_loss_%s' % lidx, tl) val_loss = tf.add_n(val_loss) else: tf.summary.scalar('validation_loss', val_loss) # Add a score for the validation set val_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO pred=val_scores, labels=val_labels) # Add aux scores if requested val_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: val_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=val_scores, labels=val_labels) # [0] # TODO: Fix for multiloss # Prepare tensorboard summaries if config.tensorboard_images: if len(val_images.get_shape()) == 4: tf_fun.image_summaries( val_images, tag='Validation') if (np.asarray( val_labels.get_shape().as_list()) > 1).sum() > 2: tf_fun.image_summaries( val_labels, tag='Validation_targets') tf_fun.image_summaries( val_scores, tag='Validation_predictions') if isinstance(val_accuracy, list): for vidx, va in enumerate(val_accuracy): tf.summary.scalar('validation_accuracy_%s' % vidx, va) else: tf.summary.scalar('validation_accuracy', val_accuracy) if config.pr_curve: if isinstance(val_scores, list): for pidx, val_score in enumerate(val_scores): val_label = val_labels[:, pidx] pr_summary.op( tag='validation_pr_%s' % pidx, predictions=tf.cast( tf.argmax( val_score, axis=-1), tf.float32), labels=tf.cast(val_label, tf.bool), display_name='validation_precision_recall_%s' % pidx) else: pr_summary.op( tag='validation_pr', predictions=tf.cast( tf.argmax( val_scores, axis=-1), tf.float32), labels=tf.cast(val_labels, tf.bool), display_name='validation_precision_recall') log.info('Added validation summaries.') # Set up summaries and saver if not hasattr(config, 'max_to_keep'): config.max_to_keep = None saver = tf.train.Saver( var_list=tf.global_variables(), max_to_keep=config.max_to_keep) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group( tf.global_variables_initializer(), tf.local_variables_initializer()) ) summary_writer = tf.summary.FileWriter(dir_list['summaries'], sess.graph) # Set up exemplar threading if placeholder_data: coord, threads = None, None else: coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # Create dictionaries of important training and validation information train_dict = { 'train_loss': train_loss, 'train_images': train_images, 'train_labels': train_labels, 'train_op': train_op, 'train_scores': train_scores } val_dict = { 'val_loss': val_loss, 'val_images': val_images, 'val_labels': val_labels, 'val_scores': val_scores, } if grad_images: train_dict['train_gradients'] = train_gradients val_dict['val_gradients'] = val_gradients if isinstance(train_accuracy, list): for tidx, (ta, va) in enumerate(zip(train_accuracy, val_accuracy)): train_dict['train_accuracy_%s' % tidx] = ta val_dict['val_accuracy_%s' % tidx] = va else: train_dict['train_accuracy_0'] = train_accuracy val_dict['val_accuracy_0'] = val_accuracy if load_and_evaluate_ckpt is not None: # Remove the train operation and add a ckpt pointer del train_dict['train_op'] if hasattr(dataset_module, 'aux_score'): # Attach auxillary scores to tensor dicts for m in dataset_module.aux_scores: train_dict['train_aux_%s' % m] = train_aux[m] val_dict['val_aux_%s' % m] = val_aux[m] # Start training loop if old_lr is not None: config.lr = old_lr np.save( os.path.join( dir_list['condition_evaluations'], 'training_config_file'), config) log.info('Starting training') if load_and_evaluate_ckpt is not None: return evaluation.evaluation_loop( config=config, db=db, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, threads=threads, summary_dir=dir_list['summaries'], checkpoint_dir=dir_list['checkpoints'], weight_dir=dir_list['weights'], train_dict=train_dict, val_dict=val_dict, train_model=model, val_model=val_model, exp_params=exp_params, placeholder_data=placeholder_data) else: output_dict = training.training_loop( config=config, db=db, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, threads=threads, summary_dir=dir_list['summaries'], checkpoint_dir=dir_list['checkpoints'], weight_dir=dir_list['weights'], train_dict=train_dict, val_dict=val_dict, train_model=model, val_model=val_model, exp_params=exp_params) log.info('Finished training.') model_name = config.model_struct.replace('/', '_') if output_dict is not None: py_utils.save_npys( data=output_dict, model_name=model_name, output_string=dir_list['experiment_evaluations'])
def start_runs(logdir, steps, run_name, thresholds, mask_every_other_prediction=False): """Generate a PR curve with precision and recall evenly weighted. Arguments: logdir: The directory into which to store all the runs' data. steps: The number of steps to run for. run_name: The name of the run. thresholds: The number of thresholds to use for PR curves. mask_every_other_prediction: Whether to mask every other prediction by alternating weights between 0 and 1. """ tf.reset_default_graph() tf.set_random_seed(42) # Create a normal distribution layer used to generate true color labels. channel_distribution = tf.distributions.Normal(loc=0., scale=142.) # Sample the distribution to generate colors. Lets generate different numbers # of each color. The first dimension is the count of examples. def clip(value, minValue, maxValue): """Clips an op to the range [minValue, maxValue]. For now, we intentionally avoid using tf.clip_by_value because it apparently exhibits slightly different behavior based on system characteristics. See tensorflow/tensorflow#18527. Tests rely on this demo, so behavior must be consistent. Args: value: The value to clip. minValue: The min value to clip by. maxValue: The max value to clip by. Returns: A TensorFlow op that outputs the clipped value. """ return tf.maximum(minValue, tf.minimum(maxValue, value)) # Generate reds. number_of_reds = 100 true_reds = clip( tf.concat([ 255. - tf.abs(channel_distribution.sample([number_of_reds, 1])), tf.abs(channel_distribution.sample([number_of_reds, 2])) ], axis=1), 0., 255.) # Generate greens. number_of_greens = 200 true_greens = clip( tf.concat([ tf.abs(channel_distribution.sample([number_of_greens, 1])), 255. - tf.abs(channel_distribution.sample([number_of_greens, 1])), tf.abs(channel_distribution.sample([number_of_greens, 1])) ], axis=1), 0., 255.) # Generate blues. number_of_blues = 150 true_blues = clip( tf.concat([ tf.abs(channel_distribution.sample([number_of_blues, 2])), 255. - tf.abs(channel_distribution.sample([number_of_blues, 1])) ], axis=1), 0., 255.) # Assign each color a vector of 3 booleans based on its true label. labels = tf.concat([ tf.tile(tf.constant([[True, False, False]]), (number_of_reds, 1)), tf.tile(tf.constant([[False, True, False]]), (number_of_greens, 1)), tf.tile(tf.constant([[False, False, True]]), (number_of_blues, 1)), ], axis=0) # We introduce 3 normal distributions. They are used to predict whether a # color falls under a certain class (based on distances from corners of the # color triangle). The distributions vary per color. We have the distributions # narrow over time. initial_standard_deviations = [v + FLAGS.steps for v in (158, 200, 242)] iteration = tf.placeholder(tf.int32, shape=[]) red_predictor = tf.distributions.Normal( loc=0., scale=tf.cast(initial_standard_deviations[0] - iteration, dtype=tf.float32)) green_predictor = tf.distributions.Normal( loc=0., scale=tf.cast(initial_standard_deviations[1] - iteration, dtype=tf.float32)) blue_predictor = tf.distributions.Normal( loc=0., scale=tf.cast(initial_standard_deviations[2] - iteration, dtype=tf.float32)) # Make predictions (assign 3 probabilities to each color based on each color's # distance to each of the 3 corners). We seek double the area in the right # tail of the normal distribution. examples = tf.concat([true_reds, true_greens, true_blues], axis=0) probabilities_colors_are_red = (1 - red_predictor.cdf( tf.norm(examples - tf.constant([255., 0, 0]), axis=1))) * 2 probabilities_colors_are_green = (1 - green_predictor.cdf( tf.norm(examples - tf.constant([0, 255., 0]), axis=1))) * 2 probabilities_colors_are_blue = (1 - blue_predictor.cdf( tf.norm(examples - tf.constant([0, 0, 255.]), axis=1))) * 2 predictions = (probabilities_colors_are_red, probabilities_colors_are_green, probabilities_colors_are_blue) # This is the crucial piece. We write data required for generating PR curves. # We create 1 summary per class because we create 1 PR curve per class. for i, color in enumerate(('red', 'green', 'blue')): description = ( 'The probabilities used to create this PR curve are ' 'generated from a normal distribution. Its standard ' 'deviation is initially %0.0f and decreases over time.' % initial_standard_deviations[i]) weights = None if mask_every_other_prediction: # Assign a weight of 0 to every even-indexed prediction. Odd-indexed # predictions are assigned a default weight of 1. consecutive_indices = tf.reshape(tf.range(tf.size(predictions[i])), tf.shape(predictions[i])) weights = tf.cast(consecutive_indices % 2, dtype=tf.float32) summary.op(name=color, labels=labels[:, i], predictions=predictions[i], num_thresholds=thresholds, weights=weights, display_name='classifying %s' % color, description=description) merged_summary_op = tf.summary.merge_all() events_directory = os.path.join(logdir, run_name) sess = tf.Session() writer = tf.summary.FileWriter(events_directory, sess.graph) for step in xrange(steps): feed_dict = { iteration: step, } merged_summary = sess.run(merged_summary_op, feed_dict=feed_dict) writer.add_summary(merged_summary, step) writer.close()
def start_runs( logdir, steps, run_name, thresholds, mask_every_other_prediction=False): """Generate a PR curve with precision and recall evenly weighted. Arguments: logdir: The directory into which to store all the runs' data. steps: The number of steps to run for. run_name: The name of the run. thresholds: The number of thresholds to use for PR curves. mask_every_other_prediction: Whether to mask every other prediction by alternating weights between 0 and 1. """ tf.reset_default_graph() tf.set_random_seed(42) # Create a normal distribution layer used to generate true color labels. distribution = tf.distributions.Normal(loc=0., scale=142.) # Sample the distribution to generate colors. Lets generate different numbers # of each color. The first dimension is the count of examples. # The calls to sample() are given fixed random seed values that are "magic" # in that they correspond to the default seeds for those ops when the PR # curve test (which depends on this code) was written. We've pinned these # instead of continuing to use the defaults since the defaults are based on # node IDs from the sequence of nodes added to the graph, which can silently # change when this code or any TF op implementations it uses are modified. # TODO(nickfelt): redo the PR curve test to avoid reliance on random seeds. # Generate reds. number_of_reds = 100 true_reds = tf.clip_by_value( tf.concat([ 255 - tf.abs(distribution.sample([number_of_reds, 1], seed=11)), tf.abs(distribution.sample([number_of_reds, 2], seed=34)) ], axis=1), 0, 255) # Generate greens. number_of_greens = 200 true_greens = tf.clip_by_value( tf.concat([ tf.abs(distribution.sample([number_of_greens, 1], seed=61)), 255 - tf.abs(distribution.sample([number_of_greens, 1], seed=82)), tf.abs(distribution.sample([number_of_greens, 1], seed=105)) ], axis=1), 0, 255) # Generate blues. number_of_blues = 150 true_blues = tf.clip_by_value( tf.concat([ tf.abs(distribution.sample([number_of_blues, 2], seed=132)), 255 - tf.abs(distribution.sample([number_of_blues, 1], seed=153)) ], axis=1), 0, 255) # Assign each color a vector of 3 booleans based on its true label. labels = tf.concat([ tf.tile(tf.constant([[True, False, False]]), (number_of_reds, 1)), tf.tile(tf.constant([[False, True, False]]), (number_of_greens, 1)), tf.tile(tf.constant([[False, False, True]]), (number_of_blues, 1)), ], axis=0) # We introduce 3 normal distributions. They are used to predict whether a # color falls under a certain class (based on distances from corners of the # color triangle). The distributions vary per color. We have the distributions # narrow over time. initial_standard_deviations = [v + FLAGS.steps for v in (158, 200, 242)] iteration = tf.placeholder(tf.int32, shape=[]) red_predictor = tf.distributions.Normal( loc=0., scale=tf.cast( initial_standard_deviations[0] - iteration, dtype=tf.float32)) green_predictor = tf.distributions.Normal( loc=0., scale=tf.cast( initial_standard_deviations[1] - iteration, dtype=tf.float32)) blue_predictor = tf.distributions.Normal( loc=0., scale=tf.cast( initial_standard_deviations[2] - iteration, dtype=tf.float32)) # Make predictions (assign 3 probabilities to each color based on each color's # distance to each of the 3 corners). We seek double the area in the right # tail of the normal distribution. examples = tf.concat([true_reds, true_greens, true_blues], axis=0) probabilities_colors_are_red = (1 - red_predictor.cdf( tf.norm(examples - tf.constant([255., 0, 0]), axis=1))) * 2 probabilities_colors_are_green = (1 - green_predictor.cdf( tf.norm(examples - tf.constant([0, 255., 0]), axis=1))) * 2 probabilities_colors_are_blue = (1 - blue_predictor.cdf( tf.norm(examples - tf.constant([0, 0, 255.]), axis=1))) * 2 predictions = ( probabilities_colors_are_red, probabilities_colors_are_green, probabilities_colors_are_blue ) # This is the crucial piece. We write data required for generating PR curves. # We create 1 summary per class because we create 1 PR curve per class. for i, color in enumerate(('red', 'green', 'blue')): description = ('The probabilities used to create this PR curve are ' 'generated from a normal distribution. Its standard ' 'deviation is initially %0.0f and decreases over time.' % initial_standard_deviations[i]) weights = None if mask_every_other_prediction: # Assign a weight of 0 to every even-indexed prediction. Odd-indexed # predictions are assigned a default weight of 1. consecutive_indices = tf.reshape( tf.range(tf.size(predictions[i])), tf.shape(predictions[i])) weights = tf.cast(consecutive_indices % 2, dtype=tf.float32) summary.op( name=color, labels=labels[:, i], predictions=predictions[i], num_thresholds=thresholds, weights=weights, display_name='classifying %s' % color, description=description) merged_summary_op = tf.summary.merge_all() events_directory = os.path.join(logdir, run_name) sess = tf.Session() writer = tf.summary.FileWriter(events_directory, sess.graph) for step in xrange(steps): feed_dict = { iteration: step, } merged_summary = sess.run(merged_summary_op, feed_dict=feed_dict) writer.add_summary(merged_summary, step) writer.close()