def testWeight1(self):
# Generate summaries for showing PR curves in TensorBoard.
with tf.Session() as sess:
summary.op(
tag='tag_bar',
labels=tf.constant([True, False, False, True, True, False]),
predictions=tf.constant([0.81, 0.8, 0.61, 0.6, 0.4, 0.2]),
num_thresholds=10)
merged_summary_op = tf.summary.merge_all()
foo_directory = os.path.join(self.logdir, 'foo')
writer = tf.summary.FileWriter(foo_directory, sess.graph)
writer.add_summary(sess.run(merged_summary_op), 1)
writer.close()
# Create a multiplexer for reading the data we just wrote.
multiplexer = event_multiplexer.EventMultiplexer()
multiplexer.AddRunsFromDirectory(self.logdir)
multiplexer.Reload()
# Verify that the metadata was correctly written.
accumulator = multiplexer.GetAccumulator('foo')
tag_content_dict = accumulator.PluginTagToContent('pr_curve')
self.assertListEqual(['tag_bar/tag_bar'],
list(tag_content_dict.keys()))
# Parse the data within the JSON string and set the proto's fields.
plugin_data = pr_curve_pb2.PrCurvePluginData()
json_format.Parse(tag_content_dict['tag_bar/tag_bar'], plugin_data)
self.assertEqual(10, plugin_data.num_thresholds)
# Test the summary contents.
tensor_events = accumulator.Tensors('tag_bar/tag_bar')
self.assertEqual(1, len(tensor_events))
tensor_event = tensor_events[0]
self.assertEqual(1, tensor_event.step)
tensor_nd_array = tf.make_ndarray(tensor_event.tensor_proto)
np.testing.assert_allclose(
[
# True positives.
[3.0, 3.0, 3.0, 3.0, 2.0, 2.0, 1.0, 1.0, 0.0, 0.0],
# False positives.
[3.0, 3.0, 2.0, 2.0, 2.0, 2.0, 1.0, 1.0, 0.0, 0.0],
# True negatives.
[0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 3.0, 3.0],
# False negatives.
[0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 2.0, 2.0, 3.0, 3.0],
# Precision.
[0.5, 0.5, 0.6, 0.6, 0.5, 0.5, 0.5, 0.5, 1.0, 1.0],
# Recall.
[1.0, 1.0, 1.0, 1.0, 2 / 3, 2 / 3, 1 / 3, 1 / 3, 0.0, 0.0],
],
tensor_nd_array)
def test_matches_op_with_updates(self):
predictions = tf.constant([0.2, 0.4, 0.5, 0.6, 0.8], dtype=tf.float32)
labels = tf.constant([False, True, True, False, True], dtype=tf.bool)
pr_curve, update_op = summary.streaming_op(
name="pr_curve",
predictions=predictions,
labels=labels,
num_thresholds=10,
)
complete_predictions = tf.tile(predictions, [3])
complete_labels = tf.tile(labels, [3])
expected_pr_curve = summary.op(
name="pr_curve",
predictions=complete_predictions,
labels=complete_labels,
num_thresholds=10,
)
with self.test_session() as sess:
sess.run(tf.compat.v1.local_variables_initializer())
sess.run([update_op])
sess.run([update_op])
sess.run([update_op])
proto = self.pb_via_op(pr_curve)
expected_proto = self.pb_via_op(expected_pr_curve)
# Need to detect and fix the automatic _1 appended to second namespace.
self.assertEqual(proto.value[0].tag, "pr_curve/pr_curves")
self.assertEqual(expected_proto.value[0].tag,
"pr_curve_1/pr_curves")
expected_proto.value[0].tag = "pr_curve/pr_curves"
self.assertProtoEquals(expected_proto, proto)
def compute_and_check_summary_pb(self,
name,
labels,
predictions,
num_thresholds,
weights=None,
display_name=None,
description=None,
feed_dict=None):
"""Use both `op` and `pb` to get a summary, asserting equality.
Returns:
a `Summary` protocol buffer
"""
labels_tensor = tf.constant(labels)
predictions_tensor = tf.constant(predictions)
weights_tensor = None if weights is None else tf.constant(weights)
op = summary.op(name=name,
labels=labels_tensor,
predictions=predictions_tensor,
num_thresholds=num_thresholds,
weights=weights_tensor,
display_name=display_name,
description=description)
pb = self.normalize_summary_pb(
summary.pb(name=name,
labels=labels,
predictions=predictions,
num_thresholds=num_thresholds,
weights=weights,
display_name=display_name,
description=description))
pb_via_op = self.normalize_summary_pb(
self.pb_via_op(op, feed_dict=feed_dict))
self.assertProtoEquals(pb, pb_via_op)
return pb
def test_add_pr_curve_plugin(live_mock_server, test_settings):
tf.compat.v1.disable_v2_behavior()
wandb.init(sync_tensorboard=True, settings=test_settings)
summ_op = pr_curve_plugin_summary.op(
name="test_pr",
labels=tf.constant([True, False, True]),
predictions=tf.constant([0.7, 0.2, 0.3]),
num_thresholds=5,
)
merged_summary_op = tf.compat.v1.summary.merge([summ_op])
sess = tf.compat.v1.Session()
writer = tf.compat.v1.summary.FileWriter(wandb.run.dir, sess.graph)
merged_summary = sess.run(merged_summary_op)
writer.add_summary(merged_summary, 0)
writer.close()
wandb.finish()
server_ctx = live_mock_server.get_ctx()
assert ("test_pr/pr_curves"
in server_ctx["config"][2]["_wandb"]["value"]["visualize"].keys())
assert (server_ctx["config"][2]["_wandb"]["value"]["visualize"]
["test_pr/pr_curves"] == PR_CURVE_PANEL_CONFIG)
wandb.tensorboard.unpatch()
def testMatchesOp(self):
predictions = tf.constant([0.2, 0.4, 0.5, 0.6, 0.8], dtype=tf.float32)
labels = tf.constant([False, True, True, False, True], dtype=tf.bool)
pr_curve, update_op = summary.streaming_op(tag='pr_curve',
predictions=predictions,
labels=labels,
num_thresholds=10)
expected_pr_curve = summary.op(tag='pr_curve',
predictions=predictions,
labels=labels,
num_thresholds=10)
with self.test_session() as sess:
sess.run(tf.local_variables_initializer())
sess.run([update_op])
proto = self.pb_via_op(pr_curve)
expected_proto = self.pb_via_op(expected_pr_curve)
# Need to detect and fix the automatic _1 appended to second namespace.
self.assertEqual(proto.value[0].tag, 'pr_curve/pr_curves')
self.assertEqual(expected_proto.value[0].tag,
'pr_curve_1/pr_curves')
expected_proto.value[0].tag = 'pr_curve/pr_curves'
self.assertProtoEquals(expected_proto, proto)
def get_train_hooks(self, labels, logits, alpha=None):
nclass = self.config['data']['task']['classes']['num']
metric_tensor = {
"batch_accuracy": metrics_lib.accuracy(logits, labels),
'global_step': tf.train.get_or_create_global_step(),
}
if nclass > 100:
logging.info('Too many classes, disable confusion matrix in train: %d' %
(nclass))
else:
metric_tensor['batch_confusion'] = \
metrics_lib.confusion_matrix(logits, labels, nclass)
tf.summary.scalar('batch_accuracy', metric_tensor['batch_accuracy'])
if alpha:
metric_tensor.update({"alignment": alpha})
# plot PR curve
true_label_bool = tf.cast(labels, tf.bool)
softmax = tf.nn.softmax(logits)
pr_summary.op(
name='pr_curve_train_batch',
labels=true_label_bool,
predictions=softmax[:, -1],
num_thresholds=16,
weights=None)
train_hooks = [
tf.train.StepCounterHook(
every_n_steps=100,
every_n_secs=None,
output_dir=None,
summary_writer=None),
tf.train.FinalOpsHook(
final_ops=[tf.train.get_or_create_global_step()],
final_ops_feed_dict=None),
tf.train.LoggingTensorHook(
tensors=metric_tensor,
every_n_iter=100,
every_n_secs=None,
at_end=False,
formatter=None),
]
return train_hooks
def set_model(self, model):
super(VisualizeTensorBoard, self).set_model(model)
if self.pr_curve:
predictions = self.model._feed_outputs[0]
labels = tf.cast(self.model._feed_targets[0], tf.bool)
self.pr_summary = pr_summary.op(
name='pr_curve',
predictions=predictions,
labels=labels,
display_name='Precision-Recall Curve')
def set_model(self, model):
super(PRTensorBoard, self).set_model(model)
if self.pr_curve:
# Get the prediction and label tensor placeholders.
predictions = self.model._feed_outputs[0]
labels = tf.cast(self.model._feed_targets[0], tf.bool)
# Create the PR summary OP.
self.pr_summary = pr_summary.op(name='pr_curve',
predictions=predictions,
labels=labels,
display_name='Precision-Recall Curve')
def log_pr_curve(self, tag, pred_probability, labels, step=0, N = 10000):
labels = labels.astype('int32')
# If values are images, then image dimensions must be flattened with the batch dimension
if len(pred_probability.shape) > 2:
Nb = pred_probability.shape[0]
Nc = pred_probability.shape[1]
Ns = np.prod(pred_probability.shape[2:])
pred_probability = np.reshape(pred_probability, [Nb, Nc, Ns])
pred_probability = np.transpose(pred_probability, [0, 2, 1])
pred_probability = np.reshape(pred_probability, [Nb * Ns, Nc])
labels = labels.flatten()
# Remove ignore-labels
keep = np.logical_not(labels == -100)
pred_probability = pred_probability[keep, :]
labels = labels[keep]
#Get size of array
Nb = pred_probability.shape[0]
Nc = pred_probability.shape[1]
# Decimate
if Nb > N:
inds = np.linspace(0, Nb-1, N).astype('int64')
labels = labels[inds]
pred_probability = pred_probability[inds,:]
Nb = pred_probability.shape[0]
# Make one-hot bool array of true labels
tmp = np.zeros([Nb, Nc])
for i in range(Nb):
if labels[i] >= 0:
tmp[i, labels[i]] = 1
labels = tmp.astype('bool')
#Add summary to dict
if tag not in self.summaries.keys():
self.summaries[tag] = pr_summary.op(name=tag, labels=self.placeholder_pr_curve_labels, predictions=self.placeholder_pr_curve_predictions, num_thresholds=21)
#Make summary
merged_summary_op = tf.summary.merge([self.summaries[tag]])
with self._tf_session() as sess:
merged_summary = sess.run(merged_summary_op, feed_dict={self.placeholder_pr_curve_labels: labels, self.placeholder_pr_curve_predictions: pred_probability})
#Write
self.writer.add_summary(merged_summary, step)
def set_model(self, model):
super().set_model(model)
if self.layout_summary:
self.writer.add_summary(self.layout_summary)
if self.pr_curve:
for class_idx in range(self.num_classes):
predictions = self.model._feed_outputs[0][..., class_idx]
labels = tf.cast(self.model._feed_targets[0][..., class_idx], tf.bool)
summary_op = pr_summary.op(
name='pr_curve_class_' + str(class_idx),
predictions=predictions,
labels=labels,
display_name='Precision-Recall (Class ' + str(class_idx) + ')')
self.pr_summary.append(summary_op)
if self.tfpn:
predictions = self.model._feed_outputs[0]
labels = tf.cast(self.model._feed_targets[0], tf.bool)
_, precision = tf.metrics.precision(labels, predictions)
_, recall = tf.metrics.recall(labels, predictions)
_, tp = tf.metrics.true_positives(labels, predictions)
_, fn = tf.metrics.false_negatives(labels, predictions)
_, fp = tf.metrics.false_positives(labels, predictions)
_, f1 = tf.contrib.metrics.f1_score(labels, predictions) # TODO this recalcs tp,fn,fp. TODO utilise existing results
self.precision_summary = scalar_summary.op(name='precision', data=precision)
self.recall_summary = scalar_summary.op(name='recall', data=recall)
self.f1_summary = scalar_summary.op(name='f1', data=f1)
self.fp_summary = scalar_summary.op(name='fp', data=fp)
self.tp_summary = scalar_summary.op(name='tp', data=tp)
self.fn_summary = scalar_summary.op(name='fn', data=fn)
self.merged = tf.summary.merge_all()
def start_runs(logdir,
steps,
run_name,
thresholds,
mask_every_other_prediction=False):
"""Generate a PR curve with precision and recall evenly weighted.
Arguments:
logdir: The directory into which to store all the runs' data.
steps: The number of steps to run for.
run_name: The name of the run.
thresholds: The number of thresholds to use for PR curves.
mask_every_other_prediction: Whether to mask every other prediction by
alternating weights between 0 and 1.
"""
tf.compat.v1.reset_default_graph()
tf.compat.v1.set_random_seed(42)
# Create a normal distribution layer used to generate true color labels.
distribution = tf.compat.v1.distributions.Normal(loc=0., scale=142.)
# Sample the distribution to generate colors. Lets generate different numbers
# of each color. The first dimension is the count of examples.
# The calls to sample() are given fixed random seed values that are "magic"
# in that they correspond to the default seeds for those ops when the PR
# curve test (which depends on this code) was written. We've pinned these
# instead of continuing to use the defaults since the defaults are based on
# node IDs from the sequence of nodes added to the graph, which can silently
# change when this code or any TF op implementations it uses are modified.
# TODO(nickfelt): redo the PR curve test to avoid reliance on random seeds.
# Generate reds.
number_of_reds = 100
true_reds = tf.clip_by_value(
tf.concat([
255 - tf.abs(distribution.sample([number_of_reds, 1], seed=11)),
tf.abs(distribution.sample([number_of_reds, 2], seed=34))
],
axis=1), 0, 255)
# Generate greens.
number_of_greens = 200
true_greens = tf.clip_by_value(
tf.concat([
tf.abs(distribution.sample([number_of_greens, 1], seed=61)),
255 - tf.abs(distribution.sample([number_of_greens, 1], seed=82)),
tf.abs(distribution.sample([number_of_greens, 1], seed=105))
],
axis=1), 0, 255)
# Generate blues.
number_of_blues = 150
true_blues = tf.clip_by_value(
tf.concat([
tf.abs(distribution.sample([number_of_blues, 2], seed=132)),
255 - tf.abs(distribution.sample([number_of_blues, 1], seed=153))
],
axis=1), 0, 255)
# Assign each color a vector of 3 booleans based on its true label.
labels = tf.concat([
tf.tile(tf.constant([[True, False, False]]), (number_of_reds, 1)),
tf.tile(tf.constant([[False, True, False]]), (number_of_greens, 1)),
tf.tile(tf.constant([[False, False, True]]), (number_of_blues, 1)),
],
axis=0)
# We introduce 3 normal distributions. They are used to predict whether a
# color falls under a certain class (based on distances from corners of the
# color triangle). The distributions vary per color. We have the distributions
# narrow over time.
initial_standard_deviations = [v + FLAGS.steps for v in (158, 200, 242)]
iteration = tf.compat.v1.placeholder(tf.int32, shape=[])
red_predictor = tf.compat.v1.distributions.Normal(
loc=0.,
scale=tf.cast(initial_standard_deviations[0] - iteration,
dtype=tf.float32))
green_predictor = tf.compat.v1.distributions.Normal(
loc=0.,
scale=tf.cast(initial_standard_deviations[1] - iteration,
dtype=tf.float32))
blue_predictor = tf.compat.v1.distributions.Normal(
loc=0.,
scale=tf.cast(initial_standard_deviations[2] - iteration,
dtype=tf.float32))
# Make predictions (assign 3 probabilities to each color based on each color's
# distance to each of the 3 corners). We seek double the area in the right
# tail of the normal distribution.
examples = tf.concat([true_reds, true_greens, true_blues], axis=0)
probabilities_colors_are_red = (1 - red_predictor.cdf(
tf.norm(tensor=examples - tf.constant([255., 0, 0]), axis=1))) * 2
probabilities_colors_are_green = (1 - green_predictor.cdf(
tf.norm(tensor=examples - tf.constant([0, 255., 0]), axis=1))) * 2
probabilities_colors_are_blue = (1 - blue_predictor.cdf(
tf.norm(tensor=examples - tf.constant([0, 0, 255.]), axis=1))) * 2
predictions = (probabilities_colors_are_red,
probabilities_colors_are_green,
probabilities_colors_are_blue)
# This is the crucial piece. We write data required for generating PR curves.
# We create 1 summary per class because we create 1 PR curve per class.
for i, color in enumerate(('red', 'green', 'blue')):
description = (
'The probabilities used to create this PR curve are '
'generated from a normal distribution. Its standard '
'deviation is initially %0.0f and decreases over time.' %
initial_standard_deviations[i])
weights = None
if mask_every_other_prediction:
# Assign a weight of 0 to every even-indexed prediction. Odd-indexed
# predictions are assigned a default weight of 1.
consecutive_indices = tf.reshape(
tf.range(tf.size(input=predictions[i])),
tf.shape(input=predictions[i]))
weights = tf.cast(consecutive_indices % 2, dtype=tf.float32)
summary.op(name=color,
labels=labels[:, i],
predictions=predictions[i],
num_thresholds=thresholds,
weights=weights,
display_name='classifying %s' % color,
description=description)
merged_summary_op = tf.compat.v1.summary.merge_all()
events_directory = os.path.join(logdir, run_name)
sess = tf.compat.v1.Session()
writer = tf.compat.v1.summary.FileWriter(events_directory, sess.graph)
for step in xrange(steps):
feed_dict = {
iteration: step,
}
merged_summary = sess.run(merged_summary_op, feed_dict=feed_dict)
writer.add_summary(merged_summary, step)
writer.close()
def main(experiment_name,
list_experiments=False,
load_and_evaluate_ckpt=None,
config_file=None,
ckpt_file=None,
gpu_device='/gpu:0'):
"""Create a tensorflow worker to run experiments in your DB."""
if list_experiments:
exps = db.list_experiments()
print '_' * 30
print 'Initialized experiments:'
print '_' * 30
for l in exps:
print l.values()[0]
print '_' * 30
if len(exps) == 0:
print 'No experiments found.'
else:
print 'You can add to the DB with: '\
'python prepare_experiments.py --experiment=%s' % \
exps[0].values()[0]
return
if experiment_name is None:
print 'No experiment specified. Pulling one out of the DB.'
experiment_name = db.get_experiment_name()
# Prepare to run the model
config = Config()
condition_label = '%s_%s' % (experiment_name, py_utils.get_dt_stamp())
experiment_label = '%s' % (experiment_name)
log = logger.get(os.path.join(config.log_dir, condition_label))
experiment_dict = experiments.experiments()[experiment_name]()
config = add_to_config(d=experiment_dict, config=config) # Globals
config.load_and_evaluate_ckpt = load_and_evaluate_ckpt
config, exp_params = process_DB_exps(
experiment_name=experiment_name, log=log,
config=config) # Update config w/ DB params
config = np.load(config_file).item()
dataset_module = py_utils.import_module(model_dir=config.dataset_info,
dataset=config.dataset)
dataset_module = dataset_module.data_processing() # hardcoded class name
train_data, train_means_image, train_means_label = get_data_pointers(
dataset=config.dataset,
base_dir=config.tf_records,
cv=dataset_module.folds.keys()[1], # TODO: SEARCH FOR INDEX.
log=log)
val_data, val_means_image, val_means_label = get_data_pointers(
dataset=config.dataset,
base_dir=config.tf_records,
cv=dataset_module.folds.keys()[0],
log=log)
# Initialize output folders
dir_list = {
'checkpoints':
os.path.join(config.checkpoints, condition_label),
'summaries':
os.path.join(config.summaries, condition_label),
'condition_evaluations':
os.path.join(config.condition_evaluations, condition_label),
'experiment_evaluations':
os.path.join( # DEPRECIATED
config.experiment_evaluations, experiment_label),
'visualization':
os.path.join(config.visualizations, condition_label),
'weights':
os.path.join(config.condition_evaluations, condition_label, 'weights')
}
[py_utils.make_dir(v) for v in dir_list.values()]
# Prepare data loaders on the cpu
if all(isinstance(i, list) for i in config.data_augmentations):
if config.data_augmentations:
config.data_augmentations = py_utils.flatten_list(
config.data_augmentations, log)
config.epochs = 1
config.shuffle = False
with tf.device('/cpu:0'):
train_images, train_labels = data_loader.inputs(
dataset=train_data,
batch_size=config.batch_size,
model_input_image_size=dataset_module.model_input_image_size,
tf_dict=dataset_module.tf_dict,
data_augmentations=config.data_augmentations,
num_epochs=config.epochs,
tf_reader_settings=dataset_module.tf_reader,
shuffle=config.shuffle_train,
resize_output=config.resize_output)
if hasattr(config, 'val_augmentations'):
val_augmentations = config.val_augmentations
else:
val_augmentations = config.data_augmentations
val_images, val_labels = data_loader.inputs(
dataset=val_data,
batch_size=config.batch_size,
model_input_image_size=dataset_module.model_input_image_size,
tf_dict=dataset_module.tf_dict,
data_augmentations=['resize_and_crop'],
num_epochs=config.epochs,
tf_reader_settings=dataset_module.tf_reader,
shuffle=config.shuffle_val,
resize_output=config.resize_output)
log.info('Created tfrecord dataloader tensors.')
# Load model specification
struct_name = config.model_struct.split(os.path.sep)[-1]
try:
model_dict = py_utils.import_module(
dataset=struct_name,
model_dir=os.path.join('models', 'structs',
experiment_name).replace(os.path.sep, '.'))
except IOError:
print 'Could not find the model structure: %s in folder %s' % (
struct_name, experiment_name)
# Inject model_dict with hyperparameters if requested
model_dict.layer_structure = hp_opt_utils.inject_model_with_hps(
layer_structure=model_dict.layer_structure, exp_params=exp_params)
# Prepare model on GPU
with tf.device(gpu_device):
with tf.variable_scope('cnn') as scope:
# Normalize labels if needed
if 'normalize_labels' in exp_params.keys():
if exp_params['normalize_labels'] == 'zscore':
train_labels -= train_means_label['mean']
train_labels /= train_means_label['std']
log.info('Z-scoring labels.')
elif exp_params['normalize_labels'] == 'mean':
train_labels -= train_means_label['mean']
log.info('Mean-centering labels.')
# Training model
if len(dataset_module.output_size) == 2:
log.warning('Found > 1 dimension for your output size.'
'Converting to a scalar.')
dataset_module.output_size = np.prod(
dataset_module.output_size)
if hasattr(model_dict, 'output_structure'):
# Use specified output layer
output_structure = model_dict.output_structure
else:
output_structure = None
model = model_utils.model_class(
mean=train_means_image,
training=True,
output_size=dataset_module.output_size)
train_scores, model_summary = model.build(
data=train_images,
layer_structure=model_dict.layer_structure,
output_structure=output_structure,
log=log,
tower_name='cnn')
eval_graph = tf.Graph()
with eval_graph.as_default():
with eval_graph.gradient_override_map({'selu': 'GradLRP'}):
train_grad_images = tf.gradients(
train_scores[0] * tf.cast(train_labels, tf.float32),
train_images)[0]
log.info('Built training model.')
log.debug(json.dumps(model_summary, indent=4), verbose=0)
print_model_architecture(model_summary)
# Check the shapes of labels and scores
if not isinstance(train_scores, list):
if len(train_scores.get_shape()) != len(
train_labels.get_shape()):
train_shape = train_scores.get_shape().as_list()
label_shape = train_labels.get_shape().as_list()
if len(train_shape) == 2 and len(
label_shape) == 1 and train_shape[-1] == 1:
train_labels = tf.expand_dims(train_labels, axis=-1)
elif len(train_shape) == 2 and len(
label_shape) == 1 and train_shape[-1] == 1:
train_scores = tf.expand_dims(train_scores, axis=-1)
# Prepare the loss function
train_loss, _ = loss_utils.loss_interpreter(
logits=train_scores, # TODO
labels=train_labels,
loss_type=config.loss_function,
weights=config.loss_weights,
dataset_module=dataset_module)
# Add loss tensorboard tracking
if isinstance(train_loss, list):
for lidx, tl in enumerate(train_loss):
tf.summary.scalar('training_loss_%s' % lidx, tl)
train_loss = tf.add_n(train_loss)
else:
tf.summary.scalar('training_loss', train_loss)
# Add weight decay if requested
if len(model.regularizations) > 0:
train_loss = loss_utils.wd_loss(
regularizations=model.regularizations,
loss=train_loss,
wd_penalty=config.regularization_strength)
train_op = loss_utils.optimizer_interpreter(
loss=train_loss,
lr=config.lr,
optimizer=config.optimizer,
constraints=config.optimizer_constraints,
model=model)
log.info('Built training loss function.')
# Add a score for the training set
train_accuracy = eval_metrics.metric_interpreter(
metric=dataset_module.score_metric, # TODO: Attach to exp cnfg
pred=train_scores, # TODO
labels=train_labels)
# Add aux scores if requested
train_aux = {}
if hasattr(dataset_module, 'aux_scores'):
for m in dataset_module.aux_scores:
train_aux[m] = eval_metrics.metric_interpreter(
metric=m, pred=train_scores,
labels=train_labels)[0] # TODO: Fix for multiloss
# Prepare remaining tensorboard summaries
if len(train_images.get_shape()) == 4:
tf_fun.image_summaries(train_images, tag='Training images')
if len(train_labels.get_shape()) > 2:
tf_fun.image_summaries(train_labels, tag='Training_targets')
tf_fun.image_summaries(train_scores,
tag='Training_predictions')
if isinstance(train_accuracy, list):
for tidx, ta in enumerate(train_accuracy):
tf.summary.scalar('training_accuracy_%s' % tidx, ta)
else:
tf.summary.scalar('training_accuracy', train_accuracy)
if config.pr_curve:
if isinstance(train_scores, list):
for pidx, train_score in enumerate(train_scores):
train_label = train_labels[:, pidx]
pr_summary.op(
tag='training_pr_%s' % pidx,
predictions=tf.cast(
tf.argmax(train_score, axis=-1), tf.float32),
labels=tf.cast(train_label, tf.bool),
display_name='training_precision_recall_%s' % pidx)
else:
pr_summary.op(tag='training_pr',
predictions=tf.cast(
tf.argmax(train_scores, axis=-1),
tf.float32),
labels=tf.cast(train_labels, tf.bool),
display_name='training_precision_recall')
log.info('Added training summaries.')
# Validation model
scope.reuse_variables()
val_model = model_utils.model_class(
mean=train_means_image, # Normalize with train data
training=False, # False,
output_size=dataset_module.output_size)
val_scores, _ = val_model.build( # Ignore summary
data=val_images,
layer_structure=model_dict.layer_structure,
output_structure=output_structure,
log=log,
tower_name='cnn')
eval_graph = tf.Graph()
with eval_graph.as_default():
with eval_graph.gradient_override_map({'selu': 'GradLRP'}):
val_grad_images = tf.gradients(
val_scores[0] * tf.cast(val_labels, tf.float32),
val_images)[0]
log.info('Built validation model.')
# Check the shapes of labels and scores
if not isinstance(train_scores, list):
if len(val_scores.get_shape()) != len(val_labels.get_shape()):
val_shape = val_scores.get_shape().as_list()
val_label_shape = val_labels.get_shape().as_list()
if len(val_shape) == 2 and len(
val_label_shape) == 1 and val_shape[-1] == 1:
val_labels = tf.expand_dims(val_labels, axis=-1)
if len(val_shape) == 2 and len(
val_label_shape) == 1 and val_shape[-1] == 1:
val_scores = tf.expand_dims(val_scores, axis=-1)
val_loss, _ = loss_utils.loss_interpreter(
logits=val_scores,
labels=val_labels,
loss_type=config.loss_function,
weights=config.loss_weights,
dataset_module=dataset_module)
# Add loss tensorboard tracking
if isinstance(val_loss, list):
for lidx, tl in enumerate(val_loss):
tf.summary.scalar('validation_loss_%s' % lidx, tl)
val_loss = tf.add_n(val_loss)
else:
tf.summary.scalar('validation_loss', val_loss)
# Add a score for the validation set
val_accuracy = eval_metrics.metric_interpreter(
metric=dataset_module.score_metric, # TODO
pred=val_scores,
labels=val_labels)
# Add aux scores if requested
val_aux = {}
if hasattr(dataset_module, 'aux_scores'):
for m in dataset_module.aux_scores:
val_aux[m] = eval_metrics.metric_interpreter(
metric=m, pred=val_scores,
labels=val_labels)[0] # TODO: Fix for multiloss
# Prepare tensorboard summaries
if len(val_images.get_shape()) == 4:
tf_fun.image_summaries(val_images, tag='Validation')
if len(val_labels.get_shape()) > 2:
tf_fun.image_summaries(val_labels, tag='Validation_targets')
tf_fun.image_summaries(val_scores,
tag='Validation_predictions')
if isinstance(val_accuracy, list):
for vidx, va in enumerate(val_accuracy):
tf.summary.scalar('validation_accuracy_%s' % vidx, va)
else:
tf.summary.scalar('validation_accuracy', val_accuracy)
if config.pr_curve:
if isinstance(val_scores, list):
for pidx, val_score in enumerate(val_scores):
val_label = val_labels[:, pidx]
pr_summary.op(
tag='validation_pr_%s' % pidx,
predictions=tf.cast(tf.argmax(val_score, axis=-1),
tf.float32),
labels=tf.cast(val_label, tf.bool),
display_name='validation_precision_recall_%s' %
pidx)
else:
pr_summary.op(tag='validation_pr',
predictions=tf.cast(
tf.argmax(val_scores, axis=-1),
tf.float32),
labels=tf.cast(val_labels, tf.bool),
display_name='validation_precision_recall')
log.info('Added validation summaries.')
# Set up summaries and saver
saver = tf.train.Saver(tf.global_variables())
# Initialize the graph
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
# Need to initialize both of these if supplying num_epochs to inputs
sess.run(
tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer()))
# Set up exemplar threading
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# Create dictionaries of important training and validation information
train_dict = {
'train_loss': train_loss,
'train_images': train_images,
'train_labels': train_labels,
'train_op': train_op,
'train_scores': train_scores,
'train_grad_images': train_grad_images
}
val_dict = {
'val_loss': val_loss,
'val_images': val_images,
'val_labels': val_labels,
'val_scores': val_scores,
'val_grad_images': val_grad_images
}
if isinstance(train_accuracy, list):
for tidx, (ta, va) in enumerate(zip(train_accuracy, val_accuracy)):
train_dict['train_accuracy_%s' % tidx] = ta
val_dict['val_accuracy_%s' % tidx] = va
else:
train_dict['train_accuracy_0'] = train_accuracy
val_dict['val_accuracy_0'] = val_accuracy
if load_and_evaluate_ckpt is not None:
# Remove the train operation and add a ckpt pointer
del train_dict['train_op']
if hasattr(dataset_module, 'aux_score'):
# Attach auxillary scores to tensor dicts
for m in dataset_module.aux_scores:
train_dict['train_aux_%s' % m] = train_aux[m]
val_dict['val_aux_%s' % m] = val_aux[m]
# Start training loop
checkpoint_dir = dir_list['checkpoints']
step = 0
train_losses, train_accs, train_aux, timesteps = {}, {}, {}, {}
val_scores, val_aux, val_labels, val_grads = {}, {}, {}, {}
train_images, val_images = {}, {}
train_scores, train_labels = {}, {}
train_aux_check = np.any(['aux_score' in k for k in train_dict.keys()])
val_aux_check = np.any(['aux_score' in k for k in val_dict.keys()])
# Restore model
saver.restore(sess, ckpt_file)
# Start evaluation
try:
while not coord.should_stop():
start_time = time.time()
train_vars = sess.run(train_dict.values())
it_train_dict = {
k: v
for k, v in zip(train_dict.keys(), train_vars)
}
duration = time.time() - start_time
train_losses[step] = it_train_dict['train_loss']
train_accs[step] = it_train_dict['train_accuracy_0']
train_images[step] = it_train_dict['train_images']
train_labels[step] = it_train_dict['train_labels']
train_scores[step] = it_train_dict['train_scores']
timesteps[step] = duration
if train_aux_check:
# Loop through to find aux scores
it_train_aux = {
itk: itv
for itk, itv in it_train_dict.iteritems()
if 'aux_score' in itk
}
train_aux[step] = it_train_aux
assert not np.isnan(it_train_dict['train_loss']).any(
), 'Model diverged with loss = NaN'
if step % config.validation_iters == 0:
it_val_scores, it_val_labels, it_val_aux, it_val_grads, it_val_ims = [], [], [], [], []
for num_vals in range(config.num_validation_evals):
# Validation accuracy as the average of n batches
val_vars = sess.run(val_dict.values())
it_val_dict = {
k: v
for k, v in zip(val_dict.keys(), val_vars)
}
it_val_labels += [it_val_dict['val_labels']]
it_val_scores += [it_val_dict['val_scores']]
it_val_grads += [it_val_dict['val_grad_images']]
it_val_ims += [it_val_dict['val_images']]
if val_aux_check:
iva = {
itk: itv
for itk, itv in it_val_dict.iteritems()
if 'aux_score' in itk
}
it_val_aux += [iva]
val_scores[step] = it_val_scores
val_labels[step] = it_val_labels
val_aux[step] = it_val_aux
val_images[step] = it_val_grads
val_grads[step] = it_val_ims
# End iteration
step += 1
except tf.errors.OutOfRangeError:
print 'Done with evaluation for %d epochs, %d steps.' % (config.epochs,
step)
print 'Saved to: %s' % checkpoint_dir
finally:
coord.request_stop()
coord.join(threads)
sess.close()
import ipdb
ipdb.set_trace()
np.savez(
'val_imgs_grads',
val_images=val_images, # it_val_dict['val_images'],
val_grads=val_grads, # it_val_dict['val_grad_images'],
val_labels=val_labels, # it_val_dict['val_labels'],
val_scores=val_scores) # it_val_dict['val_scores'][0])
def main(
experiment_name,
list_experiments=False,
load_and_evaluate_ckpt=None,
placeholder_data=None,
grad_images=False,
gpu_device='/gpu:0'):
"""Create a tensorflow worker to run experiments in your DB."""
if list_experiments:
exps = db.list_experiments()
print '_' * 30
print 'Initialized experiments:'
print '_' * 30
for l in exps:
print l.values()[0]
print '_' * 30
if len(exps) == 0:
print 'No experiments found.'
else:
print 'You can add to the DB with: '\
'python prepare_experiments.py --experiment=%s' % \
exps[0].values()[0]
return
if experiment_name is None:
print 'No experiment specified. Pulling one out of the DB.'
experiment_name = db.get_experiment_name()
# Prepare to run the model
config = Config()
condition_label = '%s_%s' % (experiment_name, py_utils.get_dt_stamp())
experiment_label = '%s' % (experiment_name)
log = logger.get(os.path.join(config.log_dir, condition_label))
assert experiment_name is not None, 'Empty experiment name.'
experiment_dict = experiments.experiments()[experiment_name]()
config = add_to_config(d=experiment_dict, config=config) # Globals
config.load_and_evaluate_ckpt = load_and_evaluate_ckpt
if load_and_evaluate_ckpt is not None:
# Remove the train operation and add a ckpt pointer
from ops import evaluation
config, exp_params = process_DB_exps(
experiment_name=experiment_name,
log=log,
config=config) # Update config w/ DB params
dataset_module = py_utils.import_module(
model_dir=config.dataset_info,
dataset=config.dataset)
dataset_module = dataset_module.data_processing() # hardcoded class name
train_key = [k for k in dataset_module.folds.keys() if 'train' in k]
if not len(train_key):
train_key = 'train'
else:
train_key = train_key[0]
train_data, train_means_image, train_means_label = get_data_pointers(
dataset=config.dataset,
base_dir=config.tf_records,
cv=train_key,
log=log)
val_key = [k for k in dataset_module.folds.keys() if 'val' in k]
if not len(val_key):
val_key = 'train'
else:
val_key = val_key[0]
val_data, val_means_image, val_means_label = get_data_pointers(
dataset=config.dataset,
base_dir=config.tf_records,
cv=val_key,
log=log)
# Initialize output folders
dir_list = {
'checkpoints': os.path.join(
config.checkpoints, condition_label),
'summaries': os.path.join(
config.summaries, condition_label),
'condition_evaluations': os.path.join(
config.condition_evaluations, condition_label),
'experiment_evaluations': os.path.join( # DEPRECIATED
config.experiment_evaluations, experiment_label),
'visualization': os.path.join(
config.visualizations, condition_label),
'weights': os.path.join(
config.condition_evaluations, condition_label, 'weights')
}
[py_utils.make_dir(v) for v in dir_list.values()]
# Prepare data loaders on the cpu
if all(isinstance(i, list) for i in config.data_augmentations):
if config.data_augmentations:
config.data_augmentations = py_utils.flatten_list(
config.data_augmentations,
log)
if load_and_evaluate_ckpt is not None:
config.epochs = 1
config.train_shuffle = False
config.val_shuffle = False
with tf.device('/cpu:0'):
if placeholder_data:
placeholder_shape = placeholder_data['train_image_shape']
placeholder_dtype = placeholder_data['train_image_dtype']
original_train_images = tf.placeholder(
dtype=placeholder_dtype,
shape=placeholder_shape,
name='train_images')
placeholder_shape = placeholder_data['train_label_shape']
placeholder_dtype = placeholder_data['train_label_dtype']
original_train_labels = tf.placeholder(
dtype=placeholder_dtype,
shape=placeholder_shape,
name='train_labels')
placeholder_shape = placeholder_data['val_image_shape']
placeholder_dtype = placeholder_data['val_image_dtype']
original_val_images = tf.placeholder(
dtype=placeholder_dtype,
shape=placeholder_shape,
name='val_images')
placeholder_shape = placeholder_data['val_label_shape']
placeholder_dtype = placeholder_data['val_label_dtype']
original_val_labels = tf.placeholder(
dtype=placeholder_dtype,
shape=placeholder_shape,
name='val_labels')
# Apply augmentations
(
train_images,
train_labels
) = data_loader.placeholder_image_augmentations(
images=original_train_images,
model_input_image_size=dataset_module.model_input_image_size,
labels=original_train_labels,
data_augmentations=config.data_augmentations,
batch_size=config.batch_size)
(
val_images,
val_labels
) = data_loader.placeholder_image_augmentations(
images=original_val_images,
model_input_image_size=dataset_module.model_input_image_size,
labels=original_val_labels,
data_augmentations=config.data_augmentations,
batch_size=config.batch_size)
# Store in the placeholder dict
placeholder_data['train_images'] = original_train_images
placeholder_data['train_labels'] = original_train_labels
placeholder_data['val_images'] = original_val_images
placeholder_data['val_labels'] = original_val_labels
else:
train_images, train_labels = data_loader.inputs(
dataset=train_data,
batch_size=config.batch_size,
model_input_image_size=dataset_module.model_input_image_size,
tf_dict=dataset_module.tf_dict,
data_augmentations=config.data_augmentations,
num_epochs=config.epochs,
tf_reader_settings=dataset_module.tf_reader,
shuffle=config.shuffle_train,
resize_output=config.resize_output)
if hasattr(config, 'val_augmentations'):
val_augmentations = config.val_augmentations
else:
val_augmentations = config.data_augmentations
val_images, val_labels = data_loader.inputs(
dataset=val_data,
batch_size=config.batch_size,
model_input_image_size=dataset_module.model_input_image_size,
tf_dict=dataset_module.tf_dict,
data_augmentations=val_augmentations,
num_epochs=config.epochs,
tf_reader_settings=dataset_module.tf_reader,
shuffle=config.shuffle_val,
resize_output=config.resize_output)
log.info('Created tfrecord dataloader tensors.')
# Load model specification
struct_name = config.model_struct.split(os.path.sep)[-1]
try:
model_dict = py_utils.import_module(
dataset=struct_name,
model_dir=os.path.join(
'models',
'structs',
experiment_name).replace(os.path.sep, '.')
)
except IOError:
print 'Could not find the model structure: %s in folder %s' % (
struct_name,
experiment_name)
# Inject model_dict with hyperparameters if requested
model_dict.layer_structure = hp_opt_utils.inject_model_with_hps(
layer_structure=model_dict.layer_structure,
exp_params=exp_params)
# Prepare variables for the models
if len(dataset_module.output_size) == 2:
log.warning(
'Found > 1 dimension for your output size.'
'Converting to a scalar.')
dataset_module.output_size = np.prod(
dataset_module.output_size)
if hasattr(model_dict, 'output_structure'):
# Use specified output layer
output_structure = model_dict.output_structure
else:
output_structure = None
# Correct number of output neurons if needed
if config.dataloader_override and\
'weights' in output_structure[-1].keys():
output_neurons = output_structure[-1]['weights'][0]
size_check = output_neurons != dataset_module.output_size
fc_check = output_structure[-1]['layers'][0] == 'fc'
if size_check and fc_check:
output_structure[-1]['weights'][0] = dataset_module.output_size
log.warning('Adjusted output neurons from %s to %s.' % (
output_neurons,
dataset_module.output_size))
# Prepare model on GPU
if not hasattr(dataset_module, 'input_normalization'):
dataset_module.input_normalization = None
with tf.device(gpu_device):
with tf.variable_scope('cnn') as scope:
# Training model
model = model_utils.model_class(
mean=train_means_image,
training=True,
output_size=dataset_module.output_size,
input_normalization=dataset_module.input_normalization)
train_scores, model_summary, _ = model.build(
data=train_images,
layer_structure=model_dict.layer_structure,
output_structure=output_structure,
log=log,
tower_name='cnn')
if grad_images:
oh_dims = int(train_scores.get_shape()[-1])
target_scores = tf.one_hot(train_labels, oh_dims) * train_scores
train_gradients = tf.gradients(target_scores, train_images)[0]
log.info('Built training model.')
log.debug(
json.dumps(model_summary, indent=4),
verbose=0)
print_model_architecture(model_summary)
# Normalize labels on GPU if needed
if 'normalize_labels' in exp_params.keys():
if exp_params['normalize_labels'] == 'zscore':
train_labels -= train_means_label['mean']
train_labels /= train_means_label['std']
val_labels -= train_means_label['mean']
val_labels /= train_means_label['std']
log.info('Z-scoring labels.')
elif exp_params['normalize_labels'] == 'mean':
train_labels -= train_means_label['mean']
val_labels -= val_means_label['mean']
log.info('Mean-centering labels.')
# Check the shapes of labels and scores
if not isinstance(train_scores, list):
if len(
train_scores.get_shape()) != len(
train_labels.get_shape()):
train_shape = train_scores.get_shape().as_list()
label_shape = train_labels.get_shape().as_list()
val_shape = val_scores.get_shape().as_list()
val_label_shape = val_labels.get_shape().as_list()
if len(
train_shape) == 2 and len(
label_shape) == 1 and train_shape[-1] == 1:
train_labels = tf.expand_dims(train_labels, axis=-1)
val_labels = tf.expand_dims(val_labels, axis=-1)
elif len(
train_shape) == 2 and len(
label_shape) == 1 and train_shape[-1] == 1:
train_scores = tf.expand_dims(train_scores, axis=-1)
val_scores = tf.expand_dims(val_scores, axis=-1)
# Prepare the loss function
train_loss, _ = loss_utils.loss_interpreter(
logits=train_scores, # TODO
labels=train_labels,
loss_type=config.loss_function,
weights=config.loss_weights,
dataset_module=dataset_module)
# Add loss tensorboard tracking
if isinstance(train_loss, list):
for lidx, tl in enumerate(train_loss):
tf.summary.scalar('training_loss_%s' % lidx, tl)
train_loss = tf.add_n(train_loss)
else:
tf.summary.scalar('training_loss', train_loss)
# Add weight decay if requested
if len(model.regularizations) > 0:
train_loss = loss_utils.wd_loss(
regularizations=model.regularizations,
loss=train_loss,
wd_penalty=config.regularization_strength)
assert config.lr is not None, 'No learning rate.' # TODO: Make a QC function
if config.lr > 1:
old_lr = config.lr
config.lr = loss_utils.create_lr_schedule(
train_batch=config.batch_size,
num_training=config.lr)
config.optimizer = 'momentum'
log.info('Forcing momentum classifier.')
else:
old_lr = None
train_op = loss_utils.optimizer_interpreter(
loss=train_loss,
lr=config.lr,
optimizer=config.optimizer,
constraints=config.optimizer_constraints,
model=model)
log.info('Built training loss function.')
# Add a score for the training set
train_accuracy = eval_metrics.metric_interpreter(
metric=dataset_module.score_metric, # TODO: Attach to exp cnfg
pred=train_scores, # TODO
labels=train_labels)
# Add aux scores if requested
train_aux = {}
if hasattr(dataset_module, 'aux_scores'):
for m in dataset_module.aux_scores:
train_aux[m] = eval_metrics.metric_interpreter(
metric=m,
pred=train_scores,
labels=train_labels) # [0] # TODO: Fix for multiloss
# Prepare remaining tensorboard summaries
if config.tensorboard_images:
if len(train_images.get_shape()) == 4:
tf_fun.image_summaries(train_images, tag='Training images')
if (np.asarray(
train_labels.get_shape().as_list()) > 1).sum() > 2:
tf_fun.image_summaries(
train_labels,
tag='Training_targets')
tf_fun.image_summaries(
train_scores,
tag='Training_predictions')
if isinstance(train_accuracy, list):
for tidx, ta in enumerate(train_accuracy):
tf.summary.scalar('training_accuracy_%s' % tidx, ta)
else:
tf.summary.scalar('training_accuracy', train_accuracy)
if config.pr_curve:
if isinstance(train_scores, list):
for pidx, train_score in enumerate(train_scores):
train_label = train_labels[:, pidx]
pr_summary.op(
tag='training_pr_%s' % pidx,
predictions=tf.cast(
tf.argmax(
train_score,
axis=-1),
tf.float32),
labels=tf.cast(train_label, tf.bool),
display_name='training_precision_recall_%s' % pidx)
else:
pr_summary.op(
tag='training_pr',
predictions=tf.cast(
tf.argmax(
train_scores,
axis=-1),
tf.float32),
labels=tf.cast(train_labels, tf.bool),
display_name='training_precision_recall')
log.info('Added training summaries.')
with tf.variable_scope('cnn', tf.AUTO_REUSE) as scope:
# Validation model
scope.reuse_variables()
val_model = model_utils.model_class(
mean=train_means_image, # Normalize with train data
training=False,
output_size=dataset_module.output_size,
input_normalization=dataset_module.input_normalization)
val_scores, _, _ = val_model.build( # Ignore summary
data=val_images,
layer_structure=model_dict.layer_structure,
output_structure=output_structure,
log=log,
tower_name='cnn')
if grad_images:
oh_dims = int(val_scores.get_shape()[-1])
target_scores = tf.one_hot(val_labels, oh_dims) * val_scores
val_gradients = tf.gradients(target_scores, val_images)[0]
log.info('Built validation model.')
# Check the shapes of labels and scores
val_loss, _ = loss_utils.loss_interpreter(
logits=val_scores,
labels=val_labels,
loss_type=config.loss_function,
weights=config.loss_weights,
dataset_module=dataset_module)
# Add loss tensorboard tracking
if isinstance(val_loss, list):
for lidx, tl in enumerate(val_loss):
tf.summary.scalar('validation_loss_%s' % lidx, tl)
val_loss = tf.add_n(val_loss)
else:
tf.summary.scalar('validation_loss', val_loss)
# Add a score for the validation set
val_accuracy = eval_metrics.metric_interpreter(
metric=dataset_module.score_metric, # TODO
pred=val_scores,
labels=val_labels)
# Add aux scores if requested
val_aux = {}
if hasattr(dataset_module, 'aux_scores'):
for m in dataset_module.aux_scores:
val_aux[m] = eval_metrics.metric_interpreter(
metric=m,
pred=val_scores,
labels=val_labels) # [0] # TODO: Fix for multiloss
# Prepare tensorboard summaries
if config.tensorboard_images:
if len(val_images.get_shape()) == 4:
tf_fun.image_summaries(
val_images,
tag='Validation')
if (np.asarray(
val_labels.get_shape().as_list()) > 1).sum() > 2:
tf_fun.image_summaries(
val_labels,
tag='Validation_targets')
tf_fun.image_summaries(
val_scores,
tag='Validation_predictions')
if isinstance(val_accuracy, list):
for vidx, va in enumerate(val_accuracy):
tf.summary.scalar('validation_accuracy_%s' % vidx, va)
else:
tf.summary.scalar('validation_accuracy', val_accuracy)
if config.pr_curve:
if isinstance(val_scores, list):
for pidx, val_score in enumerate(val_scores):
val_label = val_labels[:, pidx]
pr_summary.op(
tag='validation_pr_%s' % pidx,
predictions=tf.cast(
tf.argmax(
val_score,
axis=-1),
tf.float32),
labels=tf.cast(val_label, tf.bool),
display_name='validation_precision_recall_%s' %
pidx)
else:
pr_summary.op(
tag='validation_pr',
predictions=tf.cast(
tf.argmax(
val_scores,
axis=-1),
tf.float32),
labels=tf.cast(val_labels, tf.bool),
display_name='validation_precision_recall')
log.info('Added validation summaries.')
# Set up summaries and saver
if not hasattr(config, 'max_to_keep'):
config.max_to_keep = None
saver = tf.train.Saver(
var_list=tf.global_variables(),
max_to_keep=config.max_to_keep)
summary_op = tf.summary.merge_all()
# Initialize the graph
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
# Need to initialize both of these if supplying num_epochs to inputs
sess.run(
tf.group(
tf.global_variables_initializer(),
tf.local_variables_initializer())
)
summary_writer = tf.summary.FileWriter(dir_list['summaries'], sess.graph)
# Set up exemplar threading
if placeholder_data:
coord, threads = None, None
else:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# Create dictionaries of important training and validation information
train_dict = {
'train_loss': train_loss,
'train_images': train_images,
'train_labels': train_labels,
'train_op': train_op,
'train_scores': train_scores
}
val_dict = {
'val_loss': val_loss,
'val_images': val_images,
'val_labels': val_labels,
'val_scores': val_scores,
}
if grad_images:
train_dict['train_gradients'] = train_gradients
val_dict['val_gradients'] = val_gradients
if isinstance(train_accuracy, list):
for tidx, (ta, va) in enumerate(zip(train_accuracy, val_accuracy)):
train_dict['train_accuracy_%s' % tidx] = ta
val_dict['val_accuracy_%s' % tidx] = va
else:
train_dict['train_accuracy_0'] = train_accuracy
val_dict['val_accuracy_0'] = val_accuracy
if load_and_evaluate_ckpt is not None:
# Remove the train operation and add a ckpt pointer
del train_dict['train_op']
if hasattr(dataset_module, 'aux_score'):
# Attach auxillary scores to tensor dicts
for m in dataset_module.aux_scores:
train_dict['train_aux_%s' % m] = train_aux[m]
val_dict['val_aux_%s' % m] = val_aux[m]
# Start training loop
if old_lr is not None:
config.lr = old_lr
np.save(
os.path.join(
dir_list['condition_evaluations'], 'training_config_file'),
config)
log.info('Starting training')
if load_and_evaluate_ckpt is not None:
return evaluation.evaluation_loop(
config=config,
db=db,
coord=coord,
sess=sess,
summary_op=summary_op,
summary_writer=summary_writer,
saver=saver,
threads=threads,
summary_dir=dir_list['summaries'],
checkpoint_dir=dir_list['checkpoints'],
weight_dir=dir_list['weights'],
train_dict=train_dict,
val_dict=val_dict,
train_model=model,
val_model=val_model,
exp_params=exp_params,
placeholder_data=placeholder_data)
else:
output_dict = training.training_loop(
config=config,
db=db,
coord=coord,
sess=sess,
summary_op=summary_op,
summary_writer=summary_writer,
saver=saver,
threads=threads,
summary_dir=dir_list['summaries'],
checkpoint_dir=dir_list['checkpoints'],
weight_dir=dir_list['weights'],
train_dict=train_dict,
val_dict=val_dict,
train_model=model,
val_model=val_model,
exp_params=exp_params)
log.info('Finished training.')
model_name = config.model_struct.replace('/', '_')
if output_dict is not None:
py_utils.save_npys(
data=output_dict,
model_name=model_name,
output_string=dir_list['experiment_evaluations'])
def start_runs(logdir,
steps,
run_name,
thresholds,
mask_every_other_prediction=False):
"""Generate a PR curve with precision and recall evenly weighted.
Arguments:
logdir: The directory into which to store all the runs' data.
steps: The number of steps to run for.
run_name: The name of the run.
thresholds: The number of thresholds to use for PR curves.
mask_every_other_prediction: Whether to mask every other prediction by
alternating weights between 0 and 1.
"""
tf.reset_default_graph()
tf.set_random_seed(42)
# Create a normal distribution layer used to generate true color labels.
channel_distribution = tf.distributions.Normal(loc=0., scale=142.)
# Sample the distribution to generate colors. Lets generate different numbers
# of each color. The first dimension is the count of examples.
def clip(value, minValue, maxValue):
"""Clips an op to the range [minValue, maxValue].
For now, we intentionally avoid using tf.clip_by_value because it
apparently exhibits slightly different behavior based on system
characteristics. See tensorflow/tensorflow#18527. Tests rely on this demo,
so behavior must be consistent.
Args:
value: The value to clip.
minValue: The min value to clip by.
maxValue: The max value to clip by.
Returns:
A TensorFlow op that outputs the clipped value.
"""
return tf.maximum(minValue, tf.minimum(maxValue, value))
# Generate reds.
number_of_reds = 100
true_reds = clip(
tf.concat([
255. - tf.abs(channel_distribution.sample([number_of_reds, 1])),
tf.abs(channel_distribution.sample([number_of_reds, 2]))
],
axis=1), 0., 255.)
# Generate greens.
number_of_greens = 200
true_greens = clip(
tf.concat([
tf.abs(channel_distribution.sample([number_of_greens, 1])),
255. - tf.abs(channel_distribution.sample([number_of_greens, 1])),
tf.abs(channel_distribution.sample([number_of_greens, 1]))
],
axis=1), 0., 255.)
# Generate blues.
number_of_blues = 150
true_blues = clip(
tf.concat([
tf.abs(channel_distribution.sample([number_of_blues, 2])),
255. - tf.abs(channel_distribution.sample([number_of_blues, 1]))
],
axis=1), 0., 255.)
# Assign each color a vector of 3 booleans based on its true label.
labels = tf.concat([
tf.tile(tf.constant([[True, False, False]]), (number_of_reds, 1)),
tf.tile(tf.constant([[False, True, False]]), (number_of_greens, 1)),
tf.tile(tf.constant([[False, False, True]]), (number_of_blues, 1)),
],
axis=0)
# We introduce 3 normal distributions. They are used to predict whether a
# color falls under a certain class (based on distances from corners of the
# color triangle). The distributions vary per color. We have the distributions
# narrow over time.
initial_standard_deviations = [v + FLAGS.steps for v in (158, 200, 242)]
iteration = tf.placeholder(tf.int32, shape=[])
red_predictor = tf.distributions.Normal(
loc=0.,
scale=tf.cast(initial_standard_deviations[0] - iteration,
dtype=tf.float32))
green_predictor = tf.distributions.Normal(
loc=0.,
scale=tf.cast(initial_standard_deviations[1] - iteration,
dtype=tf.float32))
blue_predictor = tf.distributions.Normal(
loc=0.,
scale=tf.cast(initial_standard_deviations[2] - iteration,
dtype=tf.float32))
# Make predictions (assign 3 probabilities to each color based on each color's
# distance to each of the 3 corners). We seek double the area in the right
# tail of the normal distribution.
examples = tf.concat([true_reds, true_greens, true_blues], axis=0)
probabilities_colors_are_red = (1 - red_predictor.cdf(
tf.norm(examples - tf.constant([255., 0, 0]), axis=1))) * 2
probabilities_colors_are_green = (1 - green_predictor.cdf(
tf.norm(examples - tf.constant([0, 255., 0]), axis=1))) * 2
probabilities_colors_are_blue = (1 - blue_predictor.cdf(
tf.norm(examples - tf.constant([0, 0, 255.]), axis=1))) * 2
predictions = (probabilities_colors_are_red,
probabilities_colors_are_green,
probabilities_colors_are_blue)
# This is the crucial piece. We write data required for generating PR curves.
# We create 1 summary per class because we create 1 PR curve per class.
for i, color in enumerate(('red', 'green', 'blue')):
description = (
'The probabilities used to create this PR curve are '
'generated from a normal distribution. Its standard '
'deviation is initially %0.0f and decreases over time.' %
initial_standard_deviations[i])
weights = None
if mask_every_other_prediction:
# Assign a weight of 0 to every even-indexed prediction. Odd-indexed
# predictions are assigned a default weight of 1.
consecutive_indices = tf.reshape(tf.range(tf.size(predictions[i])),
tf.shape(predictions[i]))
weights = tf.cast(consecutive_indices % 2, dtype=tf.float32)
summary.op(name=color,
labels=labels[:, i],
predictions=predictions[i],
num_thresholds=thresholds,
weights=weights,
display_name='classifying %s' % color,
description=description)
merged_summary_op = tf.summary.merge_all()
events_directory = os.path.join(logdir, run_name)
sess = tf.Session()
writer = tf.summary.FileWriter(events_directory, sess.graph)
for step in xrange(steps):
feed_dict = {
iteration: step,
}
merged_summary = sess.run(merged_summary_op, feed_dict=feed_dict)
writer.add_summary(merged_summary, step)
writer.close()
def start_runs(
logdir,
steps,
run_name,
thresholds,
mask_every_other_prediction=False):
"""Generate a PR curve with precision and recall evenly weighted.
Arguments:
logdir: The directory into which to store all the runs' data.
steps: The number of steps to run for.
run_name: The name of the run.
thresholds: The number of thresholds to use for PR curves.
mask_every_other_prediction: Whether to mask every other prediction by
alternating weights between 0 and 1.
"""
tf.reset_default_graph()
tf.set_random_seed(42)
# Create a normal distribution layer used to generate true color labels.
distribution = tf.distributions.Normal(loc=0., scale=142.)
# Sample the distribution to generate colors. Lets generate different numbers
# of each color. The first dimension is the count of examples.
# The calls to sample() are given fixed random seed values that are "magic"
# in that they correspond to the default seeds for those ops when the PR
# curve test (which depends on this code) was written. We've pinned these
# instead of continuing to use the defaults since the defaults are based on
# node IDs from the sequence of nodes added to the graph, which can silently
# change when this code or any TF op implementations it uses are modified.
# TODO(nickfelt): redo the PR curve test to avoid reliance on random seeds.
# Generate reds.
number_of_reds = 100
true_reds = tf.clip_by_value(
tf.concat([
255 - tf.abs(distribution.sample([number_of_reds, 1], seed=11)),
tf.abs(distribution.sample([number_of_reds, 2], seed=34))
], axis=1),
0, 255)
# Generate greens.
number_of_greens = 200
true_greens = tf.clip_by_value(
tf.concat([
tf.abs(distribution.sample([number_of_greens, 1], seed=61)),
255 - tf.abs(distribution.sample([number_of_greens, 1], seed=82)),
tf.abs(distribution.sample([number_of_greens, 1], seed=105))
], axis=1),
0, 255)
# Generate blues.
number_of_blues = 150
true_blues = tf.clip_by_value(
tf.concat([
tf.abs(distribution.sample([number_of_blues, 2], seed=132)),
255 - tf.abs(distribution.sample([number_of_blues, 1], seed=153))
], axis=1),
0, 255)
# Assign each color a vector of 3 booleans based on its true label.
labels = tf.concat([
tf.tile(tf.constant([[True, False, False]]), (number_of_reds, 1)),
tf.tile(tf.constant([[False, True, False]]), (number_of_greens, 1)),
tf.tile(tf.constant([[False, False, True]]), (number_of_blues, 1)),
], axis=0)
# We introduce 3 normal distributions. They are used to predict whether a
# color falls under a certain class (based on distances from corners of the
# color triangle). The distributions vary per color. We have the distributions
# narrow over time.
initial_standard_deviations = [v + FLAGS.steps for v in (158, 200, 242)]
iteration = tf.placeholder(tf.int32, shape=[])
red_predictor = tf.distributions.Normal(
loc=0.,
scale=tf.cast(
initial_standard_deviations[0] - iteration,
dtype=tf.float32))
green_predictor = tf.distributions.Normal(
loc=0.,
scale=tf.cast(
initial_standard_deviations[1] - iteration,
dtype=tf.float32))
blue_predictor = tf.distributions.Normal(
loc=0.,
scale=tf.cast(
initial_standard_deviations[2] - iteration,
dtype=tf.float32))
# Make predictions (assign 3 probabilities to each color based on each color's
# distance to each of the 3 corners). We seek double the area in the right
# tail of the normal distribution.
examples = tf.concat([true_reds, true_greens, true_blues], axis=0)
probabilities_colors_are_red = (1 - red_predictor.cdf(
tf.norm(examples - tf.constant([255., 0, 0]), axis=1))) * 2
probabilities_colors_are_green = (1 - green_predictor.cdf(
tf.norm(examples - tf.constant([0, 255., 0]), axis=1))) * 2
probabilities_colors_are_blue = (1 - blue_predictor.cdf(
tf.norm(examples - tf.constant([0, 0, 255.]), axis=1))) * 2
predictions = (
probabilities_colors_are_red,
probabilities_colors_are_green,
probabilities_colors_are_blue
)
# This is the crucial piece. We write data required for generating PR curves.
# We create 1 summary per class because we create 1 PR curve per class.
for i, color in enumerate(('red', 'green', 'blue')):
description = ('The probabilities used to create this PR curve are '
'generated from a normal distribution. Its standard '
'deviation is initially %0.0f and decreases over time.' %
initial_standard_deviations[i])
weights = None
if mask_every_other_prediction:
# Assign a weight of 0 to every even-indexed prediction. Odd-indexed
# predictions are assigned a default weight of 1.
consecutive_indices = tf.reshape(
tf.range(tf.size(predictions[i])), tf.shape(predictions[i]))
weights = tf.cast(consecutive_indices % 2, dtype=tf.float32)
summary.op(
name=color,
labels=labels[:, i],
predictions=predictions[i],
num_thresholds=thresholds,
weights=weights,
display_name='classifying %s' % color,
description=description)
merged_summary_op = tf.summary.merge_all()
events_directory = os.path.join(logdir, run_name)
sess = tf.Session()
writer = tf.summary.FileWriter(events_directory, sess.graph)
for step in xrange(steps):
feed_dict = {
iteration: step,
}
merged_summary = sess.run(merged_summary_op, feed_dict=feed_dict)
writer.add_summary(merged_summary, step)
writer.close()