def build_test_metrics(self):
dense_labels = dense_labels_fn(tf.shape(self.prediction),
self.timestamps, self.mmsis)
thresholded_prediction = tf.to_int32(self.prediction > 0.5)
valid = tf.to_int32(tf.not_equal(dense_labels, -1))
ones = tf.to_int32(dense_labels > 0.5)
weights = tf.to_float(valid)
prediction = self.prediction
unclear = tf.to_int32((self.prediction > 0.333)
& (self.prediction < 0.666))
if eval_window:
b, e = eval_window
prediction = prediction[:, b:e]
dense_labels = dense_labels[:, b:e]
thresholded_prediction = thresholded_prediction[:, b:e]
valid = valid[:, b:e]
ones = ones[:, b:e]
weights = weights[:, b:e]
unclear = unclear[:, b:e]
recall = slim.metrics.streaming_recall(thresholded_prediction,
ones,
weights=weights)
precision = slim.metrics.streaming_precision(
thresholded_prediction, ones, weights=weights)
raw_metrics = {
'Test-MSE':
slim.metrics.streaming_mean_squared_error(
prediction, tf.to_float(ones), weights=weights),
'Test-accuracy':
slim.metrics.streaming_accuracy(thresholded_prediction,
ones,
weights=weights),
'Test-precision':
precision,
'Test-recall':
recall,
'Test-F1-score':
f1(recall, precision),
'Test-prediction-fraction':
slim.metrics.streaming_accuracy(thresholded_prediction,
valid,
weights=weights),
'Test-unclear-fraction':
slim.metrics.streaming_accuracy(unclear,
valid,
weights=weights),
'Test-label-fraction':
slim.metrics.streaming_accuracy(ones,
valid,
weights=weights)
}
return metrics.aggregate_metric_map({
"{}/{}".format(self.name, k): v
for (k, v) in raw_metrics.items()
})
def build_test_metrics(self):
raw_loss = self.masked_mean_error(self.prediction, self.mmsis)
return metrics.aggregate_metric_map({
'%s/Test-error' % self.name:
metrics.streaming_mean(raw_loss)
})
def eval(self, input_sequence, output_sequence, sequence_length,
control_sequence=None):
"""Evaluate on the given sequences, returning metric update ops.
Args:
input_sequence: The sequence to be fed to the encoder.
output_sequence: The sequence expected from the decoder.
sequence_length: The length of the given sequences (which must be
identical).
control_sequence: (Optional) sequence on which to condition the decoder.
Returns:
metric_update_ops: tf.metrics update ops.
"""
metric_map, scalars_to_summarize = self._compute_model_loss(
input_sequence, output_sequence, sequence_length, control_sequence)
for n, t in scalars_to_summarize.items():
metric_map[n] = tf.metrics.mean(t)
metrics_to_values, metrics_to_updates = (
contrib_metrics.aggregate_metric_map(metric_map))
for metric_name, metric_value in metrics_to_values.items():
tf.summary.scalar(metric_name, metric_value)
return list(metrics_to_updates.values())
def main(_):
assert FLAGS.train_dir, "--train_dir is required."
if tf.gfile.Exists(FLAGS.summaries_dir):
tf.gfile.DeleteRecursively(FLAGS.summaries_dir)
tf.gfile.MakeDirs(FLAGS.summaries_dir)
config = configuration.Config()
dataset_eval = loader.get_split(FLAGS.split_name,
dataset_dir=FLAGS.data_dir)
if FLAGS.preprocess_abs:
preprocess_fn = tf.abs
else:
preprocess_fn = None
# whther it is a 2d input
is_2D = common.is_2D(FLAGS.model)
series, labels, labels_one_hot = loader.load_batch(
dataset_eval,
batch_size=config.batch_size,
is_2D=is_2D,
preprocess_fn=preprocess_fn)
# Build lazy model
model = common.convert_name_to_instance(FLAGS.model, config, 'eval')
endpoints = model.build(inputs=series, is_training=False)
predictions = tf.to_int64(tf.argmax(endpoints.logits, 1))
slim.get_or_create_global_step()
# Choose the metrics to compute:
names_to_values, names_to_updates = metrics.aggregate_metric_map({
'accuracy':
metrics.streaming_accuracy(predictions, labels),
'precision':
metrics.streaming_precision(predictions, labels),
'recall':
metrics.streaming_recall(predictions, labels),
})
# Create the summary ops such that they also print out to std output:
summary_ops = []
for metric_name, metric_value in names_to_values.iteritems():
op = tf.summary.scalar(metric_name, metric_value)
op = tf.Print(op, [metric_value], metric_name)
summary_ops.append(op)
slim.evaluation.evaluation_loop(
master='',
checkpoint_dir=FLAGS.train_dir,
logdir=FLAGS.summaries_dir,
eval_op=names_to_updates.values(),
num_evals=min(FLAGS.num_batches, dataset_eval.num_samples),
eval_interval_secs=FLAGS.eval_interval_secs,
max_number_of_evaluations=FLAGS.num_of_steps,
summary_op=tf.summary.merge(summary_ops),
session_config=config.session_config,
)
def metric_def(self):
self.training_metrics = {
"learning_rate": self.learning_rate,
"adam_beta_1": self.adam_beta_1,
"adam_beta_2": self.adam_beta_2,
"ema_decay": self.ema_decay,
"cons_coefficient": self.cons_coefficient,
"train/error/1": self.mean_error_1,
"train/error/ema": self.mean_error_ema,
"train/class_cost/1": self.mean_class_cost_1,
"train/class_cost/ema": self.mean_class_cost_ema,
"train/cons_cost/mt": self.mean_cons_cost_mt,
"train/total_cost/mt": self.mean_total_cost_mt,
}
with tf.variable_scope("validation_metrics") as metrics_scope:
self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map(
{
"eval/error/1": streaming_mean(self.errors_1),
"eval/error/ema": streaming_mean(self.errors_ema),
"eval/class_cost/1": streaming_mean(self.class_costs_1),
"eval/class_cost/ema":
streaming_mean(self.class_costs_ema),
})
metric_variables = slim.get_local_variables(
scope=metrics_scope.name)
self.metric_init_op = tf.variables_initializer(metric_variables)
self.result_formatter = string_utils.DictFormatter(
order=[
"eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"
],
default_format='{name}: {value:>10.6f}',
separator=", ")
self.result_formatter.add_format('error', '{name}: {value:>6.1%}')
def evaluate():
with tf.Graph().as_default():
config = tf.ConfigProto(device_count={'GPU': 0})
images, labels = utils.load_batch(shards=VAL_SHARDS,
batch_size=FLAGS.batch_size,
train=False,
crop=False,
flip=False)
predictions = alexnet.AlexNet(images,
batch_size=FLAGS.batch_size).model
prediction = tf.to_int64(tf.argmax(predictions,
1)) # Returns index of largest
mse_op = metrics.streaming_mean_squared_error(prediction, labels)
rmse_op = metrics.streaming_root_mean_squared_error(prediction, labels)
accuracy_op = metrics.streaming_accuracy(prediction, labels)
precision_op = metrics.streaming_precision(prediction, labels)
metrics_to_values, metrics_to_updates = metrics.aggregate_metric_map({
'mse':
mse_op,
'rmse':
rmse_op,
'accuracy':
accuracy_op,
'precision':
precision_op,
})
for metric_name, metric_value in metrics_to_values.items():
tf.summary.scalar(metric_name, metric_value)
slim.evaluation.evaluation_loop('',
FLAGS.trainlog_dir,
FLAGS.evallog_dir,
num_evals=FLAGS.num_evals,
eval_op=list(
metrics_to_updates.values()),
eval_interval_secs=5,
session_config=config)
'''checkpoint_list = [FLAGS.trainlog_momentum_dir,
def build_test_metrics(self):
raw_labels = multihot_labels(self.mmsis)
mask = tf.to_float(tf.equal(tf.reduce_sum(raw_labels, 1), 1))
labels = tf.to_int32(tf.argmax(raw_labels, 1))
predictions = tf.to_int32(tf.argmax(self.prediction, 1))
metrics_map = {
'%s/Test-accuracy' % self.name:
metrics.streaming_accuracy(predictions,
labels,
weights=mask)
}
if self.metrics == 'all':
for i, cls in enumerate(self.classes):
cls_name = cls.replace(' ', '-')
trues = tf.to_int32(tf.equal(labels, i))
preds = tf.to_int32(tf.equal(predictions, i))
recall = metrics.streaming_recall(preds,
trues,
weights=mask)
precision = metrics.streaming_precision(preds,
trues,
weights=mask)
metrics_map["%s/Class-%s-Precision" %
(self.name, cls_name)] = recall
metrics_map["%s/Class-%s-Recall" %
(self.name, cls_name)] = precision
metrics_map["%s/Class-%s-F1-Score" %
(self.name, cls_name)] = f1(
recall, precision)
metrics_map["%s/Class-%s-ROC-AUC" %
(self.name,
cls_name)] = metrics.streaming_auc(
self.prediction[:, i],
trues,
weights=mask)
return metrics.aggregate_metric_map(metrics_map)
def __init__(self, result_dir):
self.checkpoint_dir = os.path.join(result_dir, 'checkpoints')
self.summary_dir = os.path.join(result_dir, 'summaries')
os.makedirs(self.checkpoint_dir)
os.makedirs(self.summary_dir)
with tf.name_scope("placeholders"):
self.images = tf.placeholder(dtype=tf.float32,
shape=(None, 32, 32, 3),
name='images')
self.labels = tf.placeholder(dtype=tf.int32,
shape=(None, ),
name='labels')
self.is_training = tf.placeholder(dtype=tf.bool,
shape=(),
name='is_training')
self.global_step = tf.Variable(0, trainable=False, name='global_step')
tf.add_to_collection("init_in_init", self.global_step)
self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS)
for var in self.hyper.variables.values():
tf.add_to_collection("init_in_init", var)
with tf.name_scope("ramps"):
sigmoid_rampup_value = sigmoid_rampup(self.global_step,
self.hyper['rampup_length'])
sigmoid_rampdown_value = sigmoid_rampdown(
self.global_step, self.hyper['rampdown_length'],
self.hyper['training_length'])
self.learning_rate = tf.multiply(sigmoid_rampup_value *
sigmoid_rampdown_value,
self.hyper['max_learning_rate'],
name='learning_rate')
self.adam_beta_1 = tf.add(
sigmoid_rampdown_value *
self.hyper['adam_beta_1_before_rampdown'],
(1 - sigmoid_rampdown_value) *
self.hyper['adam_beta_1_after_rampdown'],
name='adam_beta_1')
self.cons_coefficient = tf.multiply(
sigmoid_rampup_value,
self.hyper['max_consistency_coefficient'],
name='consistency_coefficient')
step_rampup_value = step_rampup(self.global_step,
self.hyper['rampup_length'])
self.adam_beta_2 = tf.add(
(1 - step_rampup_value) *
self.hyper['adam_beta_2_during_rampup'],
step_rampup_value * self.hyper['adam_beta_2_after_rampup'],
name='adam_beta_2')
self.ema_decay = tf.add(
(1 - step_rampup_value) *
self.hyper['ema_decay_during_rampup'],
step_rampup_value * self.hyper['ema_decay_after_rampup'],
name='ema_decay')
self.logits_1, self.logits_2, self.logits_ema = inference(
self.images,
is_training=self.is_training,
ema_decay=self.ema_decay,
normalize_input=self.hyper['normalize_input'],
flip_horizontally=self.hyper['flip_horizontally'])
with tf.name_scope("objectives"):
self.mean_error_1, self.errors_1 = errors(self.logits_1,
self.labels)
self.mean_error_ema, self.errors_ema = errors(
self.logits_ema, self.labels)
self.mean_class_cost_1, self.class_costs_1 = classification_costs(
self.logits_1, self.labels)
self.mean_class_cost_ema, self.class_costs_ema = classification_costs(
self.logits_ema, self.labels)
labeled_consistency = self.hyper['apply_consistency_to_labeled']
consistency_mask = tf.logical_or(tf.equal(self.labels, -1),
labeled_consistency)
self.mean_cons_cost_pi, self.cons_costs_pi = consistency_costs(
self.logits_1, self.logits_2, self.cons_coefficient,
consistency_mask)
self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs(
self.logits_1, self.logits_ema, self.cons_coefficient,
consistency_mask)
self.mean_total_cost_pi, self.total_costs_pi = total_costs(
self.class_costs_1, self.cons_costs_pi)
self.mean_total_cost_mt, self.total_costs_mt = total_costs(
self.class_costs_1, self.cons_costs_mt)
self.cost_to_be_minimized = tf.cond(
self.hyper['ema_consistency'], lambda: self.mean_total_cost_mt,
lambda: self.mean_total_cost_pi)
with tf.name_scope("train_step"):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self.train_step_op = nn.adam_optimizer(
self.cost_to_be_minimized,
self.global_step,
learning_rate=self.learning_rate,
beta1=self.adam_beta_1,
beta2=self.adam_beta_2,
epsilon=self.hyper['adam_epsilon'])
self.training_control = training_control(self.global_step,
self.hyper['print_span'],
self.hyper['evaluation_span'],
self.hyper['training_length'])
self.training_metrics = {
"learning_rate": self.learning_rate,
"adam_beta_1": self.adam_beta_1,
"adam_beta_2": self.adam_beta_2,
"ema_decay": self.ema_decay,
"cons_coefficient": self.cons_coefficient,
"train/error/1": self.mean_error_1,
"train/error/ema": self.mean_error_ema,
"train/class_cost/1": self.mean_class_cost_1,
"train/class_cost/ema": self.mean_class_cost_ema,
"train/cons_cost/pi": self.mean_cons_cost_pi,
"train/cons_cost/mt": self.mean_cons_cost_mt,
"train/total_cost/pi": self.mean_total_cost_pi,
"train/total_cost/mt": self.mean_total_cost_mt,
}
with tf.variable_scope("validation_metrics") as metrics_scope:
self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map(
{
"eval/error/1": streaming_mean(self.errors_1),
"eval/error/ema": streaming_mean(self.errors_ema),
# Note that the evaluation costs are not directly comparable
# to the training costs. Training batches contain unlabeled
# samples but the evaluation batches do not. Because
# classification cost is zero for unlabeled samples, the
# training costs are smaller than evaluation costs when
# doing semi-supervised learning.
"eval/class_cost/1": streaming_mean(self.class_costs_1),
"eval/class_cost/ema":
streaming_mean(self.class_costs_ema),
"eval/cons_cost/pi": streaming_mean(self.cons_costs_pi),
"eval/cons_cost/mt": streaming_mean(self.cons_costs_mt),
"eval/total_cost/pi": streaming_mean(self.total_costs_pi),
"eval/total_cost/mt": streaming_mean(self.total_costs_mt)
})
metric_variables = slim.get_local_variables(
scope=metrics_scope.name)
self.metric_init_op = tf.variables_initializer(metric_variables)
with tf.name_scope("initializers"):
init_init_variables = tf.get_collection("init_in_init")
train_init_variables = [
var for var in tf.global_variables()
if var not in init_init_variables
]
self.init_init_op = tf.variables_initializer(init_init_variables)
self.train_init_op = tf.variables_initializer(train_init_variables)
self.saver = tf.train.Saver()
self.session = tf.Session()
self.run(self.init_init_op)
def evaluate_model(config):
""" Train the model using the passed in config """
###########################################################
# Create the input pipeline
###########################################################
with tf.name_scope('input_pipeline'):
dataset = input_utils.get_dataset(config.datadir, config.dataset,
config.datasubset)
init_op, init_feed_dict, image, label = input_utils.get_data(
config.dataset,
dataset,
config.batch_size,
num_epochs=config.num_epochs,
num_readers=config.num_readers)
images, labels = tf.train.batch(
[image, label],
config.batch_size,
num_threads=config.num_preprocessing_threads,
capacity=5 * config.batch_size)
###########################################################
# Generate the model
###########################################################
outputs = create_model(config, images, dataset)
tfprof.model_analyzer.print_model_analysis(tf.get_default_graph())
###########################################################
# Setup the evaluation metrics and summaries
###########################################################
summaries = []
metrics_map = {}
for metric in tf.get_collection(graph_utils.GraphKeys.METRICS):
metrics_map[metric.op.name] = metrics.streaming_mean(metric)
predictions = tf.argmax(outputs, 1)
metrics_map['accuracy'] = metrics.streaming_accuracy(predictions, labels)
metrics_map['recall_5'] = metrics.streaming_sparse_recall_at_k(
outputs, tf.expand_dims(labels, 1), 5)
names_to_values, names_to_updates = metrics.aggregate_metric_map(
metrics_map)
# Create summaries of the metrics and print them to the screen
for name, value in names_to_values.iteritems():
summary = tf.summary.scalar(name, value, collections=[])
summaries.append(tf.Print(summary, [value], name))
summaries.extend(layers.summarize_collection(
graph_utils.GraphKeys.METRICS))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.QUANTIZED_VARIABLES))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS))
tiled_images = image_utils.tile_images(images)
summaries.append(tf.summary.image('input_batch', tiled_images))
summary_op = tf.summary.merge(summaries, name='summaries')
###########################################################
# Begin evaluation
###########################################################
checkpoint_path = FLAGS.checkpoint_path
eval_ops = tf.group(*names_to_updates.values())
scaffold = tf.train.Scaffold(init_op, init_feed_dict)
hooks = [
training.SummaryAtEndHook(FLAGS.log_dir, summary_op),
training.StopAfterNEvalsHook(
math.ceil(dataset.num_samples / float(config.batch_size)))
]
eval_kwargs = {}
eval_fn = training.evaluate_repeatedly
if FLAGS.once:
if tf.gfile.IsDirectory(checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(checkpoint_path)
eval_fn = training.evaluate_once
else:
assert tf.gfile.IsDirectory(checkpoint_path), (
'checkpoint path must be a directory when using loop evaluation')
# On Tensorflow master fd87896 fixes this, but for now just set a very large number
eval_kwargs['max_number_of_evaluations'] = sys.maxint
eval_fn(checkpoint_path,
scaffold=scaffold,
hooks=hooks,
eval_ops=eval_ops,
**eval_kwargs)
def __init__(self, run_context=None):
if run_context is not None:
self.training_log = run_context.create_train_log('training')
self.validation_log = run_context.create_train_log('validation')
self.checkpoint_path = os.path.join(run_context.transient_dir,
'checkpoint')
self.tensorboard_path = os.path.join(run_context.result_dir,
'tensorboard')
with tf.name_scope("placeholders"):
self.images = tf.placeholder(dtype=tf.float32,
shape=(None, 32, 32, 3),
name='images')
self.labels = tf.placeholder(dtype=tf.int32,
shape=(None, ),
name='labels')
self.is_training = tf.placeholder(dtype=tf.bool,
shape=(),
name='is_training')
self.global_step = tf.Variable(0, trainable=False, name='global_step')
tf.add_to_collection("init_in_init", self.global_step)
self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS)
for var in self.hyper.variables.values():
tf.add_to_collection("init_in_init", var)
with tf.name_scope("ramps"):
sigmoid_rampup_value = sigmoid_rampup(self.global_step,
self.hyper['rampup_length'])
sigmoid_rampdown_value = sigmoid_rampdown(
self.global_step, self.hyper['rampdown_length'],
self.hyper['training_length'])
self.learning_rate = tf.multiply(sigmoid_rampup_value *
sigmoid_rampdown_value,
self.hyper['max_learning_rate'],
name='learning_rate')
self.adam_beta_1 = tf.add(
sigmoid_rampdown_value *
self.hyper['adam_beta_1_before_rampdown'],
(1 - sigmoid_rampdown_value) *
self.hyper['adam_beta_1_after_rampdown'],
name='adam_beta_1')
self.cons_coefficient = tf.multiply(
sigmoid_rampup_value,
self.hyper['max_consistency_cost'],
name='consistency_coefficient')
step_rampup_value = step_rampup(self.global_step,
self.hyper['rampup_length'])
self.adam_beta_2 = tf.add(
(1 - step_rampup_value) *
self.hyper['adam_beta_2_during_rampup'],
step_rampup_value * self.hyper['adam_beta_2_after_rampup'],
name='adam_beta_2')
self.ema_decay = tf.add(
(1 - step_rampup_value) *
self.hyper['ema_decay_during_rampup'],
step_rampup_value * self.hyper['ema_decay_after_rampup'],
name='ema_decay')
((self.class_logits_1, self.cons_logits_1), (self.class_logits_2,
self.cons_logits_2),
(self.class_logits_ema, self.cons_logits_ema)) = inference(
self.images,
is_training=self.is_training,
ema_decay=self.ema_decay,
input_noise=self.hyper['input_noise'],
student_dropout_probability=self.
hyper['student_dropout_probability'],
teacher_dropout_probability=self.
hyper['teacher_dropout_probability'],
normalize_input=self.hyper['normalize_input'],
flip_horizontally=self.hyper['flip_horizontally'],
translate=self.hyper['translate'],
num_logits=self.hyper['num_logits'])
with tf.name_scope("objectives"):
self.mean_error_1, self.errors_1 = errors(self.class_logits_1,
self.labels)
self.mean_error_ema, self.errors_ema = errors(
self.class_logits_ema, self.labels)
self.mean_class_cost_1, self.class_costs_1 = classification_costs(
self.class_logits_1, self.labels)
self.mean_class_cost_ema, self.class_costs_ema = classification_costs(
self.class_logits_ema, self.labels)
labeled_consistency = self.hyper['apply_consistency_to_labeled']
consistency_mask = tf.logical_or(tf.equal(self.labels, -1),
labeled_consistency)
self.mean_cons_cost_pi, self.cons_costs_pi = consistency_costs(
self.cons_logits_1, self.class_logits_2, self.cons_coefficient,
consistency_mask, self.hyper['consistency_trust'])
self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs(
self.cons_logits_1, self.class_logits_ema,
self.cons_coefficient, consistency_mask,
self.hyper['consistency_trust'])
def l2_norms(matrix):
l2s = tf.reduce_sum(matrix**2, axis=1)
mean_l2 = tf.reduce_mean(l2s)
return mean_l2, l2s
self.mean_res_l2_1, self.res_l2s_1 = l2_norms(self.class_logits_1 -
self.cons_logits_1)
self.mean_res_l2_ema, self.res_l2s_ema = l2_norms(
self.class_logits_ema - self.cons_logits_ema)
self.res_costs_1 = self.hyper[
'logit_distance_cost'] * self.res_l2s_1
self.mean_res_cost_1 = tf.reduce_mean(self.res_costs_1)
self.res_costs_ema = self.hyper[
'logit_distance_cost'] * self.res_l2s_ema
self.mean_res_cost_ema = tf.reduce_mean(self.res_costs_ema)
self.mean_total_cost_pi, self.total_costs_pi = total_costs(
self.class_costs_1, self.cons_costs_pi, self.res_costs_1)
self.mean_total_cost_mt, self.total_costs_mt = total_costs(
self.class_costs_1, self.cons_costs_mt, self.res_costs_1)
assert_shape(self.total_costs_pi, [3])
assert_shape(self.total_costs_mt, [3])
self.cost_to_be_minimized = tf.cond(
self.hyper['ema_consistency'], lambda: self.mean_total_cost_mt,
lambda: self.mean_total_cost_pi)
with tf.name_scope("train_step"):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self.train_step_op = nn.adam_optimizer(
self.cost_to_be_minimized,
self.global_step,
learning_rate=self.learning_rate,
beta1=self.adam_beta_1,
beta2=self.adam_beta_2,
epsilon=self.hyper['adam_epsilon'])
self.training_control = training_control(self.global_step,
self.hyper['print_span'],
self.hyper['evaluation_span'],
self.hyper['training_length'])
self.training_metrics = {
"learning_rate": self.learning_rate,
"adam_beta_1": self.adam_beta_1,
"adam_beta_2": self.adam_beta_2,
"ema_decay": self.ema_decay,
"cons_coefficient": self.cons_coefficient,
"train/error/1": self.mean_error_1,
"train/error/ema": self.mean_error_ema,
"train/class_cost/1": self.mean_class_cost_1,
"train/class_cost/ema": self.mean_class_cost_ema,
"train/cons_cost/pi": self.mean_cons_cost_pi,
"train/cons_cost/mt": self.mean_cons_cost_mt,
"train/res_cost/1": self.mean_res_cost_1,
"train/res_cost/ema": self.mean_res_cost_ema,
"train/total_cost/pi": self.mean_total_cost_pi,
"train/total_cost/mt": self.mean_total_cost_mt,
}
with tf.variable_scope("validation_metrics") as metrics_scope:
self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map(
{
"eval/error/1": streaming_mean(self.errors_1),
"eval/error/ema": streaming_mean(self.errors_ema),
"eval/class_cost/1": streaming_mean(self.class_costs_1),
"eval/class_cost/ema":
streaming_mean(self.class_costs_ema),
"eval/res_cost/1": streaming_mean(self.res_costs_1),
"eval/res_cost/ema": streaming_mean(self.res_costs_ema),
})
metric_variables = slim.get_local_variables(
scope=metrics_scope.name)
self.metric_init_op = tf.variables_initializer(metric_variables)
self.result_formatter = string_utils.DictFormatter(
order=[
"eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"
],
default_format='{name}: {value:>10.6f}',
separator=", ")
self.result_formatter.add_format('error', '{name}: {value:>6.1%}')
with tf.name_scope("initializers"):
init_init_variables = tf.get_collection("init_in_init")
train_init_variables = [
var for var in tf.global_variables()
if var not in init_init_variables
]
self.init_init_op = tf.variables_initializer(init_init_variables)
self.train_init_op = tf.variables_initializer(train_init_variables)
self.saver = tf.train.Saver()
self.session = tf.Session()
self.run(self.init_init_op)
def evaluate_model(config):
""" Train the model using the passed in config """
###########################################################
# Create the input pipeline
###########################################################
with tf.name_scope('input_pipeline'):
dataset = input_utils.get_dataset(config.datadir,
config.dataset,
config.datasubset,
num_folds=config.fold_count,
fold=config.fold,
holdout=True)
init_op, init_feed_dict, image = input_utils.get_data(
config.dataset,
dataset,
config.batch_size,
num_epochs=config.num_epochs,
num_readers=config.num_readers)
images = tf.train.batch([image],
config.batch_size,
num_threads=config.num_preprocessing_threads,
capacity=5 * config.batch_size)
###########################################################
# Generate the model
###########################################################
outputs = create_model(config, images, dataset)
###########################################################
# Setup the evaluation metrics and summaries
###########################################################
summaries = []
metrics_map = {}
for loss in tf.losses.get_losses():
metrics_map[loss.op.name] = metrics.streaming_mean(loss)
for metric in tf.get_collection(graph_utils.GraphKeys.METRICS):
metrics_map[metric.op.name] = metrics.streaming_mean(metric)
total_loss = tf.losses.get_total_loss()
metrics_map[total_loss.op.name] = metrics.streaming_mean(total_loss)
names_to_values, names_to_updates = metrics.aggregate_metric_map(
metrics_map)
# Create summaries of the metrics and print them to the screen
for name, value in names_to_values.iteritems():
summary = tf.summary.scalar(name, value, collections=[])
summaries.append(tf.Print(summary, [value], name))
summaries.extend(layers.summarize_collection(tf.GraphKeys.MODEL_VARIABLES))
summaries.extend(layers.summarize_collection(
graph_utils.GraphKeys.METRICS))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.RNN_OUTPUTS))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS))
images = input_utils.reshape_images(images, config.dataset)
tiled_images = image_utils.tile_images(images)
summaries.append(tf.summary.image('input_batch', tiled_images))
# Generate the canvases that lead to the final output image
with tf.name_scope('canvases'):
for step, canvas in enumerate(outputs):
canvas = input_utils.reshape_images(canvas, config.dataset)
tiled_images = image_utils.tile_images(canvas)
summaries.append(
tf.summary.image('step{0}'.format(step), tiled_images))
summary_op = tf.summary.merge(summaries, name='summaries')
###########################################################
# Begin evaluation
###########################################################
checkpoint_path = FLAGS.checkpoint_path
eval_ops = tf.group(*names_to_updates.values())
hooks = [
training.SummaryAtEndHook(log_dir=FLAGS.log_dir,
summary_op=summary_op),
training.StopAfterNEvalsHook(
math.ceil(dataset.num_samples / float(config.batch_size)))
]
eval_kwargs = {}
eval_fn = training.evaluate_repeatedly
if FLAGS.once:
if tf.gfile.IsDirectory(checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(checkpoint_path)
eval_fn = training.evaluate_once
else:
assert tf.gfile.IsDirectory(checkpoint_path), (
'checkpoint path must be a directory when using loop evaluation')
eval_fn(checkpoint_path, hooks=hooks, eval_ops=eval_ops, **eval_kwargs)
def __init__(
self,
config,
output_dir="./output",
use_rnn=False,
testing=False,
use_best=False,
):
self.config = config
self.output_dir = output_dir
self.checkpoint_path = os.path.join(self.output_dir, "checkpoint")
self.best_ckpt_path = os.path.join(self.output_dir, "best_ckpt")
self.weights_path = os.path.join(self.output_dir, "weights")
self.log_dir = os.path.join(self.output_dir, "log")
self.use_rnn = use_rnn
# Placeholder
with tf.variable_scope("placeholders") as scope:
self.signals = tf.placeholder(dtype=tf.float32,
shape=(None,
self.config["input_size"], 1,
1),
name='signals')
self.labels = tf.placeholder(dtype=tf.int32,
shape=(None, ),
name='labels')
self.is_training = tf.placeholder(dtype=tf.bool,
shape=(),
name='is_training')
if self.use_rnn:
self.loss_weights = tf.placeholder(dtype=tf.float32,
shape=(None, ),
name='loss_weights')
self.seq_lengths = tf.placeholder(dtype=tf.int32,
shape=(None, ),
name='seq_lengths')
# Monitor global step update
self.global_step = tf.Variable(0, trainable=False, name='global_step')
# Monitor the number of epochs passed
self.global_epoch = tf.Variable(0,
trainable=False,
name='global_epoch')
# Build a network that receives inputs from placeholders
net = self.build_cnn()
if self.use_rnn:
# Check whether the corresponding config is given
if "n_rnn_layers" not in self.config:
raise Exception("Invalid config.")
# Append the RNN if needed
net = self.append_rnn(net)
# Softmax linear
net = nn.fc("softmax_linear", net, self.config["n_classes"], bias=0.0)
# Outputs
self.logits = net
self.preds = tf.argmax(self.logits, axis=1)
# Cross-entropy loss
self.loss_per_sample = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=self.labels, logits=self.logits, name="loss_ce_per_sample")
with tf.name_scope("loss_ce_mean") as scope:
if self.use_rnn:
# Weight by sequence
loss_w_seq = tf.multiply(self.loss_weights,
self.loss_per_sample)
# Weight by class
sample_weights = tf.reduce_sum(
tf.multiply(
tf.one_hot(indices=self.labels,
depth=self.config["n_classes"]),
np.asarray(self.config["class_weights"],
dtype=np.float32)), 1)
loss_w_class = tf.multiply(loss_w_seq, sample_weights)
# Computer average loss scaled with the sequence length
self.loss_ce = tf.reduce_sum(loss_w_class) / tf.reduce_sum(
self.loss_weights)
else:
self.loss_ce = tf.reduce_mean(self.loss_per_sample)
# Regularization loss
self.reg_losses = self.regularization_loss()
# Total loss
self.loss = self.loss_ce + self.reg_losses
# Metrics (used when we want to compute a metric from the output from minibatches)
with tf.variable_scope("stream_metrics") as scope:
self.metric_value_op, self.metric_update_op = contrib_metrics.aggregate_metric_map(
{
"loss":
tf.metrics.mean(values=self.loss),
"accuracy":
tf.metrics.accuracy(labels=self.labels,
predictions=self.preds),
"precision":
tf.metrics.precision(labels=self.labels,
predictions=self.preds),
"recall":
tf.metrics.recall(labels=self.labels,
predictions=self.preds),
})
# Manually create reset operations of local vars
metric_vars = contrib_slim.get_local_variables(scope=scope.name)
self.metric_init_op = tf.variables_initializer(metric_vars)
# Training outputs
self.train_outputs = {
"global_step": self.global_step,
"train/loss": self.loss,
"train/preds": self.preds,
"train/stream_metrics": self.metric_update_op,
}
if self.use_rnn:
self.train_outputs.update({
"train/init_state": self.init_state,
"train/final_state": self.final_state,
})
# Test outputs
self.test_outputs = {
"global_step": self.global_step,
"test/loss": self.loss,
"test/preds": self.preds,
}
if self.use_rnn:
self.test_outputs.update({
"test/init_state": self.init_state,
"test/final_state": self.final_state,
})
# Tensoflow
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
if not testing:
self.train_writer = tf.summary.FileWriter(
os.path.join(self.log_dir, "train"))
self.train_writer.add_graph(self.sess.graph)
logger.info("Saved tensorboard graph to {}".format(
self.train_writer.get_logdir()))
# Optimizer
if not testing:
# self.lr = tf.train.exponential_decay(
# learning_rate=self.config["learning_rate_decay"],
# global_step=self.global_step,
# decay_steps=self.config["decay_steps"],
# decay_rate=self.config["decay_rate"],
# staircase=False,
# name="learning_rate"
# )
self.lr = tf.constant(self.config["learning_rate"],
dtype=tf.float32)
with tf.variable_scope("optimizer") as scope:
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
# Pretraining
if not self.use_rnn:
self.train_step_op, self.grad_op = nn.adam_optimizer(
loss=self.loss,
training_variables=tf.trainable_variables(),
global_step=self.global_step,
# learning_rate=self.config["learning_rate"],
learning_rate=self.lr,
beta1=self.config["adam_beta_1"],
beta2=self.config["adam_beta_2"],
epsilon=self.config["adam_epsilon"],
)
# Fine-tuning
else:
# Use different learning rates for CNN and RNN
self.train_step_op, self.grad_op = nn.adam_optimizer_clip(
loss=self.loss,
training_variables=tf.trainable_variables(),
global_step=self.global_step,
# learning_rate=self.config["learning_rate"],
learning_rate=self.lr,
beta1=self.config["adam_beta_1"],
beta2=self.config["adam_beta_2"],
epsilon=self.config["adam_epsilon"],
clip_value=self.config["clip_grad_value"],
)
# Initializer
with tf.variable_scope("initializer") as scope:
# tf.trainable_variables() or tf.global_variables()
self.init_global_op = tf.variables_initializer(
tf.global_variables())
self.init_local_op = tf.variables_initializer(tf.local_variables())
# Saver for storing variables
self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=1)
self.best_saver = tf.train.Saver(tf.global_variables(), max_to_keep=1)
# Initialize variables
self.run([self.init_global_op, self.init_local_op])
# Restore variables (if possible)
is_restore = False
if use_best:
if os.path.exists(self.best_ckpt_path):
if os.path.isfile(
os.path.join(self.best_ckpt_path, "checkpoint")):
# Restore the last checkpoint
latest_checkpoint = tf.train.latest_checkpoint(
self.best_ckpt_path)
self.saver.restore(self.sess, latest_checkpoint)
logger.info("Best model restored from {}".format(
latest_checkpoint))
is_restore = True
else:
if os.path.exists(self.checkpoint_path):
if os.path.isfile(
os.path.join(self.checkpoint_path, "checkpoint")):
# Restore the last checkpoint
latest_checkpoint = tf.train.latest_checkpoint(
self.checkpoint_path)
self.saver.restore(self.sess, latest_checkpoint)
logger.info(
"Model restored from {}".format(latest_checkpoint))
is_restore = True
if not is_restore:
logger.info("Model started from random weights")
def __init__(self, run_context=None):
if run_context is not None:
self.training_log = run_context.create_train_log('training')
self.validation_log = run_context.create_train_log('validation')
self.checkpoint_path = os.path.join(run_context.transient_dir, 'checkpoint')
self.tensorboard_path = os.path.join(run_context.result_dir, 'tensorboard')
with tf.name_scope("placeholders"):
self.images = tf.placeholder(dtype=tf.float32, shape=(None, 32, 32, 3), name='images')
self.labels = tf.placeholder(dtype=tf.int32, shape=(None,), name='labels')
self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training')
self.global_step = tf.Variable(0, trainable=False, name='global_step')
tf.add_to_collection("init_in_init", self.global_step)
self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS)
for var in self.hyper.variables.values():
tf.add_to_collection("init_in_init", var)
with tf.name_scope("ramps"):
sigmoid_rampup_value = sigmoid_rampup(self.global_step, self.hyper['rampup_length'])
sigmoid_rampdown_value = sigmoid_rampdown(self.global_step,
self.hyper['rampdown_length'],
self.hyper['training_length'])
self.learning_rate = tf.multiply(sigmoid_rampup_value * sigmoid_rampdown_value,
self.hyper['max_learning_rate'],
name='learning_rate')
self.adam_beta_1 = tf.add(sigmoid_rampdown_value * self.hyper['adam_beta_1_before_rampdown'],
(1 - sigmoid_rampdown_value) * self.hyper['adam_beta_1_after_rampdown'],
name='adam_beta_1')
self.cons_coefficient = tf.multiply(sigmoid_rampup_value,
self.hyper['max_consistency_cost'],
name='consistency_coefficient')
step_rampup_value = step_rampup(self.global_step, self.hyper['rampup_length'])
self.adam_beta_2 = tf.add((1 - step_rampup_value) * self.hyper['adam_beta_2_during_rampup'],
step_rampup_value * self.hyper['adam_beta_2_after_rampup'],
name='adam_beta_2')
self.ema_decay = tf.add((1 - step_rampup_value) * self.hyper['ema_decay_during_rampup'],
step_rampup_value * self.hyper['ema_decay_after_rampup'],
name='ema_decay')
(
(self.class_logits_1, self.cons_logits_1),
(self.class_logits_2, self.cons_logits_2),
(self.class_logits_ema, self.cons_logits_ema)
) = inference(
self.images,
is_training=self.is_training,
ema_decay=self.ema_decay,
input_noise=self.hyper['input_noise'],
student_dropout_probability=self.hyper['student_dropout_probability'],
teacher_dropout_probability=self.hyper['teacher_dropout_probability'],
normalize_input=self.hyper['normalize_input'],
flip_horizontally=self.hyper['flip_horizontally'],
translate=self.hyper['translate'],
num_logits=self.hyper['num_logits'])
with tf.name_scope("objectives"):
self.mean_error_1, self.errors_1 = errors(self.class_logits_1, self.labels)
self.mean_error_ema, self.errors_ema = errors(self.class_logits_ema, self.labels)
self.mean_class_cost_1, self.class_costs_1 = classification_costs(
self.class_logits_1, self.labels)
self.mean_class_cost_ema, self.class_costs_ema = classification_costs(
self.class_logits_ema, self.labels)
labeled_consistency = self.hyper['apply_consistency_to_labeled']
consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency)
self.mean_cons_cost_pi, self.cons_costs_pi = consistency_costs(
self.cons_logits_1, self.class_logits_2, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust'])
self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs(
self.cons_logits_1, self.class_logits_ema, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust'])
def l2_norms(matrix):
l2s = tf.reduce_sum(matrix ** 2, axis=1)
mean_l2 = tf.reduce_mean(l2s)
return mean_l2, l2s
self.mean_res_l2_1, self.res_l2s_1 = l2_norms(self.class_logits_1 - self.cons_logits_1)
self.mean_res_l2_ema, self.res_l2s_ema = l2_norms(self.class_logits_ema - self.cons_logits_ema)
self.res_costs_1 = self.hyper['logit_distance_cost'] * self.res_l2s_1
self.mean_res_cost_1 = tf.reduce_mean(self.res_costs_1)
self.res_costs_ema = self.hyper['logit_distance_cost'] * self.res_l2s_ema
self.mean_res_cost_ema = tf.reduce_mean(self.res_costs_ema)
self.mean_total_cost_pi, self.total_costs_pi = total_costs(
self.class_costs_1, self.cons_costs_pi, self.res_costs_1)
self.mean_total_cost_mt, self.total_costs_mt = total_costs(
self.class_costs_1, self.cons_costs_mt, self.res_costs_1)
assert_shape(self.total_costs_pi, [3])
assert_shape(self.total_costs_mt, [3])
self.cost_to_be_minimized = tf.cond(self.hyper['ema_consistency'],
lambda: self.mean_total_cost_mt,
lambda: self.mean_total_cost_pi)
with tf.name_scope("train_step"):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self.train_step_op = nn.adam_optimizer(self.cost_to_be_minimized,
self.global_step,
learning_rate=self.learning_rate,
beta1=self.adam_beta_1,
beta2=self.adam_beta_2,
epsilon=self.hyper['adam_epsilon'])
self.training_control = training_control(self.global_step,
self.hyper['print_span'],
self.hyper['evaluation_span'],
self.hyper['training_length'])
self.training_metrics = {
"learning_rate": self.learning_rate,
"adam_beta_1": self.adam_beta_1,
"adam_beta_2": self.adam_beta_2,
"ema_decay": self.ema_decay,
"cons_coefficient": self.cons_coefficient,
"train/error/1": self.mean_error_1,
"train/error/ema": self.mean_error_ema,
"train/class_cost/1": self.mean_class_cost_1,
"train/class_cost/ema": self.mean_class_cost_ema,
"train/cons_cost/pi": self.mean_cons_cost_pi,
"train/cons_cost/mt": self.mean_cons_cost_mt,
"train/res_cost/1": self.mean_res_cost_1,
"train/res_cost/ema": self.mean_res_cost_ema,
"train/total_cost/pi": self.mean_total_cost_pi,
"train/total_cost/mt": self.mean_total_cost_mt,
}
with tf.variable_scope("validation_metrics") as metrics_scope:
self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map({
"eval/error/1": streaming_mean(self.errors_1),
"eval/error/ema": streaming_mean(self.errors_ema),
"eval/class_cost/1": streaming_mean(self.class_costs_1),
"eval/class_cost/ema": streaming_mean(self.class_costs_ema),
"eval/res_cost/1": streaming_mean(self.res_costs_1),
"eval/res_cost/ema": streaming_mean(self.res_costs_ema),
})
metric_variables = slim.get_local_variables(scope=metrics_scope.name)
self.metric_init_op = tf.variables_initializer(metric_variables)
self.result_formatter = string_utils.DictFormatter(
order=["eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"],
default_format='{name}: {value:>10.6f}',
separator=", ")
self.result_formatter.add_format('error', '{name}: {value:>6.1%}')
with tf.name_scope("initializers"):
init_init_variables = tf.get_collection("init_in_init")
train_init_variables = [
var for var in tf.global_variables() if var not in init_init_variables
]
self.init_init_op = tf.variables_initializer(init_init_variables)
self.train_init_op = tf.variables_initializer(train_init_variables)
self.saver = tf.train.Saver()
self.session = tf.Session()
self.run(self.init_init_op)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset( # 获取验证集图片数据
dataset_name=FLAGS.dataset, # 数据集名称 cityscapes 默认为 pascal_voc_seg
split_name=FLAGS.eval_split, # 指定带有val的tfrecorder数据集 默认为“val”
dataset_dir=FLAGS.dataset_dir, # 数据集目录 tfrecoder文件的数据集目录
batch_size=FLAGS.eval_batch_size, # 每个batch包含的image数量 默认为1
crop_size=[int(sz)
for sz in FLAGS.eval_crop_size], # 评估时crop_size 默认为513,513
min_resize_value=FLAGS.min_resize_value, # 默认为None
max_resize_value=FLAGS.max_resize_value, # 默认为None
resize_factor=FLAGS.resize_factor, # 默认为None
model_variant=FLAGS.model_variant, # 模型的变体 本次训练为 xception_65
num_readers=2, # 并行读取图片的数量
is_training=False, # 不训练
should_shuffle=False, # 不将输入的数据随机打乱
should_repeat=False) # 不一直重复
tf.gfile.MakeDirs(FLAGS.eval_logdir) # 创建评估目录
tf.logging.info('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next() # 获取一次迭代的验证集数据
'''
samples:
{'image_name': <tf.Tensor 'IteratorGetNext:2' shape=(?,) dtype=string>,
'width': <tf.Tensor 'IteratorGetNext:5' shape=(?,) dtype=int64>,
'image': <tf.Tensor 'IteratorGetNext:1' shape=(?, 1024, 2048, 3) dtype=float32>,
'height': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=int64>,
'label': <tf.Tensor 'IteratorGetNext:3' shape=(?, 1024, 2048, 1) dtype=int32>,
'original_image': <tf.Tensor 'IteratorGetNext:4' shape=(?, ?, ?, 3) dtype=uint8>}
'''
model_options = common.ModelOptions( # 模型参数
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
}, # {semantic: 19}
crop_size=[int(sz) for sz in FLAGS.eval_crop_size], # 1024,2048
atrous_rates=FLAGS.atrous_rates, # 6,12,18
output_stride=FLAGS.output_stride) # 16
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape( # 设置形状
[
FLAGS.eval_batch_size, # 默认为1
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]),
3
])
if tuple(FLAGS.eval_scales) == (1.0, ): # 默认 评估尺度为1
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options, # 进行每个像素点预测
image_pyramid=FLAGS.image_pyramid)
'''
predictions:
{'semantic': <tf.Tensor 'ArgMax:0' shape=(1, 1024, 2048) dtype=int64>,
'semantic_prob': <tf.Tensor 'Softmax:0' shape=(1, 1024, 2048, 19) dtype=float32>}
'''
else:
tf.logging.info('Performing multi-scale test.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
samples[common.IMAGE],
model_options=model_options,
eval_scales=FLAGS.eval_scales,
add_flipped_images=FLAGS.add_flipped_images)
predictions = predictions[common.OUTPUT_TYPE]
predictions = tf.reshape(predictions, shape=[-1]) # 预测标签
labels = tf.reshape(samples[common.LABEL], shape=[-1]) # 真实标签
weights = tf.to_float(tf.not_equal(labels,
dataset.ignore_label)) # 各标签权重
# Set ignore_label regions to label 0, because metrics.mean_iou requires
# range of labels = [0, dataset.num_classes). Note the ignore_label regions
# are not evaluated since the corresponding regions contain weights = 0.
labels = tf.where(tf.equal(labels, dataset.ignore_label),
tf.zeros_like(labels), labels)
predictions_tag = 'miou' # MIoU
predictions_tag1 = 'accuracy_pixel' # 像素精度
for eval_scale in FLAGS.eval_scales: # 默认为单尺度[1.0]
predictions_tag += '_' + str(eval_scale)
predictions_tag1 += '_' + str(eval_scale)
if FLAGS.add_flipped_images: # 默认为False 不设置左右翻转来评估模型
predictions_tag += '_flipped'
predictions_tag1 += '_flipped'
# Define the evaluation metric.
metric_map = {}
num_classes = dataset.num_of_classes # 19
metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou(
labels=labels,
predictions=predictions,
num_classes=num_classes,
weights=weights)
'''
metric_map:
{'eval/miou_1.0_overall': (<tf.Tensor 'mean_iou/Select_1:0' shape=() dtype=float32>,
<tf.Tensor 'mean_iou/AssignAdd:0' shape=(19, 19) dtype=float64_ref>)}
'''
metric_map['eval/%s_overall_accuracy_' %
predictions_tag] = tf.metrics.accuracy(
labels=labels, predictions=predictions, weights=weights)
# IoU for each class.
'''
tf.one_hot(indices, depth, on_value=None, off_value=None, axis=None, dtype=None, name=None)
Returns a one-hot tensor.
ndices表示输入的多个数值，通常是矩阵形式；depth表示输出的尺寸。
'''
one_hot_predictions = tf.one_hot(predictions, num_classes)
one_hot_predictions = tf.reshape(one_hot_predictions,
[-1, num_classes]) # 预测输出的one_hot
one_hot_labels = tf.one_hot(labels, num_classes)
one_hot_labels = tf.reshape(one_hot_labels,
[-1, num_classes]) # 真实label的one_hot
for c in range(num_classes):
predictions_tag_c = '%s_class_%d' % (predictions_tag, c
) # miou_1.0_class_c
predictions_tag_c1 = '%s_class_%d' % (predictions_tag1, c)
tp, tp_op = tf.metrics.true_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fp, fp_op = tf.metrics.false_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fn, fn_op = tf.metrics.false_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tn, tn_op = tf.metrics.true_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op)
iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp),
tf.constant(np.NaN))
ap = tf.where(tf.greater(tp + fn, 0.0),
(tp + tn) / (tp + tn + fn + fp), tf.constant(np.NaN))
metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op)
metric_map['eval/%s' % predictions_tag_c1] = (ap, tp_fp_fn_op)
(metrics_to_values,
metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map)
'''
(metrics_to_values, metrics_to_updates):
({'eval/miou_1.0_class_5': <tf.Tensor 'Select_6:0' shape=() dtype=float32>,
'eval/miou_1.0_class_18': <tf.Tensor 'Select_19:0' shape=() dtype=float32>,
'eval/miou_1.0_class_13': <tf.Tensor 'Select_14:0' shape=() dtype=float32>,
'eval/miou_1.0_class_1': <tf.Tensor 'Select_2:0' shape=() dtype=float32>,
'eval/miou_1.0_overall': <tf.Tensor 'mean_iou/Select_1:0' shape=() dtype=float32>,
'eval/miou_1.0_class_17': <tf.Tensor 'Select_18:0' shape=() dtype=float32>,
'eval/miou_1.0_class_8': <tf.Tensor 'Select_9:0' shape=() dtype=float32>,
'eval/miou_1.0_class_2': <tf.Tensor 'Select_3:0' shape=() dtype=float32>,
'eval/miou_1.0_class_0': <tf.Tensor 'Select_1:0' shape=() dtype=float32>,
'eval/miou_1.0_class_3': <tf.Tensor 'Select_4:0' shape=() dtype=float32>,
'eval/miou_1.0_class_14': <tf.Tensor 'Select_15:0' shape=() dtype=float32>,
'eval/miou_1.0_class_11': <tf.Tensor 'Select_12:0' shape=() dtype=float32>,
'eval/miou_1.0_class_6': <tf.Tensor 'Select_7:0' shape=() dtype=float32>,
'eval/miou_1.0_class_15': <tf.Tensor 'Select_16:0' shape=() dtype=float32>,
'eval/miou_1.0_class_4': <tf.Tensor 'Select_5:0' shape=() dtype=float32>,
'eval/miou_1.0_class_9': <tf.Tensor 'Select_10:0' shape=() dtype=float32>,
'eval/miou_1.0_class_16': <tf.Tensor 'Select_17:0' shape=() dtype=float32>,
'eval/miou_1.0_class_7': <tf.Tensor 'Select_8:0' shape=() dtype=float32>,
'eval/miou_1.0_class_10': <tf.Tensor 'Select_11:0' shape=() dtype=float32>,
'eval/miou_1.0_class_12': <tf.Tensor 'Select_13:0' shape=() dtype=float32>},
{'eval/miou_1.0_class_5': <tf.Operation 'group_deps_5' type=NoOp>,
'eval/miou_1.0_class_18': <tf.Operation 'group_deps_18' type=NoOp>,
'eval/miou_1.0_class_13': <tf.Operation 'group_deps_13' type=NoOp>,
'eval/miou_1.0_class_1': <tf.Operation 'group_deps_1' type=NoOp>,
'eval/miou_1.0_overall': <tf.Tensor 'mean_iou/AssignAdd:0' shape=(19, 19) dtype=float64_ref>,
'eval/miou_1.0_class_17': <tf.Operation 'group_deps_17' type=NoOp>,
'eval/miou_1.0_class_8': <tf.Operation 'group_deps_8' type=NoOp>,
'eval/miou_1.0_class_2': <tf.Operation 'group_deps_2' type=NoOp>,
'eval/miou_1.0_class_0': <tf.Operation 'group_deps' type=NoOp>,
'eval/miou_1.0_class_3': <tf.Operation 'group_deps_3' type=NoOp>,
'eval/miou_1.0_class_14': <tf.Operation 'group_deps_14' type=NoOp>,
'eval/miou_1.0_class_11': <tf.Operation 'group_deps_11' type=NoOp>,
'eval/miou_1.0_class_6': <tf.Operation 'group_deps_6' type=NoOp>,
'eval/miou_1.0_class_15': <tf.Operation 'group_deps_15' type=NoOp>,
'eval/miou_1.0_class_4': <tf.Operation 'group_deps_4' type=NoOp>,
'eval/miou_1.0_class_9': <tf.Operation 'group_deps_9' type=NoOp>,
'eval/miou_1.0_class_16': <tf.Operation 'group_deps_16' type=NoOp>,
'eval/miou_1.0_class_7': <tf.Operation 'group_deps_7' type=NoOp>,
'eval/miou_1.0_class_10': <tf.Operation 'group_deps_10' type=NoOp>,
'eval/miou_1.0_class_12': <tf.Operation 'group_deps_12' type=NoOp>})
'''
'''
tf.Print(input, data, message=None, first_n=None, summarize=None, name=None)
最低要求两个输入，input和data，input是需要打印的变量的名字，data要求是一个list，里面包含要打印的内容。
'''
summary_ops = []
for metric_name, metric_value in six.iteritems(metrics_to_values):
op = tf.summary.scalar(metric_name, metric_value) # 显示标量信息
op = tf.Print(op, [metric_value], metric_name)
summary_ops.append(op)
summary_op = tf.summary.merge(summary_ops)
summary_hook = contrib_training.SummaryAtEndHook(
log_dir=FLAGS.eval_logdir, summary_op=summary_op)
hooks = [summary_hook]
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0: # 为0 暂不考虑
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0: # -1 暂不考虑
contrib_quantize.create_eval_graph()
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
contrib_training.evaluate_repeatedly(
checkpoint_dir=FLAGS.checkpoint_dir,
master=FLAGS.master,
eval_ops=list(metrics_to_updates.values()),
max_number_of_evaluations=num_eval_iters,
hooks=hooks,
eval_interval_secs=FLAGS.eval_interval_secs)
def __init__(self, run_context=None):
self.name = "Tweet Data Class"
if run_context is not None:
self.training_log = run_context.create_train_log('training')
self.validation_log = run_context.create_train_log('validation')
self.checkpoint_path = os.path.join(run_context.transient_dir, 'checkpoint')
self.tensorboard_path = os.path.join(run_context.result_dir, 'tensorboard')
with tf.name_scope("placeholders"):
self.tweets = tf.placeholder(dtype=tf.float32, shape=(None, 500), name='tweets')
self.labels = tf.placeholder(dtype=tf.int32, shape=(None,), name='labels')
self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training')
self.global_step = tf.Variable(0, trainable=False, name='global_step')
tf.add_to_collection("init_in_init", self.global_step)
self.hyper = HyperparamVariables(self.DEFAULT_HYPERPARAMS)
for var in self.hyper.variables.values():
tf.add_to_collection("init_in_init", var)
with tf.name_scope("ramps"):
# Ramp-up and ramp-down has been removed for simplicity
# self.learning_rate = tf.constant(self.hyper['max_learning_rate'], dtype = tf.float32)
# self.adam_beta_1 = tf.constant(self.hyper['adam_beta_1_after_rampdown'], dtype = tf.float32)
# self.cons_coefficient = tf.constant(self.hyper['max_consistency_cost'], dtype = tf.float32)
# self.adam_beta_2 = tf.constant(self.hyper['adam_beta_2_after_rampup'], dtype = tf.float32)
# self.ema_decay = tf.constant(self.hyper['ema_decay_after_rampup'], dtype = tf.float32)
# self.learning_rate =self.DEFAULT_HYPERPARAMS['max_learning_rate']
# self.adam_beta_1 = self.DEFAULT_HYPERPARAMS['adam_beta_1_after_rampdown']
# self.cons_coefficient = self.DEFAULT_HYPERPARAMS['max_consistency_cost']
# self.adam_beta_2 = self.DEFAULT_HYPERPARAMS['adam_beta_2_after_rampup']
# self.ema_decay = self.DEFAULT_HYPERPARAMS['ema_decay_after_rampup']
sigmoid_rampup_value = sigmoid_rampup(self.global_step, self.hyper['rampup_length'])
sigmoid_rampdown_value = sigmoid_rampdown(self.global_step,
self.hyper['rampdown_length'],
self.hyper['training_length'])
self.learning_rate = tf.multiply(sigmoid_rampup_value * sigmoid_rampdown_value,
self.hyper['max_learning_rate'],
name='learning_rate')
self.adam_beta_1 = tf.add(sigmoid_rampdown_value * self.hyper['adam_beta_1_before_rampdown'],
(1 - sigmoid_rampdown_value) * self.hyper['adam_beta_1_after_rampdown'],
name='adam_beta_1')
self.cons_coefficient = tf.multiply(sigmoid_rampup_value,
self.hyper['max_consistency_cost'],
name='consistency_coefficient')
step_rampup_value = step_rampup(self.global_step, self.hyper['rampup_length'])
self.adam_beta_2 = tf.add((1 - step_rampup_value) * self.hyper['adam_beta_2_during_rampup'],
step_rampup_value * self.hyper['adam_beta_2_after_rampup'],
name='adam_beta_2')
self.ema_decay = tf.add((1 - step_rampup_value) * self.hyper['ema_decay_during_rampup'],
step_rampup_value * self.hyper['ema_decay_after_rampup'],
name='ema_decay')
# below is where the interesting stuff happens, mostly.
# Inference is a function which creates the towers and sets up the different logits for the two models
(
(self.class_logits_1, self.cons_logits_1),
(self.class_logits_2, self.cons_logits_2),
(self.class_logits_ema, self.cons_logits_ema)
) = inference(
self.tweets,
is_training=self.is_training,
ema_decay=self.ema_decay,
input_noise=self.hyper['input_noise'],
hidden_dims = self.DEFAULT_HYPERPARAMS['hidden_dims'],
student_dropout_probability=self.hyper['student_dropout_probability'],
teacher_dropout_probability=self.hyper['teacher_dropout_probability'],
num_logits=self.hyper['num_logits'])
with tf.name_scope("objectives"):
# something weird is done with errors for unlabeled examples.
# I think errors are only calculated for labeled, but you don't calculate it for unlabeled, so it is NaN for unlabeled
self.mean_error_1, self.errors_1 = errors(self.class_logits_1, self.labels)
self.mean_error_ema, self.errors_ema = errors(self.class_logits_ema, self.labels)
# where we calculate classification costs.
# the cost_1 should be for student and ema is for teacher
self.mean_class_cost_1, self.class_costs_1 = classification_costs(
self.class_logits_1, self.labels)
self.mean_class_cost_ema, self.class_costs_ema = classification_costs(
self.class_logits_ema, self.labels)
labeled_consistency = self.hyper['apply_consistency_to_labeled']
consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency)
self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs( self.cons_logits_1, self.class_logits_ema, self.cons_coefficient, consistency_mask)
def l2_norms(matrix):
l2s = tf.reduce_sum(matrix ** 2, axis=1)
mean_l2 = tf.reduce_mean(l2s)
return mean_l2, l2s
self.mean_res_l2_1, self.res_l2s_1 = l2_norms(self.class_logits_1 - self.cons_logits_1)
self.mean_res_l2_ema, self.res_l2s_ema = l2_norms(self.class_logits_ema - self.cons_logits_ema)
# mean total cost is what you are optimizng.
self.mean_total_cost_mt, self.total_costs_mt = total_costs(
self.class_costs_1, self.cons_costs_mt)
assert_shape(self.total_costs_mt, [2])
self.cost_to_be_minimized = self.mean_total_cost_mt
with tf.name_scope("train_step"):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self.train_step_op = adam_optimizer(self.cost_to_be_minimized,
self.global_step,
learning_rate=self.learning_rate,
beta1=self.adam_beta_1,
beta2=self.adam_beta_2,
epsilon=self.hyper['adam_epsilon'])
# TODO do we really need this?
self.training_control = training_control(self.global_step,
self.hyper['print_span'],
self.hyper['evaluation_span'],
self.hyper['training_length'])
self.training_metrics = {
# NOTE these should not need training, since we don't do ramp-up and ramp-down
"learning_rate": self.learning_rate,
"adam_beta_1": self.adam_beta_1,
"adam_beta_2": self.adam_beta_2,
"ema_decay": self.ema_decay,
"cons_coefficient": self.cons_coefficient,
"train/error/1": self.mean_error_1,
"train/error/ema": self.mean_error_ema,
"train/class_cost/1": self.mean_class_cost_1,
"train/class_cost/ema": self.mean_class_cost_ema,
"train/cons_cost/mt": self.mean_cons_cost_mt,
"train/total_cost/mt": self.mean_total_cost_mt,
}
# TODO not sure what streaming mean does?
with tf.variable_scope("validation_metrics") as metrics_scope:
self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map({
"eval/error/1": streaming_mean(self.errors_1),
"eval/error/ema": streaming_mean(self.errors_ema),
"eval/class_cost/1": streaming_mean(self.class_costs_1),
"eval/class_cost/ema": streaming_mean(self.class_costs_ema),
})
metric_variables = slim.get_local_variables(scope=metrics_scope.name)
self.metric_init_op = tf.variables_initializer(metric_variables)
# TODO string utils just formats dictionary results in a nice way for logging, not needed?
self.result_formatter = string_utils.DictFormatter(
order=["eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"],
default_format='{name}: {value:>10.6f}',
separator=", ")
self.result_formatter.add_format('error', '{name}: {value:>6.1%}')
with tf.name_scope("initializers"):
init_init_variables = tf.get_collection("init_in_init")
train_init_variables = [
var for var in tf.global_variables() if var not in init_init_variables
]
self.init_init_op = tf.variables_initializer(init_init_variables)
print("Train init variables:")
for var in train_init_variables:
print(var)
self.train_init_op = tf.variables_initializer(train_init_variables)
self.saver = tf.train.Saver()
self.session = tf.Session()
self.run(self.init_init_op)
def build():
"""Builds the Tensorflow graph."""
inputs, labels, lengths = None, None, None
if mode in ('train', 'eval'):
if isinstance(no_event_label, numbers.Number):
label_shape = []
else:
label_shape = [len(no_event_label)]
inputs, labels, lengths = magenta.common.get_padded_batch(
sequence_example_file_paths, hparams.batch_size, input_size,
label_shape=label_shape, shuffle=mode == 'train')
elif mode == 'generate':
inputs = tf.placeholder(tf.float32, [hparams.batch_size, None,
input_size])
if isinstance(encoder_decoder,
magenta.music.OneHotIndexEventSequenceEncoderDecoder):
expanded_inputs = tf.one_hot(
tf.cast(tf.squeeze(inputs, axis=-1), tf.int64),
encoder_decoder.input_depth)
else:
expanded_inputs = inputs
dropout_keep_prob = 1.0 if mode == 'generate' else hparams.dropout_keep_prob
if hparams.use_cudnn:
outputs, initial_state, final_state = make_cudnn(
expanded_inputs, hparams.rnn_layer_sizes, hparams.batch_size, mode,
dropout_keep_prob=dropout_keep_prob,
residual_connections=hparams.residual_connections)
else:
cell = make_rnn_cell(
hparams.rnn_layer_sizes,
dropout_keep_prob=dropout_keep_prob,
attn_length=hparams.attn_length,
residual_connections=hparams.residual_connections)
initial_state = cell.zero_state(hparams.batch_size, tf.float32)
outputs, final_state = tf.nn.dynamic_rnn(
cell, inputs, sequence_length=lengths, initial_state=initial_state,
swap_memory=True)
outputs_flat = magenta.common.flatten_maybe_padded_sequences(
outputs, lengths)
if isinstance(num_classes, numbers.Number):
num_logits = num_classes
else:
num_logits = sum(num_classes)
logits_flat = contrib_layers.linear(outputs_flat, num_logits)
if mode in ('train', 'eval'):
labels_flat = magenta.common.flatten_maybe_padded_sequences(
labels, lengths)
if isinstance(num_classes, numbers.Number):
softmax_cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=labels_flat, logits=logits_flat)
predictions_flat = tf.argmax(logits_flat, axis=1)
else:
logits_offsets = np.cumsum([0] + num_classes)
softmax_cross_entropy = []
predictions = []
for i in range(len(num_classes)):
softmax_cross_entropy.append(
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=labels_flat[:, i],
logits=logits_flat[
:, logits_offsets[i]:logits_offsets[i + 1]]))
predictions.append(
tf.argmax(logits_flat[
:, logits_offsets[i]:logits_offsets[i + 1]], axis=1))
predictions_flat = tf.stack(predictions, 1)
correct_predictions = tf.to_float(
tf.equal(labels_flat, predictions_flat))
event_positions = tf.to_float(tf.not_equal(labels_flat, no_event_label))
no_event_positions = tf.to_float(tf.equal(labels_flat, no_event_label))
# Compute the total number of time steps across all sequences in the
# batch. For some models this will be different from the number of RNN
# steps.
def batch_labels_to_num_steps(batch_labels, lengths):
num_steps = 0
for labels, length in zip(batch_labels, lengths):
num_steps += encoder_decoder.labels_to_num_steps(labels[:length])
return np.float32(num_steps)
num_steps = tf.py_func(
batch_labels_to_num_steps, [labels, lengths], tf.float32)
if mode == 'train':
loss = tf.reduce_mean(softmax_cross_entropy)
perplexity = tf.exp(loss)
accuracy = tf.reduce_mean(correct_predictions)
event_accuracy = (
tf.reduce_sum(correct_predictions * event_positions) /
tf.reduce_sum(event_positions))
no_event_accuracy = (
tf.reduce_sum(correct_predictions * no_event_positions) /
tf.reduce_sum(no_event_positions))
loss_per_step = tf.reduce_sum(softmax_cross_entropy) / num_steps
perplexity_per_step = tf.exp(loss_per_step)
optimizer = tf.train.AdamOptimizer(learning_rate=hparams.learning_rate)
train_op = contrib_slim.learning.create_train_op(
loss, optimizer, clip_gradient_norm=hparams.clip_norm)
tf.add_to_collection('train_op', train_op)
vars_to_summarize = {
'loss': loss,
'metrics/perplexity': perplexity,
'metrics/accuracy': accuracy,
'metrics/event_accuracy': event_accuracy,
'metrics/no_event_accuracy': no_event_accuracy,
'metrics/loss_per_step': loss_per_step,
'metrics/perplexity_per_step': perplexity_per_step,
}
elif mode == 'eval':
vars_to_summarize, update_ops = contrib_metrics.aggregate_metric_map({
'loss':
tf.metrics.mean(softmax_cross_entropy),
'metrics/accuracy':
tf.metrics.accuracy(labels_flat, predictions_flat),
'metrics/per_class_accuracy':
tf.metrics.mean_per_class_accuracy(labels_flat,
predictions_flat,
num_classes),
'metrics/event_accuracy':
tf.metrics.recall(event_positions, correct_predictions),
'metrics/no_event_accuracy':
tf.metrics.recall(no_event_positions, correct_predictions),
'metrics/loss_per_step':
tf.metrics.mean(
tf.reduce_sum(softmax_cross_entropy) / num_steps,
weights=num_steps),
})
for updates_op in update_ops.values():
tf.add_to_collection('eval_ops', updates_op)
# Perplexity is just exp(loss) and doesn't need its own update op.
vars_to_summarize['metrics/perplexity'] = tf.exp(
vars_to_summarize['loss'])
vars_to_summarize['metrics/perplexity_per_step'] = tf.exp(
vars_to_summarize['metrics/loss_per_step'])
for var_name, var_value in six.iteritems(vars_to_summarize):
tf.summary.scalar(var_name, var_value)
tf.add_to_collection(var_name, var_value)
elif mode == 'generate':
temperature = tf.placeholder(tf.float32, [])
if isinstance(num_classes, numbers.Number):
softmax_flat = tf.nn.softmax(
tf.div(logits_flat, tf.fill([num_classes], temperature)))
softmax = tf.reshape(
softmax_flat, [hparams.batch_size, -1, num_classes])
else:
logits_offsets = np.cumsum([0] + num_classes)
softmax = []
for i in range(len(num_classes)):
sm = tf.nn.softmax(
tf.div(
logits_flat[:, logits_offsets[i]:logits_offsets[i + 1]],
tf.fill([num_classes[i]], temperature)))
sm = tf.reshape(sm, [hparams.batch_size, -1, num_classes[i]])
softmax.append(sm)
tf.add_to_collection('inputs', inputs)
tf.add_to_collection('temperature', temperature)
tf.add_to_collection('softmax', softmax)
# Flatten state tuples for metagraph compatibility.
for state in tf_nest.flatten(initial_state):
tf.add_to_collection('initial_state', state)
for state in tf_nest.flatten(final_state):
tf.add_to_collection('final_state', state)
def __init__(self, run_context=None, hyper_dict={}):
#inilization of hyper
for i in hyper_dict:
assert i in self.hyper, "Wrong hyper dict '{}'!".format(i)
self.hyper[i] = hyper_dict[i]
# inilize bg noise input
if self.hyper['bg_noise']:
self.bg_noise_input = tf.convert_to_tensor(self.hyper['bg_noise_input'],dtype=tf.float32)
else:
self.bg_noise_input = tf.convert_to_tensor(np.zeros((32,32)),dtype=tf.float32)
# inilization model
print('{} is initliazed!'.format(self.hyper['cnn']))
self.cnn = getattr(model,self.hyper['cnn'])
if run_context is not None:
self.training_log = run_context.create_train_log('training')
self.validation_log = run_context.create_train_log('validation')
self.checkpoint_path = os.path.join(run_context.transient_dir, 'checkpoint')
self.tensorboard_path = os.path.join(run_context.result_dir, 'tensorboard')
with tf.name_scope("placeholders"):
self.images = tf.placeholder(dtype=tf.float32, shape=(None,) + self.hyper['input_dim'], name='images')
self.labels = tf.placeholder(dtype=tf.int32, shape=(None,) + self.hyper['label_dim'], name='labels')
self.is_training = tf.placeholder(dtype=tf.bool, shape=(), name='is_training')
self.global_step = tf.Variable(0, trainable=False, name='global_step')
tf.add_to_collection("init_in_init", self.global_step)
with tf.name_scope("ramps"):
self.learning_rate, self.cons_coefficient, \
self.adam_beta_1, self.adam_beta_2, \
self.ema_decay = ramp_value(self.global_step,self.hyper)
( self.class_logits_1,
self.class_logits_ema
) = self.inference(
self.images,
is_training=self.is_training,
ema_decay=self.ema_decay,
input_noise=self.hyper['input_noise'],
student_dropout_probability=self.hyper['student_dropout_probability'],
teacher_dropout_probability=self.hyper['teacher_dropout_probability'],
normalize_input=self.hyper['normalize_input'],
flip_horizontally=self.hyper['flip_horizontally'],
translate=self.hyper['translate'])
with tf.name_scope("objectives"):
self.mean_error_1, self.errors_1 = errors(self.class_logits_1, self.labels,sig = self.hyper['sig'])
self.mean_error_ema, self.errors_ema = errors(self.class_logits_ema, self.labels,sig = self.hyper['sig'])
self.mean_class_cost_1, self.class_costs_1 = classification_costs(
self.class_logits_1, self.labels,sig = self.hyper['sig'])
self.mean_class_cost_ema, self.class_costs_ema = classification_costs(
self.class_logits_ema, self.labels,sig = self.hyper['sig'])
labeled_consistency = self.hyper['apply_consistency_to_labeled']
consistency_mask = tf.logical_or(tf.equal(self.labels, -1), labeled_consistency)
self.mean_cons_cost_mt, self.cons_costs_mt = consistency_costs(
self.class_logits_1, self.class_logits_ema, self.cons_coefficient, consistency_mask, self.hyper['consistency_trust'],sig = 'softmax')
self.mean_total_cost_mt, self.total_costs_mt = total_costs(
self.class_costs_1, self.cons_costs_mt)
self.cost_to_be_minimized = self.mean_total_cost_mt
with tf.name_scope("train_step"):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
if self.hyper['optimizer']=='adam':
self.train_step_op = nn.adam_optimizer(self.cost_to_be_minimized,
self.global_step,
learning_rate=self.learning_rate,
beta1=self.adam_beta_1,
beta2=self.adam_beta_2,
epsilon=self.hyper['adam_epsilon'])
elif self.hyper['optimizer']=='sgd':
self.train_step_op = nn.sgd_optimizer(self.cost_to_be_minimized,
self.global_step,
learning_rate=self.hyper['max_learning_rate'])
else:
assert False, 'Wrong optimizer input!'
self.training_metrics = {
"learning_rate": self.learning_rate,
"adam_beta_1": self.adam_beta_1,
"adam_beta_2": self.adam_beta_2,
"ema_decay": self.ema_decay,
"cons_coefficient": self.cons_coefficient,
"train/error/1": self.mean_error_1,
"train/error/ema": self.mean_error_ema,
"train/class_cost/1": self.mean_class_cost_1,
"train/class_cost/ema": self.mean_class_cost_ema,
"train/cons_cost/mt": self.mean_cons_cost_mt,
"train/total_cost/mt": self.mean_total_cost_mt,
}
with tf.variable_scope("validation_metrics") as metrics_scope:
self.metric_values, self.metric_update_ops = metrics.aggregate_metric_map({
"eval/error/1": streaming_mean(self.errors_1),
"eval/error/ema": streaming_mean(self.errors_ema),
"eval/class_cost/1": streaming_mean(self.class_costs_1),
"eval/class_cost/ema": streaming_mean(self.class_costs_ema),
})
metric_variables = slim.get_local_variables(scope=metrics_scope.name)
self.metric_init_op = tf.variables_initializer(metric_variables)
self.result_formatter = string_utils.DictFormatter(
order=["eval/error/ema", "error/1", "class_cost/1", "cons_cost/mt"],
default_format='{name}: {value:>10.6f}',
separator=", ")
self.result_formatter.add_format('error', '{name}: {value:>6.1%}')
with tf.name_scope("initializers"):
init_init_variables = tf.get_collection("init_in_init")
train_init_variables = [
var for var in tf.global_variables() if var not in init_init_variables
]
self.init_init_op = tf.variables_initializer(init_init_variables)
self.train_init_op = tf.variables_initializer(train_init_variables)
self.saver = tf.train.Saver()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.session = tf.Session(config=config)
self.run(self.init_init_op)
self.save_tensorboard_graph()
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset(
dataset_name=FLAGS.dataset,
split_name=FLAGS.eval_split,
dataset_dir=FLAGS.dataset_dir,
batch_size=FLAGS.eval_batch_size,
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
min_resize_value=FLAGS.min_resize_value,
max_resize_value=FLAGS.max_resize_value,
resize_factor=FLAGS.resize_factor,
model_variant=FLAGS.model_variant,
num_readers=2,
is_training=False,
should_shuffle=False,
should_repeat=False,
with_cls=True,
cls_only=False,
output_valid=True)
tf.gfile.MakeDirs(FLAGS.eval_logdir)
tf.logging.info('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next()
model_options = common.ModelOptions(
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
},
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape([
FLAGS.eval_batch_size,
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]), 3
])
if tuple(FLAGS.eval_scales) == (1.0, ):
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options,
image_pyramid=FLAGS.image_pyramid)
else:
tf.logging.info('Performing multi-scale test.')
raise NotImplementedError('Multi-scale is not supported yet!')
metric_map = {}
## Extract cls logits
if FLAGS.weakly:
_, end_points = feature_extractor.extract_features(
samples[common.IMAGE],
output_stride=model_options.output_stride,
multi_grid=model_options.multi_grid,
model_variant=model_options.model_variant,
depth_multiplier=model_options.depth_multiplier,
divisible_by=model_options.divisible_by,
reuse=tf.AUTO_REUSE,
is_training=False,
preprocessed_images_dtype=model_options.
preprocessed_images_dtype,
global_pool=True,
num_classes=dataset.num_of_classes - 1)
# ResNet beta version has an additional suffix in FLAGS.model_variant, but
# it shares the same variable names with original version. Add a special
# handling here for beta version ResNet.
logits = end_points['{}/logits'.format(
FLAGS.model_variant).replace('_beta', '')]
logits = tf.reshape(logits, [-1, dataset.num_of_classes - 1])
cls_pred = tf.sigmoid(logits)
# Multi-label classification evaluation
cls_label = samples['cls_label']
cls_pred = tf.cast(tf.greater_equal(cls_pred, 0.5), tf.int32)
## For classification
metric_map['eval/cls_overall'] = tf.metrics.accuracy(
labels=cls_label, predictions=cls_pred)
metric_map['eval/cls_precision'] = tf.metrics.precision(
labels=cls_label, predictions=cls_pred)
metric_map['eval/cls_recall'] = tf.metrics.recall(
labels=cls_label, predictions=cls_pred)
## For segmentation branch eval
predictions = predictions[common.OUTPUT_TYPE]
predictions = tf.reshape(predictions, shape=[-1])
labels = tf.reshape(samples[common.LABEL], shape=[-1])
weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label))
# Set ignore_label regions to label 0, because metrics.mean_iou requires
# range of labels = [0, dataset.num_classes). Note the ignore_label regions
# are not evaluated since the corresponding regions contain weights = 0.
labels = tf.where(tf.equal(labels, dataset.ignore_label),
tf.zeros_like(labels), labels)
predictions_tag = 'miou'
# Define the evaluation metric.
num_classes = dataset.num_of_classes
## For segmentation
metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou(
labels=labels,
predictions=predictions,
num_classes=num_classes,
weights=weights)
# IoU for each class.
one_hot_predictions = tf.one_hot(predictions, num_classes)
one_hot_predictions = tf.reshape(one_hot_predictions,
[-1, num_classes])
one_hot_labels = tf.one_hot(labels, num_classes)
one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes])
for c in range(num_classes):
predictions_tag_c = '%s_class_%d' % (predictions_tag, c)
tp, tp_op = tf.metrics.true_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fp, fp_op = tf.metrics.false_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fn, fn_op = tf.metrics.false_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op)
iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp),
tf.constant(np.NaN))
metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op)
(metrics_to_values,
metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map)
summary_ops = []
for metric_name, metric_value in six.iteritems(metrics_to_values):
op = tf.summary.scalar(metric_name, metric_value)
op = tf.Print(op, [metric_value], metric_name)
summary_ops.append(op)
summary_op = tf.summary.merge(summary_ops)
summary_hook = contrib_training.SummaryAtEndHook(
log_dir=FLAGS.eval_logdir, summary_op=summary_op)
hooks = [summary_hook]
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0:
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0:
contrib_quantize.create_eval_graph()
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
contrib_training.evaluate_repeatedly(
checkpoint_dir=FLAGS.checkpoint_dir,
master=FLAGS.master,
eval_ops=list(metrics_to_updates.values()),
max_number_of_evaluations=num_eval_iters,
hooks=hooks,
eval_interval_secs=FLAGS.eval_interval_secs)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset(
dataset_name=FLAGS.dataset,
split_name=FLAGS.eval_split,
dataset_dir=FLAGS.dataset_dir,
batch_size=FLAGS.eval_batch_size,
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
min_resize_value=FLAGS.min_resize_value,
max_resize_value=FLAGS.max_resize_value,
resize_factor=FLAGS.resize_factor,
model_variant=FLAGS.model_variant,
num_readers=2,
is_training=False,
should_shuffle=False,
should_repeat=False)
tf.gfile.MakeDirs(FLAGS.eval_logdir)
tf.logging.info('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next()
model_options = common.ModelOptions(
model_name=FLAGS.model_name,
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
},
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape([
FLAGS.eval_batch_size,
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]), 3
])
if tuple(FLAGS.eval_scales) == (1.0, ):
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options,
image_pyramid=FLAGS.image_pyramid)
else:
tf.logging.info('Performing multi-scale test.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
samples[common.IMAGE],
model_options=model_options,
eval_scales=FLAGS.eval_scales,
add_flipped_images=FLAGS.add_flipped_images)
predictions = predictions[common.OUTPUT_TYPE]
predictions = tf.reshape(predictions, shape=[-1])
labels = tf.reshape(samples[common.LABEL], shape=[-1])
weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label))
# Set ignore_label regions to label 0, because metrics.mean_iou requires
# range of labels = [0, dataset.num_classes). Note the ignore_label regions
# are not evaluated since the corresponding regions contain weights = 0.
labels = tf.where(tf.equal(labels, dataset.ignore_label),
tf.zeros_like(labels), labels)
predictions_tag = 'miou'
for eval_scale in FLAGS.eval_scales:
predictions_tag += '_' + str(eval_scale)
if FLAGS.add_flipped_images:
predictions_tag += '_flipped'
# Define the evaluation metric.
metric_map = {}
num_classes = dataset.num_of_classes
metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou(
labels=labels,
predictions=predictions,
num_classes=num_classes,
weights=weights)
# IoU for each class.
one_hot_predictions = tf.one_hot(predictions, num_classes)
one_hot_predictions = tf.reshape(one_hot_predictions,
[-1, num_classes])
one_hot_labels = tf.one_hot(labels, num_classes)
one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes])
for c in range(num_classes):
predictions_tag_c = '%s_class_%d' % (predictions_tag, c)
tp, tp_op = tf.metrics.true_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fp, fp_op = tf.metrics.false_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fn, fn_op = tf.metrics.false_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op)
iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp),
tf.constant(np.NaN))
metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op)
(metrics_to_values,
metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map)
summary_ops = []
for metric_name, metric_value in six.iteritems(metrics_to_values):
op = tf.summary.scalar(metric_name, metric_value)
op = tf.Print(op, [metric_value], metric_name)
summary_ops.append(op)
summary_op = tf.summary.merge(summary_ops)
summary_hook = contrib_training.SummaryAtEndHook(
log_dir=FLAGS.eval_logdir, summary_op=summary_op)
hooks = [summary_hook]
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0:
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0:
contrib_quantize.create_eval_graph()
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
contrib_training.evaluate_repeatedly(
checkpoint_dir=FLAGS.checkpoint_dir,
master=FLAGS.master,
eval_ops=list(metrics_to_updates.values()),
max_number_of_evaluations=num_eval_iters,
hooks=hooks,
eval_interval_secs=FLAGS.eval_interval_secs)
def build():
"""Builds the Tensorflow graph."""
inputs, lengths = None, None
if mode in ('train', 'eval'):
inputs, _, lengths = magenta.common.get_padded_batch(
sequence_example_file_paths,
hparams.batch_size,
input_size,
shuffle=mode == 'train')
elif mode == 'generate':
inputs = tf.placeholder(tf.float32,
[hparams.batch_size, None, input_size])
cell = events_rnn_graph.make_rnn_cell(
hparams.rnn_layer_sizes,
dropout_keep_prob=hparams.dropout_keep_prob
if mode == 'train' else 1.0,
attn_length=hparams.attn_length,
residual_connections=hparams.residual_connections)
rnn_nade = RnnNade(cell,
num_dims=input_size,
num_hidden=hparams.nade_hidden_units)
if mode in ('train', 'eval'):
log_probs, cond_probs = rnn_nade.log_prob(inputs, lengths)
inputs_flat = tf.to_float(
magenta.common.flatten_maybe_padded_sequences(inputs, lengths))
predictions_flat = tf.to_float(tf.greater_equal(cond_probs, .5))
if mode == 'train':
loss = tf.reduce_mean(-log_probs)
perplexity = tf.reduce_mean(tf.exp(log_probs))
correct_predictions = tf.to_float(
tf.equal(inputs_flat, predictions_flat))
accuracy = tf.reduce_mean(correct_predictions)
precision = (tf.reduce_sum(inputs_flat * predictions_flat) /
tf.reduce_sum(predictions_flat))
recall = (tf.reduce_sum(inputs_flat * predictions_flat) /
tf.reduce_sum(inputs_flat))
optimizer = tf.train.AdamOptimizer(
learning_rate=hparams.learning_rate)
train_op = contrib_slim.learning.create_train_op(
loss, optimizer, clip_gradient_norm=hparams.clip_norm)
tf.add_to_collection('train_op', train_op)
vars_to_summarize = {
'loss': loss,
'metrics/perplexity': perplexity,
'metrics/accuracy': accuracy,
'metrics/precision': precision,
'metrics/recall': recall,
}
elif mode == 'eval':
vars_to_summarize, update_ops = contrib_metrics.aggregate_metric_map(
{
'loss':
tf.metrics.mean(-log_probs),
'metrics/perplexity':
tf.metrics.mean(tf.exp(log_probs)),
'metrics/accuracy':
tf.metrics.accuracy(inputs_flat, predictions_flat),
'metrics/precision':
tf.metrics.precision(inputs_flat, predictions_flat),
'metrics/recall':
tf.metrics.recall(inputs_flat, predictions_flat),
})
for updates_op in update_ops.values():
tf.add_to_collection('eval_ops', updates_op)
precision = vars_to_summarize['metrics/precision']
recall = vars_to_summarize['metrics/precision']
f1_score = tf.where(
tf.greater(precision + recall, 0),
2 * ((precision * recall) / (precision + recall)), 0)
vars_to_summarize['metrics/f1_score'] = f1_score
for var_name, var_value in vars_to_summarize.items():
tf.summary.scalar(var_name, var_value)
tf.add_to_collection(var_name, var_value)
elif mode == 'generate':
initial_state = rnn_nade.zero_state(hparams.batch_size)
final_state = rnn_nade.steps(inputs, initial_state)
samples, log_prob = rnn_nade.sample_single(initial_state)
tf.add_to_collection('inputs', inputs)
tf.add_to_collection('sample', samples)
tf.add_to_collection('log_prob', log_prob)
# Flatten state tuples for metagraph compatibility.
for state in tf_nest.flatten(initial_state):
tf.add_to_collection('initial_state', state)
for state in tf_nest.flatten(final_state):
tf.add_to_collection('final_state', state)
