def test_histogram_summary(self, scope, skip_summary=False):
scoped_summary = _ScopedSummary(scope, skip_summary)
with self.test_session() as s:
i = tf.ones((5, 4, 4, 3))
with tf.name_scope("outer"):
summ_op = scoped_summary.histogram("inner", i)
summary_str = s.run(summ_op)
if skip_summary:
self.assertEqual("", decode(summary_str))
return
summary = tf.Summary()
summary.ParseFromString(summary_str)
self.assertLen(summary.value, 1)
self.assertEqual(summary.value[0].tag, "outer/inner")
def write_summary(eval_results,
summary_writer,
current_step,
predictions=None):
"""Write out eval results for the checkpoint."""
with tf.Graph().as_default():
summaries = []
for metric in eval_results:
summaries.append(
tf.Summary.Value(tag=metric,
simple_value=eval_results[metric]))
tf_summary = tf.Summary(value=list(summaries))
summary_writer.add_summary(tf_summary, current_step)
write_image_summary(predictions, summary_writer, current_step)
def _eval(sess, test_set, model, imgs, img_list, texts):
"""評価を行う"""
auc_sum = 0.0
for _, uij in DataInputTest(test_set, FLAGS.test_batch_size, imgs,
img_list, texts):
auc_sum += model.eval(sess, uij) * len(uij[0])
test_auc = auc_sum / len(test_set)
model.eval_writer.add_summary(summary=tf.Summary(
value=[tf.Summary.Value(tag='Eval AUC', simple_value=test_auc)]),
global_step=model.global_step.eval())
return test_auc
def call(step, **value_dict):
"""Adds values into the summary."""
summary = tf.Summary()
for tag, value in value_dict.items():
if isinstance(value, (float, np.float32)):
summary.value.add(tag=tag, simple_value=value)
else:
l = len(value)
for i in range(l):
summary.value.add(tag=tag + '_' + str(i),
simple_value=value[i])
writer.add_summary(summary, step)
writer.flush()
def test_merge_all(self, nest_graph):
c0 = tf.constant(0)
c1 = tf.constant(1)
scoped_summary0 = _ScopedSummary()
scoped_summary0.scalar("c0", c0)
scoped_summary0.scalar("c1", c1)
scoped_summary1 = _ScopedSummary("scope1")
scoped_summary1.scalar("c0", c0)
scoped_summary1.scalar("c1", c1)
scoped_summary2 = _ScopedSummary("scope2")
scoped_summary2.scalar("c0", c0)
scoped_summary2.scalar("c1", c1)
config = tf.compat.v1.ConfigProto(
gpu_options=tf.compat.v1.GPUOptions(allow_growth=True))
if nest_graph:
with tf.Graph().as_default():
scoped_summary2.scalar("c2", tf.constant(2))
with tf.Session(config=config) as sess:
summaries = scoped_summary2.merge_all()
tf.logging.warn("summaries %s", summaries)
summary = tf.Summary()
summary.ParseFromString(sess.run(tf.summary.merge(summaries)))
self.assertEqual(["c2"], [s.tag for s in summary.value])
self.assertEqual([2], [s.simple_value for s in summary.value])
with tf.Session(config=config) as sess:
for scoped_summary in [scoped_summary0, scoped_summary1, scoped_summary2]:
summaries = scoped_summary.merge_all()
summary = tf.Summary()
summary.ParseFromString(sess.run(tf.summary.merge(summaries)))
self.assertEqual(["c0", "c1"], [s.tag for s in summary.value])
self.assertEqual([0, 1], [s.simple_value for s in summary.value])
def eval_on_entire_dataset(sess, model, input_x, input_y, batch_size,
tb_prefix_and_iter, tb_writer):
#grad_sums = np.zeros(dim_sum)
num_batches = int(input_y.shape[0] / batch_size)
total_acc = 0
total_loss = 0
total_loss_no_reg = 0 # loss without counting l2 penalty
for i in range(num_batches):
# slice indices (should be large)
s_start = batch_size * i
s_end = s_start + batch_size
fetch_dict = {
'accuracy': model.accuracy,
'loss': model.loss,
'loss_no_reg': model.loss_cross_ent
}
result_dict = sess_run_dict(sess,
fetch_dict,
feed_dict={
model.input_images:
input_x[s_start:s_end],
model.input_labels:
input_y[s_start:s_end],
learning_phase(): 0,
batchnorm_learning_phase(): 1
}) # do not use nor update moving averages
total_acc += result_dict['accuracy']
total_loss += result_dict['loss']
total_loss_no_reg += result_dict['loss_no_reg']
acc = total_acc / num_batches
loss = total_loss / num_batches
loss_no_reg = total_loss_no_reg / num_batches
# tensorboard
if tb_writer:
tb_prefix, iterations = tb_prefix_and_iter
summary = tf.Summary()
summary.value.add(tag='%s_acc' % tb_prefix, simple_value=acc)
summary.value.add(tag='%s_loss' % tb_prefix, simple_value=loss)
summary.value.add(tag='%s_loss_no_reg' % tb_prefix,
simple_value=loss_no_reg)
tb_writer.add_summary(summary, iterations)
return acc, loss_no_reg
def test_image_summary_with_family(self, scope):
scoped_summary = _ScopedSummary(scope)
with self.test_session() as s:
i = tf.ones((5, 2, 3, 1))
with tf.name_scope("outer"):
im = scoped_summary.image("inner", i, max_outputs=3, family="family")
summary_str = s.run(im)
summary = tf.Summary()
summary.ParseFromString(summary_str)
values = summary.value
self.assertLen(values, 3)
tags = sorted(v.tag for v in values)
expected = sorted(
"family/outer/family/inner/image/{}".format(i) for i in range(3))
self.assertEqual(tags, expected)
def test_summarizing_variable(self, scope):
scoped_summary = _ScopedSummary(scope)
with self.test_session() as s:
c = tf.constant(42.0)
v = tf.Variable(c)
ss = scoped_summary.scalar("summary", v)
init = tf.global_variables_initializer()
s.run(init)
summ_str = s.run(ss)
summary = tf.Summary()
summary.ParseFromString(summ_str)
self.assertLen(summary.value, 1)
value = summary.value[0]
self.assertEqual(value.tag, "summary")
self.assertEqual(value.simple_value, 42.0)
def video_summary(name, video, step=None, fps=5):
name = name if isinstance(name, str) else name.decode('utf-8')
if np.issubdtype(video.dtype, np.floating):
video = np.clip(255 * video, 0, 255).astype(np.uint8)
B, T, H, W, C = video.shape
# try:
frames = video.transpose((1, 2, 0, 3, 4)).reshape((T, H, B * W, C))
summary = tf1.Summary()
image = tf1.Summary.Image(height=B * H, width=T * W, colorspace=C)
image.encoded_image_string = encode_gif(frames, fps)
summary.value.add(tag=name + '/gif', image=image)
tf.summary.experimental.write_raw_pb(summary.SerializeToString(), step)
# print('GIF summaries require ffmpeg in $PATH.', e)
frames = video.transpose((0, 2, 1, 3, 4)).reshape((1, B * H, T * W, C))
tf.summary.image(name + '/grid', frames, step)
def calc_running_avg_loss(loss,
running_avg_loss,
summary_writer,
step,
decay=0.99):
if running_avg_loss == 0: # on the first iteration just take the loss
running_avg_loss = loss
else:
running_avg_loss = running_avg_loss * decay + (1 - decay) * loss
running_avg_loss = min(running_avg_loss, 12) # clip
loss_sum = tf.Summary()
tag_name = 'running_avg_loss/decay=%f' % (decay)
loss_sum.value.add(tag=tag_name, simple_value=running_avg_loss)
summary_writer.add_summary(loss_sum, step)
return running_avg_loss
def get_metric(hparams, predictions, current_step):
"""Run inference and compute metric."""
predicted_ids = []
for prediction in predictions:
predicted_ids.append(prediction["predictions"])
if hparams.examples_to_infer < len(predicted_ids):
predicted_ids = predicted_ids[0:hparams.examples_to_infer]
translations = _convert_ids_to_strings(hparams.tgt_vocab_file,
predicted_ids)
trans_file = os.path.join(
hparams.out_dir, "newstest2014_out_{}.tok.de".format(current_step))
trans_dir = os.path.dirname(trans_file)
if not tf.gfile.Exists(trans_dir):
tf.gfile.MakeDirs(trans_dir)
tf.logging.info("Writing to file %s" % trans_file)
with codecs.getwriter("utf-8")(tf.gfile.GFile(trans_file,
mode="wb")) as trans_f:
trans_f.write("") # Write empty string to ensure file is created.
for translation in translations:
sentence = nmt_utils.get_translation(
translation,
tgt_eos=hparams.eos,
subword_option=hparams.subword_option)
trans_f.write((sentence + b"\n").decode("utf-8"))
# Evaluation
output_dir = os.path.join(hparams.out_dir, "eval")
tf.gfile.MakeDirs(output_dir)
summary_writer = tf.summary.FileWriter(output_dir)
ref_file = "%s.%s" % (hparams.test_prefix, hparams.tgt)
if hparams.use_REDACTED:
score = evaluation_utils.evaluate(ref_file, trans_file)
else:
score = get_sacrebleu(trans_file, hparams.detokenizer_file)
with tf.Graph().as_default():
summaries = []
summaries.append(tf.Summary.Value(tag="sacrebleu", simple_value=score))
tf_summary = tf.Summary(value=list(summaries))
summary_writer.add_summary(tf_summary, current_step)
misc_utils.print_out(" %s: %.1f" % ("sacrebleu", score))
summary_writer.close()
return score
def _write_tb(self, lam, lam_lb, lam_ub, cf_found, X_current, **kwargs):
if self.model:
scalars_tf = [
self.global_step, self.learning_rate, self.dist[0],
self.loss_pred[0], self.loss_opt[0], self.pred_proba_class[0]
]
gs, lr, dist, loss_pred, loss_opt, pred = self.sess.run(
scalars_tf, feed_dict={self.lam: lam})
else:
scalars_tf = [
self.global_step, self.learning_rate, self.dist[0],
self.loss_opt[0]
]
gs, lr, dist, loss_opt = self.sess.run(scalars_tf,
feed_dict={self.lam: lam})
loss_pred = kwargs['loss_pred']
pred = kwargs['pred']
try:
found = kwargs['found']
not_found = kwargs['not_found']
except KeyError:
found = 0
not_found = 0
summary = tf.Summary()
summary.value.add(tag='lr/global_step', simple_value=gs)
summary.value.add(tag='lr/lr', simple_value=lr)
summary.value.add(tag='lambda/lambda', simple_value=lam[0])
summary.value.add(tag='lambda/l_bound', simple_value=lam_lb[0])
summary.value.add(tag='lambda/u_bound', simple_value=lam_ub[0])
summary.value.add(tag='losses/dist', simple_value=dist)
summary.value.add(tag='losses/loss_pred', simple_value=loss_pred)
summary.value.add(tag='losses/loss_opt', simple_value=loss_opt)
summary.value.add(tag='losses/pred_div_dist',
simple_value=loss_pred / (lam[0] * dist))
summary.value.add(tag='Y/pred_proba_class', simple_value=pred)
summary.value.add(tag='Y/pred_class_fn(X_current)',
simple_value=self.predict_class_fn(X_current))
summary.value.add(tag='Y/n_cf_found', simple_value=cf_found[0].sum())
summary.value.add(tag='Y/found', simple_value=found)
summary.value.add(tag='Y/not_found', simple_value=not_found)
self.writer.add_summary(summary)
self.writer.flush()
def write_to_tensorboard(writer, tags, values, step):
"""Write metrics to tensorboard.
Args:
writer: tensorboard writer
tags: a list of tags of metrics
values: a list of values of metrics
step: step for the summary
"""
if writer is None:
return
summary = tf.Summary()
for tag, val in zip(tags, values):
summary.value.add(tag=tag, simple_value=val)
writer.add_summary(summary, step)
writer.flush()
def _save_tensorboard_summaries(self, iteration,
num_episodes,
average_reward):
"""Save statistics as tensorboard summaries.
Args:
iteration: int, The current iteration number.
num_episodes: int, number of episodes run.
average_reward: float, The average reward.
"""
summary = tf.Summary(value=[
tf.Summary.Value(tag='Play/NumEpisodes', simple_value=num_episodes),
tf.Summary.Value(tag='Play/AverageReturns',
simple_value=average_reward),
])
self._summary_writer.add_summary(summary, iteration)
def test_scalar_summary(self, scope, skip_summary=False):
scoped_summary = _ScopedSummary(scope, skip_summary)
with self.test_session() as s:
i = tf.constant(3)
with tf.name_scope("outer"):
im = scoped_summary.scalar("inner", i)
summary_str = s.run(im)
if skip_summary:
self.assertEqual("", decode(summary_str))
return
summary = tf.Summary()
summary.ParseFromString(summary_str)
values = summary.value
self.assertLen(values, 1)
self.assertEqual(values[0].tag, "outer/inner")
self.assertEqual(values[0].simple_value, 3.0)
def _write_scores_to_summary(scores: Mapping[str, Union[float, int]],
summary_writer: tf.summary.FileWriter,
current_step: int) -> None:
"""Writes eval scores to tf summary file."""
for metric_name, score in scores.items():
if metric_name.startswith("sp"):
metric_name = f"sp/{metric_name}"
elif metric_name.startswith("joint"):
metric_name = f"joint/{metric_name}"
else:
metric_name = f"ans/{metric_name}"
summary_writer.add_summary(tf.Summary(value=[
tf.Summary.Value(tag=metric_name, simple_value=score),
]),
global_step=current_step)
summary_writer.flush()
def _video_summary(self, name, video, step):
import tensorflow as tf
import tensorflow.compat.v1 as tf1
name = name if isinstance(name, str) else name.decode('utf-8')
if np.issubdtype(video.dtype, np.floating):
video = np.clip(255 * video, 0, 255).astype(np.uint8)
try:
T, H, W, C = video.shape
summary = tf1.Summary()
image = tf1.Summary.Image(height=H, width=W, colorspace=C)
image.encoded_image_string = encode_gif(video, self._fps)
summary.value.add(tag=name, image=image)
tf.summary.experimental.write_raw_pb(summary.SerializeToString(),
step)
except (IOError, OSError) as e:
print('GIF summaries require ffmpeg in $PATH.', e)
tf.summary.image(name, video, step)
def test_audio_summary(self, scope, skip_summary=False):
scoped_summary = _ScopedSummary(scope, skip_summary)
with self.test_session() as s:
i = tf.ones((5, 3, 4))
with tf.name_scope("outer"):
aud = scoped_summary.audio("inner", i, 0.2, max_outputs=3)
summary_str = s.run(aud)
if skip_summary:
self.assertEqual("", decode(summary_str))
return
summary = tf.Summary()
summary.ParseFromString(summary_str)
values = summary.value
self.assertLen(values, 3)
tags = sorted(v.tag for v in values)
expected = sorted("outer/inner/audio/{}".format(i) for i in range(3))
self.assertEqual(tags, expected)
def train_epoch(self, sess, epoch, summary_writer=None):
self._model.set_is_training(True)
cur_epoch = epoch
iter = 0
loss_sum = 0
acc_sum = 0
for batch_idx in range(0, self._dataset.batch_count()):
self.global_iteration += 1
iter += 1
X, Y = self._dataset.next_batch(batch_idx)
im = X
label = Y
_, loss, acc, cur_lr = sess.run(
[
self._train_op, self._loss_op, self._accuracy_op,
self._lr_op
],
feed_dict={
self._model.image: im,
self._model.label: label,
self._model.lr: self._lr
})
loss_sum += loss
acc_sum += acc
if iter % 100 == 0:
print('Iteration: {}, loss: {:.4f}, accuracy: {:.4f}'.format(
self.global_iteration, loss_sum * 1.0 / iter,
acc_sum * 1.0 / iter))
print('epoch: {}, loss: {:.4f}, accuracy: {:.4f}, lr:{}'.format(
cur_epoch, loss_sum * 1.0 / iter, acc_sum * 1.0 / iter, cur_lr))
self._dataset.on_epoch_end()
if summary_writer is not None:
s = tf.Summary()
s.value.add(tag='train/loss', simple_value=loss_sum * 1.0 / iter)
s.value.add(tag='train/accuracy',
simple_value=acc_sum * 1.0 / iter)
summary_writer.add_summary(s, self.global_iteration)
def flush(self):
"""Output metrics to tensorboard for global step."""
metrics_d = self.get_current_metrics()
if self._verbose:
logging.info('COMPUTING METRICS COMPLETE. FLUSHING ALL METRICS')
metric_keys = ['ld_metric', 'f0_metric', 'f0_outlier_ratio']
metrics_str = ' | '.join(
'{}: {:0.3f}'.format(m, metrics_d[m]) for m in metric_keys)
logging.info(metrics_str)
counts_keys = [
'f0_gen_trackable_pitch_count', 'f0_gen_pitch_outlier_count',
'f0_gen_untrackable_pitch_count',
'f0_ground_truth_untrackable_pitch_count'
]
counts_str = ' | '.join(
'{}: {}'.format(c, metrics_d[c]) for c in counts_keys)
logging.info(counts_str)
summary = tf.Summary()
for value_name in [
'ld_metric',
'f0_metric',
'f0_outlier_ratio',
]:
summary.value.add(
tag='rt_metrics/' + value_name, simple_value=metrics_d[value_name])
for value_name in [
'f0_gen_trackable_pitch_count', 'f0_gen_pitch_outlier_count',
'f0_gen_untrackable_pitch_count',
'f0_ground_truth_untrackable_pitch_count'
]:
summary.value.add(
tag='counts/' + value_name, simple_value=metrics_d[value_name])
self._file_writer.add_summary(summary, self._global_step)
self._file_writer.flush()
logging.info('Wrote metric summaries for step %s to %s', self._global_step,
self._summary_dir)
spectral_ops.reset_crepe() # Reset CREPE global state
def update(self, gen_audio_outputs, ground_truth_feats, tensor_dict):
"""Update metrics dictionary given a batch of audio."""
gt = ground_truth_feats['audio']
a1 = tensor_dict.get('additive_audio')
a2 = tensor_dict.get('noise_audio')
a3 = gen_audio_outputs
def _plot(summary, sample_idx, length=None, prefix='waveform'):
"""Plots all waveforms."""
waveforms = []
labels = []
for a, label in zip([gt, a1, a2, a3],
['gt', 'additive', 'noise', 'synthesized']):
if a is not None:
x = check_and_squeeze_to_vector(a[sample_idx])
if length is not None:
x = x[:length]
waveforms.append(x)
labels.append(label)
# Manually specify exact size of fig for tensorboard
num_subplots = len(labels)
fig = plt.figure(figsize=(2.5 * num_subplots, 10))
for i in range(num_subplots):
ax = fig.add_subplot(num_subplots, 1, i + 1)
ax.plot(waveforms[i])
ax.set_title(labels[i])
# Format and save plot to image
buf = io.BytesIO()
fig.savefig(buf, format='png')
image_summary = tf.Summary.Image(
encoded_image_string=buf.getvalue())
plt.close(fig)
summary.value.add(tag='{}/{}'.format(prefix, sample_idx),
image=image_summary)
summary = tf.Summary()
for sample_idx in range(self._batch_size):
_plot(summary, sample_idx, length=None, prefix='waveform_4s')
_plot(summary, sample_idx, length=2000, prefix='waveform_125ms')
self._file_writer.add_summary(summary, self._global_step)
def end(self, session):
results = run_attack_helper(self._estimator, self._in_train_input_fn,
self._out_train_input_fn,
self._in_train_labels,
self._out_train_labels,
self._attack_classifiers)
print('all_thresh_loss_advantage',
results['all_thresh_loss_advantage'])
logging.info(results)
if self._writer:
summary = tf.Summary()
summary.value.add(
tag='attack advantage',
simple_value=results['all_thresh_loss_advantage'])
global_step = self._estimator.get_variable_value('global_step')
self._writer.add_summary(summary, global_step)
self._writer.flush()
def video_summary(name, video, step=None, fps=20):
# print("name:", name)
# name = name if isinstance(name, str) else name.decode("utf-8")
name = str(name)
if np.issubdtype(video.dtype, np.floating):
video = np.clip(255 * video, 0, 255).astype(np.uint8)
B, T, H, W, C = video.shape
try:
frames = video.transpose((1, 2, 0, 3, 4)).reshape((T, H, B * W, C))
summary = tf1.Summary()
image = tf1.Summary.Image(height=B * H, width=T * W, colorspace=C)
image.encoded_image_string = encode_gif(frames, fps)
summary.value.add(tag=name + "/gif", image=image)
tf.summary.experimental.write_raw_pb(summary.SerializeToString(), step)
except (IOError, OSError) as e:
print("GIF summaries require ffmpeg in $PATH.", e)
frames = video.transpose((0, 2, 1, 3, 4)).reshape((1, B * H, T * W, C))
tf.summary.image(name + "/grid", frames, step)
def testGifSummary(self):
for c in (1, 3):
images_shape = (1, 12, 48, 64, c) # batch, time, height, width, channels
images = np.random.randint(256, size=images_shape).astype(np.uint8)
with self.test_session():
summary = common_video.gif_summary(
"gif", tf.convert_to_tensor(images), fps=10)
summary_string = summary.eval()
summary = tf.Summary()
summary.ParseFromString(summary_string)
self.assertEqual(1, len(summary.value))
self.assertTrue(summary.value[0].HasField("image"))
encoded = summary.value[0].image.encoded_image_string
self.assertEqual(encoded, common_video._encode_gif(images[0], fps=10)) # pylint: disable=protected-access
def image_summary(self, tag, images, step):
img_summaries = []
for i, img in enumerate(images):
# Write the image to a string
s = BytesIO()
Image.fromarray(img).save(s, format="png")
# Create an Image object
img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(),
height=img.shape[0],
width=img.shape[1])
# Create a Summary value
img_summaries.append(
tf.Summary.Value(tag='%s/%d' % (tag, i), image=img_sum))
# Create and write Summary
summary = tf.Summary(value=img_summaries)
self.writer.add_summary(summary, step)
def test_summary_name_conversion(self, scope):
scoped_summary = _ScopedSummary(scope)
c = tf.constant(3)
summary = tf.Summary()
with self.test_session() as sess:
s = scoped_summary.scalar("name with spaces", c)
summary.ParseFromString(sess.run(s))
self.assertEqual(summary.value[0].tag, "name_with_spaces")
s2 = scoped_summary.scalar("name with many $#illegal^: characters!", c)
summary.ParseFromString(sess.run(s2))
self.assertEqual(summary.value[0].tag,
"name_with_many___illegal___characters_")
s3 = scoped_summary.scalar("/name/with/leading/slash", c)
summary.ParseFromString(sess.run(s3))
self.assertEqual(summary.value[0].tag, "name/with/leading/slash")
def _log_epoch(self, epoch):
# Log info about epoch
self.logger.log_tabular("Epoch", epoch)
self.logger.log_tabular("EpRet", with_min_and_max=True)
self.logger.log_tabular("EpWinRate", average_only=True)
self.logger.log_tabular("EpLen", average_only=True)
self.logger.log_tabular("VVals", with_min_and_max=True)
self.logger.log_tabular("LossPi", average_only=True)
self.logger.log_tabular("LossV", average_only=True)
self.logger.log_tabular("DeltaLossPi", average_only=True)
self.logger.log_tabular("DeltaLossV", average_only=True)
self.logger.log_tabular("Entropy", average_only=True)
self.logger.log_tabular("KL", average_only=True)
self.logger.log_tabular("ClipFrac", average_only=True)
self.logger.log_tabular("StopIter", average_only=True)
if proc_id() == 0:
scalar_summary = tf.Summary()
for field in [
"MinEpRet",
"MaxEpRet",
"AverageEpRet",
"StdEpRet",
"EpWinRate",
"EpLen",
"MinVVals",
"MaxVVals",
"AverageVVals",
"StdVVals",
"LossPi",
"LossV",
"DeltaLossPi",
"DeltaLossV",
"Entropy",
"KL",
"ClipFrac",
"StopIter",
]:
val = self.logger.log_current_row[field]
scalar_summary.value.add(tag=field, simple_value=val)
self.writer.add_summary(scalar_summary, global_step=epoch)
self.last_epoch_wr = self.logger.log_current_row["EpWinRate"]
self.logger.dump_tabular()
def eval_finish_fn(cur_step, eval_output, summary_writer):
"""Executed after every eval."""
global run_steps
global masked_lm_accuracy
cur_step_corrected = cur_step + FLAGS.iterations_per_loop
run_steps = cur_step_corrected
masked_lm_weighted_correct = eval_output["masked_lm_weighted_correct"]
masked_lm_weighted_count = eval_output["masked_lm_weighted_count"]
masked_lm_accuracy = np.sum(masked_lm_weighted_correct) / np.sum(
masked_lm_weighted_count)
# the eval_output may mix up the order of the two arrays
# swap the order if it did got mix up
if masked_lm_accuracy > 1:
masked_lm_accuracy = 1 / masked_lm_accuracy
if summary_writer:
with tf.Graph().as_default():
summary_writer.add_summary(
tf.Summary(value=[
tf.Summary.Value(tag="masked_lm_accuracy",
simple_value=masked_lm_accuracy)
]), cur_step_corrected)
mlp_log.mlperf_print(
"block_stop",
None,
metadata={
"first_epoch_num": cur_step_corrected,
})
# While BERT pretraining does not have epochs,
# to make the logging consistent with other mlperf models,
# in all the mlp_log, epochs are steps, and examples are sequences.
mlp_log.mlperf_print(
"eval_accuracy",
float(masked_lm_accuracy),
metadata={"epoch_num": cur_step_corrected})
if (masked_lm_accuracy >= FLAGS.stop_threshold and
cur_step_corrected >= FLAGS.iterations_per_loop * 6):
mlp_log.mlperf_print("run_stop", None, metadata={"status": "success"})
return True
else:
return False
def train(config):
"""Trains and returns an Safe RNN agent."""
_set_experiment_name(config)
logging.info('Launching experiment id: %s', config['experiment_name'])
_setup_directories(config)
envs = _envs_builder(config, config['num_episodes_per_update'])
config_str = fg_core.to_json(config)
config = types.SimpleNamespace(**config)
pkl.dump(config, open('config.pkl', 'wb'))
json.dump(config_str, open('config_.json', 'w'))
print("Config has been saved")
agent = _agent_builder(envs[0], config)
writer = tf.summary.FileWriter(
os.path.join(config.results_dir,
'runs/{}'.format(config.experiment_name)))
while config.warm_start.initial_batch < config.num_updates:
last_checkpoint = _training_loop(envs, agent, config, writer)
agent.set_batch_size(1)
metrics = evaluation.evaluate_agent(
agent,
envs[0],
alpha=config.alpha,
num_users=config.num_users_eval,
deterministic=config.eval_deterministic)
agent.set_batch_size(len(envs))
step = config.warm_start.initial_batch * config.num_episodes_per_update
print(step, last_checkpoint, metrics)
log_tboard(metrics, writer, step)
summary = tf.Summary(value=[
tf.Summary.Value(tag='training_var', simple_value=agent.var)
])
writer.add_summary(summary, step)
# Do one final eval at the end.
agent.set_batch_size(1)
metrics = evaluation.evaluate_agent(
agent,
envs[0],
alpha=config.alpha,
num_users=config.num_users_eval_final,
deterministic=config.eval_deterministic)
agent.set_batch_size(len(envs))
return agent
def eval_once(saver, summary_writer, top_k_op, summary_op):
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split('/')[-1].split(
'-')[-1]
else:
print('No checkpoint file found')
return
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(
qr.create_threads(sess,
coord=coord,
daemon=True,
start=True))
num_iter = int(math.ceil(FLAGS.num_examples / FLAGS.batch_size))
true_count = 0
total_sample_count = num_iter * FLAGS.batch_size
step = 0
while step < num_iter and not coord.should_stop():
predictions = sess.run([top_k_op])
true_count += np.sum(predictions)
step += 1
precision = true_count / total_sample_count
print('%s: precision @ 1 = %.3f' % (datetime.now(), precision))
summary = tf.Summary()
summary.ParseFromString(sess.run(summary_op))
summary.value.add(tag='Precision @ 1', simple_value=precision)
summary_writer.add_summary(summary, global_step)
except Exception as e:
coord.request_stop(e)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
