def evaluate_on_validation_set(validationset, step: int, session, model, summary_writer, validation_summary,
val_loss, workspace: Workspace):
"""Convenience function for evaluating network on a validation set
Parameters
-------
validationset : dataset reader
Dataset reader for the validation set
step : int
Current step in training
session : tf.session
Tensorflow session to use
model : network model
Network model
val_loss : tensor
Tensor representing the validation loss computation
Returns
-------
: float
Loss averaged over validation set
"""
loss = 0
_pbw = ['Evaluating on validation set: ', progressbar.ETA()]
progress = progressbar.ProgressBar(widgets=_pbw, maxval=validationset.n_mbs - 1, redirect_stdout=True).start()
mb_validation = validationset.batch_loader()
with Timer(verbose=True, name="Evaluate on Validation Set"):
for mb_i, mb in enumerate(mb_validation):
# Abort if indicated by file
check_kill_file(workspace)
val_summary, cur_loss = session.run([validation_summary, val_loss],
feed_dict={model.X: mb['X'], model.y_: mb['y']})
loss += cur_loss
progress.update(mb_i)
mb.clear()
del mb
progress.finish()
avg_loss = loss / validationset.n_mbs
summary_writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag="Validation Loss", simple_value=avg_loss)]),
step)
print("Validation scores:\n\tstep {} validation loss {}".format(step, avg_loss))
sys.stdout.flush()
return avg_loss
def initialize_tf_variables(self, reset_optimizer_on_restore=False):
"""
Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint).
:param reset_optimizer_on_restore: Flag indicating whether to reset the optimizer(s) given that this
function call includes a restore operation.
:return: updated TeLL session
"""
session = self.tf_session
checkpoint = self.workspace.get_checkpoint()
#
# Initialize or load variables
#
with Timer(name="Initializing variables"):
session.run(tf.global_variables_initializer())
session.run(tf.local_variables_initializer())
if checkpoint is not None:
# restore from checkpoint
self.tf_saver.restore(session, checkpoint)
# get step number from checkpoint
step = session.run(self.__global_step_placeholder) + 1
self.global_step = step
# reopen summaries
for _, summary in self.tf_summaries.items():
summary.reopen()
summary.add_session_log(
tf.SessionLog(status=tf.SessionLog.START),
global_step=step)
print("Resuming from checkpoint '{}' at iteration {}".format(
checkpoint, step))
if self.config.get_value('optimizer', None) is not None:
if reset_optimizer_on_restore:
if isinstance(self.tf_optimizer, list):
for optimizer in self.tf_optimizer:
self.reset_optimizer(optimizer)
else:
self.reset_optimizer(self.tf_optimizer)
else:
for _, summary in self.tf_summaries.items():
summary.add_graph(session.graph)
return self
def main(_):
config = Config()
np.random.seed(config.get_value("random_seed", 12345))
# PARAMETERS
n_epochs = config.get_value("epochs", 100)
batchsize = config.get_value("batchsize", 8)
n_classes = config.get_value("n_classes", 13)
dropout = config.get_value("dropout", 0.25) # TODO
num_threads = config.get_value("num_threads", 5)
initial_val = config.get_value("initial_val", True)
# READER, LOADER
readers = invoke_dataset_from_config(config)
reader_train = readers["train"]
reader_val = readers["val"]
train_loader = torch.utils.data.DataLoader(reader_train,
batch_size=config.batchsize,
shuffle=True,
num_workers=num_threads)
val_loader = torch.utils.data.DataLoader(reader_val,
batch_size=1,
shuffle=False,
num_workers=num_threads)
# CONFIG
tell = TeLLSession(config=config,
model_params={"shape": reader_train.shape})
# Get some members from the session for easier usage
session = tell.tf_session
model = tell.model
workspace, config = tell.workspace, tell.config
prediction = tf.sigmoid(model.output)
prediction_val = tf.reduce_mean(tf.sigmoid(model.output),
axis=0,
keepdims=True)
# LOSS
if hasattr(model, "loss"):
loss = model.loss()
else:
with tf.name_scope("Loss_per_Class"):
loss = 0
for i in range(n_classes):
loss_batch = tf.nn.sigmoid_cross_entropy_with_logits(
logits=model.output[:, i], labels=model.y_[:, i])
loss_mean = tf.reduce_mean(loss_batch)
loss += loss_mean
# Validation loss after patching
if hasattr(model, "loss"):
loss_val = model.loss()
else:
with tf.name_scope("Loss_per_Class_Patching"):
loss_val = 0
for i in range(n_classes):
loss_batch = tf.nn.sigmoid_cross_entropy_with_logits(
logits=tf.reduce_mean(model.output[:, i],
axis=0,
keepdims=True),
labels=model.y_[:, i])
loss_mean = tf.reduce_mean(loss_batch)
loss_val += loss_mean
# REGULARIZATION
reg_penalty = regularize(layers=model.layers,
l1=config.l1,
l2=config.l2,
regularize_weights=True,
regularize_biases=True)
# LEARNING RATE (SCHEDULE)
# if a LRS is defined always use MomentumOptimizer and pass learning rate to optimizer
lrs_plateu = False
if config.get_value("lrs", None) is not None:
lr_sched_type = config.lrs["type"]
if lr_sched_type == "plateau":
lrs_plateu = True
learning_rate = tf.placeholder(tf.float32, [],
name='learning_rate')
lrs_learning_rate = config.get_value(
"optimizer_params")["learning_rate"]
lrs_n_bad_epochs = 0 # counter for plateu LRS
lrs_patience = config.lrs["patience"]
lrs_factor = config.lrs["factor"]
lrs_threshold = config.lrs["threshold"]
lrs_mode = config.lrs["mode"]
lrs_best = -np.inf if lrs_mode == "max" else np.inf
lrs_is_better = lambda old, new: (new > old * (
1 + lrs_threshold)) if lrs_mode == "max" else (new < old * (
1 - lrs_threshold))
else:
learning_rate = None # if no LRS is defined the default optimizer is used with its defined learning rate
# LOAD WEIGHTS and get list of trainables if specified
assign_loaded_variables = None
trainables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
if config.get_value("checkpoint", None) is not None:
with Timer(name="Loading Checkpoint", verbose=True):
assign_loaded_variables, trainables = tell.load_weights(
config.get_value("checkpoint", None),
config.get_value("freeze", False),
config.get_value("exclude_weights", None),
config.get_value("exclude_freeze", None))
# Update step
if len(trainables) > 0:
update, gradients, gradient_name_dict = update_step(
loss + reg_penalty,
config,
tell,
lr=learning_rate,
trainables=trainables)
# INITIALIZE Tensorflow VARIABLES
step = tell.initialize_tf_variables().global_step
# ASSING LOADED WEIGHTS (overriding initializations) if available
if assign_loaded_variables is not None:
session.run(assign_loaded_variables)
# -------------------------------------------------------------------------
# Start training
# -------------------------------------------------------------------------
try:
n_mbs = len(train_loader)
epoch = int((step * batchsize) / (n_mbs * batchsize))
epochs = range(epoch, n_epochs)
if len(trainables) == 0:
validate(val_loader, n_classes, session, loss_val, prediction_val,
model, workspace, step, batchsize, tell)
return
print("Epoch: {}/{} (step: {}, nmbs: {}, batchsize: {})".format(
epoch + 1, n_epochs, step, n_mbs, batchsize))
for ep in epochs:
if ep == 0 and initial_val:
f1 = validate(val_loader, n_classes, session, loss_val,
prediction_val, model, workspace, step,
batchsize, tell)
else:
if config.has_value("lrs_best") and config.has_value(
"lrs_learning_rate") and config.has_value(
"lrs_n_bad_epochs"):
f1 = config.get_value("lrs_f1")
lrs_best = config.get_value("lrs_best")
lrs_learning_rate = config.get_value("lrs_learning_rate")
lrs_n_bad_epochs = config.get_value("lrs_n_bad_epochs")
else:
f1 = 0
# LRS "Plateu"
if lrs_plateu:
# update scheduler
if lrs_is_better(lrs_best, f1):
lrs_best = f1
lrs_n_bad_epochs = 0
else:
lrs_n_bad_epochs += 1
# update learning rate
if lrs_n_bad_epochs > lrs_patience:
lrs_learning_rate = max(lrs_learning_rate * lrs_factor, 0)
lrs_n_bad_epochs = 0
with tqdm(total=len(train_loader),
desc="Training [{}/{}]".format(ep + 1,
len(epochs))) as pbar:
for mbi, mb in enumerate(train_loader):
# LRS "Plateu"
if lrs_plateu:
feed_dict = {
model.X: mb['input'].numpy(),
model.y_: mb['target'].numpy(),
model.dropout: dropout,
learning_rate: lrs_learning_rate
}
else:
feed_dict = {
model.X: mb['input'].numpy(),
model.y_: mb['target'].numpy(),
model.dropout: dropout
}
# TRAINING
pred, loss_train, _ = session.run(
[prediction, loss, update], feed_dict=feed_dict)
# Update status
pbar.set_description_str(
"Training [{}/{}] Loss: {:.4f}".format(
ep + 1, len(epochs), loss_train))
pbar.update()
step += 1
validate(val_loader, n_classes, session, loss_val, prediction_val,
model, workspace, step, batchsize, tell)
except AbortRun:
print("Aborting...")
finally:
tell.close(global_step=step, save_checkpoint=True)
def main(_):
# ------------------------------------------------------------------------------------------------------------------
# Setup training
# ------------------------------------------------------------------------------------------------------------------
# Initialize config, parses command line and reads specified file; also supports overriding of values from cmd
config = Config()
#
# Load datasets for training and validation
#
with Timer(name="Loading Data", verbose=True):
# Make sure datareader is reproducible
random_seed = config.get_value('random_seed', 12345)
np.random.seed(
random_seed) # not threadsafe, use rnd_gen object where possible
rnd_gen = np.random.RandomState(seed=random_seed)
print("Loading training data...")
trainingset = MovingDotDataset(n_timesteps=5,
n_samples=50,
batchsize=config.batchsize,
rnd_gen=rnd_gen)
print("Loading validation data...")
validationset = MovingDotDataset(n_timesteps=5,
n_samples=25,
batchsize=config.batchsize,
rnd_gen=rnd_gen)
#
# Initialize TeLL session
#
tell = TeLLSession(config=config,
summaries=["train", "validation"],
model_params={"dataset": trainingset})
# Get some members from the session for easier usage
sess = tell.tf_session
summary_writer_train, summary_writer_validation = tell.tf_summaries[
"train"], tell.tf_summaries["validation"]
model = tell.model
workspace, config = tell.workspace, tell.config
#
# Define loss functions and update steps
#
print("Initializing loss calculation...")
pos_target_weight = np.prod(
trainingset.y_shape[2:]
) - 1 # only 1 pixel per sample is of positive class -> up-weight!
loss = tf.reduce_mean(
tf.nn.weighted_cross_entropy_with_logits(targets=model.y_,
logits=model.output,
pos_weight=pos_target_weight))
# loss = tf.reduce_mean(-tf.reduce_sum((model.y_ * tf.log(model.output)) *
# -tf.reduce_sum(model.y_ - 1) / tf.reduce_sum(model.y_),
# axis=[1, 2, 3, 4]))
train_summary = tf.summary.scalar(
"Training Loss", loss) # create summary to add to tensorboard
# Loss function for validationset
val_loss = tf.reduce_mean(
tf.nn.weighted_cross_entropy_with_logits(targets=model.y_,
logits=model.output,
pos_weight=pos_target_weight))
# val_loss = tf.reduce_mean(-tf.reduce_sum(model.y_ * tf.log(model.output) *
# -tf.reduce_sum(model.y_ - 1) / tf.reduce_sum(model.y_),
# axis=[1, 2, 3, 4]))
val_loss_summary = tf.summary.scalar(
"Validation Loss", val_loss) # create summary to add to tensorboard
# Regularization
reg_penalty = regularize(layers=model.get_layers(),
l1=config.l1,
l2=config.l2,
regularize_weights=True,
regularize_biases=True)
regpen_summary = tf.summary.scalar(
"Regularization Penalty",
reg_penalty) # create summary to add to tensorboard
# Update step for weights
update = update_step(loss + reg_penalty, config)
#
# Prepare plotting
#
plot_elements_sym = list(model.get_plot_dict().values())
plot_elements = list()
plot_ranges = model.get_plot_range_dict()
#
# Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint)
#
global_step = tell.initialize_tf_variables().global_step
#
# Finalize graph
# This makes our tensorflow graph read-only and prevents further additions to the graph
#
sess.graph.finalize()
if sess.graph.finalized:
print("Graph is finalized!")
else:
raise ValueError("Could not finalize graph!")
sys.stdout.flush()
# ------------------------------------------------------------------------------------------------------------------
# Start training
# ------------------------------------------------------------------------------------------------------------------
try:
epoch = int(global_step / trainingset.n_mbs)
epochs = range(epoch, config.n_epochs)
# Loop through epochs
print("Starting training")
for ep in epochs:
epoch = ep
print("Starting training epoch: {}".format(ep))
# Initialize variables for over-all loss per epoch
train_loss = 0
# Load one minibatch at a time and perform a training step
t_mb = Timer(verbose=True, name="Load Minibatch")
mb_training = trainingset.batch_loader(rnd_gen=rnd_gen)
#
# Loop through minibatches
#
for mb_i, mb in enumerate(mb_training):
sys.stdout.flush()
# Print minibatch load time
t_mb.print()
# Abort if indicated by file
check_kill_file(workspace)
#
# Calculate scores on validation set
#
if global_step % config.score_at == 0:
print("Starting scoring on validation set...")
evaluate_on_validation_set(validationset, global_step,
sess, model,
summary_writer_validation,
val_loss_summary, val_loss,
workspace)
#
# Perform weight updates and do plotting
#
if (mb_i % config.plot_at) == 0 and os.path.isfile(
workspace.get_plot_file()):
# Perform weight update, return summary_str and values for plotting
with Timer(verbose=True, name="Weight Update"):
train_summ, regpen_summ, _, cur_loss, cur_output, *plot_elements = sess.run(
[
train_summary, regpen_summary, update, loss,
model.output, *plot_elements_sym
],
feed_dict={
model.X: mb['X'],
model.y_: mb['y']
})
# Add current summary values to tensorboard
summary_writer_train.add_summary(train_summ,
global_step=global_step)
summary_writer_train.add_summary(regpen_summ,
global_step=global_step)
# Re-associate returned tensorflow values to plotting keys
plot_dict = OrderedDict(
zip(list(model.get_plot_dict().keys()), plot_elements))
#
# Plot subplots in plot_dict
# Loop through each element in plotlist and pass it to the save_subplots function for plotting
# (adapt this to your needs for plotting)
#
with Timer(verbose=True, name="Plotting",
precision="msec"):
for plotlist_i, plotlist in enumerate(
model.get_plotsink()):
for frame in range(len(plot_dict[plotlist[0]])):
subplotlist = []
subfigtitles = []
subplotranges = []
n_cols = int(np.ceil(np.sqrt(len(plotlist))))
for col_i, col_i in enumerate(range(n_cols)):
subfigtitles.append(
plotlist[n_cols *
col_i:n_cols * col_i +
n_cols])
subplotlist.append([
plot_dict[p]
[frame *
(frame < len(plot_dict[p])), :]
for p in plotlist[n_cols *
col_i:n_cols *
col_i + n_cols]
])
subplotranges.append([
plot_ranges.get(p, False)
for p in plotlist[n_cols *
col_i:n_cols *
col_i + n_cols]
])
# remove rows/columns without images
subplotlist = [
p for p in subplotlist if p != []
]
plot_args = dict(
images=subplotlist,
filename=os.path.join(
workspace.get_result_dir(),
"plot{}_ep{}_mb{}_fr{}.png".format(
plotlist_i, ep, mb_i, frame)),
subfigtitles=subfigtitles,
subplotranges=subplotranges)
plotter.set_plot_kwargs(plot_args)
plotter.plot()
# Plot outputs and cell states over frames if specified
if config.store_states and 'ConvLSTMLayer_h' in plot_dict:
convh = plot_dict['ConvLSTMLayer_h']
convrh = [c[0, :, :, 0] for c in convh]
convrh = [
convrh[:6], convrh[6:12], convrh[12:18],
convrh[18:24], convrh[24:]
]
plot_args = dict(images=convrh,
filename=os.path.join(
workspace.get_result_dir(),
"plot{}_ep{}_mb{}_h.png".format(
plotlist_i, ep, mb_i)))
plotter.set_plot_kwargs(plot_args)
plotter.plot()
if config.store_states and 'ConvLSTMLayer_c' in plot_dict:
convc = plot_dict['ConvLSTMLayer_c']
convrc = [c[0, :, :, 0] for c in convc]
convrc = [
convrc[:6], convrc[6:12], convrc[12:18],
convrc[18:24], convrc[24:]
]
plot_args = dict(images=convrc,
filename=os.path.join(
workspace.get_result_dir(),
"plot{}_ep{}_mb{}_c.png".format(
plotlist_i, ep, mb_i)))
plotter.set_plot_kwargs(plot_args)
plotter.plot()
else:
#
# Perform weight update without plotting
#
with Timer(verbose=True, name="Weight Update"):
train_summ, regpen_summ, _, cur_loss = sess.run(
[train_summary, regpen_summary, update, loss],
feed_dict={
model.X: mb['X'],
model.y_: mb['y']
})
# Add current summary values to tensorboard
summary_writer_train.add_summary(train_summ,
global_step=global_step)
summary_writer_train.add_summary(regpen_summ,
global_step=global_step)
# Add current loss to running average loss
train_loss += cur_loss
# Print some status info
print("ep {} mb {} loss {} (avg. loss {})".format(
ep, mb_i, cur_loss, train_loss / (mb_i + 1)))
# Reset timer
t_mb = Timer(name="Load Minibatch")
# Free the memory allocated for the minibatch data
mb.clear()
del mb
global_step += 1
#
# Calculate scores on validation set
#
# Perform scoring on validation set
print("Starting scoring on validation set...")
evaluate_on_validation_set(validationset, global_step, sess, model,
summary_writer_validation,
val_loss_summary, val_loss, workspace)
# Save the model
tell.save_checkpoint(global_step=global_step)
# Abort if indicated by file
check_kill_file(workspace)
except AbortRun:
print("Detected kill file, aborting...")
finally:
#
# If the program executed correctly or an error was raised, close the data readers and save the model and exit
#
trainingset.close()
validationset.close()
tell.close(save_checkpoint=True, global_step=global_step)
plotter.close()
def main(_):
np.random.seed(0)
rng = np.random.RandomState(seed=0)
config = Config()
#
# Load Data
#
with Timer(name="Load data"):
training_data = BouncingMNISTDataHandler(
config, config.mnist_train_images, config.mnist_train_labels, rng)
test_data = BouncingMNISTDataHandler(
config, config.mnist_test_images, config.mnist_test_labels, rng)
dataset = DataSet((config.batch_size, config.num_frames, config.image_size, config.image_size, 1),
(config.batch_size, config.num_frames, config.image_size, config.image_size))
# Create new TeLL session with two summary writers
tell = TeLLSession(config=config, summaries=["train", "validation"], model_params={"dataset": dataset})
# Get some members from the session for easier usage
session = tell.tf_session
summary_writer_train, summary_writer_validation = tell.tf_summaries["train"], tell.tf_summaries["validation"]
model = tell.model
workspace, config = tell.workspace, tell.config
# Parameters
learning_rate = config.get_value("learning_rate", 1e-3)
iterations = config.get_value("iterations", 1000)
batch_size = config.get_value("batch_size", 256)
display_step = config.get_value("display_step", 10)
calc_statistics = config.get_value("calc_statistics", False)
blur_filter_size = config.get_value("blur_filter_size", None)
training_summary_tensors = OrderedDict()
# Define loss and optimizer
#with tf.name_scope("Cost"):
# sem_seg_loss, _ = image_crossentropy(pred=model.output, target=model.y_,
# calc_statistics=calc_statistics, reduce_by="sum")
# optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(sem_seg_loss)
# tf.summary.scalar("Loss", sem_seg_loss)
# Evaluate model
validation_summary_tensors = OrderedDict()
# validationset always uses class weights for loss calculation
with tf.name_scope('Cost'):
blur_sampling_range = tf.placeholder(tf.float32)
if blur_filter_size is not None:
sem_seg_loss = blurred_cross_entropy(output=model.output, target=model.y_,
filter_size=blur_filter_size,
sampling_range=blur_sampling_range)
else:
sem_seg_loss, _ = image_crossentropy(pred=model.output, target=model.y_,
reduce_by="mean", calc_statistics=calc_statistics)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(sem_seg_loss)
iou, iou_op = tf.contrib.metrics.streaming_mean_iou(
predictions=tf.squeeze(tf.arg_max(model.output, 4)),
labels=tf.squeeze(model.y_),
num_classes=model.output.get_shape()[-1])
loss_prot = tf.summary.scalar("Loss", sem_seg_loss)
iou_prot = tf.summary.scalar("IoU", iou)
train_summaries = tf.summary.merge([loss_prot])
valid_summaries = tf.summary.merge([loss_prot, iou_prot])
# Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint)
step = tell.initialize_tf_variables().global_step
# -------------------------------------------------------------------------
# Start training
# -------------------------------------------------------------------------
plot_elements_sym = list(model.get_plot_dict().values())
plot_elements = list()
plot_ranges = model.get_plot_range_dict()
try:
while step < iterations:
check_kill_file(workspace=workspace)
batch_x, batch_y = training_data.GetBatch()
i = step * batch_size
if step % display_step == 0:
mean_loss = 0.
for j in range(10):
test_x, test_y = test_data.GetBatch()
summary, loss, _, *plot_elements = session.run([valid_summaries, sem_seg_loss, iou_op, *plot_elements_sym],
feed_dict={model.X: test_x,
model.y_feed: test_y,
blur_sampling_range: 3.5})
summary_writer_validation.add_summary(summary, i)
mean_loss += loss
# Re-associate returned tensorflow values to plotting keys
plot_dict = OrderedDict(zip(list(model.get_plot_dict().keys()), plot_elements))
# Plot outputs and cell states over frames if specified
if config.store_states and 'ConvLSTMLayer_h' in plot_dict and step % config.plot_at == 0:
convh = plot_dict['ConvLSTMLayer_h']
convrh = [c[0, :, :, 0] for c in convh]
convrh = [convrh[:6], convrh[6:12], convrh[12:18], convrh[18:24], convrh[24:]]
plot_args = dict(images=convrh,
filename=os.path.join(workspace.get_result_dir(),
"step{}_h.png".format(step)))
plotter.set_plot_kwargs(plot_args)
plotter.plot()
if config.store_states and 'ConvLSTMLayer_c' in plot_dict and step % config.plot_at == 0:
convc = plot_dict['ConvLSTMLayer_c']
convrc = [c[0, :, :, 0] for c in convc]
convrc = [convrc[:6], convrc[6:12], convrc[12:18], convrc[18:24], convrc[24:]]
plot_args = dict(images=convrc,
filename=os.path.join(workspace.get_result_dir(),
"step{}_c.png".format(step)))
plotter.set_plot_kwargs(plot_args)
plotter.plot()
print('Validation Loss at step {}: {}'.format(i, mean_loss / 10))
summary, loss, _ = session.run([train_summaries, sem_seg_loss, optimizer],
feed_dict={model.X: batch_x,
model.y_feed: batch_y,
blur_sampling_range: 3.5})
summary_writer_train.add_summary(summary, i)
step += 1
print("Training Finished!")
# Final Eval
mean_loss = 0.
for j in range(100):
test_x, test_y = test_data.GetBatch()
summary, loss, _ = session.run([valid_summaries, sem_seg_loss, iou_op],
feed_dict={model.X: test_x,
model.y_feed: test_y,
blur_sampling_range: 3.5})
mean_loss += loss
test_x, test_y = test_data.GetBatch()
pred = session.run(tf.argmax(model.output, 4), feed_dict={model.X: test_x})
pred = to_color(pred)
true = to_color(test_y)
out = to_image(pred, true)
for i in range(pred.shape[0]):
imsave(tell.workspace.get_result_dir() + '/sample_{:02d}.png'.format(i), out[i,])
print("Validation Loss {}".format(mean_loss / 100))
except AbortRun:
print("Aborting...")
finally:
tell.close(global_step=step)
plotter.close()
def main(_):
# ------------------------------------------------------------------------------------------------------------------
# Setup training
# ------------------------------------------------------------------------------------------------------------------
# Initialize config, parses command line and reads specified file; also supports overriding of values from cmd
config = Config()
#
# Prepare input data
#
# Make sure datareader is reproducible
random_seed = config.get_value('random_seed', 12345)
np.random.seed(random_seed) # not threadsafe, use rnd_gen object where possible
rnd_gen = np.random.RandomState(seed=random_seed)
# Set datareaders
n_timesteps = config.get_value('mnist_n_timesteps', 20)
# Load datasets for trainingset
with Timer(name="Loading Data"):
readers = initialize_datareaders(config, required=("train", "val"))
# Set Preprocessing
trainingset = Normalize(readers["train"], apply_to=['X', 'y'])
validationset = Normalize(readers["val"], apply_to=['X', 'y'])
# Set minibatch loaders
trainingset = DataLoader(trainingset, batchsize=2, batchsize_method='zeropad', verbose=False)
validationset = DataLoader(validationset, batchsize=2, batchsize_method='zeropad', verbose=False)
#
# Initialize TeLL session
#
tell = TeLLSession(config=config, summaries=["train", "validation"], model_params={"dataset": trainingset})
# Get some members from the session for easier usage
sess = tell.tf_session
summary_writer_train, summary_writer_validation = tell.tf_summaries["train"], tell.tf_summaries["validation"]
model = tell.model
workspace, config = tell.workspace, tell.config
#
# Define loss functions and update steps
#
print("Initializing loss calculation...")
loss, _ = image_crossentropy(target=model.y_[:, 10:, :, :], pred=model.output[:, 10:, :, :, :],
pixel_weights=model.pixel_weights[:, 10:, :, :], reduce_by='mean')
train_summary = tf.summary.scalar("Training Loss", loss) # create summary to add to tensorboard
# Loss function for validationset
val_loss = loss
val_loss_summary = tf.summary.scalar("Validation Loss", val_loss) # create summary to add to tensorboard
# Regularization
reg_penalty = regularize(layers=model.get_layers(), l1=config.l1, l2=config.l2,
regularize_weights=True, regularize_biases=True)
regpen_summary = tf.summary.scalar("Regularization Penalty", reg_penalty) # create summary to add to tensorboard
# Update step for weights
update = update_step(loss + reg_penalty, config)
#
# Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint)
#
global_step = tell.initialize_tf_variables().global_step
#
# Set up plotting
# (store tensors we want to plot in a dictionary for easier tensor-evaluation)
#
# We want to plot input, output and target for the 1st sample, 1st frame, and 1st channel in subplot 1
tensors_subplot1 = OrderedDict()
tensors_subplot2 = OrderedDict()
tensors_subplot3 = OrderedDict()
for frame in range(n_timesteps):
tensors_subplot1['input_{}'.format(frame)] = model.X[0, frame, :, :]
tensors_subplot2['target_{}'.format(frame)] = model.y_[0, frame, :, :] - 1
tensors_subplot3['network_output_{}'.format(frame)] = tf.argmax(model.output[0, frame, :, :, :], axis=-1) - 1
# We also want to plot the cell states and hidden states for each frame (again of the 1st sample and 1st lstm unit)
# in subplot 2 and 3
tensors_subplot4 = OrderedDict()
tensors_subplot5 = OrderedDict()
for frame in range(len(model.lstm_layer.c)):
tensors_subplot4['hiddenstate_{}'.format(frame)] = model.lstm_layer.h[frame][0, :, :, 0]
tensors_subplot5['cellstate_{}'.format(frame)] = model.lstm_layer.c[frame][0, :, :, 0]
# Create a list to store all symbolic tensors for plotting
plotting_tensors = list(tensors_subplot1.values()) + list(tensors_subplot2.values()) + \
list(tensors_subplot3.values()) + list(tensors_subplot4.values()) + \
list(tensors_subplot5.values())
#
# Finalize graph
# This makes our tensorflow graph read-only and prevents further additions to the graph
#
sess.graph.finalize()
if sess.graph.finalized:
print("Graph is finalized!")
else:
raise ValueError("Could not finalize graph!")
sys.stdout.flush()
# ------------------------------------------------------------------------------------------------------------------
# Start training
# ------------------------------------------------------------------------------------------------------------------
try:
epoch = int(global_step / trainingset.n_mbs)
epochs = range(epoch, config.n_epochs)
# Loop through epochs
print("Starting training")
for ep in epochs:
epoch = ep
print("Starting training epoch: {}".format(ep))
# Initialize variables for over-all loss per epoch
train_loss = 0
# Load one minibatch at a time and perform a training step
t_mb = Timer(verbose=True, name="Load Minibatch")
mb_training = trainingset.batch_loader(rnd_gen=rnd_gen)
#
# Loop through minibatches
#
for mb_i, mb in enumerate(mb_training):
sys.stdout.flush()
# Print minibatch load time
t_mb.print()
# Abort if indicated by file
check_kill_file(workspace)
#
# Calculate scores on validation set
#
if global_step % config.score_at == 0:
print("Starting scoring on validation set...")
evaluate_on_validation_set(validationset, global_step, sess, model, summary_writer_validation,
val_loss_summary, val_loss, workspace)
#
# Perform weight updates and do plotting
#
if (mb_i % config.plot_at) == 0 and os.path.isfile(workspace.get_plot_file()):
# Perform weight update, return summary values and values for plotting
with Timer(verbose=True, name="Weight Update"):
plotting_values = []
train_summ, regpen_summ, _, cur_loss, *plotting_values = sess.run(
[train_summary, regpen_summary, update, loss, *plotting_tensors],
feed_dict={model.X: mb['X'], model.y_: mb['y']})
# Add current summary values to tensorboard
summary_writer_train.add_summary(train_summ, global_step=global_step)
summary_writer_train.add_summary(regpen_summ, global_step=global_step)
# Create and save subplot 1 (input)
save_subplots(images=plotting_values[:len(tensors_subplot1)],
subfigtitles=list(tensors_subplot1.keys()),
subplotranges=[(0, 1)] * n_timesteps, colorbar=True, automatic_positioning=True,
tight_layout=True,
filename=os.path.join(workspace.get_result_dir(),
"input_ep{}_mb{}.png".format(ep, mb_i)))
del plotting_values[:len(tensors_subplot1)]
# Create and save subplot 2 (target)
save_subplots(images=plotting_values[:len(tensors_subplot2)],
subfigtitles=list(tensors_subplot2.keys()),
subplotranges=[(0, 10) * n_timesteps], colorbar=True, automatic_positioning=True,
tight_layout=True,
filename=os.path.join(workspace.get_result_dir(),
"target_ep{}_mb{}.png".format(ep, mb_i)))
del plotting_values[:len(tensors_subplot2)]
# Create and save subplot 3 (output)
save_subplots(images=plotting_values[:len(tensors_subplot3)],
subfigtitles=list(tensors_subplot3.keys()),
# subplotranges=[(0, 10)] * n_timesteps,
colorbar=True, automatic_positioning=True,
tight_layout=True,
filename=os.path.join(workspace.get_result_dir(),
"output_ep{}_mb{}.png".format(ep, mb_i)))
del plotting_values[:len(tensors_subplot3)]
# Create and save subplot 2 (hidden states, i.e. ConvLSTM outputs)
save_subplots(images=plotting_values[:len(tensors_subplot4)],
subfigtitles=list(tensors_subplot4.keys()),
title='ConvLSTM hidden states (outputs)', colorbar=True, automatic_positioning=True,
tight_layout=True,
filename=os.path.join(workspace.get_result_dir(),
"hidden_ep{}_mb{}.png".format(ep, mb_i)))
del plotting_values[:len(tensors_subplot4)]
# Create and save subplot 3 (cell states)
save_subplots(images=plotting_values[:len(tensors_subplot5)],
subfigtitles=list(tensors_subplot5.keys()),
title='ConvLSTM cell states', colorbar=True, automatic_positioning=True,
tight_layout=True,
filename=os.path.join(workspace.get_result_dir(),
"cell_ep{}_mb{}.png".format(ep, mb_i)))
del plotting_values[:len(tensors_subplot5)]
else:
#
# Perform weight update without plotting
#
with Timer(verbose=True, name="Weight Update"):
train_summ, regpen_summ, _, cur_loss = sess.run([
train_summary, regpen_summary, update, loss],
feed_dict={model.X: mb['X'], model.y_: mb['y']})
# Add current summary values to tensorboard
summary_writer_train.add_summary(train_summ, global_step=global_step)
summary_writer_train.add_summary(regpen_summ, global_step=global_step)
# Add current loss to running average loss
train_loss += cur_loss
# Print some status info
print("ep {} mb {} loss {} (avg. loss {})".format(ep, mb_i, cur_loss, train_loss / (mb_i + 1)))
# Reset timer
t_mb = Timer(name="Load Minibatch")
# Free the memory allocated for the minibatch data
mb.clear()
del mb
global_step += 1
#
# Calculate scores on validation set
#
# Perform scoring on validation set
print("Starting scoring on validation set...")
evaluate_on_validation_set(validationset, global_step, sess, model, summary_writer_validation,
val_loss_summary, val_loss, workspace)
# Save the model
tell.save_checkpoint(global_step=global_step)
# Abort if indicated by file
check_kill_file(workspace)
except AbortRun:
print("Detected kill file, aborting...")
finally:
#
# If the program executed correctly or an error was raised, close the data readers and save the model and exit
#
trainingset.close()
validationset.close()
tell.close(save_checkpoint=True, global_step=global_step)
def main(_):
config = Config()
# Create new TeLL session with two summary writers
tell = TeLLSession(config=config, summaries=["train", "validation"])
# Get some members from the session for easier usage
session = tell.tf_session
summary_writer_train, summary_writer_validation = tell.tf_summaries["train"], tell.tf_summaries["validation"]
model = tell.model
workspace, config = tell.workspace, tell.config
# Parameters
learning_rate = config.get_value("learning_rate", 1e-3)
iterations = config.get_value("iterations", 1000)
batchsize = config.get_value("batchsize", 250)
display_step = config.get_value("display_step", 10)
dropout = config.get_value("dropout_prob", 0.25)
#
# Load Data
#
with Timer(name="Load data"):
mnist = input_data.read_data_sets("../MNIST_data", one_hot=True)
# Define loss and optimizer
with tf.name_scope("Cost"):
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=model.output, labels=model.y_))
##entropy = tf.reduce_mean(tf.contrib.bayesflow.entropy.entropy_shannon(
## tf.contrib.distributions.Categorical(p=tf.nn.softmax(logits=model.output))))
probs = tf.nn.softmax(logits=model.output)
entropy = tf.reduce_mean(-tf.reduce_sum(tf.log(tf.maximum(probs, 1e-15)) * probs, 1))
# test decor regularization
#decor_penalty(model.hidden1, model.y_, 10, [1], 0.)
#decor_penalty(model.hidden2, model.y_, 10, [1], 0.)
optimizer = tell.tf_optimizer.minimize(cost - config.get_value("entropy_w", 0.) * entropy)
tf.summary.scalar("Loss", cost)
#tf.summary.scalar("Decor", decor1 + decor2)
#tf.summary.scalar("Entropy", entropy)
tf.summary.scalar("O-Prob", tf.reduce_mean(tf.reduce_sum(tf.nn.softmax(logits=model.output) * model.y_, 1)))
# Evaluate model
with tf.name_scope("Accuracy"):
correct_pred = tf.equal(tf.argmax(model.output, 1), tf.argmax(model.y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
tf.summary.scalar("Accuracy", accuracy)
merged_summaries = tf.summary.merge_all()
# Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint)
step = tell.initialize_tf_variables(reset_optimizer_on_restore=True).global_step
# -------------------------------------------------------------------------
# Start training
# -------------------------------------------------------------------------
acc_train = 0.
val_acc_best = 0.
try:
while step < iterations:
check_kill_file(workspace=workspace)
batch_x, batch_y = mnist.train.next_batch(batchsize)
i = step * batchsize
if step % display_step == 0:
summary, acc = session.run([merged_summaries, accuracy],
feed_dict={model.X: mnist.validation.images[:2048],
model.y_: mnist.validation.labels[:2048],
model.dropout: 0})
summary_writer_validation.add_summary(summary, i)
print('step {}: train acc {}, valid acc {}'.format(i, acc_train, acc))
if acc > val_acc_best:
val_acc_best = acc
else:
summary, acc_train, _ = session.run([merged_summaries, accuracy, optimizer],
feed_dict={model.X: batch_x, model.y_: batch_y,
model.dropout: dropout})
summary_writer_train.add_summary(summary, i)
step += 1
print("Training Finished! best valid acc {}".format(val_acc_best))
# Final Eval
print("Test Accuracy:",
session.run(accuracy, feed_dict={model.X: mnist.test.images[:2048],
model.y_: mnist.test.labels[:2048],
model.dropout: 0}))
except AbortRun:
print("Aborting...")
finally:
tell.close(global_step=step)
def main(_):
# ------------------------------------------------------------------------------------------------------------------
# Setup training
# ------------------------------------------------------------------------------------------------------------------
# Initialize config, parses command line and reads specified file; also supports overriding of values from cmd
config = Config()
random_seed = config.get_value('random_seed', 12345)
np.random.seed(random_seed) # not threadsafe, use rnd_gen object where possible
rnd_gen = np.random.RandomState(seed=random_seed)
# Load datasets for trainingset
with Timer(name="Loading Data"):
readers = initialize_datareaders(config, required=("train", "val"))
trainingset = DataLoader(readers["train"], batchsize=config.batchsize)
#validationset = DataLoader(readers["val"], batchsize=config.batchsize)
# Initialize TeLL session
tell = TeLLSession(config=config, model_params={"input_shape": [300]})
# Get some members from the session for easier usage
session = tell.tf_session
model = tell.model
workspace, config = tell.workspace, tell.config
# Initialize Tensorflow variables
global_step = tell.initialize_tf_variables().global_step
sys.stdout.flush()
# ------------------------------------------------------------------------------------------------------------------
# Start training
# ------------------------------------------------------------------------------------------------------------------
try:
epoch = int(global_step / trainingset.n_mbs)
epochs = range(epoch, config.n_epochs)
#
# Loop through epochs
#
print("Starting training")
for ep in epochs:
print("Starting training epoch: {}".format(ep))
# Initialize variables for over-all loss per epoch
train_loss = 0
# Load one minibatch at a time and perform a training step
t_mb = Timer(name="Load Minibatch")
mb_training = trainingset.batch_loader(rnd_gen=rnd_gen)
#
# Loop through minibatches
#
for mb_i, mb in enumerate(mb_training):
sys.stdout.flush()
#Print minibatch load time
t_mb.print()
# Abort if indicated by file
check_kill_file(workspace)
#
# Calculate scores on validation set
#
if global_step % config.score_at == 0:
print("Starting scoring on validation set...")
# Get new sample
training_sample = np.ones(shape=(1,np.random.randint(low=20,high=100),300))
#
# Perform weight update
#
with Timer(name="Weight Update"):
#
# Set placeholder values
#
placeholder_values = OrderedDict(
input_placeholder=training_sample,
sequence_length_placeholder = training_sample.shape[1]
)
feed_dict = dict(((model.placeholders[k], placeholder_values[k]) for k in placeholder_values.keys()))
#
# Decide which tensors to compute
#
data_keys = ['lstm_internals_enc', 'lstm_internals_dec', 'lstm_h_enc',
'lstm_h_dec', 'loss' , 'loss_last_time_prediction','loss_last_time_prediction', 'reg_loss']
data_tensors = [model.data_tensors[k] for k in data_keys]
operation_keys = ['ae_update']
operation_tensors = [model.operation_tensors[k] for k in operation_keys]
summary_keys = ['all_summaries']
summary_tensors = [model.summaries[k] for k in summary_keys]
#
# Run graph and re-associate return values with keys in dictionary
#
ret = session.run(data_tensors + summary_tensors + operation_tensors, feed_dict)
data_keys = ['loss']
data_tensors = [model.data_tensors[k] for k in data_keys]
session.run(model.data_tensors['loss'] , feed_dict)
session.run(model.data_tensors['latent_space'], feed_dict)
ret_dict = OrderedDict(((k, ret[i]) for i, k in enumerate(data_keys)))
del ret[:len(data_keys)]
ret_dict.update(OrderedDict(((k, ret[i]) for i, k in enumerate(summary_keys))))
# Print some status info
#print("ep {} mb {} loss {} (avg. loss {})".format(ep, mb_i, cur_loss, train_loss / (mb_i + 1)))
# Reset timer
#t_mb = Timer(name="Load Minibatch")
# Free the memory allocated for the minibatch data
#mb.clear()
#del mb
global_step += 1
#
# Calculate scores on validation set after training is done
#
# Perform scoring on validation set
print("Starting scoring on validation set...")
tell.save_checkpoint(global_step=global_step)
# Abort if indicated by file
check_kill_file(workspace)
except AbortRun:
print("Detected kill file, aborting...")
finally:
tell.close(save_checkpoint=True, global_step=global_step)
def main(_):
# ------------------------------------------------------------------------------------------------------------------
# Setup training
# ------------------------------------------------------------------------------------------------------------------
# Initialize config, parses command line and reads specified file; also supports overriding of values from cmd
config = Config()
# Load datasets for trainingset
with Timer(name="Loading Training Data"):
# Make sure datareader is reproducible
random_seed = config.get_value('random_seed', 12345)
np.random.seed(
random_seed) # not threadsafe, use rnd_gen object where possible
rnd_gen = np.random.RandomState(seed=random_seed)
print("Loading training data...")
trainingset = ShortLongDataset(n_timesteps=250,
n_samples=3000,
batchsize=config.batchsize,
rnd_gen=rnd_gen)
# Load datasets for validationset
validationset = ShortLongDataset(n_timesteps=250,
n_samples=300,
batchsize=config.batchsize,
rnd_gen=rnd_gen)
# Initialize TeLL session
tell = TeLLSession(config=config,
summaries=["train"],
model_params={"dataset": trainingset})
# Get some members from the session for easier usage
session = tell.tf_session
summary_writer = tell.tf_summaries["train"]
model = tell.model
workspace, config = tell.workspace, tell.config
# Loss function for trainingset
print("Initializing loss calculation...")
loss = tf.reduce_mean(
tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(
model.y_, model.output,
-tf.reduce_sum(model.y_ - 1) / tf.reduce_sum(model.y_)),
axis=[1]))
train_summary = tf.summary.scalar("Training Loss",
loss) # add loss to tensorboard
# Loss function for validationset
val_loss = tf.reduce_mean(
tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(
model.y_, model.output,
-tf.reduce_sum(model.y_ - 1) / tf.reduce_sum(model.y_)),
axis=[1]))
val_loss_summary = tf.summary.scalar(
"Validation Loss", val_loss) # add val_loss to tensorboard
# Regularization
reg_penalty = regularize(layers=model.get_layers(),
l1=config.l1,
l2=config.l2,
regularize_weights=True,
regularize_biases=True)
regpen_summary = tf.summary.scalar(
"Regularization Penalty",
reg_penalty) # add reg_penalty to tensorboard
# Update step for weights
update = update_step(loss + reg_penalty, config)
# Initialize Tensorflow variables
global_step = tell.initialize_tf_variables().global_step
sys.stdout.flush()
# ------------------------------------------------------------------------------------------------------------------
# Start training
# ------------------------------------------------------------------------------------------------------------------
try:
epoch = int(global_step / trainingset.n_mbs)
epochs = range(epoch, config.n_epochs)
#
# Loop through epochs
#
print("Starting training")
for ep in epochs:
print("Starting training epoch: {}".format(ep))
# Initialize variables for over-all loss per epoch
train_loss = 0
# Load one minibatch at a time and perform a training step
t_mb = Timer(name="Load Minibatch")
mb_training = trainingset.batch_loader(rnd_gen=rnd_gen)
#
# Loop through minibatches
#
for mb_i, mb in enumerate(mb_training):
sys.stdout.flush()
# Print minibatch load time
t_mb.print()
# Abort if indicated by file
check_kill_file(workspace)
#
# Calculate scores on validation set
#
if global_step % config.score_at == 0:
print("Starting scoring on validation set...")
evaluate_on_validation_set(validationset, global_step,
session, model, summary_writer,
val_loss_summary, val_loss,
workspace)
#
# Perform weight update
#
with Timer(name="Weight Update"):
train_summ, regpen_summ, _, cur_loss = session.run(
[train_summary, regpen_summary, update, loss],
feed_dict={
model.X: mb['X'],
model.y_: mb['y']
})
# Add current summary values to tensorboard
summary_writer.add_summary(train_summ, global_step=global_step)
summary_writer.add_summary(regpen_summ,
global_step=global_step)
# Add current loss to running average loss
train_loss += cur_loss
# Print some status info
print("ep {} mb {} loss {} (avg. loss {})".format(
ep, mb_i, cur_loss, train_loss / (mb_i + 1)))
# Reset timer
t_mb = Timer(name="Load Minibatch")
# Free the memory allocated for the minibatch data
mb.clear()
del mb
global_step += 1
#
# Calculate scores on validation set after training is done
#
# Perform scoring on validation set
print("Starting scoring on validation set...")
evaluate_on_validation_set(validationset, global_step, session,
model, summary_writer, val_loss_summary,
val_loss, workspace)
tell.save_checkpoint(global_step=global_step)
# Abort if indicated by file
check_kill_file(workspace)
except AbortRun:
print("Detected kill file, aborting...")
finally:
tell.close(save_checkpoint=True, global_step=global_step)
def evaluate(self,
step: int,
summary_writer,
prefix='validation ',
num_cached=5,
num_threads=3,
rnd_gen=None,
plotter=None,
model=None):
# Reset streaming measures
self.reset_tensors()
# Get tensors to evaluate for plotting
if plotter is not None:
plot_tensors = plotter.get_tensors()
# Set up progress bar
_pbw = ['Evaluating on {}set:'.format(prefix), progressbar.ETA()]
progress = progressbar.ProgressBar(widgets=_pbw,
maxval=self.dataset.n_mbs -
1).start()
#
# Iterate over dataset minibatches
#
mb_validation = self.dataset.batch_loader(num_cached=num_cached,
num_threads=num_threads,
rnd_gen=rnd_gen)
with Timer(verbose=True, name="Evaluate on {}set".format(prefix)):
summary_values_filled = None
for mb_i, mb in enumerate(mb_validation):
# Abort if indicated by file
check_kill_file(self.workspace)
if mb.get('pixel_weights', None) is None:
feed_dict = {self.model.X: mb['X'], self.model.y_: mb['y']}
else:
feed_dict = {
self.model.X: mb['X'],
self.model.y_: mb['y'],
self.model.pixel_weights: mb['pixel_weights']
}
if plotter is not None:
evaluated_tensors = self.session.run([
*self.summary_ops, *self.summary_tensors, *plot_tensors
],
feed_dict=feed_dict)
else:
evaluated_tensors = self.session.run(
[*self.summary_ops, *self.summary_tensors],
feed_dict=feed_dict)
# Discard return values from summary_ops (=update operations)
evaluated_tensors = evaluated_tensors[len(self.summary_ops):]
summary_values = evaluated_tensors[:len(self.summary_tensors)]
# Perform plotting
if plotter is not None:
plotter.set_tensor_values(
evaluated_tensors[len(self.summary_tensors
):len(self.plot_tensors) +
len(plot_tensors)])
plotter.plot(evaluate_tensors=False)
# Re-associate returned tensorflow values to keys and incorporate new minibatch values
if summary_values_filled is None:
# Fill summary_values_filled for the first time
summary_values_filled = OrderedDict(
zip(list(self.summary_tensor_dict.keys()),
summary_values))
for key_i, key in enumerate(summary_values_filled.keys()):
if not self.summary_tensor_is_op[key_i]:
if isinstance(summary_values_filled[key],
np.ndarray):
summary_values_filled[key] = [
summary_values_filled[key]
]
elif np.isfinite(summary_values_filled[key]):
summary_values_filled[key] = [
summary_values_filled[key]
]
else:
summary_values_filled[key] = []
else:
for key_i, key in enumerate(summary_values_filled.keys()):
if not self.summary_tensor_is_op[key_i]:
if isinstance(summary_values[key_i], np.ndarray):
summary_values_filled[key].append(
summary_values[key_i])
elif np.isfinite(summary_values[key_i]):
summary_values_filled[key].append(
summary_values[key_i])
else:
summary_values_filled[key] = summary_values[key_i]
# Update progress bar and clear minibatch
progress.update(mb_i)
mb.clear()
del mb
progress.finish()
#
# Divide sums by number of samples for tensors that do not have an update function
#
if len(summary_values_filled):
for key_i, key in enumerate(summary_values_filled.keys()):
if not self.summary_tensor_is_op[key_i]:
if len(summary_values_filled[key]):
if not isinstance(summary_values_filled[key][0],
np.ndarray):
summary_values_filled[key] = np.mean(
summary_values_filled[key])
else:
summary_values_filled[key] = np.concatenate(
summary_values_filled[key])
else:
summary_values_filled[key] = np.nan
#
# Go through values to use as summaries, create histograms if values are not scalars
#
values_to_print = OrderedDict()
if len(summary_values_filled):
for key_i, key in enumerate(summary_values_filled.keys()):
if not isinstance(summary_values_filled[key], np.ndarray):
values_to_print.update({key: summary_values_filled[key]})
summary = tf.Summary(value=[
tf.Summary.Value(tag=prefix + key,
simple_value=float(
summary_values_filled[key]))
])
else:
hist = custom_tensorflow_histogram(
summary_values_filled[key], bins=100)
summary = tf.Summary(
value=[tf.Summary.Value(tag=prefix + key, histo=hist)])
summary_writer.add_summary(summary, step)
print("{}scores:\n\tstep {}, {}".format(prefix, step, values_to_print))
summary_writer.flush()
sys.stdout.flush()