def evaluate_on_validation_set(validationset, step: int, session, model, summary_writer, validation_summary,
val_loss, workspace: Workspace):
"""Convenience function for evaluating network on a validation set
Parameters
-------
validationset : dataset reader
Dataset reader for the validation set
step : int
Current step in training
session : tf.session
Tensorflow session to use
model : network model
Network model
val_loss : tensor
Tensor representing the validation loss computation
Returns
-------
: float
Loss averaged over validation set
"""
loss = 0
_pbw = ['Evaluating on validation set: ', progressbar.ETA()]
progress = progressbar.ProgressBar(widgets=_pbw, maxval=validationset.n_mbs - 1, redirect_stdout=True).start()
mb_validation = validationset.batch_loader()
with Timer(verbose=True, name="Evaluate on Validation Set"):
for mb_i, mb in enumerate(mb_validation):
# Abort if indicated by file
check_kill_file(workspace)
val_summary, cur_loss = session.run([validation_summary, val_loss],
feed_dict={model.X: mb['X'], model.y_: mb['y']})
loss += cur_loss
progress.update(mb_i)
mb.clear()
del mb
progress.finish()
avg_loss = loss / validationset.n_mbs
summary_writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag="Validation Loss", simple_value=avg_loss)]),
step)
print("Validation scores:\n\tstep {} validation loss {}".format(step, avg_loss))
sys.stdout.flush()
return avg_loss
def main(_):
# ------------------------------------------------------------------------------------------------------------------
# Setup training
# ------------------------------------------------------------------------------------------------------------------
# Initialize config, parses command line and reads specified file; also supports overriding of values from cmd
config = Config()
#
# Load datasets for training and validation
#
with Timer(name="Loading Data", verbose=True):
# Make sure datareader is reproducible
random_seed = config.get_value('random_seed', 12345)
np.random.seed(
random_seed) # not threadsafe, use rnd_gen object where possible
rnd_gen = np.random.RandomState(seed=random_seed)
print("Loading training data...")
trainingset = MovingDotDataset(n_timesteps=5,
n_samples=50,
batchsize=config.batchsize,
rnd_gen=rnd_gen)
print("Loading validation data...")
validationset = MovingDotDataset(n_timesteps=5,
n_samples=25,
batchsize=config.batchsize,
rnd_gen=rnd_gen)
#
# Initialize TeLL session
#
tell = TeLLSession(config=config,
summaries=["train", "validation"],
model_params={"dataset": trainingset})
# Get some members from the session for easier usage
sess = tell.tf_session
summary_writer_train, summary_writer_validation = tell.tf_summaries[
"train"], tell.tf_summaries["validation"]
model = tell.model
workspace, config = tell.workspace, tell.config
#
# Define loss functions and update steps
#
print("Initializing loss calculation...")
pos_target_weight = np.prod(
trainingset.y_shape[2:]
) - 1 # only 1 pixel per sample is of positive class -> up-weight!
loss = tf.reduce_mean(
tf.nn.weighted_cross_entropy_with_logits(targets=model.y_,
logits=model.output,
pos_weight=pos_target_weight))
# loss = tf.reduce_mean(-tf.reduce_sum((model.y_ * tf.log(model.output)) *
# -tf.reduce_sum(model.y_ - 1) / tf.reduce_sum(model.y_),
# axis=[1, 2, 3, 4]))
train_summary = tf.summary.scalar(
"Training Loss", loss) # create summary to add to tensorboard
# Loss function for validationset
val_loss = tf.reduce_mean(
tf.nn.weighted_cross_entropy_with_logits(targets=model.y_,
logits=model.output,
pos_weight=pos_target_weight))
# val_loss = tf.reduce_mean(-tf.reduce_sum(model.y_ * tf.log(model.output) *
# -tf.reduce_sum(model.y_ - 1) / tf.reduce_sum(model.y_),
# axis=[1, 2, 3, 4]))
val_loss_summary = tf.summary.scalar(
"Validation Loss", val_loss) # create summary to add to tensorboard
# Regularization
reg_penalty = regularize(layers=model.get_layers(),
l1=config.l1,
l2=config.l2,
regularize_weights=True,
regularize_biases=True)
regpen_summary = tf.summary.scalar(
"Regularization Penalty",
reg_penalty) # create summary to add to tensorboard
# Update step for weights
update = update_step(loss + reg_penalty, config)
#
# Prepare plotting
#
plot_elements_sym = list(model.get_plot_dict().values())
plot_elements = list()
plot_ranges = model.get_plot_range_dict()
#
# Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint)
#
global_step = tell.initialize_tf_variables().global_step
#
# Finalize graph
# This makes our tensorflow graph read-only and prevents further additions to the graph
#
sess.graph.finalize()
if sess.graph.finalized:
print("Graph is finalized!")
else:
raise ValueError("Could not finalize graph!")
sys.stdout.flush()
# ------------------------------------------------------------------------------------------------------------------
# Start training
# ------------------------------------------------------------------------------------------------------------------
try:
epoch = int(global_step / trainingset.n_mbs)
epochs = range(epoch, config.n_epochs)
# Loop through epochs
print("Starting training")
for ep in epochs:
epoch = ep
print("Starting training epoch: {}".format(ep))
# Initialize variables for over-all loss per epoch
train_loss = 0
# Load one minibatch at a time and perform a training step
t_mb = Timer(verbose=True, name="Load Minibatch")
mb_training = trainingset.batch_loader(rnd_gen=rnd_gen)
#
# Loop through minibatches
#
for mb_i, mb in enumerate(mb_training):
sys.stdout.flush()
# Print minibatch load time
t_mb.print()
# Abort if indicated by file
check_kill_file(workspace)
#
# Calculate scores on validation set
#
if global_step % config.score_at == 0:
print("Starting scoring on validation set...")
evaluate_on_validation_set(validationset, global_step,
sess, model,
summary_writer_validation,
val_loss_summary, val_loss,
workspace)
#
# Perform weight updates and do plotting
#
if (mb_i % config.plot_at) == 0 and os.path.isfile(
workspace.get_plot_file()):
# Perform weight update, return summary_str and values for plotting
with Timer(verbose=True, name="Weight Update"):
train_summ, regpen_summ, _, cur_loss, cur_output, *plot_elements = sess.run(
[
train_summary, regpen_summary, update, loss,
model.output, *plot_elements_sym
],
feed_dict={
model.X: mb['X'],
model.y_: mb['y']
})
# Add current summary values to tensorboard
summary_writer_train.add_summary(train_summ,
global_step=global_step)
summary_writer_train.add_summary(regpen_summ,
global_step=global_step)
# Re-associate returned tensorflow values to plotting keys
plot_dict = OrderedDict(
zip(list(model.get_plot_dict().keys()), plot_elements))
#
# Plot subplots in plot_dict
# Loop through each element in plotlist and pass it to the save_subplots function for plotting
# (adapt this to your needs for plotting)
#
with Timer(verbose=True, name="Plotting",
precision="msec"):
for plotlist_i, plotlist in enumerate(
model.get_plotsink()):
for frame in range(len(plot_dict[plotlist[0]])):
subplotlist = []
subfigtitles = []
subplotranges = []
n_cols = int(np.ceil(np.sqrt(len(plotlist))))
for col_i, col_i in enumerate(range(n_cols)):
subfigtitles.append(
plotlist[n_cols *
col_i:n_cols * col_i +
n_cols])
subplotlist.append([
plot_dict[p]
[frame *
(frame < len(plot_dict[p])), :]
for p in plotlist[n_cols *
col_i:n_cols *
col_i + n_cols]
])
subplotranges.append([
plot_ranges.get(p, False)
for p in plotlist[n_cols *
col_i:n_cols *
col_i + n_cols]
])
# remove rows/columns without images
subplotlist = [
p for p in subplotlist if p != []
]
plot_args = dict(
images=subplotlist,
filename=os.path.join(
workspace.get_result_dir(),
"plot{}_ep{}_mb{}_fr{}.png".format(
plotlist_i, ep, mb_i, frame)),
subfigtitles=subfigtitles,
subplotranges=subplotranges)
plotter.set_plot_kwargs(plot_args)
plotter.plot()
# Plot outputs and cell states over frames if specified
if config.store_states and 'ConvLSTMLayer_h' in plot_dict:
convh = plot_dict['ConvLSTMLayer_h']
convrh = [c[0, :, :, 0] for c in convh]
convrh = [
convrh[:6], convrh[6:12], convrh[12:18],
convrh[18:24], convrh[24:]
]
plot_args = dict(images=convrh,
filename=os.path.join(
workspace.get_result_dir(),
"plot{}_ep{}_mb{}_h.png".format(
plotlist_i, ep, mb_i)))
plotter.set_plot_kwargs(plot_args)
plotter.plot()
if config.store_states and 'ConvLSTMLayer_c' in plot_dict:
convc = plot_dict['ConvLSTMLayer_c']
convrc = [c[0, :, :, 0] for c in convc]
convrc = [
convrc[:6], convrc[6:12], convrc[12:18],
convrc[18:24], convrc[24:]
]
plot_args = dict(images=convrc,
filename=os.path.join(
workspace.get_result_dir(),
"plot{}_ep{}_mb{}_c.png".format(
plotlist_i, ep, mb_i)))
plotter.set_plot_kwargs(plot_args)
plotter.plot()
else:
#
# Perform weight update without plotting
#
with Timer(verbose=True, name="Weight Update"):
train_summ, regpen_summ, _, cur_loss = sess.run(
[train_summary, regpen_summary, update, loss],
feed_dict={
model.X: mb['X'],
model.y_: mb['y']
})
# Add current summary values to tensorboard
summary_writer_train.add_summary(train_summ,
global_step=global_step)
summary_writer_train.add_summary(regpen_summ,
global_step=global_step)
# Add current loss to running average loss
train_loss += cur_loss
# Print some status info
print("ep {} mb {} loss {} (avg. loss {})".format(
ep, mb_i, cur_loss, train_loss / (mb_i + 1)))
# Reset timer
t_mb = Timer(name="Load Minibatch")
# Free the memory allocated for the minibatch data
mb.clear()
del mb
global_step += 1
#
# Calculate scores on validation set
#
# Perform scoring on validation set
print("Starting scoring on validation set...")
evaluate_on_validation_set(validationset, global_step, sess, model,
summary_writer_validation,
val_loss_summary, val_loss, workspace)
# Save the model
tell.save_checkpoint(global_step=global_step)
# Abort if indicated by file
check_kill_file(workspace)
except AbortRun:
print("Detected kill file, aborting...")
finally:
#
# If the program executed correctly or an error was raised, close the data readers and save the model and exit
#
trainingset.close()
validationset.close()
tell.close(save_checkpoint=True, global_step=global_step)
plotter.close()
def main(_):
np.random.seed(0)
rng = np.random.RandomState(seed=0)
config = Config()
#
# Load Data
#
with Timer(name="Load data"):
training_data = BouncingMNISTDataHandler(
config, config.mnist_train_images, config.mnist_train_labels, rng)
test_data = BouncingMNISTDataHandler(
config, config.mnist_test_images, config.mnist_test_labels, rng)
dataset = DataSet((config.batch_size, config.num_frames, config.image_size, config.image_size, 1),
(config.batch_size, config.num_frames, config.image_size, config.image_size))
# Create new TeLL session with two summary writers
tell = TeLLSession(config=config, summaries=["train", "validation"], model_params={"dataset": dataset})
# Get some members from the session for easier usage
session = tell.tf_session
summary_writer_train, summary_writer_validation = tell.tf_summaries["train"], tell.tf_summaries["validation"]
model = tell.model
workspace, config = tell.workspace, tell.config
# Parameters
learning_rate = config.get_value("learning_rate", 1e-3)
iterations = config.get_value("iterations", 1000)
batch_size = config.get_value("batch_size", 256)
display_step = config.get_value("display_step", 10)
calc_statistics = config.get_value("calc_statistics", False)
blur_filter_size = config.get_value("blur_filter_size", None)
training_summary_tensors = OrderedDict()
# Define loss and optimizer
#with tf.name_scope("Cost"):
# sem_seg_loss, _ = image_crossentropy(pred=model.output, target=model.y_,
# calc_statistics=calc_statistics, reduce_by="sum")
# optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(sem_seg_loss)
# tf.summary.scalar("Loss", sem_seg_loss)
# Evaluate model
validation_summary_tensors = OrderedDict()
# validationset always uses class weights for loss calculation
with tf.name_scope('Cost'):
blur_sampling_range = tf.placeholder(tf.float32)
if blur_filter_size is not None:
sem_seg_loss = blurred_cross_entropy(output=model.output, target=model.y_,
filter_size=blur_filter_size,
sampling_range=blur_sampling_range)
else:
sem_seg_loss, _ = image_crossentropy(pred=model.output, target=model.y_,
reduce_by="mean", calc_statistics=calc_statistics)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(sem_seg_loss)
iou, iou_op = tf.contrib.metrics.streaming_mean_iou(
predictions=tf.squeeze(tf.arg_max(model.output, 4)),
labels=tf.squeeze(model.y_),
num_classes=model.output.get_shape()[-1])
loss_prot = tf.summary.scalar("Loss", sem_seg_loss)
iou_prot = tf.summary.scalar("IoU", iou)
train_summaries = tf.summary.merge([loss_prot])
valid_summaries = tf.summary.merge([loss_prot, iou_prot])
# Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint)
step = tell.initialize_tf_variables().global_step
# -------------------------------------------------------------------------
# Start training
# -------------------------------------------------------------------------
plot_elements_sym = list(model.get_plot_dict().values())
plot_elements = list()
plot_ranges = model.get_plot_range_dict()
try:
while step < iterations:
check_kill_file(workspace=workspace)
batch_x, batch_y = training_data.GetBatch()
i = step * batch_size
if step % display_step == 0:
mean_loss = 0.
for j in range(10):
test_x, test_y = test_data.GetBatch()
summary, loss, _, *plot_elements = session.run([valid_summaries, sem_seg_loss, iou_op, *plot_elements_sym],
feed_dict={model.X: test_x,
model.y_feed: test_y,
blur_sampling_range: 3.5})
summary_writer_validation.add_summary(summary, i)
mean_loss += loss
# Re-associate returned tensorflow values to plotting keys
plot_dict = OrderedDict(zip(list(model.get_plot_dict().keys()), plot_elements))
# Plot outputs and cell states over frames if specified
if config.store_states and 'ConvLSTMLayer_h' in plot_dict and step % config.plot_at == 0:
convh = plot_dict['ConvLSTMLayer_h']
convrh = [c[0, :, :, 0] for c in convh]
convrh = [convrh[:6], convrh[6:12], convrh[12:18], convrh[18:24], convrh[24:]]
plot_args = dict(images=convrh,
filename=os.path.join(workspace.get_result_dir(),
"step{}_h.png".format(step)))
plotter.set_plot_kwargs(plot_args)
plotter.plot()
if config.store_states and 'ConvLSTMLayer_c' in plot_dict and step % config.plot_at == 0:
convc = plot_dict['ConvLSTMLayer_c']
convrc = [c[0, :, :, 0] for c in convc]
convrc = [convrc[:6], convrc[6:12], convrc[12:18], convrc[18:24], convrc[24:]]
plot_args = dict(images=convrc,
filename=os.path.join(workspace.get_result_dir(),
"step{}_c.png".format(step)))
plotter.set_plot_kwargs(plot_args)
plotter.plot()
print('Validation Loss at step {}: {}'.format(i, mean_loss / 10))
summary, loss, _ = session.run([train_summaries, sem_seg_loss, optimizer],
feed_dict={model.X: batch_x,
model.y_feed: batch_y,
blur_sampling_range: 3.5})
summary_writer_train.add_summary(summary, i)
step += 1
print("Training Finished!")
# Final Eval
mean_loss = 0.
for j in range(100):
test_x, test_y = test_data.GetBatch()
summary, loss, _ = session.run([valid_summaries, sem_seg_loss, iou_op],
feed_dict={model.X: test_x,
model.y_feed: test_y,
blur_sampling_range: 3.5})
mean_loss += loss
test_x, test_y = test_data.GetBatch()
pred = session.run(tf.argmax(model.output, 4), feed_dict={model.X: test_x})
pred = to_color(pred)
true = to_color(test_y)
out = to_image(pred, true)
for i in range(pred.shape[0]):
imsave(tell.workspace.get_result_dir() + '/sample_{:02d}.png'.format(i), out[i,])
print("Validation Loss {}".format(mean_loss / 100))
except AbortRun:
print("Aborting...")
finally:
tell.close(global_step=step)
plotter.close()
def main(_):
# ------------------------------------------------------------------------------------------------------------------
# Setup training
# ------------------------------------------------------------------------------------------------------------------
# Initialize config, parses command line and reads specified file; also supports overriding of values from cmd
config = Config()
#
# Prepare input data
#
# Make sure datareader is reproducible
random_seed = config.get_value('random_seed', 12345)
np.random.seed(random_seed) # not threadsafe, use rnd_gen object where possible
rnd_gen = np.random.RandomState(seed=random_seed)
# Set datareaders
n_timesteps = config.get_value('mnist_n_timesteps', 20)
# Load datasets for trainingset
with Timer(name="Loading Data"):
readers = initialize_datareaders(config, required=("train", "val"))
# Set Preprocessing
trainingset = Normalize(readers["train"], apply_to=['X', 'y'])
validationset = Normalize(readers["val"], apply_to=['X', 'y'])
# Set minibatch loaders
trainingset = DataLoader(trainingset, batchsize=2, batchsize_method='zeropad', verbose=False)
validationset = DataLoader(validationset, batchsize=2, batchsize_method='zeropad', verbose=False)
#
# Initialize TeLL session
#
tell = TeLLSession(config=config, summaries=["train", "validation"], model_params={"dataset": trainingset})
# Get some members from the session for easier usage
sess = tell.tf_session
summary_writer_train, summary_writer_validation = tell.tf_summaries["train"], tell.tf_summaries["validation"]
model = tell.model
workspace, config = tell.workspace, tell.config
#
# Define loss functions and update steps
#
print("Initializing loss calculation...")
loss, _ = image_crossentropy(target=model.y_[:, 10:, :, :], pred=model.output[:, 10:, :, :, :],
pixel_weights=model.pixel_weights[:, 10:, :, :], reduce_by='mean')
train_summary = tf.summary.scalar("Training Loss", loss) # create summary to add to tensorboard
# Loss function for validationset
val_loss = loss
val_loss_summary = tf.summary.scalar("Validation Loss", val_loss) # create summary to add to tensorboard
# Regularization
reg_penalty = regularize(layers=model.get_layers(), l1=config.l1, l2=config.l2,
regularize_weights=True, regularize_biases=True)
regpen_summary = tf.summary.scalar("Regularization Penalty", reg_penalty) # create summary to add to tensorboard
# Update step for weights
update = update_step(loss + reg_penalty, config)
#
# Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint)
#
global_step = tell.initialize_tf_variables().global_step
#
# Set up plotting
# (store tensors we want to plot in a dictionary for easier tensor-evaluation)
#
# We want to plot input, output and target for the 1st sample, 1st frame, and 1st channel in subplot 1
tensors_subplot1 = OrderedDict()
tensors_subplot2 = OrderedDict()
tensors_subplot3 = OrderedDict()
for frame in range(n_timesteps):
tensors_subplot1['input_{}'.format(frame)] = model.X[0, frame, :, :]
tensors_subplot2['target_{}'.format(frame)] = model.y_[0, frame, :, :] - 1
tensors_subplot3['network_output_{}'.format(frame)] = tf.argmax(model.output[0, frame, :, :, :], axis=-1) - 1
# We also want to plot the cell states and hidden states for each frame (again of the 1st sample and 1st lstm unit)
# in subplot 2 and 3
tensors_subplot4 = OrderedDict()
tensors_subplot5 = OrderedDict()
for frame in range(len(model.lstm_layer.c)):
tensors_subplot4['hiddenstate_{}'.format(frame)] = model.lstm_layer.h[frame][0, :, :, 0]
tensors_subplot5['cellstate_{}'.format(frame)] = model.lstm_layer.c[frame][0, :, :, 0]
# Create a list to store all symbolic tensors for plotting
plotting_tensors = list(tensors_subplot1.values()) + list(tensors_subplot2.values()) + \
list(tensors_subplot3.values()) + list(tensors_subplot4.values()) + \
list(tensors_subplot5.values())
#
# Finalize graph
# This makes our tensorflow graph read-only and prevents further additions to the graph
#
sess.graph.finalize()
if sess.graph.finalized:
print("Graph is finalized!")
else:
raise ValueError("Could not finalize graph!")
sys.stdout.flush()
# ------------------------------------------------------------------------------------------------------------------
# Start training
# ------------------------------------------------------------------------------------------------------------------
try:
epoch = int(global_step / trainingset.n_mbs)
epochs = range(epoch, config.n_epochs)
# Loop through epochs
print("Starting training")
for ep in epochs:
epoch = ep
print("Starting training epoch: {}".format(ep))
# Initialize variables for over-all loss per epoch
train_loss = 0
# Load one minibatch at a time and perform a training step
t_mb = Timer(verbose=True, name="Load Minibatch")
mb_training = trainingset.batch_loader(rnd_gen=rnd_gen)
#
# Loop through minibatches
#
for mb_i, mb in enumerate(mb_training):
sys.stdout.flush()
# Print minibatch load time
t_mb.print()
# Abort if indicated by file
check_kill_file(workspace)
#
# Calculate scores on validation set
#
if global_step % config.score_at == 0:
print("Starting scoring on validation set...")
evaluate_on_validation_set(validationset, global_step, sess, model, summary_writer_validation,
val_loss_summary, val_loss, workspace)
#
# Perform weight updates and do plotting
#
if (mb_i % config.plot_at) == 0 and os.path.isfile(workspace.get_plot_file()):
# Perform weight update, return summary values and values for plotting
with Timer(verbose=True, name="Weight Update"):
plotting_values = []
train_summ, regpen_summ, _, cur_loss, *plotting_values = sess.run(
[train_summary, regpen_summary, update, loss, *plotting_tensors],
feed_dict={model.X: mb['X'], model.y_: mb['y']})
# Add current summary values to tensorboard
summary_writer_train.add_summary(train_summ, global_step=global_step)
summary_writer_train.add_summary(regpen_summ, global_step=global_step)
# Create and save subplot 1 (input)
save_subplots(images=plotting_values[:len(tensors_subplot1)],
subfigtitles=list(tensors_subplot1.keys()),
subplotranges=[(0, 1)] * n_timesteps, colorbar=True, automatic_positioning=True,
tight_layout=True,
filename=os.path.join(workspace.get_result_dir(),
"input_ep{}_mb{}.png".format(ep, mb_i)))
del plotting_values[:len(tensors_subplot1)]
# Create and save subplot 2 (target)
save_subplots(images=plotting_values[:len(tensors_subplot2)],
subfigtitles=list(tensors_subplot2.keys()),
subplotranges=[(0, 10) * n_timesteps], colorbar=True, automatic_positioning=True,
tight_layout=True,
filename=os.path.join(workspace.get_result_dir(),
"target_ep{}_mb{}.png".format(ep, mb_i)))
del plotting_values[:len(tensors_subplot2)]
# Create and save subplot 3 (output)
save_subplots(images=plotting_values[:len(tensors_subplot3)],
subfigtitles=list(tensors_subplot3.keys()),
# subplotranges=[(0, 10)] * n_timesteps,
colorbar=True, automatic_positioning=True,
tight_layout=True,
filename=os.path.join(workspace.get_result_dir(),
"output_ep{}_mb{}.png".format(ep, mb_i)))
del plotting_values[:len(tensors_subplot3)]
# Create and save subplot 2 (hidden states, i.e. ConvLSTM outputs)
save_subplots(images=plotting_values[:len(tensors_subplot4)],
subfigtitles=list(tensors_subplot4.keys()),
title='ConvLSTM hidden states (outputs)', colorbar=True, automatic_positioning=True,
tight_layout=True,
filename=os.path.join(workspace.get_result_dir(),
"hidden_ep{}_mb{}.png".format(ep, mb_i)))
del plotting_values[:len(tensors_subplot4)]
# Create and save subplot 3 (cell states)
save_subplots(images=plotting_values[:len(tensors_subplot5)],
subfigtitles=list(tensors_subplot5.keys()),
title='ConvLSTM cell states', colorbar=True, automatic_positioning=True,
tight_layout=True,
filename=os.path.join(workspace.get_result_dir(),
"cell_ep{}_mb{}.png".format(ep, mb_i)))
del plotting_values[:len(tensors_subplot5)]
else:
#
# Perform weight update without plotting
#
with Timer(verbose=True, name="Weight Update"):
train_summ, regpen_summ, _, cur_loss = sess.run([
train_summary, regpen_summary, update, loss],
feed_dict={model.X: mb['X'], model.y_: mb['y']})
# Add current summary values to tensorboard
summary_writer_train.add_summary(train_summ, global_step=global_step)
summary_writer_train.add_summary(regpen_summ, global_step=global_step)
# Add current loss to running average loss
train_loss += cur_loss
# Print some status info
print("ep {} mb {} loss {} (avg. loss {})".format(ep, mb_i, cur_loss, train_loss / (mb_i + 1)))
# Reset timer
t_mb = Timer(name="Load Minibatch")
# Free the memory allocated for the minibatch data
mb.clear()
del mb
global_step += 1
#
# Calculate scores on validation set
#
# Perform scoring on validation set
print("Starting scoring on validation set...")
evaluate_on_validation_set(validationset, global_step, sess, model, summary_writer_validation,
val_loss_summary, val_loss, workspace)
# Save the model
tell.save_checkpoint(global_step=global_step)
# Abort if indicated by file
check_kill_file(workspace)
except AbortRun:
print("Detected kill file, aborting...")
finally:
#
# If the program executed correctly or an error was raised, close the data readers and save the model and exit
#
trainingset.close()
validationset.close()
tell.close(save_checkpoint=True, global_step=global_step)
def main(_):
config = Config()
# Create new TeLL session with two summary writers
tell = TeLLSession(config=config, summaries=["train", "validation"])
# Get some members from the session for easier usage
session = tell.tf_session
summary_writer_train, summary_writer_validation = tell.tf_summaries["train"], tell.tf_summaries["validation"]
model = tell.model
workspace, config = tell.workspace, tell.config
# Parameters
learning_rate = config.get_value("learning_rate", 1e-3)
iterations = config.get_value("iterations", 1000)
batchsize = config.get_value("batchsize", 250)
display_step = config.get_value("display_step", 10)
dropout = config.get_value("dropout_prob", 0.25)
#
# Load Data
#
with Timer(name="Load data"):
mnist = input_data.read_data_sets("../MNIST_data", one_hot=True)
# Define loss and optimizer
with tf.name_scope("Cost"):
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=model.output, labels=model.y_))
##entropy = tf.reduce_mean(tf.contrib.bayesflow.entropy.entropy_shannon(
## tf.contrib.distributions.Categorical(p=tf.nn.softmax(logits=model.output))))
probs = tf.nn.softmax(logits=model.output)
entropy = tf.reduce_mean(-tf.reduce_sum(tf.log(tf.maximum(probs, 1e-15)) * probs, 1))
# test decor regularization
#decor_penalty(model.hidden1, model.y_, 10, [1], 0.)
#decor_penalty(model.hidden2, model.y_, 10, [1], 0.)
optimizer = tell.tf_optimizer.minimize(cost - config.get_value("entropy_w", 0.) * entropy)
tf.summary.scalar("Loss", cost)
#tf.summary.scalar("Decor", decor1 + decor2)
#tf.summary.scalar("Entropy", entropy)
tf.summary.scalar("O-Prob", tf.reduce_mean(tf.reduce_sum(tf.nn.softmax(logits=model.output) * model.y_, 1)))
# Evaluate model
with tf.name_scope("Accuracy"):
correct_pred = tf.equal(tf.argmax(model.output, 1), tf.argmax(model.y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
tf.summary.scalar("Accuracy", accuracy)
merged_summaries = tf.summary.merge_all()
# Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint)
step = tell.initialize_tf_variables(reset_optimizer_on_restore=True).global_step
# -------------------------------------------------------------------------
# Start training
# -------------------------------------------------------------------------
acc_train = 0.
val_acc_best = 0.
try:
while step < iterations:
check_kill_file(workspace=workspace)
batch_x, batch_y = mnist.train.next_batch(batchsize)
i = step * batchsize
if step % display_step == 0:
summary, acc = session.run([merged_summaries, accuracy],
feed_dict={model.X: mnist.validation.images[:2048],
model.y_: mnist.validation.labels[:2048],
model.dropout: 0})
summary_writer_validation.add_summary(summary, i)
print('step {}: train acc {}, valid acc {}'.format(i, acc_train, acc))
if acc > val_acc_best:
val_acc_best = acc
else:
summary, acc_train, _ = session.run([merged_summaries, accuracy, optimizer],
feed_dict={model.X: batch_x, model.y_: batch_y,
model.dropout: dropout})
summary_writer_train.add_summary(summary, i)
step += 1
print("Training Finished! best valid acc {}".format(val_acc_best))
# Final Eval
print("Test Accuracy:",
session.run(accuracy, feed_dict={model.X: mnist.test.images[:2048],
model.y_: mnist.test.labels[:2048],
model.dropout: 0}))
except AbortRun:
print("Aborting...")
finally:
tell.close(global_step=step)
def main(_):
# ------------------------------------------------------------------------------------------------------------------
# Setup training
# ------------------------------------------------------------------------------------------------------------------
# Initialize config, parses command line and reads specified file; also supports overriding of values from cmd
config = Config()
random_seed = config.get_value('random_seed', 12345)
np.random.seed(random_seed) # not threadsafe, use rnd_gen object where possible
rnd_gen = np.random.RandomState(seed=random_seed)
# Load datasets for trainingset
with Timer(name="Loading Data"):
readers = initialize_datareaders(config, required=("train", "val"))
trainingset = DataLoader(readers["train"], batchsize=config.batchsize)
#validationset = DataLoader(readers["val"], batchsize=config.batchsize)
# Initialize TeLL session
tell = TeLLSession(config=config, model_params={"input_shape": [300]})
# Get some members from the session for easier usage
session = tell.tf_session
model = tell.model
workspace, config = tell.workspace, tell.config
# Initialize Tensorflow variables
global_step = tell.initialize_tf_variables().global_step
sys.stdout.flush()
# ------------------------------------------------------------------------------------------------------------------
# Start training
# ------------------------------------------------------------------------------------------------------------------
try:
epoch = int(global_step / trainingset.n_mbs)
epochs = range(epoch, config.n_epochs)
#
# Loop through epochs
#
print("Starting training")
for ep in epochs:
print("Starting training epoch: {}".format(ep))
# Initialize variables for over-all loss per epoch
train_loss = 0
# Load one minibatch at a time and perform a training step
t_mb = Timer(name="Load Minibatch")
mb_training = trainingset.batch_loader(rnd_gen=rnd_gen)
#
# Loop through minibatches
#
for mb_i, mb in enumerate(mb_training):
sys.stdout.flush()
#Print minibatch load time
t_mb.print()
# Abort if indicated by file
check_kill_file(workspace)
#
# Calculate scores on validation set
#
if global_step % config.score_at == 0:
print("Starting scoring on validation set...")
# Get new sample
training_sample = np.ones(shape=(1,np.random.randint(low=20,high=100),300))
#
# Perform weight update
#
with Timer(name="Weight Update"):
#
# Set placeholder values
#
placeholder_values = OrderedDict(
input_placeholder=training_sample,
sequence_length_placeholder = training_sample.shape[1]
)
feed_dict = dict(((model.placeholders[k], placeholder_values[k]) for k in placeholder_values.keys()))
#
# Decide which tensors to compute
#
data_keys = ['lstm_internals_enc', 'lstm_internals_dec', 'lstm_h_enc',
'lstm_h_dec', 'loss' , 'loss_last_time_prediction','loss_last_time_prediction', 'reg_loss']
data_tensors = [model.data_tensors[k] for k in data_keys]
operation_keys = ['ae_update']
operation_tensors = [model.operation_tensors[k] for k in operation_keys]
summary_keys = ['all_summaries']
summary_tensors = [model.summaries[k] for k in summary_keys]
#
# Run graph and re-associate return values with keys in dictionary
#
ret = session.run(data_tensors + summary_tensors + operation_tensors, feed_dict)
data_keys = ['loss']
data_tensors = [model.data_tensors[k] for k in data_keys]
session.run(model.data_tensors['loss'] , feed_dict)
session.run(model.data_tensors['latent_space'], feed_dict)
ret_dict = OrderedDict(((k, ret[i]) for i, k in enumerate(data_keys)))
del ret[:len(data_keys)]
ret_dict.update(OrderedDict(((k, ret[i]) for i, k in enumerate(summary_keys))))
# Print some status info
#print("ep {} mb {} loss {} (avg. loss {})".format(ep, mb_i, cur_loss, train_loss / (mb_i + 1)))
# Reset timer
#t_mb = Timer(name="Load Minibatch")
# Free the memory allocated for the minibatch data
#mb.clear()
#del mb
global_step += 1
#
# Calculate scores on validation set after training is done
#
# Perform scoring on validation set
print("Starting scoring on validation set...")
tell.save_checkpoint(global_step=global_step)
# Abort if indicated by file
check_kill_file(workspace)
except AbortRun:
print("Detected kill file, aborting...")
finally:
tell.close(save_checkpoint=True, global_step=global_step)
def main(_):
# ------------------------------------------------------------------------------------------------------------------
# Setup training
# ------------------------------------------------------------------------------------------------------------------
# Initialize config, parses command line and reads specified file; also supports overriding of values from cmd
config = Config()
# Load datasets for trainingset
with Timer(name="Loading Training Data"):
# Make sure datareader is reproducible
random_seed = config.get_value('random_seed', 12345)
np.random.seed(
random_seed) # not threadsafe, use rnd_gen object where possible
rnd_gen = np.random.RandomState(seed=random_seed)
print("Loading training data...")
trainingset = ShortLongDataset(n_timesteps=250,
n_samples=3000,
batchsize=config.batchsize,
rnd_gen=rnd_gen)
# Load datasets for validationset
validationset = ShortLongDataset(n_timesteps=250,
n_samples=300,
batchsize=config.batchsize,
rnd_gen=rnd_gen)
# Initialize TeLL session
tell = TeLLSession(config=config,
summaries=["train"],
model_params={"dataset": trainingset})
# Get some members from the session for easier usage
session = tell.tf_session
summary_writer = tell.tf_summaries["train"]
model = tell.model
workspace, config = tell.workspace, tell.config
# Loss function for trainingset
print("Initializing loss calculation...")
loss = tf.reduce_mean(
tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(
model.y_, model.output,
-tf.reduce_sum(model.y_ - 1) / tf.reduce_sum(model.y_)),
axis=[1]))
train_summary = tf.summary.scalar("Training Loss",
loss) # add loss to tensorboard
# Loss function for validationset
val_loss = tf.reduce_mean(
tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(
model.y_, model.output,
-tf.reduce_sum(model.y_ - 1) / tf.reduce_sum(model.y_)),
axis=[1]))
val_loss_summary = tf.summary.scalar(
"Validation Loss", val_loss) # add val_loss to tensorboard
# Regularization
reg_penalty = regularize(layers=model.get_layers(),
l1=config.l1,
l2=config.l2,
regularize_weights=True,
regularize_biases=True)
regpen_summary = tf.summary.scalar(
"Regularization Penalty",
reg_penalty) # add reg_penalty to tensorboard
# Update step for weights
update = update_step(loss + reg_penalty, config)
# Initialize Tensorflow variables
global_step = tell.initialize_tf_variables().global_step
sys.stdout.flush()
# ------------------------------------------------------------------------------------------------------------------
# Start training
# ------------------------------------------------------------------------------------------------------------------
try:
epoch = int(global_step / trainingset.n_mbs)
epochs = range(epoch, config.n_epochs)
#
# Loop through epochs
#
print("Starting training")
for ep in epochs:
print("Starting training epoch: {}".format(ep))
# Initialize variables for over-all loss per epoch
train_loss = 0
# Load one minibatch at a time and perform a training step
t_mb = Timer(name="Load Minibatch")
mb_training = trainingset.batch_loader(rnd_gen=rnd_gen)
#
# Loop through minibatches
#
for mb_i, mb in enumerate(mb_training):
sys.stdout.flush()
# Print minibatch load time
t_mb.print()
# Abort if indicated by file
check_kill_file(workspace)
#
# Calculate scores on validation set
#
if global_step % config.score_at == 0:
print("Starting scoring on validation set...")
evaluate_on_validation_set(validationset, global_step,
session, model, summary_writer,
val_loss_summary, val_loss,
workspace)
#
# Perform weight update
#
with Timer(name="Weight Update"):
train_summ, regpen_summ, _, cur_loss = session.run(
[train_summary, regpen_summary, update, loss],
feed_dict={
model.X: mb['X'],
model.y_: mb['y']
})
# Add current summary values to tensorboard
summary_writer.add_summary(train_summ, global_step=global_step)
summary_writer.add_summary(regpen_summ,
global_step=global_step)
# Add current loss to running average loss
train_loss += cur_loss
# Print some status info
print("ep {} mb {} loss {} (avg. loss {})".format(
ep, mb_i, cur_loss, train_loss / (mb_i + 1)))
# Reset timer
t_mb = Timer(name="Load Minibatch")
# Free the memory allocated for the minibatch data
mb.clear()
del mb
global_step += 1
#
# Calculate scores on validation set after training is done
#
# Perform scoring on validation set
print("Starting scoring on validation set...")
evaluate_on_validation_set(validationset, global_step, session,
model, summary_writer, val_loss_summary,
val_loss, workspace)
tell.save_checkpoint(global_step=global_step)
# Abort if indicated by file
check_kill_file(workspace)
except AbortRun:
print("Detected kill file, aborting...")
finally:
tell.close(save_checkpoint=True, global_step=global_step)
def evaluate(self,
step: int,
summary_writer,
prefix='validation ',
num_cached=5,
num_threads=3,
rnd_gen=None,
plotter=None,
model=None):
# Reset streaming measures
self.reset_tensors()
# Get tensors to evaluate for plotting
if plotter is not None:
plot_tensors = plotter.get_tensors()
# Set up progress bar
_pbw = ['Evaluating on {}set:'.format(prefix), progressbar.ETA()]
progress = progressbar.ProgressBar(widgets=_pbw,
maxval=self.dataset.n_mbs -
1).start()
#
# Iterate over dataset minibatches
#
mb_validation = self.dataset.batch_loader(num_cached=num_cached,
num_threads=num_threads,
rnd_gen=rnd_gen)
with Timer(verbose=True, name="Evaluate on {}set".format(prefix)):
summary_values_filled = None
for mb_i, mb in enumerate(mb_validation):
# Abort if indicated by file
check_kill_file(self.workspace)
if mb.get('pixel_weights', None) is None:
feed_dict = {self.model.X: mb['X'], self.model.y_: mb['y']}
else:
feed_dict = {
self.model.X: mb['X'],
self.model.y_: mb['y'],
self.model.pixel_weights: mb['pixel_weights']
}
if plotter is not None:
evaluated_tensors = self.session.run([
*self.summary_ops, *self.summary_tensors, *plot_tensors
],
feed_dict=feed_dict)
else:
evaluated_tensors = self.session.run(
[*self.summary_ops, *self.summary_tensors],
feed_dict=feed_dict)
# Discard return values from summary_ops (=update operations)
evaluated_tensors = evaluated_tensors[len(self.summary_ops):]
summary_values = evaluated_tensors[:len(self.summary_tensors)]
# Perform plotting
if plotter is not None:
plotter.set_tensor_values(
evaluated_tensors[len(self.summary_tensors
):len(self.plot_tensors) +
len(plot_tensors)])
plotter.plot(evaluate_tensors=False)
# Re-associate returned tensorflow values to keys and incorporate new minibatch values
if summary_values_filled is None:
# Fill summary_values_filled for the first time
summary_values_filled = OrderedDict(
zip(list(self.summary_tensor_dict.keys()),
summary_values))
for key_i, key in enumerate(summary_values_filled.keys()):
if not self.summary_tensor_is_op[key_i]:
if isinstance(summary_values_filled[key],
np.ndarray):
summary_values_filled[key] = [
summary_values_filled[key]
]
elif np.isfinite(summary_values_filled[key]):
summary_values_filled[key] = [
summary_values_filled[key]
]
else:
summary_values_filled[key] = []
else:
for key_i, key in enumerate(summary_values_filled.keys()):
if not self.summary_tensor_is_op[key_i]:
if isinstance(summary_values[key_i], np.ndarray):
summary_values_filled[key].append(
summary_values[key_i])
elif np.isfinite(summary_values[key_i]):
summary_values_filled[key].append(
summary_values[key_i])
else:
summary_values_filled[key] = summary_values[key_i]
# Update progress bar and clear minibatch
progress.update(mb_i)
mb.clear()
del mb
progress.finish()
#
# Divide sums by number of samples for tensors that do not have an update function
#
if len(summary_values_filled):
for key_i, key in enumerate(summary_values_filled.keys()):
if not self.summary_tensor_is_op[key_i]:
if len(summary_values_filled[key]):
if not isinstance(summary_values_filled[key][0],
np.ndarray):
summary_values_filled[key] = np.mean(
summary_values_filled[key])
else:
summary_values_filled[key] = np.concatenate(
summary_values_filled[key])
else:
summary_values_filled[key] = np.nan
#
# Go through values to use as summaries, create histograms if values are not scalars
#
values_to_print = OrderedDict()
if len(summary_values_filled):
for key_i, key in enumerate(summary_values_filled.keys()):
if not isinstance(summary_values_filled[key], np.ndarray):
values_to_print.update({key: summary_values_filled[key]})
summary = tf.Summary(value=[
tf.Summary.Value(tag=prefix + key,
simple_value=float(
summary_values_filled[key]))
])
else:
hist = custom_tensorflow_histogram(
summary_values_filled[key], bins=100)
summary = tf.Summary(
value=[tf.Summary.Value(tag=prefix + key, histo=hist)])
summary_writer.add_summary(summary, step)
print("{}scores:\n\tstep {}, {}".format(prefix, step, values_to_print))
summary_writer.flush()
sys.stdout.flush()
def main(_):
config = Config()
# Create new TeLL session with two summary writers
tell = TeLLSession(config=config, summaries=["train", "validation"])
# Get some members from the session for easier usage
session = tell.tf_session
summary_writer_train, summary_writer_validation = tell.tf_summaries[
"train"], tell.tf_summaries["validation"]
model = tell.model
workspace, config = tell.workspace, tell.config
# Parameters
learning_rate = config.get_value("learning_rate", 1e-3)
iterations = config.get_value("iterations", 1000)
batchsize = config.get_value("batchsize", 250)
display_step = config.get_value("display_step", 10)
dropout = config.get_value("dropout_prob", 0.25)
#
# Prepare input data
#
# Set datareaders
training_reader = MNISTReader(dset='train')
validation_reader = MNISTReader(dset='validation')
test_reader = MNISTReader(dset='test')
# Set Preprocessing
training_data_preprocessed = DataProcessing(training_reader, apply_to='X')
training_data_preprocessed = Normalize(training_data_preprocessed,
apply_to='X')
training_data_preprocessed = Normalize(training_data_preprocessed,
apply_to=['X', 'Y'])
# Set minibatch loaders
training_loader = DataLoader(training_data_preprocessed,
batchsize=50,
batchsize_method='zeropad')
validation_loader = DataLoader(validation_reader,
batchsize=50,
batchsize_method='zeropad')
test_loader = DataLoader(test_reader,
batchsize=50,
batchsize_method='zeropad')
#
# Define loss and optimizer
#
with tf.name_scope("Cost"):
cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=model.output,
labels=model.y_))
decor1 = decor_penalty(model.hidden1, model.y_, 10, [1], 0.)
decor2 = decor_penalty(model.hidden2, model.y_, 10, [1], 6e-5)
optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(cost + decor1 + decor2)
tf.summary.scalar("Loss", cost)
tf.summary.scalar("Decor", decor1 + decor2)
# Evaluate model
with tf.name_scope("Accuracy"):
correct_pred = tf.equal(tf.argmax(model.output, 1),
tf.argmax(model.y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
tf.summary.scalar("Accuracy", accuracy)
merged_summaries = tf.summary.merge_all()
# Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint)
step = tell.initialize_tf_variables().global_step
# -------------------------------------------------------------------------
# Start training
# -------------------------------------------------------------------------
acc_train = 0.
try:
while step < iterations:
# Loop through training set
for mb_i, mb in enumerate(
training_loader.batch_loader(num_cached=5, num_threads=3)):
check_kill_file(workspace=workspace)
# Perform weight update
summary, acc_train, _ = session.run(
[merged_summaries, accuracy, optimizer],
feed_dict={
model.X: mb['X'],
model.y_: mb['y'],
model.dropout: dropout
})
summary_writer_train.add_summary(summary,
mb_i + step * batchsize)
if step % display_step == 0:
# Loop through validation set
cos_sum, acc_sum, cor_sum = (0, 0, 0)
for vmb_i, vmb in enumerate(
validation_loader.batch_loader(num_cached=5,
num_threads=3)):
cos, acc, cor = session.run(
[cost, accuracy, correct_pred],
feed_dict={
model.X: vmb['X'],
model.y_: vmb['y'],
model.dropout: 0
})
cos_sum += cos
acc_sum += acc
cor_sum += cor
print('step {}: train acc {}, valid acc {}'.format(
mb_i + step * batchsize, cos_sum / vmb_i,
acc_sum / vmb_i, cor_sum / vmb_i))
step += 1
if step >= iterations:
break
print("Training Finished!")
# Final Eval
for tmb_i, tmb in enumerate(
test_loader.batch_loader(num_cached=len(
test_reader.get_sample_keys()),
num_threads=1)):
print(
"Test Accuracy:",
session.run(accuracy,
feed_dict={
model.X: tmb['X'],
model.y_: tmb['y'],
model.dropout: 0
}))
except AbortRun:
print("Aborting...")
finally:
tell.close(global_step=step)