def fit(self,
dataloader,
nb_iter=None,
nb_epoch=None,
iter_per_epoch=None,
callbacks=[],
verbose=0):
"""Trains the underlying Keras model.
Args:
dataloader (StandardDataLoader): Manages the loading of data to
model.
nb_iter (int): The number of iterations to train the model.
nb_epoch (int): The number of epochs to train the model.
iter_per_epoch (int): Defines the number of iterations per epoch.
callbacks (list): List of Keras callbacks to run during training.
"""
nb_iter, iter_per_epoch = self._get_iterations(nb_iter, nb_epoch,
iter_per_epoch)
callbacks = CallbackList(callbacks)
callbacks._set_model(self)
callbacks.on_train_begin()
try:
epoch = 0
self.stop_training = False
for i in xrange(nb_iter):
# Begin epoch
if i % iter_per_epoch == 0:
callbacks.on_epoch_begin(epoch)
# Execution
callbacks.on_batch_begin(i)
if verbose > 0:
import time
time.sleep(0.001)
j = i % iter_per_epoch
perc = int(100 * (j + 1) / iter_per_epoch)
prog = ''.join(['='] * (perc / 2))
string = "[{:50s}] {:3d}%\r".format(prog, perc)
sys.stdout.write(string)
sys.stdout.flush()
losses = self.keras_model.train_on_batch(
*dataloader.get_training_batch())
callbacks.on_batch_end(i)
# End epoch
if (i + 1) % iter_per_epoch == 0:
callbacks.on_epoch_end(epoch, logs={'losses': losses})
epoch += 1
if self.stop_training:
break
except KeyboardInterrupt:
print "\n[BayesNet] Abort: KeyboardInterrupt"
raise
callbacks.on_train_end()
def fit(self,
train_dataset,
test_dataset=None,
batch_size=32,
epochs=1,
optimizer=tf.keras.optimizers.Adam(),
callbacks=None):
# log_dir = './logs/test/'
# writer = tf.summary.create_file_writer(log_dir)
train_loss = tf.keras.metrics.Mean()
train_reconstruction_loss = tf.keras.metrics.Mean()
test_loss = tf.keras.metrics.Mean()
test_reconstruction_loss = tf.keras.metrics.Mean()
train_dataset = self._check_tf_dataset_instance(train_dataset,
batch_size=batch_size)
callbacks = CallbackList(callbacks)
callback_model = self._get_callback_model()
callbacks.set_model(callback_model)
for epoch in range(1, epochs + 1):
callbacks.on_epoch_begin(epoch)
epoch_logs = {}
for x_train in train_dataset:
self.compute_apply_gradients(x_train, optimizer=optimizer)
train_loss(self.compute_loss(x_train))
train_reconstruction_loss(
self.compute_reconstruction_error(x_train))
epoch_logs['train_loss'] = train_loss.result().numpy()
epoch_logs[
'train_reconstruction_error'] = train_reconstruction_loss.result(
).numpy()
train_loss.reset_states()
train_reconstruction_loss.reset_states()
if test_dataset is not None:
test_dataset = self._check_tf_dataset_instance(
test_dataset, batch_size=batch_size)
for x_test in test_dataset:
test_loss(self.compute_loss(x_test))
test_reconstruction_loss(
self.compute_reconstruction_error(x_test))
epoch_logs['test_loss'] = test_loss.result().numpy()
epoch_logs[
'test_reconstruction_error'] = test_reconstruction_loss.result(
).numpy()
test_loss.reset_states()
test_reconstruction_loss.reset_states()
callbacks.on_epoch_end(epoch, logs=epoch_logs)
def fit(self, dataloader, nb_iter=None, nb_epoch=None, iter_per_epoch=None,
callbacks=[], verbose=0):
"""Trains the underlying Keras model.
Args:
dataloader (StandardDataLoader): Manages the loading of data to
model.
nb_iter (int): The number of iterations to train the model.
nb_epoch (int): The number of epochs to train the model.
iter_per_epoch (int): Defines the number of iterations per epoch.
callbacks (list): List of Keras callbacks to run during training.
"""
nb_iter, iter_per_epoch = self._get_iterations(
nb_iter, nb_epoch, iter_per_epoch)
callbacks = CallbackList(callbacks)
callbacks._set_model(self)
callbacks.on_train_begin()
try:
epoch = 0
self.stop_training = False
for i in xrange(nb_iter):
# Begin epoch
if i % iter_per_epoch == 0:
callbacks.on_epoch_begin(epoch)
# Execution
callbacks.on_batch_begin(i)
if verbose > 0:
import time
time.sleep(0.001)
j = i % iter_per_epoch
perc = int(100 * (j + 1) /iter_per_epoch)
prog = ''.join(['='] * (perc/2))
string = "[{:50s}] {:3d}%\r".format(prog, perc)
sys.stdout.write(string); sys.stdout.flush()
losses = self.keras_model.train_on_batch(
*dataloader.get_training_batch())
callbacks.on_batch_end(i)
# End epoch
if (i + 1) % iter_per_epoch == 0:
callbacks.on_epoch_end(epoch, logs={'losses': losses})
epoch += 1
if self.stop_training:
break
except KeyboardInterrupt:
print "\n[BayesNet] Abort: KeyboardInterrupt"
raise
callbacks.on_train_end()
def fit(self, X_train, epochs=1, batch_size=256, callbacks=None):
global_step = 0
step_size = 2 * np.ceil(X_train.shape[0] / batch_size)
train_dataset = tf.data.Dataset.from_tensor_slices(X_train).shuffle(
X_train.shape[0]).batch(batch_size)
callbacks = CallbackList(callbacks)
callback_model = self._get_callback_model()
callbacks.set_model(callback_model)
for epoch in range(1, epochs + 1):
callbacks.on_epoch_begin(epoch)
epoch_logs = {}
# Learning rate schedule
if epoch in [60, 100, 300]:
self.base_lr = self.base_lr / 2
self.max_lr = self.max_lr / 2
step_size = step_size / 2
# print('learning rate changed!')
epoch_ae_loss_avg = tf.metrics.Mean()
epoch_dc_loss_avg = tf.metrics.Mean()
epoch_dc_acc_avg = tf.metrics.Mean()
epoch_gen_loss_avg = tf.metrics.Mean()
for batch, (x_batch) in enumerate(train_dataset):
# -------------------------------------------------------------------------------------------------------------
# Calculate cyclic learning rate
global_step = global_step + 1
cycle = np.floor(1 + global_step / (2 * step_size))
x_lr = np.abs(global_step / step_size - 2 * cycle + 1)
clr = self.base_lr + (self.max_lr - self.base_lr) * max(
0, 1 - x_lr)
self.ae_optimizer.lr = clr
self.dc_optimizer.lr = clr
self.gen_optimizer.lr = clr
ae_loss, dc_loss, dc_acc, gen_loss = self.train_step(x_batch)
epoch_ae_loss_avg(ae_loss)
epoch_dc_loss_avg(dc_loss)
epoch_dc_acc_avg(dc_acc)
epoch_gen_loss_avg(gen_loss)
epoch_logs['ae_loss'] = epoch_ae_loss_avg.result().numpy()
epoch_logs['dc_loss'] = epoch_dc_loss_avg.result().numpy()
epoch_logs['dc_acc'] = epoch_dc_acc_avg.result().numpy()
epoch_logs['gen_loss'] = epoch_gen_loss_avg.result().numpy()
callbacks.on_epoch_end(epoch, logs=epoch_logs)
def _train_by_batch(self):
# batch finite generator should be loaded within epoch loop
logger.info('Start training by batch')
self.validation_xy = self.load_data('val', feed_mode='all')
do_validation = bool(self.validation_xy)
# prepare display labels in tensorboard
out_labels = self.model._get_deduped_metrics_names()
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
self.model.history = History()
callbacks = [BaseLogger()] + (self.callbacks
or []) + [self.model.history]
# callbacks = (self.callbacks or []) + [self.model.history]
if self.verbose:
callbacks += [ProgbarLogger(count_mode='samples')]
callbacks = CallbackList(callbacks)
# it's possible to callback a different model than this model
if hasattr(self.model, 'callback_model') and self.model.callback_model:
callback_model = self.model.callback_model
else:
callback_model = self.model
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': self.epochs,
'samples': self.data.nb_train,
'verbose': self.verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
for epoch in range(self.epochs):
start_e = time()
callbacks.on_epoch_begin(epoch)
xy_gen = self.load_data('train', feed_mode='batch')
logger.info('New training epoch')
for batch_index, (x, y) in enumerate(xy_gen):
# build batch logs
batch_logs = {}
if isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
outs = self.model.train_on_batch(x, y)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if (batch_index + 1) % 1000 == 0 and do_validation:
val_outs = self.model.evaluate(*self.validation_xy,
batch_size=81920,
verbose=0)
batch_logs = {}
if not isinstance(val_outs, list):
val_outs = [val_outs]
for l, o in zip(out_labels, val_outs):
batch_logs['val_' + l] = o
print(' - Eval inside: %.6f' % val_outs[0])
for cb in self.callbacks:
if cb.__class__ == tensorBoard:
cb.on_batch_end(batch_index,
batch_logs,
count=False)
epoch_logs = {}
if do_validation:
val_outs = self.model.evaluate(*self.validation_xy,
batch_size=81920,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_batch_end(epoch, epoch_logs)
callbacks.on_epoch_end(epoch, epoch_logs)
elapsed_e = timedelta(seconds=int(time() - start_e))
self.send_metric('elapsed_per_epoch', elapsed_e)
if not self.no_save and do_validation and (epoch !=
self.epochs - 1):
self.model.save(
'results/trained_models/%s_ctr_model_%.4f_epoch_%d.h5' %
(self.sess_id, val_outs[0], epoch))
callbacks.on_train_end()
return self.model.history
def predict_image(version, image_path, batch_size, overlap, data_format=None):
def current_time_millis():
return int(round(time.time() * 1000))
def offset(size, diff, overlap):
return math.floor(diff / math.ceil(diff / (size * (1 - overlap))))
def map_c(i, j, b, l):
return int(((i * b) + j) / l)
def map_r(i, j, b, l):
return ((i * b) + j) % l
if data_format is None:
data_format = K.image_data_format()
if data_format not in {'channels_first', 'channels_last'}:
raise ValueError('Unknown data_format:', data_format)
path = version.model_file.name
print(_('Loading model "%(path)s".') % {'path': path})
model = load_model(os.path.join(settings.MEDIA_ROOT, path))
if len(model.inputs) != 1:
raise RuntimeError('Models with more than one input are not'
' supported at the moment.')
inputs = []
for i in range(len(model.inputs)):
name = model.inputs[i].name
pos = min(
name.index('/') if '/' in name else len(name),
name.index(':') if ':' in name else len(name))
name = name[:pos]
inputs.append({'shape': model.inputs[i].shape.as_list(), 'name': name})
if data_format == 'channels_first':
inputs[i]['grayscale'] = inputs[i]['shape'][1] == 1
inputs[i]['r'] = inputs[i]['shape'][2]
inputs[i]['c'] = inputs[i]['shape'][3]
elif data_format == 'channels_last':
inputs[i]['r'] = inputs[i]['shape'][1]
inputs[i]['c'] = inputs[i]['shape'][2]
inputs[i]['grayscale'] = inputs[i]['shape'][3] == 1
inputs[i]['img'] = img_to_array(
load_img(image_path, inputs[i]['grayscale']))
inputs[i]['img'] *= 1. / 255
if data_format == 'channels_first':
inputs[i]['img_r'] = inputs[i]['img'].shape[1]
inputs[i]['img_c'] = inputs[i]['img'].shape[2]
elif data_format == 'channels_last':
inputs[i]['img_r'] = inputs[i]['img'].shape[0]
inputs[i]['img_c'] = inputs[i]['img'].shape[1]
inputs[i]['diff_r'] = inputs[i]['img_r'] - inputs[i]['r']
inputs[i]['diff_c'] = inputs[i]['img_c'] - inputs[i]['c']
inputs[i]['offset_r'] = offset(inputs[i]['r'], inputs[i]['diff_r'],
overlap)
inputs[i]['offset_c'] = offset(inputs[i]['c'], inputs[i]['diff_c'],
overlap)
inputs[i]['nb_r'] = math.ceil(
inputs[i]['diff_r'] / inputs[i]['offset_r']) + 1
inputs[i]['nb_c'] = math.ceil(
inputs[i]['diff_c'] / inputs[i]['offset_c']) + 1
inputs = inputs[0]
N = inputs['nb_r'] * inputs['nb_c']
steps = math.ceil(N / batch_size)
metrics = []
outputs = []
for i in range(len(model.outputs)):
tshape = model.outputs[i].shape.as_list()
name = model.outputs[i].name
pos = min(
name.index('/') if '/' in name else len(name),
name.index(':') if ':' in name else len(name))
name = name[:pos]
activation = model.get_layer(name).activation.__name__.lower()
outputs.append({'name': name, 'shape': tshape})
if len(tshape) == 2:
if activation == 'softmax':
outputs[i]['t'] = 'class'
else:
outputs[i]['t'] = 'multi'
nb_classes = tshape[1]
if nb_classes is None:
nb_classes = model.get_layer(name).output_shape[1]
nb_classes = int(nb_classes)
metrics += ['%s:%s' % (name, i) for i in range(nb_classes)]
if data_format == 'channels_first':
shape = (nb_classes, inputs['nb_r'], inputs['nb_c'])
elif data_format == 'channels_last':
shape = (inputs['nb_r'], inputs['nb_c'], nb_classes)
elif len(tshape) == 4:
if activation == 'softmax':
outputs[i]['t'] = 'class'
else:
outputs[i]['t'] = 'img'
shape = (inputs['nb_r'], inputs['nb_c']) + tuple(tshape[1:])
outputs[i]['p'] = np.zeros(shape)
history = History()
callbacks = CallbackList([BaseLogger(), history, ProgbarLogger()])
callbacks.set_model(model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': 1,
'steps': steps,
'samples': N,
'verbose': 1,
'do_validation': False,
'metrics': metrics,
})
callbacks.on_train_begin()
callbacks.on_epoch_begin(0)
start_time = current_time_millis()
for b in range(steps):
current_index = (b * batch_size) % N
if N >= current_index + batch_size:
current_batch_size = batch_size
else:
current_batch_size = N - current_index
batch_logs = {'batch': b, 'size': current_batch_size}
for metric in metrics:
batch_logs[metric] = 0
callbacks.on_batch_begin(b, batch_logs)
bX = np.zeros((current_batch_size, ) + tuple(inputs['shape'][1:]))
for j in range(current_batch_size):
idx_r = map_r(b, j, batch_size, inputs['nb_r'])
idx_c = map_c(b, j, batch_size, inputs['nb_r'])
top = min(idx_r * inputs['offset_r'],
inputs['img_r'] - inputs['r'])
bottom = min(idx_r * inputs['offset_r'] + inputs['r'],
inputs['img_r'])
left = min(idx_c * inputs['offset_c'],
inputs['img_c'] - inputs['c'])
right = min(idx_c * inputs['offset_c'] + inputs['c'],
inputs['img_c'])
if data_format == 'channels_first':
bX[j] = inputs['img'][:, top:bottom, left:right]
elif data_format == 'channels_last':
bX[j] = inputs['img'][top:bottom, left:right, :]
p = model.predict_on_batch(bX)
if type(p) != list:
p = [p]
for j in range(current_batch_size):
for i in range(len(outputs)):
idx_r = map_r(b, j, batch_size, inputs['nb_r'])
idx_c = map_c(b, j, batch_size, inputs['nb_r'])
if len(outputs[i]['p'].shape) == 3:
if data_format == 'channels_first':
outputs[i]['p'][:, idx_r, idx_c] = p[i][j]
elif data_format == 'channels_last':
outputs[i]['p'][idx_r, idx_c, :] = p[i][j]
metric = metrics[p[i][j].argmax()]
batch_logs[metric] += 1. / current_batch_size
elif len(outputs[i]['p'].shape) == 5:
outputs[i]['p'][idx_r, idx_c, :, :, :] = p[i][j]
callbacks.on_batch_end(b, batch_logs)
runtime = (current_time_millis() - start_time) / 1000
callbacks.on_epoch_end(0, {'runtime': runtime})
callbacks.on_train_end()
for i in range(len(outputs)):
if len(outputs[i]['shape']) == 2:
if data_format == 'channels_first':
shape = (outputs[i]['p'].shape[0], inputs['img_r'],
inputs['img_c'])
elif data_format == 'channels_last':
shape = (inputs['img_r'], inputs['img_c'],
outputs[i]['p'].shape[2])
elif len(tshape) == 4:
if data_format == 'channels_first':
shape = (outputs[i]['p'].shape[2], inputs['img_r'],
inputs['img_c'])
elif data_format == 'channels_last':
shape = (inputs['img_r'], inputs['img_c'],
outputs[i]['p'].shape[4])
count = np.zeros(shape)
outputs[i]['img'] = np.zeros(shape)
if len(outputs[i]['p'].shape) == 3:
if data_format == 'channels_first':
nb_rows = outputs[i]['p'].shape[1]
nb_cols = outputs[i]['p'].shape[2]
elif data_format == 'channels_last':
nb_rows = outputs[i]['p'].shape[0]
nb_cols = outputs[i]['p'].shape[1]
elif len(outputs[i]['p'].shape) == 5:
nb_rows = outputs[i]['p'].shape[0]
nb_cols = outputs[i]['p'].shape[1]
for j in range(nb_rows):
for k in range(nb_cols):
top = min(j * inputs['offset_r'],
inputs['img_r'] - inputs['r'])
bottom = min(j * inputs['offset_r'] + inputs['r'],
inputs['img_r'])
left = min(k * inputs['offset_c'],
inputs['img_c'] - inputs['c'])
right = min(k * inputs['offset_c'] + inputs['c'],
inputs['img_c'])
if data_format == 'channels_first':
outputs[i]['img'][:, top:bottom, left:right] += \
outputs[i]['p'][:, j, k]
count[:, top:bottom, left:right] += 1
elif data_format == 'channels_last':
outputs[i]['img'][top:bottom, left:right, :] += \
outputs[i]['p'][j, k, :]
count[top:bottom, left:right, :] += 1
outputs[i]['img'] /= count
del outputs[i]['p']
del outputs[i]['shape']
return history.history, outputs
def evaluate(model, save_path, num_outputs, liver_only=False, **kwargs):
model, callbacks, gen = prepare_model(model=model,
save_path=save_path,
num_outputs=num_outputs,
liver_only=liver_only,
**kwargs)
print(' > Evaluating the model...')
from scipy.misc import imsave
# Create directory, if needed
save_predictions_to = os.path.join(save_path, "predictions")
if not os.path.exists(save_predictions_to):
os.makedirs(save_predictions_to)
# Initialize callbacks
val_callback_list = [BaseLogger()]
if not liver_only:
val_callback_list.extend(
[callbacks['dice_lesion'], callbacks['dice_lesion_inliver']])
if len(model.outputs) == 2 or liver_only:
val_callback_list.append(callbacks['dice_liver'])
val_callbacks = CallbackList(val_callback_list)
val_callbacks.set_params({
'nb_epoch': 0,
'nb_sample': 0,
'verbose': False,
'do_validation': True,
'metrics': model.metrics_names
})
val_callbacks.on_train_begin()
val_callbacks.on_epoch_begin(0)
# Create theano function
inputs = model.inputs + model.targets + model.sample_weights
if model.uses_learning_phase and \
not isinstance(K.learning_phase(), int):
inputs += [K.learning_phase()]
predict_and_test_function = K.function( \
inputs,
model.outputs+[model.total_loss]+model.metrics_tensors,
updates=model.state_updates)
# Loop through batches, applying function and callbacks
flow = repeat_flow(gen['valid_callback'].flow(), num_outputs=num_outputs)
for batch_num, batch in enumerate(flow):
x, y, sample_weights = model._standardize_user_data(batch[0], batch[1])
ins = x + y + sample_weights
if model.uses_learning_phase and \
not isinstance(K.learning_phase(), int):
ins += [0.]
outputs = predict_and_test_function(ins)
if num_outputs == 1:
predictions = outputs[0:1]
val_metrics = outputs[1:]
else:
predictions = outputs[0:2]
val_metrics = outputs[2:]
## Save images
#def process_slice(s):
#s = np.squeeze(s).copy()
#s[s<0]=0
#s[s>1]=1
#s[0,0]=1
#s[0,1]=0
#return s
#for i in range(len(batch[0])):
#s_pred_list = []
#if num_outputs==1:
#s_pred_list = [process_slice(predictions[i])]
#else:
#for j in range(num_outputs):
#s_pred_list.append(process_slice(predictions[j][i]))
#s_input = process_slice(batch[0][i])
#if num_outputs==1:
#s_truth = process_slice(batch[1][i]/2.)
#else:
#s_truth = process_slice(batch[1][0][i]/2.)
#out_image = np.concatenate([s_input]+s_pred_list+[s_truth],
#axis=1)
#imsave(os.path.join(save_predictions_to,
#"{}_{}.png".format(batch_num, i)),
#out_image)
# Update metrics
val_logs = OrderedDict(zip(model.metrics_names, val_metrics))
val_logs.update({'batch': batch_num, 'size': len(batch[0])})
val_callbacks.on_batch_end(batch_num, val_logs)
# Update metrics
val_callbacks.on_epoch_end(0, val_logs)
# Output metrics
for m in val_logs:
if m not in ['batch', 'size']:
print("{}: {}".format(m, val_logs[m]))
def fit(self,
x,
y,
batch_size=None,
nsteps=None,
epochs=1,
verbose=1,
callbacks=None,
validation_data=None):
assert self.is_compiled, "Must compile model first"
assert epochs > 0
x = x if type(x) is list else [x]
y = y if type(y) is list else [y]
if nsteps is None:
total_len = len(y[0]) if type(y) is list else len(y)
nsteps = total_len // batch_size
# BaseLogger should always be the first metric since it computes the stats on epoch end
base_logger = BaseLogger(
stateful_metrics=["val_%s" % m for m in self.metrics_name] +
['val_loss', 'size'])
base_logger_params = {'metrics': ['loss'] + self.metrics_name}
if validation_data:
base_logger_params['metrics'] += [
'val_%s' % m for m in base_logger_params['metrics']
]
base_logger.set_params(base_logger_params)
hist = History()
if callbacks is None:
callbacks = [base_logger] + [hist]
elif type(callbacks) is list:
callbacks = [base_logger] + callbacks + [hist]
else:
callbacks = [base_logger] + [callbacks] + [hist]
callback_list = CallbackList(callbacks=callbacks)
callback_list.set_model(self)
callback_list.on_train_begin()
self.callbacks = callback_list
for epoch in range(epochs):
g = batchify(x, y, batch_size) if batch_size else None
t = trange(nsteps) if verbose == 1 else range(nsteps)
callback_list.on_epoch_begin(epoch)
for it in t:
x_, y_ = next(g) if g else (None, None)
batch_logs = self.train_on_batch(x_, y_)
callback_list.on_batch_end(it, batch_logs)
curr_loss = base_logger.totals['loss'] / base_logger.seen
if verbose == 1:
t.set_postfix(loss="%.4f" % curr_loss)
if verbose == 2:
if it % 1000 == 0:
print("%s %i/%i, loss=%.5f" %
(datetime.datetime.now().strftime("%H:%M:%S"),
it, nsteps, curr_loss),
flush=True)
if validation_data:
val_logs = self.evaluate(validation_data[0],
validation_data[1])
base_logger.on_batch_end(None, val_logs)
epoch_logs = {}
callback_list.on_epoch_end(epoch=epoch, logs=epoch_logs)
if verbose:
if validation_data:
to_print = ['loss'] + self.metrics_name + ['val_loss'] + [
'val_%s' % m for m in self.metrics_name
]
else:
to_print = ['loss'] + self.metrics_name
prog = ", ".join([
"%s=%.4f" % (name, hist.history[name][-1])
for name in to_print
])
print("Epoch %i, %s" % (epoch, prog), flush=True)
if self.stop_training:
break
return hist.history
def fit(self,
x,
y,
batch_size,
n_epochs=1,
callbacks=None,
validation_data=None):
"""Trains the network on the given data for a fixed number of epochs
:param x: input data to train on
:type x: torch.Tensor
:param y: target data to train on
:type y: torch.Tensor
:param batch_size: number of samples to use per forward and backward
pass
:type batch_size: int
:param n_epochs: number of epochs (iterations of the dataset) to train
the model
:type n_epochs: int
:param callbacks: callbacks to be used during training
:type callbacks: list[object]
:param validation_data: data on which to evaluate the loss and metrics
at the end of each epoch
:type validation_data: tuple(numpy.ndarray)
"""
default_callbacks = self._load_default_callbacks()
default_callbacks.append(ProgbarLogger(count_mode='samples'))
if callbacks:
default_callbacks.extend(callbacks)
callbacks = CallbackList(default_callbacks)
self._assert_compiled()
if self.device:
self.network.to(self.device)
metrics = ['loss']
if self.n_outputs > 1:
for idx_output in range(1, self.n_outputs + 1):
metrics.append('loss{}'.format(idx_output))
if validation_data is not None:
metrics.append('val_loss')
if self.n_outputs > 1:
for idx_output in range(1, self.n_outputs + 1):
metrics.append('val_loss{}'.format(idx_output))
for metric_name in self.metric_names:
metrics.append(metric_name)
if validation_data is not None:
metrics.append('val_{}'.format(metric_name))
index_array = np.arange(x.shape[0])
callbacks.set_params({
'batch_size': batch_size,
'epochs': n_epochs,
'metrics': metrics,
'steps': None,
'samples': x.shape[0],
'verbose': True
})
callbacks.set_model(self)
callbacks.on_train_begin()
for idx_epoch in range(n_epochs):
if self.stop_training:
break
epoch_logs = {}
callbacks.on_epoch_begin(idx_epoch)
np.random.shuffle(index_array)
batches = make_batches(len(index_array), batch_size)
for idx_batch, (idx_start, idx_end) in enumerate(batches):
batch_logs = {'batch': idx_batch, 'size': idx_end - idx_start}
callbacks.on_batch_begin(idx_batch, batch_logs)
inputs = x[index_array[idx_start:idx_end]]
if self.n_outputs > 1:
targets = []
for idx_output in range(self.n_outputs):
targets.append(
y[idx_output][index_array[idx_start:idx_end]])
else:
targets = y[index_array[idx_start:idx_end]]
train_outputs = self.train_on_batch(inputs, targets)
batch_logs['loss'] = train_outputs[0]
if self.n_outputs > 1:
for idx_output in range(1, self.n_outputs + 1):
batch_logs['loss{}'.format(idx_output)] = (
train_outputs[idx_output])
idx_metric_values = (1 if self.n_outputs == 1 else
self.n_outputs + 1)
it = zip(self.metric_names, train_outputs[idx_metric_values:])
for metric_name, train_output in it:
batch_logs[metric_name] = train_output
callbacks.on_batch_end(idx_batch, batch_logs)
if self.stop_training:
break
if validation_data:
val_outputs = self.evaluate(validation_data[0],
validation_data[1], batch_size)
epoch_logs['val_loss'] = val_outputs[0]
if self.n_outputs > 1:
for idx_output in range(1, self.n_outputs + 1):
epoch_logs['val_loss{}'.format(idx_output)] = (
val_outputs[idx_output])
idx_metric_values = (1 if self.n_outputs == 1 else
self.n_outputs + 1)
it = zip(self.metric_names, val_outputs[idx_metric_values:])
for metric_name, val_output in it:
metric_name = 'val_{}'.format(metric_name)
epoch_logs[metric_name] = val_output
callbacks.on_epoch_end(idx_epoch, epoch_logs)
callbacks.on_train_end()
def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
log_path = 'logging/vae_nlg_{}'.format(int(round(time.time() * 1000)))
os.mkdir(log_path)
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='{}/evolution.log'.format(log_path),
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
batch_size = config_data['batch_size']
epochs = config_data['nb_epochs']
discriminator_iterations = config_data['discriminator_iterations']
tweets_path = config_data['tweets_path']
vocab_path = config_data['vocab_path']
vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb'))
#== == == == == == =
# Load all the Data
#== == == == == == =
delimiter = ''
noutputs = 11
logging.info('Load Training Data')
train_input, train_output, train_weights, train_lex = load_text_gen_data(
join(tweets_path, 'trainset.csv'),
config_data,
vocab,
noutputs,
word_based=False)
logging.info('Load Validation Data')
valid_input, valid_output, _, valid_lex = load_text_gen_data(
join(tweets_path, 'devset.csv'),
config_data,
vocab,
noutputs,
word_based=False)
logging.info('Load Output Validation Data')
valid_dev_input, valid_dev_output, _, valid_dev_lex = load_text_gen_data(
join(tweets_path, 'devset_reduced.csv'),
config_data,
vocab,
noutputs,
random_output=False,
word_based=False)
step = K.variable(1., name='step_varialbe')
steps_per_epoch = ceil(train_output[0].shape[0] /
config_data['batch_size'])
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
vae_model_train, vae_model_test, vae_vanilla_train_model, vae_vanilla_test_model, discriminator_model, decoder_test, discriminator = get_vae_gan_model(
config_data, vocab, step)
with open(os.path.join(log_path, 'models.txt'), 'wt') as fh:
fh.write('VAE Model Train\n')
fh.write('---------\n')
vae_model_train.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Test\n')
fh.write('--------------\n')
vae_model_test.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Pretrain\n')
fh.write('---------------------------\n')
vae_vanilla_train_model.summary(
print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Pretrain Test\n')
fh.write('---------------------------\n')
vae_vanilla_test_model.summary(
print_fn=lambda x: fh.write(x + '\n'))
fh.write('Decoder Test\n')
fh.write('-------------------\n')
decoder_test.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Discriminator Models\n')
fh.write('-------------------\n')
discriminator_model.summary(print_fn=lambda x: fh.write(x + '\n'))
terminate_on_nan = TerminateOnNaN()
output_callback = LexOutputCallback(
vae_vanilla_test_model,
valid_dev_input,
valid_dev_lex,
1,
vocab,
delimiter,
fname='{}/test_output'.format(log_path))
#output_callback_full = LexOutputCallback(vae_vanilla_test_model, valid_dev_input, valid_dev_lex, 1, vocab, delimiter, fname='{}/test_output'.format(log_path))
#
# vae_vanilla_train_model.fit_generator(
# generator=generate_data_stream(config_data['pretrain_path'], config_data, vocab, config_data['batch_size'], noutputs=3),
# steps_per_epoch=steps_per_epoch,
# epochs=ceil(config_data['pretrain_samples']/config_data['pretrain_samples_per_epoch']),
# callbacks=[output_callback, terminate_on_nan],
# validation_data=(valid_input, valid_output[:3]),
# )
vae_vanilla_train_model.fit(
x=train_input,
y=train_output[:2],
epochs=config_data['pretrain_epochs'],
batch_size=batch_size,
validation_data=(valid_input, valid_output[:2]),
sample_weight=train_weights[:2],
callbacks=[output_callback, terminate_on_nan])
terminate_on_nan = TerminateOnNaN()
model_checkpoint = ModelCheckpoint(
'models/vae_model/weights.{epoch:02d}.hdf5',
period=10,
save_weights_only=True)
out_labels = [
'enc_' + s for s in vae_model_train._get_deduped_metrics_names()
]
out_labels += [
'dis_' + s
for s in discriminator_model._get_deduped_metrics_names()
]
callback_metrics = out_labels + ['val_' + n for n in out_labels]
tensorboard = TensorBoard(log_dir='logging/tensorboard',
histogram_freq=0,
write_grads=True,
write_images=True)
step_callback = StepCallback(step, steps_per_epoch)
output_callback = LexOutputCallback(
vae_vanilla_test_model,
valid_dev_input,
valid_dev_lex,
1,
vocab,
delimiter,
fname='{}/test_output'.format(log_path))
output_callback_full = LexOutputCallback(
vae_vanilla_test_model,
valid_input,
valid_lex,
5,
vocab,
delimiter,
fname='{}/test_valid_output'.format(log_path))
callbacks = CallbackList([
BaseLogger(),
ProgbarLogger(count_mode='steps'), step_callback, tensorboard,
output_callback, output_callback_full, model_checkpoint,
terminate_on_nan
])
callbacks.set_model(vae_model_train)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': True,
'do_validation': True,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
initial_epoch = 0
num_train_samples = train_input[0].shape[0]
index_array = np.arange(num_train_samples)
steps = 0
epoch = initial_epoch
while epoch < epochs:
epoch_logs = {}
callbacks.on_epoch_begin(epoch)
index_array = _batch_shuffle(index_array, batch_size)
steps_done = 0
batches = _make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_logs = {}
batch_ids = index_array[batch_start:batch_end]
X = _slice_arrays(train_input, batch_ids)
y = _slice_arrays(train_output, batch_ids)
sample_weights = _slice_arrays(train_weights, batch_ids)
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
set_trainability(discriminator, trainable=False)
enc_outs = vae_model_train.train_on_batch(
x=X, y=y, sample_weight=sample_weights)
set_trainability(discriminator, trainable=True)
list_disc_loss_real = []
if steps < 25 or steps % 500 == 0:
disc_iterations = 25
else:
disc_iterations = discriminator_iterations
for disc_it in range(disc_iterations):
real_idx = np.random.choice(train_input[0].shape[0],
len(batch_ids),
replace=False)
disX_train = train_input[-1][
real_idx] #take input 8 as train input and the rest as targets
disy_train = [x[real_idx] for x in train_input[:8]
] #take input 1-7 as targets
#train on real data
dis_outs_real = discriminator_model.train_on_batch(
disX_train, disy_train)
list_disc_loss_real.append(dis_outs_real)
loss_d_real = np.mean(list_disc_loss_real, axis=0)
outs = np.concatenate((enc_outs, loss_d_real))
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
batch_index += 1
steps_done += 1
steps += 1
# Epoch finished.
if steps_done >= steps_per_epoch:
valid_len = valid_output[0].shape[0]
enc_val_outs = vae_model_train.evaluate(valid_input,
valid_output,
verbose=False)
dis_val_outs = discriminator_model.evaluate(
valid_input[-1], valid_input[:8], verbose=False)
val_outs = enc_val_outs + dis_val_outs
#val_outs = full_model.evaluate(valid_input, valid_output, verbose=False)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
callbacks.on_train_end()
def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='logging/vae_gan/evolution.log',
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
tweets_path = config_data['tweets_path']
vocab_path = config_data['vocab_path']
vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb'))
#== == == == == == =
# Load all the Data
#== == == == == == =
noutputs = 5
logging.info('Load Training Data')
train_input, train_output = load_data(
join(tweets_path, 'en_train.tsv'), config_data, vocab, noutputs)
logging.info('Load Validation Data')
valid_input, valid_output = load_data(
join(tweets_path, 'en_valid15.tsv'), config_data, vocab, noutputs)
logging.info('Load Validation Data')
valid_input2, valid_output2 = load_data(
join(tweets_path, 'en_test16.tsv'), config_data, vocab, noutputs)
logging.info('Load Test Data')
test_input, test_output = load_data(join(tweets_path, 'en_test17.tsv'),
config_data, vocab, noutputs)
step = K.variable(1.)
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
full_model, encoding_train_model, decoder_train_model, discriminator_train_model, decoder_inference, encoder, decoder, discriminator, discriminator_pretrain_model = vae_gan_model(
config_data, vocab, step)
#full_model.summary()
encoding_train_model.summary()
decoder_train_model.summary()
discriminator_train_model.summary()
decoder_inference.summary()
encoder.summary()
decoder.summary()
discriminator.summary()
#pretrain_discriminator(discriminator_pretrain_model, train_input, vocab)
model_path = 'models/vae_model/'
steps_per_epoch = int(
ceil(config_data['samples_per_epoch'] / config_data['batch_size']))
epochs = int(
ceil(config_data['nb_epochs'] *
(config_data['nsamples'] / config_data['samples_per_epoch'])))
batch_size = config_data['batch_size']
initial_epoch = 0
skip_texts = 0
terminate_on_nan = TerminateOnNaN()
model_checkpoint = ModelCheckpoint(
'models/vae_model/weights.{epoch:02d}.hdf5',
period=10,
save_weights_only=True)
generator = generate_data_stream(config_data['training_path'],
config_data,
vocab,
config_data['batch_size'],
skip_data=skip_texts,
noutputs=noutputs)
enqueuer = GeneratorEnqueuer(generator,
use_multiprocessing=False,
wait_time=0.01)
enqueuer.start(workers=1, max_queue_size=10)
output_generator = enqueuer.get()
enc_out_labels = [
'enc_' + s
for s in encoding_train_model._get_deduped_metrics_names()
]
dec_out_labels = [
'dec_' + s
for s in decoder_train_model._get_deduped_metrics_names()
]
dis_out_labels = [
'dis_' + s
for s in discriminator_train_model._get_deduped_metrics_names()
]
out_labels = enc_out_labels + dec_out_labels + dis_out_labels
#out_labels = full_model._get_deduped_metrics_names()
callback_metrics = out_labels + ['val_' + n for n in out_labels]
step_callback = NewCallback(step, steps_per_epoch)
output_callback = OutputCallback(decoder_inference, valid_input, 15,
vocab, '')
callbacks = CallbackList([
BaseLogger(),
ProgbarLogger(count_mode='steps'), step_callback, output_callback
])
callbacks.set_model(full_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': True,
'do_validation': True,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
epoch = initial_epoch
while epoch < epochs:
epoch_logs = {}
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
X, y = next(output_generator)
callbacks.on_batch_begin(batch_index, batch_logs)
set_trainability(encoder, trainable=True)
set_trainability(decoder, trainable=False)
set_trainability(discriminator, trainable=False)
enc_outs = encoding_train_model.train_on_batch(X, y[:3])
set_trainability(encoder, trainable=False)
set_trainability(decoder, trainable=True)
set_trainability(discriminator, trainable=False)
dec_outs = decoder_train_model.train_on_batch(X, y[:4])
set_trainability(encoder, trainable=False)
set_trainability(decoder, trainable=False)
set_trainability(discriminator, trainable=True)
dis_outs = discriminator_train_model.train_on_batch(X, y[0])
outs = enc_outs + dec_outs + [dis_outs]
#outs = full_model.train_on_batch(X, y)
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
batch_index += 1
steps_done += 1
# Epoch finished.
if steps_done >= steps_per_epoch:
enc_val_outs = encoding_train_model.evaluate(
valid_input, valid_output[:3], verbose=False)
dec_val_outs = decoder_train_model.evaluate(
valid_input, valid_output[:4], verbose=False)
dis_val_outs = discriminator_train_model.evaluate(
valid_input, valid_output[0], verbose=False)
val_outs = enc_val_outs + dec_val_outs + [dis_val_outs]
#val_outs = full_model.evaluate(valid_input, valid_output, verbose=False)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
callbacks.on_train_end()
def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
log_path = 'logging/vae_nlg_{}'.format(int(round(time.time() * 1000)))
os.mkdir(log_path)
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='{}/evolution.log'.format(log_path),
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
batch_size = config_data['batch_size']
epochs = config_data['nb_epochs']
discriminator_iterations = config_data['discriminator_iterations']
tweets_path = config_data['tweets_path']
vocab_path = config_data['vocab_path']
vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb'))
#== == == == == == =
# Load all the Data
#== == == == == == =
noutputs = 5
logging.info('Load Training Data')
train_input, train_output = load_text_pairs(
join(tweets_path, 'training_set.tsv'), config_data, vocab,
noutputs)
logging.info('Load Validation Data')
valid_input, valid_output = load_text_pairs(
join(tweets_path, 'vaild_set.tsv'), config_data, vocab, noutputs)
logging.info('Load Output Validation Data')
valid_dev_input, valid_dev_output = load_text_pairs(
join(tweets_path, 'test_set.tsv'), config_data, vocab, noutputs)
#train_input = [x[:1213] for x in train_input]
#train_output = [x[:1213] for x in train_output]
noise_valid_input = np.zeros(shape=(valid_input[0].shape[0],
config_data['z_size']))
step = K.variable(1.)
steps_per_epoch = ceil(train_output[0].shape[0] /
config_data['batch_size'])
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
vae_model, vae_model_test, decoder_discr_model, decoder_test_model, discriminator_model, discriminator = get_vae_gan_model(
config_data, vocab, step)
with open(os.path.join(log_path, 'models.txt'), 'wt') as fh:
fh.write('VAE Model\n')
fh.write('---------\n')
vae_model.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Test\n')
fh.write('--------------\n')
vae_model_test.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Decoder Discriminator Model\n')
fh.write('---------------------------\n')
decoder_discr_model.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Decoder Test Model\n')
fh.write('---------------------------\n')
decoder_test_model.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Discriminator Model\n')
fh.write('-------------------\n')
discriminator_model.summary(print_fn=lambda x: fh.write(x + '\n'))
terminate_on_nan = TerminateOnNaN()
model_checkpoint = ModelCheckpoint(
'models/vae_model/weights.{epoch:02d}.hdf5',
period=10,
save_weights_only=True)
enc_out_labels = [
'enc_' + s for s in vae_model._get_deduped_metrics_names()
]
dec_out_labels = [
'dec_' + s
for s in decoder_discr_model._get_deduped_metrics_names()
]
dis_out_labels = [
'dis_' + s
for s in discriminator_model._get_deduped_metrics_names()
]
out_labels = enc_out_labels + dec_out_labels + [
'dis_real', 'dis_gen', 'dis_noise'
]
#out_labels = full_model._get_deduped_metrics_names()
callback_metrics = out_labels + ['val_' + n for n in out_labels]
step_callback = StepCallback(step, steps_per_epoch)
output_callback = GANOutputCallback(
vae_model_test,
valid_dev_input[0],
1,
vocab,
'',
fname='{}/test_output'.format(log_path))
callbacks = CallbackList([
BaseLogger(),
ProgbarLogger(count_mode='steps'), step_callback, output_callback,
model_checkpoint, terminate_on_nan
])
callbacks.set_model(vae_model_test)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': True,
'do_validation': True,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
initial_epoch = 0
num_train_samples = train_input[0].shape[0]
index_array = np.arange(num_train_samples)
steps = 0
epoch = initial_epoch
while epoch < epochs:
epoch_logs = {}
callbacks.on_epoch_begin(epoch)
index_array = _batch_shuffle(index_array, batch_size)
steps_done = 0
batches = _make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_logs = {}
batch_ids = index_array[batch_start:batch_end]
X = _slice_arrays(train_input, batch_ids)
y = _slice_arrays(train_output, batch_ids)
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
set_trainability(discriminator, trainable=False)
enc_outs = vae_model.train_on_batch(x=X, y=y[:3])
set_trainability(discriminator, trainable=True)
list_disc_loss_real = []
list_disc_loss_gen = []
list_disc_loss_noise = []
if steps < 25 or steps % 500 == 0:
disc_iterations = 100
else:
disc_iterations = discriminator_iterations
noise_input = np.zeros(shape=(len(batch_ids),
config_data['z_size']))
for disc_it in range(disc_iterations):
#clip_weights(discriminator)
real_idx = np.random.choice(train_input[0].shape[0],
len(batch_ids),
replace=False)
train_real_batch = [x[real_idx] for x in train_input]
#train on real data
x_fake = vae_model_test.predict_on_batch(
x=train_real_batch[0])
x_noise_fake = decoder_test_model.predict_on_batch(
x=noise_input)
train_input_discr = np.concatenate(
(train_real_batch[0], train_real_batch[0],
train_real_batch[0]))
train_output_discr = np.concatenate(
(train_real_batch[1], x_fake, x_noise_fake))
labels = np.asarray(
len(batch_ids) * [1] + 2 * len(batch_ids) * [-1])
index_array_discr = np.arange(len(labels))
np.random.shuffle(index_array_discr)
discr_batch = [
train_input_discr[index_array_discr],
train_output_discr[index_array_discr]
]
discr_batch_labels = labels[index_array_discr]
dis_outs_real = discriminator_model.train_on_batch(
discr_batch, discr_batch_labels)
#dis_outs_real = discriminator_model.train_on_batch(train_real_batch, -np.ones(shape=(len(batch_ids), 1)))
#dis_outs_gen = discriminator_model.train_on_batch([train_real_batch[0], x_fake], np.ones(shape=(len(batch_ids), 1)))
#dis_outs_gen_noise = discriminator_model.train_on_batch([train_real_batch[0], x_noise_fake], np.ones(shape=(len(batch_ids), 1)))
list_disc_loss_real.append(dis_outs_real)
#list_disc_loss_gen.append(dis_outs_gen)
#list_disc_loss_noise.append(dis_outs_gen_noise)
loss_d_real = -np.mean(list_disc_loss_real)
loss_d_gen = np.mean(list_disc_loss_gen)
loss_d_noise = np.mean(list_disc_loss_noise)
set_trainability(discriminator, trainable=False)
decoder_discr_input = [X[0], noise_input]
dec_outs = decoder_discr_model.train_on_batch(
x=decoder_discr_input,
y=-np.ones(shape=(len(batch_ids), 1)))
outs = enc_outs + [dec_outs] + [loss_d_real]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
batch_index += 1
steps_done += 1
steps += 1
# Epoch finished.
if steps_done >= steps_per_epoch:
valid_len = valid_output[0].shape[0]
enc_val_outs = vae_model.evaluate(valid_input,
valid_output[:3],
verbose=False)
dec_val_outs = decoder_discr_model.evaluate(
[valid_input[0], noise_valid_input],
-np.ones(shape=(valid_len, 1)),
verbose=False)
dis_val_outs = discriminator_model.evaluate(
valid_input,
-np.ones(shape=(valid_len, 1)),
verbose=False)
val_outs = enc_val_outs + [dec_val_outs] + [dis_val_outs]
#val_outs = full_model.evaluate(valid_input, valid_output, verbose=False)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
callbacks.on_train_end()
'batch_size': batch_size,
'nb_epoch': nb_epoch,
'nb_sample': nb_train_sample,
'verbose': 1,
'do_validation': do_validation,
'metrics': metrics,
})
##########################
# TRAINING
##########################
callbacks.on_train_begin()
model.stop_training = False
for epoch in range(nb_epoch):
callbacks.on_epoch_begin(epoch)
if shuffle_on_epoch_start:
X_train, y_train = util.shuffle_data(X_train, y_train)
# train
util.train_on_batch(model, X_train, y_train, nb_classes,
callbacks=callbacks,
normalize=normalize_data,
batch_size=batch_size,
class_weight=class_weight,
shuffle=False)
epoch_logs = {}
##########################
def fit_generator(self,
generator,
n_steps_per_epoch,
n_epochs=1,
validation_data=None,
n_validation_steps=None,
callbacks=None):
"""Train the network on batches of data generated from `generator`
:param generator: a generator yielding batches indefinitely, where each
batch is a tuple of (inputs, targets)
:type generator: generator
:param n_steps_per_epoch: number of batches to train on in one epoch
:type n_steps_per_epoch: int
:param n_epochs: number of epochs to train the model
:type n_epochs: int
:param validation_data: generator yielding batches to evaluate the loss
on at the end of each epoch, where each batch is a tuple of (inputs,
targets)
:type validation_data: generator
:param n_validation_steps: number of batches to evaluate on from
`validation_data`
:param callbacks: callbacks to be used during training
:type callbacks: list[object]
:raises RuntimeError: if only one of `validation_data` and
`n_validation_steps` are passed in
"""
default_callbacks = self._load_default_callbacks()
default_callbacks.append(ProgbarLogger(count_mode='steps'))
if callbacks:
default_callbacks.extend(callbacks)
callbacks = CallbackList(default_callbacks)
self._assert_compiled()
invalid_inputs = ((validation_data is not None
and not n_validation_steps)
or (n_validation_steps and validation_data is None))
if invalid_inputs:
msg = ('`validation_data` and `n_validation_steps` must both be '
'passed, or neither.')
raise RuntimeError(msg)
if self.device:
self.network.to(self.device)
metrics = ['loss']
if self.n_outputs > 1:
for idx_output in range(1, self.n_outputs + 1):
metrics.append('loss{}'.format(idx_output))
if validation_data is not None:
metrics.append('val_loss')
if self.n_outputs > 1:
for idx_output in range(1, self.n_outputs + 1):
metrics.append('val_loss{}'.format(idx_output))
for metric_name in self.metric_names:
metrics.append(metric_name)
if validation_data is not None:
metrics.append('val_{}'.format(metric_name))
callbacks.set_params({
'epochs': n_epochs,
'metrics': metrics,
'steps': n_steps_per_epoch,
'verbose': True
})
callbacks.set_model(self)
callbacks.on_train_begin()
for idx_epoch in range(n_epochs):
if self.stop_training:
break
epoch_logs = {}
callbacks.on_epoch_begin(idx_epoch)
for idx_batch in range(n_steps_per_epoch):
batch_logs = {'batch': idx_batch, 'size': 1}
callbacks.on_batch_begin(idx_batch, batch_logs)
generator_output = next(generator)
if len(generator_output) != 2:
msg = ('Output of generator should be a tuple of '
'(inputs, targets), but instead got a {}: '
'{}.').format(type(generator_output),
str(generator_output))
inputs, targets = generator_output
train_outputs = self.train_on_batch(inputs, targets)
batch_logs['loss'] = train_outputs[0]
if self.n_outputs > 1:
for idx_output in range(1, self.n_outputs + 1):
batch_logs['loss{}'.format(idx_output)] = (
train_outputs[idx_output])
idx_metric_values = (1 if self.n_outputs == 1 else
self.n_outputs + 1)
it = zip(self.metric_names, train_outputs[idx_metric_values:])
for metric_name, train_output in it:
batch_logs[metric_name] = train_output
callbacks.on_batch_end(idx_batch, batch_logs)
if self.stop_training:
break
if validation_data:
val_outputs = self.evaluate_generator(validation_data,
n_validation_steps)
epoch_logs['val_loss'] = val_outputs[0]
if self.n_outputs > 1:
for idx_output in range(1, self.n_outputs + 1):
epoch_logs['val_loss{}'.format(idx_output)] = (
val_outputs[idx_output])
idx_metric_values = (1 if self.n_outputs == 1 else
self.n_outputs + 1)
it = zip(self.metric_names, val_outputs[idx_metric_values:])
for metric_name, val_output in it:
metric_name = 'val_{}'.format(metric_name)
epoch_logs[metric_name] = val_output
callbacks.on_epoch_end(idx_epoch, epoch_logs)
callbacks.on_train_end()
def train_wgan_with_grad_penalty(prior_gen,
generator,
data_gen,
critic,
batch_size,
epochs,
batches_per_epoch=100,
optimizer=Adam(lr=1e-4, beta_1=0, beta_2=0.9),
grad_pen_coef=10.,
critic_gen_train_ratio=2,
callbacks=None):
# build model to train the critic
data_shape = critic.input_shape[1:]
real_critic_input = Input(shape=data_shape, name='real_in')
fake_critic_input = Input(shape=data_shape, name='fake_in')
interp_critic_input = Input(shape=data_shape, name='interp_in')
real_critic_score = critic(real_critic_input)
fake_critic_score = critic(fake_critic_input)
interp_critic_score = critic(interp_critic_input)
critic_loss = subtract([fake_critic_score, real_critic_score])
gradient_penalty = GradPenLayer()(
[interp_critic_input, interp_critic_score])
critic_train_mdl = Model(
[real_critic_input, fake_critic_input, interp_critic_input],
[critic_loss, gradient_penalty])
critic_train_mdl.compile(optimizer=optimizer,
loss=lambda y_true, y_pred: y_pred,
loss_weights=[1., grad_pen_coef])
# build model to train generator
prior_input = Input(shape=generator.input_shape[1:], name='prior_in')
critic.trainable = False
critic_on_generator_score = critic(generator(prior_input))
generator_train_mdl = Model(prior_input, critic_on_generator_score)
generator_train_mdl.compile(optimizer=optimizer,
loss=lambda y_true, y_pred: -y_pred)
# init callbacks
callbacks = callbacks or []
callbacks = CallbackList(callbacks)
callbacks.set_model({'generator': generator, 'critic': critic})
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': batches_per_epoch,
'samples': batches_per_epoch * batch_size,
'prior_gen': prior_gen,
'data_gen': data_gen,
})
# train
print('Training on {} samples for {} epochs'.format(
batches_per_epoch * batch_size, epochs))
callbacks.on_train_begin()
for e in range(epochs):
print('Epoch {}/{}'.format(e + 1, epochs))
callbacks.on_epoch_begin(e)
progbar = Progbar(target=batches_per_epoch * batch_size)
dummy_y = np.array([None] * batch_size)
for b in range(batches_per_epoch):
callbacks.on_batch_begin(b)
batch_losses = np.zeros(shape=3)
for critic_upd in range(critic_gen_train_ratio):
real_batch = data_gen(batch_size)
fake_batch = generator.predict(prior_gen(batch_size))
weights = np.random.uniform(size=batch_size)
weights = weights.reshape((-1, ) + (1, ) *
(len(real_batch.shape) - 1))
interp_batch = weights * real_batch + (1. -
weights) * fake_batch
x_batch = {
'real_in': real_batch,
'fake_in': fake_batch,
'interp_in': interp_batch
}
cur_losses = np.array(
critic_train_mdl.train_on_batch(x=x_batch,
y=[dummy_y, dummy_y]))
batch_losses += cur_losses
generator_train_mdl.train_on_batch(x=prior_gen(batch_size),
y=dummy_y)
losses_names = ('total_loss', 'critic_loss', 'gradient_pen')
progbar.add(batch_size, zip(losses_names, batch_losses))
callbacks.on_batch_end(b)
progbar.update(batches_per_epoch * batch_size)
callbacks.on_epoch_end(e)
callbacks.on_train_end()
def fit_dataset(self,
dataset,
steps_per_epoch=None,
batch_size=32,
epochs=1,
verbose=1,
callbacks=None,
on_sample=None,
on_scores=None):
"""Train the model on the given dataset for a given number of epochs.
Arguments
---------
dataset: Instance of `BaseDataset` that provides the data
to train on.
steps_per_epoch: int or None, number of gradient updates before
considering an epoch has passed. If None it is set
to be `len(dataset.train_data) / batch_size`.
batch_size: int, number of samples per gradient update
epochs: int, number of times to iterate `steps_per_epoch` times
verbose: {0, >0}, whether to employ the progress bar Keras
callback or not
callbacks: list of Keras callbacks to be called during training
on_sample: callable that accepts the sampler, idxs, w, scores
on_scores: callable that accepts the sampler and scores
"""
try:
if len(dataset.train_data) < batch_size:
raise ValueError(("The model cannot be trained with "
"batch_size > training set"))
except RuntimeError as e:
assert "no size" in str(e)
# Set steps_per_epoch properly
if steps_per_epoch is None:
steps_per_epoch = len(dataset.train_data) // batch_size
# Create the callbacks list
self.history = History()
callbacks = [BaseLogger()] + (callbacks or []) + [self.history]
if verbose > 0:
callbacks += [ProgbarLogger(count_mode="steps")]
callbacks = CallbackList(callbacks)
#TODO: Should we be making it possible to call back a different model
# than self.model.model?
callbacks.set_model(self.model.model)
callbacks.set_params({
"epochs":
epochs,
"steps":
steps_per_epoch,
"verbose":
verbose,
"do_validation":
len(dataset.test_data) > 0,
"metrics":
self._get_metric_names() +
["val_" + n for n in self._get_metric_names()]
})
# Create the sampler
sampler = self.sampler(dataset, batch_size, steps_per_epoch, epochs)
# Start the training loop
epoch = 0
self.model.model.stop_training = False
callbacks.on_train_begin()
while epoch < epochs:
callbacks.on_epoch_begin(epoch)
for step in range(steps_per_epoch):
batch_logs = {"batch": step, "size": batch_size}
callbacks.on_batch_begin(step, batch_logs)
# Importance sampling is done here
idxs, (x, y), w = sampler.sample(batch_size)
# Train on the sampled data
loss, metrics, scores = self.model.train_batch(x, y, w)
# Update the sampler
sampler.update(idxs, scores)
values = map(lambda x: x.mean(), [loss] + metrics)
for l, o in zip(self._get_metric_names(), values):
batch_logs[l] = o
callbacks.on_batch_end(step, batch_logs)
if on_scores is not None:
on_scores(sampler, self._latest_scores)
if on_sample is not None:
on_sample(sampler, self._latest_sample_event["idxs"],
self._latest_sample_event["w"],
self._latest_sample_event["predicted_scores"])
if self.model.model.stop_training:
break
# Evaluate now that an epoch passed
epoch_logs = {}
if len(dataset.test_data) > 0:
val = self.model.evaluate(*dataset.test_data[:],
batch_size=batch_size)
epoch_logs = {
"val_" + l: o
for l, o in zip(self._get_metric_names(), val)
}
callbacks.on_epoch_end(epoch, epoch_logs)
if self.model.model.stop_training:
break
epoch += 1
callbacks.on_train_end()
return self.history
def fit(self,
train_dataset,
test_dataset=None,
batch_size=32,
epochs=1,
callbacks=None):
epoch_ae_loss_avg = tf.metrics.Mean()
epoch_dc_loss_avg = tf.metrics.Mean()
epoch_dc_acc_avg = tf.metrics.Mean()
epoch_gen_loss_avg = tf.metrics.Mean()
train_reconstruction_loss = tf.keras.metrics.Mean()
test_reconstruction_loss = tf.keras.metrics.Mean()
train_dataset = self._check_tf_dataset_instance(train_dataset,
batch_size=batch_size)
callbacks = CallbackList(callbacks)
callback_model = self._get_callback_model()
callbacks.set_model(callback_model)
for epoch in range(1, epochs + 1):
callbacks.on_epoch_begin(epoch)
epoch_logs = {}
for x_train in train_dataset:
ae_loss, dc_loss, dc_acc, gen_loss = self.train_step(x_train)
epoch_ae_loss_avg(ae_loss)
epoch_dc_loss_avg(dc_loss)
epoch_dc_acc_avg(dc_acc)
epoch_gen_loss_avg(gen_loss)
train_reconstruction_loss(
self.compute_reconstruction_error(x_train))
epoch_logs['ae_loss'] = epoch_ae_loss_avg.result().numpy()
epoch_logs['dc_loss'] = epoch_dc_loss_avg.result().numpy()
epoch_logs['dc_acc'] = epoch_dc_acc_avg.result().numpy()
epoch_logs['gen_loss'] = epoch_gen_loss_avg.result().numpy()
epoch_logs[
'train_reconstruction_error'] = train_reconstruction_loss.result(
).numpy()
epoch_ae_loss_avg.reset_states()
epoch_dc_loss_avg.reset_states()
epoch_dc_acc_avg.reset_states()
epoch_gen_loss_avg.reset_states()
train_reconstruction_loss.reset_states()
if test_dataset is not None:
test_dataset = self._check_tf_dataset_instance(
test_dataset, batch_size=batch_size)
for x_test in test_dataset:
test_reconstruction_loss(
self.compute_reconstruction_error(x_test))
epoch_logs[
'test_reconstruction_error'] = test_reconstruction_loss.result(
).numpy()
test_reconstruction_loss.reset_states()
callbacks.on_epoch_end(epoch, logs=epoch_logs)
'batch_size': batch_size,
'nb_epoch': nb_epoch,
'nb_sample': nb_train_sample,
'verbose': 1,
'do_validation': do_validation,
'metrics': metrics,
})
##########################
# TRAINING
##########################
callbacks.on_train_begin()
model.stop_training = False
for epoch in range(nb_epoch):
callbacks.on_epoch_begin(epoch)
if shuffle_on_epoch_start:
X_train, y_train = util.shuffle_data(X_train, y_train)
# train
util.train_on_batch(model,
X_train,
y_train,
nb_classes,
callbacks=callbacks,
normalize=normalize_data,
batch_size=batch_size,
class_weight=class_weight,
shuffle=False)
def predict(model,
batch_size,
num_outputs,
save_path,
evaluate=False,
liver_only=False,
save_predictions=False,
initial_epoch=0,
**kwargs):
model, callbacks, gen = prepare_model(model=model,
num_outputs=num_outputs,
liver_only=liver_only,
evaluate=evaluate,
**kwargs)
# Set up prediction file.
if save_predictions:
save_path = os.path.join(save_path, "predictions.zarr")
if os.path.exists(save_path):
os.remove(save_path)
# Initialize callbacks
val_callback_list = [BaseLogger()]
if not liver_only:
val_callback_list.extend(
[callbacks['dice_lesion'], callbacks['dice_lesion_inliver']])
if len(model.outputs) == 2 or liver_only:
val_callback_list.append(callbacks['dice_liver'])
val_callbacks = CallbackList(val_callback_list)
val_callbacks.set_params({
'nb_epoch': 0,
'nb_sample': 0,
'verbose': False,
'do_validation': True,
'metrics': model.metrics_names
})
val_callbacks.on_train_begin()
val_callbacks.on_epoch_begin(0)
# Create theano function
if evaluate:
inputs = model.inputs + model.targets + model.sample_weights
if model.uses_learning_phase and \
not isinstance(K.learning_phase(), int):
inputs += [K.learning_phase()]
predict_function = K.function(inputs,
model.outputs + [model.total_loss] +
model.metrics_tensors,
updates=model.state_updates)
else:
inputs = model.inputs
if model.uses_learning_phase and \
not isinstance(K.learning_phase(), int):
inputs += [K.learning_phase()]
predict_function = K.function(inputs,
model.outputs,
updates=model.state_updates)
# Predict for all data.
print(' > Predicting...')
for key in gen:
print(' - DATA: {}'.format(key))
# Duplicate inputs and outputs (and add outputs) as necessary.
flow = repeat_flow(gen[key].flow(), num_outputs=num_outputs)
# Set up file.
if save_predictions:
zgroup = zarr.open_group(store=save_path, mode='a', path="/")
zarr_kwargs = {
'chunks': (1, 512, 512),
'compressor': zarr.Blosc(cname='lz4', clevel=9, shuffle=1)
}
# Predict and write to file.
batch_num = 0
for vol_num, volume in enumerate(flow):
print("Predicting on `{}` - {}/{}"
"".format(key, vol_num + 1, len(gen[key])))
# Begin writing to file.
if save_predictions:
vol_idx = volume[-1]
subgroup = zgroup.create_group(str(vol_idx))
num_channels = np.sum(model.output_shape[i][1] \
for i in range(num_outputs))
output_shape = \
(len(volume[0]), num_channels)+model.output_shape[0][2:]
subgroup.empty("volume",
shape=output_shape,
dtype=np.float32,
**zarr_kwargs)
segmentation = volume[1]
if isinstance(segmentation, list):
segmentation = segmentation[0]
subgroup.create_dataset("segmentation",
shape=segmentation.shape,
data=segmentation,
dtype=np.int16,
**zarr_kwargs)
# Iterate through volume batch-wise.
for idx0, idx1 in zip(
range(0, len(volume[0]), batch_size),
range(batch_size,
len(volume[0]) + batch_size + 1, batch_size)):
# Prepare data for joint evaluation and prediction.
if evaluate:
batch = (volume[0][idx0:idx1], volume[1][idx0:idx1])
x, y, sample_weights = model._standardize_user_data(
batch[0], batch[1])
ins = x + y + sample_weights
else:
batch = (volume[0][idx0:idx1], )
ins = _standardize_input_data(batch[0],
model._feed_input_names,
model._feed_input_shapes,
check_batch_axis=False,
exception_prefix='input')
if model.uses_learning_phase and \
not isinstance(K.learning_phase(), int):
ins += [0.]
# Jointly evaluate and predict.
outputs = predict_function(ins)
if num_outputs == 1:
predictions = outputs[0:1]
if evaluate:
val_metrics = outputs[1:]
elif num_outputs == 2:
predictions = outputs[0:2]
if evaluate:
val_metrics = outputs[2:]
else:
raise ValueError("num_outputs must be 1 or 2")
# Write predictions.
predictions = np.concatenate(predictions, axis=1)
subgroup['volume'][idx0:idx1] = predictions
# Update metrics
if evaluate:
val_logs = OrderedDict(
zip(model.metrics_names, val_metrics))
val_logs.update({
'batch': batch_num,
'size': len(batch[0])
})
val_callbacks.on_batch_end(batch_num, val_logs)
batch_num += 1
if evaluate:
# Update metrics
val_callbacks.on_epoch_end(0, val_logs)
# Output metrics
for m in val_logs:
if m not in ['batch', 'size']:
print("{}: {}".format(m, val_logs[m]))