def test_validate_callbacks_predefined_callbacks(self):
supported_predefined_callbacks = [
callbacks.TensorBoard(),
callbacks.CSVLogger(filename='./log.csv'),
callbacks.EarlyStopping(),
callbacks.ModelCheckpoint(filepath='./checkpoint'),
callbacks.TerminateOnNaN(),
callbacks.ProgbarLogger(),
callbacks.History(),
callbacks.RemoteMonitor()
]
distributed_training_utils_v1.validate_callbacks(
supported_predefined_callbacks, adam.Adam())
unsupported_predefined_callbacks = [
callbacks.ReduceLROnPlateau(),
callbacks.LearningRateScheduler(schedule=lambda epoch: 0.001)
]
for callback in unsupported_predefined_callbacks:
with self.assertRaisesRegex(
ValueError, 'You must specify a Keras Optimizer V2'):
distributed_training_utils_v1.validate_callbacks(
[callback], tf.compat.v1.train.AdamOptimizer())
def __init__(self,
model,
optimizer,
comm,
batch_iterator,
batch_size,
num_replicas=None,
warmup_steps=1000,
lr=0.01,
num_batches_minimum=100):
# random.seed(task_index)
self.epoch = 0
self.num_so_far = 0
self.num_so_far_accum = 0
self.num_so_far_indiv = 0
self.model = model
self.optimizer = optimizer
self.max_lr = 0.1
self.DUMMY_LR = 0.001
self.comm = comm
self.batch_size = batch_size
self.batch_iterator = batch_iterator
self.set_batch_iterator_func()
self.warmup_steps = warmup_steps
self.num_batches_minimum = num_batches_minimum
self.num_workers = comm.Get_size()
self.task_index = comm.Get_rank()
self.history = cbks.History()
if num_replicas is None or num_replicas < 1 or num_replicas > self.num_workers:
self.num_replicas = self.num_workers
else:
self.num_replicas = num_replicas
self.lr = lr / (1.0 + self.num_replicas / 100.0) if (
lr < self.max_lr) else self.max_lr / (1.0 +
self.num_replicas / 100.0)
def setup_callback_list(self, model_name):
if model_name in self.callback_lists:
return self.callback_lists[model_name]
model = self.models[model_name]
callbacks = self.callbacks[model_name] \
if model_name in self.callbacks else []
# Prepare callbacks for autoencoder model
all_callbacks = [cbks.BaseLogger()] + callbacks + [cbks.History()]
all_callbacks = cbks.CallbackList(all_callbacks)
out_labels = model.metrics_names
if self.do_validation:
callback_metrics = copy.copy(out_labels) + \
["val_" + l for l in out_labels]
else:
callback_metrics = copy.copy(out_labels)
callback_list = cbks.CallbackList(all_callbacks)
callback_list.set_params({
'batch_size': self.batch_size,
'epochs': self.epochs,
'verbose': 2,
'do_validation': model_name in self.do_validation,
'metrics': callback_metrics or [],
})
callback_list.set_model(model)
return callback_list
def cb():
# pl = callbacks.ProgbarLogger(count_mode='steps')
history = callbacks.History()
ch = callbacks.ModelCheckpoint(
'./weights/weights.{epoch:02d}-{val_loss:.2f}.hdf5',
monitor='val_loss',
verbose=0,
save_best_only=False,
save_weights_only=False,
mode='auto',
period=1)
es = callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=3,
verbose=0,
mode='auto')
cb = [history, ch, es]
#学習率の変更をしたいならこれ
#keras.callbacks.LearningRateScheduler(schedule, verbose=0)
#テンソルボードに記述したいならこれ
#keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=32, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None)
#評価値の改善が止まった時に学習率を減らします．
#モデルは訓練が停滞した時に学習率を2〜10で割ることで恩恵を受けることがあります． このコールバックは評価値を監視し，'patience'で指定されたエポック数の間改善が見られなかった場合，学習率を減らします．
#keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=10, verbose=0, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0)
return cb, history
def __init__(self, filepath, verbose=0, period=1, targets=None, is_each=True):
super(HistoryCheckpoint, self).__init__()
self.__verbose = verbose
self.__filepath = filepath
self.__period = period
self.__epochs_since_last_save = 0
self.__history_callback = KC.History()
self.__targets = [TargetHistory.Loss]
if isinstance(targets, list):
self.__targets = targets
self.__is_each = is_each
def get_callbacks(self, opt):
# ModelCheckpoints: saving model after each epoch
fn1 = (os.path.basename(self.model_params_file_path).replace(
'.json', ''))
fn = (f'{fn1}___{self.start_time}'
f'___model_%s{"_TEST" if opt.test else ""}.h5' % ('{epoch:02d}'))
filepath = os.path.join(self.path_nn_model, self.model.name, fn)
del fn
checkpoint = callbacks.ModelCheckpoint(filepath,
monitor='val_loss',
verbose=opt.verbose)
# TerminateOnNaN
tonan = callbacks.TerminateOnNaN()
# History
history = callbacks.History()
# CSV logger: saves epoch train and valid loss to a log file
fn1 = (os.path.basename(self.model_params_file_path).replace(
'.json', ''))
fn = (f'{fn1}___{self.start_time}'
f'___training{"_TEST" if opt.test else ""}.log')
filepath = os.path.join(self.path_nn_model, self.model.name, fn)
csv_logger = callbacks.CSVLogger(filepath, separator=',', append=True)
# Learning rate scheduler
def exp_decay(epoch,
initial_lrate=self.model_params['keras_train']['lr'],
decay=self.model_params['keras_train']['lr_decay']):
lrate = initial_lrate * np.exp(-decay * epoch)
return lrate
def learning_rate_decay(
epoch,
initial_lrate=self.model_params['keras_train']['lr'],
decay=self.model_params['keras_train']['lr_decay']):
lrate = initial_lrate * (1 - decay)**epoch
return lrate
lrs = callbacks.LearningRateScheduler(learning_rate_decay)
callbacks_list = [
tonan, checkpoint, history, csv_logger, csv_logger, lrs
]
# Early stopping: stops training if validation loss does not improves
if (self.model_params['keras_train'].get('early_stopping_n')
is not None):
es = callbacks.EarlyStopping(
monitor='val_loss',
min_delta=0,
patience=self.model_params['keras_train']['early_stopping_n'],
verbose=opt.verbose)
callbacks_list.append(es)
return callbacks_list
def fit(self, freeze_indices, optimizers, warmup_epochs=5):
# callbacks
filepath = 'models/transfer_CNN_reg.h5'
mc = callbacks.ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
hist = callbacks.History()
es = callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=2,
verbose=1,
mode='auto')
if not os.path.exists('tensorboard_logs/transfer_CNN_tensorboard_reg'):
os.makedirs('tensorboard_logs/transfer_CNN_tensorboard_reg')
tensorboard = callbacks.TensorBoard(
log_dir='tensorboard_logs/transfer_CNN_tensorboard_reg',
histogram_freq=0,
batch_size=self.batch_size,
write_graph=True,
embeddings_freq=0,
write_images=False)
# change head from default
self._create_transfer_model()
# train head, then chunks
histories = []
for i, freeze in enumerate(freeze_indices):
if i == 0:
e = warmup_epochs
opt = optimizers[0]
else:
e = self.epochs
opt = optimizers[1]
self._change_trainable_layers(freeze)
self.model.compile(optimizer=opt, loss=root_mean_squared_error)
history = self.model.fit_generator(
self.train_generator,
steps_per_epoch=len(self.train_generator),
epochs=e,
validation_data=self.validation_generator,
validation_steps=len(self.validation_generator),
callbacks=[mc, tensorboard, hist])
histories.append(history.history)
return histories
def _fit(self,
f,
nb_train_sample,
nb_batches,
batch_size=128,
nb_epoch=100,
verbose=1,
callbacks=[],
shuffle=True,
metrics=[]):
"""
Abstract fit function for f(*ins). Assume that f returns a list,
labelled by out_labels. """
history = cbks.History()
callbacks = [cbks.BaseLogger()] + callbacks + [history]
if verbose:
callbacks = callbacks + [cbks.ProgbarLogger()]
callbacks = cbks.CallbackList(callbacks)
callbacks._set_model(self)
callbacks._set_params({
'batch_size': nb_train_sample // nb_batches,
'nb_epoch': nb_epoch,
'nb_sample': nb_train_sample,
'verbose': verbose,
'do_validation': False,
'metrics': metrics,
})
callbacks.on_train_begin()
self.stop_training = False
for epoch in range(nb_epoch):
callbacks.on_epoch_begin(epoch)
for batch_index in range(nb_batches):
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
f(self, batch_index, batch_logs)
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
callbacks.on_epoch_end(epoch, epoch_logs)
if self.stop_training:
break
callbacks.on_train_end()
return history
def train_model(input_to_softmax,
pickle_path,
save_model_path,
train_json='train_corpus.json',
valid_json='valid_corpus.json',
minibatch_size=16, # You will want to change this depending on the GPU you are training on
spectrogram=True,
mfcc_dim=13,
optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False, clipnorm=1, clipvalue=.5),
epochs=30, # You will want to change this depending on the model you are training and data you are using
verbose=1,
sort_by_duration=False,
max_duration=10.0):
# Obtain batches of data
audio_gen = AudioGenerator(minibatch_size=minibatch_size,
spectrogram=spectrogram, mfcc_dim=mfcc_dim, max_duration=max_duration,
sort_by_duration=sort_by_duration)
# Load the datasets
audio_gen.load_train_data(train_json)
audio_gen.load_validation_data(valid_json)
# Calculate steps per epoch
num_train_examples=len(audio_gen.train_audio_paths)
steps_per_epoch = num_train_examples//minibatch_size
# Calculate validation steps
num_valid_samples = len(audio_gen.valid_audio_paths)
validation_steps = num_valid_samples//minibatch_size
# Add custom CTC loss function to the nn
model = add_ctc_loss(input_to_softmax)
# Dummy lambda function for loss since CTC loss is implemented above
model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=optimizer)
# Make initial results/ directory for saving model pickles
if not os.path.exists('results'):
os.makedirs('results')
# Add callbacks
checkpointer = ModelCheckpoint(filepath='results/'+save_model_path, verbose=0)
terminator = callbacks.TerminateOnNaN()
time_machiner = callbacks.History()
logger = callbacks.CSVLogger('training.log')
tensor_boarder = callbacks.TensorBoard(log_dir='./logs', batch_size=16,
write_graph=True, write_grads=True, write_images=True,)
# Fit/train model
hist = model.fit_generator(generator=audio_gen.next_train(), steps_per_epoch=steps_per_epoch,
epochs=epochs, validation_data=audio_gen.next_valid(), validation_steps=validation_steps,
callbacks=[checkpointer, terminator, logger, time_machiner, tensor_boarder], verbose=verbose)
# Save model loss
with open('results/'+pickle_path, 'wb') as f:
pickle.dump(hist.history, f)
def fit_generator(self,
generator,
nb_epoch,
nb_batches_per_epoch,
callbacks=[],
batch_size=None,
verbose=False):
if batch_size is None:
batch_size = 2 * len(next(generator)[0])
out_labels = ['g', 'd', 'm']
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + callbacks + [self.history]
if verbose:
callbacks += [cbks.ProgbarLogger()]
callbacks = cbks.CallbackList(callbacks)
callbacks.set_model(self)
callbacks.set_params({
'nb_epoch': nb_epoch,
'nb_sample': nb_batches_per_epoch * batch_size,
'verbose': verbose,
'metrics': out_labels,
})
callbacks.on_train_begin()
for e in range(nb_epoch):
callbacks.on_epoch_begin(e)
for batch_index, (seq_input, real) in enumerate(generator):
callbacks.on_batch_begin(batch_index)
batch_logs = dict()
batch_logs['batch'] = batch_index
batch_logs['size'] = len(real) + len(seq_input)
outs = self.train_on_batch(seq_input, real)
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if batch_index + 1 == nb_batches_per_epoch:
break
callbacks.on_epoch_end(e)
callbacks.on_train_end()
def __init__(self,
model,
optimizer,
comm,
batch_iterator,
batch_size,
num_replicas=None,
warmup_steps=1000,
lr=0.01,
num_batches_minimum=100,
conf=None):
random.seed(g.task_index)
np.random.seed(g.task_index)
self.conf = conf
self.start_time = time.time()
self.epoch = 0
self.num_so_far = 0
self.num_so_far_accum = 0
self.num_so_far_indiv = 0
self.model = model
self.optimizer = optimizer
self.max_lr = 0.1
self.DUMMY_LR = 0.001
self.batch_size = batch_size
self.batch_iterator = batch_iterator
self.set_batch_iterator_func()
self.warmup_steps = warmup_steps
self.num_batches_minimum = num_batches_minimum
# TODO(KGF): duplicate/may be in conflict with global_vars.py
self.comm = comm
self.num_workers = comm.Get_size()
self.task_index = comm.Get_rank()
self.history = cbks.History()
self.model.stop_training = False
if (num_replicas is None or num_replicas < 1
or num_replicas > self.num_workers):
self.num_replicas = self.num_workers
else:
self.num_replicas = num_replicas
self.lr = (lr / (1.0 + self.num_replicas / 100.0) if
(lr < self.max_lr) else self.max_lr /
(1.0 + self.num_replicas / 100.0))
def online_fit(autoencoder, X_train, model_name, test_size=0.2, batch_size = 8, epochs = 100, verbose=True) \
-> Model:
'''
Fits the model
:param autoencoder: the autoencoder to fit
:param X_train:
:param model_name:
:param test_size:
:param batch_size:
:param epochs:
:param verbose:
:return: the fit model
'''
dt = datetime.today()
currentDate = ''.join([str(dt.year), str(dt.month), str(dt.day)])
bestModelName = ''.join(
[currentDate, model_name, '_AutoEncoder', '.h5'])
bestModelFilepath = os.path.join('.', 'models', bestModelName)
checkpoint = callbacks.ModelCheckpoint(bestModelFilepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
early_stopping = callbacks.EarlyStopping(monitor='val_loss',
patience=3,
verbose=True)
history = callbacks.History()
autoencoder.fit(X_train,
X_train,
batch_size=batch_size,
epochs=epochs,
callbacks=[early_stopping, checkpoint, history],
verbose=verbose,
validation_split=test_size)
return autoencoder
def callbacks(model, callbacks, params):
model.history = cbks.History()
_callbacks = [
cbks.BaseLogger(stateful_metrics=model.stateful_metric_names)
]
_callbacks.append(
cbks.ProgbarLogger(count_mode='steps',
stateful_metrics=model.stateful_metric_names))
_callbacks += (callbacks or []) + [model.history]
callbacks = cbks.CallbackList(_callbacks)
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
out_labels = model.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
callbacks.set_params({
**params,
'metrics': callback_metrics,
})
return callbacks
def fit(self):
filepath = 'models/Simple_CNN.h5'
mc = callbacks.ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
hist = callbacks.History()
es = callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=4,
verbose=1,
mode='auto')
if not os.path.exists('tensorboard_logs/Simple_CNN_tensorboard'):
os.makedirs('tensorboard_logs/Simple_CNN_tensorboard')
tensorboard = callbacks.TensorBoard(
log_dir='tensorboard_logs/Simple_CNN_tensorboard',
histogram_freq=0,
batch_size=self.batch_size,
write_graph=True,
embeddings_freq=0,
write_images=False)
self._find_class_weights()
self.history = self.model.fit_generator(
self.train_generator,
steps_per_epoch=len(self.train_generator),
epochs=self.nb_epoch,
class_weight=self.class_weights,
validation_data=self.validation_generator,
validation_steps=len(self.validation_generator),
callbacks=[mc, hist, es, tensorboard])
return self.history
def init_callbacks(self, for_worker=False):
"""Prepares all keras callbacks to be used in training.
Automatically attaches a History callback to the end of the callback list.
If for_worker is True, leaves out callbacks that only make sense
with validation enabled."""
import keras.callbacks as cbks
remove_for_worker = [cbks.EarlyStopping, cbks.ModelCheckpoint]
if for_worker:
for obj in remove_for_worker:
self.callbacks_list = [
c for c in self.callbacks_list if not isinstance(c, obj)
]
self.model.history = cbks.History()
self.callbacks = cbks.CallbackList(self.callbacks_list +
[self.model.history])
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(self.model, 'callback_model') and self.model.callback_model:
self.callback_model = self.model.callback_model
else:
self.callback_model = self.model
self.callbacks.set_model(self.callback_model)
self.callback_model.stop_training = False
"""
from __future__ import print_function
from keras.models import Sequential
from keras.layers import Dense
import pandas
import matplotlib.pyplot as plt
from keras import optimizers
from keras import callbacks
import numpy
import keras.regularizers as regularizers
from sklearn.preprocessing import StandardScaler
batch_size = 300
history = callbacks.History()
print('Loading data...')
dataframeTrain = pandas.DataFrame(pandas.read_csv('train.txt', sep = ' '))
dataframeTest = pandas.DataFrame(pandas.read_csv('test.txt', sep = ' '))
train = dataframeTrain.values
test = dataframeTest.values
# split into input (X) and output (Y) variables
X_train = train[0:20000,6:26]
y_train = train[0:20000,27]
X_test = test[53:209,6:26]
y_test = test[53:209,27]
#Trainingsdatensatz Standardisieren
def run(GP):
# set the seed
if GP['seed']:
np.random.seed(GP['seed'])
else:
np.random.seed(np.random.randint(10000))
# Set paths
if not os.path.isdir(GP['home_dir']):
print('Keras home directory not set')
sys.exit(0)
sys.path.append(GP['home_dir'])
# Setup loggin
args = candle.ArgumentStruct(**GP)
# set_seed(args.rng_seed)
# ext = extension_from_parameters(args)
candle.verify_path(args.save_path)
prefix = args.save_path # + ext
logfile = args.logfile if args.logfile else prefix + '.log'
candle.set_up_logger(logfile, logger, False) #args.verbose
logger.info('Params: {}'.format(GP))
import p2b1 as hf
reload(hf)
#import keras_model_utils as KEU
#reload(KEU)
#reload(p2ck)
#reload(p2ck.optimizers)
maps = hf.autoencoder_preprocess()
from keras.optimizers import SGD, RMSprop, Adam
from keras.datasets import mnist
from keras.callbacks import LearningRateScheduler, ModelCheckpoint
from keras import callbacks
from keras.layers.advanced_activations import ELU
from keras.preprocessing.image import ImageDataGenerator
# GP=hf.ReadConfig(opts.config_file)
batch_size = GP['batch_size']
learning_rate = GP['learning_rate']
kerasDefaults = candle.keras_default_config()
##### Read Data ########
import helper
(data_files, fields) = p2b1.get_list_of_data_files(GP)
# Read from local directoy
#(data_files, fields) = helper.get_local_files('/p/gscratchr/brainusr/datasets/cancer/pilot2/3k_run16_10us.35fs-DPPC.20-DIPC.60-CHOL.20.dir/')
#(data_files, fields) = helper.get_local_files('3k_run16', '/p/lscratchf/brainusr/datasets/cancer/pilot2/')
# Define datagenerator
datagen = hf.ImageNoiseDataGenerator(corruption_level=GP['noise_factor'])
# get data dimension ##
num_samples = 0
for f in data_files:
# Seperate different arrays from the data
(X, nbrs, resnums) = helper.get_data_arrays(f)
num_samples += X.shape[0]
(X, nbrs, resnums) = helper.get_data_arrays(data_files[0])
print('\nData chunk shape: ', X.shape)
molecular_hidden_layers = GP['molecular_num_hidden']
if not molecular_hidden_layers:
X_train = hf.get_data(X, case=GP['case'])
input_dim = X_train.shape[1]
else:
# computing input dimension for outer AE
input_dim = X.shape[1] * molecular_hidden_layers[-1]
print('\nState AE input/output dimension: ', input_dim)
# get data dimension for molecular autoencoder
molecular_nbrs = np.int(GP['molecular_nbrs'])
num_molecules = X.shape[1]
num_beads = X.shape[2]
if GP['nbr_type'] == 'relative':
# relative x, y, z positions
num_loc_features = 3
loc_feat_vect = ['rel_x', 'rel_y', 'rel_z']
elif GP['nbr_type'] == 'invariant':
# relative distance and angle
num_loc_features = 2
loc_feat_vect = ['rel_dist', 'rel_angle']
else:
print('Invalid nbr_type!!')
exit()
if not GP['type_bool']:
# only consider molecular location coordinates
num_type_features = 0
type_feat_vect = []
else:
num_type_features = 5
type_feat_vect = list(fields.keys())[3:8]
num_features = num_loc_features + num_type_features + num_beads
dim = np.prod([num_beads, num_features, molecular_nbrs + 1])
bead_kernel_size = num_features
molecular_input_dim = dim
mol_kernel_size = num_beads
feature_vector = loc_feat_vect + type_feat_vect + list(fields.keys())[8:]
print('\nMolecular AE input/output dimension: ', molecular_input_dim)
print(
'\nData Format:\n[Frames (%s), Molecules (%s), Beads (%s), %s (%s)]' %
(num_samples, num_molecules, num_beads, feature_vector, num_features))
### Define Model, Solver and Compile ##########
print('\nDefine the model and compile')
opt = candle.build_optimizer(GP['optimizer'], learning_rate, kerasDefaults)
model_type = 'mlp'
memo = '%s_%s' % (GP['base_memo'], model_type)
######## Define Molecular Model, Solver and Compile #########
molecular_nonlinearity = GP['molecular_nonlinearity']
len_molecular_hidden_layers = len(molecular_hidden_layers)
conv_bool = GP['conv_bool']
full_conv_bool = GP['full_conv_bool']
if conv_bool:
molecular_model, molecular_encoder = AE_models.conv_dense_mol_auto(
bead_k_size=bead_kernel_size,
mol_k_size=mol_kernel_size,
weights_path=None,
input_shape=(1, molecular_input_dim, 1),
nonlinearity=molecular_nonlinearity,
hidden_layers=molecular_hidden_layers,
l2_reg=GP['l2_reg'],
drop=float(GP['drop_prob']))
elif full_conv_bool:
molecular_model, molecular_encoder = AE_models.full_conv_mol_auto(
bead_k_size=bead_kernel_size,
mol_k_size=mol_kernel_size,
weights_path=None,
input_shape=(1, molecular_input_dim, 1),
nonlinearity=molecular_nonlinearity,
hidden_layers=molecular_hidden_layers,
l2_reg=GP['l2_reg'],
drop=float(GP['drop_prob']))
else:
molecular_model, molecular_encoder = AE_models.dense_auto(
weights_path=None,
input_shape=(molecular_input_dim, ),
nonlinearity=molecular_nonlinearity,
hidden_layers=molecular_hidden_layers,
l2_reg=GP['l2_reg'],
drop=float(GP['drop_prob']))
if GP['loss'] == 'mse':
loss_func = 'mse'
elif GP['loss'] == 'custom':
loss_func = helper.combined_loss
molecular_model.compile(
optimizer=opt,
loss=loss_func,
metrics=['mean_squared_error', 'mean_absolute_error'])
print('\nModel Summary: \n')
molecular_model.summary()
##### set up callbacks and cooling for the molecular_model ##########
drop = 0.5
mb_epochs = GP['epochs']
initial_lrate = GP['learning_rate']
epochs_drop = 1 + int(np.floor(mb_epochs / 3))
def step_decay(epoch):
global initial_lrate, epochs_drop, drop
lrate = initial_lrate * np.power(drop,
np.floor((1 + epoch) / epochs_drop))
return lrate
lr_scheduler = LearningRateScheduler(step_decay)
history = callbacks.History()
# callbacks=[history,lr_scheduler]
history_logger = candle.LoggingCallback(logger.debug)
candleRemoteMonitor = candle.CandleRemoteMonitor(params=GP)
timeoutMonitor = candle.TerminateOnTimeOut(TIMEOUT)
callbacks = [history, history_logger, candleRemoteMonitor, timeoutMonitor]
loss = 0.
#### Save the Model to disk
if GP['save_path'] != None:
save_path = GP['save_path']
if not os.path.exists(save_path):
os.makedirs(save_path)
else:
save_path = '.'
model_json = molecular_model.to_json()
with open(save_path + '/model.json', "w") as json_file:
json_file.write(model_json)
encoder_json = molecular_encoder.to_json()
with open(save_path + '/encoder.json', "w") as json_file:
json_file.write(encoder_json)
print('Saved model to disk')
#### Train the Model
if GP['train_bool']:
ct = hf.Candle_Molecular_Train(
molecular_model,
molecular_encoder,
data_files,
mb_epochs,
callbacks,
batch_size=batch_size,
nbr_type=GP['nbr_type'],
save_path=GP['save_path'],
len_molecular_hidden_layers=len_molecular_hidden_layers,
molecular_nbrs=molecular_nbrs,
conv_bool=conv_bool,
full_conv_bool=full_conv_bool,
type_bool=GP['type_bool'],
sampling_density=GP['sampling_density'])
frame_loss, frame_mse = ct.train_ac()
else:
frame_mse = []
frame_loss = []
return frame_loss, frame_mse
def fit_tfrecord(self,
steps_per_epoch,
epochs=1,
verbose=1,
callbacks=None,
validation_steps=None,
initial_epoch=0):
epoch = initial_epoch
self._make_tfrecord_train_function()
do_validation = bool(len(self.val_inputs) > 0)
if do_validation and not validation_steps:
raise ValueError('When using a validation batch, '
'you must specify a value for '
'`validation_steps`.')
# Prepare display labels.
out_labels = self._get_deduped_metrics_names()
if do_validation:
callback_metrics = copy.copy(out_labels) + [
'val_' + n for n in out_labels
]
else:
callback_metrics = copy.copy(out_labels)
# prepare callbacks
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
callbacks += [cbks.ProgbarLogger(count_mode='steps')]
callbacks = cbks.CallbackList(callbacks)
# it's possible to callback a different model than self:
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
if do_validation:
val_sample_weight = None
for cbk in callbacks:
cbk.validation_data = [
self.val_inputs, self.y_val, val_sample_weight
]
try:
sess = K.get_session()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
callback_model.stop_training = False
while epoch < epochs:
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
# build batch logs
batch_logs = {
'batch': batch_index,
'size': self.inputs[0].shape[0].value
}
callbacks.on_batch_begin(batch_index, batch_logs)
if self.uses_learning_phase and not isinstance(
K.learning_phase(), int):
ins = [1.]
else:
ins = []
outs = self.train_function(ins)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
# Construct epoch logs.
epoch_logs = {}
batch_index += 1
steps_done += 1
# Epoch finished.
if steps_done >= steps_per_epoch and do_validation:
val_outs = self._validate_tfrecord(
steps=validation_steps)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
if callback_model.stop_training:
break
finally:
# TODO: If you close the queue, you can't open it again..
# coord.request_stop()
# coord.join(threads)
pass
callbacks.on_train_end()
return self.history
def _fit(self,
f,
ins,
out_labels=[],
batch_size=128,
nb_epoch=100,
verbose=1,
callbacks=[],
val_f=None,
val_ins=None,
shuffle=True,
metrics=[]):
'''
Abstract fit function for f(*ins). Assume that f returns a list, labelled by out_labels.
'''
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose:
print("Train on %d samples, validate on %d samples" %
(len(ins[0]), len(val_ins[0])))
nb_train_sample = len(ins[0])
index_array = np.arange(nb_train_sample)
history = cbks.History()
if verbose:
callbacks = [history, cbks.BaseLogger()] + callbacks
else:
callbacks = [history] + callbacks
callbacks = cbks.CallbackList(callbacks)
callbacks._set_model(self)
callbacks._set_params({
'batch_size': batch_size,
'nb_epoch': nb_epoch,
'nb_sample': nb_train_sample,
'verbose': verbose,
'do_validation': do_validation,
'metrics': metrics,
})
callbacks.on_train_begin()
self.stop_training = False
for epoch in range(nb_epoch):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
ins_batch = slice_X(ins, batch_ids)
except TypeError as err:
raise Exception('TypeError while preparing batch. \
If using HDF5 input data, pass shuffle="batch".\n')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(*ins_batch)
if type(outs) != list:
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
if batch_index == len(batches) - 1: # last batch
# validation
if do_validation:
# replace with self._evaluate
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
verbose=0)
if type(val_outs) != list:
val_outs = [val_outs]
# same labels assumed
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if self.stop_training:
break
callbacks.on_train_end()
return history
def _fit_loop(self,
f,
ins,
out_labels=None,
batch_size=32,
epochs=100,
verbose=1,
callbacks=None,
val_f=None,
val_ins=None,
shuffle=True,
callback_metrics=None,
initial_epoch=0,
steps_per_epoch=None,
validation_steps=None):
"""Abstract fit function for f(ins).
Assume that f returns a list, labeled by out_labels.
# Arguments
f: Keras function returning a list of tensors
ins: List of tensors to be fed to `f`
out_labels: List of strings, display names of
the outputs of `f`
batch_size: Integer batch size or None if unknown.
epochs: Number of times to iterate over the data
verbose: Verbosity mode, 0, 1 or 2
callbacks: List of callbacks to be called during training
val_f: Keras function to call for validation
val_ins: List of tensors to be fed to `val_f`
shuffle: Whether to shuffle the data at the beginning of each epoch
callback_metrics: List of strings, the display names of the metrics
passed to the callbacks. They should be the
concatenation of list the display names of the outputs of
`f` and the list of display names of the outputs of `f_val`.
initial_epoch: Epoch at which to start training
(useful for resuming a previous training run)
steps_per_epoch: Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. Ignored with the default value of `None`.
validation_steps: Number of steps to run validation for
(only if doing validation from data tensors).
Ignored with the default value of `None`.
# Returns
`History` object.
[A tweaked version.]
"""
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose and ins and hasattr(ins[0], 'shape') and hasattr(
val_ins[0], 'shape'):
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
if validation_steps:
do_validation = True
if steps_per_epoch is None:
raise ValueError('Can only use `validation_steps` '
'when doing step-wise '
'training, i.e. `steps_per_epoch` '
'must be set.')
num_train_samples = self._check_num_samples(ins, batch_size,
steps_per_epoch,
'steps_per_epoch')
if num_train_samples is not None:
index_array = np.arange(num_train_samples)
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
if steps_per_epoch is not None:
count_mode = 'steps'
else:
count_mode = 'samples'
callbacks += [cbks.ProgbarLogger(count_mode)]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
# for cbk in callbacks:
# cbk.validation_data = val_ins
for epoch in range(initial_epoch, epochs):
callbacks.on_epoch_begin(epoch)
epoch_logs = {}
if steps_per_epoch is not None:
for step_index in range(steps_per_epoch):
batch_logs = {}
batch_logs['batch'] = step_index
batch_logs['size'] = 1
callbacks.on_batch_begin(step_index, batch_logs)
outs = f(ins)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(step_index, batch_logs)
if callback_model.stop_training:
break
if do_validation:
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
steps=validation_steps,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
else:
if shuffle == 'batch':
index_array = _batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = _make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# do not slice the training phase flag
ins_batch = _slice_arrays(ins[:-1],
batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
batch_logs['ids'] = batch_ids
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if callback_model.stop_training:
break
if batch_index == len(batches) - 1: # last batch.
if do_validation:
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# same labels assumed
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return self.history
def fit_generator_Ndiff(model,
generator,
steps_per_epoch=None,
batch_size=1,
N_diff=5,
margin=0.5,
epochs=1,
verbose=1,
callbacks=None,
validation_data=None,
validation_steps=None,
class_weight=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False,
shuffle=True,
initial_epoch=0):
"""Trains the model on data yielded batch-by-batch by a Python generator.
The generator is run in parallel to the model, for efficiency.
For instance, this allows you to do real-time data augmentation
on images on CPU in parallel to training your model on GPU.
The use of `keras.utils.Sequence` guarantees the ordering
and guarantees the single use of every input per epoch when
using `use_multiprocessing=True`.
# Arguments
generator: A generator or an instance of `Sequence`
(`keras.utils.Sequence`) object in order to avoid
duplicate data when using multiprocessing.
The output of the generator must be either
- a tuple `(inputs, targets)`
- a tuple `(inputs, targets, sample_weights)`.
This tuple (a single output of the generator) makes a single
batch. Therefore, all arrays in this tuple must have the same
length (equal to the size of this batch). Different batches
may have different sizes. For example, the last batch of the
epoch is commonly smaller than the others, if the size of the
dataset is not divisible by the batch size.
The generator is expected to loop over its data
indefinitely. An epoch finishes when `steps_per_epoch`
batches have been seen by the model.
steps_per_epoch: Integer.
Total number of steps (batches of samples)
to yield from `generator` before declaring one epoch
finished and starting the next epoch. It should typically
be equal to the number of samples of your dataset
divided by the batch size.
Optional for `Sequence`: if unspecified, will use
the `len(generator)` as a number of steps.
epochs: Integer. Number of epochs to train the model.
An epoch is an iteration over the entire data provided,
as defined by `steps_per_epoch`.
Note that in conjunction with `initial_epoch`,
`epochs` is to be understood as "final epoch".
The model is not trained for a number of iterations
given by `epochs`, but merely until the epoch
of index `epochs` is reached.
verbose: Integer. 0, 1, or 2. Verbosity mode.
0 = silent, 1 = progress bar, 2 = one line per epoch.
callbacks: List of `keras.callbacks.Callback` instances.
List of callbacks to apply during training.
See [callbacks](/callbacks).
validation_data: This can be either
- a generator for the validation data
- tuple `(x_val, y_val)`
- tuple `(x_val, y_val, val_sample_weights)`
on which to evaluate
the loss and any model metrics at the end of each epoch.
The model will not be trained on this data.
validation_steps: Only relevant if `validation_data`
is a generator. Total number of steps (batches of samples)
to yield from `validation_data` generator before stopping.
Optional for `Sequence`: if unspecified, will use
the `len(validation_data)` as a number of steps.
class_weight: Optional dictionary mapping class indices (integers)
to a weight (float) value, used for weighting the loss function
(during training only).
This can be useful to tell the model to
"pay more attention" to samples from
an under-represented class.
max_queue_size: Integer. Maximum size for the generator queue.
If unspecified, `max_queue_size` will default to 10.
workers: Integer. Maximum number of processes to spin up
when using process based threading.
If unspecified, `workers` will default to 1. If 0, will
execute the generator on the main thread.
use_multiprocessing: Boolean. If True, use process based threading.
If unspecified, `use_multiprocessing` will default to False.
Note that because
this implementation relies on multiprocessing,
you should not pass
non picklable arguments to the generator
as they can't be passed
easily to children processes.
shuffle: Boolean. Whether to shuffle the training data
in batch-sized chunks before each epoch.
Only used with instances of `Sequence` (`keras.utils.Sequence`).
initial_epoch: Integer.
Epoch at which to start training
(useful for resuming a previous training run).
# Returns
A `History` object. Its `History.history` attribute is
a record of training loss values and metrics values
at successive epochs, as well as validation loss values
and validation metrics values (if applicable).
# Example
```python
def generate_arrays_from_file(path):
while 1:
with open(path) as f:
for line in f:
# create numpy arrays of input data
# and labels, from each line in the file
x1, x2, y = process_line(line)
yield ({'input_1': x1, 'input_2': x2}, {'output': y})
model.fit_generator(generate_arrays_from_file('/my_file.txt'),
steps_per_epoch=10000, epochs=10)
```
# Raises
ValueError: In case the generator yields
data in an invalid format.
"""
wait_time = 0.01 # in seconds
epoch = initial_epoch
do_validation = bool(validation_data)
# self._make_train_function()
# if do_validation:
# self._make_test_function()
is_sequence = isinstance(generator, Sequence)
# do_validation = True if is_sequence else False
if not is_sequence and use_multiprocessing and workers > 1:
warnings.warn(
UserWarning('Using a generator with `use_multiprocessing=True`'
' and multiple workers may duplicate your data.'
' Please consider using the`keras.utils.Sequence'
' class.'))
if steps_per_epoch is None:
if is_sequence:
steps_per_epoch = len(generator)
else:
raise ValueError('`steps_per_epoch=None` is only valid for a'
' generator based on the `keras.utils.Sequence`'
' class. Please specify `steps_per_epoch` or use'
' the `keras.utils.Sequence` class.')
# python 2 has 'next', 3 has '__next__'
# avoid any explicit version checks
val_gen = (hasattr(validation_data, 'next')
or hasattr(validation_data, '__next__')
or isinstance(validation_data, Sequence))
if (val_gen and not isinstance(validation_data, Sequence)
and not validation_steps):
raise ValueError('`validation_steps=None` is only valid for a'
' generator based on the `keras.utils.Sequence`'
' class. Please specify `validation_steps` or use'
' the `keras.utils.Sequence` class.')
# Prepare display labels.
out_labels = model._get_deduped_metrics_names()
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [history]
if verbose:
callbacks += [cbks.ProgbarLogger(count_mode='steps')]
callbacks = cbks.CallbackList(callbacks)
# # it's possible to callback a different model than self:
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
enqueuer = None
val_enqueuer = None
try:
if do_validation:
if val_gen:
if workers > 0:
if isinstance(validation_data, Sequence):
val_enqueuer = OrderedEnqueuer(
validation_data,
use_multiprocessing=use_multiprocessing)
if validation_steps is None:
validation_steps = len(validation_data)
else:
val_enqueuer = GeneratorEnqueuer(
validation_data,
use_multiprocessing=use_multiprocessing,
wait_time=wait_time)
val_enqueuer.start(workers=workers,
max_queue_size=max_queue_size)
validation_generator = val_enqueuer.get()
else:
validation_generator = validation_data
else:
pass
# if len(validation_data) == 2:
# val_x, val_y = validation_data
# val_sample_weights = None
# elif len(validation_data) == 3:
# val_x, val_y, val_sample_weights = validation_data
# else:
# raise ValueError('`validation_data` should be a tuple '
# '`(val_x, val_y, val_sample_weight)` '
# 'or `(val_x, val_y)`. Found: ' +
# str(validation_data))
# val_x, val_y, val_sample_weights = _standardize_user_data(
# val_x, val_y, val_sample_weight)
# val_data = val_x + val_y + val_sample_weights
# if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
# val_data += [0.]
# for cbk in callbacks:
# cbk.validation_data = val_data
if workers > 0:
if is_sequence:
enqueuer = OrderedEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
shuffle=shuffle)
else:
enqueuer = GeneratorEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
wait_time=wait_time)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
else:
output_generator = generator
callback_model.stop_training = False
# Construct epoch logs.
epoch_logs = {}
while epoch < epochs:
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
generator_output = next(output_generator)
if not hasattr(generator_output, '__len__'):
raise ValueError('Output of generator should be '
'batch_size lists ' +
str(generator_output))
if len(generator_output) == batch_size:
# ii_ndiff: the index of the negative sample
gen_out = generator_output
sample_weight = None
else:
raise ValueError('Output of generator should be '
'batch_size lists ' +
str(generator_output))
# build batch logs
batch_logs = {}
# if isinstance(x, list):
# batch_size = x[0].shape[0]
# elif isinstance(x, dict):
# batch_size = list(x.values())[0].shape[0]
# else:
# batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
# aggregate the losses by inner index n_diff
loss_mat = np.zeros((batch_size, N_diff))
for ii_ndiff in range(N_diff):
# get the maximum sequence length
len_anchor_max, len_same_max, len_diff_max = \
get_maximum_length(batch_size=batch_size,
generator_output=gen_out,
index=[ii_ndiff]*batch_size)
print(len_anchor_max, len_same_max, len_diff_max)
# organize the input for the prediction
input_anchor, input_same, input_diff = \
make_same_length_batch(batch_size=batch_size,
len_anchor_max=len_anchor_max,
len_same_max=len_same_max,
len_diff_max=len_diff_max,
generator_output=gen_out,
index=[ii_ndiff]*batch_size)
output_batch_pred = model.predict_on_batch(
[input_anchor, input_same, input_diff])
loss = K.eval(
triplet_loss_no_mean(output_batch_pred, margin))
loss_mat[:, ii_ndiff] = loss
# this the index of the input which has the maximum loss for each N_diff pairs
index_max_loss = np.argmax(loss_mat, axis=-1)
len_anchor_max, len_same_max, len_diff_max = get_maximum_length(
batch_size=batch_size,
generator_output=gen_out,
index=index_max_loss)
input_anchor, input_same, input_diff = \
make_same_length_batch(batch_size=batch_size,
len_anchor_max=len_anchor_max,
len_same_max=len_same_max,
len_diff_max=len_diff_max,
generator_output=gen_out,
index=index_max_loss)
outs = model.train_on_batch(
[input_anchor, input_same, input_diff],
None,
sample_weight=sample_weight,
class_weight=class_weight)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
batch_index += 1
steps_done += 1
# Epoch finished.
if steps_done >= steps_per_epoch and do_validation:
if val_gen:
val_outs = evaluate_generator(
model=model,
generator=validation_generator,
steps=validation_steps,
batch_size=batch_size,
margin=margin,
N_diff=N_diff,
workers=0)
else:
pass
# # No need for try/except because
# # data has already been validated.
# val_outs = model.evaluate(
# val_x, val_y,
# batch_size=batch_size,
# sample_weight=val_sample_weights,
# verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
if callback_model.stop_training:
break
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
if callback_model.stop_training:
break
finally:
try:
if enqueuer is not None:
enqueuer.stop()
finally:
if val_enqueuer is not None:
val_enqueuer.stop()
callbacks.on_train_end()
return history
def fit_generator_autosized(
model,
generator,
epochs=1,
#steps_per_epoch=None,
verbose=1,
callbacks=None,
validation_data=None,
validation_steps=None,
validation_callbacks=None,
class_weight=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False,
shuffle=True,
initial_epoch=0):
"""See docstring for `Model.fit_generator`."""
wait_time = 0.01 # in seconds
epoch = initial_epoch
do_validation = bool(validation_data)
model._make_train_function()
if do_validation:
model._make_test_function()
is_sequence = isinstance(generator, Sequence)
if not is_sequence and use_multiprocessing and workers > 1:
warnings.warn(
UserWarning('Using a generator with `use_multiprocessing=True`'
' and multiple workers may duplicate your data.'
' Please consider using the`keras.utils.Sequence'
' class.'))
# if steps_per_epoch is None:
# if is_sequence:
# steps_per_epoch = len(generator)
# else:
# raise ValueError('`steps_per_epoch=None` is only valid for a'
# ' generator based on the '
# '`keras.utils.Sequence`'
# ' class. Please specify `steps_per_epoch` '
# 'or use the `keras.utils.Sequence` class.')
# python 2 has 'next', 3 has '__next__'
# avoid any explicit version checks
val_gen = (hasattr(validation_data, 'next')
or hasattr(validation_data, '__next__')
or isinstance(validation_data, Sequence))
# if (val_gen and not isinstance(validation_data, Sequence) and
# not validation_steps):
# raise ValueError('`validation_steps=None` is only valid for a'
# ' generator based on the `keras.utils.Sequence`'
# ' class. Please specify `validation_steps` or use'
# ' the `keras.utils.Sequence` class.')
# Prepare display labels.
out_labels = model.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
model.history = cbks.History()
_callbacks = [
cbks.BaseLogger(stateful_metrics=model.stateful_metric_names)
]
# instead of ProgbarLogger (but only for first epoch):
if verbose:
print('Epoch 1/%d' % epochs)
progbar = Progbar(target=None,
verbose=1,
stateful_metrics=model.stateful_metric_names)
_callbacks += (callbacks or []) + [model.history]
callbacks = cbks.CallbackList(_callbacks)
# it's possible to callback a different model than self:
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': None, # will be refined during first epoch
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
enqueuer = None
val_enqueuer = None
try:
if do_validation and not val_gen:
# Prepare data for validation
if len(validation_data) == 2:
val_x, val_y = validation_data
val_sample_weight = None
elif len(validation_data) == 3:
val_x, val_y, val_sample_weight = validation_data
else:
raise ValueError('`validation_data` should be a tuple '
'`(val_x, val_y, val_sample_weight)` '
'or `(val_x, val_y)`. Found: ' +
str(validation_data))
val_x, val_y, val_sample_weights = model._standardize_user_data(
val_x, val_y, val_sample_weight)
val_data = val_x + val_y + val_sample_weights
if model.uses_learning_phase and not isinstance(
K.learning_phase(), int):
val_data += [0.]
for cbk in callbacks:
cbk.validation_data = val_data
if workers > 0:
if is_sequence:
enqueuer = OrderedEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
shuffle=shuffle)
else:
enqueuer = GeneratorEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
wait_time=wait_time)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
else:
if is_sequence:
output_generator = iter(generator)
else:
output_generator = generator
callback_model.stop_training = False
# Construct epoch logs.
epoch_logs = {}
while epoch < epochs:
for m in model.stateful_metric_functions:
m.reset_states()
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
for generator_output in output_generator:
if not generator_output: # end of epoch?
break
if not hasattr(generator_output, '__len__'):
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
# build batch logs
batch_logs = {}
if not x:
# Handle data tensors support when no input given
# step-size = 1 for data tensors
batch_size = 1
elif isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
outs = model.train_on_batch(x,
y,
sample_weight=sample_weight,
class_weight=class_weight)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if epoch == initial_epoch and verbose:
log_values = []
for k in callback_metrics:
if k in batch_logs:
log_values.append((k, batch_logs[k]))
progbar.update(steps_done, log_values)
batch_index += 1
steps_done += 1
if callback_model.stop_training:
break
if epoch == initial_epoch:
if verbose:
log_values = []
for k in callback_metrics:
if k in batch_logs:
log_values.append((k, batch_logs[k]))
progbar.update(steps_done, log_values)
# Epoch finished.
if do_validation:
if val_gen:
val_outs, validation_steps = evaluate_generator_autosized(
model,
validation_data,
steps=validation_steps,
callbacks=validation_callbacks,
workers=workers,
use_multiprocessing=use_multiprocessing,
max_queue_size=max_queue_size,
verbose=1)
else:
# No need for try/except because
# data has already been validated.
val_outs = model.evaluate(val_x,
val_y,
batch_size=batch_size,
sample_weight=val_sample_weights,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
if callback_model.stop_training:
break
callbacks.on_epoch_end(epoch, epoch_logs)
if epoch == initial_epoch:
if verbose:
print()
progbar = cbks.ProgbarLogger(
count_mode='steps',
stateful_metrics=model.stateful_metric_names)
progbar.set_model(callback_model)
callbacks.append(progbar)
callbacks.set_params({
'epochs': epochs,
'steps': steps_done, # refine
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
if verbose:
progbar.on_train_begin()
epoch += 1
if callback_model.stop_training:
break
finally:
try:
if enqueuer is not None:
enqueuer.stop()
finally:
if val_enqueuer is not None:
val_enqueuer.stop()
callbacks.on_train_end()
return model.history
async def fit_generator(model,
generator,
steps_per_epoch=None,
epochs=1,
verbose=1,
callbacks=None,
validation_data=None,
validation_steps=None,
class_weight=None,
shuffle=True,
initial_epoch=0):
"""See docstring for `Model.fit_generator`."""
epoch = initial_epoch
do_validation = bool(validation_data)
model._make_train_function()
if do_validation:
model._make_test_function()
if steps_per_epoch is None:
steps_per_epoch = len(generator)
# Prepare display labels.
out_labels = model.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
model.history = cbks.History()
_callbacks = [
cbks.BaseLogger(stateful_metrics=model.stateful_metric_names)
]
if verbose:
_callbacks.append(
cbks.ProgbarLogger(count_mode='steps',
stateful_metrics=model.stateful_metric_names))
_callbacks += (callbacks or []) + [model.history]
callbacks = cbks.CallbackList(_callbacks)
# it's possible to callback a different model than self:
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
output_generator = generator.async_next
callback_model.stop_training = False
# Construct epoch logs.
epoch_logs = {}
while epoch < epochs:
for m in model.stateful_metric_functions:
m.reset_states()
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
generator_output = await output_generator()
if not hasattr(generator_output, '__len__'):
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
# build batch logs
batch_logs = {}
if x is None or len(x) == 0:
# Handle data tensors support when no input given
# step-size = 1 for data tensors
batch_size = 1
elif isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
outs = model.train_on_batch(x,
y,
sample_weight=sample_weight,
class_weight=class_weight)
outs = to_list(outs)
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
batch_index += 1
steps_done += 1
# Epoch finished.
if steps_done >= steps_per_epoch and do_validation:
val_outs = await evaluate_generator(model, validation_data,
validation_steps)
val_outs = to_list(val_outs)
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
if callback_model.stop_training:
break
generator.on_epoch_end()
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
if callback_model.stop_training:
break
callbacks.on_train_end()
return model.history
def _fit_loop(self,
f,
ins,
out_labels=None,
batch_size=32,
epochs=100,
verbose=1,
callbacks=None,
val_f=None,
val_ins=None,
shuffle=True,
callback_metrics=None,
initial_epoch=0,
steps_per_epoch=None):
"""Abstract fit function for `f(ins)`.
Assume that f returns a list, labeled by out_labels.
# Arguments
f: Keras function returning a list of tensors
ins: list of tensors to be fed to `f`
out_labels: list of strings, display names of
the outputs of `f`
batch_size: integer batch size
epochs: number of times to iterate over the data
verbose: verbosity mode, 0, 1 or 2
callbacks: list of callbacks to be called during training
val_f: Keras function to call for validation
val_ins: list of tensors to be fed to `val_f`
shuffle: whether to shuffle the data at the beginning of each epoch
callback_metrics: list of strings, the display names of the metrics
passed to the callbacks. They should be the
concatenation of list the display names of the outputs of
`f` and the list of display names of the outputs of `f_val`.
initial_epoch: epoch at which to start training
(useful for resuming a previous training run)
steps_per_epoch: Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. The default `None` is equal to the number
of unique samples in your dataset divided by the batch
size, or 1 if that cannot be determined.
# Returns
`History` object.
"""
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose and ins and hasattr(ins[0], 'shape'):
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
if steps_per_epoch is not None:
num_train_samples = steps_per_epoch
else:
if ins and hasattr(ins[0], 'shape'):
num_train_samples = ins[0].shape[0]
else:
# May happen if we are running `fit` without Numpy input data,
# i.e. if all inputs to the models are data tensors
# instead of placeholders.
# In that case we will run `fit` over a single batch.
num_train_samples = batch_size
verbose = 2
index_array = np.arange(num_train_samples)
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
# callbacks += [cbks.ProgbarLogger()]
callbacks += [ProgbarLogger_TFRecord()]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
for cbk in callbacks:
cbk.validation_data = val_ins
for epoch in range(initial_epoch, epochs):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = _batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = _make_batches(num_train_samples, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# Do not slice the training phase flag.
ins_batch = \
_slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if callback_model.stop_training:
break
if batch_index == len(batches) - 1: # Last batch.
if do_validation:
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return self.history
def fit_models(callback_model,
models,
generators,
metrics_names,
batch_size,
steps_per_epoch=None,
epochs=1,
verbose=1,
callbacks=None,
initial_epoch=0):
epoch = initial_epoch
# Prepare display labels.
callback_metrics = [n for m in metrics_names for n in m.keys()]
# prepare callbacks
stateful_metric_names = []
for model in models:
model.history = cbks.History()
try:
stateful_metric_names.extend(model.stateful_metric_names)
except AttributeError:
stateful_metric_names.extend(model.model.stateful_metric_names)
_callbacks = [cbks.BaseLogger(stateful_metrics=stateful_metric_names)]
if verbose:
_callbacks.append(
cbks.ProgbarLogger(count_mode='steps',
stateful_metrics=stateful_metric_names))
_callbacks += (callbacks or []) + [model.history for model in models]
callbacks = cbks.CallbackList(_callbacks)
# it's possible to callback a different model than self:
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': False,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
try:
callback_model.stop_training = False
# Construct epoch logs.
epoch_logs = {}
while epoch < epochs:
for model in models:
try:
stateful_metric_functions = model.stateful_metric_functions
except AttributeError:
stateful_metric_functions = model.model.stateful_metric_functions
for m in stateful_metric_functions:
m.reset_states()
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
# build batch logs
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
for model, output_generator, metrics in zip(
models, generators, metrics_names):
generator_output = next(output_generator)
if not hasattr(generator_output, '__len__'):
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
outs = model.train_on_batch(x,
y,
sample_weight=sample_weight)
if not isinstance(outs, list):
outs = [outs]
for name, i in metrics.items():
batch_logs[name] = outs[i]
callbacks.on_batch_end(batch_index, batch_logs)
batch_index += 1
steps_done += 1
# Epoch finished.
if callback_model.stop_training:
break
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
if callback_model.stop_training:
break
finally:
pass
callbacks.on_train_end()
return [model.history for model in models]
def _fit_loop(self, f, ins, out_labels=None, batch_size=32,
nb_epoch=100, verbose=1, callbacks=None,
val_f=None, val_ins=None, shuffle=True,
callback_metrics=None, initial_epoch=0):
"""Abstract fit function for f(ins).
Assume that f returns a list, labeled by out_labels.
# Arguments
f: Keras function returning a list of tensors
ins: list of tensors to be fed to `f`
out_labels: list of strings, display names of
the outputs of `f`
batch_size: integer batch size
nb_epoch: number of times to iterate over the data
verbose: verbosity mode, 0, 1 or 2
callbacks: list of callbacks to be called during training
val_f: Keras function to call for validation
val_ins: list of tensors to be fed to `val_f`
shuffle: whether to shuffle the data at the beginning of each epoch
callback_metrics: list of strings, the display names of the metrics
passed to the callbacks. They should be the
concatenation of list the display names of the outputs of
`f` and the list of display names of the outputs of `f_val`.
initial_epoch: epoch at which to start training
(useful for resuming a previous training run)
# Returns
`History` object.
[A tweaked version.]
"""
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose:
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
nb_train_sample = ins[0].shape[0]
index_array = np.arange(nb_train_sample)
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
callbacks += [cbks.ProgbarLogger()]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'nb_epoch': nb_epoch,
'nb_sample': nb_train_sample,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
self.validation_data = val_ins
for epoch in range(initial_epoch, nb_epoch):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
batch_logs['ids'] = batch_ids
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if batch_index == len(batches) - 1: # last batch
# validation
if do_validation:
# replace with self._evaluate
val_outs = self._test_loop(val_f, val_ins,
batch_size=batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# same labels assumed
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return self.history
def fit_dataflow(self,
dflow,
steps_per_epoch,
epochs=1,
verbose=1,
callbacks=None,
validation_data=None,
validation_steps=None,
class_weight=None,
max_q_size=10,
workers=1,
pickle_safe=False,
initial_epoch=0):
"""Fits the model on data yielded batch-by-batch by a Python generator.
The generator is run in parallel to the model, for efficiency.
For instance, this allows you to do real-time data augmentation
on images on CPU in parallel to training your model on GPU.
# Arguments
dflow: a dataflow object a-la-carte Tensorpack.
The output of the generator must be either
- a tuple (inputs, targets)
- a tuple (inputs, targets, sample_weights).
All arrays should contain the same number of samples.
The generator is expected to loop over its data
indefinitely. An epoch finishes when `steps_per_epoch`
samples have been seen by the model.
steps_per_epoch: Total number of steps (batches of samples)
to yield from `generator` before declaring one epoch
finished and starting the next epoch. It should typically
be equal to the number of unique samples if your dataset
divided by the batch size.
epochs: integer, total number of iterations on the data.
verbose: verbosity mode, 0, 1, or 2.
callbacks: list of callbacks to be called during training.
validation_data: this can be either
- a generator for the validation data
- a tuple (inputs, targets)
- a tuple (inputs, targets, sample_weights).
validation_steps: Only relevant if `validation_data`
is a generator. Total number of steps (batches of samples)
to yield from `generator` before stopping.
class_weight: dictionary mapping class indices to a weight
for the class.
max_q_size: maximum size for the generator queue
workers: maximum number of processes to spin up
when using process based threading
pickle_safe: if True, use process based threading.
Note that because
this implementation relies on multiprocessing,
you should not pass
non picklable arguments to the generator
as they can't be passed
easily to children processes.
initial_epoch: epoch at which to start training
(useful for resuming a previous training run)
# Returns
A `History` object.
# Example
```python
def generate_arrays_from_file(path):
while 1:
f = open(path)
for line in f:
# create numpy arrays of input data
# and labels, from each line in the file
x1, x2, y = process_line(line)
yield ({'input_1': x1, 'input_2': x2}, {'output': y})
f.close()
model.fit_generator(generate_arrays_from_file('/my_file.txt'),
steps_per_epoch=10000, epochs=10)
```
# Raises
ValueError: In case the generator yields
data in an invalid format.
"""
# wait_time = 0.01 # in seconds
epoch = initial_epoch
do_validation = bool(validation_data)
self._make_train_function()
if do_validation:
self._make_test_function()
# python 2 has 'next', 3 has '__next__'
# avoid any explicit version checks
val_gen = (hasattr(validation_data, 'next')
or hasattr(validation_data, '__next__'))
if val_gen and not validation_steps:
raise ValueError('When using a generator for validation data, '
'you must specify a value for '
'`validation_steps`.')
out_labels = self.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
callbacks += [cbks.ProgbarLogger(count_mode='steps')]
callbacks = cbks.CallbackList(callbacks)
# it's possible to callback a different model than self:
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
if do_validation and not val_gen:
if len(validation_data) == 2:
val_x, val_y = validation_data
val_sample_weight = None
elif len(validation_data) == 3:
val_x, val_y, val_sample_weight = validation_data
else:
raise ValueError('validation_data should be a tuple '
'`(val_x, val_y, val_sample_weight)` '
'or `(val_x, val_y)`. Found: ' +
str(validation_data))
val_x, val_y, val_sample_weights = self._standardize_user_data(
val_x, val_y, val_sample_weight)
for cbk in callbacks:
cbk.validation_data = val_x + [val_y, val_sample_weights]
# enqueuer = None
# TODO: Tensorpack does some kind of acceleratn using
# QueueInputTrainer, QueueInput, and EnqueueThread. The
# implementation below corresponds to SimpleTrainer which
# Tensorpack notes as being slow. I still cannot decipher what
# exactly is going on in Tensorpack. For the same per-GPU batchsize
# the runtime per epoch seems on par. Perhaps with Tensorpack
# implementation using Queue+Thread for datafalow the feed_dict
# would be faster. The keras fit_generator does use an enqueuer,
# but I did not notice performance difference between using
# fit_generator or this mixed-in fit_dataflow method.
try:
# enqueuer = GeneratorEnqueuer(generator, pickle_safe=pickle_safe)
# enqueuer.start(max_q_size=max_q_size, workers=workers)
dflow.reset_state()
_generator = dflow.get_data()
callback_model.stop_training = False
while epoch < epochs:
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
# generator_output = None
generator_output = next(_generator)
# while enqueuer.is_running():
# if not enqueuer.queue.empty():
# generator_output = enqueuer.queue.get()
# break
# else:
# time.sleep(wait_time)
if not hasattr(generator_output, '__len__'):
raise ValueError('output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
# build batch logs
batch_logs = {}
if isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
outs = self.train_on_batch(x,
y,
sample_weight=sample_weight,
class_weight=class_weight)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
# Construct epoch logs.
epoch_logs = {}
batch_index += 1
steps_done += 1
# Epoch finished.
if steps_done >= steps_per_epoch and do_validation:
if val_gen:
val_outs = self.evaluate_generator(
validation_data,
validation_steps,
max_q_size=max_q_size,
workers=workers,
pickle_safe=pickle_safe)
else:
# No need for try/except because
# data has already been validated.
val_outs = self.evaluate(
val_x,
val_y,
batch_size=batch_size,
sample_weight=val_sample_weights,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
if callback_model.stop_training:
break
finally:
# if enqueuer is not None:
# enqueuer.stop()
pass
callbacks.on_train_end()
return self.history
def fit(self,
model_dir_path,
image_label_pairs,
epochs=None,
batch_size=None,
snapshot_dir_path=None,
snapshot_interval=None):
# Change the epoch value
if epochs is None:
epochs = 100
if batch_size is None:
batch_size = 64
if snapshot_interval is None:
snapshot_interval = 20
self.config = dict()
self.config['img_width'] = self.img_width
self.config['img_height'] = self.img_height
self.config['random_input_dim'] = self.random_input_dim
self.config['text_input_dim'] = self.text_input_dim
self.config['img_channels'] = self.img_channels
self.config['glove_source_dir_path'] = self.glove_source_dir_path
self.glove_model.load(data_dir_path=self.glove_source_dir_path,
embedding_dim=self.text_input_dim)
config_file_path = Capsules.get_config_file_path(model_dir_path)
np.save(config_file_path, self.config)
noise = np.zeros((batch_size, self.random_input_dim))
text_batch = np.zeros((batch_size, self.text_input_dim))
self.create_model()
batch_count = int(image_label_pairs.shape[0] / batch_size)
print(batch_count)
#exp_replay = [] # array to store sample for experience replay
for epoch in range(epochs):
cum_d_loss = 0
cum_g_loss = 0
cum_g_acc = 0
cum_d_acc = 0
print('-' * 15, 'Epoch %d' % epoch, '-' * 15)
for batch_index in tqdm(range(batch_count)):
# Step 1: train the discriminator
image_label_pair_batch = image_label_pairs[batch_index *
batch_size:
(batch_index + 1) *
batch_size]
image_batch = []
for index in range(batch_size):
image_label_pair = image_label_pair_batch[index]
normalized_img = image_label_pair[0]
text = image_label_pair[1]
image_batch.append(normalized_img)
text_batch[index, :] = self.glove_model.encode_doc(
text, self.text_input_dim)
noise[index, :] = np.random.uniform(
-1, 1, self.random_input_dim)
image_batch = np.array(image_batch)
#image_batch = np.transpose(image_batch, (0, 2, 3, 1))
generated_images = self.generator.predict([noise, text_batch],
verbose=0)
# Train on soft targets (add noise to targets as well)
noise_prop = 0.05 # Randomly flip 5% of targets
# Prepare labels for real data
true_labels = np.zeros((batch_size, 1)) + np.random.uniform(
low=0.0, high=0.1, size=(batch_size, 1))
flipped_idx = np.random.choice(np.arange(len(true_labels)),
size=int(noise_prop *
len(true_labels)))
true_labels[flipped_idx] = 1 - true_labels[flipped_idx]
# Prepare labels for generated data
gene_labels = np.ones((batch_size, 1)) - np.random.uniform(
low=0.0, high=0.1, size=(batch_size, 1))
flipped_idx = np.random.choice(np.arange(len(gene_labels)),
size=int(noise_prop *
len(gene_labels)))
gene_labels[flipped_idx] = 1 - gene_labels[flipped_idx]
'''
# Store a random point for experience replay
r_idx = np.random.randint(batch_size)
exp_replay.append([generated_images[r_idx], text_batch[r_idx], gene_labels[r_idx]])
'''
if (
epoch * batch_size + batch_index
) % snapshot_interval == 0 and snapshot_dir_path is not None:
self.save_snapshots(generated_images,
snapshot_dir_path=snapshot_dir_path,
epoch=epoch,
batch_index=batch_index)
self.discriminator.trainable = True
# Train discriminator on real data
d_loss_true = self.discriminator.train_on_batch(
[image_batch, text_batch], true_labels)
# Train discriminator on generated data
d_loss_gene = self.discriminator.train_on_batch(
[generated_images, text_batch], gene_labels)
d_loss = ((np.asarray(d_loss_true) + np.asarray(d_loss_gene)) *
0.5).tolist()
cum_d_loss += d_loss[0]
cum_d_acc += d_loss[1]
'''
#Adversarial Model Training
#If we have enough points, do experience replay
if len(exp_replay) == batch_size:
generated_images = np.array([p[0] for p in exp_replay])
text_batch = np.array([p[1] for p in exp_replay])
gene_labels = np.array([p[2] for p in exp_replay])
expprep_loss_gene = self.discriminator.train_on_batch([generated_images, text_batch], gene_labels)
exp_replay = []
break
'''
#step 2: train the generator
for index in range(batch_size):
image_label_pair = image_label_pair_batch[index]
text = image_label_pair[1]
text_batch[index, :] = self.glove_model.encode_doc(
text, self.text_input_dim)
noise[index, :] = np.random.uniform(
-1, 1, self.random_input_dim)
self.discriminator.trainable = False
g_loss = self.model.train_on_batch([noise, text_batch],
np.zeros((batch_size, 1)))
cum_g_loss += g_loss[0]
cum_g_acc += g_loss[1]
#for index in range(batch_size):
#noise[index, :] = np.random.uniform(-1, 1, self.random_input_dim)
#g_loss = self.model.train_on_batch([noise, text_batch], np.array([1] * batch_size))
#print('\tEpoch: {}, Generator Loss: {}, Discriminator Loss: {}'.format(epoch+1, cum_g_loss/batch_count, cum_d_loss/batch_count))
print(
'\tEpoch: {}, Generator Loss: {}, Generator Accuracy: {}, Discriminator Loss: {}, Disciminator Accuracy: {}'
.format(epoch + 1, cum_g_loss / batch_count,
cum_g_acc / batch_count, cum_d_loss / batch_count,
cum_d_acc / batch_count))
D_L.append(cum_d_loss / batch_count)
D_A.append(cum_d_acc / batch_count)
G_L.append(cum_g_loss / batch_count)
G_A.append(cum_g_acc / batch_count)
#if (epoch * batch_size + batch_index) % 10 == 9:
self.generator.save_weights(
Capsules.get_weight_file_path(model_dir_path, 'generator'),
True)
self.discriminator.save_weights(
Capsules.get_weight_file_path(model_dir_path, 'discriminator'),
True)
self.generator.save_weights(
Capsules.get_weight_file_path(model_dir_path, 'generator'), True)
self.discriminator.save_weights(
Capsules.get_weight_file_path(model_dir_path, 'discriminator'),
True)
callbacks.History()
callbacks.ModelCheckpoint(os.path.join(model_dir_path, 'capgans.h5'),
monitor='cum_d_loss',
verbose=0,
save_best_only=False,
save_weights_only=False,
mode='auto',
period=1)
def fit_and_predict_generator_with_sceneinst_metrics(
model,
generator,
params,
multithreading_metrics=False,
steps_per_epoch=None,
epochs=1,
verbose=1,
callbacks=None,
validation_data=None,
validation_steps=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False,
shuffle=True,
initial_epoch=0):
"""See docstring for `Model.fit_generator`."""
wait_time = 0.01 # in seconds
epoch = initial_epoch
do_validation = bool(validation_data)
model._make_train_function()
if do_validation:
model._make_test_function()
is_sequence = isinstance(generator, Sequence)
if not is_sequence and use_multiprocessing and workers > 1:
warnings.warn(
UserWarning('Using a generator with `use_multiprocessing=True`'
' and multiple workers may duplicate your data.'
' Please consider using the`keras.utils.Sequence'
' class.'))
if steps_per_epoch is None:
if is_sequence:
steps_per_epoch = len(generator)
else:
raise ValueError('`steps_per_epoch=None` is only valid for a'
' generator based on the '
'`keras.utils.Sequence`'
' class. Please specify `steps_per_epoch` '
'or use the `keras.utils.Sequence` class.')
# python 2 has 'next', 3 has '__next__'
# avoid any explicit version checks
val_gen = (hasattr(validation_data, 'next')
or hasattr(validation_data, '__next__')
or isinstance(validation_data, Sequence))
if (val_gen and not isinstance(validation_data, Sequence)
and not validation_steps):
raise ValueError('`validation_steps=None` is only valid for a'
' generator based on the `keras.utils.Sequence`'
' class. Please specify `validation_steps` or use'
' the `keras.utils.Sequence` class.')
# Prepare display labels.
out_labels = model.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
model.history = cbks.History()
_callbacks = [
cbks.BaseLogger(stateful_metrics=model.stateful_metric_names)
]
if verbose:
_callbacks.append(
cbks.ProgbarLogger(count_mode='steps',
stateful_metrics=model.stateful_metric_names))
_callbacks += (callbacks or []) + [model.history]
callbacks = cbks.CallbackList(_callbacks)
# it's possible to callback a different model than self:
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
enqueuer = None
val_enqueuer = None
try:
if do_validation:
if val_gen and workers > 0:
# Create an Enqueuer that can be reused
val_data = validation_data
if isinstance(val_data, Sequence):
val_enqueuer = OrderedEnqueuer(
val_data, use_multiprocessing=use_multiprocessing)
validation_steps = len(val_data)
else:
val_enqueuer = GeneratorEnqueuer(
val_data, use_multiprocessing=use_multiprocessing)
val_enqueuer.start(workers=workers,
max_queue_size=max_queue_size)
val_enqueuer_gen = val_enqueuer.get()
elif val_gen:
val_data = validation_data
if isinstance(val_data, Sequence):
val_enqueuer_gen = iter(val_data)
else:
val_enqueuer_gen = val_data
else:
# Prepare data for validation
if len(validation_data) == 2:
val_x, val_y = validation_data
val_sample_weight = None
elif len(validation_data) == 3:
val_x, val_y, val_sample_weight = validation_data
else:
raise ValueError('`validation_data` should be a tuple '
'`(val_x, val_y, val_sample_weight)` '
'or `(val_x, val_y)`. Found: ' +
str(validation_data))
val_x, val_y, val_sample_weights = model._standardize_user_data(
val_x, val_y, val_sample_weight)
val_data = val_x + val_y + val_sample_weights
if model.uses_learning_phase and not isinstance(
K.learning_phase(), int):
val_data += [0.]
for cbk in callbacks:
cbk.validation_data = val_data
if workers > 0:
if is_sequence:
enqueuer = OrderedEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
shuffle=shuffle)
else:
enqueuer = GeneratorEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
wait_time=wait_time)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
else:
if is_sequence:
output_generator = iter(generator)
else:
output_generator = generator
callback_model.stop_training = False
# Construct epoch logs.
epoch_logs = {}
while epoch < epochs:
# setup scene instance dictionary
model.scene_instance_id_metrics_dict_train = {}
# create thread for asynchronous batch metrics calculation (one thread per epoch, joined before final metrics calculation)
if multithreading_metrics:
label_queue = queue.Queue(
) # threadsafe queue into which we will push (y_pred, y) tuples
trainmetrics_thread = threading.Thread(
target=metrics_per_batch_thread_handler,
args=(label_queue,
model.scene_instance_id_metrics_dict_train,
params['mask_value'], steps_per_epoch))
trainmetrics_thread.start()
#print('thread for calculating the batch train metrics has been started')
for m in model.stateful_metric_functions:
m.reset_states()
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
runtime_generator_cumulated = 0.
runtime_train_and_predict_on_batch_cumulated = 0.
runtime_class_accuracies_cumulated = 0.
skip_runtime_avg = 5 # skipping the first few batches to reduce bias due to inital extra time
while steps_done < steps_per_epoch:
t_start_batch = time()
t_start = time()
generator_output = next(output_generator)
runtime_generator_next = time() - t_start
if batch_index >= skip_runtime_avg:
runtime_generator_cumulated += runtime_generator_next
if not hasattr(generator_output, '__len__'):
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
# build batch logs
batch_logs = {}
if x is None or len(x) == 0:
# Handle data tensors support when no input given
# step-size = 1 for data tensors
batch_size = 1
elif isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
t_start = time()
callbacks.on_batch_begin(batch_index, batch_logs)
runtime_callbacks_on_batch_begin = time() - t_start
# remark on label shape: last (fourth) dimension contains in 0 the true labels, in 1 the corresponding sceneinstid (millioncode)
t_start = time()
# set sample weights
if params['nosceneinstweights']:
sample_weight = None
else:
sample_weight = heiner_calculate_sample_weights_batch(
y[:, :, 0, 1], generator.length_dict,
generator.scene_instance_ids_dict, 'train')
# run forward and backward pass and do the gradient descent step
batch_loss, y_pred_logits, gradient_norm = heiner_train_and_predict_on_batch(
model,
x,
y[:, :, :, 0],
sample_weight=sample_weight,
calc_global_gradient_norm=not params['nocalcgradientnorm'])
runtime_train_and_predict_on_batch = time() - t_start
if batch_index >= skip_runtime_avg:
runtime_train_and_predict_on_batch_cumulated += runtime_train_and_predict_on_batch
batch_logs['loss'] = batch_loss
model.gradient_norm = gradient_norm
t_start = time()
# from logits to predicted class probabilities
y_pred_probs = sigmoid(y_pred_logits,
out=y_pred_logits) # last arg: inplace
# from probabilities to hard class decisions
y_pred = np.greater_equal(
y_pred_probs, params['outputthreshold'],
out=y_pred_probs) # last arg: inplace
# increment metrics for scene instances in batch
if multithreading_metrics:
# the following two arrays need to be unchanged in order for being thread-safe
# assumption 1: batchloader yields array copies (true for moritz loader)
# assumption 2: *_and_predict_on_batch return newly allocated arrays
label_queue.put((y_pred, y))
else:
heiner_calculate_class_accuracies_metrics_per_scene_instance_in_batch(
model.scene_instance_id_metrics_dict_train, y_pred, y,
params['mask_value'])
runtime_class_accuracies = time() - t_start
if batch_index >= skip_runtime_avg:
runtime_class_accuracies_cumulated += runtime_class_accuracies
t_start = time()
callbacks.on_batch_end(batch_index, batch_logs)
runtime_callbacks_on_batch_end = time() - t_start
runtime_batch = time() - t_start_batch
# print((' ----> batch {} in epoch {} took in total {:.2f} sec => generator {:.2f} ' +
# 'train_and_predict {:.2f}, metrics {:.2f}')
# .format(batch_index + 1, epoch + 1, runtime_batch, runtime_generator_next,
# runtime_train_and_predict_on_batch,
# runtime_class_accuracies))
batch_index += 1
steps_done += 1
if steps_done > skip_runtime_avg and steps_done == steps_per_epoch - 1:
print(
' --> batch {} we have average runtimes: generator {:.2f}, train_predict {:.2f}, metrics {:.2f}'
.format(
batch_index, runtime_generator_cumulated /
(steps_done - skip_runtime_avg),
runtime_train_and_predict_on_batch_cumulated /
(steps_done - skip_runtime_avg),
runtime_class_accuracies_cumulated /
(steps_done - skip_runtime_avg)))
# Epoch finished.
if steps_done >= steps_per_epoch and do_validation:
if val_gen:
val_outs = evaluate_and_predict_generator_with_sceneinst_metrics(
model,
val_enqueuer_gen,
params,
multithreading_metrics,
validation_steps,
workers=0,
verbose=1)
else:
# No need for try/except because
# data has already been validated.
val_outs = model.evaluate(
val_x,
val_y,
batch_size=batch_size,
sample_weight=val_sample_weights,
verbose=0)
val_outs = to_list(val_outs)
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
if callback_model.stop_training:
break
if multithreading_metrics:
trainmetrics_thread.join()
print(
' --> both threads for calculating the batch metrics -- training and validation -- finished all their work'
)
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
if callback_model.stop_training:
break
finally:
try:
if enqueuer is not None:
enqueuer.stop()
finally:
if val_enqueuer is not None:
val_enqueuer.stop()
if multithreading_metrics:
trainmetrics_thread.join() # joined again (harmless)
callbacks.on_train_end()
return model.history
def fit_with_pseudo_label(self,
steps_per_epoch,
validation_steps=None,
use_checkpoints=True,
class_labels=None,
verbose=1,
use_multiprocessing=False,
shuffle=False,
workers=1,
max_queue_size=10):
# Default value if validation steps is none
if (validation_steps == None):
validation_steps = self.validation_generator.samples // self.batch_size
wait_time = 0.01 # in seconds
self.model._make_train_function()
# Create a checkpoint callback
checkpoint = ModelCheckpoint("../models_checkpoints/" +
str(self.h5_filename) + ".h5",
monitor='val_acc',
verbose=1,
save_best_only=True,
save_weights_only=True,
mode='auto',
period=1)
# Generate callbacks
callback_list = []
if use_checkpoints:
callback_list.append(checkpoint)
# Init train counters
epoch = 0
validation_data = self.validation_generator
do_validation = bool(validation_data)
self.model._make_train_function()
if do_validation:
self.model._make_test_function()
val_gen = (hasattr(validation_data, 'next')
or hasattr(validation_data, '__next__')
or isinstance(validation_data, Sequence))
if (val_gen and not isinstance(validation_data, Sequence)
and not validation_steps):
raise ValueError('`validation_steps=None` is only valid for a'
' generator based on the `keras.utils.Sequence`'
' class. Please specify `validation_steps` or use'
' the `keras.utils.Sequence` class.')
# Prepare display labels.
out_labels = self.model.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# Prepare train callbacks
self.model.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callback_list or []) + \
[self.model.history]
if verbose:
callbacks += [cbks.ProgbarLogger(count_mode='steps')]
callbacks = cbks.CallbackList(callbacks)
# it's possible to callback a different model than self:
if hasattr(self.model, 'callback_model') and self.model.callback_model:
callback_model = self.model.callback_model
else:
callback_model = self.model
callbacks.set_model(callback_model)
is_sequence = isinstance(self.train_generator, Sequence)
if not is_sequence and use_multiprocessing and workers > 1:
warnings.warn(
UserWarning('Using a generator with `use_multiprocessing=True`'
' and multiple workers may duplicate your data.'
' Please consider using the`keras.utils.Sequence'
' class.'))
if is_sequence:
steps_per_epoch = len(self.train_generator)
enqueuer = None
val_enqueuer = None
callbacks.set_params({
'epochs': self.epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
try:
if do_validation and not val_gen:
# Prepare data for validation
if len(validation_data) == 2:
val_x, val_y = validation_data
val_sample_weight = None
elif len(validation_data) == 3:
val_x, val_y, val_sample_weight = validation_data
else:
raise ValueError('`validation_data` should be a tuple '
'`(val_x, val_y, val_sample_weight)` '
'or `(val_x, val_y)`. Found: ' +
str(validation_data))
val_x, val_y, val_sample_weights = self.model._standardize_user_data(
val_x, val_y, val_sample_weight)
val_data = val_x + val_y + val_sample_weights
if self.model.uses_learning_phase and not isinstance(
K.learning_phase(), int):
val_data += [0.]
for cbk in callbacks:
cbk.validation_data = val_data
if is_sequence:
enqueuer = OrderedEnqueuer(
self.train_generator,
use_multiprocessing=use_multiprocessing,
shuffle=shuffle)
else:
enqueuer = GeneratorEnqueuer(
self.train_generator,
use_multiprocessing=use_multiprocessing,
wait_time=wait_time)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
# Train the model
# Construct epoch logs.
epoch_logs = {}
# Epochs
while epoch < self.epochs:
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
# Steps per epoch
while steps_done < steps_per_epoch:
generator_output = next(output_generator)
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
#==========================
# Mini-batch
#==========================
if (self.print_pseudo_generate):
print ''
print 'Generating pseudo-labels...'
verbose = 1
else:
verbose = 0
if self.no_label_generator.samples > 0:
no_label_output = self.model.predict_generator(
self.no_label_generator,
self.no_label_generator.samples,
verbose=verbose)
# One-hot encoded
self.no_label_generator.classes = np.argmax(
no_label_output, axis=1)
# Concat Pseudo labels with true labels
x_pseudo, y_pseudo = next(self.no_label_generator)
x, y = np.concatenate((x, x_pseudo),
axis=0), np.concatenate(
(y, y_pseudo), axis=0)
# build batch logs
batch_logs = {}
if isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
# Runs a single gradient update on a single batch of data
scalar_training_loss = self.model.train_on_batch(x=x, y=y)
if not isinstance(scalar_training_loss, list):
scalar_training_loss = [scalar_training_loss]
for l, o in zip(out_labels, scalar_training_loss):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
#==========================
# end Mini-batch
#==========================
batch_index += 1
steps_done += 1
if steps_done >= steps_per_epoch and do_validation:
if val_gen:
val_outs = self.model.evaluate_generator(
validation_data,
validation_steps,
workers=workers,
use_multiprocessing=use_multiprocessing,
max_queue_size=max_queue_size)
else:
# No need for try/except because
# data has already been validated.
val_outs = self.model.evaluate(
val_x,
val_y,
batch_size=batch_size,
sample_weight=val_sample_weights,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
# Epoch finished.
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
finally:
try:
if enqueuer is not None:
enqueuer.stop()
finally:
if val_enqueuer is not None:
val_enqueuer.stop()
callbacks.on_train_end()
return self.model.history
