def NetworkTrain(self, model: keras.engine.training.Model, train_x: list,
train_y: np.ndarray):
try:
print("#######################################\n")
print("########### Starts Training ###########")
base_lr = BaseLogger()
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.2,
patience=5,
min_lr=self._min_lr,
verbose=self.VERBOSE)
es = EarlyStopping(monitor='val_loss', mode='min', patience=100)
#mc = ModelCheckpoint(self.MODEL_DESC+'best_model.h5', monitor='val_acc', mode='max', verbose=1, save_best_only=True)
return model.fit(train_x,
train_y,
batch_size=self.BATCH_SIZE,
epochs=self.EPOCHS,
verbose=self.VERBOSE,
shuffle=self.SHUFFLE_DATASET,
validation_split=self.VALIDATION_SPLIT,
callbacks=[base_lr, reduce_lr, es])
except Exception as ex:
template = "An exception of type {0} occurred in [Main.NetworkTrain]. Arguments:\n{1!r}"
message = template.format(type(ex).__name__, ex.args)
print(message)
print(ex)
def train(model, X_train, X_test, epochs, batch_size, modeldir, logdir):
modelpath = modeldir + "/" + "autoencoder_epoch-{epoch:02d}_loss-{loss:.4f}_valloss-{val_loss:.4f}.h5"
CHK = ModelCheckpoint(filepath=modelpath, verbose=0, save_best_only=True)
# CHK = ModelCheckpoint(modelpath, monitor="val_kauc", verbose=1, save_best_only=True, save_weights_only=False, mode="max", period=1)
TB = TensorBoard(log_dir=logdir,
histogram_freq=0,
write_graph=True,
write_images=False)
ES = EarlyStopping(monitor="val_loss", min_delta=ES_MIN_DELTA, patience=ES_PATIENCE, verbose=2, mode="auto")
TB = TensorBoard(log_dir=logdir, histogram_freq=0, write_graph=True, write_images=False)
RLRP = ReduceLROnPlateau(monitor="val_loss", factor=RLRP_FACTOR, patience=RLRP_PATIENCE, min_lr=RLRP_MIN_LR)
BL = BaseLogger()
# NOTE: training on X_train/X_train (not Y_train) since autoencoder
# tries to train only on good data (errors in prediction will be bad values).
# Similarly, the validation set is X_test/X_test.
print("X_train: ", X_train.shape)
print("X_test: ", X_test.shape)
history = model.fit(X_train, X_train,
epochs=epochs, batch_size=batch_size,
validation_split=0.2,
callbacks=[CHK, ES, RLRP, TB])
# history = model.fit(
# train_x, train_y,
# epochs=epochs, batch_size=batch_size,
# class_weight=class_weight,
# callbacks=[ES, TB, BL, RLRP, roc_callback])
return history
def get_callbacks():
callbacks = list()
callbacks.append(EarlyStopping(monitor='val_loss', patience=100,
verbose=1))
weights_file = os.path.join("weights",
"weights_ep_{epoch:02d}_{val_acc:.5f}.hd5f")
callbacks.append(
ModelCheckpoint(weights_file,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max',
save_weights_only=True))
callbacks.append(
ReduceLROnPlateau(monitor='val_loss',
verbose=1,
factor=0.3,
patience=30,
min_lr=MIN_LR))
callbacks.append(BaseLogger())
callbacks.append(TensorBoard())
return callbacks
def fit(self,
x,
y,
epochs=25,
callbacks=[BaseLogger()],
validation_split=0.1):
self.model.fit(x=x,
y=y,
epochs=epochs,
validation_split=validation_split,
callbacks=callbacks)
def test_training(self):
#np.save("resources/XY_train/X_copy.npy", self.X)
callback = [BaseLogger()]
model: Model = training(model=self.model,
X=self.X,
Y=self.Y,
callbacks=callback)
loss, acc = model.evaluate(self.X_dev, self.Y_dev)
print(acc) # 9312872886657715
model.save('resources/model/test_model.h5')
def fit(self, timeseries, lag=7, epochs=10000, verbose=0, optimizer='sgd'):
self.timeseries = np.array(
timeseries, dtype="float64"
) # Apply log transformation por variance stationarity
self.lag = lag
self.y = None
self.n = len(timeseries)
if self.lag >= self.n:
raise ValueError("Lag is higher than length of the timeseries")
self.X = np.zeros((self.n - self.lag, self.lag), dtype="float64")
with np.errstate(divide='ignore'):
self.y = np.log(self.timeseries[self.lag:])
self.epochs = epochs
self.scaler = StandardScaler()
self.verbose = verbose
self.optimizer = optimizer
# Building X matrix
for i in range(0, self.n - lag):
self.X[i, :] = self.timeseries[range(i, i + lag)]
#
# if verbose:
# print "Scaling data"
self.scaler.fit(self.X)
self.X = self.scaler.transform(self.X)
# Neural net architecture
self.nn = Sequential()
self.nn.add(Dense(self.X.shape[1], self.hidden_layers[0]))
self.nn.add(Activation(self.activation_functions[0]))
self.nn.add(Dropout(0.25))
for i, layer in enumerate(self.hidden_layers[:-1]):
self.nn.add(Dense(self.hidden_layers[i],
self.hidden_layers[i + 1]))
self.nn.add(Activation(self.activation_functions[i]))
self.nn.add(Dropout(0.25))
# Add final node
self.nn.add(Dense(self.hidden_layers[-1], 1))
self.nn.compile(loss='mean_absolute_error', optimizer=self.optimizer)
# Train neural net
self.nn.fit(self.X,
self.y,
nb_epoch=self.epochs,
verbose=self.verbose,
callbacks=[BaseLogger()])
def _load_default_callbacks(self):
"""Return default callbacks automatically applied during training
By default, the following callbacks are automatically applied during
training:
- tensorflow.keras.callbacks.BaseLogger
- tensorflow.keras.callbacks.ProgbarLogger
- tensorflow.keras.callbacks.History (which is the `Model.history`
attribute set in `Model.__init__`)
:return: callbacks automatically applied to every Model
:rtype: list
"""
default_callbacks = [BaseLogger(), self.history]
return default_callbacks
def Train(self,
train_x,
train_y,
val_split: float = 0.2,
eps: int = 20,
batches: int = 64):
"""
This method executes the model training process.
:param train_x: train input
:param train_y: train desired result
:param val_split:float: validation split of the train set
:param eps:int: train epochs
:param batches:int: the dataset batch size
"""
try:
verbose: int = 1
base_lr = BaseLogger()
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.02,
patience=5,
min_lr=0.000125,
verbose=verbose)
es = EarlyStopping(monitor='val_loss',
mode='min',
verbose=verbose,
patience=75)
mc = ModelCheckpoint(self._model_folder + 'best_model.h5',
monitor='val_acc',
mode='max',
verbose=verbose,
save_best_only=True)
return self._model.fit(train_x,
train_y,
validation_split=val_split,
epochs=eps,
batch_size=batches,
shuffle=True,
callbacks=[base_lr, reduce_lr, es, mc])
except Exception as ex:
template = "An exception of type {0} occurred in [Model.Train]. Arguments:\n{1!r}"
message = template.format(type(ex).__name__, ex.args)
print(message)
def fit_lstm(X, y, batch_size, nb_epoch, neurons):
#X, y = train[:, 0:-1], train[:, -1]
# X = X.reshape(X.shape[0], 1, X.shape[1])
# print('X=', X, 'y=', y)
print('X=', X.shape, 'y=', y.shape)
model = Sequential()
model.add(
LSTM(neurons,
batch_input_shape=(batch_size, X.shape[1], X.shape[2]),
stateful=True))
model.add(Dense(X.shape[2]))
model.compile(loss='mean_squared_error', optimizer='adam')
for i in range(nb_epoch):
print("epoch:", i)
model.fit(X,
y,
epochs=1,
batch_size=batch_size,
verbose=0,
shuffle=False,
callbacks=[BaseLogger()])
model.reset_states()
return model
def _prepare_callbacks(self, callbacks: List[Callback],
val_ins: List[numpy.array], epochs: int,
batch_size: int, num_train_samples: int,
callback_metrics: List[str], do_validation: bool,
verbose: int):
"""
Sets up Keras callbacks to perform various monitoring functions during training.
"""
self.history = History() # pylint: disable=attribute-defined-outside-init
callbacks = [BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
callbacks += [ProgbarLogger()]
callbacks = CallbackList(callbacks)
# it's possible to callback a different model than self
# (used by Sequential models).
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self # pylint: disable=redefined-variable-type
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
for cbk in callbacks:
cbk.validation_data = val_ins
return callbacks, callback_model
def train(model, train_x, train_y, epochs, batch_size, modeldir, logdir):
class_weight = dict(
zip([0, 1], compute_class_weight('balanced', [0, 1], train_y)))
print("Unbalanced data, actual class_weight: ", class_weight)
# class_weight = {0: 1, 1: 1000}
print("Unbalanced data, using class_weight: ", class_weight)
modelpath = modeldir + '/' + 'model_epoch-{epoch:02d}_loss-{loss:.4f}_valloss-{val_loss:.4f}_val_kauc-{val_kauc:.4f}.h5'
# CHK = ModelCheckpoint(modelpath, monitor="val_kauc", verbose=1, save_best_only=True, save_weights_only=False, mode="max", period=1)
ES = EarlyStopping(monitor="val_kauc",
min_delta=ES_MIN_DELTA,
patience=ES_PATIENCE,
verbose=2,
mode="max")
TB = TensorBoard(log_dir=logdir,
histogram_freq=0,
write_graph=True,
write_images=False)
RLRP = ReduceLROnPlateau(monitor="val_loss",
factor=RLRP_FACTOR,
patience=RLRP_PATIENCE,
min_lr=RLRP_MIN_LR)
BL = BaseLogger()
roc_callback = ROC_Callback()
history = model.fit(train_x,
train_y,
validation_split=0.2,
epochs=epochs,
batch_size=batch_size,
class_weight=class_weight,
callbacks=[ES, TB, BL, RLRP, roc_callback])
return history
model.add(Dense(50))
model.add(Activation('relu'))
print(model.output_shape)
model.add(Dense(1))
model.add(Activation('tanh'))
print(model.output_shape)
if save_model_and_data:
print("--------------- Save model architecture -----------------")
json_string = model.to_json()
f = open('model.json', 'w')
f.write(json_string)
f.close()
print("--------------- TRAINING THE MODEL -----------------")
# model.compile(loss='mse', optimizer='rmsprop')
model.compile(loss='mse', optimizer='adam')
# model.compile(loss='mse', optimizer='rmsprop')
model.fit(X_train,
y_train,
batch_size=batch,
nb_epoch=epoch,
verbose=1,
callbacks=[BaseLogger()],
validation_split=validation)
print("--------------- Save weights -----------------")
model.save_weights('model.h5')
print("--------------- ALL DONE -----------------")
models['adam'] = create_model()
models['adam'].summary()
plot_model(models['adam'], to_file = model_name + '-model.png', show_shapes=False)
Image(filename = model_name + '-model.png')
"""## Train Model"""
s1_data.shape
results = []
for name, model in models.items():
callbacks = [
BaseLogger(),
ReduceLROnPlateau(monitor = 'val_loss', factor=0.2, patience=5, min_lr=0.001),
TensorBoard(log_dir='./' + model_name + '-' + name + '-logs', histogram_freq=0, write_graph=True, write_images=True),
ModelCheckpoint(model_name + '-' + name + '-checkpoint-weights.{epoch:02d}-{val_acc:.2f}.hdf5', monitor='val_acc', save_best_only=True)
]
start_time = time.time()
print('')
print('Start learning %s at %d' % (name, start_time))
print('Epochs: %d' % epochs)
print('Batch size: %d' % batch_size)
history = model.fit([s1_data, s2_data],
labels,
epochs = 30,
batch_size = batch_size,
def _train_by_batch(self):
# batch finite generator should be loaded within epoch loop
logger.info('Start training by batch')
self.validation_xy = self.load_data('val', feed_mode='all')
do_validation = bool(self.validation_xy)
# prepare display labels in tensorboard
out_labels = self.model._get_deduped_metrics_names()
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
self.model.history = History()
callbacks = [BaseLogger()] + (self.callbacks
or []) + [self.model.history]
# callbacks = (self.callbacks or []) + [self.model.history]
if self.verbose:
callbacks += [ProgbarLogger(count_mode='samples')]
callbacks = CallbackList(callbacks)
# it's possible to callback a different model than this model
if hasattr(self.model, 'callback_model') and self.model.callback_model:
callback_model = self.model.callback_model
else:
callback_model = self.model
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': self.epochs,
'samples': self.data.nb_train,
'verbose': self.verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
for epoch in range(self.epochs):
start_e = time()
callbacks.on_epoch_begin(epoch)
xy_gen = self.load_data('train', feed_mode='batch')
logger.info('New training epoch')
for batch_index, (x, y) in enumerate(xy_gen):
# build batch logs
batch_logs = {}
if isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
outs = self.model.train_on_batch(x, y)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if (batch_index + 1) % 1000 == 0 and do_validation:
val_outs = self.model.evaluate(*self.validation_xy,
batch_size=81920,
verbose=0)
batch_logs = {}
if not isinstance(val_outs, list):
val_outs = [val_outs]
for l, o in zip(out_labels, val_outs):
batch_logs['val_' + l] = o
print(' - Eval inside: %.6f' % val_outs[0])
for cb in self.callbacks:
if cb.__class__ == tensorBoard:
cb.on_batch_end(batch_index,
batch_logs,
count=False)
epoch_logs = {}
if do_validation:
val_outs = self.model.evaluate(*self.validation_xy,
batch_size=81920,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_batch_end(epoch, epoch_logs)
callbacks.on_epoch_end(epoch, epoch_logs)
elapsed_e = timedelta(seconds=int(time() - start_e))
self.send_metric('elapsed_per_epoch', elapsed_e)
if not self.no_save and do_validation and (epoch !=
self.epochs - 1):
self.model.save(
'results/trained_models/%s_ctr_model_%.4f_epoch_%d.h5' %
(self.sess_id, val_outs[0], epoch))
callbacks.on_train_end()
return self.model.history
def predict_image(version, image_path, batch_size, overlap, data_format=None):
def current_time_millis():
return int(round(time.time() * 1000))
def offset(size, diff, overlap):
return math.floor(diff / math.ceil(diff / (size * (1 - overlap))))
def map_c(i, j, b, l):
return int(((i * b) + j) / l)
def map_r(i, j, b, l):
return ((i * b) + j) % l
if data_format is None:
data_format = K.image_data_format()
if data_format not in {'channels_first', 'channels_last'}:
raise ValueError('Unknown data_format:', data_format)
path = version.model_file.name
print(_('Loading model "%(path)s".') % {'path': path})
model = load_model(os.path.join(settings.MEDIA_ROOT, path))
if len(model.inputs) != 1:
raise RuntimeError('Models with more than one input are not'
' supported at the moment.')
inputs = []
for i in range(len(model.inputs)):
name = model.inputs[i].name
pos = min(
name.index('/') if '/' in name else len(name),
name.index(':') if ':' in name else len(name))
name = name[:pos]
inputs.append({'shape': model.inputs[i].shape.as_list(), 'name': name})
if data_format == 'channels_first':
inputs[i]['grayscale'] = inputs[i]['shape'][1] == 1
inputs[i]['r'] = inputs[i]['shape'][2]
inputs[i]['c'] = inputs[i]['shape'][3]
elif data_format == 'channels_last':
inputs[i]['r'] = inputs[i]['shape'][1]
inputs[i]['c'] = inputs[i]['shape'][2]
inputs[i]['grayscale'] = inputs[i]['shape'][3] == 1
inputs[i]['img'] = img_to_array(
load_img(image_path, inputs[i]['grayscale']))
inputs[i]['img'] *= 1. / 255
if data_format == 'channels_first':
inputs[i]['img_r'] = inputs[i]['img'].shape[1]
inputs[i]['img_c'] = inputs[i]['img'].shape[2]
elif data_format == 'channels_last':
inputs[i]['img_r'] = inputs[i]['img'].shape[0]
inputs[i]['img_c'] = inputs[i]['img'].shape[1]
inputs[i]['diff_r'] = inputs[i]['img_r'] - inputs[i]['r']
inputs[i]['diff_c'] = inputs[i]['img_c'] - inputs[i]['c']
inputs[i]['offset_r'] = offset(inputs[i]['r'], inputs[i]['diff_r'],
overlap)
inputs[i]['offset_c'] = offset(inputs[i]['c'], inputs[i]['diff_c'],
overlap)
inputs[i]['nb_r'] = math.ceil(
inputs[i]['diff_r'] / inputs[i]['offset_r']) + 1
inputs[i]['nb_c'] = math.ceil(
inputs[i]['diff_c'] / inputs[i]['offset_c']) + 1
inputs = inputs[0]
N = inputs['nb_r'] * inputs['nb_c']
steps = math.ceil(N / batch_size)
metrics = []
outputs = []
for i in range(len(model.outputs)):
tshape = model.outputs[i].shape.as_list()
name = model.outputs[i].name
pos = min(
name.index('/') if '/' in name else len(name),
name.index(':') if ':' in name else len(name))
name = name[:pos]
activation = model.get_layer(name).activation.__name__.lower()
outputs.append({'name': name, 'shape': tshape})
if len(tshape) == 2:
if activation == 'softmax':
outputs[i]['t'] = 'class'
else:
outputs[i]['t'] = 'multi'
nb_classes = tshape[1]
if nb_classes is None:
nb_classes = model.get_layer(name).output_shape[1]
nb_classes = int(nb_classes)
metrics += ['%s:%s' % (name, i) for i in range(nb_classes)]
if data_format == 'channels_first':
shape = (nb_classes, inputs['nb_r'], inputs['nb_c'])
elif data_format == 'channels_last':
shape = (inputs['nb_r'], inputs['nb_c'], nb_classes)
elif len(tshape) == 4:
if activation == 'softmax':
outputs[i]['t'] = 'class'
else:
outputs[i]['t'] = 'img'
shape = (inputs['nb_r'], inputs['nb_c']) + tuple(tshape[1:])
outputs[i]['p'] = np.zeros(shape)
history = History()
callbacks = CallbackList([BaseLogger(), history, ProgbarLogger()])
callbacks.set_model(model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': 1,
'steps': steps,
'samples': N,
'verbose': 1,
'do_validation': False,
'metrics': metrics,
})
callbacks.on_train_begin()
callbacks.on_epoch_begin(0)
start_time = current_time_millis()
for b in range(steps):
current_index = (b * batch_size) % N
if N >= current_index + batch_size:
current_batch_size = batch_size
else:
current_batch_size = N - current_index
batch_logs = {'batch': b, 'size': current_batch_size}
for metric in metrics:
batch_logs[metric] = 0
callbacks.on_batch_begin(b, batch_logs)
bX = np.zeros((current_batch_size, ) + tuple(inputs['shape'][1:]))
for j in range(current_batch_size):
idx_r = map_r(b, j, batch_size, inputs['nb_r'])
idx_c = map_c(b, j, batch_size, inputs['nb_r'])
top = min(idx_r * inputs['offset_r'],
inputs['img_r'] - inputs['r'])
bottom = min(idx_r * inputs['offset_r'] + inputs['r'],
inputs['img_r'])
left = min(idx_c * inputs['offset_c'],
inputs['img_c'] - inputs['c'])
right = min(idx_c * inputs['offset_c'] + inputs['c'],
inputs['img_c'])
if data_format == 'channels_first':
bX[j] = inputs['img'][:, top:bottom, left:right]
elif data_format == 'channels_last':
bX[j] = inputs['img'][top:bottom, left:right, :]
p = model.predict_on_batch(bX)
if type(p) != list:
p = [p]
for j in range(current_batch_size):
for i in range(len(outputs)):
idx_r = map_r(b, j, batch_size, inputs['nb_r'])
idx_c = map_c(b, j, batch_size, inputs['nb_r'])
if len(outputs[i]['p'].shape) == 3:
if data_format == 'channels_first':
outputs[i]['p'][:, idx_r, idx_c] = p[i][j]
elif data_format == 'channels_last':
outputs[i]['p'][idx_r, idx_c, :] = p[i][j]
metric = metrics[p[i][j].argmax()]
batch_logs[metric] += 1. / current_batch_size
elif len(outputs[i]['p'].shape) == 5:
outputs[i]['p'][idx_r, idx_c, :, :, :] = p[i][j]
callbacks.on_batch_end(b, batch_logs)
runtime = (current_time_millis() - start_time) / 1000
callbacks.on_epoch_end(0, {'runtime': runtime})
callbacks.on_train_end()
for i in range(len(outputs)):
if len(outputs[i]['shape']) == 2:
if data_format == 'channels_first':
shape = (outputs[i]['p'].shape[0], inputs['img_r'],
inputs['img_c'])
elif data_format == 'channels_last':
shape = (inputs['img_r'], inputs['img_c'],
outputs[i]['p'].shape[2])
elif len(tshape) == 4:
if data_format == 'channels_first':
shape = (outputs[i]['p'].shape[2], inputs['img_r'],
inputs['img_c'])
elif data_format == 'channels_last':
shape = (inputs['img_r'], inputs['img_c'],
outputs[i]['p'].shape[4])
count = np.zeros(shape)
outputs[i]['img'] = np.zeros(shape)
if len(outputs[i]['p'].shape) == 3:
if data_format == 'channels_first':
nb_rows = outputs[i]['p'].shape[1]
nb_cols = outputs[i]['p'].shape[2]
elif data_format == 'channels_last':
nb_rows = outputs[i]['p'].shape[0]
nb_cols = outputs[i]['p'].shape[1]
elif len(outputs[i]['p'].shape) == 5:
nb_rows = outputs[i]['p'].shape[0]
nb_cols = outputs[i]['p'].shape[1]
for j in range(nb_rows):
for k in range(nb_cols):
top = min(j * inputs['offset_r'],
inputs['img_r'] - inputs['r'])
bottom = min(j * inputs['offset_r'] + inputs['r'],
inputs['img_r'])
left = min(k * inputs['offset_c'],
inputs['img_c'] - inputs['c'])
right = min(k * inputs['offset_c'] + inputs['c'],
inputs['img_c'])
if data_format == 'channels_first':
outputs[i]['img'][:, top:bottom, left:right] += \
outputs[i]['p'][:, j, k]
count[:, top:bottom, left:right] += 1
elif data_format == 'channels_last':
outputs[i]['img'][top:bottom, left:right, :] += \
outputs[i]['p'][j, k, :]
count[top:bottom, left:right, :] += 1
outputs[i]['img'] /= count
del outputs[i]['p']
del outputs[i]['shape']
return history.history, outputs
kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed)
# Define a model generator
def generate_model():
_model = Sequential()
_model.add(Dense(22, input_dim=28))
_model.add(SReLU())
_model.add(Dropout(0.2))
_model.add(Dense(1, activation='sigmoid'))
_model.compile(loss='binary_crossentropy', optimizer='adam')
return _model
# Define callbacks
baselogger = BaseLogger()
earlystop = EarlyStopping(monitor='val_loss',
min_delta=1e-4,
patience=5,
verbose=0,
mode='auto')
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.2,
patience=5,
min_lr=0.001)
tensor_board = TensorBoard(log_dir='./logs',
histogram_freq=0,
write_graph=True,
write_images=False)
# Storage
def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
log_path = 'logging/vae_nlg_{}'.format(int(round(time.time() * 1000)))
os.mkdir(log_path)
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='{}/evolution.log'.format(log_path),
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
batch_size = config_data['batch_size']
epochs = config_data['nb_epochs']
discriminator_iterations = config_data['discriminator_iterations']
tweets_path = config_data['tweets_path']
vocab_path = config_data['vocab_path']
vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb'))
#== == == == == == =
# Load all the Data
#== == == == == == =
noutputs = 5
logging.info('Load Training Data')
train_input, train_output = load_text_pairs(
join(tweets_path, 'training_set.tsv'), config_data, vocab,
noutputs)
logging.info('Load Validation Data')
valid_input, valid_output = load_text_pairs(
join(tweets_path, 'vaild_set.tsv'), config_data, vocab, noutputs)
logging.info('Load Output Validation Data')
valid_dev_input, valid_dev_output = load_text_pairs(
join(tweets_path, 'test_set.tsv'), config_data, vocab, noutputs)
#train_input = [x[:1213] for x in train_input]
#train_output = [x[:1213] for x in train_output]
noise_valid_input = np.zeros(shape=(valid_input[0].shape[0],
config_data['z_size']))
step = K.variable(1.)
steps_per_epoch = ceil(train_output[0].shape[0] /
config_data['batch_size'])
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
vae_model, vae_model_test, decoder_discr_model, decoder_test_model, discriminator_model, discriminator = get_vae_gan_model(
config_data, vocab, step)
with open(os.path.join(log_path, 'models.txt'), 'wt') as fh:
fh.write('VAE Model\n')
fh.write('---------\n')
vae_model.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Test\n')
fh.write('--------------\n')
vae_model_test.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Decoder Discriminator Model\n')
fh.write('---------------------------\n')
decoder_discr_model.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Decoder Test Model\n')
fh.write('---------------------------\n')
decoder_test_model.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Discriminator Model\n')
fh.write('-------------------\n')
discriminator_model.summary(print_fn=lambda x: fh.write(x + '\n'))
terminate_on_nan = TerminateOnNaN()
model_checkpoint = ModelCheckpoint(
'models/vae_model/weights.{epoch:02d}.hdf5',
period=10,
save_weights_only=True)
enc_out_labels = [
'enc_' + s for s in vae_model._get_deduped_metrics_names()
]
dec_out_labels = [
'dec_' + s
for s in decoder_discr_model._get_deduped_metrics_names()
]
dis_out_labels = [
'dis_' + s
for s in discriminator_model._get_deduped_metrics_names()
]
out_labels = enc_out_labels + dec_out_labels + [
'dis_real', 'dis_gen', 'dis_noise'
]
#out_labels = full_model._get_deduped_metrics_names()
callback_metrics = out_labels + ['val_' + n for n in out_labels]
step_callback = StepCallback(step, steps_per_epoch)
output_callback = GANOutputCallback(
vae_model_test,
valid_dev_input[0],
1,
vocab,
'',
fname='{}/test_output'.format(log_path))
callbacks = CallbackList([
BaseLogger(),
ProgbarLogger(count_mode='steps'), step_callback, output_callback,
model_checkpoint, terminate_on_nan
])
callbacks.set_model(vae_model_test)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': True,
'do_validation': True,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
initial_epoch = 0
num_train_samples = train_input[0].shape[0]
index_array = np.arange(num_train_samples)
steps = 0
epoch = initial_epoch
while epoch < epochs:
epoch_logs = {}
callbacks.on_epoch_begin(epoch)
index_array = _batch_shuffle(index_array, batch_size)
steps_done = 0
batches = _make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_logs = {}
batch_ids = index_array[batch_start:batch_end]
X = _slice_arrays(train_input, batch_ids)
y = _slice_arrays(train_output, batch_ids)
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
set_trainability(discriminator, trainable=False)
enc_outs = vae_model.train_on_batch(x=X, y=y[:3])
set_trainability(discriminator, trainable=True)
list_disc_loss_real = []
list_disc_loss_gen = []
list_disc_loss_noise = []
if steps < 25 or steps % 500 == 0:
disc_iterations = 100
else:
disc_iterations = discriminator_iterations
noise_input = np.zeros(shape=(len(batch_ids),
config_data['z_size']))
for disc_it in range(disc_iterations):
#clip_weights(discriminator)
real_idx = np.random.choice(train_input[0].shape[0],
len(batch_ids),
replace=False)
train_real_batch = [x[real_idx] for x in train_input]
#train on real data
x_fake = vae_model_test.predict_on_batch(
x=train_real_batch[0])
x_noise_fake = decoder_test_model.predict_on_batch(
x=noise_input)
train_input_discr = np.concatenate(
(train_real_batch[0], train_real_batch[0],
train_real_batch[0]))
train_output_discr = np.concatenate(
(train_real_batch[1], x_fake, x_noise_fake))
labels = np.asarray(
len(batch_ids) * [1] + 2 * len(batch_ids) * [-1])
index_array_discr = np.arange(len(labels))
np.random.shuffle(index_array_discr)
discr_batch = [
train_input_discr[index_array_discr],
train_output_discr[index_array_discr]
]
discr_batch_labels = labels[index_array_discr]
dis_outs_real = discriminator_model.train_on_batch(
discr_batch, discr_batch_labels)
#dis_outs_real = discriminator_model.train_on_batch(train_real_batch, -np.ones(shape=(len(batch_ids), 1)))
#dis_outs_gen = discriminator_model.train_on_batch([train_real_batch[0], x_fake], np.ones(shape=(len(batch_ids), 1)))
#dis_outs_gen_noise = discriminator_model.train_on_batch([train_real_batch[0], x_noise_fake], np.ones(shape=(len(batch_ids), 1)))
list_disc_loss_real.append(dis_outs_real)
#list_disc_loss_gen.append(dis_outs_gen)
#list_disc_loss_noise.append(dis_outs_gen_noise)
loss_d_real = -np.mean(list_disc_loss_real)
loss_d_gen = np.mean(list_disc_loss_gen)
loss_d_noise = np.mean(list_disc_loss_noise)
set_trainability(discriminator, trainable=False)
decoder_discr_input = [X[0], noise_input]
dec_outs = decoder_discr_model.train_on_batch(
x=decoder_discr_input,
y=-np.ones(shape=(len(batch_ids), 1)))
outs = enc_outs + [dec_outs] + [loss_d_real]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
batch_index += 1
steps_done += 1
steps += 1
# Epoch finished.
if steps_done >= steps_per_epoch:
valid_len = valid_output[0].shape[0]
enc_val_outs = vae_model.evaluate(valid_input,
valid_output[:3],
verbose=False)
dec_val_outs = decoder_discr_model.evaluate(
[valid_input[0], noise_valid_input],
-np.ones(shape=(valid_len, 1)),
verbose=False)
dis_val_outs = discriminator_model.evaluate(
valid_input,
-np.ones(shape=(valid_len, 1)),
verbose=False)
val_outs = enc_val_outs + [dec_val_outs] + [dis_val_outs]
#val_outs = full_model.evaluate(valid_input, valid_output, verbose=False)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
callbacks.on_train_end()
def fit(self,
X_train,
Y_train,
X_test,
modeldir=".",
logdir=".",
epochs=2,
batch_size=20000):
max_app = self._config["max_app"]
max_ch = self._config["max_ch"]
max_dev = self._config["max_dev"]
max_os = self._config["max_os"]
max_h = self._config["max_h"]
max_dqh = self._config["max_dqh"]
max_qh = self._config["max_qh"]
max_d = self._config["max_d"]
max_wd = self._config["max_wd"]
max_qty = self._config["max_qty"]
max_c1 = self._config["max_c1"]
max_c2 = self._config["max_c2"]
max_c3 = self._config["max_c3"]
max_c4 = self._config["max_c4"]
max_c5 = self._config["max_c5"]
X_train = self.convert(X_train)
# Establish the network
emb_n = 100
dense_n = 1000
in_app = Input(shape=[1], name='app')
emb_app = Embedding(max_app, emb_n)(in_app)
in_ch = Input(shape=[1], name='ch')
emb_ch = Embedding(max_ch, emb_n)(in_ch)
in_dev = Input(shape=[1], name='dev')
emb_dev = Embedding(max_dev, emb_n)(in_dev)
in_os = Input(shape=[1], name='os')
emb_os = Embedding(max_os, emb_n)(in_os)
in_h = Input(shape=[1], name='h')
emb_h = Embedding(max_h, emb_n)(in_h)
in_dqh = Input(shape=[1], name='dqh')
emb_dqh = Embedding(max_dqh, emb_n)(in_dqh)
in_qh = Input(shape=[1], name='qh')
emb_qh = Embedding(max_qh, emb_n)(in_qh)
in_d = Input(shape=[1], name='d')
emb_d = Embedding(max_d, emb_n)(in_d)
in_wd = Input(shape=[1], name='wd')
emb_wd = Embedding(max_wd, emb_n)(in_wd)
in_qty = Input(shape=[1], name='qty')
emb_qty = Embedding(max_qty, emb_n)(in_qty)
in_c1 = Input(shape=[1], name='c1')
emb_c1 = Embedding(max_c1, emb_n)(in_c1)
in_c2 = Input(shape=[1], name='c2')
emb_c2 = Embedding(max_c2, emb_n)(in_c2)
in_c3 = Input(shape=[1], name='c3')
emb_c3 = Embedding(max_c3, emb_n)(in_c3)
in_c4 = Input(shape=[1], name='c4')
emb_c4 = Embedding(max_c4, emb_n)(in_c4)
in_c5 = Input(shape=[1], name='c5')
emb_c5 = Embedding(max_c5, emb_n)(in_c5)
fe = concatenate([(emb_app), (emb_ch), (emb_dev), (emb_os), (emb_h),
(emb_qh), (emb_dqh), (emb_d), (emb_wd), (emb_qty),
(emb_c1), (emb_c2), (emb_c3), (emb_c4), (emb_c5)])
s_dout = SpatialDropout1D(0.2)(fe)
fl = Flatten()(s_dout)
x = Dropout(0.2)(Dense(dense_n, activation='relu')(fl))
x = Dropout(0.2)(Dense(dense_n, activation='relu')(x))
gl = MaxPooling1D(pool_size=1, strides=1)(s_dout)
fl = Flatten()(gl)
x = concatenate([(x), (fl)])
outp = Dense(1, activation='sigmoid')(x)
model = Model(inputs=[
in_app, in_ch, in_dev, in_os, in_h, in_dqh, in_qh, in_d, in_wd,
in_qty, in_c1, in_c2, in_c3, in_c4, in_c5
],
outputs=outp)
# Compile the model
exp_decay = lambda init, fin, steps: (init / fin)**(1 /
(steps - 1)) - 1
steps = int(len(X_train) / batch_size) * epochs
# lr_init, lr_fin = 0.001, 0.0001
# lr_init, lr_fin = 0.00005, 0.000005
# lr_decay = exp_decay(lr_init, lr_fin, steps)
# lr_decay = 0.0000001
# lr = lr_init
# optimizer_adam = optimizers.Adam(lr=lr, decay=lr_decay)
# lr = DenseModelOne.OPT_LEARNING_RATE
# lr_decay = DenseModelOne.OPT_DECAY
# optimizer_adam = optimizers.adam(lr=lr, decay=lr_decay)
# print("Using learning init rate: ", lr, ", decay: ", lr_decay)
lr = DenseModelOne.OPT_LEARNING_RATE
optimizer_adam = optimizers.Adam(lr=lr)
print("Using learning init rate: ", lr)
model.compile(loss="binary_crossentropy",
optimizer=optimizer_adam,
metrics=["accuracy"])
model.summary()
self._model = model
# Establish class weights since this is a very unbalanced dataset
class_weight = dict(
zip([0, 1], compute_class_weight('balanced', [0, 1], Y_train)))
print("Unbalanced data, actual class_weight: ", class_weight)
class_weight = {0: .01, 1: .99}
print("Unbalanced data, using class_weight: ", class_weight)
# Establish callbacks for training
modelpath = modeldir + '/' + 'dense-model-checkpoint.h5'
CHK = ModelCheckpoint(modelpath,
monitor="val_loss",
verbose=1,
save_best_only=True,
save_weights_only=False,
mode="auto",
period=1)
ES = EarlyStopping(monitor="val_loss",
min_delta=DenseModelOne.ES_MIN_DELTA,
patience=DenseModelOne.ES_PATIENCE,
verbose=2,
mode="auto")
TB = TensorBoard(log_dir=logdir,
histogram_freq=0,
write_graph=True,
write_images=False)
RLRP = ReduceLROnPlateau(monitor="val_loss",
factor=DenseModelOne.RLRP_FACTOR,
patience=DenseModelOne.RLRP_PATIENCE,
min_lr=DenseModelOne.RLRP_MIN_LR,
verbose=1)
BL = BaseLogger()
# ROC = ROCCallback.ROCCallback()
callbacks = [ES, TB, BL, RLRP, CHK]
if self._X_validation is not None:
print("Using AUCCallback...")
AUC = AUCCallback.AUCCallback(self,
self._X_validation,
self._Y_validation,
convert=True)
callbacks.append(AUC)
validation_data = None
if self._X_validation is not None:
print("Using validation_data (overrides validation_split)...")
X_validation = self.convert(self._X_validation)
Y_validation = self._Y_validation
validation_data = (X_validation, Y_validation)
# Train the model
# (NOTE: if validation_data exists then it will OVERRIDE the validation split)
history = self._model.fit(X_train,
Y_train,
batch_size=batch_size,
epochs=epochs,
validation_split=0.2,
validation_data=validation_data,
class_weight=class_weight,
callbacks=callbacks,
shuffle=True,
verbose=1)
return history
def fit(self,
X_train,
Y_train,
X_test,
modeldir=".",
logdir=".",
epochs=2,
batch_size=20000):
max_app = self._config["max_app"]
max_ch = self._config["max_ch"]
max_dev = self._config["max_dev"]
max_os = self._config["max_os"]
max_h = self._config["max_h"]
max_dqh = self._config["max_dqh"]
max_qh = self._config["max_qh"]
max_d = self._config["max_d"]
max_wd = self._config["max_wd"]
max_qty = self._config["max_qty"]
max_c1 = self._config["max_c1"]
max_c2 = self._config["max_c2"]
max_c3 = self._config["max_c3"]
max_c4 = self._config["max_c4"]
max_c5 = self._config["max_c5"]
X_train = self.convert(X_train)
# Establish the network
emb_n = 50
dense_n1 = 1000
dense_n2 = 1400
dense_n3 = 200
dense_n4 = 40
in_app = Input(shape=[1], name="app")
in_ch = Input(shape=[1], name="ch")
in_dev = Input(shape=[1], name="dev")
in_os = Input(shape=[1], name="os")
in_h = Input(shape=[1], name="h")
in_dqh = Input(shape=[1], name="dqh")
in_qh = Input(shape=[1], name="qh")
in_d = Input(shape=[1], name="d")
in_wd = Input(shape=[1], name="wd")
in_qty = Input(shape=[1], name="qty")
in_c1 = Input(shape=[1], name="c1")
in_c2 = Input(shape=[1], name="c2")
in_c3 = Input(shape=[1], name="c3")
in_c4 = Input(shape=[1], name="c4")
in_c5 = Input(shape=[1], name="c5")
emb_app = Embedding(max_app, emb_n)(in_app)
emb_ch = Embedding(max_ch, int(emb_n / 2))(in_ch)
emb_dev = Embedding(max_dev, int(emb_n * 3))(in_dev)
emb_os = Embedding(max_os, emb_n)(in_os)
emb_h = Embedding(max_h, int(emb_n / 5))(in_h)
emb_dqh = Embedding(max_dqh, int(emb_n / 2))(in_dqh)
emb_qh = Embedding(max_qh, int(emb_n / 10))(in_qh)
emb_d = Embedding(max_d, int(emb_n / 5))(in_d)
emb_wd = Embedding(max_wd, int(emb_n / 5))(in_wd)
emb_qty = Embedding(max_qty, emb_n)(in_qty)
emb_c1 = Embedding(max_c1, emb_n)(in_c1)
emb_c2 = Embedding(max_c2, emb_n)(in_c2)
emb_c3 = Embedding(max_c3, emb_n)(in_c3)
emb_c4 = Embedding(max_c4, emb_n)(in_c4)
emb_c5 = Embedding(max_c5, emb_n)(in_c5)
# in_ip_freq = Input(shape=[1], name="ip_freq")
# in_app_freq = Input(shape=[1], name="app_freq")
# in_device_freq = Input(shape=[1], name="device_freq")
# in_os_freq = Input(shape=[1], name="os_freq")
# in_channel_freq = Input(shape=[1], name="channel_freq")
in_app_channel_freq = Input(shape=[1], name="app_channel_freq")
in_app_os_freq = Input(shape=[1], name="app_os_freq")
in_app_device_freq = Input(shape=[1], name="app_device_freq")
main = concatenate([
# in_ip_freq,
# in_app_freq,
# in_device_freq,
# in_os_freq,
# in_channel_freq,
in_app_channel_freq,
in_app_os_freq,
in_app_device_freq,
Flatten()(emb_app),
Flatten()(emb_ch),
Flatten()(emb_dev),
Flatten()(emb_os),
Flatten()(emb_h),
Flatten()(emb_dqh),
Flatten()(emb_qh),
Flatten()(emb_d),
Flatten()(emb_wd),
Flatten()(emb_qty),
Flatten()(emb_c1),
Flatten()(emb_c2),
Flatten()(emb_c3),
Flatten()(emb_c4),
Flatten()(emb_c5)
])
# s_dout = SpatialDropout1D(0.2)(main)
# fl = Flatten()(s_dout)
x = Dropout(0.2)(Dense(dense_n1, activation='relu')(main))
x = Dropout(0.2)(Dense(dense_n1, activation='relu')(x))
# gl = MaxPooling1D(pool_size=1, strides=1)(main)
# fl = Flatten()(gl)
# x = concatenate([(x), (fl)])
outp = Dense(1, activation='sigmoid')(x)
model = Model(
inputs=[
# in_ip_freq, in_app_freq, in_device_freq, in_os_freq, in_channel_freq,
in_app_channel_freq,
in_app_os_freq,
in_app_device_freq,
in_app,
in_ch,
in_dev,
in_os,
in_h,
in_dqh,
in_qh,
in_d,
in_wd,
in_qty,
in_c1,
in_c2,
in_c3,
in_c4,
in_c5
],
outputs=outp)
# Compile the model
lr = DenseModelFour.OPT_LEARNING_RATE
optimizer_adam = optimizers.Adam(lr=lr)
print("Using learning init rate: ", lr)
model.compile(loss="binary_crossentropy",
optimizer=optimizer_adam,
metrics=["accuracy"])
model.summary()
self._model = model
# Establish class weights since this is a very unbalanced dataset
class_weight = dict(
zip([0, 1], compute_class_weight("balanced", [0, 1], Y_train)))
print("Unbalanced data, actual class_weight: ", class_weight)
class_weight = {0: .01, 1: .99}
print("Unbalanced data, using class_weight: ", class_weight)
# Establish callbacks for training
modelpath = modeldir + "/" + "dense-model-checkpoint.h5"
CHK = ModelCheckpoint(modelpath,
monitor="val_loss",
verbose=1,
save_best_only=True,
save_weights_only=False,
mode="auto",
period=1)
ES = EarlyStopping(monitor="val_loss",
min_delta=DenseModelFour.ES_MIN_DELTA,
patience=DenseModelFour.ES_PATIENCE,
verbose=2,
mode="auto")
TB = TensorBoard(log_dir=logdir,
histogram_freq=0,
write_graph=True,
write_images=False)
RLRP = ReduceLROnPlateau(monitor="val_loss",
factor=DenseModelFour.RLRP_FACTOR,
patience=DenseModelFour.RLRP_PATIENCE,
min_lr=DenseModelFour.RLRP_MIN_LR,
verbose=1)
BL = BaseLogger()
# ROC = ROCCallback.ROCCallback()
callbacks = [ES, TB, BL, RLRP, CHK]
if self._X_validation is not None:
print("Using AUCCallback...")
AUC = AUCCallback.AUCCallback(self,
self._X_validation,
self._Y_validation,
convert=True)
callbacks.append(AUC)
validation_data = None
if self._X_validation is not None:
print("Using validation_data (overrides validation_split)...")
X_validation = self.convert(self._X_validation)
Y_validation = self._Y_validation
validation_data = (X_validation, Y_validation)
# Train the model
# (NOTE: if validation_data exists then it will OVERRIDE the validation split)
history = self._model.fit(X_train,
Y_train,
batch_size=batch_size,
epochs=epochs,
validation_split=0.2,
validation_data=validation_data,
class_weight=class_weight,
callbacks=callbacks,
shuffle=True,
verbose=1)
return history
def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='logging/vae_gan/evolution.log',
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
tweets_path = config_data['tweets_path']
vocab_path = config_data['vocab_path']
vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb'))
#== == == == == == =
# Load all the Data
#== == == == == == =
noutputs = 5
logging.info('Load Training Data')
train_input, train_output = load_data(
join(tweets_path, 'en_train.tsv'), config_data, vocab, noutputs)
logging.info('Load Validation Data')
valid_input, valid_output = load_data(
join(tweets_path, 'en_valid15.tsv'), config_data, vocab, noutputs)
logging.info('Load Validation Data')
valid_input2, valid_output2 = load_data(
join(tweets_path, 'en_test16.tsv'), config_data, vocab, noutputs)
logging.info('Load Test Data')
test_input, test_output = load_data(join(tweets_path, 'en_test17.tsv'),
config_data, vocab, noutputs)
step = K.variable(1.)
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
full_model, encoding_train_model, decoder_train_model, discriminator_train_model, decoder_inference, encoder, decoder, discriminator, discriminator_pretrain_model = vae_gan_model(
config_data, vocab, step)
#full_model.summary()
encoding_train_model.summary()
decoder_train_model.summary()
discriminator_train_model.summary()
decoder_inference.summary()
encoder.summary()
decoder.summary()
discriminator.summary()
#pretrain_discriminator(discriminator_pretrain_model, train_input, vocab)
model_path = 'models/vae_model/'
steps_per_epoch = int(
ceil(config_data['samples_per_epoch'] / config_data['batch_size']))
epochs = int(
ceil(config_data['nb_epochs'] *
(config_data['nsamples'] / config_data['samples_per_epoch'])))
batch_size = config_data['batch_size']
initial_epoch = 0
skip_texts = 0
terminate_on_nan = TerminateOnNaN()
model_checkpoint = ModelCheckpoint(
'models/vae_model/weights.{epoch:02d}.hdf5',
period=10,
save_weights_only=True)
generator = generate_data_stream(config_data['training_path'],
config_data,
vocab,
config_data['batch_size'],
skip_data=skip_texts,
noutputs=noutputs)
enqueuer = GeneratorEnqueuer(generator,
use_multiprocessing=False,
wait_time=0.01)
enqueuer.start(workers=1, max_queue_size=10)
output_generator = enqueuer.get()
enc_out_labels = [
'enc_' + s
for s in encoding_train_model._get_deduped_metrics_names()
]
dec_out_labels = [
'dec_' + s
for s in decoder_train_model._get_deduped_metrics_names()
]
dis_out_labels = [
'dis_' + s
for s in discriminator_train_model._get_deduped_metrics_names()
]
out_labels = enc_out_labels + dec_out_labels + dis_out_labels
#out_labels = full_model._get_deduped_metrics_names()
callback_metrics = out_labels + ['val_' + n for n in out_labels]
step_callback = NewCallback(step, steps_per_epoch)
output_callback = OutputCallback(decoder_inference, valid_input, 15,
vocab, '')
callbacks = CallbackList([
BaseLogger(),
ProgbarLogger(count_mode='steps'), step_callback, output_callback
])
callbacks.set_model(full_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': True,
'do_validation': True,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
epoch = initial_epoch
while epoch < epochs:
epoch_logs = {}
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
X, y = next(output_generator)
callbacks.on_batch_begin(batch_index, batch_logs)
set_trainability(encoder, trainable=True)
set_trainability(decoder, trainable=False)
set_trainability(discriminator, trainable=False)
enc_outs = encoding_train_model.train_on_batch(X, y[:3])
set_trainability(encoder, trainable=False)
set_trainability(decoder, trainable=True)
set_trainability(discriminator, trainable=False)
dec_outs = decoder_train_model.train_on_batch(X, y[:4])
set_trainability(encoder, trainable=False)
set_trainability(decoder, trainable=False)
set_trainability(discriminator, trainable=True)
dis_outs = discriminator_train_model.train_on_batch(X, y[0])
outs = enc_outs + dec_outs + [dis_outs]
#outs = full_model.train_on_batch(X, y)
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
batch_index += 1
steps_done += 1
# Epoch finished.
if steps_done >= steps_per_epoch:
enc_val_outs = encoding_train_model.evaluate(
valid_input, valid_output[:3], verbose=False)
dec_val_outs = decoder_train_model.evaluate(
valid_input, valid_output[:4], verbose=False)
dis_val_outs = discriminator_train_model.evaluate(
valid_input, valid_output[0], verbose=False)
val_outs = enc_val_outs + dec_val_outs + [dis_val_outs]
#val_outs = full_model.evaluate(valid_input, valid_output, verbose=False)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
callbacks.on_train_end()
def predict(model,
batch_size,
num_outputs,
save_path,
evaluate=False,
liver_only=False,
save_predictions=False,
initial_epoch=0,
**kwargs):
model, callbacks, gen = prepare_model(model=model,
num_outputs=num_outputs,
liver_only=liver_only,
evaluate=evaluate,
**kwargs)
# Set up prediction file.
if save_predictions:
save_path = os.path.join(save_path, "predictions.zarr")
if os.path.exists(save_path):
os.remove(save_path)
# Initialize callbacks
val_callback_list = [BaseLogger()]
if not liver_only:
val_callback_list.extend(
[callbacks['dice_lesion'], callbacks['dice_lesion_inliver']])
if len(model.outputs) == 2 or liver_only:
val_callback_list.append(callbacks['dice_liver'])
val_callbacks = CallbackList(val_callback_list)
val_callbacks.set_params({
'nb_epoch': 0,
'nb_sample': 0,
'verbose': False,
'do_validation': True,
'metrics': model.metrics_names
})
val_callbacks.on_train_begin()
val_callbacks.on_epoch_begin(0)
# Create theano function
if evaluate:
inputs = model.inputs + model.targets + model.sample_weights
if model.uses_learning_phase and \
not isinstance(K.learning_phase(), int):
inputs += [K.learning_phase()]
predict_function = K.function(inputs,
model.outputs + [model.total_loss] +
model.metrics_tensors,
updates=model.state_updates)
else:
inputs = model.inputs
if model.uses_learning_phase and \
not isinstance(K.learning_phase(), int):
inputs += [K.learning_phase()]
predict_function = K.function(inputs,
model.outputs,
updates=model.state_updates)
# Predict for all data.
print(' > Predicting...')
for key in gen:
print(' - DATA: {}'.format(key))
# Duplicate inputs and outputs (and add outputs) as necessary.
flow = repeat_flow(gen[key].flow(), num_outputs=num_outputs)
# Set up file.
if save_predictions:
zgroup = zarr.open_group(store=save_path, mode='a', path="/")
zarr_kwargs = {
'chunks': (1, 512, 512),
'compressor': zarr.Blosc(cname='lz4', clevel=9, shuffle=1)
}
# Predict and write to file.
batch_num = 0
for vol_num, volume in enumerate(flow):
print("Predicting on `{}` - {}/{}"
"".format(key, vol_num + 1, len(gen[key])))
# Begin writing to file.
if save_predictions:
vol_idx = volume[-1]
subgroup = zgroup.create_group(str(vol_idx))
num_channels = np.sum(model.output_shape[i][1] \
for i in range(num_outputs))
output_shape = \
(len(volume[0]), num_channels)+model.output_shape[0][2:]
subgroup.empty("volume",
shape=output_shape,
dtype=np.float32,
**zarr_kwargs)
segmentation = volume[1]
if isinstance(segmentation, list):
segmentation = segmentation[0]
subgroup.create_dataset("segmentation",
shape=segmentation.shape,
data=segmentation,
dtype=np.int16,
**zarr_kwargs)
# Iterate through volume batch-wise.
for idx0, idx1 in zip(
range(0, len(volume[0]), batch_size),
range(batch_size,
len(volume[0]) + batch_size + 1, batch_size)):
# Prepare data for joint evaluation and prediction.
if evaluate:
batch = (volume[0][idx0:idx1], volume[1][idx0:idx1])
x, y, sample_weights = model._standardize_user_data(
batch[0], batch[1])
ins = x + y + sample_weights
else:
batch = (volume[0][idx0:idx1], )
ins = _standardize_input_data(batch[0],
model._feed_input_names,
model._feed_input_shapes,
check_batch_axis=False,
exception_prefix='input')
if model.uses_learning_phase and \
not isinstance(K.learning_phase(), int):
ins += [0.]
# Jointly evaluate and predict.
outputs = predict_function(ins)
if num_outputs == 1:
predictions = outputs[0:1]
if evaluate:
val_metrics = outputs[1:]
elif num_outputs == 2:
predictions = outputs[0:2]
if evaluate:
val_metrics = outputs[2:]
else:
raise ValueError("num_outputs must be 1 or 2")
# Write predictions.
predictions = np.concatenate(predictions, axis=1)
subgroup['volume'][idx0:idx1] = predictions
# Update metrics
if evaluate:
val_logs = OrderedDict(
zip(model.metrics_names, val_metrics))
val_logs.update({
'batch': batch_num,
'size': len(batch[0])
})
val_callbacks.on_batch_end(batch_num, val_logs)
batch_num += 1
if evaluate:
# Update metrics
val_callbacks.on_epoch_end(0, val_logs)
# Output metrics
for m in val_logs:
if m not in ['batch', 'size']:
print("{}: {}".format(m, val_logs[m]))
def fit_dataset(self,
dataset,
steps_per_epoch=None,
batch_size=32,
epochs=1,
verbose=1,
callbacks=None,
on_sample=None,
on_scores=None):
"""Train the model on the given dataset for a given number of epochs.
Arguments
---------
dataset: Instance of `BaseDataset` that provides the data
to train on.
steps_per_epoch: int or None, number of gradient updates before
considering an epoch has passed. If None it is set
to be `len(dataset.train_data) / batch_size`.
batch_size: int, number of samples per gradient update
epochs: int, number of times to iterate `steps_per_epoch` times
verbose: {0, >0}, whether to employ the progress bar Keras
callback or not
callbacks: list of Keras callbacks to be called during training
on_sample: callable that accepts the sampler, idxs, w, scores
on_scores: callable that accepts the sampler and scores
"""
try:
if len(dataset.train_data) < batch_size:
raise ValueError(("The model cannot be trained with "
"batch_size > training set"))
except RuntimeError as e:
assert "no size" in str(e)
# Set steps_per_epoch properly
if steps_per_epoch is None:
steps_per_epoch = len(dataset.train_data) // batch_size
# Create the callbacks list
self.history = History()
callbacks = [BaseLogger()] + (callbacks or []) + [self.history]
if verbose > 0:
callbacks += [ProgbarLogger(count_mode="steps")]
callbacks = CallbackList(callbacks)
#TODO: Should we be making it possible to call back a different model
# than self.model.model?
callbacks.set_model(self.model.model)
callbacks.set_params({
"epochs":
epochs,
"steps":
steps_per_epoch,
"verbose":
verbose,
"do_validation":
len(dataset.test_data) > 0,
"metrics":
self._get_metric_names() +
["val_" + n for n in self._get_metric_names()]
})
# Create the sampler
sampler = self.sampler(dataset, batch_size, steps_per_epoch, epochs)
# Start the training loop
epoch = 0
self.model.model.stop_training = False
callbacks.on_train_begin()
while epoch < epochs:
callbacks.on_epoch_begin(epoch)
for step in range(steps_per_epoch):
batch_logs = {"batch": step, "size": batch_size}
callbacks.on_batch_begin(step, batch_logs)
# Importance sampling is done here
idxs, (x, y), w = sampler.sample(batch_size)
# Train on the sampled data
loss, metrics, scores = self.model.train_batch(x, y, w)
# Update the sampler
sampler.update(idxs, scores)
values = map(lambda x: x.mean(), [loss] + metrics)
for l, o in zip(self._get_metric_names(), values):
batch_logs[l] = o
callbacks.on_batch_end(step, batch_logs)
if on_scores is not None:
on_scores(sampler, self._latest_scores)
if on_sample is not None:
on_sample(sampler, self._latest_sample_event["idxs"],
self._latest_sample_event["w"],
self._latest_sample_event["predicted_scores"])
if self.model.model.stop_training:
break
# Evaluate now that an epoch passed
epoch_logs = {}
if len(dataset.test_data) > 0:
val = self.model.evaluate(*dataset.test_data[:],
batch_size=batch_size)
epoch_logs = {
"val_" + l: o
for l, o in zip(self._get_metric_names(), val)
}
callbacks.on_epoch_end(epoch, epoch_logs)
if self.model.model.stop_training:
break
epoch += 1
callbacks.on_train_end()
return self.history
from model import model
from properties import *
from keras.callbacks import BaseLogger, ModelCheckpoint
from data_generator import midi_input_generator, generate_song_array, save_array_to_midi
print "Training model"
#print generate_song_array(model)
#save_array_to_midi([generate_song_array(model)], 'Generated_zero.mid')
cp = ModelCheckpoint(model_check_point_file,
monitor='loss',
verbose=1,
save_best_only=True,
mode='max')
model.fit_generator(midi_input_generator(),
num_seq_in_epoch,
nb_epoch=num_epochs,
callbacks=[BaseLogger(), cp])
model.save_weights('net_dump.nw')
save_array_to_midi([generate_song_array(model)], 'Generated.mid')
def evaluate(model, save_path, num_outputs, liver_only=False, **kwargs):
model, callbacks, gen = prepare_model(model=model,
save_path=save_path,
num_outputs=num_outputs,
liver_only=liver_only,
**kwargs)
print(' > Evaluating the model...')
from scipy.misc import imsave
# Create directory, if needed
save_predictions_to = os.path.join(save_path, "predictions")
if not os.path.exists(save_predictions_to):
os.makedirs(save_predictions_to)
# Initialize callbacks
val_callback_list = [BaseLogger()]
if not liver_only:
val_callback_list.extend(
[callbacks['dice_lesion'], callbacks['dice_lesion_inliver']])
if len(model.outputs) == 2 or liver_only:
val_callback_list.append(callbacks['dice_liver'])
val_callbacks = CallbackList(val_callback_list)
val_callbacks.set_params({
'nb_epoch': 0,
'nb_sample': 0,
'verbose': False,
'do_validation': True,
'metrics': model.metrics_names
})
val_callbacks.on_train_begin()
val_callbacks.on_epoch_begin(0)
# Create theano function
inputs = model.inputs + model.targets + model.sample_weights
if model.uses_learning_phase and \
not isinstance(K.learning_phase(), int):
inputs += [K.learning_phase()]
predict_and_test_function = K.function( \
inputs,
model.outputs+[model.total_loss]+model.metrics_tensors,
updates=model.state_updates)
# Loop through batches, applying function and callbacks
flow = repeat_flow(gen['valid_callback'].flow(), num_outputs=num_outputs)
for batch_num, batch in enumerate(flow):
x, y, sample_weights = model._standardize_user_data(batch[0], batch[1])
ins = x + y + sample_weights
if model.uses_learning_phase and \
not isinstance(K.learning_phase(), int):
ins += [0.]
outputs = predict_and_test_function(ins)
if num_outputs == 1:
predictions = outputs[0:1]
val_metrics = outputs[1:]
else:
predictions = outputs[0:2]
val_metrics = outputs[2:]
## Save images
#def process_slice(s):
#s = np.squeeze(s).copy()
#s[s<0]=0
#s[s>1]=1
#s[0,0]=1
#s[0,1]=0
#return s
#for i in range(len(batch[0])):
#s_pred_list = []
#if num_outputs==1:
#s_pred_list = [process_slice(predictions[i])]
#else:
#for j in range(num_outputs):
#s_pred_list.append(process_slice(predictions[j][i]))
#s_input = process_slice(batch[0][i])
#if num_outputs==1:
#s_truth = process_slice(batch[1][i]/2.)
#else:
#s_truth = process_slice(batch[1][0][i]/2.)
#out_image = np.concatenate([s_input]+s_pred_list+[s_truth],
#axis=1)
#imsave(os.path.join(save_predictions_to,
#"{}_{}.png".format(batch_num, i)),
#out_image)
# Update metrics
val_logs = OrderedDict(zip(model.metrics_names, val_metrics))
val_logs.update({'batch': batch_num, 'size': len(batch[0])})
val_callbacks.on_batch_end(batch_num, val_logs)
# Update metrics
val_callbacks.on_epoch_end(0, val_logs)
# Output metrics
for m in val_logs:
if m not in ['batch', 'size']:
print("{}: {}".format(m, val_logs[m]))
def fit(self, X_train, modeldir=".", logdir=".", epochs=2, batch_size=20000):
input_dim = X_train.shape[1]
encoding_dim = int(input_dim/2)
output_dim = input_dim
print("Using input_dim: ", input_dim)
print("Using encoding_dim: ", encoding_dim)
print("Using output_dim: ", output_dim)
L1 = 1000
L2 = int(L1*2.25)
L3 = int(L2*0.5)
L4 = int(L3*0.5)
L5 = int(L4*0.5)
L6 = int(L5*0.5)
D1 = 0.25
D2 = 0.25
D3 = 0.25
D4 = 0.5
D5 = 0.5
seed = 0
model = Sequential([
Dense(units=L1, input_dim=input_dim, kernel_initializer="normal", activation="tanh"),
Dropout(D1, seed=seed),
Dense(units=L2, activation="tanh"),
Dropout(D2, seed=seed),
# Dense(L3, kernel_initializer="normal", activation="tanh"),
# Dropout(D3, seed=seed),
# Dense(L4, kernel_initializer="normal", activation="tanh"),
# Dropout(D4, seed=seed),
# Dense(l\L5, kernel_initializer="normal", activation="tanh"),
# Dropout(D5, seed=seed),
# Dense(L6, kernel_initializer="normal", activation="tanh"),
# Dropout(L6, seed=seed),
# Dense(output_dim, kernel_initializer="uniform", activity_regularizer=regularizers.l1(1e-5), activation="sigmoid")
Dense(output_dim, kernel_initializer="uniform", activation="sigmoid")
])
# model = Sequential([
# Dense(units=l1, input_dim=input_dim, activation="tanh"),
# Dense(encoding_dim, activation="relu"),
# Dense(int(encoding_dim / 2), activation="relu"),
# Dense(int(encoding_dim / 2), activation="tanh"),
# Dense(output_dim, activation="relu")
# ])
optimizer = optimizers.adam(lr=AutoEncoderModel.OPT_LEARNING_RATE)
# model.compile(optimizer=optimizer, loss="mean_squared_error", metrics=["accuracy"])
model.compile(optimizer=optimizer, loss="mean_squared_logarithmic_error", metrics=["accuracy"])
model.summary()
self._model = model
# Establish callbacks for training
modelpath = modeldir + '/' + 'autoencoder-model-checkpoint.h5'
CHK = ModelCheckpoint(
modelpath, monitor="val_loss", verbose=1,
save_best_only=True, save_weights_only=False,
mode="auto", period=1)
ES = EarlyStopping(
monitor="val_loss", min_delta=AutoEncoderModel.ES_MIN_DELTA,
patience=AutoEncoderModel.ES_PATIENCE, verbose=2, mode="auto")
TB = TensorBoard(
log_dir=logdir, histogram_freq=0,
write_graph=True, write_images=False)
RLRP = ReduceLROnPlateau(
monitor="val_loss", factor=AutoEncoderModel.RLRP_FACTOR,
patience=AutoEncoderModel.RLRP_PATIENCE, min_lr=AutoEncoderModel.RLRP_MIN_LR,
verbose=1)
BL = BaseLogger()
# ROC = ROCCallback.ROCCallback()
# callbacks=[ES, TB, BL, RLRP, CHK]
callbacks=[ES, TB, BL, RLRP]
if self._X_validation is not None:
print("Using AUCCallback...")
AUC = AUCCallback.AUCCallback(self, self._X_validation, self._Y_validation, convert=False)
callbacks.append(AUC)
print("X_train: ", X_train.shape)
validation_data = None
# if self._X_validation is not None:
# print("Using validation_data (overrides validation_split)...")
# X_validation = self._X_validation
# Y_validation = self._Y_validation
# validation_data = (X_validation, Y_validation)
# print("X_validation: ", X_validation.shape)
# print("Y_validation: ", Y_validation.shape)
# Train the model
# (NOTE: if validation_data exists then it will OVERRIDE the validation split)
history = self._model.fit(
X_train, X_train, batch_size=batch_size, epochs=epochs,
validation_split=0.2,
validation_data=validation_data,
callbacks=callbacks,
shuffle=True,
verbose=1)
return history
def main(args):
try:
opts, args = getopt.getopt(args, "c:s", ["config="])
except getopt.GetoptError:
print('usage: -c config.json')
sys.exit(2)
start_from_model = False
for opt, arg in opts:
if opt in ("-c", "--config"):
config_fname = os.path.join('configurations', arg)
elif opt == '-s':
start_from_model = True
if start_from_model:
filemode = 'a'
else:
filemode = 'w'
log_path = 'logging/vae_nlg_{}'.format(int(round(time.time() * 1000)))
os.mkdir(log_path)
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
filename='{}/evolution.log'.format(log_path),
filemode=filemode)
with open(config_fname, 'r') as json_data:
config_data = json.load(json_data)
batch_size = config_data['batch_size']
epochs = config_data['nb_epochs']
discriminator_iterations = config_data['discriminator_iterations']
tweets_path = config_data['tweets_path']
vocab_path = config_data['vocab_path']
vocab = cPickle.load(open(join(vocab_path, 'vocabulary.pkl'), 'rb'))
#== == == == == == =
# Load all the Data
#== == == == == == =
delimiter = ''
noutputs = 11
logging.info('Load Training Data')
train_input, train_output, train_weights, train_lex = load_text_gen_data(
join(tweets_path, 'trainset.csv'),
config_data,
vocab,
noutputs,
word_based=False)
logging.info('Load Validation Data')
valid_input, valid_output, _, valid_lex = load_text_gen_data(
join(tweets_path, 'devset.csv'),
config_data,
vocab,
noutputs,
word_based=False)
logging.info('Load Output Validation Data')
valid_dev_input, valid_dev_output, _, valid_dev_lex = load_text_gen_data(
join(tweets_path, 'devset_reduced.csv'),
config_data,
vocab,
noutputs,
random_output=False,
word_based=False)
step = K.variable(1., name='step_varialbe')
steps_per_epoch = ceil(train_output[0].shape[0] /
config_data['batch_size'])
# == == == == == == == == == == =
# Define and load the CNN model
# == == == == == == == == == == =
vae_model_train, vae_model_test, vae_vanilla_train_model, vae_vanilla_test_model, discriminator_model, decoder_test, discriminator = get_vae_gan_model(
config_data, vocab, step)
with open(os.path.join(log_path, 'models.txt'), 'wt') as fh:
fh.write('VAE Model Train\n')
fh.write('---------\n')
vae_model_train.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Test\n')
fh.write('--------------\n')
vae_model_test.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Pretrain\n')
fh.write('---------------------------\n')
vae_vanilla_train_model.summary(
print_fn=lambda x: fh.write(x + '\n'))
fh.write('VAE Model Pretrain Test\n')
fh.write('---------------------------\n')
vae_vanilla_test_model.summary(
print_fn=lambda x: fh.write(x + '\n'))
fh.write('Decoder Test\n')
fh.write('-------------------\n')
decoder_test.summary(print_fn=lambda x: fh.write(x + '\n'))
fh.write('Discriminator Models\n')
fh.write('-------------------\n')
discriminator_model.summary(print_fn=lambda x: fh.write(x + '\n'))
terminate_on_nan = TerminateOnNaN()
output_callback = LexOutputCallback(
vae_vanilla_test_model,
valid_dev_input,
valid_dev_lex,
1,
vocab,
delimiter,
fname='{}/test_output'.format(log_path))
#output_callback_full = LexOutputCallback(vae_vanilla_test_model, valid_dev_input, valid_dev_lex, 1, vocab, delimiter, fname='{}/test_output'.format(log_path))
#
# vae_vanilla_train_model.fit_generator(
# generator=generate_data_stream(config_data['pretrain_path'], config_data, vocab, config_data['batch_size'], noutputs=3),
# steps_per_epoch=steps_per_epoch,
# epochs=ceil(config_data['pretrain_samples']/config_data['pretrain_samples_per_epoch']),
# callbacks=[output_callback, terminate_on_nan],
# validation_data=(valid_input, valid_output[:3]),
# )
vae_vanilla_train_model.fit(
x=train_input,
y=train_output[:2],
epochs=config_data['pretrain_epochs'],
batch_size=batch_size,
validation_data=(valid_input, valid_output[:2]),
sample_weight=train_weights[:2],
callbacks=[output_callback, terminate_on_nan])
terminate_on_nan = TerminateOnNaN()
model_checkpoint = ModelCheckpoint(
'models/vae_model/weights.{epoch:02d}.hdf5',
period=10,
save_weights_only=True)
out_labels = [
'enc_' + s for s in vae_model_train._get_deduped_metrics_names()
]
out_labels += [
'dis_' + s
for s in discriminator_model._get_deduped_metrics_names()
]
callback_metrics = out_labels + ['val_' + n for n in out_labels]
tensorboard = TensorBoard(log_dir='logging/tensorboard',
histogram_freq=0,
write_grads=True,
write_images=True)
step_callback = StepCallback(step, steps_per_epoch)
output_callback = LexOutputCallback(
vae_vanilla_test_model,
valid_dev_input,
valid_dev_lex,
1,
vocab,
delimiter,
fname='{}/test_output'.format(log_path))
output_callback_full = LexOutputCallback(
vae_vanilla_test_model,
valid_input,
valid_lex,
5,
vocab,
delimiter,
fname='{}/test_valid_output'.format(log_path))
callbacks = CallbackList([
BaseLogger(),
ProgbarLogger(count_mode='steps'), step_callback, tensorboard,
output_callback, output_callback_full, model_checkpoint,
terminate_on_nan
])
callbacks.set_model(vae_model_train)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': True,
'do_validation': True,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
initial_epoch = 0
num_train_samples = train_input[0].shape[0]
index_array = np.arange(num_train_samples)
steps = 0
epoch = initial_epoch
while epoch < epochs:
epoch_logs = {}
callbacks.on_epoch_begin(epoch)
index_array = _batch_shuffle(index_array, batch_size)
steps_done = 0
batches = _make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_logs = {}
batch_ids = index_array[batch_start:batch_end]
X = _slice_arrays(train_input, batch_ids)
y = _slice_arrays(train_output, batch_ids)
sample_weights = _slice_arrays(train_weights, batch_ids)
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
set_trainability(discriminator, trainable=False)
enc_outs = vae_model_train.train_on_batch(
x=X, y=y, sample_weight=sample_weights)
set_trainability(discriminator, trainable=True)
list_disc_loss_real = []
if steps < 25 or steps % 500 == 0:
disc_iterations = 25
else:
disc_iterations = discriminator_iterations
for disc_it in range(disc_iterations):
real_idx = np.random.choice(train_input[0].shape[0],
len(batch_ids),
replace=False)
disX_train = train_input[-1][
real_idx] #take input 8 as train input and the rest as targets
disy_train = [x[real_idx] for x in train_input[:8]
] #take input 1-7 as targets
#train on real data
dis_outs_real = discriminator_model.train_on_batch(
disX_train, disy_train)
list_disc_loss_real.append(dis_outs_real)
loss_d_real = np.mean(list_disc_loss_real, axis=0)
outs = np.concatenate((enc_outs, loss_d_real))
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
batch_index += 1
steps_done += 1
steps += 1
# Epoch finished.
if steps_done >= steps_per_epoch:
valid_len = valid_output[0].shape[0]
enc_val_outs = vae_model_train.evaluate(valid_input,
valid_output,
verbose=False)
dis_val_outs = discriminator_model.evaluate(
valid_input[-1], valid_input[:8], verbose=False)
val_outs = enc_val_outs + dis_val_outs
#val_outs = full_model.evaluate(valid_input, valid_output, verbose=False)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
callbacks.on_train_end()
def fit(self,
x,
y,
batch_size=None,
nsteps=None,
epochs=1,
verbose=1,
callbacks=None,
validation_data=None):
assert self.is_compiled, "Must compile model first"
assert epochs > 0
x = x if type(x) is list else [x]
y = y if type(y) is list else [y]
if nsteps is None:
total_len = len(y[0]) if type(y) is list else len(y)
nsteps = total_len // batch_size
# BaseLogger should always be the first metric since it computes the stats on epoch end
base_logger = BaseLogger(
stateful_metrics=["val_%s" % m for m in self.metrics_name] +
['val_loss', 'size'])
base_logger_params = {'metrics': ['loss'] + self.metrics_name}
if validation_data:
base_logger_params['metrics'] += [
'val_%s' % m for m in base_logger_params['metrics']
]
base_logger.set_params(base_logger_params)
hist = History()
if callbacks is None:
callbacks = [base_logger] + [hist]
elif type(callbacks) is list:
callbacks = [base_logger] + callbacks + [hist]
else:
callbacks = [base_logger] + [callbacks] + [hist]
callback_list = CallbackList(callbacks=callbacks)
callback_list.set_model(self)
callback_list.on_train_begin()
self.callbacks = callback_list
for epoch in range(epochs):
g = batchify(x, y, batch_size) if batch_size else None
t = trange(nsteps) if verbose == 1 else range(nsteps)
callback_list.on_epoch_begin(epoch)
for it in t:
x_, y_ = next(g) if g else (None, None)
batch_logs = self.train_on_batch(x_, y_)
callback_list.on_batch_end(it, batch_logs)
curr_loss = base_logger.totals['loss'] / base_logger.seen
if verbose == 1:
t.set_postfix(loss="%.4f" % curr_loss)
if verbose == 2:
if it % 1000 == 0:
print("%s %i/%i, loss=%.5f" %
(datetime.datetime.now().strftime("%H:%M:%S"),
it, nsteps, curr_loss),
flush=True)
if validation_data:
val_logs = self.evaluate(validation_data[0],
validation_data[1])
base_logger.on_batch_end(None, val_logs)
epoch_logs = {}
callback_list.on_epoch_end(epoch=epoch, logs=epoch_logs)
if verbose:
if validation_data:
to_print = ['loss'] + self.metrics_name + ['val_loss'] + [
'val_%s' % m for m in self.metrics_name
]
else:
to_print = ['loss'] + self.metrics_name
prog = ", ".join([
"%s=%.4f" % (name, hist.history[name][-1])
for name in to_print
])
print("Epoch %i, %s" % (epoch, prog), flush=True)
if self.stop_training:
break
return hist.history
def monkey_patched_BaseLogger(*args, **kwargs):
return KerasAggregateCallback(BaseLogger(*args, **kwargs),
NeptuneLogger(experiment_getter=experiment_getter))
def fit_generator(self,
generator,
epochs=1,
validation_data=None,
callbacks=None,
verbose=True):
method = self._model.optimizer.method
x0 = self._collect_weights()
history = History()
_callbacks = [BaseLogger(stateful_metrics=self._model.metrics_names)]
_callbacks += (callbacks or []) + [history]
callback_list = CallbackList(_callbacks)
callback_list.set_model(self._model)
callback_list.set_params({
'epochs': epochs,
'verbose': False,
'metrics': list(self._model.metrics_names),
})
state = {
'epoch': 0,
'verbose': verbose,
'callbacks': callback_list,
'in_epoch': False,
'epoch_logs': {},
}
min_options = {
'maxiter': epochs,
'maxfun': epochs * 10,
'maxcor': 50,
'maxls': 50,
'ftol': np.finfo(float).eps,
'gtol': 1e-10,
'eps': 1e-8,
}
val_generator = None
if validation_data is not None:
if isinstance(validation_data, keras.utils.Sequence):
val_generator = validation_data
elif isinstance(validation_data,
tuple) and len(validation_data) == 2:
val_generator = GeneratorWrapper(*validation_data)
def on_iteration_end(xk):
cb = state['callbacks']
if val_generator is not None:
self._validate(xk, val_generator, state)
cb.on_epoch_end(state['epoch'], state['epoch_logs'])
# if state['verbose']:
# epoch_logs = state['epoch_logs']
# print('epoch: ', state['epoch'],
# ', '.join([' {0}: {1:.3e}'.format(k, v) for k, v in epoch_logs.items()]))
state['epoch'] += 1
state['in_epoch'] = False
state['epoch_logs'] = {}
callback_list.on_train_begin()
result = minimize(self._fun_generator,
x0,
method=method,
jac=True,
options=min_options,
callback=on_iteration_end,
args=(generator, state))
self._update_weights(result['x'])
callback_list.on_train_end()
return history