def get_fitted_model(cache, args):
# load model
input_model = args.fitted_model
input_weights = args.model_weights
compiled_model = args.compiled_model
model = None
if input_model is not None:
model = helpers.import_model(input_model)
elif input_weights is not None:
model = cache['compiled_model']
if compiled_model is not None:
model = helpers.import_model(compiled_model)
if model is None:
print "Error loading fitted model: weights were provided, but a model was not compiled. Exiting"
weights = helpers.import_weights(input_weights)
model.set_weights(weights)
elif 'fitted_model' in cache:
model = cache['fitted_model']
else:
print "Error loading fitted model: no fitted model provided: exiting"
exit()
return model
def get_fitted_model(cache, args):
# load model
input_model = args.fitted_model
input_weights = args.model_weights
compiled_model = args.compiled_model
model = None
if input_model is not None:
model = helpers.import_model(input_model)
elif input_weights is not None:
model = cache['compiled_model']
if compiled_model is not None:
model = helpers.import_model(compiled_model)
if model is None:
print "Error loading fitted model: weights were provided, but a model was not compiled. Exiting"
weights = helpers.import_weights(input_weights)
model.set_weights(weights)
elif 'fitted_model' in cache:
model = cache['fitted_model']
else:
print "Error loading fitted model: no fitted model provided: exiting"
exit()
return model
def h_train(cache, args):
sets = {}
req = ['X_emb', 'Y_tokens', 'Y_emb', 'M', 'maxlen']
show_multiple = s2b(get_required_arg(args, 'show_multiple'))
# load embeddings
embedding_src = get_embedding(cache, args, 'embedding_src')
embedding_dst = get_embedding(cache, args, 'embedding_dst')
# load train dataset
train_src = get_required_arg(args, 'train_src')
train_dst = get_required_arg(args, 'train_dst')
maxlen = get_required_arg(args, 'maxlen')
sets['train'] = helpers.load_datasets(req,
embedding_src, embedding_dst,
train_src, train_dst,
maxlen)
X_train = sets['train']['X_emb']
Y_train = sets['train']['Y_emb']
M_train = sets['train']['M']
# load validation dataset
validation_src = get_required_arg(args, 'validation_src')
validation_dst = get_required_arg(args, 'validation_dst')
maxlen = get_required_arg(args, 'maxlen')
sets['validate'] = helpers.load_datasets(req,
embedding_src, embedding_dst,
validation_src, validation_dst,
maxlen)
X_validation = sets['validate']['X_emb']
Y_validation = sets['validate']['Y_emb']
M_validation = sets['validate']['M']
# load model
modelfile = args.compiled_model
if modelfile is not None:
model = helpers.import_model(modelfile)
elif 'compiled_model' in cache:
model = cache['compiled_model']
else:
print "Error in train: no compiled model provided: exiting"
exit()
# load weights (if applicable)
input_weights = args.model_weights
if input_weights is not None:
log('loading weights')
weights = helpers.import_weights(input_weights)
model.set_weights(weights)
log('done')
# train model
batch_size = get_required_arg(args, 'batch_size')
validation_skip = get_required_arg(args, 'validation_skip')
snapshot_skip = get_required_arg(args, 'snapshot_skip')
lr_A = get_required_arg(args, 'lr_encoder')
lr_B = get_required_arg(args, 'lr_decoder')
epoch_start = get_required_arg(args, 'epoch_start')
timer_start = None
def get_elapsed():
return (datetime.utcnow() - timer_start).total_seconds()
# policy evaluated at beginning of each epoch:
# stop training if any thresholds are met
def continue_training(epoch, callback_result):
def check_lte(left, right):
return left is not None and right is not None and left <= right
def check_gte(left, right):
return left is not None and right is not None and left >= right
def dict_or_None(l):
try:
return l()
except KeyError:
return None
except TypeError:
return None
loss = dict_or_None(lambda: callback_result['sets']['train']['loss'])
val_loss = dict_or_None(lambda: callback_result['sets']['validate']['loss'])
error = dict_or_None(lambda: 1 - callback_result['sets']['train']['summary']['normal, L2']['avg_correct_pct'])
val_error = dict_or_None(lambda: 1 - callback_result['sets']['validate']['summary']['normal, L2']['avg_correct_pct'])
if (loss is not None and math.isnan(loss)) or (val_loss is not None and math.isnan(val_loss)):
return False
return not (
check_gte(epoch, args.epochs) or \
check_gte(get_elapsed(), args.seconds) or \
(check_lte(loss, args.loss) and check_lte(val_loss, args.loss)) or \
(check_lte(error, args.error) and check_lte(val_error, args.error))
)
sample_size = get_required_arg(args, 'sample_size')
# end of epoch callback: output stats, take snapshot
snapshot_prefix = args.output_snapshot_prefix
def epoch_callback(round_stats, epoch):
elapsed = get_elapsed()
log("Begin epoch callback for epoch {0}".format(epoch))
if validation_skip > 0 and (epoch + 1) % validation_skip == 0:
DLs = [('normal, L2', None, embedding_dst)]
set_dicts = output_dumps.full_stats(round_stats, sets, DLs,
model, sample_size=sample_size, log=lambda *_: None,
show_multiple=show_multiple)
else:
set_dicts = round_stats
s = {'epoch': epoch, 'time': elapsed, 'sets': set_dicts}
stat.info("{0},".format(s))
# take snapshot
if snapshot_prefix and snapshot_skip > 0 and (epoch + 1) % snapshot_skip == 0:
snapshot_name = "{0}.{1}".format(snapshot_prefix, epoch)
log("Exporting snapshot weights for epoch {0} to {1}".format(epoch, snapshot_name))
helpers.export_weights(model, snapshot_name)
log("Exported snapshot weights for epoch {0}".format(epoch))
log("End epoch callback for epoch {0}".format(epoch))
return s
model.X1 = [[embedding_dst.start_1h]]
log("Training model...")
timer_start = datetime.utcnow()
model.fit(
X_train, Y_train, M_train,
X_validation, Y_validation, M_validation,
lr_A=lr_A, lr_B=lr_B,
batch_size=batch_size,
verbose=1, shuffle=True,
epoch_start=epoch_start,
continue_training=continue_training,
epoch_callback=epoch_callback
)
output_model = args.output_fitted_model
if output_model is not None:
log("Exporting fitted model to {0}".format(output_model))
helpers.export_model(model, output_model)
output_weights = args.output_model_weights
if output_weights is not None:
log("Exporting fitted weights to {0}".format(output_weights))
helpers.export_weights(model, output_weights)
cache['fitted_model'] = model
def h_train(cache, args):
sets = {}
req = ['X_emb', 'Y_tokens', 'Y_emb', 'M', 'maxlen']
show_multiple = s2b(get_required_arg(args, 'show_multiple'))
# load embeddings
embedding_src = get_embedding(cache, args, 'embedding_src')
embedding_dst = get_embedding(cache, args, 'embedding_dst')
# load train dataset
train_src = get_required_arg(args, 'train_src')
train_dst = get_required_arg(args, 'train_dst')
maxlen = get_required_arg(args, 'maxlen')
sets['train'] = helpers.load_datasets(req, embedding_src, embedding_dst,
train_src, train_dst, maxlen)
X_train = sets['train']['X_emb']
Y_train = sets['train']['Y_emb']
M_train = sets['train']['M']
# load validation dataset
validation_src = get_required_arg(args, 'validation_src')
validation_dst = get_required_arg(args, 'validation_dst')
maxlen = get_required_arg(args, 'maxlen')
sets['validate'] = helpers.load_datasets(req, embedding_src, embedding_dst,
validation_src, validation_dst,
maxlen)
X_validation = sets['validate']['X_emb']
Y_validation = sets['validate']['Y_emb']
M_validation = sets['validate']['M']
# load model
modelfile = args.compiled_model
if modelfile is not None:
model = helpers.import_model(modelfile)
elif 'compiled_model' in cache:
model = cache['compiled_model']
else:
print "Error in train: no compiled model provided: exiting"
exit()
# load weights (if applicable)
input_weights = args.model_weights
if input_weights is not None:
log('loading weights')
weights = helpers.import_weights(input_weights)
model.set_weights(weights)
log('done')
# train model
batch_size = get_required_arg(args, 'batch_size')
validation_skip = get_required_arg(args, 'validation_skip')
snapshot_skip = get_required_arg(args, 'snapshot_skip')
lr_A = get_required_arg(args, 'lr_encoder')
lr_B = get_required_arg(args, 'lr_decoder')
epoch_start = get_required_arg(args, 'epoch_start')
timer_start = None
def get_elapsed():
return (datetime.utcnow() - timer_start).total_seconds()
# policy evaluated at beginning of each epoch:
# stop training if any thresholds are met
def continue_training(epoch, callback_result):
def check_lte(left, right):
return left is not None and right is not None and left <= right
def check_gte(left, right):
return left is not None and right is not None and left >= right
def dict_or_None(l):
try:
return l()
except KeyError:
return None
except TypeError:
return None
loss = dict_or_None(lambda: callback_result['sets']['train']['loss'])
val_loss = dict_or_None(
lambda: callback_result['sets']['validate']['loss'])
error = dict_or_None(lambda: 1 - callback_result['sets']['train'][
'summary']['normal, L2']['avg_correct_pct'])
val_error = dict_or_None(lambda: 1 - callback_result['sets'][
'validate']['summary']['normal, L2']['avg_correct_pct'])
if (loss is not None
and math.isnan(loss)) or (val_loss is not None
and math.isnan(val_loss)):
return False
return not (
check_gte(epoch, args.epochs) or \
check_gte(get_elapsed(), args.seconds) or \
(check_lte(loss, args.loss) and check_lte(val_loss, args.loss)) or \
(check_lte(error, args.error) and check_lte(val_error, args.error))
)
sample_size = get_required_arg(args, 'sample_size')
# end of epoch callback: output stats, take snapshot
snapshot_prefix = args.output_snapshot_prefix
def epoch_callback(round_stats, epoch):
elapsed = get_elapsed()
log("Begin epoch callback for epoch {0}".format(epoch))
if validation_skip > 0 and (epoch + 1) % validation_skip == 0:
DLs = [('normal, L2', None, embedding_dst)]
set_dicts = output_dumps.full_stats(round_stats,
sets,
DLs,
model,
sample_size=sample_size,
log=lambda *_: None,
show_multiple=show_multiple)
else:
set_dicts = round_stats
s = {'epoch': epoch, 'time': elapsed, 'sets': set_dicts}
stat.info("{0},".format(s))
# take snapshot
if snapshot_prefix and snapshot_skip > 0 and (epoch +
1) % snapshot_skip == 0:
snapshot_name = "{0}.{1}".format(snapshot_prefix, epoch)
log("Exporting snapshot weights for epoch {0} to {1}".format(
epoch, snapshot_name))
helpers.export_weights(model, snapshot_name)
log("Exported snapshot weights for epoch {0}".format(epoch))
log("End epoch callback for epoch {0}".format(epoch))
return s
model.X1 = [[embedding_dst.start_1h]]
log("Training model...")
timer_start = datetime.utcnow()
model.fit(X_train,
Y_train,
M_train,
X_validation,
Y_validation,
M_validation,
lr_A=lr_A,
lr_B=lr_B,
batch_size=batch_size,
verbose=1,
shuffle=True,
epoch_start=epoch_start,
continue_training=continue_training,
epoch_callback=epoch_callback)
output_model = args.output_fitted_model
if output_model is not None:
log("Exporting fitted model to {0}".format(output_model))
helpers.export_model(model, output_model)
output_weights = args.output_model_weights
if output_weights is not None:
log("Exporting fitted weights to {0}".format(output_weights))
helpers.export_weights(model, output_weights)
cache['fitted_model'] = model
