def epoch_callback(round_stats, epoch):
elapsed = get_elapsed()
log("Begin epoch callback for epoch {0}".format(epoch))
if validation_skip > 0 and (epoch + 1) % validation_skip == 0:
DLs = [('normal, L2', None, embedding_dst)]
set_dicts = output_dumps.full_stats(round_stats, sets, DLs,
model, sample_size=sample_size, log=lambda *_: None,
show_multiple=show_multiple)
else:
set_dicts = round_stats
s = {'epoch': epoch, 'time': elapsed, 'sets': set_dicts}
stat.info("{0},".format(s))
# take snapshot
if snapshot_prefix and snapshot_skip > 0 and (epoch + 1) % snapshot_skip == 0:
snapshot_name = "{0}.{1}".format(snapshot_prefix, epoch)
log("Exporting snapshot weights for epoch {0} to {1}".format(epoch, snapshot_name))
helpers.export_weights(model, snapshot_name)
log("Exported snapshot weights for epoch {0}".format(epoch))
log("End epoch callback for epoch {0}".format(epoch))
return s
def epoch_callback(round_stats, epoch):
elapsed = get_elapsed()
log("Begin epoch callback for epoch {0}".format(epoch))
if validation_skip > 0 and (epoch + 1) % validation_skip == 0:
DLs = [('normal, L2', None, embedding_dst)]
set_dicts = output_dumps.full_stats(round_stats,
sets,
DLs,
model,
sample_size=sample_size,
log=lambda *_: None,
show_multiple=show_multiple)
else:
set_dicts = round_stats
s = {'epoch': epoch, 'time': elapsed, 'sets': set_dicts}
stat.info("{0},".format(s))
# take snapshot
if snapshot_prefix and snapshot_skip > 0 and (epoch +
1) % snapshot_skip == 0:
snapshot_name = "{0}.{1}".format(snapshot_prefix, epoch)
log("Exporting snapshot weights for epoch {0} to {1}".format(
epoch, snapshot_name))
helpers.export_weights(model, snapshot_name)
log("Exported snapshot weights for epoch {0}".format(epoch))
log("End epoch callback for epoch {0}".format(epoch))
return s
def h_test(cache, args):
sets = {}
req = ['X_emb', 'Y_tokens']
show_multiple = s2b(get_required_arg(args, 'show_multiple'))
# load embeddings
embedding_src = get_embedding(cache, args, 'embedding_src')
embedding_dst = get_embedding(cache, args, 'embedding_dst')
# load dataset
test_src = get_required_arg(args, 'test_src')
test_dst = get_required_arg(args, 'test_dst')
maxlen = get_required_arg(args, 'maxlen')
sets['test'] = helpers.load_datasets(req, embedding_src, embedding_dst,
test_src, test_dst, maxlen)
# load model
log('loading model')
model = get_fitted_model(cache, args)
log('done loading model')
sample_size = get_required_arg(args, 'sample_size')
# compute test
round_stats = {'test': {}}
DLs = [('normal, L2', None, embedding_dst)]
set_dicts = output_dumps.full_stats(round_stats,
sets,
DLs,
model,
sample_size=sample_size,
log=lambda *_: None,
show_multiple=show_multiple)
print set_dicts
log("done test")
def h_test(cache, args):
sets = {}
req = ['X_emb', 'Y_tokens']
show_multiple = s2b(get_required_arg(args, 'show_multiple'))
# load embeddings
embedding_src = get_embedding(cache, args, 'embedding_src')
embedding_dst = get_embedding(cache, args, 'embedding_dst')
# load dataset
test_src = get_required_arg(args, 'test_src')
test_dst = get_required_arg(args, 'test_dst')
maxlen = get_required_arg(args, 'maxlen')
sets['test'] = helpers.load_datasets(req,
embedding_src, embedding_dst,
test_src, test_dst,
maxlen)
# load model
log('loading model')
model = get_fitted_model(cache, args)
log('done loading model')
sample_size = get_required_arg(args, 'sample_size')
# compute test
round_stats = {'test':{}}
DLs = [('normal, L2', None, embedding_dst)]
set_dicts = output_dumps.full_stats(round_stats, sets, DLs,
model, sample_size=sample_size, log=lambda *_: None,
show_multiple=show_multiple)
print set_dicts
log("done test")