def main():
in_arg = get_input_args()
check_command_line_arguments(in_arg)
image_path = in_arg.img
# Load flower categories
cat_to_name = load_flower_categories(in_arg.json_file)
print(len(cat_to_name), 'flower categories/names loaded.\n')
loader_dict, data_dict = load_datasets()
model = load_checkpoint(in_arg.checkpoint)
cuda = train_on_gpu(in_arg.cpu)
if cuda:
model = model.cuda()
# See what the model predicts for that image
probabilities, classes = predict(image_path, model, data_dict['train'],
in_arg.topk, cuda)
print("\n\nThe model predicts that your image is a " +
cat_to_name[classes[0]])
print("\nTop k with percentages:\n")
for i in range(len(classes)):
print(
str(int(probabilities[i] * 100)) + "%: " + cat_to_name[classes[i]])
def main():
print("Starting model trainer")
start_time = time()
in_arg = get_input_args()
check_command_line_arguments(in_arg)
loader_dict, data_dict = load_datasets()
# Load flower categories
cat_to_name = load_flower_categories('cat_to_name.json')
print(len(cat_to_name), 'flower categories/names loaded.\n')
input_nodes = {
'densenet121': 1024,
'vgg16': 25088,
'vgg13': 25088,
'alexnet': 9216
}
model = make_model(in_arg.arch, cat_to_name, input_nodes)
cuda = train_on_gpu(in_arg.cpu)
if cuda:
model = model.cuda()
# Train a model with a pre-trained network
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.classifier.parameters(), lr=in_arg.lr)
train_model(model, loader_dict, in_arg.epochs, criterion, optimizer, cuda)
check_accuracy_on_test(model, loader_dict['test'], cuda, criterion)
save_checkpoint(model, data_dict['train'], in_arg, input_nodes)
# Measure total program runtime by collecting end time
end_time = time()
# Computes overall runtime in seconds & prints it in hh:mm:ss format
tot_time = end_time - start_time
print(
"\n** Total Elapsed Runtime (h:m:s):",
str(int((tot_time / 3600))) + ":" + str(int(
(tot_time % 3600) / 60)) + ":" + str(int((tot_time % 3600) % 60)))
def h_test(cache, args):
sets = {}
req = ['X_emb', 'Y_tokens']
show_multiple = s2b(get_required_arg(args, 'show_multiple'))
# load embeddings
embedding_src = get_embedding(cache, args, 'embedding_src')
embedding_dst = get_embedding(cache, args, 'embedding_dst')
# load dataset
test_src = get_required_arg(args, 'test_src')
test_dst = get_required_arg(args, 'test_dst')
maxlen = get_required_arg(args, 'maxlen')
sets['test'] = helpers.load_datasets(req, embedding_src, embedding_dst,
test_src, test_dst, maxlen)
# load model
log('loading model')
model = get_fitted_model(cache, args)
log('done loading model')
sample_size = get_required_arg(args, 'sample_size')
# compute test
round_stats = {'test': {}}
DLs = [('normal, L2', None, embedding_dst)]
set_dicts = output_dumps.full_stats(round_stats,
sets,
DLs,
model,
sample_size=sample_size,
log=lambda *_: None,
show_multiple=show_multiple)
print set_dicts
log("done test")
def h_test(cache, args):
sets = {}
req = ['X_emb', 'Y_tokens']
show_multiple = s2b(get_required_arg(args, 'show_multiple'))
# load embeddings
embedding_src = get_embedding(cache, args, 'embedding_src')
embedding_dst = get_embedding(cache, args, 'embedding_dst')
# load dataset
test_src = get_required_arg(args, 'test_src')
test_dst = get_required_arg(args, 'test_dst')
maxlen = get_required_arg(args, 'maxlen')
sets['test'] = helpers.load_datasets(req,
embedding_src, embedding_dst,
test_src, test_dst,
maxlen)
# load model
log('loading model')
model = get_fitted_model(cache, args)
log('done loading model')
sample_size = get_required_arg(args, 'sample_size')
# compute test
round_stats = {'test':{}}
DLs = [('normal, L2', None, embedding_dst)]
set_dicts = output_dumps.full_stats(round_stats, sets, DLs,
model, sample_size=sample_size, log=lambda *_: None,
show_multiple=show_multiple)
print set_dicts
log("done test")
def h_train(cache, args):
sets = {}
req = ['X_emb', 'Y_tokens', 'Y_emb', 'M', 'maxlen']
show_multiple = s2b(get_required_arg(args, 'show_multiple'))
# load embeddings
embedding_src = get_embedding(cache, args, 'embedding_src')
embedding_dst = get_embedding(cache, args, 'embedding_dst')
# load train dataset
train_src = get_required_arg(args, 'train_src')
train_dst = get_required_arg(args, 'train_dst')
maxlen = get_required_arg(args, 'maxlen')
sets['train'] = helpers.load_datasets(req,
embedding_src, embedding_dst,
train_src, train_dst,
maxlen)
X_train = sets['train']['X_emb']
Y_train = sets['train']['Y_emb']
M_train = sets['train']['M']
# load validation dataset
validation_src = get_required_arg(args, 'validation_src')
validation_dst = get_required_arg(args, 'validation_dst')
maxlen = get_required_arg(args, 'maxlen')
sets['validate'] = helpers.load_datasets(req,
embedding_src, embedding_dst,
validation_src, validation_dst,
maxlen)
X_validation = sets['validate']['X_emb']
Y_validation = sets['validate']['Y_emb']
M_validation = sets['validate']['M']
# load model
modelfile = args.compiled_model
if modelfile is not None:
model = helpers.import_model(modelfile)
elif 'compiled_model' in cache:
model = cache['compiled_model']
else:
print "Error in train: no compiled model provided: exiting"
exit()
# load weights (if applicable)
input_weights = args.model_weights
if input_weights is not None:
log('loading weights')
weights = helpers.import_weights(input_weights)
model.set_weights(weights)
log('done')
# train model
batch_size = get_required_arg(args, 'batch_size')
validation_skip = get_required_arg(args, 'validation_skip')
snapshot_skip = get_required_arg(args, 'snapshot_skip')
lr_A = get_required_arg(args, 'lr_encoder')
lr_B = get_required_arg(args, 'lr_decoder')
epoch_start = get_required_arg(args, 'epoch_start')
timer_start = None
def get_elapsed():
return (datetime.utcnow() - timer_start).total_seconds()
# policy evaluated at beginning of each epoch:
# stop training if any thresholds are met
def continue_training(epoch, callback_result):
def check_lte(left, right):
return left is not None and right is not None and left <= right
def check_gte(left, right):
return left is not None and right is not None and left >= right
def dict_or_None(l):
try:
return l()
except KeyError:
return None
except TypeError:
return None
loss = dict_or_None(lambda: callback_result['sets']['train']['loss'])
val_loss = dict_or_None(lambda: callback_result['sets']['validate']['loss'])
error = dict_or_None(lambda: 1 - callback_result['sets']['train']['summary']['normal, L2']['avg_correct_pct'])
val_error = dict_or_None(lambda: 1 - callback_result['sets']['validate']['summary']['normal, L2']['avg_correct_pct'])
if (loss is not None and math.isnan(loss)) or (val_loss is not None and math.isnan(val_loss)):
return False
return not (
check_gte(epoch, args.epochs) or \
check_gte(get_elapsed(), args.seconds) or \
(check_lte(loss, args.loss) and check_lte(val_loss, args.loss)) or \
(check_lte(error, args.error) and check_lte(val_error, args.error))
)
sample_size = get_required_arg(args, 'sample_size')
# end of epoch callback: output stats, take snapshot
snapshot_prefix = args.output_snapshot_prefix
def epoch_callback(round_stats, epoch):
elapsed = get_elapsed()
log("Begin epoch callback for epoch {0}".format(epoch))
if validation_skip > 0 and (epoch + 1) % validation_skip == 0:
DLs = [('normal, L2', None, embedding_dst)]
set_dicts = output_dumps.full_stats(round_stats, sets, DLs,
model, sample_size=sample_size, log=lambda *_: None,
show_multiple=show_multiple)
else:
set_dicts = round_stats
s = {'epoch': epoch, 'time': elapsed, 'sets': set_dicts}
stat.info("{0},".format(s))
# take snapshot
if snapshot_prefix and snapshot_skip > 0 and (epoch + 1) % snapshot_skip == 0:
snapshot_name = "{0}.{1}".format(snapshot_prefix, epoch)
log("Exporting snapshot weights for epoch {0} to {1}".format(epoch, snapshot_name))
helpers.export_weights(model, snapshot_name)
log("Exported snapshot weights for epoch {0}".format(epoch))
log("End epoch callback for epoch {0}".format(epoch))
return s
model.X1 = [[embedding_dst.start_1h]]
log("Training model...")
timer_start = datetime.utcnow()
model.fit(
X_train, Y_train, M_train,
X_validation, Y_validation, M_validation,
lr_A=lr_A, lr_B=lr_B,
batch_size=batch_size,
verbose=1, shuffle=True,
epoch_start=epoch_start,
continue_training=continue_training,
epoch_callback=epoch_callback
)
output_model = args.output_fitted_model
if output_model is not None:
log("Exporting fitted model to {0}".format(output_model))
helpers.export_model(model, output_model)
output_weights = args.output_model_weights
if output_weights is not None:
log("Exporting fitted weights to {0}".format(output_weights))
helpers.export_weights(model, output_weights)
cache['fitted_model'] = model
def h_train(cache, args):
sets = {}
req = ['X_emb', 'Y_tokens', 'Y_emb', 'M', 'maxlen']
show_multiple = s2b(get_required_arg(args, 'show_multiple'))
# load embeddings
embedding_src = get_embedding(cache, args, 'embedding_src')
embedding_dst = get_embedding(cache, args, 'embedding_dst')
# load train dataset
train_src = get_required_arg(args, 'train_src')
train_dst = get_required_arg(args, 'train_dst')
maxlen = get_required_arg(args, 'maxlen')
sets['train'] = helpers.load_datasets(req, embedding_src, embedding_dst,
train_src, train_dst, maxlen)
X_train = sets['train']['X_emb']
Y_train = sets['train']['Y_emb']
M_train = sets['train']['M']
# load validation dataset
validation_src = get_required_arg(args, 'validation_src')
validation_dst = get_required_arg(args, 'validation_dst')
maxlen = get_required_arg(args, 'maxlen')
sets['validate'] = helpers.load_datasets(req, embedding_src, embedding_dst,
validation_src, validation_dst,
maxlen)
X_validation = sets['validate']['X_emb']
Y_validation = sets['validate']['Y_emb']
M_validation = sets['validate']['M']
# load model
modelfile = args.compiled_model
if modelfile is not None:
model = helpers.import_model(modelfile)
elif 'compiled_model' in cache:
model = cache['compiled_model']
else:
print "Error in train: no compiled model provided: exiting"
exit()
# load weights (if applicable)
input_weights = args.model_weights
if input_weights is not None:
log('loading weights')
weights = helpers.import_weights(input_weights)
model.set_weights(weights)
log('done')
# train model
batch_size = get_required_arg(args, 'batch_size')
validation_skip = get_required_arg(args, 'validation_skip')
snapshot_skip = get_required_arg(args, 'snapshot_skip')
lr_A = get_required_arg(args, 'lr_encoder')
lr_B = get_required_arg(args, 'lr_decoder')
epoch_start = get_required_arg(args, 'epoch_start')
timer_start = None
def get_elapsed():
return (datetime.utcnow() - timer_start).total_seconds()
# policy evaluated at beginning of each epoch:
# stop training if any thresholds are met
def continue_training(epoch, callback_result):
def check_lte(left, right):
return left is not None and right is not None and left <= right
def check_gte(left, right):
return left is not None and right is not None and left >= right
def dict_or_None(l):
try:
return l()
except KeyError:
return None
except TypeError:
return None
loss = dict_or_None(lambda: callback_result['sets']['train']['loss'])
val_loss = dict_or_None(
lambda: callback_result['sets']['validate']['loss'])
error = dict_or_None(lambda: 1 - callback_result['sets']['train'][
'summary']['normal, L2']['avg_correct_pct'])
val_error = dict_or_None(lambda: 1 - callback_result['sets'][
'validate']['summary']['normal, L2']['avg_correct_pct'])
if (loss is not None
and math.isnan(loss)) or (val_loss is not None
and math.isnan(val_loss)):
return False
return not (
check_gte(epoch, args.epochs) or \
check_gte(get_elapsed(), args.seconds) or \
(check_lte(loss, args.loss) and check_lte(val_loss, args.loss)) or \
(check_lte(error, args.error) and check_lte(val_error, args.error))
)
sample_size = get_required_arg(args, 'sample_size')
# end of epoch callback: output stats, take snapshot
snapshot_prefix = args.output_snapshot_prefix
def epoch_callback(round_stats, epoch):
elapsed = get_elapsed()
log("Begin epoch callback for epoch {0}".format(epoch))
if validation_skip > 0 and (epoch + 1) % validation_skip == 0:
DLs = [('normal, L2', None, embedding_dst)]
set_dicts = output_dumps.full_stats(round_stats,
sets,
DLs,
model,
sample_size=sample_size,
log=lambda *_: None,
show_multiple=show_multiple)
else:
set_dicts = round_stats
s = {'epoch': epoch, 'time': elapsed, 'sets': set_dicts}
stat.info("{0},".format(s))
# take snapshot
if snapshot_prefix and snapshot_skip > 0 and (epoch +
1) % snapshot_skip == 0:
snapshot_name = "{0}.{1}".format(snapshot_prefix, epoch)
log("Exporting snapshot weights for epoch {0} to {1}".format(
epoch, snapshot_name))
helpers.export_weights(model, snapshot_name)
log("Exported snapshot weights for epoch {0}".format(epoch))
log("End epoch callback for epoch {0}".format(epoch))
return s
model.X1 = [[embedding_dst.start_1h]]
log("Training model...")
timer_start = datetime.utcnow()
model.fit(X_train,
Y_train,
M_train,
X_validation,
Y_validation,
M_validation,
lr_A=lr_A,
lr_B=lr_B,
batch_size=batch_size,
verbose=1,
shuffle=True,
epoch_start=epoch_start,
continue_training=continue_training,
epoch_callback=epoch_callback)
output_model = args.output_fitted_model
if output_model is not None:
log("Exporting fitted model to {0}".format(output_model))
helpers.export_model(model, output_model)
output_weights = args.output_model_weights
if output_weights is not None:
log("Exporting fitted weights to {0}".format(output_weights))
helpers.export_weights(model, output_weights)
cache['fitted_model'] = model
