def get_fitted_model(cache, args): # load model input_model = args.fitted_model input_weights = args.model_weights compiled_model = args.compiled_model model = None if input_model is not None: model = helpers.import_model(input_model) elif input_weights is not None: model = cache['compiled_model'] if compiled_model is not None: model = helpers.import_model(compiled_model) if model is None: print "Error loading fitted model: weights were provided, but a model was not compiled. Exiting" weights = helpers.import_weights(input_weights) model.set_weights(weights) elif 'fitted_model' in cache: model = cache['fitted_model'] else: print "Error loading fitted model: no fitted model provided: exiting" exit() return model
def h_train(cache, args): sets = {} req = ['X_emb', 'Y_tokens', 'Y_emb', 'M', 'maxlen'] show_multiple = s2b(get_required_arg(args, 'show_multiple')) # load embeddings embedding_src = get_embedding(cache, args, 'embedding_src') embedding_dst = get_embedding(cache, args, 'embedding_dst') # load train dataset train_src = get_required_arg(args, 'train_src') train_dst = get_required_arg(args, 'train_dst') maxlen = get_required_arg(args, 'maxlen') sets['train'] = helpers.load_datasets(req, embedding_src, embedding_dst, train_src, train_dst, maxlen) X_train = sets['train']['X_emb'] Y_train = sets['train']['Y_emb'] M_train = sets['train']['M'] # load validation dataset validation_src = get_required_arg(args, 'validation_src') validation_dst = get_required_arg(args, 'validation_dst') maxlen = get_required_arg(args, 'maxlen') sets['validate'] = helpers.load_datasets(req, embedding_src, embedding_dst, validation_src, validation_dst, maxlen) X_validation = sets['validate']['X_emb'] Y_validation = sets['validate']['Y_emb'] M_validation = sets['validate']['M'] # load model modelfile = args.compiled_model if modelfile is not None: model = helpers.import_model(modelfile) elif 'compiled_model' in cache: model = cache['compiled_model'] else: print "Error in train: no compiled model provided: exiting" exit() # load weights (if applicable) input_weights = args.model_weights if input_weights is not None: log('loading weights') weights = helpers.import_weights(input_weights) model.set_weights(weights) log('done') # train model batch_size = get_required_arg(args, 'batch_size') validation_skip = get_required_arg(args, 'validation_skip') snapshot_skip = get_required_arg(args, 'snapshot_skip') lr_A = get_required_arg(args, 'lr_encoder') lr_B = get_required_arg(args, 'lr_decoder') epoch_start = get_required_arg(args, 'epoch_start') timer_start = None def get_elapsed(): return (datetime.utcnow() - timer_start).total_seconds() # policy evaluated at beginning of each epoch: # stop training if any thresholds are met def continue_training(epoch, callback_result): def check_lte(left, right): return left is not None and right is not None and left <= right def check_gte(left, right): return left is not None and right is not None and left >= right def dict_or_None(l): try: return l() except KeyError: return None except TypeError: return None loss = dict_or_None(lambda: callback_result['sets']['train']['loss']) val_loss = dict_or_None(lambda: callback_result['sets']['validate']['loss']) error = dict_or_None(lambda: 1 - callback_result['sets']['train']['summary']['normal, L2']['avg_correct_pct']) val_error = dict_or_None(lambda: 1 - callback_result['sets']['validate']['summary']['normal, L2']['avg_correct_pct']) if (loss is not None and math.isnan(loss)) or (val_loss is not None and math.isnan(val_loss)): return False return not ( check_gte(epoch, args.epochs) or \ check_gte(get_elapsed(), args.seconds) or \ (check_lte(loss, args.loss) and check_lte(val_loss, args.loss)) or \ (check_lte(error, args.error) and check_lte(val_error, args.error)) ) sample_size = get_required_arg(args, 'sample_size') # end of epoch callback: output stats, take snapshot snapshot_prefix = args.output_snapshot_prefix def epoch_callback(round_stats, epoch): elapsed = get_elapsed() log("Begin epoch callback for epoch {0}".format(epoch)) if validation_skip > 0 and (epoch + 1) % validation_skip == 0: DLs = [('normal, L2', None, embedding_dst)] set_dicts = output_dumps.full_stats(round_stats, sets, DLs, model, sample_size=sample_size, log=lambda *_: None, show_multiple=show_multiple) else: set_dicts = round_stats s = {'epoch': epoch, 'time': elapsed, 'sets': set_dicts} stat.info("{0},".format(s)) # take snapshot if snapshot_prefix and snapshot_skip > 0 and (epoch + 1) % snapshot_skip == 0: snapshot_name = "{0}.{1}".format(snapshot_prefix, epoch) log("Exporting snapshot weights for epoch {0} to {1}".format(epoch, snapshot_name)) helpers.export_weights(model, snapshot_name) log("Exported snapshot weights for epoch {0}".format(epoch)) log("End epoch callback for epoch {0}".format(epoch)) return s model.X1 = [[embedding_dst.start_1h]] log("Training model...") timer_start = datetime.utcnow() model.fit( X_train, Y_train, M_train, X_validation, Y_validation, M_validation, lr_A=lr_A, lr_B=lr_B, batch_size=batch_size, verbose=1, shuffle=True, epoch_start=epoch_start, continue_training=continue_training, epoch_callback=epoch_callback ) output_model = args.output_fitted_model if output_model is not None: log("Exporting fitted model to {0}".format(output_model)) helpers.export_model(model, output_model) output_weights = args.output_model_weights if output_weights is not None: log("Exporting fitted weights to {0}".format(output_weights)) helpers.export_weights(model, output_weights) cache['fitted_model'] = model
def h_train(cache, args): sets = {} req = ['X_emb', 'Y_tokens', 'Y_emb', 'M', 'maxlen'] show_multiple = s2b(get_required_arg(args, 'show_multiple')) # load embeddings embedding_src = get_embedding(cache, args, 'embedding_src') embedding_dst = get_embedding(cache, args, 'embedding_dst') # load train dataset train_src = get_required_arg(args, 'train_src') train_dst = get_required_arg(args, 'train_dst') maxlen = get_required_arg(args, 'maxlen') sets['train'] = helpers.load_datasets(req, embedding_src, embedding_dst, train_src, train_dst, maxlen) X_train = sets['train']['X_emb'] Y_train = sets['train']['Y_emb'] M_train = sets['train']['M'] # load validation dataset validation_src = get_required_arg(args, 'validation_src') validation_dst = get_required_arg(args, 'validation_dst') maxlen = get_required_arg(args, 'maxlen') sets['validate'] = helpers.load_datasets(req, embedding_src, embedding_dst, validation_src, validation_dst, maxlen) X_validation = sets['validate']['X_emb'] Y_validation = sets['validate']['Y_emb'] M_validation = sets['validate']['M'] # load model modelfile = args.compiled_model if modelfile is not None: model = helpers.import_model(modelfile) elif 'compiled_model' in cache: model = cache['compiled_model'] else: print "Error in train: no compiled model provided: exiting" exit() # load weights (if applicable) input_weights = args.model_weights if input_weights is not None: log('loading weights') weights = helpers.import_weights(input_weights) model.set_weights(weights) log('done') # train model batch_size = get_required_arg(args, 'batch_size') validation_skip = get_required_arg(args, 'validation_skip') snapshot_skip = get_required_arg(args, 'snapshot_skip') lr_A = get_required_arg(args, 'lr_encoder') lr_B = get_required_arg(args, 'lr_decoder') epoch_start = get_required_arg(args, 'epoch_start') timer_start = None def get_elapsed(): return (datetime.utcnow() - timer_start).total_seconds() # policy evaluated at beginning of each epoch: # stop training if any thresholds are met def continue_training(epoch, callback_result): def check_lte(left, right): return left is not None and right is not None and left <= right def check_gte(left, right): return left is not None and right is not None and left >= right def dict_or_None(l): try: return l() except KeyError: return None except TypeError: return None loss = dict_or_None(lambda: callback_result['sets']['train']['loss']) val_loss = dict_or_None( lambda: callback_result['sets']['validate']['loss']) error = dict_or_None(lambda: 1 - callback_result['sets']['train'][ 'summary']['normal, L2']['avg_correct_pct']) val_error = dict_or_None(lambda: 1 - callback_result['sets'][ 'validate']['summary']['normal, L2']['avg_correct_pct']) if (loss is not None and math.isnan(loss)) or (val_loss is not None and math.isnan(val_loss)): return False return not ( check_gte(epoch, args.epochs) or \ check_gte(get_elapsed(), args.seconds) or \ (check_lte(loss, args.loss) and check_lte(val_loss, args.loss)) or \ (check_lte(error, args.error) and check_lte(val_error, args.error)) ) sample_size = get_required_arg(args, 'sample_size') # end of epoch callback: output stats, take snapshot snapshot_prefix = args.output_snapshot_prefix def epoch_callback(round_stats, epoch): elapsed = get_elapsed() log("Begin epoch callback for epoch {0}".format(epoch)) if validation_skip > 0 and (epoch + 1) % validation_skip == 0: DLs = [('normal, L2', None, embedding_dst)] set_dicts = output_dumps.full_stats(round_stats, sets, DLs, model, sample_size=sample_size, log=lambda *_: None, show_multiple=show_multiple) else: set_dicts = round_stats s = {'epoch': epoch, 'time': elapsed, 'sets': set_dicts} stat.info("{0},".format(s)) # take snapshot if snapshot_prefix and snapshot_skip > 0 and (epoch + 1) % snapshot_skip == 0: snapshot_name = "{0}.{1}".format(snapshot_prefix, epoch) log("Exporting snapshot weights for epoch {0} to {1}".format( epoch, snapshot_name)) helpers.export_weights(model, snapshot_name) log("Exported snapshot weights for epoch {0}".format(epoch)) log("End epoch callback for epoch {0}".format(epoch)) return s model.X1 = [[embedding_dst.start_1h]] log("Training model...") timer_start = datetime.utcnow() model.fit(X_train, Y_train, M_train, X_validation, Y_validation, M_validation, lr_A=lr_A, lr_B=lr_B, batch_size=batch_size, verbose=1, shuffle=True, epoch_start=epoch_start, continue_training=continue_training, epoch_callback=epoch_callback) output_model = args.output_fitted_model if output_model is not None: log("Exporting fitted model to {0}".format(output_model)) helpers.export_model(model, output_model) output_weights = args.output_model_weights if output_weights is not None: log("Exporting fitted weights to {0}".format(output_weights)) helpers.export_weights(model, output_weights) cache['fitted_model'] = model