Example #1
0
def get_fitted_model(cache, args):
    # load model
    input_model = args.fitted_model
    input_weights = args.model_weights
    compiled_model = args.compiled_model
    model = None
    if input_model is not None:
        model = helpers.import_model(input_model)
    elif input_weights is not None:
        model = cache['compiled_model']
        if compiled_model is not None:
            model = helpers.import_model(compiled_model)
        if model is None:
            print "Error loading fitted model: weights were provided, but a model was not compiled. Exiting"
        weights = helpers.import_weights(input_weights)
        model.set_weights(weights)
    elif 'fitted_model' in cache:
        model = cache['fitted_model']
    else:
        print "Error loading fitted model: no fitted model provided: exiting"
        exit()
    return model
Example #2
0
def get_fitted_model(cache, args):
    # load model
    input_model = args.fitted_model
    input_weights = args.model_weights
    compiled_model = args.compiled_model
    model = None
    if input_model is not None:
        model = helpers.import_model(input_model)
    elif input_weights is not None:
        model = cache['compiled_model']
        if compiled_model is not None:
            model = helpers.import_model(compiled_model)
        if model is None:
            print "Error loading fitted model: weights were provided, but a model was not compiled. Exiting"
        weights = helpers.import_weights(input_weights)
        model.set_weights(weights)
    elif 'fitted_model' in cache:
        model = cache['fitted_model']
    else:
        print "Error loading fitted model: no fitted model provided: exiting"
        exit()
    return model
Example #3
0
def h_train(cache, args):
    sets = {}
    req = ['X_emb', 'Y_tokens', 'Y_emb', 'M', 'maxlen']
    show_multiple = s2b(get_required_arg(args, 'show_multiple'))

    # load embeddings
    embedding_src = get_embedding(cache, args, 'embedding_src')
    embedding_dst = get_embedding(cache, args, 'embedding_dst')

    # load train dataset
    train_src = get_required_arg(args, 'train_src')
    train_dst = get_required_arg(args, 'train_dst')
    maxlen = get_required_arg(args, 'maxlen')

    sets['train'] = helpers.load_datasets(req,
            embedding_src, embedding_dst,
            train_src, train_dst,
            maxlen)
    X_train = sets['train']['X_emb']
    Y_train = sets['train']['Y_emb']
    M_train = sets['train']['M']

    # load validation dataset
    validation_src = get_required_arg(args, 'validation_src')
    validation_dst = get_required_arg(args, 'validation_dst')
    maxlen = get_required_arg(args, 'maxlen')

    sets['validate'] = helpers.load_datasets(req,
            embedding_src, embedding_dst,
            validation_src, validation_dst,
            maxlen)
    X_validation = sets['validate']['X_emb']
    Y_validation = sets['validate']['Y_emb']
    M_validation = sets['validate']['M']

    # load model
    modelfile = args.compiled_model
    if modelfile is not None:
        model = helpers.import_model(modelfile)
    elif 'compiled_model' in cache:
        model = cache['compiled_model']
    else:
        print "Error in train: no compiled model provided: exiting"
        exit()

    # load weights (if applicable)
    input_weights = args.model_weights
    if input_weights is not None:
        log('loading weights')
        weights = helpers.import_weights(input_weights)
        model.set_weights(weights)
        log('done')

    # train model
    batch_size = get_required_arg(args, 'batch_size')
    validation_skip = get_required_arg(args, 'validation_skip')
    snapshot_skip = get_required_arg(args, 'snapshot_skip')
    lr_A = get_required_arg(args, 'lr_encoder')
    lr_B = get_required_arg(args, 'lr_decoder')
    epoch_start = get_required_arg(args, 'epoch_start')

    timer_start = None

    def get_elapsed():
        return (datetime.utcnow() - timer_start).total_seconds()

    # policy evaluated at beginning of each epoch:
    # stop training if any thresholds are met
    def continue_training(epoch, callback_result):
        def check_lte(left, right):
            return left is not None and right is not None and left <= right
        def check_gte(left, right):
            return left is not None and right is not None and left >= right
        def dict_or_None(l):
            try:
                return l()
            except KeyError:
                return None
            except TypeError:
                return None

        loss      = dict_or_None(lambda: callback_result['sets']['train']['loss'])
        val_loss  = dict_or_None(lambda: callback_result['sets']['validate']['loss'])
        error     = dict_or_None(lambda: 1 - callback_result['sets']['train']['summary']['normal, L2']['avg_correct_pct'])
        val_error = dict_or_None(lambda: 1 - callback_result['sets']['validate']['summary']['normal, L2']['avg_correct_pct'])

        if (loss is not None and math.isnan(loss)) or (val_loss is not None and math.isnan(val_loss)):
            return False

        return not (
            check_gte(epoch, args.epochs) or \
            check_gte(get_elapsed(), args.seconds) or \
            (check_lte(loss, args.loss) and check_lte(val_loss, args.loss)) or \
            (check_lte(error, args.error) and check_lte(val_error, args.error))
        )

    sample_size = get_required_arg(args, 'sample_size')

    # end of epoch callback: output stats, take snapshot
    snapshot_prefix = args.output_snapshot_prefix
    def epoch_callback(round_stats, epoch):
        elapsed = get_elapsed()

        log("Begin epoch callback for epoch {0}".format(epoch))

        if validation_skip > 0 and (epoch + 1) % validation_skip == 0:
            DLs = [('normal, L2', None, embedding_dst)]
            set_dicts = output_dumps.full_stats(round_stats, sets, DLs,
                    model, sample_size=sample_size, log=lambda *_: None,
                    show_multiple=show_multiple)
        else:
            set_dicts = round_stats

        s = {'epoch': epoch, 'time': elapsed, 'sets': set_dicts}

        stat.info("{0},".format(s))

        # take snapshot
        if snapshot_prefix and snapshot_skip > 0 and (epoch + 1) % snapshot_skip == 0:
            snapshot_name = "{0}.{1}".format(snapshot_prefix, epoch)
            log("Exporting snapshot weights for epoch {0} to {1}".format(epoch, snapshot_name))
            helpers.export_weights(model, snapshot_name)
            log("Exported snapshot weights for epoch {0}".format(epoch))

        log("End epoch callback for epoch {0}".format(epoch))
        return s

    model.X1 = [[embedding_dst.start_1h]]

    log("Training model...")
    timer_start = datetime.utcnow()
    model.fit(
            X_train, Y_train, M_train,
            X_validation, Y_validation, M_validation,
            lr_A=lr_A, lr_B=lr_B,
            batch_size=batch_size,
            verbose=1, shuffle=True,
            epoch_start=epoch_start,
            continue_training=continue_training,
            epoch_callback=epoch_callback
    )

    output_model = args.output_fitted_model
    if output_model is not None:
        log("Exporting fitted model to {0}".format(output_model))
        helpers.export_model(model, output_model)

    output_weights = args.output_model_weights
    if output_weights is not None:
        log("Exporting fitted weights to {0}".format(output_weights))
        helpers.export_weights(model, output_weights)

    cache['fitted_model'] = model
Example #4
0
def h_train(cache, args):
    sets = {}
    req = ['X_emb', 'Y_tokens', 'Y_emb', 'M', 'maxlen']
    show_multiple = s2b(get_required_arg(args, 'show_multiple'))

    # load embeddings
    embedding_src = get_embedding(cache, args, 'embedding_src')
    embedding_dst = get_embedding(cache, args, 'embedding_dst')

    # load train dataset
    train_src = get_required_arg(args, 'train_src')
    train_dst = get_required_arg(args, 'train_dst')
    maxlen = get_required_arg(args, 'maxlen')

    sets['train'] = helpers.load_datasets(req, embedding_src, embedding_dst,
                                          train_src, train_dst, maxlen)
    X_train = sets['train']['X_emb']
    Y_train = sets['train']['Y_emb']
    M_train = sets['train']['M']

    # load validation dataset
    validation_src = get_required_arg(args, 'validation_src')
    validation_dst = get_required_arg(args, 'validation_dst')
    maxlen = get_required_arg(args, 'maxlen')

    sets['validate'] = helpers.load_datasets(req, embedding_src, embedding_dst,
                                             validation_src, validation_dst,
                                             maxlen)
    X_validation = sets['validate']['X_emb']
    Y_validation = sets['validate']['Y_emb']
    M_validation = sets['validate']['M']

    # load model
    modelfile = args.compiled_model
    if modelfile is not None:
        model = helpers.import_model(modelfile)
    elif 'compiled_model' in cache:
        model = cache['compiled_model']
    else:
        print "Error in train: no compiled model provided: exiting"
        exit()

    # load weights (if applicable)
    input_weights = args.model_weights
    if input_weights is not None:
        log('loading weights')
        weights = helpers.import_weights(input_weights)
        model.set_weights(weights)
        log('done')

    # train model
    batch_size = get_required_arg(args, 'batch_size')
    validation_skip = get_required_arg(args, 'validation_skip')
    snapshot_skip = get_required_arg(args, 'snapshot_skip')
    lr_A = get_required_arg(args, 'lr_encoder')
    lr_B = get_required_arg(args, 'lr_decoder')
    epoch_start = get_required_arg(args, 'epoch_start')

    timer_start = None

    def get_elapsed():
        return (datetime.utcnow() - timer_start).total_seconds()

    # policy evaluated at beginning of each epoch:
    # stop training if any thresholds are met
    def continue_training(epoch, callback_result):
        def check_lte(left, right):
            return left is not None and right is not None and left <= right

        def check_gte(left, right):
            return left is not None and right is not None and left >= right

        def dict_or_None(l):
            try:
                return l()
            except KeyError:
                return None
            except TypeError:
                return None

        loss = dict_or_None(lambda: callback_result['sets']['train']['loss'])
        val_loss = dict_or_None(
            lambda: callback_result['sets']['validate']['loss'])
        error = dict_or_None(lambda: 1 - callback_result['sets']['train'][
            'summary']['normal, L2']['avg_correct_pct'])
        val_error = dict_or_None(lambda: 1 - callback_result['sets'][
            'validate']['summary']['normal, L2']['avg_correct_pct'])

        if (loss is not None
                and math.isnan(loss)) or (val_loss is not None
                                          and math.isnan(val_loss)):
            return False

        return not (
            check_gte(epoch, args.epochs) or \
            check_gte(get_elapsed(), args.seconds) or \
            (check_lte(loss, args.loss) and check_lte(val_loss, args.loss)) or \
            (check_lte(error, args.error) and check_lte(val_error, args.error))
        )

    sample_size = get_required_arg(args, 'sample_size')

    # end of epoch callback: output stats, take snapshot
    snapshot_prefix = args.output_snapshot_prefix

    def epoch_callback(round_stats, epoch):
        elapsed = get_elapsed()

        log("Begin epoch callback for epoch {0}".format(epoch))

        if validation_skip > 0 and (epoch + 1) % validation_skip == 0:
            DLs = [('normal, L2', None, embedding_dst)]
            set_dicts = output_dumps.full_stats(round_stats,
                                                sets,
                                                DLs,
                                                model,
                                                sample_size=sample_size,
                                                log=lambda *_: None,
                                                show_multiple=show_multiple)
        else:
            set_dicts = round_stats

        s = {'epoch': epoch, 'time': elapsed, 'sets': set_dicts}

        stat.info("{0},".format(s))

        # take snapshot
        if snapshot_prefix and snapshot_skip > 0 and (epoch +
                                                      1) % snapshot_skip == 0:
            snapshot_name = "{0}.{1}".format(snapshot_prefix, epoch)
            log("Exporting snapshot weights for epoch {0} to {1}".format(
                epoch, snapshot_name))
            helpers.export_weights(model, snapshot_name)
            log("Exported snapshot weights for epoch {0}".format(epoch))

        log("End epoch callback for epoch {0}".format(epoch))
        return s

    model.X1 = [[embedding_dst.start_1h]]

    log("Training model...")
    timer_start = datetime.utcnow()
    model.fit(X_train,
              Y_train,
              M_train,
              X_validation,
              Y_validation,
              M_validation,
              lr_A=lr_A,
              lr_B=lr_B,
              batch_size=batch_size,
              verbose=1,
              shuffle=True,
              epoch_start=epoch_start,
              continue_training=continue_training,
              epoch_callback=epoch_callback)

    output_model = args.output_fitted_model
    if output_model is not None:
        log("Exporting fitted model to {0}".format(output_model))
        helpers.export_model(model, output_model)

    output_weights = args.output_model_weights
    if output_weights is not None:
        log("Exporting fitted weights to {0}".format(output_weights))
        helpers.export_weights(model, output_weights)

    cache['fitted_model'] = model