from keras.models import Model, Sequential
from keras.layers import Dense, Dropout, Input
from keras.callbacks import Callback, ModelCheckpoint

import p1b1

EPOCH = 2
BATCH = 50

P = 60025  # 245 x 245
N1 = 2000
NE = 600  # encoded dim
F_MAX = 33.3
DR = 0.2

X_train, X_test = p1b1.load_data()

input_dim = X_train.shape[1]
output_dim = input_dim

input_vector = Input(shape=(input_dim, ))
x = Dense(N1, activation='sigmoid')(input_vector)
# x = Dropout(DR)(x)
x = Dense(NE, activation='sigmoid')(x)
encoded = x

x = Dense(N1, activation='sigmoid')(encoded)
# x = Dropout(DR)(x)
x = Dense(output_dim, activation='sigmoid')(x)
decoded = x
def run(params):

    args = candle.ArgumentStruct(**params)
    seed = args.rng_seed
    candle.set_seed(seed)

    # # Construct extension to save model
    # ext = p1b1.extension_from_parameters(params, '.keras')
    # candle.verify_path(params['save_path'])
    # prefix = '{}{}'.format(params['save_path'], ext)
    # logfile = params['logfile'] if params['logfile'] else prefix+'.log'
    # candle.set_up_logger(logfile, p1b1.logger, params['verbose'])
    #p1b1.logger.info('Params: {}'.format(params))

    # Get default parameters for initialization and optimizer functions
    keras_defaults = candle.keras_default_config()

    # Load dataset
    x_train, y_train, x_val, y_val, x_test, y_test, x_labels, y_labels = p1b1.load_data(
        params, seed)

    # cache_file = 'data_l1000_cache.h5'
    # save_cache(cache_file, x_train, y_train, x_val, y_val, x_test, y_test, x_labels, y_labels)
    # x_train, y_train, x_val, y_val, x_test, y_test, x_labels, y_labels = load_cache(cache_file)

    # p1b1.logger.info("Shape x_train: {}".format(x_train.shape))
    # p1b1.logger.info("Shape x_val:   {}".format(x_val.shape))
    #p1b1.logger.info("Shape x_test:  {}".format(x_test.shape))

    # p1b1.logger.info("Range x_train: [{:.3g}, {:.3g}]".format(np.min(x_train), np.max(x_train)))
    # p1b1.logger.info("Range x_val:   [{:.3g}, {:.3g}]".format(np.min(x_val), np.max(x_val)))
    #p1b1.logger.info("Range x_test:  [{:.3g}, {:.3g}]".format(np.min(x_test), np.max(x_test)))

    # p1b1.logger.debug('Class labels')
    # for i, label in enumerate(y_labels):
    #     p1b1.logger.debug('  {}: {}'.format(i, label))

    # clf = build_type_classifier(x_train, y_train, x_val, y_val)

    n_classes = len(y_labels)
    cond_train = y_train
    cond_val = y_val
    cond_test = y_test

    input_dim = x_train.shape[1]
    cond_dim = cond_train.shape[1]
    latent_dim = params['latent_dim']

    activation = params['activation']
    dropout = params['dropout']
    dense_layers = params['dense']
    dropout_layer = AlphaDropout if params['alpha_dropout'] else Dropout

    # Initialize weights and learning rule
    initializer_weights = candle.build_initializer(params['initialization'],
                                                   keras_defaults, seed)
    initializer_bias = candle.build_initializer('constant', keras_defaults, 0.)

    if dense_layers is not None:
        if type(dense_layers) != list:
            dense_layers = list(dense_layers)
    else:
        dense_layers = []

    # Encoder Part
    x_input = Input(shape=(input_dim, ))
    cond_input = Input(shape=(cond_dim, ))
    h = x_input
    if params['model'] == 'cvae':
        h = keras.layers.concatenate([x_input, cond_input])

    for i, layer in enumerate(dense_layers):
        if layer > 0:
            x = h
            h = Dense(layer,
                      activation=activation,
                      kernel_initializer=initializer_weights,
                      bias_initializer=initializer_bias)(h)
            if params['residual']:
                try:
                    h = keras.layers.add([h, x])
                except ValueError:
                    pass
            if params['batch_normalization']:
                h = BatchNormalization()(h)
            if dropout > 0:
                h = dropout_layer(dropout)(h)

    if params['model'] == 'ae':
        encoded = Dense(latent_dim,
                        activation=activation,
                        kernel_initializer=initializer_weights,
                        bias_initializer=initializer_bias)(h)
    else:
        epsilon_std = params['epsilon_std']
        z_mean = Dense(latent_dim, name='z_mean')(h)
        z_log_var = Dense(latent_dim, name='z_log_var')(h)
        encoded = z_mean

        def vae_loss(x, x_decoded_mean):
            xent_loss = binary_crossentropy(x, x_decoded_mean)
            kl_loss = -0.5 * K.sum(
                1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
            return K.mean(xent_loss + kl_loss / input_dim)

        def sampling(params):
            z_mean_, z_log_var_ = params
            batch_size = K.shape(z_mean_)[0]
            epsilon = K.random_normal(shape=(batch_size, latent_dim),
                                      mean=0.,
                                      stddev=epsilon_std)
            return z_mean_ + K.exp(z_log_var_ / 2) * epsilon

        z = Lambda(sampling, output_shape=(latent_dim, ))([z_mean, z_log_var])
        if params['model'] == 'cvae':
            z_cond = keras.layers.concatenate([z, cond_input])

    # Build autoencoder model
    if params['model'] == 'cvae':
        # encoder = Model([x_input, cond_input], encoded)
        # decoder = Model([decoder_input, cond_input], decoded)
        # model = Model([x_input, cond_input], decoder([z, cond_input]))
        loss = vae_loss
        metrics = [xent, corr, mse]
    elif params['model'] == 'vae':
        # encoder = Model(x_input, encoded)
        # decoder = Model(decoder_input, decoded)
        # model = Model(x_input, decoder(z))
        loss = vae_loss
        metrics = [xent, corr, mse]
    else:
        # encoder = Model(x_input, encoded)
        # decoder = Model(decoder_input, decoded)
        # model = Model(x_input, decoder(encoded))
        loss = params['loss']
        metrics = [xent, corr]

    # Define optimizer
    # optimizer = candle.build_optimizer(params['optimizer'],
    #                                             params['learning_rate'],
    #                                             keras_defaults)
    optimizer = optimizers.deserialize({
        'class_name': params['optimizer'],
        'config': {}
    })
    base_lr = params['base_lr'] or K.get_value(optimizer.lr)
    if params['learning_rate']:
        K.set_value(optimizer.lr, params['learning_rate'])

    if params['model'] == 'cvae':
        # inputs = [x_train, cond_train]
        # val_inputs = [x_val, cond_val]
        test_inputs = [x_test, cond_test]
    else:
        # inputs = x_train
        # val_inputs = x_val
        test_inputs = x_test

    test_outputs = x_test

    model_name = params['model_name']

    # load json and create model
    json_file = open('{}.{}.model.json'.format(model_name, params['model']),
                     'r')
    loaded_model_json = json_file.read()
    json_file.close()
    loaded_model_json = model_from_json(loaded_model_json)

    # load weights into new model
    loaded_model_json.load_weights('{}.{}.weights.h5'.format(
        model_name, params['model']))
    print("Loaded model from disk")

    # evaluate loaded model on test data
    loaded_model_json.compile(loss=loss, optimizer=optimizer, metrics=metrics)
    x_pred = loaded_model_json.predict(test_inputs)
    scores = p1b1.evaluate_autoencoder(x_pred, x_test)
    # p1b1.logger.info('\nEvaluation on test data: {}'.format(scores))
    print('Evaluation on test data: {}'.format(scores))

    # load encoder
    encoder = load_model('{}.{}.encoder.h5'.format(model_name,
                                                   params['model']))
    print("Loaded encoder from disk")

    x_test_encoded = encoder.predict(test_inputs,
                                     batch_size=params['batch_size'])
    y_test_classes = np.argmax(y_test, axis=1)
    candle.plot_scatter(x_test_encoded, y_test_classes,
                        '{}.{}.latent'.format(model_name, params['model']))

    if params['tsne']:
        tsne = TSNE(n_components=2, random_state=seed)
        x_test_encoded_tsne = tsne.fit_transform(x_test_encoded)
        candle.plot_scatter(
            x_test_encoded_tsne, y_test_classes,
            '{}.{}.latent.tsne'.format(model_name, params['model']))
Exemple #3
0
def run(params):

    args = candle.ArgumentStruct(**params)
    seed = args.rng_seed
    candle.set_seed(seed)

    # Construct extension to save model
    ext = p1b1.extension_from_parameters(params, '.keras')
    candle.verify_path(params['save'])
    prefix = '{}{}'.format(params['save'], ext)
    logfile = params['logfile'] if params['logfile'] else prefix + '.log'
    candle.set_up_logger(logfile, p1b1.logger, params['verbose'])
    p1b1.logger.info('Params: {}'.format(params))

    # Get default parameters for initialization and optimizer functions
    keras_defaults = candle.keras_default_config()

    # Load dataset
    x_train, y_train, x_val, y_val, x_test, y_test, x_labels, y_labels = p1b1.load_data(
        params, seed)

    # cache_file = 'data_l1000_cache.h5'
    # save_cache(cache_file, x_train, y_train, x_val, y_val, x_test, y_test, x_labels, y_labels)
    # x_train, y_train, x_val, y_val, x_test, y_test, x_labels, y_labels = load_cache(cache_file)

    p1b1.logger.info("Shape x_train: {}".format(x_train.shape))
    p1b1.logger.info("Shape x_val:   {}".format(x_val.shape))
    p1b1.logger.info("Shape x_test:  {}".format(x_test.shape))

    p1b1.logger.info("Range x_train: [{:.3g}, {:.3g}]".format(
        np.min(x_train), np.max(x_train)))
    p1b1.logger.info("Range x_val:   [{:.3g}, {:.3g}]".format(
        np.min(x_val), np.max(x_val)))
    p1b1.logger.info("Range x_test:  [{:.3g}, {:.3g}]".format(
        np.min(x_test), np.max(x_test)))

    p1b1.logger.debug('Class labels')
    for i, label in enumerate(y_labels):
        p1b1.logger.debug('  {}: {}'.format(i, label))

    # clf = build_type_classifier(x_train, y_train, x_val, y_val)

    n_classes = len(y_labels)
    cond_train = y_train
    cond_val = y_val
    cond_test = y_test

    input_dim = x_train.shape[1]
    cond_dim = cond_train.shape[1]
    latent_dim = params['latent_dim']

    activation = params['activation']
    dropout = params['drop']
    dense_layers = params['dense']
    dropout_layer = keras.layers.noise.AlphaDropout if params[
        'alpha_dropout'] else Dropout

    # Initialize weights and learning rule
    initializer_weights = candle.build_initializer(params['initialization'],
                                                   keras_defaults, seed)
    initializer_bias = candle.build_initializer('constant', keras_defaults, 0.)

    if dense_layers is not None:
        if type(dense_layers) != list:
            dense_layers = list(dense_layers)
    else:
        dense_layers = []

    # Encoder Part
    x_input = Input(shape=(input_dim, ))
    cond_input = Input(shape=(cond_dim, ))
    h = x_input
    if params['model'] == 'cvae':
        h = keras.layers.concatenate([x_input, cond_input])

    for i, layer in enumerate(dense_layers):
        if layer > 0:
            x = h
            h = Dense(layer,
                      activation=activation,
                      kernel_initializer=initializer_weights,
                      bias_initializer=initializer_bias)(h)
            if params['residual']:
                try:
                    h = keras.layers.add([h, x])
                except ValueError:
                    pass
            if params['batch_normalization']:
                h = BatchNormalization()(h)
            if dropout > 0:
                h = dropout_layer(dropout)(h)

    if params['model'] == 'ae':
        encoded = Dense(latent_dim,
                        activation=activation,
                        kernel_initializer=initializer_weights,
                        bias_initializer=initializer_bias)(h)
    else:
        epsilon_std = params['epsilon_std']
        z_mean = Dense(latent_dim, name='z_mean')(h)
        z_log_var = Dense(latent_dim, name='z_log_var')(h)
        encoded = z_mean

        def vae_loss(x, x_decoded_mean):
            xent_loss = binary_crossentropy(x, x_decoded_mean)
            kl_loss = -0.5 * K.sum(
                1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
            return K.mean(xent_loss + kl_loss / input_dim)

        def sampling(params):
            z_mean_, z_log_var_ = params
            batch_size = K.shape(z_mean_)[0]
            epsilon = K.random_normal(shape=(batch_size, latent_dim),
                                      mean=0.,
                                      stddev=epsilon_std)
            return z_mean_ + K.exp(z_log_var_ / 2) * epsilon

        z = Lambda(sampling, output_shape=(latent_dim, ))([z_mean, z_log_var])
        if params['model'] == 'cvae':
            z_cond = keras.layers.concatenate([z, cond_input])

    # Decoder Part
    decoder_input = Input(shape=(latent_dim, ))
    h = decoder_input
    if params['model'] == 'cvae':
        h = keras.layers.concatenate([decoder_input, cond_input])

    for i, layer in reversed(list(enumerate(dense_layers))):
        if layer > 0:
            x = h
            h = Dense(layer,
                      activation=activation,
                      kernel_initializer=initializer_weights,
                      bias_initializer=initializer_bias)(h)
            if params['residual']:
                try:
                    h = keras.layers.add([h, x])
                except ValueError:
                    pass
            if params['batch_normalization']:
                h = BatchNormalization()(h)
            if dropout > 0:
                h = dropout_layer(dropout)(h)

    decoded = Dense(input_dim,
                    activation='sigmoid',
                    kernel_initializer=initializer_weights,
                    bias_initializer=initializer_bias)(h)

    # Build autoencoder model
    if params['model'] == 'cvae':
        encoder = Model([x_input, cond_input], encoded)
        decoder = Model([decoder_input, cond_input], decoded)
        model = Model([x_input, cond_input], decoder([z, cond_input]))
        loss = vae_loss
        metrics = [xent, corr, mse]
    elif params['model'] == 'vae':
        encoder = Model(x_input, encoded)
        decoder = Model(decoder_input, decoded)
        model = Model(x_input, decoder(z))
        loss = vae_loss
        metrics = [xent, corr, mse]
    else:
        encoder = Model(x_input, encoded)
        decoder = Model(decoder_input, decoded)
        model = Model(x_input, decoder(encoded))
        loss = params['loss']
        metrics = [xent, corr]

    model.summary()
    decoder.summary()

    if params['cp']:
        model_json = model.to_json()
        with open(prefix + '.model.json', 'w') as f:
            print(model_json, file=f)

    # Define optimizer
    # optimizer = candle.build_optimizer(params['optimizer'],
    #                                             params['learning_rate'],
    #                                             keras_defaults)
    optimizer = optimizers.deserialize({
        'class_name': params['optimizer'],
        'config': {}
    })
    base_lr = params['base_lr'] or K.get_value(optimizer.lr)
    if params['learning_rate']:
        K.set_value(optimizer.lr, params['learning_rate'])

    model.compile(loss=loss, optimizer=optimizer, metrics=metrics)

    # calculate trainable and non-trainable params
    params.update(candle.compute_trainable_params(model))

    def warmup_scheduler(epoch):
        lr = params['learning_rate'] or base_lr * params['batch_size'] / 100
        if epoch <= 5:
            K.set_value(model.optimizer.lr,
                        (base_lr * (5 - epoch) + lr * epoch) / 5)
        p1b1.logger.debug('Epoch {}: lr={}'.format(
            epoch, K.get_value(model.optimizer.lr)))
        return K.get_value(model.optimizer.lr)

    reduce_lr = ReduceLROnPlateau(monitor='val_loss',
                                  factor=0.5,
                                  patience=5,
                                  min_lr=0.00001)
    warmup_lr = LearningRateScheduler(warmup_scheduler)
    checkpointer = ModelCheckpoint(params['save'] + ext + '.weights.h5',
                                   save_best_only=True,
                                   save_weights_only=True)
    tensorboard = TensorBoard(log_dir="tb/tb{}".format(ext))
    candle_monitor = candle.CandleRemoteMonitor(params=params)
    timeout_monitor = candle.TerminateOnTimeOut(params['timeout'])
    history_logger = LoggingCallback(p1b1.logger.debug)

    callbacks = [candle_monitor, timeout_monitor, history_logger]
    if params['reduce_lr']:
        callbacks.append(reduce_lr)
    if params['warmup_lr']:
        callbacks.append(warmup_lr)
    if params['cp']:
        callbacks.append(checkpointer)
    if params['tb']:
        callbacks.append(tensorboard)

    x_val2 = np.copy(x_val)
    np.random.shuffle(x_val2)
    start_scores = p1b1.evaluate_autoencoder(x_val, x_val2)
    p1b1.logger.info('\nBetween random pairs of validation samples: {}'.format(
        start_scores))

    if params['model'] == 'cvae':
        inputs = [x_train, cond_train]
        val_inputs = [x_val, cond_val]
        test_inputs = [x_test, cond_test]
    else:
        inputs = x_train
        val_inputs = x_val
        test_inputs = x_test

    outputs = x_train
    val_outputs = x_val
    test_outputs = x_test

    history = model.fit(inputs,
                        outputs,
                        verbose=2,
                        batch_size=params['batch_size'],
                        epochs=params['epochs'],
                        callbacks=callbacks,
                        validation_data=(val_inputs, val_outputs))

    if params['cp']:
        encoder.save(prefix + '.encoder.h5')
        decoder.save(prefix + '.decoder.h5')

    plot_history(prefix, history, 'loss')
    plot_history(prefix, history, 'corr', 'streaming pearson correlation')

    # Evalute model on test set
    x_pred = model.predict(test_inputs)
    scores = p1b1.evaluate_autoencoder(x_pred, x_test)
    p1b1.logger.info('\nEvaluation on test data: {}'.format(scores))

    x_test_encoded = encoder.predict(test_inputs,
                                     batch_size=params['batch_size'])
    y_test_classes = np.argmax(y_test, axis=1)
    plot_scatter(x_test_encoded, y_test_classes, prefix + '.latent')

    if params['tsne']:
        tsne = TSNE(n_components=2, random_state=seed)
        x_test_encoded_tsne = tsne.fit_transform(x_test_encoded)
        plot_scatter(x_test_encoded_tsne, y_test_classes,
                     prefix + '.latent.tsne')

    # diff = x_pred - x_test
    # plt.hist(diff.ravel(), bins='auto')
    # plt.title("Histogram of Errors with 'auto' bins")
    # plt.savefig('histogram_keras.png')

    # generate synthetic data
    # epsilon_std = 1.0
    # for i in range(1000):
    #     z_sample = np.random.normal(size=(1, 2)) * epsilon_std
    #     x_decoded = decoder.predict(z_sample)

    p1b1.logger.handlers = []

    return history
def main():

    # Get command-line parameters
    parser = get_p1b1_parser()
    args = parser.parse_args()
    #print('Args:', args)
    # Get parameters from configuration file
    fileParameters = p1b1.read_config_file(args.config_file)
    #print ('Params:', fileParameters)
    # Consolidate parameter set. Command-line parameters overwrite file configuration
    gParameters = p1_common.args_overwrite_config(args, fileParameters)
    print('Params:', gParameters)

    # Construct extension to save model
    ext = p1b1.extension_from_parameters(gParameters, '.pt')
    logfile = args.logfile if args.logfile else args.save + ext + '.log'
    p1b1.logger.info('Params: {}'.format(gParameters))

    # Get default parameters for initialization and optimizer functions
    kerasDefaults = p1_common.keras_default_config()
    seed = gParameters['rng_seed']

    # Load dataset
    X_train, X_val, X_test = p1b1.load_data(gParameters, seed)

    print("Shape X_train: ", X_train.shape)
    print("Shape X_val: ", X_val.shape)
    print("Shape X_test: ", X_test.shape)

    print("Range X_train --> Min: ", np.min(X_train), ", max: ",
          np.max(X_train))
    print("Range X_val --> Min: ", np.min(X_val), ", max: ", np.max(X_val))
    print("Range X_test --> Min: ", np.min(X_test), ", max: ", np.max(X_test))

    # Set input and target to X_train
    train_data = torch.from_numpy(X_train)
    train_tensor = data.TensorDataset(train_data, train_data)
    train_iter = data.DataLoader(train_tensor,
                                 batch_size=gParameters['batch_size'],
                                 shuffle=gParameters['shuffle'])

    # Validation set
    val_data = torch.from_numpy(X_val)
    val_tensor = torch.utils.data.TensorDataset(val_data, val_data)
    val_iter = torch.utils.data.DataLoader(
        val_tensor,
        batch_size=gParameters['batch_size'],
        shuffle=gParameters['shuffle'])

    # Test set
    test_data = torch.from_numpy(X_test)
    test_tensor = torch.utils.data.TensorDataset(test_data, test_data)
    test_iter = torch.utils.data.DataLoader(
        test_tensor,
        batch_size=gParameters['batch_size'],
        shuffle=gParameters['shuffle'])

    #net = mx.sym.Variable('data')
    #out = mx.sym.Variable('softmax_label')
    input_dim = X_train.shape[1]
    output_dim = input_dim

    # Define Autoencoder architecture
    layers = gParameters['dense']
    activation = p1_common_pytorch.build_activation(gParameters['activation'])
    loss_fn = p1_common_pytorch.get_function(gParameters['loss'])
    '''
    N1 = layers[0]
    NE = layers[1]

    net = nn.Sequential(
      nn.Linear(input_dim,N1),
      activation,
      nn.Linear(N1,NE),
      activation,
      nn.Linear(NE,N1),
      activation,
      nn.Linear(N1,output_dim),
      activation,
    )
    '''

    # Documentation indicates this should work
    net = nn.Sequential()

    if layers != None:
        if type(layers) != list:
            layers = list(layers)
        # Encoder Part
        for i, l in enumerate(layers):
            if i == 0:
                net.add_module('in_dense', nn.Linear(input_dim, l))
                net.add_module('in_act', activation)
                insize = l
            else:
                net.add_module('en_dense%d' % i, nn.Linear(insize, l))
                net.add_module('en_act%d' % i, activation)
                insize = l

        # Decoder Part
        for i, l in reversed(list(enumerate(layers))):
            if i < len(layers) - 1:
                net.add_module('de_dense%d' % i, nn.Linear(insize, l))
                net.add_module('de_act%d' % i, activation)
                insize = l

    net.add_module('out_dense', nn.Linear(insize, output_dim))
    net.add_module('out_act', activation)

    # Initialize weights
    for m in net.modules():
        if isinstance(m, nn.Linear):
            p1_common_pytorch.build_initializer(m.weight,
                                                gParameters['initialization'],
                                                kerasDefaults)
            p1_common_pytorch.build_initializer(m.bias, 'constant',
                                                kerasDefaults, 0.0)

    # Display model
    print(net)

    # Define context

    # Define optimizer
    optimizer = p1_common_pytorch.build_optimizer(net,
                                                  gParameters['optimizer'],
                                                  gParameters['learning_rate'],
                                                  kerasDefaults)

    # Seed random generator for training
    torch.manual_seed(seed)

    #use_gpu = torch.cuda.is_available()
    use_gpu = 0

    train_loss = 0

    freq_log = 1
    for epoch in range(gParameters['epochs']):
        for batch, (in_train, _) in enumerate(train_iter):
            in_train = Variable(in_train)
            #print(in_train.data.shape())
            if use_gpu:
                in_train = in_train.cuda()
            optimizer.zero_grad()
            output = net(in_train)

            loss = loss_fn(output, in_train)
            loss.backward()
            train_loss += loss.data[0]
            optimizer.step()
            if batch % freq_log == 0:
                print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                    epoch, batch * len(in_train), len(train_iter.dataset),
                    100. * batch / len(train_iter),
                    loss.data[0]))  # / len(in_train)))
        print('====> Epoch: {} Average loss: {:.4f}'.format(
            epoch, train_loss / len(train_iter.dataset)))

        # model save
        #save_filepath = "model_ae_" + ext
        #ae.save(save_filepath)

        # Evalute model on valdation set
        for i, (in_val, _) in enumerate(val_iter):
            in_val = Variable(in_val)
            X_pred = net(in_val).data.numpy()
            if i == 0:
                in_all = in_val.data.numpy()
                out_all = X_pred
            else:
                in_all = np.append(in_all, in_val.data.numpy(), axis=0)
                out_all = np.append(out_all, X_pred, axis=0)

        #print ("Shape in_all: ", in_all.shape)
        #print ("Shape out_all: ", out_all.shape)

        scores = p1b1.evaluate_autoencoder(in_all, out_all)
        print('Evaluation on validation data:', scores)

    # Evalute model on test set
    for i, (in_test, _) in enumerate(test_iter):
        in_test = Variable(in_test)
        X_pred = net(in_test).data.numpy()
        if i == 0:
            in_all = in_test.data.numpy()
            out_all = X_pred
        else:
            in_all = np.append(in_all, in_test.data.numpy(), axis=0)
            out_all = np.append(out_all, X_pred, axis=0)

    #print ("Shape in_all: ", in_all.shape)
    #print ("Shape out_all: ", out_all.shape)

    scores = p1b1.evaluate_autoencoder(in_all, out_all)
    print('Evaluation on test data:', scores)

    diff = in_all - out_all
    plt.hist(diff.ravel(), bins='auto')
    plt.title("Histogram of Errors with 'auto' bins")
    plt.savefig('histogram_mx.pdf')
Exemple #5
0
def main():
    # Get command-line parameters
    parser = get_p1b1_parser()
    args = parser.parse_args()
    #print('Args:', args)
    # Get parameters from configuration file
    fileParameters = p1b1.read_config_file(args.config_file)
    #print ('Params:', fileParameters)

    # Correct for arguments set by default by neon parser
    # (i.e. instead of taking the neon parser default value fall back to the config file,
    # if effectively the command-line was used, then use the command-line value)
    # This applies to conflictive parameters: batch_size, epochs and rng_seed
    if not any("--batch_size" in ag or "-z" in ag for ag in sys.argv):
        args.batch_size = fileParameters['batch_size']
    if not any("--epochs" in ag or "-e" in ag for ag in sys.argv):
        args.epochs = fileParameters['epochs']
    if not any("--rng_seed" in ag or "-r" in ag for ag in sys.argv):
        args.rng_seed = fileParameters['rng_seed']

    # Consolidate parameter set. Command-line parameters overwrite file configuration
    gParameters = p1_common.args_overwrite_config(args, fileParameters)
    print('Params:', gParameters)

    # Determine verbosity level
    loggingLevel = logging.DEBUG if args.verbose else logging.INFO
    logging.basicConfig(level=loggingLevel, format='')
    # Construct extension to save model
    ext = p1b1.extension_from_parameters(gParameters, '.neon')

    # Get default parameters for initialization and optimizer functions
    kerasDefaults = p1_common.keras_default_config()
    seed = gParameters['rng_seed']

    # Load dataset
    X_train, X_val, X_test = p1b1.load_data(gParameters, seed)

    print("Shape X_train: ", X_train.shape)
    print("Shape X_val: ", X_val.shape)
    print("Shape X_test: ", X_test.shape)

    print("Range X_train --> Min: ", np.min(X_train), ", max: ",
          np.max(X_train))
    print("Range X_val --> Min: ", np.min(X_val), ", max: ", np.max(X_val))
    print("Range X_test --> Min: ", np.min(X_test), ", max: ", np.max(X_test))

    input_dim = X_train.shape[1]
    output_dim = input_dim

    # Re-generate the backend after consolidating parsing and file config
    gen_backend(backend=args.backend,
                rng_seed=seed,
                device_id=args.device_id,
                batch_size=gParameters['batch_size'],
                datatype=gParameters['datatype'],
                max_devices=args.max_devices,
                compat_mode=args.compat_mode)

    # Set input and target to X_train
    train = ArrayIterator(X_train)
    val = ArrayIterator(X_val)
    test = ArrayIterator(X_test)

    # Initialize weights and learning rule
    initializer_weights = p1_common_neon.build_initializer(
        gParameters['initialization'], kerasDefaults)
    initializer_bias = p1_common_neon.build_initializer(
        'constant', kerasDefaults, 0.)

    activation = p1_common_neon.get_function(gParameters['activation'])()

    # Define Autoencoder architecture
    layers = []
    reshape = None

    # Autoencoder
    layers_params = gParameters['dense']

    if layers_params != None:
        if type(layers_params) != list:
            layers_params = list(layers_params)
        # Encoder Part
        for i, l in enumerate(layers_params):
            layers.append(
                Affine(nout=l,
                       init=initializer_weights,
                       bias=initializer_bias,
                       activation=activation))
        # Decoder Part
        for i, l in reversed(list(enumerate(layers_params))):
            if i < len(layers) - 1:
                layers.append(
                    Affine(nout=l,
                           init=initializer_weights,
                           bias=initializer_bias,
                           activation=activation))

    layers.append(
        Affine(nout=output_dim,
               init=initializer_weights,
               bias=initializer_bias,
               activation=activation))

    # Build Autoencoder model
    ae = Model(layers=layers)

    # Define cost and optimizer
    cost = GeneralizedCost(p1_common_neon.get_function(gParameters['loss'])())
    optimizer = p1_common_neon.build_optimizer(gParameters['optimizer'],
                                               gParameters['learning_rate'],
                                               kerasDefaults)

    callbacks = Callbacks(ae, eval_set=val, eval_freq=1)

    # Seed random generator for training
    np.random.seed(seed)

    ae.fit(train,
           optimizer=optimizer,
           num_epochs=gParameters['epochs'],
           cost=cost,
           callbacks=callbacks)

    # model save
    #save_fname = "model_ae_W" + ext
    #ae.save_params(save_fname)

    # Compute errors
    X_pred = ae.get_outputs(test)
    scores = p1b1.evaluate_autoencoder(X_pred, X_test)
    print('Evaluation on test data:', scores)

    diff = X_pred - X_test
    # Plot histogram of errors comparing input and output of autoencoder
    plt.hist(diff.ravel(), bins='auto')
    plt.title("Histogram of Errors with 'auto' bins")
    plt.savefig('histogram_neon.png')
def main():

    # Get command-line parameters
    parser = get_p1b1_parser()
    args = parser.parse_args()
    #print('Args:', args)
    # Get parameters from configuration file
    fileParameters = p1b1.read_config_file(args.config_file)
    #print ('Params:', fileParameters)
    # Consolidate parameter set. Command-line parameters overwrite file configuration
    gParameters = p1_common.args_overwrite_config(args, fileParameters)
    print ('Params:', gParameters)

    # Construct extension to save model
    ext = p1b1.extension_from_parameters(gParameters, '.mx')
    logfile = args.logfile if args.logfile else args.save+ext+'.log'
    p1b1.logger.info('Params: {}'.format(gParameters))

    # Get default parameters for initialization and optimizer functions
    kerasDefaults = p1_common.keras_default_config()
    seed = gParameters['rng_seed']

    # Load dataset
    X_train, X_val, X_test = p1b1.load_data(gParameters, seed)
    
    print ("Shape X_train: ", X_train.shape)
    print ("Shape X_val: ", X_val.shape)
    print ("Shape X_test: ", X_test.shape)

    print ("Range X_train --> Min: ", np.min(X_train), ", max: ", np.max(X_train))
    print ("Range X_val --> Min: ", np.min(X_val), ", max: ", np.max(X_val))
    print ("Range X_test --> Min: ", np.min(X_test), ", max: ", np.max(X_test))


    # Set input and target to X_train
    train_iter = mx.io.NDArrayIter(X_train, X_train, gParameters['batch_size'], shuffle=gParameters['shuffle'])
    val_iter = mx.io.NDArrayIter(X_val, X_val, gParameters['batch_size'])
    test_iter = mx.io.NDArrayIter(X_test, X_test, gParameters['batch_size'])
    
    net = mx.sym.Variable('data')
    out = mx.sym.Variable('softmax_label')
    input_dim = X_train.shape[1]
    output_dim = input_dim

    # Initialize weights and learning rule
    initializer_weights = p1_common_mxnet.build_initializer(gParameters['initialization'], kerasDefaults)
    initializer_bias = p1_common_mxnet.build_initializer('constant', kerasDefaults, 0.)
    init = mx.initializer.Mixed(['bias', '.*'], [initializer_bias, initializer_weights])
    
    activation = gParameters['activation']

    # Define Autoencoder architecture
    layers = gParameters['dense']
    
    if layers != None:
        if type(layers) != list:
            layers = list(layers)
        # Encoder Part
        for i,l in enumerate(layers):
            net = mx.sym.FullyConnected(data=net, num_hidden=l)
            net = mx.sym.Activation(data=net, act_type=activation)
        # Decoder Part
        for i,l in reversed( list(enumerate(layers)) ):
            if i < len(layers)-1:
                net = mx.sym.FullyConnected(data=net, num_hidden=l)
                net = mx.sym.Activation(data=net, act_type=activation)
                    
    net = mx.sym.FullyConnected(data=net, num_hidden=output_dim)
    #net = mx.sym.Activation(data=net, act_type=activation)
    net = mx.symbol.LinearRegressionOutput(data=net, label=out)


    # Display model
    p1_common_mxnet.plot_network(net, 'net'+ext)

    # Define context
    devices = mx.cpu()
    if gParameters['gpus']:
        devices = [mx.gpu(i) for i in gParameters['gpus']]
    

    # Build Autoencoder model
    ae = mx.mod.Module(symbol=net, context=devices)

    # Define optimizer
    optimizer = p1_common_mxnet.build_optimizer(gParameters['optimizer'],
                                                gParameters['learning_rate'],
                                                kerasDefaults)

    # Seed random generator for training
    mx.random.seed(seed)

    freq_log = 1
    ae.fit(train_iter, eval_data=val_iter,
           eval_metric=gParameters['loss'],
           optimizer=optimizer,
           num_epoch=gParameters['epochs'])#,
           #epoch_end_callback = mx.callback.Speedometer(gParameters['batch_size'], freq_log))

    # model save
    #save_filepath = "model_ae_" + ext
    #ae.save(save_filepath)

    # Evalute model on test set
    X_pred = ae.predict(test_iter).asnumpy()
    #print ("Shape X_pred: ", X_pred.shape)
    
    scores = p1b1.evaluate_autoencoder(X_pred, X_test)
    print('Evaluation on test data:', scores)

    diff = X_pred - X_test
    plt.hist(diff.ravel(), bins='auto')
    plt.title("Histogram of Errors with 'auto' bins")
    plt.savefig('histogram_mx.png')
Exemple #7
0
import p1b1


EPOCH = 10
BATCH = 50

P     = 60025    # 245 x 245
N1    = 1000
N2    = 500
N3    = 250
NE    = 100      # encoded dim
DR    = 0.2      # dropout

ACT = 'sigmoid'

X_train, X_test = p1b1.load_data()

input_dim = X_train.shape[1]
output_dim = input_dim

input_vector = Input(shape=(input_dim,))
x = Dense(N1, activation=ACT)(input_vector)
# x = Dropout(DR)(x)
x = Dense(N2, activation=ACT)(x)
x = Dense(N3, activation=ACT)(x)

x = Dense(NE, activation=ACT)(x)
encoded = x

x = Dense(N3, activation=ACT)(encoded)
x = Dense(N2, activation=ACT)(x)