def test_autoencoder_advanced():
encoder = containers.Sequential([core.Dense(5, input_shape=(10,))])
decoder = containers.Sequential([core.Dense(10, input_shape=(5,))])
X_train = np.random.random((100, 10))
X_test = np.random.random((100, 10))
model = Sequential()
model.add(core.Dense(output_dim=10, input_dim=10))
autoencoder = core.AutoEncoder(encoder=encoder, decoder=decoder,
output_reconstruction=True)
model.add(autoencoder)
# training the autoencoder:
model.compile(optimizer='sgd', loss='mse')
assert autoencoder.output_reconstruction
model.fit(X_train, X_train, nb_epoch=1, batch_size=32)
# predicting compressed representations of inputs:
autoencoder.output_reconstruction = False # the autoencoder has to be recompiled after modifying this property
assert not autoencoder.output_reconstruction
model.compile(optimizer='sgd', loss='mse')
representations = model.predict(X_test)
assert representations.shape == (100, 5)
# the model is still trainable, although it now expects compressed representations as targets:
model.fit(X_test, representations, nb_epoch=1, batch_size=32)
# to keep training against the original inputs, just switch back output_reconstruction to True:
autoencoder.output_reconstruction = True
model.compile(optimizer='sgd', loss='mse')
model.fit(X_train, X_train, nb_epoch=1)
reconstructions = model.predict(X_test)
assert reconstructions.shape == (100, 10)
def model(labels, data, go_id):
batch_size = 64
nb_classes = 2
nb_epoch = 1
train, val, test = train_val_test_split(labels,
data,
batch_size=batch_size)
train_label, train_data = train
val_label, val_data = val
test_label, test_data = test
test_label_rep = test_label
train_data = train_data.reshape(-1, 500 * 20)
test_data = test_data.reshape(-1, 500 * 20)
val_data = val_data.reshape(-1, 500 * 20)
# convert class vectors to binary class matrices
enc_wt = []
#creating the autoencoder
ae = Sequential()
encoder = containers.Sequential([Dense(5000, input_dim=10000), Dense(100)])
decoder = containers.Sequential([Dense(5000, input_dim=100), Dense(10000)])
ae.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=True))
ae.compile(loss='mean_squared_error', optimizer='rmsprop')
ae.fit(train_data,
train_data,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=False,
verbose=1,
validation_data=[val_data, val_data])
model = Sequential()
model.add(encoder)
model.add(Dense(100, nb_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
score = model.evaluate(val_data, val_label, show_accuracy=True, verbose=0)
print('Test score before fine turning:', score[0])
print('Test accuracy after fine turning:', score[1])
model.fit(train_data,
train_label,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=True,
validation_data=(val_data, val_label))
score = model.evaluate(val_data, val_label, show_accuracy=True, verbose=0)
print('Test score after fine turning:', score[0])
print('Test accuracy after fine turning:', score[1])
def _pre_train(self, X):
logger.info(u"Pre-training the network")
encoders = []
layers = self._hidden_layers[:] # Copy the hidden layers list
layers.insert(0, self.input_dim)
for i, (n_in, n_out) in enumerate(zip(layers[:-1], layers[1:]),
start=1):
logger.info(u"Training layer {}: Input {} -> Output {}".format(
i, n_in, n_out))
autoencoder = models.Sequential()
encoder = containers.Sequential()
encoder.add(core.Dropout(self._dropout_ratio,
input_shape=(n_in, )))
encoder.add(
core.Dense(input_dim=n_in,
output_dim=n_out,
activation=self._activation,
init=self._weight_init))
decoder = containers.Sequential()
decoder.add(
core.Dense(input_dim=n_out,
output_dim=n_in,
activation=self._activation,
init=self._weight_init))
autoencoder.add(
core.AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=False))
logger.info(u"Compiling the autoencoder")
autoencoder.compile(optimizer=self._optimizer,
loss='mean_squared_error')
logger.info(u"Fitting the data")
autoencoder.fit(X,
X,
batch_size=self._batch_size,
nb_epoch=self._pre_train_epochs)
# Store trained weight and update training data
encoders.append(autoencoder.layers[0].encoder)
X = autoencoder.predict(X)
pass
return encoders
def unroll_deep_ae(autoencoder, params, tie_weights=True):
'''
Takes an autoencoder list generated by `pretrain_deep_ae` and
unrolls it to make a deep autoencoder. NOTE this doesn't
compile anything! This is simply a wrapper around the
unrolling process to make it easier.
Args:
autoencoder (list): a list of keras layers.
params (dict): the param dict returned by `pretrain_deep_ae`
tie_weights (bool): whether or not to make the weights tied.
Usage:
>>> params = {
....'structure' : [625, 512, 128, 64],
....'activations' : 3 * [('sigmoid', 'relu')],
....'noise' : [GaussianNoise(0.01), None, None],
....'optimizer' : Adam(),
....'loss' : ['mse', 'mse', 'mse']
....}
>>> model = unroll_deep_ae(*pretrain_deep_ae(params, X))
Returns:
keras.Sequential: a keras sequential model with one layer
which is the unrolled autoencoder.
'''
encoder = []
decoder = []
structure = params['structure']
for (layer_nb, (inputs, hidden)), (enc_act, dec_act) in zip(
enumerate(zip(structure, structure[1:])), params['activations']):
logger.info('Unpacking structure from level {}.'.format(layer_nb))
encoder.append(Dense(inputs, hidden, activation=enc_act))
encoder[-1].set_weights(autoencoder[layer_nb].get_weights()[:2])
decoder.insert(0, Dense(hidden, inputs, activation=dec_act))
decoder[0].set_weights(autoencoder[layer_nb].get_weights()[2:])
encoder_sequence = containers.Sequential(encoder)
decoder_sequence = containers.Sequential(decoder)
stacked_autoencoder = Sequential()
stacked_autoencoder.add(
AutoEncoder(encoder=encoder_sequence,
decoder=decoder_sequence,
output_reconstruction=False,
tie_weights=tie_weights))
return stacked_autoencoder
def build_deep_classical_autoencoder(autoencoder):
encoder = containers.Sequential([
Dense(input_dim, hidden_dim, activation=activation),
Dense(hidden_dim, hidden_dim / 2, activation=activation)
])
decoder = containers.Sequential([
Dense(hidden_dim / 2, hidden_dim, activation=activation),
Dense(hidden_dim, input_dim, activation=activation)
])
autoencoder.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=False))
return autoencoder
def convLayer(self, inp_dim, embedding_dim, nb_filter, filter_length):
c = containers.Graph()
c.add_input(name='input', input_shape=(inp_dim, embedding_dim))
inps = []
for i in filter_length:
c.add_node(containers.Sequential([
Convolution1D(
nb_filter=nb_filter,
filter_length=i,
border_mode='valid',
activation='relu',
subsample_length=1,
input_shape=(inp_dim, embedding_dim),
),
MaxPooling1D(pool_length=2),
Flatten()
]),
name='Conv{}'.format(i),
input='input')
inps.append('Conv{}'.format(i))
if len(inps) == 1:
c.add_output('output', input=inps[0])
else:
c.add_output('output', inputs=inps)
return c
def build_deep_denoising_autoencoder(autoencoder):
encoder = containers.Sequential([
Dense(input_dim, hidden_dim, activation=activation),
Dense(hidden_dim, hidden_dim / 2, activation=activation)
])
decoder = containers.Sequential([
Dense(hidden_dim / 2, hidden_dim, activation=activation),
Dense(hidden_dim, input_dim, activation=activation)
])
autoencoder.add(Dense(input_dim, input_dim))
autoencoder.add(
DenoisingAutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=False,
tie_weights=True))
return autoencoder
def new_encdecs(self, compile=True, use_dropout=None, use_noise=None):
self.enc_decs = []
if not use_dropout is None:
self.enc_use_drop = self.drop_rate > 0 and use_dropout
if not use_noise is None:
self.enc_use_noise = self.sigma_base > 0 and use_noise
if self.l1 != 0 or self.l2 != 0:
regularizer = WeightRegularizer(l1=self.l1, l2=self.l2)
else:
regularizer = None
for (i, (n_in, n_out)) in enumerate(
zip(self.layer_sizes[:-1], self.layer_sizes[1:])):
ae = Sequential()
enc_l = []
if self.enc_use_noise:
enc_l.append(
noise.GaussianNoise(self.sigma_base *
(self.sigma_fact**-i),
input_shape=(n_in, )))
enc_l.append(
core.Dense(input_dim=n_in,
output_dim=n_out,
activation='sigmoid',
W_regularizer=regularizer))
if self.enc_use_drop:
enc_l.append(core.Dropout(self.drop_rate))
enc = containers.Sequential(enc_l)
dec = containers.Sequential([
core.Dense(input_dim=n_out,
output_dim=n_in,
activation='sigmoid')
])
ae.add(
core.AutoEncoder(encoder=enc,
decoder=dec,
output_reconstruction=True))
if compile:
ae.compile(loss='mse', optimizer=self.enc_opt)
self.enc_decs.append(ae)
def __call__(self,
vocabulary_size=5000,
maxlen=100,
embedding_dim=300,
nb_filter=300,
filter_length=[3],
layer=-1,
hidden_dim=250,
nb_class=2,
drop_out_prob=0.5,
use_my_embedding=False,
embedding_weights=None):
self.log_params(locals())
model = Sequential()
maxlen = self.add_embedding(model, vocabulary_size, embedding_dim,
maxlen, use_my_embedding,
embedding_weights)
model.add(Reshape((1, maxlen, embedding_dim)))
c = containers.Graph()
c.add_input(name='input', input_shape=(1, maxlen, embedding_dim))
inps = []
for filter_h in filter_length:
pool_size = (maxlen - filter_h + 1, 1)
c.add_node(containers.Sequential([
Convolution2D(nb_filter=nb_filter,
nb_row=filter_h,
nb_col=embedding_dim,
border_mode='valid',
activation='relu',
init='uniform',
input_shape=(1, maxlen, embedding_dim)),
MaxPooling2D(pool_size=pool_size),
Flatten()
]),
name='Conv{}'.format(filter_h),
input='input')
inps.append('Conv{}'.format(filter_h))
if len(inps) == 1:
c.add_output('output', input=inps[0])
else:
c.add_output('output', inputs=inps)
model.add(c)
self.add_full(model, hidden_dim, drop_out_prob, nb_class)
return model
blend_test_j[:, i] = clf.predict_proba(X_test)[:,1]
blend_test[:, j] = blend_test_j.mean(1)
print 'Y_dev.shape = %s' % (Y_dev.shape)
def multiple_layer_autoencoder(X_train, X_test, activation = 'linear', batch_size = 100, nb_epoch = 20, last_dim = 64):
nb_hidden_layers = [X_train.shape[1], 256, 128, last_dim]
X_train_tmp = np.copy(X_train)
#X_test_tmp = np.copy(X_test)
encoders = []
for i, (n_in, n_out) in enumerate(zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]), start=1):
print('Training the layer {}: Input {} -> Output {}'.format(i, n_in, n_out))
# Create AE and training
ae = Sequential()
encoder = containers.Sequential([Dense(n_in, n_out, activation=activation, , W_regularizer=regularizers.l2(0.0001), b_regularizer= regularizers.l1(0.0001))])
decoder = containers.Sequential([Dense(n_out, n_in, activation=activation)])
ae.add(AutoEncoder(encoder=encoder, decoder=decoder,
output_reconstruction=False))
ae.add(Dropout(0.6))
#sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
ae.compile(loss='mean_squared_error', optimizer='adam')#'rmsprop')
ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=False, verbose=0, callbacks = [EarlyStopping(monitor='val_acc', patience=2)])
encoders.append(ae)
X_train_tmp = ae.predict(X_train_tmp)
print X_train_tmp.shape
return encoders
def autoencoder_two_subnetwork_fine_tuning(X_train1, X_train2, Y_train, X_test1, X_test2, Y_test = None, batch_size =100, nb_epoch = 20):
print 'autoencode learning'
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
# Layer-wise pretraining
encoders = []
nb_hidden_layers = [784, 600, 500, 400]
X_train_tmp = np.copy(X_train)
for i, (n_in, n_out) in enumerate(zip(nb_hidden_layers[:-1],
nb_hidden_layers[1:]),
start=1):
print('Training the layer {}: Input {} -> Output {}'.format(
i, n_in, n_out))
# Create AE and training
ae = Sequential()
encoder = containers.Sequential([Dense(n_in, n_out, activation='sigmoid')])
decoder = containers.Sequential([Dense(n_out, n_in, activation='sigmoid')])
ae.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=False,
tie_weights=True))
ae.compile(loss='mean_squared_error', optimizer='rmsprop')
ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size, nb_epoch=nb_epoch)
# Store trainined weight and update training data
encoders.append(ae.layers[0].encoder)
X_train_tmp = ae.predict(X_train_tmp)
# Fine-turning
model = Sequential()
for encoder in encoders:
#X_train = np.random.random((1000, data_dim))
#X_test = np.random.random((1000, data_dim))
# X_train = np.array([[1,0,0,0], [0,1,0,0],[0,0,1,0], [0,0,0,1]])
# X_test = np.array([[1,0,0,0], [0,1,0,0], [0,0,1,0], [0,0,0,1]])
X_train = np.array([[1, 0, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0],
[0, 0, 1, 0, 0, 0], [0, 0, 0, 1, 0, 0],
[0, 0, 0, 0, 1, 0], [0, 0, 0, 0, 0, 1]])
X_test = np.array([[1, 0, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0],
[0, 0, 1, 0, 0, 0], [0, 0, 0, 1, 0, 0],
[0, 0, 0, 0, 1, 0], [0, 0, 0, 0, 0, 1]])
# X_train = X_train.astype("float32")
# X_test = X_test.astype("float32")
# input shape: (nb_samples, 32)
encoder = containers.Sequential(
[Dense(4, input_dim=6),
Dense(2, input_dim=4, activation='sigmoid')])
decoder = containers.Sequential(
[Dense(4, input_dim=2),
Dense(6, input_dim=4, activation='sigmoid')])
#output_reconstruction参数为True,则dim(input) = dim(output)
autoencoder = AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=True)
# print autoencoder.get_output()1
model = models.Sequential()
model.add(autoencoder)
# 训练autoencoder
import numpy as np
import pandas as pd
from keras.layers import containers, AutoEncoder, Dense
from keras import models
from keras.datasets import mnist
# input shape: (nb_samples, 32)
encoder = containers.Sequential([Dense(300, input_dim=784), Dense(10)])
decoder = containers.Sequential([Dense(300, input_dim=10), Dense(784)])
# the data, shuffled and split between train and test sets
(X_train, _), (X_test, _) = mnist.load_data()
X_train = X_train.reshape(-1, 784)
X_test = X_test.reshape(-1, 784)
X_train = X_train.astype("float32") / 255.0
X_test = X_test.astype("float32") / 255.0
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')
autoencoder = AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=True)
model = models.Sequential()
model.add(autoencoder)
# training the autoencoder:
model.compile(optimizer='sgd', loss='mse')
model.fit(X_train, X_train, nb_epoch=10)
X_predicted = model.predict(X_train)
M = 6040
N = 3952
if __name__ == "__main__":
(trainI, testI) = datasets.read_instances(sys.argv[1], sys.argv[2])
X = np.zeros((M, N))
testX = np.zeros((M, N))
trainS = [set() for i in range(M)]
for ins in trainI:
X[ins[0], ins[1]] = ins[2]
trainS[ins[0]].add(ins[1])
for ins in testI:
testX[ins[0], ins[1]] = ins[2]
#X = np.array([[5, 3, 0, 1], [4, 0, 0, 1], [1, 1, 0, 5], [1, 0, 0, 4],[0, 1, 5, 4]] )
encoder = containers.Sequential([
Dense(3952, 300, activation='sigmoid'),
Dense(300, 100, activation='sigmoid')
])
decoder = containers.Sequential([
Dense(100, 300, activation='sigmoid'),
Dense(300, 3952, activation='sigmoid')
])
#encoder = containers.Sequential([Dense(3952, 100)])
#decoder = containers.Sequential([Dense(100, 3952)])
model = Sequential()
model.add(AutoEncoder(encoder=encoder, decoder=decoder))
sgd = SGD(lr=0.2)
model.compile(loss='binary_crossentropy', optimizer=sgd)
#model.load_weights("/tmp/DA_weights.mod.hdf5")
"""
score = -1
for k in range(60):
#the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train = [[0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 0, 1]]
X_test = X_test.reshape(10000, 784)
X_train = X_train.astype("float64")
X_test = X_test.astype("float64")
X_train /= 255
X_test /= 255
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')
#convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
#creating the autoencoder
#first autoencoder
AE1_output_reconstruction = True
ae1 = Sequential()
encoder1 = containers.Sequential([Dense(4, 2, activation='tanh')])
decoder1 = containers.Sequential([Dense(2, 4, activation='tanh')])
ae1.add(AutoEncoder(encoder=encoder1, decoder=decoder1,
output_reconstruction=AE1_output_reconstruction, tie_weights=True))
#training the first autoencoder
ae1.compile(loss='mean_squared_error', optimizer=RMSprop())
ae1.fit(X_train, X_train, batch_size=batch_size, nb_epoch=nb_epoch,
show_accuracy=False, verbose=1, validation_data=[X_test, X_test])
def stackedAutoencoder(parameters):
# Stacked Autoencoder
# Train the autoencoder
# Source: https://github.com/fchollet/keras/issues/358
random.seed(3)
np.random.seed(3)
nb_epoch_pretraining = 10
batch_size_pretraining = 500
# Layer-wise pretraining
encoders = []
decoders = []
nb_hidden_layers = [parameters['X_train'].shape[1], parameters['neuron'][0], parameters['neuron'][1]]
X_train_tmp = np.copy(parameters['X_train'])
print('original X_train_tmp SIZE:',X_train_tmp.shape)
dense_layers = []
for i, (n_in, n_out) in enumerate(zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]), start=1):
print('Training the layer {}: Input {} -> Output {}'.format(i, n_in, n_out))
# Create AE and training
ae = Sequential()
if n_out >= 100:
encoder = containers.Sequential([Dense(output_dim=n_out, input_dim=n_in, activation='tanh'), Dropout(0.5)])
else:
encoder = containers.Sequential([Dense(output_dim=n_out, input_dim=n_in, activation='tanh')])
decoder = containers.Sequential([Dense(output_dim=n_in, input_dim=n_out, activation='tanh')])
ae.add(AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False))
sgd = SGD(lr=2, decay=1e-6, momentum=0.0, nesterov=True)
ae.compile(loss='mse', optimizer=parameters['optimizer'])
ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size_pretraining, nb_epoch=parameters['epoch_pretraining'], verbose = True, shuffle=True)
# Store trainined weight and update training data
encoders.append(ae.layers[0].encoder)
decoders.append(ae.layers[0].decoder)
X_train_tmp = ae.predict(X_train_tmp)
print('X_train_tmp SIZE:',X_train_tmp.shape)
##############
#End to End Autoencoder training
if len(nb_hidden_layers) > 2:
full_encoder = containers.Sequential()
for encoder in encoders:
full_encoder.add(encoder)
full_decoder = containers.Sequential()
for decoder in reversed(decoders):
full_decoder.add(decoder)
full_ae = Sequential()
full_ae.add(AutoEncoder(encoder=full_encoder, decoder=full_decoder, output_reconstruction=False))
full_ae.compile(loss='mse', optimizer=parameters['optimizer'])
print "Pretraining of full AE"
full_ae.fit(parameters['X_train'], parameters['X_train'], batch_size=batch_size_pretraining, nb_epoch=parameters['epoch_pretraining'], verbose = True, shuffle=True)
#######################################
nb_epoch = parameters['epoch']
batch_size = 100
model = Sequential()
for encoder in encoders:
model.add(encoder)
model.add(Dense(output_dim=nb_labels, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=parameters['optimizer'])
score = model.evaluate(parameters['X_test'], parameters['y_test'], show_accuracy=True, verbose=0)
print('Test score before fine turning:', score[0])
print('Test accuracy before fine turning:', score[1])
model.fit(parameters['X_train'], parameters['y_train'], batch_size=batch_size, nb_epoch=parameters['epoch'],
show_accuracy=True, validation_data=(parameters['X_test'], parameters['y_test']), shuffle=True)
score = model.evaluate(parameters['X_test'], parameters['y_test'], show_accuracy=True, verbose=0)
print('Test score after fine turning:', score[0])
print('Test accuracy after fine turning:', score[1])
TestScore = score[0]
TestAccuracy = score[1]
return TestScore, TestAccuracy
'optimizer' : Adam(),
'loss' : 2 * ['binary_crossentropy'] + 1 * ['binary_crossentropy']
}
ae, config = pretrain_deep_ae(params, X, nb_epoch=30, batch_size=512)
model = unroll_deep_ae(ae, config)
model.compile(loss='binary_crossentropy', optimizer=Adam())
im = Sequential()
im.add(Embedding(1, 625, W_constraint=NonNeg()))
im.add(containers.Sequential(model.layers[0].encoder.layers[:-1]))
w = model.layers[0].encoder.layers[-1].get_weights()
im.add(Dense(64, 1, weights=[w[0][:, 0].reshape(64, 1), np.array(w[1][0]).reshape((1, ))]))
im.add(Activation('sigmoid'))
im.compile(loss='mse', optimizer=Adam())
weights = model.layers[0].encoder.layers[0].get_weights()
# model.fit(X, X, batch_size=512)
def generate_dev_batches(number):
count = 0
while count < X_dev.shape[0]:
yield X_dev[count:count + number], Y_dev[count:count + number]
count += number
print("Constructing model...", end=" ")
# embedding layer
encoder = containers.Sequential([
Embedding(input_dim=vocabulary_size,
output_dim=n_d,
input_length=batch_size,
init="glorot_normal"),
Dropout(0.5),
LSTM(output_dim=n_d,
input_shape=(batch_size, n_d),
activation="tanh",
return_sequences=True)
])
# rnn layer and output layers
decoder = containers.Sequential([
Dropout(0.5, input_shape=(batch_size, n_d)),
TimeDistributedDense(output_dim=vocabulary_size,
input_dim=n_d,
input_length=batch_size),
Activation("softmax")
])
model.add_input(name='input', input_shape=(maxlen, ), dtype='int')
# Start off with an efficient embedding layer which maps
# our vocab indices into embedding_dims dimensions
embedding_layer = Embedding(max_features,
embedding_dims,
input_length=maxlen,
weights=[w2v_word_vec],
trainable=True)
model.add_node(embedding_layer, name='embedding', input='input')
model.add_node(Dropout(0.5), name='embedding_dropout', input='embedding')
# Add several Convolutional1D, learning nb_filter word group filters
# with different filter_lengths
for filter_length in filter_lengths:
sequential = containers.Sequential()
sequential.add(
Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode='valid',
activation='relu',
subsample_length=1,
input_shape=(maxlen, embedding_dims)))
sequential.add(MaxPooling1D(pool_length=pool_length))
sequential.add(Flatten())
model.add_node(sequential,
name='unit_' + str(filter_length),
input='embedding_dropout')
fc = containers.Sequential()
# classical_score = model_classical.evaluate(X_test, Y_test, verbose=0, show_accuracy=True)
# print('classical_score:', classical_score)
#
# y_true, y_pred = y_test, model_classical.predict_classes(X_test)
# print(classification_report(y_true, y_pred))
##########################
# autoencoder model test #
##########################
# plotExamples(X_train, number = 25, title="Original_")
# print("Training AutoEncoder for feature viz")
# AutoEncoder for feature visualization
encoder = containers.Sequential([
Dense(input_dim, hidden_dim, activation=activation),
Dense(hidden_dim, final_dim, activation=activation)
])
decoder = containers.Sequential([
Dense(final_dim, hidden_dim, activation=activation),
Dense(hidden_dim, input_dim, activation=activation)
])
# autoencoder.add(AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=True))
# autoencoder.compile(loss='mse', optimizer='adadelta')
# # Do NOT use validation data with return output_reconstruction=True
# autoencoder.fit(X_train, X_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=False, verbose=0)
# # plotExamples(autoencoder.predict(X_train), number = 25, title="Reproduction_")
# vals = getActivationsLayer0(autoencoder.layers[0].encoder.layers)[0].T
# # plotExamples(vals, number = hidden_dim, title="Neuron_features_")
def train():
model = Sequential()
X_train = np.load(home + '/gabor/numpyFiles/Training Set.npy')
X_test = np.load(home + '/gabor/numpyFiles/TestSet.npy')
Y_train = np.load(home + '/gabor/numpyFiles/Training Labels.npy')
Y_test = np.load(home + '/gabor/numpyFiles/TestSet Labels.npy')
#X_test = X_test.reshape(-1, 1, 30, 96)
Y_test = np_utils.to_categorical(Y_test, 447)
#X_train = X_train.reshape(-1, 1, 30, 96)
Y_train = np_utils.to_categorical(Y_train, 447)
print("X_test.shape == {};".format(X_test.shape))
print("Y_test.shape == {};".format(Y_test.shape))
print("X_test.shape == {};".format(X_train.shape))
print("Y_test.shape == {};".format(Y_train.shape))
nb_hidden_layers = [len(X_train[0]), 700, 500, 300]
XtrainKeras = X_train
print 'shape of XTrain Keras is ', XtrainKeras.shape
YtrainKeras = np_utils.to_categorical(Y_train, nb_classes)
op1 = RMSprop(lr=0.01, rho=0.5, epsilon=1e-8)
X_train_tmp = XtrainKeras
trained_encoders = []
#XtrainKeras=XwhiteTrain.reshape(-1,1,len(XwhiteTrain),len(XwhiteTrain[0]))
#YtrainKeras=np_utils.to_categorical(Y_train, nb_classes)
#XtestKeras=X_test.reshape(-1,1,imageWidth,imageHeight)
#YtestKeras=np_utils.to_categorical(Y_test, nb_classes)
#X_train_tmp=XtrainKeras
for n_in, n_out in zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]):
print('Pre-training the layer: Input {} -> Output {}'.format(
n_in, n_out))
# Create AE and training
ae = Sequential()
encoder = containers.Sequential(
[Dense(n_out, input_dim=n_in, activation='sigmoid')])
decoder = containers.Sequential(
[Dense(n_in, input_dim=n_out, activation='sigmoid')])
ae.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=False))
ae.compile(loss='mean_squared_error', optimizer=op1)
hist = ae.fit(X_train_tmp,
X_train_tmp,
batch_size=batch_size,
nb_epoch=nb_epoch)
print(hist.history)
Fname = prefix + 'autoencoder_n_in=' + str(n_in) + '_n_out= ' + str(
n_out) + '.json'
weightName = prefix + 'Weights_autoencoder_n_in=' + str(
n_in) + '_n_out= ' + str(n_out) + '.h5'
json_string = model.to_json()
open(Fname, 'w').write(json_string)
model.save_weights(weightName, overwrite=True)
# Store trainined weight
trained_encoders.append(ae.layers[0].encoder)
# Update training data
X_train_tmp = ae.predict(X_train_tmp)
#ae1=Sequential()
#encoder1=containers.Sequential([Dense(len(XwhiteTrain[0])-200,len(XwhiteTrain[0]),activation='sigmoid')])
Y_test = np_utils.to_categorical(Y_test, nb_classes)
#X_test=X_test.reshape(-1,len(X_test[0]))
print 'shape of X_test is ', X_test.shape
print('Fine-tuning')
sgd = SGD(lr=0.01, momentum=0.5, decay=0., nesterov=False)
i = 1
model = Sequential()
for encoder in trained_encoders:
model.add(encoder)
model.add(
Dense(nb_classes, input_dim=nb_hidden_layers[-1],
activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=sgd)
hist = model.fit(XtrainKeras,
YtrainKeras,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=True,
validation_data=(X_test, Y_test))
score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])
res = model.predict_classes(X_test)
print 'res is ', res
Fname = prefix + '2 FineTuning_model=' + '.json'
weightName = prefix + 'Fine Tunes Weights_autoencoder_i=' + str(i) + '.h5'
json_string = model.to_json()
open(Fname, 'w').write(json_string)
model.save_weights(weightName, overwrite=True)
#print DataTr[['MeanSalesNotPromo','MeanSalesPromo','MeanHolidaySales0','MeanHolidaySales1','MeanHolidaySales2','MeanHolidaySales3']]
print('Number of input neurons...', len(columns))
print('Getting data ...')
X, Y = get_training_dataset_simple(DataTr, columns)
Xtest = get_test_dataset_simple(data_test, columns)
in_neurons = len(columns)
hidden_neurons = 300
hidden_neurons_2 = 250
hidden_neurons_3 = 75
out_neurons = 1
nb_epoch = 13
evaluation = []
# lets create autoencoder
encoder = containers.Sequential(
[Dense(in_neurons, hidden_neurons, activation='tanh')])
decoder = containers.Sequential(
[Dense(hidden_neurons, in_neurons, activation='tanh')])
ae = Sequential()
ae.add(
AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False))
ae.compile(loss='mean_squared_error', optimizer='rmsprop')
ae.fit(X, X, verbose=1, nb_epoch=4)
ae.fit(Xtest, Xtest, verbose=1, nb_epoch=2)
# lets create second autoencoder
X2 = ae.predict(X)
X2test = ae.predict(Xtest)
encoder2 = containers.Sequential(
batch_size = 64
nb_classes = 10
nb_epoch = 1
# the data, shuffled and split between train and test sets
(X_train, _), (X_test, _) = mnist.load_data()
X_train = X_train.reshape(-1, 784)
X_test = X_test.reshape(-1, 784)
X_train = X_train.astype("float32") / 255.0
X_test = X_test.astype("float32") / 255.0
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')
#creating the autoencoder
ae = Sequential()
encoder = containers.Sequential([Dense(784, 700), Dense(700, 600)])
decoder = containers.Sequential([Dense(600, 700), Dense(700, 784)])
ae.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=True,
tie_weights=True))
ae.compile(loss='mean_squared_error', optimizer=RMSprop())
ae.fit(X_train,
X_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=False,
verbose=1,
validation_data=[X_test, X_test])
# plt.ylim()
# plt.xlim(0, 1)
plt.colorbar()
plt.title(r"QCD Jet mass vs. $\hat{y}$, " + '\n' + r'Jet $p_T\in[200, 1000]$ $\mathrm{GeV},\vert\eta\vert<2$, $m_{\mathrm{jet}}\in (65, 110)$ GeV')
plt.savefig(PLOT_DIR % 'dl-output-mass-likelihood-bkg.pdf')
plt.show()
# -- hidden activations
hidden_act = Sequential()
hidden_act.add(
containers.Sequential(dl.layers[:-3])
)
hidden_act.compile('sgd', 'mse')
R = hidden_act.predict(X_, verbose=True)
def normalize_rows(x):
def norm1d(a):
return a / a.sum()
x = np.array([norm1d(r) for r in x])
return x
H, b_x, b_y = np.histogram2d(
import numpy as np
import pandas as pd
from keras.layers import containers, AutoEncoder, Dense
from keras import models
from sklearn import preprocessing
# input shape: (nb_samples, 32)
encoder = containers.Sequential([Dense(4, input_dim=4), Dense(2)])
decoder = containers.Sequential([Dense(4, input_dim=2), Dense(4)])
df = pd.read_csv("8.csv")
X_train = df.values.copy()
#loss in 0.23
#X_train = preprocessing.scale(X_train.astype("float32"))
#loss is 0.02
X_train = preprocessing.normalize(X_train.astype("float32"), norm='l2')
autoencoder = AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=True)
model = models.Sequential()
model.add(autoencoder)
# training the autoencoder:
model.compile(optimizer='sgd', loss='mse')
model.fit(X_train, X_train, nb_epoch=20)
X_predicted = model.predict(X_train)
mx = magntData.shape[0]
my = magntData.shape[1]
mz = magntData.shape[2]
print phaseData.shape
print magntData.shape
kMagnData = np.copy(magntData)
for encInd in range(0, 5):
encMag = Sequential()
midlay = encInd * 200
outlay = (encInd + 1) * 200
encoder = containers.Sequential([
TimeDistributedDense(input_dim=pz - midlay,
output_dim=pz - outlay,
activation='tanh')
])
decoder = containers.Sequential([
TimeDistributedDense(input_dim=pz - outlay,
output_dim=pz - midlay,
activation='linear')
])
encMag.add(Dropout(p=0.1, input_shape=(py, pz - midlay)))
encMag.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=False))
#encMag.load_weights('magnitude-layer-' + str(encInd))
encMag.compile(loss='mean_squared_error', optimizer='rmsprop')
startLoss = 999
print(len(X_test), 'test sequences')
nb_classes = np.max(y_train) + 1
print(nb_classes, 'classes')
print("Vectorizing sequence data...")
tokenizer = Tokenizer(nb_words=max_words)
X_train = tokenizer.sequences_to_matrix(X_train, mode="binary")
X_test = tokenizer.sequences_to_matrix(X_test, mode="binary")
xw = X_train.transpose()
userfea = 1000
itemfea = 8982
print("Building model...")
encoder = containers.Sequential([Dense(1000, 700), Dense(700, 500)])
decoder = containers.Sequential([Dense(500, 700), Dense(700, 1000)])
model = Sequential()
model.add(
AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False))
model.compile(loss='mean_squared_error', optimizer='adam')
model.fit(X_train,
X_train,
nb_epoch=15,
batch_size=1024,
verbose=1,
show_accuracy=True,
validation_split=0.1)
X_pca = pca(X_train)
plotScatter(X_pca,
y_train,
title="2_PCA reduction (2d) of raw data (%dd)" % X_train.shape[1])
print("Performing TSNE")
model = TSNE(n_components=2, random_state=0, init="pca")
toPlot = model.fit_transform(X_train[:1000])
plotTSNE(toPlot, y_train[:1000], nb_classes, "3_t-SNE embedding of raw data ")
print("Training AutoEncoder for feature viz")
# AutoEncoder for feature visualization
autoencoder = Sequential()
encoder = containers.Sequential(
[Dense(input_dim, hidden_dim),
Dropout(0.3),
Activation(activation)])
decoder = containers.Sequential(
[Dense(hidden_dim, input_dim, activation=activation)])
autoencoder.add(
AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=True))
autoencoder.compile(loss='mean_squared_error', optimizer='adam')
# Do NOT use validation data with return output_reconstruction=True
autoencoder.fit(X_train,
X_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=False,
verbose=0)
plotExamples(autoencoder.predict(X_train), number=25, title="4_Reproduction_")
def train():
model = Sequential()
print 'X.shape is ', XwhiteTrain.shape
print 'Y.shape is ', Y_train.shape
nb_hidden_layers = [len(XwhiteTrain[0]), 700, 500, 300]
XtrainKeras = XwhiteTrain.reshape(-1, len(XwhiteTrain[0]))
pretraining_files = [
'Layer_1_PCAWhitened.txt', 'Layer_1_PCAWhitened.txt',
'Layer_3_PCAWhitened.txt'
]
print 'shape of XTrain Keras is ', XtrainKeras.shape
YtrainKeras = np_utils.to_categorical(Y_train, nb_classes)
op1 = RMSprop(lr=0.01, rho=0.8, epsilon=1e-3)
#datagen=ImageDataGenerator(zca_whitening=True)
#datagen.fit()
X_train_tmp = XtrainKeras
trained_encoders = []
#XtrainKeras=XwhiteTrain.reshape(-1,1,len(XwhiteTrain),len(XwhiteTrain[0]))
#YtrainKeras=np_utils.to_categorical(Y_train, nb_classes)
#XtestKeras=X_test.reshape(-1,1,imageWidth,imageHeight)
#YtestKeras=np_utils.to_categorical(Y_test, nb_classes)
#X_train_tmp=XtrainKeras
i = 0
for n_in, n_out in zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]):
print('Pre-training the layer: Input {} -> Output {}'.format(
n_in, n_out))
# Create AE and training
ae = Sequential()
encoder = containers.Sequential(
[Dense(n_out, input_dim=n_in, activation='sigmoid')])
decoder = containers.Sequential(
[Dense(n_in, input_dim=n_out, activation='sigmoid')])
ae.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=False))
ae.compile(loss='mean_squared_error', optimizer=op1)
hist = ae.fit(X_train_tmp,
X_train_tmp,
batch_size=batch_size,
nb_epoch=nb_epoch)
#f.write(str(hist.history))
Fname = prefix + 'autoencoder_n_in=' + str(n_in) + '_n_out= ' + str(
n_out) + '.json'
weightName = prefix + 'Weights_autoencoder_n_in=' + str(
n_in) + '_n_out= ' + str(n_out) + '.h5'
json_string = model.to_json()
f2 = open(pretraining_files[i], 'wb')
f2.write(json.dumps(hist.history))
f2.close()
i = i + 1
open(Fname, 'w').write(json_string)
model.save_weights(weightName, overwrite=True)
# Store trainined weight
trained_encoders.append(ae.layers[0].encoder)
# Update training data
X_train_tmp = ae.predict(X_train_tmp)
f.close()
#ae1=Sequential()
#encoder1=containers.Sequential([Dense(len(XwhiteTrain[0])-200,len(XwhiteTrain[0]),activation='sigmoid')])
X_test = np.load(Xtest_file)
Y_test = np.load(Ytest_file)
Y_test = np_utils.to_categorical(Y_test, nb_classes)
X_test = X_test.reshape(-1, len(X_test[0]))
print 'shape of X_test is ', X_test.shape
print('Fine-tuning')
sgd = SGD(lr=0.01, momentum=0.5, decay=0., nesterov=False)
i = 1
model = Sequential()
for encoder in trained_encoders:
model.add(encoder)
model.add(
Dense(nb_classes, input_dim=nb_hidden_layers[-1],
activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=sgd)
hist = model.fit(XtrainKeras,
YtrainKeras,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=True,
validation_data=(XtrainKeras, YtrainKeras))
score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])
#f.write()
f3 = open('AutoEncoderOnPca.txt', 'wb')
f3.write(json.dumps(hist.history))
f3.close()
Fname = prefix + '2 FineTuning_model=' + '.json'
weightName = prefix + 'Fine Tunes Weights_autoencoder_i=' + str(i) + '.h5'
json_string = model.to_json()
open(Fname, 'w').write(json_string)
model.save_weights(weightName, overwrite=True)
# Layer-wise pretraining
encoders = []
decoders = []
nb_hidden_layers = [X_train_narray.shape[1], 150, 2]
X_train_tmp = np.copy(X_train_norm_narray)
print('original X_train_tmp SIZE:',X_train_tmp.shape)
dense_layers = []
for i, (n_in, n_out) in enumerate(zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]), start=1):
print('Training the layer {}: Input {} -> Output {}'.format(i, n_in, n_out))
# Create AE and training
ae = Sequential()
if n_out >= 100:
encoder = containers.Sequential([Dense(output_dim=n_out, input_dim=n_in, activation='tanh'), Dropout(0.5)])
else:
encoder = containers.Sequential([Dense(output_dim=n_out, input_dim=n_in, activation='tanh')])
decoder = containers.Sequential([Dense(output_dim=n_in, input_dim=n_out, activation='tanh')])
ae.add(AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False))
sgd = SGD(lr=2, decay=1e-6, momentum=0.0, nesterov=True)
ae.compile(loss='mse', optimizer='adam')
ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size_pretraining, nb_epoch=nb_epoch_pretraining, verbose = True, shuffle=True)
# Store trainined weight and update training data
encoders.append(ae.layers[0].encoder)
decoders.append(ae.layers[0].decoder)
X_train_tmp = ae.predict(X_train_tmp)
print('X_train_tmp SIZE:',X_train_tmp.shape)
