def create_auto_encoder(encoder, decoder):
print "\tCreating Auto Encoder..."
auto_encoder = AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=True)
print "\tAuto Encoder created."
return auto_encoder
def add_autoencoder(self, encoder_sizes=[], decoder_sizes=[]):
assert(len(encoder_sizes) != 0 and len(decoder_sizes) != 0)
assert(len(encoder_sizes) == len(decoder_sizes))
self.encoder_sizes = encoder_sizes
self.decoder_sizes = decoder_sizes
# self.models = [Sequential() for i in range(len(encoder_sizes))]
self.models = [Sequential()]
encoders = Sequential()
decoders = Sequential()
for i in range(0, len(encoder_sizes) - 1):
encoders.add(Dense(encoder_sizes[i], encoder_sizes[i + 1]
, init=self.conf['--initialization']
, activation=self.conf['--activation']
, W_regularizer=l2()))
decoders.add(Dense(decoder_sizes[i], decoder_sizes[i + 1]
, init=self.conf['--initialization']
, activation=self.conf['--activation']
, W_regularizer=l2()))
self.models[0].add(AutoEncoder(encoder=encoders
, decoder=decoders
, output_reconstruction=(i == 0)))
return self.models
def add_conv_autoencoder(self, encoder_sizes=[], decoder_sizes=[]):
assert(len(encoder_sizes) != 0 and len(decoder_sizes) != 0)
assert(len(encoder_sizes) == len(decoder_sizes))
self.encoder_sizes = encoder_sizes
self.decoder_sizes = decoder_sizes
# self.models = [Sequential() for i in range(len(encoder_sizes))]
self.models = [Sequential()]
encoders = Sequential()
decoders = Sequential()
for i in range(0, len(encoder_sizes) - 1):
encoders.add(Convolution1D(32, 3, 3
, activation=self.conf['--activation']
, init=self.conf['--initialization']
, border_mode='valid'))
encoders.add(Activation('relu'))
encoders.add(MaxPooling1D())
encoders.add(Convolution1D(32, 1, 1
, activation=self.conf['--activation']
, init=self.conf['--initialization']
, border_mode='valid'))
decoders.add(Convolution1D(32, 1, 1
, activation=self.conf['--activation']
, init=self.conf['--initialization']
, border_mode='valid'))
decoders.add(Activation('relu'))
decoders.add(MaxPooling1D())
self.models[0].add(AutoEncoder(encoder=encoders
, decoder=decoders
, output_reconstruction=(i == 0)))
return self.models
def multiple_layer_autoencoder(X_train, X_test, activation = 'linear', batch_size = 100, nb_epoch = 20, last_dim = 64):
nb_hidden_layers = [X_train.shape[1], 256, 128, last_dim]
X_train_tmp = np.copy(X_train)
#X_test_tmp = np.copy(X_test)
encoders = []
for i, (n_in, n_out) in enumerate(zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]), start=1):
print('Training the layer {}: Input {} -> Output {}'.format(i, n_in, n_out))
# Create AE and training
ae = Sequential()
num_hidden = 128
max_features = X_train.shape[1]
encoder = Embedding(max_features, 256)
encoder = LSTM(256, num_hidden, activation='sigmoid', inner_activation='hard_sigmoid')
encoder = Dense(n_in, n_out, activation=activation, init='glorot_normal', W_constraint = unitnorm())
decoder = Attention(max_features, 256)
decoder = LSTM(256, num_hidden, activation='sigmoid', inner_activation='hard_sigmoid')
decoder = Dense(n_out, n_in, activation=activation, init='glorot_normal'), W_constraint = unitnorm())
ae.add(AutoEncoder(encoder=encoder, decoder=decoder,
output_reconstruction=False))
ae.add(Dropout(0.5))
#sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
adadelta = Adadelta(lr=1.0, rho=0.95, epsilon=1e-08)
ae.compile(loss='mean_squared_error', optimizer=adadelta)#'rmsprop')
ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=False, verbose=0, callbacks = [EarlyStopping(monitor='val_acc', patience=2)])
encoders.append(ae)
X_train_tmp = ae.predict(X_train_tmp)
print X_train_tmp.shape
#X_test_tmp = ae.predict(X_test_tmp)
#return encoders, X_train_tmp, X_test_tmp
return encoders
def add_lstm_autoencoder(self, encoder_sizes=[], decoder_sizes=[]):
assert(len(encoder_sizes) != 0 and len(decoder_sizes) != 0)
assert(len(encoder_sizes) == len(decoder_sizes))
self.encoder_sizes = encoder_sizes
self.decoder_sizes = decoder_sizes
# self.models = [Sequential() for i in range(len(encoder_sizes))]
self.models = [Sequential()]
encoders = Sequential()
decoders = Sequential()
for i in range(0, len(encoder_sizes) - 1):
encoders.add(LSTM(encoder_sizes[i], encoder_sizes[i + 1]
, activation=self.conf['--activation']
, inner_activation=self.conf['--inner_activation']
, init=self.conf['--initialization']
, inner_init=self.conf['--inner_init']
, truncate_gradient=int(self.conf['--truncated_gradient'])
, return_sequences=True))
decoders.add(LSTM(decoder_sizes[i], decoder_sizes[i + 1]
, activation=self.conf['--activation']
, inner_activation=self.conf['--inner_activation']
, init=self.conf['--initialization']
, inner_init=self.conf['--inner_init']
, truncate_gradient=int(int(self.conf['--truncated_gradient']))
, return_sequences=not (i == len(encoder_sizes) - 1)))
self.models[0].add(AutoEncoder(encoder=encoders
, decoder=decoders
, output_reconstruction=(i == 0)))
return self.models
def __init__(self,
n_in,
n_hid,
lr=1e-2,
l2reg=3e-6,
corruption_level=0.3,
act='sigmoid'):
self.lr = lr
self.l2reg = l2reg
self.corruption_level = corruption_level
self.ae = Sequential()
encoder = Sequential()
encoder.add(Dense(n_in, n_hid, init='uniform',
W_regularizer=l2(l2reg)))
encoder.add(Activation(act))
decoder = Sequential()
decoder.add(Dense(n_hid, n_in, init='uniform',
W_regularizer=l2(l2reg)))
decoder.add(Activation(act))
self.ae.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=True))
opt = RMSprop(lr=lr, rho=0.9, epsilon=1e-6)
self.ae.compile(loss='mean_squared_error', optimizer=opt)
def model(labels, data, go_id):
batch_size = 64
nb_classes = 2
nb_epoch = 1
train, val, test = train_val_test_split(labels,
data,
batch_size=batch_size)
train_label, train_data = train
val_label, val_data = val
test_label, test_data = test
test_label_rep = test_label
train_data = train_data.reshape(-1, 500 * 20)
test_data = test_data.reshape(-1, 500 * 20)
val_data = val_data.reshape(-1, 500 * 20)
# convert class vectors to binary class matrices
enc_wt = []
#creating the autoencoder
ae = Sequential()
encoder = containers.Sequential([Dense(5000, input_dim=10000), Dense(100)])
decoder = containers.Sequential([Dense(5000, input_dim=100), Dense(10000)])
ae.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=True))
ae.compile(loss='mean_squared_error', optimizer='rmsprop')
ae.fit(train_data,
train_data,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=False,
verbose=1,
validation_data=[val_data, val_data])
model = Sequential()
model.add(encoder)
model.add(Dense(100, nb_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
score = model.evaluate(val_data, val_label, show_accuracy=True, verbose=0)
print('Test score before fine turning:', score[0])
print('Test accuracy after fine turning:', score[1])
model.fit(train_data,
train_label,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=True,
validation_data=(val_data, val_label))
score = model.evaluate(val_data, val_label, show_accuracy=True, verbose=0)
print('Test score after fine turning:', score[0])
print('Test accuracy after fine turning:', score[1])
def createAutoEncoder(self, hSize, activation, optimizer):
ae = Sequential()
encoder = Dense(input_dim=self.input_dim,
output_dim=hSize,
activation=activation)
decoder = Dense(input_dim=hSize, output_dim=self.input_dim)
ae.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=True))
ae.compile(loss=self.loss, optimizer=optimizer)
return ae
def unroll_deep_ae(autoencoder, params, tie_weights=True):
'''
Takes an autoencoder list generated by `pretrain_deep_ae` and
unrolls it to make a deep autoencoder. NOTE this doesn't
compile anything! This is simply a wrapper around the
unrolling process to make it easier.
Args:
autoencoder (list): a list of keras layers.
params (dict): the param dict returned by `pretrain_deep_ae`
tie_weights (bool): whether or not to make the weights tied.
Usage:
>>> params = {
....'structure' : [625, 512, 128, 64],
....'activations' : 3 * [('sigmoid', 'relu')],
....'noise' : [GaussianNoise(0.01), None, None],
....'optimizer' : Adam(),
....'loss' : ['mse', 'mse', 'mse']
....}
>>> model = unroll_deep_ae(*pretrain_deep_ae(params, X))
Returns:
keras.Sequential: a keras sequential model with one layer
which is the unrolled autoencoder.
'''
encoder = []
decoder = []
structure = params['structure']
for (layer_nb, (inputs, hidden)), (enc_act, dec_act) in zip(
enumerate(zip(structure, structure[1:])), params['activations']):
logger.info('Unpacking structure from level {}.'.format(layer_nb))
encoder.append(Dense(inputs, hidden, activation=enc_act))
encoder[-1].set_weights(autoencoder[layer_nb].get_weights()[:2])
decoder.insert(0, Dense(hidden, inputs, activation=dec_act))
decoder[0].set_weights(autoencoder[layer_nb].get_weights()[2:])
encoder_sequence = containers.Sequential(encoder)
decoder_sequence = containers.Sequential(decoder)
stacked_autoencoder = Sequential()
stacked_autoencoder.add(
AutoEncoder(encoder=encoder_sequence,
decoder=decoder_sequence,
output_reconstruction=False,
tie_weights=tie_weights))
return stacked_autoencoder
def build_deep_classical_autoencoder(autoencoder):
encoder = containers.Sequential([
Dense(input_dim, hidden_dim, activation=activation),
Dense(hidden_dim, hidden_dim / 2, activation=activation)
])
decoder = containers.Sequential([
Dense(hidden_dim / 2, hidden_dim, activation=activation),
Dense(hidden_dim, input_dim, activation=activation)
])
autoencoder.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=False))
return autoencoder
def build_deep_classical_autoencoder(autoencoder):
encoders = [
Dense(input_dim, hidden_dim, activation=activation),
Dense(hidden_dim, hidden_dim / 2, activation=activation)
]
decoders = [
Dense(hidden_dim / 2, hidden_dim, activation=activation),
Dense(hidden_dim, input_dim, activation=activation)
]
autoencoder.add(
AutoEncoder(encoders=encoders,
decoders=decoders,
output_reconstruction=False,
tie_weights=True))
return autoencoder
def build_lstm_autoencoder(self, autoencoder):
# The TimeDistributedDense isn't really necessary, however you need a lot of GPU memory to do 784x394-394x784
autoencoder.add(
TimeDistributedDense(input_dim=self.input_dim,
output_dim=self.input_dim))
autoencoder.add(
AutoEncoder(encoder=LSTM(input_dim=self.input_dim,
output_dim=50,
activation='tanh',
return_sequences=True),
decoder=LSTM(input_dim=50,
output_dim=self.input_dim,
activation='tanh',
return_sequences=True),
output_reconstruction=False))
return autoencoder
def build_lstm_autoencoder(autoencoder, X_train, X_test):
X_train = X_train[:, np.newaxis, :]
X_test = X_test[:, np.newaxis, :]
print("Modified X_train: ", X_train.shape)
print("Modified X_test: ", X_test.shape)
# The TimeDistributedDense isn't really necessary, however you need a lot of GPU memory to do 784x394-394x784
autoencoder.add(TimeDistributedDense(input_dim, 16))
autoencoder.add(
AutoEncoder(encoder=LSTM(16,
8,
activation=activation,
return_sequences=True),
decoder=LSTM(8,
input_dim,
activation=activation,
return_sequences=True),
output_reconstruction=False))
return autoencoder, X_train, X_test
def test_denoising_autoencoder(self):
# # load data
X_train, Y_train, X_test, Y_test = data.load_whale_data(
WHALE_TRAIN_DATA, WHALE_TEST_DATA, dim=1)
# autoencoder parameters
batch_size = 32
nb_epoch = 1000
nb_neurons = 600
img_dim = X_train.shape[1]
# create and train
X_train_tmp = np.copy(X_train)
ae = Sequential()
ae.add(
AutoEncoder(
encoder=Dense(nb_neurons,
input_dim=img_dim,
activation='sigmoid'),
decoder=Dense(img_dim,
input_dim=nb_neurons,
activation='sigmoid'),
output_reconstruction=True,
))
# compile
ae.compile(loss="mean_squared_error", optimizer="adadelta")
# fit
ae.fit(X_train_tmp,
X_train_tmp,
batch_size=batch_size,
nb_epoch=nb_epoch)
results = ae.predict(X_train, verbose=1)
np.save("results_2.npy", results)
img_data = np.concatenate((X_train[0:9], results[0:9]))
data.plot_multiple_mnist_images(img_data[0:9], (6, 3))
def pretrain_deep_ae(params,
X,
tie_weights=True,
batch_size=100,
nb_epoch=5,
validation_data=None):
'''
A function for building and greedily pretraining (interactively)
a deep autoencoder.
Args:
params (dict): A dictionary with the following fields:
* `structure`: a list of ints that describe the
structure of the net, i.e., [10, 13, 2]
* `activations`: a list of tuples of strings of
length len(structure - 1) that describe the
encoding and decoding activation function.
For example, [('sigmoid', 'relu'), ('sigmoid', 'relu')]
* `noise` (optional): a list of keras layers or None that describe
the noise you want to add. i.e., [GaussianNoise(0.01), None]
* `optimizer` (optional): one of the keras optimizers
* `loss` (optional): a list of the loss functions to use.
X (numpy.ndarray): the data to perform the unsupervised pretraining on.
tie_weights (bool): tied or untied autoencoders.
batch_size and nb_epoch: should be self explanatory...
Usage:
>>> params = {
....'structure' : [625, 512, 128, 64],
....'activations' : 3 * [('sigmoid', 'relu')],
....'noise' : [GaussianNoise(0.01), None, None],
....'optimizer' : Adam(),
....'loss' : ['mse', 'mse', 'mse']
....}
>>> ae, p = pretrain_deep_ae(params, X)
Returns:
a tuple (list, params), where list is a list of keras.Sequential().
'''
# -- check for logic errors.
if type(params) is not dict:
raise TypeError('params must be of class `dict`.')
for k in ['structure', 'activations']:
if k not in params.keys():
raise KeyError('key: `{}` must be in params dict'.format(k))
if len(params['structure']) != (len(params['activations']) + 1):
raise ValueError(
'length of activations must be one less than length of structure.')
if 'noise' not in params.keys():
logger.info('noise specifications not specified -- default to None')
params['noise'] = len(params['activations']) * [None]
if 'optimizer' not in params.keys():
logger.info(
'optimization specifications not specified -- using Adam()')
params['optimizer'] = Adam()
if 'loss' not in params.keys():
logger.info('loss specifications not specified -- using MSE')
params['optimizer'] = len(params['activations']) * ['mse']
structure = params['structure']
autoencoder = []
# -- loop through the parameters
for (inputs, hidden), (enc_act, dec_act), noise, loss in zip(
zip(
structure, # -- number of inputs
structure[1:] # -- number of outputs
),
params['activations'],
params['noise'],
params['loss']):
logger.info('Building {} x {} structure.'.format(inputs, hidden))
autoencoder.append(Sequential())
if noise is not None:
# -- noise should be a keras layer, so it can be in a Sequential()
logger.info('using noise of type {}'.format(type(noise)))
encoder = containers.Sequential([
noise,
Dense(inputs, hidden, activation=enc_act),
ActivityRegularization(l1=0.001)
])
else:
# -- just a regular (non-denoising) ae.
encoder = containers.Sequential([
Dense(inputs, hidden, activation=enc_act),
ActivityRegularization(l1=0.001)
])
# encoder = Dense(inputs, hidden, activation=enc_act)
# -- each element of the list is a Sequential(), so we add.
autoencoder[-1].add(
AutoEncoder(encoder=encoder,
decoder=Dense(hidden, inputs, activation=dec_act),
output_reconstruction=False,
tie_weights=tie_weights))
logger.info('Compiling...')
# -- each layer has it's own loss, but there is a global optimizer.
logger.info('Loss: {}, Optimizer: {}'.format(loss,
type(
params['optimizer'])))
autoencoder[-1].compile(loss=loss, optimizer=params['optimizer'])
logger.info('Training...')
# -- we allow people to end the training of each unit early.
try:
autoencoder[-1].fit(X,
X,
batch_size=batch_size,
nb_epoch=nb_epoch,
validation_data=validation_data)
except KeyboardInterrupt:
logger.info('Training ended early...')
# -- embed in the new code space.
X = autoencoder[-1].predict(X)
return autoencoder, params
encoders = []
nb_hidden_layers = [784, 600, 500, 400]
X_train_tmp = np.copy(X_train)
for i, (n_in, n_out) in enumerate(zip(nb_hidden_layers[:-1],
nb_hidden_layers[1:]),
start=1):
print('Training the layer {}: Input {} -> Output {}'.format(
i, n_in, n_out))
# Create AE and training
ae = Sequential()
encoder = containers.Sequential([Dense(n_in, n_out, activation='sigmoid')])
decoder = containers.Sequential([Dense(n_out, n_in, activation='sigmoid')])
ae.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=False,
tie_weights=True))
ae.compile(loss='mean_squared_error', optimizer='rmsprop')
ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size, nb_epoch=nb_epoch)
# Store trainined weight and update training data
encoders.append(ae.layers[0].encoder)
X_train_tmp = ae.predict(X_train_tmp)
# Fine-turning
model = Sequential()
for encoder in encoders:
model.add(encoder)
model.add(Dense(nb_hidden_layers[-1], nb_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0)
input_shape=(batch_size, n_d),
activation="tanh",
return_sequences=True)
])
# rnn layer and output layers
decoder = containers.Sequential([
Dropout(0.5, input_shape=(batch_size, n_d)),
TimeDistributedDense(output_dim=vocabulary_size,
input_dim=n_d,
input_length=batch_size),
Activation("softmax")
])
model = Sequential()
model.add(AutoEncoder(encoder, decoder, output_reconstruction=False))
# Compile the model
model.compile(loss="categorical_crossentropy", optimizer="rmsprop")
json_string = model.to_json()
with open("autoencoder_architecture.json", "w") as f:
f.write(json_string)
print("done.")
ts = []
ds = []
data_file = "autoencoder_accuracy.tsv"
def preds2seq(predictions):
def train():
model = Sequential()
X_train = np.load(home + '/gabor/numpyFiles/Training Set.npy')
X_test = np.load(home + '/gabor/numpyFiles/TestSet.npy')
Y_train = np.load(home + '/gabor/numpyFiles/Training Labels.npy')
Y_test = np.load(home + '/gabor/numpyFiles/TestSet Labels.npy')
#X_test = X_test.reshape(-1, 1, 30, 96)
Y_test = np_utils.to_categorical(Y_test, 447)
#X_train = X_train.reshape(-1, 1, 30, 96)
Y_train = np_utils.to_categorical(Y_train, 447)
print("X_test.shape == {};".format(X_test.shape))
print("Y_test.shape == {};".format(Y_test.shape))
print("X_test.shape == {};".format(X_train.shape))
print("Y_test.shape == {};".format(Y_train.shape))
nb_hidden_layers = [len(X_train[0]), 700, 500, 300]
XtrainKeras = X_train
print 'shape of XTrain Keras is ', XtrainKeras.shape
YtrainKeras = np_utils.to_categorical(Y_train, nb_classes)
op1 = RMSprop(lr=0.01, rho=0.5, epsilon=1e-8)
X_train_tmp = XtrainKeras
trained_encoders = []
#XtrainKeras=XwhiteTrain.reshape(-1,1,len(XwhiteTrain),len(XwhiteTrain[0]))
#YtrainKeras=np_utils.to_categorical(Y_train, nb_classes)
#XtestKeras=X_test.reshape(-1,1,imageWidth,imageHeight)
#YtestKeras=np_utils.to_categorical(Y_test, nb_classes)
#X_train_tmp=XtrainKeras
for n_in, n_out in zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]):
print('Pre-training the layer: Input {} -> Output {}'.format(
n_in, n_out))
# Create AE and training
ae = Sequential()
encoder = containers.Sequential(
[Dense(n_out, input_dim=n_in, activation='sigmoid')])
decoder = containers.Sequential(
[Dense(n_in, input_dim=n_out, activation='sigmoid')])
ae.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=False))
ae.compile(loss='mean_squared_error', optimizer=op1)
hist = ae.fit(X_train_tmp,
X_train_tmp,
batch_size=batch_size,
nb_epoch=nb_epoch)
print(hist.history)
Fname = prefix + 'autoencoder_n_in=' + str(n_in) + '_n_out= ' + str(
n_out) + '.json'
weightName = prefix + 'Weights_autoencoder_n_in=' + str(
n_in) + '_n_out= ' + str(n_out) + '.h5'
json_string = model.to_json()
open(Fname, 'w').write(json_string)
model.save_weights(weightName, overwrite=True)
# Store trainined weight
trained_encoders.append(ae.layers[0].encoder)
# Update training data
X_train_tmp = ae.predict(X_train_tmp)
#ae1=Sequential()
#encoder1=containers.Sequential([Dense(len(XwhiteTrain[0])-200,len(XwhiteTrain[0]),activation='sigmoid')])
Y_test = np_utils.to_categorical(Y_test, nb_classes)
#X_test=X_test.reshape(-1,len(X_test[0]))
print 'shape of X_test is ', X_test.shape
print('Fine-tuning')
sgd = SGD(lr=0.01, momentum=0.5, decay=0., nesterov=False)
i = 1
model = Sequential()
for encoder in trained_encoders:
model.add(encoder)
model.add(
Dense(nb_classes, input_dim=nb_hidden_layers[-1],
activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=sgd)
hist = model.fit(XtrainKeras,
YtrainKeras,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=True,
validation_data=(X_test, Y_test))
score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])
res = model.predict_classes(X_test)
print 'res is ', res
Fname = prefix + '2 FineTuning_model=' + '.json'
weightName = prefix + 'Fine Tunes Weights_autoencoder_i=' + str(i) + '.h5'
json_string = model.to_json()
open(Fname, 'w').write(json_string)
model.save_weights(weightName, overwrite=True)
#the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train = [[0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 0, 1]]
X_test = X_test.reshape(10000, 784)
X_train = X_train.astype("float64")
X_test = X_test.astype("float64")
X_train /= 255
X_test /= 255
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')
#convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
#creating the autoencoder
#first autoencoder
AE1_output_reconstruction = True
ae1 = Sequential()
encoder1 = containers.Sequential([Dense(4, 2, activation='tanh')])
decoder1 = containers.Sequential([Dense(2, 4, activation='tanh')])
ae1.add(AutoEncoder(encoder=encoder1, decoder=decoder1,
output_reconstruction=AE1_output_reconstruction, tie_weights=True))
#training the first autoencoder
ae1.compile(loss='mean_squared_error', optimizer=RMSprop())
ae1.fit(X_train, X_train, batch_size=batch_size, nb_epoch=nb_epoch,
show_accuracy=False, verbose=1, validation_data=[X_test, X_test])
trainS[ins[0]].add(ins[1])
for ins in testI:
testX[ins[0], ins[1]] = ins[2]
#X = np.array([[5, 3, 0, 1], [4, 0, 0, 1], [1, 1, 0, 5], [1, 0, 0, 4],[0, 1, 5, 4]] )
encoder = containers.Sequential([
Dense(3952, 300, activation='sigmoid'),
Dense(300, 100, activation='sigmoid')
])
decoder = containers.Sequential([
Dense(100, 300, activation='sigmoid'),
Dense(300, 3952, activation='sigmoid')
])
#encoder = containers.Sequential([Dense(3952, 100)])
#decoder = containers.Sequential([Dense(100, 3952)])
model = Sequential()
model.add(AutoEncoder(encoder=encoder, decoder=decoder))
sgd = SGD(lr=0.2)
model.compile(loss='binary_crossentropy', optimizer=sgd)
#model.load_weights("/tmp/DA_weights.mod.hdf5")
"""
score = -1
for k in range(60):
print k
model.fit(X, X, nb_epoch = 1, batch_size = 128)
P = model.predict(X)
(score_in, score_out) = eval_AUC(P, X, testX)
print score_in, score_out
if score_in > score:
score = score_in
model.save_weights("/tmp/DA_weights.mod.hdf5", overwrite = True)
"""
def add_layer(mdl, ld, arch, first=False, last=False):
global output_payload
global tlogs
global dbmodel
global dbt
num_inp = arch['num_inp']
num_out = arch['num_out']
if ld['type'] == 'dense':
if first:
mdl.add(Dense(ld['nodes'], input_dim=num_inp, init=ld['init']))
elif last:
mdl.add(Dense(output_dim=num_out, init=ld['init']))
else:
mdl.add(Dense(ld['nodes'], init=ld['init']))
try:
if ld['type'] == 'convolutional':
logstr(dbt, "Adding convolutional layer")
if first:
mdl.add(
Convolution1D(ld['nodes'],
filter_length=3,
input_dim=num_inp,
init=ld['init']))
elif last:
mdl.add(
Convolution1D(num_out, filter_length=3, init=ld['init']))
else:
mdl.add(
Convolution1D(ld['nodes'],
filter_length=3,
input_dim=ld['num_inp'],
init=ld['init']))
except Exception as e:
logstr(str(e))
try:
if ld['type'] == 'timedistdense':
if first:
mdl.add(
TimeDistributedDense(ld['nodes'],
input_dim=num_inp,
init=ld['init']))
elif last:
mdl.add(TimeDistributedDense(num_out, init=ld['init']))
else:
mdl.add(TimeDistributedDense(ld['nodes'], init=ld['init']))
except Exception as e:
logstr(str(e))
if ld['type'] == 'autoencoder':
if first:
mdl.add(
AutoEncoder(
Dense(ld['nodes'], input_dim=num_inp, init=ld['init']),
Dense(ld['nodes'], input_dim=num_inp, init=ld['init'])))
elif last:
mdl.add(
AutoEncoder(Dense(num_out, init=ld['init']),
Dense(num_out, init=ld['init'])))
else:
mdl.add(
AutoEncoder(Dense(ld['nodes'], init=ld['init']),
Dense(ld['nodes'], init=ld['init'])))
if ld['activation'] and ld['activation'] != 'none':
mdl.add(Activation(ld['activation']))
return mdl
def stackedAutoencoder(parameters):
# Stacked Autoencoder
# Train the autoencoder
# Source: https://github.com/fchollet/keras/issues/358
random.seed(3)
np.random.seed(3)
nb_epoch_pretraining = 10
batch_size_pretraining = 500
# Layer-wise pretraining
encoders = []
decoders = []
nb_hidden_layers = [parameters['X_train'].shape[1], parameters['neuron'][0], parameters['neuron'][1]]
X_train_tmp = np.copy(parameters['X_train'])
print('original X_train_tmp SIZE:',X_train_tmp.shape)
dense_layers = []
for i, (n_in, n_out) in enumerate(zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]), start=1):
print('Training the layer {}: Input {} -> Output {}'.format(i, n_in, n_out))
# Create AE and training
ae = Sequential()
if n_out >= 100:
encoder = containers.Sequential([Dense(output_dim=n_out, input_dim=n_in, activation='tanh'), Dropout(0.5)])
else:
encoder = containers.Sequential([Dense(output_dim=n_out, input_dim=n_in, activation='tanh')])
decoder = containers.Sequential([Dense(output_dim=n_in, input_dim=n_out, activation='tanh')])
ae.add(AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False))
sgd = SGD(lr=2, decay=1e-6, momentum=0.0, nesterov=True)
ae.compile(loss='mse', optimizer=parameters['optimizer'])
ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size_pretraining, nb_epoch=parameters['epoch_pretraining'], verbose = True, shuffle=True)
# Store trainined weight and update training data
encoders.append(ae.layers[0].encoder)
decoders.append(ae.layers[0].decoder)
X_train_tmp = ae.predict(X_train_tmp)
print('X_train_tmp SIZE:',X_train_tmp.shape)
##############
#End to End Autoencoder training
if len(nb_hidden_layers) > 2:
full_encoder = containers.Sequential()
for encoder in encoders:
full_encoder.add(encoder)
full_decoder = containers.Sequential()
for decoder in reversed(decoders):
full_decoder.add(decoder)
full_ae = Sequential()
full_ae.add(AutoEncoder(encoder=full_encoder, decoder=full_decoder, output_reconstruction=False))
full_ae.compile(loss='mse', optimizer=parameters['optimizer'])
print "Pretraining of full AE"
full_ae.fit(parameters['X_train'], parameters['X_train'], batch_size=batch_size_pretraining, nb_epoch=parameters['epoch_pretraining'], verbose = True, shuffle=True)
#######################################
nb_epoch = parameters['epoch']
batch_size = 100
model = Sequential()
for encoder in encoders:
model.add(encoder)
model.add(Dense(output_dim=nb_labels, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=parameters['optimizer'])
score = model.evaluate(parameters['X_test'], parameters['y_test'], show_accuracy=True, verbose=0)
print('Test score before fine turning:', score[0])
print('Test accuracy before fine turning:', score[1])
model.fit(parameters['X_train'], parameters['y_train'], batch_size=batch_size, nb_epoch=parameters['epoch'],
show_accuracy=True, validation_data=(parameters['X_test'], parameters['y_test']), shuffle=True)
score = model.evaluate(parameters['X_test'], parameters['y_test'], show_accuracy=True, verbose=0)
print('Test score after fine turning:', score[0])
print('Test accuracy after fine turning:', score[1])
TestScore = score[0]
TestAccuracy = score[1]
return TestScore, TestAccuracy
X_train_tmp = np.copy(X_train_norm_narray)
print('original X_train_tmp SIZE:',X_train_tmp.shape)
dense_layers = []
for i, (n_in, n_out) in enumerate(zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]), start=1):
print('Training the layer {}: Input {} -> Output {}'.format(i, n_in, n_out))
# Create AE and training
ae = Sequential()
if n_out >= 100:
encoder = containers.Sequential([Dense(output_dim=n_out, input_dim=n_in, activation='tanh'), Dropout(0.5)])
else:
encoder = containers.Sequential([Dense(output_dim=n_out, input_dim=n_in, activation='tanh')])
decoder = containers.Sequential([Dense(output_dim=n_in, input_dim=n_out, activation='tanh')])
ae.add(AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False))
sgd = SGD(lr=2, decay=1e-6, momentum=0.0, nesterov=True)
ae.compile(loss='mse', optimizer='adam')
ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size_pretraining, nb_epoch=nb_epoch_pretraining, verbose = True, shuffle=True)
# Store trainined weight and update training data
encoders.append(ae.layers[0].encoder)
decoders.append(ae.layers[0].decoder)
X_train_tmp = ae.predict(X_train_tmp)
print('X_train_tmp SIZE:',X_train_tmp.shape)
##############
print("Vectorizing sequence data...")
tokenizer = Tokenizer(nb_words=max_words)
X_train = tokenizer.sequences_to_matrix(X_train, mode="binary")
X_test = tokenizer.sequences_to_matrix(X_test, mode="binary")
xw = X_train.transpose()
userfea = 1000
itemfea = 8982
print("Building model...")
encoder = containers.Sequential([Dense(1000, 700), Dense(700, 500)])
decoder = containers.Sequential([Dense(500, 700), Dense(700, 1000)])
model = Sequential()
model.add(
AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False))
model.compile(loss='mean_squared_error', optimizer='adam')
model.fit(X_train,
X_train,
nb_epoch=15,
batch_size=1024,
verbose=1,
show_accuracy=True,
validation_split=0.1)
df = open(r'f:\autoencoderrep.txt')
dh = model.predict(X_train)
for doc in dh:
for v in doc:
df.write("%s " % v)
y_train[count][col] = data
col += 1
else:
X_train[count - 1][col - 1] = data
col += 1
count += 1
print "finish data"
# First autoencoder
AE_0 = Sequential()
encoder = Sequential([Dense(30, input_dim=19, activation='relu')])
decoder = Sequential([Dense(19, input_dim=30, activation='relu')])
AE_0.add(
AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=True))
AE_0.compile(loss='mse', optimizer='rmsprop')
AE_0.fit(X_train, X_train, nb_epoch=10)
temp = Sequential()
temp.add(encoder)
temp.compile(loss='mse', optimizer='rmsprop')
first_output = temp.predict(X_train)
print first_output.shape
# Second Autoencoder
AE_1 = Sequential()
encoder_0 = Sequential([Dense(50, input_dim=30, activation='relu')])
decoder_0 = Sequential([Dense(30, input_dim=50, activation='relu')])
def train():
model = Sequential()
print 'X.shape is ', XwhiteTrain.shape
print 'Y.shape is ', Y_train.shape
nb_hidden_layers = [len(XwhiteTrain[0]), 700, 500, 300]
XtrainKeras = XwhiteTrain.reshape(-1, len(XwhiteTrain[0]))
pretraining_files = [
'Layer_1_PCAWhitened.txt', 'Layer_1_PCAWhitened.txt',
'Layer_3_PCAWhitened.txt'
]
print 'shape of XTrain Keras is ', XtrainKeras.shape
YtrainKeras = np_utils.to_categorical(Y_train, nb_classes)
op1 = RMSprop(lr=0.01, rho=0.8, epsilon=1e-3)
#datagen=ImageDataGenerator(zca_whitening=True)
#datagen.fit()
X_train_tmp = XtrainKeras
trained_encoders = []
#XtrainKeras=XwhiteTrain.reshape(-1,1,len(XwhiteTrain),len(XwhiteTrain[0]))
#YtrainKeras=np_utils.to_categorical(Y_train, nb_classes)
#XtestKeras=X_test.reshape(-1,1,imageWidth,imageHeight)
#YtestKeras=np_utils.to_categorical(Y_test, nb_classes)
#X_train_tmp=XtrainKeras
i = 0
for n_in, n_out in zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]):
print('Pre-training the layer: Input {} -> Output {}'.format(
n_in, n_out))
# Create AE and training
ae = Sequential()
encoder = containers.Sequential(
[Dense(n_out, input_dim=n_in, activation='sigmoid')])
decoder = containers.Sequential(
[Dense(n_in, input_dim=n_out, activation='sigmoid')])
ae.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=False))
ae.compile(loss='mean_squared_error', optimizer=op1)
hist = ae.fit(X_train_tmp,
X_train_tmp,
batch_size=batch_size,
nb_epoch=nb_epoch)
#f.write(str(hist.history))
Fname = prefix + 'autoencoder_n_in=' + str(n_in) + '_n_out= ' + str(
n_out) + '.json'
weightName = prefix + 'Weights_autoencoder_n_in=' + str(
n_in) + '_n_out= ' + str(n_out) + '.h5'
json_string = model.to_json()
f2 = open(pretraining_files[i], 'wb')
f2.write(json.dumps(hist.history))
f2.close()
i = i + 1
open(Fname, 'w').write(json_string)
model.save_weights(weightName, overwrite=True)
# Store trainined weight
trained_encoders.append(ae.layers[0].encoder)
# Update training data
X_train_tmp = ae.predict(X_train_tmp)
f.close()
#ae1=Sequential()
#encoder1=containers.Sequential([Dense(len(XwhiteTrain[0])-200,len(XwhiteTrain[0]),activation='sigmoid')])
X_test = np.load(Xtest_file)
Y_test = np.load(Ytest_file)
Y_test = np_utils.to_categorical(Y_test, nb_classes)
X_test = X_test.reshape(-1, len(X_test[0]))
print 'shape of X_test is ', X_test.shape
print('Fine-tuning')
sgd = SGD(lr=0.01, momentum=0.5, decay=0., nesterov=False)
i = 1
model = Sequential()
for encoder in trained_encoders:
model.add(encoder)
model.add(
Dense(nb_classes, input_dim=nb_hidden_layers[-1],
activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=sgd)
hist = model.fit(XtrainKeras,
YtrainKeras,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=True,
validation_data=(XtrainKeras, YtrainKeras))
score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])
#f.write()
f3 = open('AutoEncoderOnPca.txt', 'wb')
f3.write(json.dumps(hist.history))
f3.close()
Fname = prefix + '2 FineTuning_model=' + '.json'
weightName = prefix + 'Fine Tunes Weights_autoencoder_i=' + str(i) + '.h5'
json_string = model.to_json()
open(Fname, 'w').write(json_string)
model.save_weights(weightName, overwrite=True)
print("{1} train samples, {2} channel{0}, {3}x{4}".format("" if train_shape[1] == 1 else "s", *train_shape))
print("{1} test samples, {2} channel{0}, {3}x{4}".format("" if test_shape[1] == 1 else "s", *test_shape))
# flatten
num_train_samples = train_shape[0]
num_test_samples = test_shape[0]
input_dim = train_shape[1] * train_shape[2] * train_shape[3]
X_train = X_train.reshape(num_train_samples, input_dim)
X_test = X_test.reshape(num_test_samples, input_dim)
# parameters for autoencoder
hidden_dim = int(input_dim / 2)
final_dim = int(hidden_dim / 2)
activation = 'linear'
# build model
model = Sequential()
encoder = containers.Sequential([
Dense(hidden_dim, activation=activation, input_shape=(input_dim,)),
Dense(final_dim, activation=activation)])
decoder = containers.Sequential([
Dense(hidden_dim, activation=activation, input_shape=(final_dim,)),
Dense(input_dim, activation=activation)])
model.add(AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=True))
model.compile(loss='mean_squared_error', optimizer='adam')
model.fit(X_train, X_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=1, validation_data=(X_test, X_test))
# not sure if this is a valid way to evaluate the autoencoder...
score = model.evaluate(X_test, X_test, show_accuracy=True, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])
