def create_auto_encoder(encoder, decoder): print "\tCreating Auto Encoder..." auto_encoder = AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=True) print "\tAuto Encoder created." return auto_encoder
def add_autoencoder(self, encoder_sizes=[], decoder_sizes=[]): assert(len(encoder_sizes) != 0 and len(decoder_sizes) != 0) assert(len(encoder_sizes) == len(decoder_sizes)) self.encoder_sizes = encoder_sizes self.decoder_sizes = decoder_sizes # self.models = [Sequential() for i in range(len(encoder_sizes))] self.models = [Sequential()] encoders = Sequential() decoders = Sequential() for i in range(0, len(encoder_sizes) - 1): encoders.add(Dense(encoder_sizes[i], encoder_sizes[i + 1] , init=self.conf['--initialization'] , activation=self.conf['--activation'] , W_regularizer=l2())) decoders.add(Dense(decoder_sizes[i], decoder_sizes[i + 1] , init=self.conf['--initialization'] , activation=self.conf['--activation'] , W_regularizer=l2())) self.models[0].add(AutoEncoder(encoder=encoders , decoder=decoders , output_reconstruction=(i == 0))) return self.models
def add_conv_autoencoder(self, encoder_sizes=[], decoder_sizes=[]): assert(len(encoder_sizes) != 0 and len(decoder_sizes) != 0) assert(len(encoder_sizes) == len(decoder_sizes)) self.encoder_sizes = encoder_sizes self.decoder_sizes = decoder_sizes # self.models = [Sequential() for i in range(len(encoder_sizes))] self.models = [Sequential()] encoders = Sequential() decoders = Sequential() for i in range(0, len(encoder_sizes) - 1): encoders.add(Convolution1D(32, 3, 3 , activation=self.conf['--activation'] , init=self.conf['--initialization'] , border_mode='valid')) encoders.add(Activation('relu')) encoders.add(MaxPooling1D()) encoders.add(Convolution1D(32, 1, 1 , activation=self.conf['--activation'] , init=self.conf['--initialization'] , border_mode='valid')) decoders.add(Convolution1D(32, 1, 1 , activation=self.conf['--activation'] , init=self.conf['--initialization'] , border_mode='valid')) decoders.add(Activation('relu')) decoders.add(MaxPooling1D()) self.models[0].add(AutoEncoder(encoder=encoders , decoder=decoders , output_reconstruction=(i == 0))) return self.models
def multiple_layer_autoencoder(X_train, X_test, activation = 'linear', batch_size = 100, nb_epoch = 20, last_dim = 64): nb_hidden_layers = [X_train.shape[1], 256, 128, last_dim] X_train_tmp = np.copy(X_train) #X_test_tmp = np.copy(X_test) encoders = [] for i, (n_in, n_out) in enumerate(zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]), start=1): print('Training the layer {}: Input {} -> Output {}'.format(i, n_in, n_out)) # Create AE and training ae = Sequential() num_hidden = 128 max_features = X_train.shape[1] encoder = Embedding(max_features, 256) encoder = LSTM(256, num_hidden, activation='sigmoid', inner_activation='hard_sigmoid') encoder = Dense(n_in, n_out, activation=activation, init='glorot_normal', W_constraint = unitnorm()) decoder = Attention(max_features, 256) decoder = LSTM(256, num_hidden, activation='sigmoid', inner_activation='hard_sigmoid') decoder = Dense(n_out, n_in, activation=activation, init='glorot_normal'), W_constraint = unitnorm()) ae.add(AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False)) ae.add(Dropout(0.5)) #sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) adadelta = Adadelta(lr=1.0, rho=0.95, epsilon=1e-08) ae.compile(loss='mean_squared_error', optimizer=adadelta)#'rmsprop') ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=False, verbose=0, callbacks = [EarlyStopping(monitor='val_acc', patience=2)]) encoders.append(ae) X_train_tmp = ae.predict(X_train_tmp) print X_train_tmp.shape #X_test_tmp = ae.predict(X_test_tmp) #return encoders, X_train_tmp, X_test_tmp return encoders
def add_lstm_autoencoder(self, encoder_sizes=[], decoder_sizes=[]): assert(len(encoder_sizes) != 0 and len(decoder_sizes) != 0) assert(len(encoder_sizes) == len(decoder_sizes)) self.encoder_sizes = encoder_sizes self.decoder_sizes = decoder_sizes # self.models = [Sequential() for i in range(len(encoder_sizes))] self.models = [Sequential()] encoders = Sequential() decoders = Sequential() for i in range(0, len(encoder_sizes) - 1): encoders.add(LSTM(encoder_sizes[i], encoder_sizes[i + 1] , activation=self.conf['--activation'] , inner_activation=self.conf['--inner_activation'] , init=self.conf['--initialization'] , inner_init=self.conf['--inner_init'] , truncate_gradient=int(self.conf['--truncated_gradient']) , return_sequences=True)) decoders.add(LSTM(decoder_sizes[i], decoder_sizes[i + 1] , activation=self.conf['--activation'] , inner_activation=self.conf['--inner_activation'] , init=self.conf['--initialization'] , inner_init=self.conf['--inner_init'] , truncate_gradient=int(int(self.conf['--truncated_gradient'])) , return_sequences=not (i == len(encoder_sizes) - 1))) self.models[0].add(AutoEncoder(encoder=encoders , decoder=decoders , output_reconstruction=(i == 0))) return self.models
def __init__(self, n_in, n_hid, lr=1e-2, l2reg=3e-6, corruption_level=0.3, act='sigmoid'): self.lr = lr self.l2reg = l2reg self.corruption_level = corruption_level self.ae = Sequential() encoder = Sequential() encoder.add(Dense(n_in, n_hid, init='uniform', W_regularizer=l2(l2reg))) encoder.add(Activation(act)) decoder = Sequential() decoder.add(Dense(n_hid, n_in, init='uniform', W_regularizer=l2(l2reg))) decoder.add(Activation(act)) self.ae.add( AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=True)) opt = RMSprop(lr=lr, rho=0.9, epsilon=1e-6) self.ae.compile(loss='mean_squared_error', optimizer=opt)
def model(labels, data, go_id): batch_size = 64 nb_classes = 2 nb_epoch = 1 train, val, test = train_val_test_split(labels, data, batch_size=batch_size) train_label, train_data = train val_label, val_data = val test_label, test_data = test test_label_rep = test_label train_data = train_data.reshape(-1, 500 * 20) test_data = test_data.reshape(-1, 500 * 20) val_data = val_data.reshape(-1, 500 * 20) # convert class vectors to binary class matrices enc_wt = [] #creating the autoencoder ae = Sequential() encoder = containers.Sequential([Dense(5000, input_dim=10000), Dense(100)]) decoder = containers.Sequential([Dense(5000, input_dim=100), Dense(10000)]) ae.add( AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=True)) ae.compile(loss='mean_squared_error', optimizer='rmsprop') ae.fit(train_data, train_data, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=False, verbose=1, validation_data=[val_data, val_data]) model = Sequential() model.add(encoder) model.add(Dense(100, nb_classes, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') score = model.evaluate(val_data, val_label, show_accuracy=True, verbose=0) print('Test score before fine turning:', score[0]) print('Test accuracy after fine turning:', score[1]) model.fit(train_data, train_label, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, validation_data=(val_data, val_label)) score = model.evaluate(val_data, val_label, show_accuracy=True, verbose=0) print('Test score after fine turning:', score[0]) print('Test accuracy after fine turning:', score[1])
def createAutoEncoder(self, hSize, activation, optimizer): ae = Sequential() encoder = Dense(input_dim=self.input_dim, output_dim=hSize, activation=activation) decoder = Dense(input_dim=hSize, output_dim=self.input_dim) ae.add( AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=True)) ae.compile(loss=self.loss, optimizer=optimizer) return ae
def unroll_deep_ae(autoencoder, params, tie_weights=True): ''' Takes an autoencoder list generated by `pretrain_deep_ae` and unrolls it to make a deep autoencoder. NOTE this doesn't compile anything! This is simply a wrapper around the unrolling process to make it easier. Args: autoencoder (list): a list of keras layers. params (dict): the param dict returned by `pretrain_deep_ae` tie_weights (bool): whether or not to make the weights tied. Usage: >>> params = { ....'structure' : [625, 512, 128, 64], ....'activations' : 3 * [('sigmoid', 'relu')], ....'noise' : [GaussianNoise(0.01), None, None], ....'optimizer' : Adam(), ....'loss' : ['mse', 'mse', 'mse'] ....} >>> model = unroll_deep_ae(*pretrain_deep_ae(params, X)) Returns: keras.Sequential: a keras sequential model with one layer which is the unrolled autoencoder. ''' encoder = [] decoder = [] structure = params['structure'] for (layer_nb, (inputs, hidden)), (enc_act, dec_act) in zip( enumerate(zip(structure, structure[1:])), params['activations']): logger.info('Unpacking structure from level {}.'.format(layer_nb)) encoder.append(Dense(inputs, hidden, activation=enc_act)) encoder[-1].set_weights(autoencoder[layer_nb].get_weights()[:2]) decoder.insert(0, Dense(hidden, inputs, activation=dec_act)) decoder[0].set_weights(autoencoder[layer_nb].get_weights()[2:]) encoder_sequence = containers.Sequential(encoder) decoder_sequence = containers.Sequential(decoder) stacked_autoencoder = Sequential() stacked_autoencoder.add( AutoEncoder(encoder=encoder_sequence, decoder=decoder_sequence, output_reconstruction=False, tie_weights=tie_weights)) return stacked_autoencoder
def build_deep_classical_autoencoder(autoencoder): encoder = containers.Sequential([ Dense(input_dim, hidden_dim, activation=activation), Dense(hidden_dim, hidden_dim / 2, activation=activation) ]) decoder = containers.Sequential([ Dense(hidden_dim / 2, hidden_dim, activation=activation), Dense(hidden_dim, input_dim, activation=activation) ]) autoencoder.add( AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False)) return autoencoder
def build_deep_classical_autoencoder(autoencoder): encoders = [ Dense(input_dim, hidden_dim, activation=activation), Dense(hidden_dim, hidden_dim / 2, activation=activation) ] decoders = [ Dense(hidden_dim / 2, hidden_dim, activation=activation), Dense(hidden_dim, input_dim, activation=activation) ] autoencoder.add( AutoEncoder(encoders=encoders, decoders=decoders, output_reconstruction=False, tie_weights=True)) return autoencoder
def build_lstm_autoencoder(self, autoencoder): # The TimeDistributedDense isn't really necessary, however you need a lot of GPU memory to do 784x394-394x784 autoencoder.add( TimeDistributedDense(input_dim=self.input_dim, output_dim=self.input_dim)) autoencoder.add( AutoEncoder(encoder=LSTM(input_dim=self.input_dim, output_dim=50, activation='tanh', return_sequences=True), decoder=LSTM(input_dim=50, output_dim=self.input_dim, activation='tanh', return_sequences=True), output_reconstruction=False)) return autoencoder
def build_lstm_autoencoder(autoencoder, X_train, X_test): X_train = X_train[:, np.newaxis, :] X_test = X_test[:, np.newaxis, :] print("Modified X_train: ", X_train.shape) print("Modified X_test: ", X_test.shape) # The TimeDistributedDense isn't really necessary, however you need a lot of GPU memory to do 784x394-394x784 autoencoder.add(TimeDistributedDense(input_dim, 16)) autoencoder.add( AutoEncoder(encoder=LSTM(16, 8, activation=activation, return_sequences=True), decoder=LSTM(8, input_dim, activation=activation, return_sequences=True), output_reconstruction=False)) return autoencoder, X_train, X_test
def test_denoising_autoencoder(self): # # load data X_train, Y_train, X_test, Y_test = data.load_whale_data( WHALE_TRAIN_DATA, WHALE_TEST_DATA, dim=1) # autoencoder parameters batch_size = 32 nb_epoch = 1000 nb_neurons = 600 img_dim = X_train.shape[1] # create and train X_train_tmp = np.copy(X_train) ae = Sequential() ae.add( AutoEncoder( encoder=Dense(nb_neurons, input_dim=img_dim, activation='sigmoid'), decoder=Dense(img_dim, input_dim=nb_neurons, activation='sigmoid'), output_reconstruction=True, )) # compile ae.compile(loss="mean_squared_error", optimizer="adadelta") # fit ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size, nb_epoch=nb_epoch) results = ae.predict(X_train, verbose=1) np.save("results_2.npy", results) img_data = np.concatenate((X_train[0:9], results[0:9])) data.plot_multiple_mnist_images(img_data[0:9], (6, 3))
def pretrain_deep_ae(params, X, tie_weights=True, batch_size=100, nb_epoch=5, validation_data=None): ''' A function for building and greedily pretraining (interactively) a deep autoencoder. Args: params (dict): A dictionary with the following fields: * `structure`: a list of ints that describe the structure of the net, i.e., [10, 13, 2] * `activations`: a list of tuples of strings of length len(structure - 1) that describe the encoding and decoding activation function. For example, [('sigmoid', 'relu'), ('sigmoid', 'relu')] * `noise` (optional): a list of keras layers or None that describe the noise you want to add. i.e., [GaussianNoise(0.01), None] * `optimizer` (optional): one of the keras optimizers * `loss` (optional): a list of the loss functions to use. X (numpy.ndarray): the data to perform the unsupervised pretraining on. tie_weights (bool): tied or untied autoencoders. batch_size and nb_epoch: should be self explanatory... Usage: >>> params = { ....'structure' : [625, 512, 128, 64], ....'activations' : 3 * [('sigmoid', 'relu')], ....'noise' : [GaussianNoise(0.01), None, None], ....'optimizer' : Adam(), ....'loss' : ['mse', 'mse', 'mse'] ....} >>> ae, p = pretrain_deep_ae(params, X) Returns: a tuple (list, params), where list is a list of keras.Sequential(). ''' # -- check for logic errors. if type(params) is not dict: raise TypeError('params must be of class `dict`.') for k in ['structure', 'activations']: if k not in params.keys(): raise KeyError('key: `{}` must be in params dict'.format(k)) if len(params['structure']) != (len(params['activations']) + 1): raise ValueError( 'length of activations must be one less than length of structure.') if 'noise' not in params.keys(): logger.info('noise specifications not specified -- default to None') params['noise'] = len(params['activations']) * [None] if 'optimizer' not in params.keys(): logger.info( 'optimization specifications not specified -- using Adam()') params['optimizer'] = Adam() if 'loss' not in params.keys(): logger.info('loss specifications not specified -- using MSE') params['optimizer'] = len(params['activations']) * ['mse'] structure = params['structure'] autoencoder = [] # -- loop through the parameters for (inputs, hidden), (enc_act, dec_act), noise, loss in zip( zip( structure, # -- number of inputs structure[1:] # -- number of outputs ), params['activations'], params['noise'], params['loss']): logger.info('Building {} x {} structure.'.format(inputs, hidden)) autoencoder.append(Sequential()) if noise is not None: # -- noise should be a keras layer, so it can be in a Sequential() logger.info('using noise of type {}'.format(type(noise))) encoder = containers.Sequential([ noise, Dense(inputs, hidden, activation=enc_act), ActivityRegularization(l1=0.001) ]) else: # -- just a regular (non-denoising) ae. encoder = containers.Sequential([ Dense(inputs, hidden, activation=enc_act), ActivityRegularization(l1=0.001) ]) # encoder = Dense(inputs, hidden, activation=enc_act) # -- each element of the list is a Sequential(), so we add. autoencoder[-1].add( AutoEncoder(encoder=encoder, decoder=Dense(hidden, inputs, activation=dec_act), output_reconstruction=False, tie_weights=tie_weights)) logger.info('Compiling...') # -- each layer has it's own loss, but there is a global optimizer. logger.info('Loss: {}, Optimizer: {}'.format(loss, type( params['optimizer']))) autoencoder[-1].compile(loss=loss, optimizer=params['optimizer']) logger.info('Training...') # -- we allow people to end the training of each unit early. try: autoencoder[-1].fit(X, X, batch_size=batch_size, nb_epoch=nb_epoch, validation_data=validation_data) except KeyboardInterrupt: logger.info('Training ended early...') # -- embed in the new code space. X = autoencoder[-1].predict(X) return autoencoder, params
encoders = [] nb_hidden_layers = [784, 600, 500, 400] X_train_tmp = np.copy(X_train) for i, (n_in, n_out) in enumerate(zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]), start=1): print('Training the layer {}: Input {} -> Output {}'.format( i, n_in, n_out)) # Create AE and training ae = Sequential() encoder = containers.Sequential([Dense(n_in, n_out, activation='sigmoid')]) decoder = containers.Sequential([Dense(n_out, n_in, activation='sigmoid')]) ae.add( AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False, tie_weights=True)) ae.compile(loss='mean_squared_error', optimizer='rmsprop') ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size, nb_epoch=nb_epoch) # Store trainined weight and update training data encoders.append(ae.layers[0].encoder) X_train_tmp = ae.predict(X_train_tmp) # Fine-turning model = Sequential() for encoder in encoders: model.add(encoder) model.add(Dense(nb_hidden_layers[-1], nb_classes, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0)
input_shape=(batch_size, n_d), activation="tanh", return_sequences=True) ]) # rnn layer and output layers decoder = containers.Sequential([ Dropout(0.5, input_shape=(batch_size, n_d)), TimeDistributedDense(output_dim=vocabulary_size, input_dim=n_d, input_length=batch_size), Activation("softmax") ]) model = Sequential() model.add(AutoEncoder(encoder, decoder, output_reconstruction=False)) # Compile the model model.compile(loss="categorical_crossentropy", optimizer="rmsprop") json_string = model.to_json() with open("autoencoder_architecture.json", "w") as f: f.write(json_string) print("done.") ts = [] ds = [] data_file = "autoencoder_accuracy.tsv" def preds2seq(predictions):
def train(): model = Sequential() X_train = np.load(home + '/gabor/numpyFiles/Training Set.npy') X_test = np.load(home + '/gabor/numpyFiles/TestSet.npy') Y_train = np.load(home + '/gabor/numpyFiles/Training Labels.npy') Y_test = np.load(home + '/gabor/numpyFiles/TestSet Labels.npy') #X_test = X_test.reshape(-1, 1, 30, 96) Y_test = np_utils.to_categorical(Y_test, 447) #X_train = X_train.reshape(-1, 1, 30, 96) Y_train = np_utils.to_categorical(Y_train, 447) print("X_test.shape == {};".format(X_test.shape)) print("Y_test.shape == {};".format(Y_test.shape)) print("X_test.shape == {};".format(X_train.shape)) print("Y_test.shape == {};".format(Y_train.shape)) nb_hidden_layers = [len(X_train[0]), 700, 500, 300] XtrainKeras = X_train print 'shape of XTrain Keras is ', XtrainKeras.shape YtrainKeras = np_utils.to_categorical(Y_train, nb_classes) op1 = RMSprop(lr=0.01, rho=0.5, epsilon=1e-8) X_train_tmp = XtrainKeras trained_encoders = [] #XtrainKeras=XwhiteTrain.reshape(-1,1,len(XwhiteTrain),len(XwhiteTrain[0])) #YtrainKeras=np_utils.to_categorical(Y_train, nb_classes) #XtestKeras=X_test.reshape(-1,1,imageWidth,imageHeight) #YtestKeras=np_utils.to_categorical(Y_test, nb_classes) #X_train_tmp=XtrainKeras for n_in, n_out in zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]): print('Pre-training the layer: Input {} -> Output {}'.format( n_in, n_out)) # Create AE and training ae = Sequential() encoder = containers.Sequential( [Dense(n_out, input_dim=n_in, activation='sigmoid')]) decoder = containers.Sequential( [Dense(n_in, input_dim=n_out, activation='sigmoid')]) ae.add( AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False)) ae.compile(loss='mean_squared_error', optimizer=op1) hist = ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size, nb_epoch=nb_epoch) print(hist.history) Fname = prefix + 'autoencoder_n_in=' + str(n_in) + '_n_out= ' + str( n_out) + '.json' weightName = prefix + 'Weights_autoencoder_n_in=' + str( n_in) + '_n_out= ' + str(n_out) + '.h5' json_string = model.to_json() open(Fname, 'w').write(json_string) model.save_weights(weightName, overwrite=True) # Store trainined weight trained_encoders.append(ae.layers[0].encoder) # Update training data X_train_tmp = ae.predict(X_train_tmp) #ae1=Sequential() #encoder1=containers.Sequential([Dense(len(XwhiteTrain[0])-200,len(XwhiteTrain[0]),activation='sigmoid')]) Y_test = np_utils.to_categorical(Y_test, nb_classes) #X_test=X_test.reshape(-1,len(X_test[0])) print 'shape of X_test is ', X_test.shape print('Fine-tuning') sgd = SGD(lr=0.01, momentum=0.5, decay=0., nesterov=False) i = 1 model = Sequential() for encoder in trained_encoders: model.add(encoder) model.add( Dense(nb_classes, input_dim=nb_hidden_layers[-1], activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer=sgd) hist = model.fit(XtrainKeras, YtrainKeras, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, validation_data=(X_test, Y_test)) score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0) print('Test score:', score[0]) print('Test accuracy:', score[1]) res = model.predict_classes(X_test) print 'res is ', res Fname = prefix + '2 FineTuning_model=' + '.json' weightName = prefix + 'Fine Tunes Weights_autoencoder_i=' + str(i) + '.h5' json_string = model.to_json() open(Fname, 'w').write(json_string) model.save_weights(weightName, overwrite=True)
#the data, shuffled and split between train and test sets (X_train, y_train), (X_test, y_test) = mnist.load_data() X_train = [[0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 0, 1]] X_test = X_test.reshape(10000, 784) X_train = X_train.astype("float64") X_test = X_test.astype("float64") X_train /= 255 X_test /= 255 print(X_train.shape[0], 'train samples') print(X_test.shape[0], 'test samples') #convert class vectors to binary class matrices Y_train = np_utils.to_categorical(y_train, nb_classes) Y_test = np_utils.to_categorical(y_test, nb_classes) #creating the autoencoder #first autoencoder AE1_output_reconstruction = True ae1 = Sequential() encoder1 = containers.Sequential([Dense(4, 2, activation='tanh')]) decoder1 = containers.Sequential([Dense(2, 4, activation='tanh')]) ae1.add(AutoEncoder(encoder=encoder1, decoder=decoder1, output_reconstruction=AE1_output_reconstruction, tie_weights=True)) #training the first autoencoder ae1.compile(loss='mean_squared_error', optimizer=RMSprop()) ae1.fit(X_train, X_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=False, verbose=1, validation_data=[X_test, X_test])
trainS[ins[0]].add(ins[1]) for ins in testI: testX[ins[0], ins[1]] = ins[2] #X = np.array([[5, 3, 0, 1], [4, 0, 0, 1], [1, 1, 0, 5], [1, 0, 0, 4],[0, 1, 5, 4]] ) encoder = containers.Sequential([ Dense(3952, 300, activation='sigmoid'), Dense(300, 100, activation='sigmoid') ]) decoder = containers.Sequential([ Dense(100, 300, activation='sigmoid'), Dense(300, 3952, activation='sigmoid') ]) #encoder = containers.Sequential([Dense(3952, 100)]) #decoder = containers.Sequential([Dense(100, 3952)]) model = Sequential() model.add(AutoEncoder(encoder=encoder, decoder=decoder)) sgd = SGD(lr=0.2) model.compile(loss='binary_crossentropy', optimizer=sgd) #model.load_weights("/tmp/DA_weights.mod.hdf5") """ score = -1 for k in range(60): print k model.fit(X, X, nb_epoch = 1, batch_size = 128) P = model.predict(X) (score_in, score_out) = eval_AUC(P, X, testX) print score_in, score_out if score_in > score: score = score_in model.save_weights("/tmp/DA_weights.mod.hdf5", overwrite = True) """
def add_layer(mdl, ld, arch, first=False, last=False): global output_payload global tlogs global dbmodel global dbt num_inp = arch['num_inp'] num_out = arch['num_out'] if ld['type'] == 'dense': if first: mdl.add(Dense(ld['nodes'], input_dim=num_inp, init=ld['init'])) elif last: mdl.add(Dense(output_dim=num_out, init=ld['init'])) else: mdl.add(Dense(ld['nodes'], init=ld['init'])) try: if ld['type'] == 'convolutional': logstr(dbt, "Adding convolutional layer") if first: mdl.add( Convolution1D(ld['nodes'], filter_length=3, input_dim=num_inp, init=ld['init'])) elif last: mdl.add( Convolution1D(num_out, filter_length=3, init=ld['init'])) else: mdl.add( Convolution1D(ld['nodes'], filter_length=3, input_dim=ld['num_inp'], init=ld['init'])) except Exception as e: logstr(str(e)) try: if ld['type'] == 'timedistdense': if first: mdl.add( TimeDistributedDense(ld['nodes'], input_dim=num_inp, init=ld['init'])) elif last: mdl.add(TimeDistributedDense(num_out, init=ld['init'])) else: mdl.add(TimeDistributedDense(ld['nodes'], init=ld['init'])) except Exception as e: logstr(str(e)) if ld['type'] == 'autoencoder': if first: mdl.add( AutoEncoder( Dense(ld['nodes'], input_dim=num_inp, init=ld['init']), Dense(ld['nodes'], input_dim=num_inp, init=ld['init']))) elif last: mdl.add( AutoEncoder(Dense(num_out, init=ld['init']), Dense(num_out, init=ld['init']))) else: mdl.add( AutoEncoder(Dense(ld['nodes'], init=ld['init']), Dense(ld['nodes'], init=ld['init']))) if ld['activation'] and ld['activation'] != 'none': mdl.add(Activation(ld['activation'])) return mdl
def stackedAutoencoder(parameters): # Stacked Autoencoder # Train the autoencoder # Source: https://github.com/fchollet/keras/issues/358 random.seed(3) np.random.seed(3) nb_epoch_pretraining = 10 batch_size_pretraining = 500 # Layer-wise pretraining encoders = [] decoders = [] nb_hidden_layers = [parameters['X_train'].shape[1], parameters['neuron'][0], parameters['neuron'][1]] X_train_tmp = np.copy(parameters['X_train']) print('original X_train_tmp SIZE:',X_train_tmp.shape) dense_layers = [] for i, (n_in, n_out) in enumerate(zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]), start=1): print('Training the layer {}: Input {} -> Output {}'.format(i, n_in, n_out)) # Create AE and training ae = Sequential() if n_out >= 100: encoder = containers.Sequential([Dense(output_dim=n_out, input_dim=n_in, activation='tanh'), Dropout(0.5)]) else: encoder = containers.Sequential([Dense(output_dim=n_out, input_dim=n_in, activation='tanh')]) decoder = containers.Sequential([Dense(output_dim=n_in, input_dim=n_out, activation='tanh')]) ae.add(AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False)) sgd = SGD(lr=2, decay=1e-6, momentum=0.0, nesterov=True) ae.compile(loss='mse', optimizer=parameters['optimizer']) ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size_pretraining, nb_epoch=parameters['epoch_pretraining'], verbose = True, shuffle=True) # Store trainined weight and update training data encoders.append(ae.layers[0].encoder) decoders.append(ae.layers[0].decoder) X_train_tmp = ae.predict(X_train_tmp) print('X_train_tmp SIZE:',X_train_tmp.shape) ############## #End to End Autoencoder training if len(nb_hidden_layers) > 2: full_encoder = containers.Sequential() for encoder in encoders: full_encoder.add(encoder) full_decoder = containers.Sequential() for decoder in reversed(decoders): full_decoder.add(decoder) full_ae = Sequential() full_ae.add(AutoEncoder(encoder=full_encoder, decoder=full_decoder, output_reconstruction=False)) full_ae.compile(loss='mse', optimizer=parameters['optimizer']) print "Pretraining of full AE" full_ae.fit(parameters['X_train'], parameters['X_train'], batch_size=batch_size_pretraining, nb_epoch=parameters['epoch_pretraining'], verbose = True, shuffle=True) ####################################### nb_epoch = parameters['epoch'] batch_size = 100 model = Sequential() for encoder in encoders: model.add(encoder) model.add(Dense(output_dim=nb_labels, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer=parameters['optimizer']) score = model.evaluate(parameters['X_test'], parameters['y_test'], show_accuracy=True, verbose=0) print('Test score before fine turning:', score[0]) print('Test accuracy before fine turning:', score[1]) model.fit(parameters['X_train'], parameters['y_train'], batch_size=batch_size, nb_epoch=parameters['epoch'], show_accuracy=True, validation_data=(parameters['X_test'], parameters['y_test']), shuffle=True) score = model.evaluate(parameters['X_test'], parameters['y_test'], show_accuracy=True, verbose=0) print('Test score after fine turning:', score[0]) print('Test accuracy after fine turning:', score[1]) TestScore = score[0] TestAccuracy = score[1] return TestScore, TestAccuracy
X_train_tmp = np.copy(X_train_norm_narray) print('original X_train_tmp SIZE:',X_train_tmp.shape) dense_layers = [] for i, (n_in, n_out) in enumerate(zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]), start=1): print('Training the layer {}: Input {} -> Output {}'.format(i, n_in, n_out)) # Create AE and training ae = Sequential() if n_out >= 100: encoder = containers.Sequential([Dense(output_dim=n_out, input_dim=n_in, activation='tanh'), Dropout(0.5)]) else: encoder = containers.Sequential([Dense(output_dim=n_out, input_dim=n_in, activation='tanh')]) decoder = containers.Sequential([Dense(output_dim=n_in, input_dim=n_out, activation='tanh')]) ae.add(AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False)) sgd = SGD(lr=2, decay=1e-6, momentum=0.0, nesterov=True) ae.compile(loss='mse', optimizer='adam') ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size_pretraining, nb_epoch=nb_epoch_pretraining, verbose = True, shuffle=True) # Store trainined weight and update training data encoders.append(ae.layers[0].encoder) decoders.append(ae.layers[0].decoder) X_train_tmp = ae.predict(X_train_tmp) print('X_train_tmp SIZE:',X_train_tmp.shape) ##############
print("Vectorizing sequence data...") tokenizer = Tokenizer(nb_words=max_words) X_train = tokenizer.sequences_to_matrix(X_train, mode="binary") X_test = tokenizer.sequences_to_matrix(X_test, mode="binary") xw = X_train.transpose() userfea = 1000 itemfea = 8982 print("Building model...") encoder = containers.Sequential([Dense(1000, 700), Dense(700, 500)]) decoder = containers.Sequential([Dense(500, 700), Dense(700, 1000)]) model = Sequential() model.add( AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False)) model.compile(loss='mean_squared_error', optimizer='adam') model.fit(X_train, X_train, nb_epoch=15, batch_size=1024, verbose=1, show_accuracy=True, validation_split=0.1) df = open(r'f:\autoencoderrep.txt') dh = model.predict(X_train) for doc in dh: for v in doc: df.write("%s " % v)
y_train[count][col] = data col += 1 else: X_train[count - 1][col - 1] = data col += 1 count += 1 print "finish data" # First autoencoder AE_0 = Sequential() encoder = Sequential([Dense(30, input_dim=19, activation='relu')]) decoder = Sequential([Dense(19, input_dim=30, activation='relu')]) AE_0.add( AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=True)) AE_0.compile(loss='mse', optimizer='rmsprop') AE_0.fit(X_train, X_train, nb_epoch=10) temp = Sequential() temp.add(encoder) temp.compile(loss='mse', optimizer='rmsprop') first_output = temp.predict(X_train) print first_output.shape # Second Autoencoder AE_1 = Sequential() encoder_0 = Sequential([Dense(50, input_dim=30, activation='relu')]) decoder_0 = Sequential([Dense(30, input_dim=50, activation='relu')])
def train(): model = Sequential() print 'X.shape is ', XwhiteTrain.shape print 'Y.shape is ', Y_train.shape nb_hidden_layers = [len(XwhiteTrain[0]), 700, 500, 300] XtrainKeras = XwhiteTrain.reshape(-1, len(XwhiteTrain[0])) pretraining_files = [ 'Layer_1_PCAWhitened.txt', 'Layer_1_PCAWhitened.txt', 'Layer_3_PCAWhitened.txt' ] print 'shape of XTrain Keras is ', XtrainKeras.shape YtrainKeras = np_utils.to_categorical(Y_train, nb_classes) op1 = RMSprop(lr=0.01, rho=0.8, epsilon=1e-3) #datagen=ImageDataGenerator(zca_whitening=True) #datagen.fit() X_train_tmp = XtrainKeras trained_encoders = [] #XtrainKeras=XwhiteTrain.reshape(-1,1,len(XwhiteTrain),len(XwhiteTrain[0])) #YtrainKeras=np_utils.to_categorical(Y_train, nb_classes) #XtestKeras=X_test.reshape(-1,1,imageWidth,imageHeight) #YtestKeras=np_utils.to_categorical(Y_test, nb_classes) #X_train_tmp=XtrainKeras i = 0 for n_in, n_out in zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]): print('Pre-training the layer: Input {} -> Output {}'.format( n_in, n_out)) # Create AE and training ae = Sequential() encoder = containers.Sequential( [Dense(n_out, input_dim=n_in, activation='sigmoid')]) decoder = containers.Sequential( [Dense(n_in, input_dim=n_out, activation='sigmoid')]) ae.add( AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False)) ae.compile(loss='mean_squared_error', optimizer=op1) hist = ae.fit(X_train_tmp, X_train_tmp, batch_size=batch_size, nb_epoch=nb_epoch) #f.write(str(hist.history)) Fname = prefix + 'autoencoder_n_in=' + str(n_in) + '_n_out= ' + str( n_out) + '.json' weightName = prefix + 'Weights_autoencoder_n_in=' + str( n_in) + '_n_out= ' + str(n_out) + '.h5' json_string = model.to_json() f2 = open(pretraining_files[i], 'wb') f2.write(json.dumps(hist.history)) f2.close() i = i + 1 open(Fname, 'w').write(json_string) model.save_weights(weightName, overwrite=True) # Store trainined weight trained_encoders.append(ae.layers[0].encoder) # Update training data X_train_tmp = ae.predict(X_train_tmp) f.close() #ae1=Sequential() #encoder1=containers.Sequential([Dense(len(XwhiteTrain[0])-200,len(XwhiteTrain[0]),activation='sigmoid')]) X_test = np.load(Xtest_file) Y_test = np.load(Ytest_file) Y_test = np_utils.to_categorical(Y_test, nb_classes) X_test = X_test.reshape(-1, len(X_test[0])) print 'shape of X_test is ', X_test.shape print('Fine-tuning') sgd = SGD(lr=0.01, momentum=0.5, decay=0., nesterov=False) i = 1 model = Sequential() for encoder in trained_encoders: model.add(encoder) model.add( Dense(nb_classes, input_dim=nb_hidden_layers[-1], activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer=sgd) hist = model.fit(XtrainKeras, YtrainKeras, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, validation_data=(XtrainKeras, YtrainKeras)) score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0) print('Test score:', score[0]) print('Test accuracy:', score[1]) #f.write() f3 = open('AutoEncoderOnPca.txt', 'wb') f3.write(json.dumps(hist.history)) f3.close() Fname = prefix + '2 FineTuning_model=' + '.json' weightName = prefix + 'Fine Tunes Weights_autoencoder_i=' + str(i) + '.h5' json_string = model.to_json() open(Fname, 'w').write(json_string) model.save_weights(weightName, overwrite=True)
print("{1} train samples, {2} channel{0}, {3}x{4}".format("" if train_shape[1] == 1 else "s", *train_shape)) print("{1} test samples, {2} channel{0}, {3}x{4}".format("" if test_shape[1] == 1 else "s", *test_shape)) # flatten num_train_samples = train_shape[0] num_test_samples = test_shape[0] input_dim = train_shape[1] * train_shape[2] * train_shape[3] X_train = X_train.reshape(num_train_samples, input_dim) X_test = X_test.reshape(num_test_samples, input_dim) # parameters for autoencoder hidden_dim = int(input_dim / 2) final_dim = int(hidden_dim / 2) activation = 'linear' # build model model = Sequential() encoder = containers.Sequential([ Dense(hidden_dim, activation=activation, input_shape=(input_dim,)), Dense(final_dim, activation=activation)]) decoder = containers.Sequential([ Dense(hidden_dim, activation=activation, input_shape=(final_dim,)), Dense(input_dim, activation=activation)]) model.add(AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=True)) model.compile(loss='mean_squared_error', optimizer='adam') model.fit(X_train, X_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=1, validation_data=(X_test, X_test)) # not sure if this is a valid way to evaluate the autoencoder... score = model.evaluate(X_test, X_test, show_accuracy=True, verbose=0) print('Test score:', score[0]) print('Test accuracy:', score[1])