def target_RNN(wv, tweet_max_length, aspect_max_length, classes=2, **kwargs): ###################################################### # HyperParameters ###################################################### noise = kwargs.get("noise", 0) trainable = kwargs.get("trainable", False) rnn_size = kwargs.get("rnn_size", 75) rnn_type = kwargs.get("rnn_type", LSTM) final_size = kwargs.get("final_size", 100) final_type = kwargs.get("final_type", "linear") use_final = kwargs.get("use_final", False) drop_text_input = kwargs.get("drop_text_input", 0.) drop_text_rnn = kwargs.get("drop_text_rnn", 0.) drop_text_rnn_U = kwargs.get("drop_text_rnn_U", 0.) drop_target_rnn = kwargs.get("drop_target_rnn", 0.) drop_rep = kwargs.get("drop_rep", 0.) drop_final = kwargs.get("drop_final", 0.) activity_l2 = kwargs.get("activity_l2", 0.) clipnorm = kwargs.get("clipnorm", 5) bi = kwargs.get("bi", False) lr = kwargs.get("lr", 0.001) attention = kwargs.get("attention", "simple") ##################################################### shared_RNN = get_RNN(rnn_type, rnn_size, bi=bi, return_sequences=True, dropout_U=drop_text_rnn_U) input_tweet = Input(shape=[tweet_max_length], dtype='int32') input_aspect = Input(shape=[aspect_max_length], dtype='int32') # Embeddings tweets_emb = embeddings_layer(max_length=tweet_max_length, embeddings=wv, trainable=trainable, masking=True)( input_tweet) tweets_emb = GaussianNoise(noise)(tweets_emb) tweets_emb = Dropout(drop_text_input)(tweets_emb) aspects_emb = embeddings_layer(max_length=aspect_max_length, embeddings=wv, trainable=trainable, masking=True)( input_aspect) aspects_emb = GaussianNoise(noise)(aspects_emb) # Recurrent NN h_tweets = shared_RNN(tweets_emb) h_tweets = Dropout(drop_text_rnn)(h_tweets) h_aspects = shared_RNN(aspects_emb) h_aspects = Dropout(drop_target_rnn)(h_aspects) h_aspects = MeanOverTime()(h_aspects) h_aspects = RepeatVector(tweet_max_length)(h_aspects) # Merge of Aspect + Tweet representation = concatenate([h_tweets, h_aspects]) # apply attention over the hidden outputs of the RNN's att_layer = AttentionWithContext if attention == "context" else Attention representation = att_layer()(representation) representation = Dropout(drop_rep)(representation) if use_final: if final_type == "maxout": representation = MaxoutDense(final_size)(representation) else: representation = Dense(final_size, activation=final_type)( representation) representation = Dropout(drop_final)(representation) ###################################################### # Probabilities ###################################################### probabilities = Dense(1 if classes == 2 else classes, activation="sigmoid" if classes == 2 else "softmax", activity_regularizer=l2(activity_l2))(representation) model = Model(input=[input_aspect, input_tweet], output=probabilities) loss = "binary_crossentropy" if classes == 2 else "categorical_crossentropy" model.compile(optimizer=Adam(clipnorm=clipnorm, lr=lr), loss=loss) return model
(X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at)) (y_train, y_val) = (y[:split_at], y[split_at:]) print(X_train.shape) print(y_train.shape) chars = '0123456789' ctable = CharacterTable(chars, MAXLEN) print('Build model...') model = Sequential() # "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE # note: in a situation where your input sequences have a variable length, # use input_shape=(None, nb_feature). model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN, len(chars)))) # For the decoder's input, we repeat the encoded input for each time step model.add(RepeatVector(DIGITS)) # The decoder RNN could be multiple layers stacked or a single layer for _ in range(LAYERS): model.add(RNN(HIDDEN_SIZE, return_sequences=True)) # For each of step of the output sequence, decide which character should be chosen model.add(TimeDistributed(Dense(len(chars)))) model.add(Activation('softmax')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) print (model.summary()) print (X_train[0]) print (y_train[0])
Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, test_size=0.2, random_state=42) EMBED_SIZE = 128 HIDDEN_SIZE = 64 BATCH_SIZE = 32 NUM_EPOCHS = 1 # GRU model = Sequential() model.add(Embedding(s_vocabsize, EMBED_SIZE, input_length=MAX_SEQLEN)) model.add(Dropout(0.2)) model.add(GRU(HIDDEN_SIZE, dropout=0.2, recurrent_dropout=0.2)) model.add(RepeatVector(MAX_SEQLEN)) model.add(GRU(HIDDEN_SIZE, return_sequences=True)) model.add(TimeDistributed(Dense(t_vocabsize))) model.add(Activation("softmax")) model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS, validation_data=[Xtest, Ytest]) score, acc = model.evaluate(Xtest, Ytest, batch_size=BATCH_SIZE) print("Test score: {:.3f}, accuracy: {:.3f}".format(score, acc)) # LSTM
input_img = Input((32, 100, 3)) # CNN x = Conv2D(32, (3, 3), activation='relu', padding='same')(input_img) x = MaxPooling2D((2, 2))(x) x = Conv2D(32, (3, 3), activation='relu', padding='same')(x) x = MaxPooling2D((2, 2))(x) x = Conv2D(64, (3, 3), activation='relu', padding='same')(x) x = MaxPooling2D((2, 2))(x) x = Flatten()(x) # FC_layer x = Dense(1024, activation='relu')(x) x = RepeatVector(18)(x) # RNN x = LSTM(512, return_sequences=True)(x) x = LSTM(512, return_sequences=True)(x) output_word = TimeDistributed(Dense(64, activation='softmax'), input_shape=(18, 64))(x) recognizer = Model(input_img, output_word) with open('./models/Reco_archi.json', 'w') as f: f.write(recognizer.to_json()) print(recognizer.summary())
file_path = 'snopes.npy' with open(file_path, 'rb') as f: record = pickle.load(f) emb_idx = get_emb_idx('glove.6B.100d.txt') word_emb = create_emb_matrix( vocabulary_size, emb_idx, record['claim_text'] + record['evidence']) # + record['evidence_source']) art_wrd = Input(shape=(100, )) clm_wrd = Input(shape=(100, )) clm_wrd_emb = Embedding(vocabulary_size, 100, input_length=100, weights=[word_emb], trainable=False)(clm_wrd) mean_clm_wrd_emb = RepeatVector(100)(mean(clm_wrd_emb, axis=-1)) art_wrd_emb = Embedding(vocabulary_size, 100, input_length=100, weights=[word_emb], trainable=False)(art_wrd) ip_to_dense = Concatenate(axis=-1)([mean_clm_wrd_emb, art_wrd_emb]) attn_weights = Dense(128, activation='tanh')(clm_wrd_emb) attn_weights = Activation('softmax')(attn_weights) model_attn = Model(inputs=[art_wrd, clm_wrd], outputs=attn_weights) lstm_op = Bidirectional(LSTM(lstm_op_dim, return_sequences=True), merge_mode='concat')(art_wrd_emb) model_lstm = Model(inputs=art_wrd, outputs=lstm_op) inner_pdt = Dot(axes=1)([model_attn.output, model_lstm.output])
def model_construct(): # CONFIG config = ConfigParser() config.read('./config.ini') question_input = Input(shape=(config.getint('pre', 'question_maximum_length'), ), dtype='int32', name="question_input") relation_all_input = Input(shape=(config.getint( 'pre', 'relation_word_maximum_length'), ), dtype='int32', name="relation_all_input") relation_input = Input(shape=(config.getint('pre', 'relation_maximum_length'), ), dtype='int32', name="relation_input") question_emd = np.load('./question_emd_matrix.npy') relation_emd = np.load('./relation_emd_matrix.npy') relation_all_emd = np.load('./relation_all_emd_matrix.npy') question_emd = Embedding(question_emd.shape[0], config.getint('pre', 'word_emd_length'), weights=[question_emd], input_length=config.getint( 'pre', 'question_maximum_length'), trainable=False, name="question_emd")(question_input) sharedEmbd_r_w = Embedding(relation_all_emd.shape[0], config.getint('pre', 'word_emd_length'), weights=[relation_all_emd], input_length=config.getint( 'pre', 'relation_word_maximum_length'), trainable=False, name="sharedEmbd_r_w") relation_word_emd = sharedEmbd_r_w(relation_all_input) sharedEmbd_r = Embedding(relation_emd.shape[0], config.getint('pre', 'word_emd_length'), weights=[relation_emd], input_length=config.getint( 'pre', 'relation_maximum_length'), trainable=True, name="sharedEmbd_r") relation_emd = sharedEmbd_r(relation_input) bilstem_layer = Bidirectional(LSTM(units=200, return_sequences=True, implementation=2), name="bilstem_layer") question_bilstm_1 = bilstem_layer(question_emd) # question_bilstm_2 = Bidirectional(LSTM(units=200, return_sequences=True, implementation=2),name="question_bilstm_2")(question_bilstm_1) relation_word_bilstm = bilstem_layer(relation_word_emd) relation_bilstm = bilstem_layer(relation_emd) # question_res = Add()([question_bilstm_1, question_bilstm_2]) relation_con = Concatenate(axis=-2)( [relation_word_bilstm, relation_bilstm]) relation_res = MaxPooling1D(400, padding='same')(relation_con) relation_flatten = Flatten()(relation_res) fc_layer1 = Dense(400, use_bias=True, activation='tanh') fc_layer2 = Dense(1, use_bias=False, activation='softmax') rel_expand = RepeatVector(30)(relation_flatten) inputs = Concatenate()([question_bilstm_1, rel_expand]) weights = fc_layer2(fc_layer1(inputs)) question_att = MaxPooling1D(400, padding='same')( Multiply()([question_bilstm_1, weights])) # relation_maxpool = MaxPooling1D(400, padding='same')(relation_bilstm) # relation_word_maxpool = MaxPooling1D(400, padding='same')(relation_word_bilstm) # relation_res = Add()([relation_maxpool, relation_word_maxpool]) result = Dot(axes=-1, normalize=True)([question_att, relation_flatten]) model = Model(inputs=[ question_input, relation_input, relation_all_input, ], outputs=result) model.compile(optimizer=Adam(), loss=ranking_loss) return model
# $$ # # For safety, $y_0$ is defined as $\vec{0}$. # # # In[7]: # Define part of the attention layer gloablly so as to # share the same layers for each attention step. def softmax(x): return K.softmax(x, axis=1) at_repeat = RepeatVector(Tx) at_concatenate = Concatenate(axis=-1) at_dense1 = Dense(8, activation="tanh") at_dense2 = Dense(1, activation="relu") at_softmax = Activation(softmax, name='attention_weights') at_dot = Dot(axes=1) def one_step_of_attention(h_prev, a): """ Get the context. Input: h_prev - Previous hidden state of a RNN layer (m, n_h) a - Input data, possibly processed (m, Tx, n_a)
X = X.reshape((1, X.shape[0], X.shape[1])) y = y.reshape((1, y.shape[0], y.shape[1])) return (X, y) X, y = get_pair(5, 2, 50) n_features = 50 n_timesteps_in = 5 n_timesteps_out = 2 # model architecture model = Sequential() model.add(LSTM(150, input_shape=(n_timesteps_in, n_features))) model.add(RepeatVector(n_timesteps_in)) model.add(LSTM(150, return_sequences=True)) model.add(TimeDistributed(Dense(n_features, activation='softmax'))) # Summary print(model.summary()) # compile model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc']) for epoch in range(5000): X, y = get_pair(n_timesteps_in, n_timesteps_out, n_features) model.fit(X, y) # Evaluate
count = 0 # ## Let's create the model # In[48]: embedding_size = 300 # Input dimension is 4096 since we will feed it the encoded version of the image. # In[49]: image_model = Sequential([ Dense(embedding_size, input_shape=(2048, ), activation='relu'), RepeatVector(max_len) ]) # Since we are going to predict the next word using the previous words(length of previous words changes with every iteration over the caption), we have to set return_sequences = True. # In[50]: caption_model = Sequential([ Embedding(vocab_size, embedding_size, input_length=max_len), LSTM(256, return_sequences=True), TimeDistributed(Dense(300)) ]) # Merging the models and creating a softmax classifier # In[51]:
def build_model(self, rnn_layer_sizes=np.array([20, 20, 20]), dense_layer_sequential_sizes=np.array([32, 20]), dropout_prob_rnn=0.3, dropout_prob_dense=0.3, lambda_reg_rnn=0.001, lambda_reg_dense=0.001, multiple_concatenate=False): self.rnn_layer_sizes = rnn_layer_sizes self.dense_layer_sequential_sizes = dense_layer_sequential_sizes self.dropout_prob_rnn = dropout_prob_rnn self.dropout_prob_dense = dropout_prob_dense self.lambda_reg_rnn = lambda_reg_rnn self.lambda_reg_dense = lambda_reg_dense self.multiple_concatenate = multiple_concatenate # define inputs tracks_input = Input(self.input_shape_tracks, dtype='float32', name='tracks_input') session_input = Input(self.input_shape_sessions, dtype='float32', name='session_input') # Concatenate sessions and tracks session_rep = RepeatVector(20)(session_input) x_input = concatenate([tracks_input, session_rep], axis=-1) # RNN part x = LSTM(self.rnn_layer_sizes[0], return_sequences=True, kernel_regularizer=l2(self.lambda_reg_rnn))(x_input) if self.multiple_concatenate: for i in range(1, self.rnn_layer_sizes.size): x = concatenate([x, session_rep], axis=-1) x = LSTM(self.rnn_layer_sizes[i], return_sequences=True, kernel_regularizer=l2(self.lambda_reg_rnn))(x) else: for i in range(1, self.rnn_layer_sizes.size): x = LSTM(self.rnn_layer_sizes[i], return_sequences=True, kernel_regularizer=l2(self.lambda_reg_rnn))(x) x = BatchNormalization()(x) x = Dropout(self.dropout_prob_rnn)(x) for i in range(self.dense_layer_sequential_sizes.size - 1): x = Dense(self.dense_layer_sequential_sizes[i], activation='relu', kernel_regularizer=l2(self.lambda_reg_dense))(x) x = Dense(self.dense_layer_sequential_sizes[-1], activation='linear', kernel_regularizer=l2(self.lambda_reg_dense))(x) output = Reshape(self.output_shape, name='output')(x) # create model self.model = K_Model(inputs=[tracks_input, session_input], outputs=[output])
z = Lambda(sampling, name='LatentVector', output_shape=(latent_dim, ))([z_mean, z_log_sigma]) #VAE Loss def vae_loss(inputs, decoded): xent_loss = K.sum(K.binary_crossentropy(inputs, decoded), axis=1) kl_loss = -0.5 * K.sum( 1 + z_log_sigma - K.square(z_mean) - K.exp(z_log_sigma), axis=-1) return K.mean(xent_loss + kl_loss) #decoder LSTM decoded = RepeatVector(7, name='EmbeddingtoTimeSeries')(z) #timesteps decoded = LSTM(Intermediate_dim, name='DecoderLSTM1', return_sequences=True)(decoded) #intermediate dimensions decoded = LSTM(1, name='DecoderLSTM2', return_sequences=True)(decoded) #input_dim #decoded=TimeDistributed(Dense(1, name='Wrapper'), name='TimeDistributed')(decoded) v_autoencoder = Model(inputs, decoded) encoder = Model(inputs, z_mean) #v_autoencoder.summary() v_autoencoder.compile(optimizer=optimizer, loss=vae_loss) v_autoencoder.fit(X, X, nb_epoch=nb_epoch, batch_size=batch_size)
def build_model(self, rnn_layer_sizes=np.array([20, 20, 20]), dense_layer_parallel_sizes=np.array([10, 20]), dense_layer_sequential_sizes=np.array([32, 20]), dropout_prob_rnn=0.3, dropout_prob_dense=0.3, lambda_reg_rnn=0.001, lambda_reg_dense=0.001, merge='multiply'): self.rnn_layer_sizes = rnn_layer_sizes self.dense_layer_parallel_sizes = dense_layer_parallel_sizes self.dense_layer_sequential_sizes = dense_layer_sequential_sizes self.dropout_prob_rnn = dropout_prob_rnn self.dropout_prob_dense = dropout_prob_dense self.lambda_reg_rnn = lambda_reg_rnn self.lambda_reg_dense = lambda_reg_dense self.merge = merge if dense_layer_parallel_sizes[-1] != rnn_layer_sizes[-1]: print( 'Dimensions of last layers of RNN and of parallel dense network must agree!' ) return # define inputs tracks_input = Input(self.input_shape_tracks, dtype='float32', name='tracks_input') session_input = Input(self.input_shape_sessions, dtype='float32', name='session_input') # RNN side x_rnn = LSTM(self.rnn_layer_sizes[0], return_sequences=True, kernel_regularizer=l2(self.lambda_reg_rnn))(tracks_input) for i in range(1, self.rnn_layer_sizes.size): x_rnn = LSTM(self.rnn_layer_sizes[i], return_sequences=True, kernel_regularizer=l2(self.lambda_reg_rnn))(x_rnn) x_rnn = BatchNormalization()(x_rnn) out_rnn = Dropout(self.dropout_prob_rnn)(x_rnn) # dense side x_fc = Dense(self.dense_layer_parallel_sizes[0], activation='relu', kernel_regularizer=l2( self.lambda_reg_dense))(session_input) for i in range(1, self.dense_layer_parallel_sizes.size): x_fc = Dense(self.dense_layer_parallel_sizes[i], activation='relu', kernel_regularizer=l2(self.lambda_reg_dense))(x_fc) x_fc = BatchNormalization()(x_fc) out_fc = Dropout(self.dropout_prob_dense)(x_fc) x = [] # merge RNN and dense side if self.merge == 'multiply': x = multiply([out_rnn, out_fc]) elif self.merge == 'add': out_fc = RepeatVector(20)(out_fc) x = add([out_rnn, out_fc]) elif self.merge == 'concatenate': out_fc = RepeatVector(20)(out_fc) x = concatenate([out_rnn, out_fc], axis=-1) elif self.merge == 'maximum': out_fc = RepeatVector(20)(out_fc) x = maximum([out_rnn, out_fc]) else: print( 'Choose proper merge variation: multiply, add, concatenate or maximum' ) for i in range(self.dense_layer_sequential_sizes.size - 1): x = Dense(self.dense_layer_sequential_sizes[i], activation='relu', kernel_regularizer=l2(self.lambda_reg_dense))(x) x = Dense(self.dense_layer_sequential_sizes[-1], activation='linear', kernel_regularizer=l2(self.lambda_reg_dense))(x) output = Reshape(self.output_shape, name='output')(x) # create model and compile it self.model = K_Model(inputs=[tracks_input, session_input], outputs=[output])
def main(model=None): X, y = prepare_data(TRAINING_SIZE) print('Split data ... ') # Shuffle (X, y) #indices = np.arange(len(y)) #np.random.shuffle(indices) #X = X[indices] #y = y[indices] # Explicitly set apart 10% for validation data that we never train over split_at = len(X) - len(X) / 10 (X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at)) (y_train, y_val) = (y[:split_at], y[split_at:]) print(X_train.shape) print(y_train.shape) #print(X_train) #print(y_train) #return X_train, y_train if model is None: print('Build model...') model = Sequential() model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN_q, MAXW_q))) model.add(RepeatVector(MAXLEN_a)) for _ in range(LAYERS): model.add(RNN(HIDDEN_SIZE, return_sequences=True)) model.add(TimeDistributed(Dense(MAXW_a))) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) else: print('Use existed model...') # Train the model each generation and show predictions against the validation dataset for iteration in range(1, 50): print() print('-' * 50) print('Iteration', iteration) model.fit( X_train, y_train, batch_size=BATCH_SIZE, nb_epoch=10, #validation_split=0.1, validation_data=(X_val, y_val)) ### # Select samples from the validation set at random so we can visualize errors for i in range(5): ind = np.random.randint(0, len(X_val)) rowX, rowy = X_val[np.array([ind])], y_val[np.array([ind])] preds = model.predict_classes(rowX, verbose=0) #print(rowX) #print(rowy) #print(preds) q = rowX[0] correct = rowy[0] guess = preds[0] #print('Q', q) print('T', correct) #print('G', preds) print( colors.ok + '☑' + colors.close if correct[0][guess[0]] > 0 else colors.fail + '☒' + colors.close, guess) print('---') print('Save model ... ') model.save('stk_rnn2.h5')
sequence_in = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9]) # rehape input into [samples, timesteps, features] timestep_in = len(sequence_in) sequence_in = sequence_in.reshape((1, timestep_in, 1)) # Prepare output sequence sequence_out = sequence_in[:, 1:, :] sequence_out timestep_out = timestep_in - 1 # define model model = Sequential() # Encoder model.add(LSTM(100, activation='relu', input_shape=(timestep_in, 1))) model.add(RepeatVector(timestep_out)) model.add(LSTM(100, activation='relu', return_sequences=True)) model.add(TimeDistributed(Dense(1))) model.compile(optimizer='adam', loss='mse') #plot_model(model, show_shapes=True, to_file='predict_lstm_autoencoder.png') # fit model model.fit(sequence_in, sequence_out, epochs=100) # demonstrate prediction yhat = model.predict(sequence_in) print(yhat) # Test on new data sequence_test1 = np.array([11, 12, 13, 14, 15, 16, 17, 18, 19])
def extreme(self): print('Now operaring in Extreme.........') #input channel 1 start = time.time() input1 = Input(shape=(2, 1, 2, self.X_train.shape[2])) conv1 = ConvLSTM2D(filters=32, kernel_size=(1, 2), activation='relu')(input1) #pool1 = MaxPooling1D(pool_size=2)(conv1) # conv12 = Conv1D(filters=32, # activation='relu', # kernel_size=1)(pool1) # pool12 =MaxPooling1D(pool_size=1)(conv12) # flat1 = Flatten()(pool12) # #input channel 2 # inputs2 =Input(shape=(self.X_train.shape[1],self.X_train.shape[2])) # conv2= Conv1D(filters=96, # kernel_size=3, # activation='relu')(inputs2) # pool2 = MaxPooling1D(pool_size=2)(conv2) # conv22 = Conv1D(filters=32, # activation='relu', # kernel_size=1)(pool2) # pool22 =MaxPooling1D(pool_size=1)(conv22) # flat2 = Flatten()(pool22) # #input channel 3 # inputs3 =Input(shape=(self.X_train.shape[1],self.X_train.shape[2])) # conv3= Conv1D(filters=96, # kernel_size=3, # activation='relu')(inputs3) # pool3 = MaxPooling1D(pool_size=2)(conv3) # conv32 = Conv1D(filters=32, # activation='relu', # kernel_size=1)(pool3) # pool32 =MaxPooling1D(pool_size=1)(conv32) # flat3 = Flatten()(pool32) # #merged # merged =concatenate([flat1,flat2, flat3]) flat1 = Flatten()(conv1) rv = RepeatVector(self.w)(flat1) lstm1 = LSTM(128, activation='relu', return_sequences=True, dropout=0.2)(rv, training=True) lstm2 = LSTM(32, activation='relu', dropout=0.2)(lstm1) dense1 = Dense(50)(lstm2) out10 = Dense(1)(dense1) out50 = Dense(1)(dense1) out90 = Dense(1)(dense1) model = Model([input1], [out10, out50, out90]) losses = [ lambda y, f: self.loss(self.q, y, f), lambda y, f: self.loss(0.5, y, f), lambda y, f: self.loss(1 - self.q, y, f) ] #optimizer=BayesianOptimization() model.compile(loss=losses, optimizer='adam', metrics=['mae'], loss_weights=[0.2, 0.2, 0.2]) history = model.fit( [self.X_train.reshape(-1, 2, 1, 2, self.X_train.shape[2])], [self.y_train, self.y_train, self.y_train], epochs=1, batch_size=self.batch_size, verbose=1, shuffle=True) self.modele, self.historye = model, history ypred = np.array( self.modele.predict(self.X_test.reshape(-1, 2, 1, 2, self.X_train.shape[2]), verbose=1)).reshape(3, -1) self.yextreme = ypred[1] self.extremet = time.time() - start print('Execution time: {}'.format(self.extremet), end='\n')
def get_model_helper(): repeator = RepeatVector(Tx) concatenator = Concatenate(axis=-1) densor1 = Dense(10, activation="tanh") densor2 = Dense(1, activation="relu") activator = Activation(softmax, name='attention_weights') # customed softmax dotor = Dot(axes=1) def one_step_attention(a, s_prev): """ Performs one step of attention: Outputs a context vector computed as a dot product of the attention weights "alphas" and the hidden states "a" of the Bi-LSTM. Arguments: a -- hidden state output of the Bi-LSTM, numpy-array of shape (m, Tx, 2*n_a) s_prev -- previous hidden state of the (post-attention) LSTM, numpy-array of shape (m, n_s) Returns: context -- context vector, input of the next (post-attetion) LSTM cell """ # Use repeator to repeat s_prev to be of shape (m, Tx, n_s) so that you can concatenate it with all hidden states "a" (≈ 1 line) s_prev = repeator(s_prev) # Use concatenator to concatenate a and s_prev on the last axis (≈ 1 line) concat = concatenator([a, s_prev]) # Use densor1 to propagate concat through a small fully-connected neural network to compute the "intermediate energies" variable e. (≈1 lines) e = densor1(concat) # Use densor2 to propagate e through a small fully-connected neural network to compute the "energies" variable energies. (≈1 lines) energies = densor2(e) # Use "activator" on "energies" to compute the attention weights "alphas" (≈ 1 line) alphas = activator(energies) # Use dotor together with "alphas" and "a" to compute the context vector to be given to the next (post-attention) LSTM-cell (≈ 1 line) context = dotor([alphas, a]) return context #define inference model for evaluation def inference_model(vocab_inp_size, vocab_tar_size, Tx, Ty, n_s, embedding_dim): """ Arguments: Tx -- length of the input sequence Ty -- length of the output sequence embedding_dim -- embedding layer output size n_s -- hidden state size of the post-attention LSTM vocab_inp_size -- size of the python dictionary "vocab_inp_size" vocab_tar_size -- size of the python dictionary "vocab_tar_size" Returns: inference_model -- Keras inference model instance """ # Define the inputs of your model with a shape (Tx, vocab_inp_size) # Define s0 and c0, initial hidden state for the decoder LSTM of shape (n_s,) X0 = Input(shape=(Tx, ), name='X') # (m,Tx) X = encoder_embedding(X0) # (m,Tx,embedding_dim) s0 = Input(shape=(n_s, ), name='s0') c0 = Input(shape=(n_s, ), name='c0') s = s0 c = c0 outputs = [] decoder_X0 = Input(shape=(1, )) decoder_X = decoder_X0 #shape=(m, 1) #shape is not (m, Ty) because we manually iterate Ty timesteps #Define encoder as Bi-LSTM encoder_output, forward_h, forward_c, backward_h, backward_c = encoder_layer( X) encoder_hidden = Concatenate(axis=-1)([forward_h, backward_h]) encoder_cell = Concatenate(axis=-1)([forward_c, backward_c]) s = encoder_hidden c = encoder_cell for t in range(Ty): # one step of the attention mechanism to get back the context vector at step t context = one_step_attention(encoder_output, encoder_hidden) #(m, 1, n_s) decoder_X = decoder_embedding(decoder_X) #(m,1) - (m,1,embedding_dim) decoder_inputs = enc_concat([decoder_X, context]) #shape--(m, 1, n_s+embedding_dim) # Apply the post-attention LSTM cell to the "context" vector. #initial_state = [hidden state, cell state] decoder_X, s, c = decoder_cell(decoder_inputs, initial_state=[s, c]) #decoder_X.shape--(m,n_s) # Step 2.C: Apply Dense layer to the hidden state output of the decoder LSTM decoder_X = output_layer(decoder_X) #(m,vocab_tar_size) out = decoder_X #trick to add a dimension of 1 to tensor decoder_X = RepeatVector(1)(decoder_X) decoder_X = Lambda(lambda x: K.argmax(x))(decoder_X) #sampling #shape--(m,1) so that it can fit embedding layer outputs.append(out) model = Model([X0, s0, c0, decoder_X0], outputs) return model inf_model = inference_model(ger_vocab_size, eng_vocab_size, ger_length, eng_length, n_s, embedding_dim) inf_model.compile(optimizer=optimizers.RMSprop(lr=0.001), loss='categorical_crossentropy') #there is issues serializing the model architecture, so only save weights inf_model.load_weights('../models/inf_model_wts.h5') return inf_model
def one_hot(x): x = K.argmax(x) x = tf.one_hot(x, 78) x = RepeatVector(1)(x) return x
def inference_model(vocab_inp_size, vocab_tar_size, Tx, Ty, n_s, embedding_dim): """ Arguments: Tx -- length of the input sequence Ty -- length of the output sequence embedding_dim -- embedding layer output size n_s -- hidden state size of the post-attention LSTM vocab_inp_size -- size of the python dictionary "vocab_inp_size" vocab_tar_size -- size of the python dictionary "vocab_tar_size" Returns: inference_model -- Keras inference model instance """ # Define the inputs of your model with a shape (Tx, vocab_inp_size) # Define s0 and c0, initial hidden state for the decoder LSTM of shape (n_s,) X0 = Input(shape=(Tx, ), name='X') # (m,Tx) X = encoder_embedding(X0) # (m,Tx,embedding_dim) s0 = Input(shape=(n_s, ), name='s0') c0 = Input(shape=(n_s, ), name='c0') s = s0 c = c0 outputs = [] decoder_X0 = Input(shape=(1, )) decoder_X = decoder_X0 #shape=(m, 1) #shape is not (m, Ty) because we manually iterate Ty timesteps #Define encoder as Bi-LSTM encoder_output, forward_h, forward_c, backward_h, backward_c = encoder_layer( X) encoder_hidden = Concatenate(axis=-1)([forward_h, backward_h]) encoder_cell = Concatenate(axis=-1)([forward_c, backward_c]) s = encoder_hidden c = encoder_cell for t in range(Ty): # one step of the attention mechanism to get back the context vector at step t context = one_step_attention(encoder_output, encoder_hidden) #(m, 1, n_s) decoder_X = decoder_embedding(decoder_X) #(m,1) - (m,1,embedding_dim) decoder_inputs = enc_concat([decoder_X, context]) #shape--(m, 1, n_s+embedding_dim) # Apply the post-attention LSTM cell to the "context" vector. #initial_state = [hidden state, cell state] decoder_X, s, c = decoder_cell(decoder_inputs, initial_state=[s, c]) #decoder_X.shape--(m,n_s) # Step 2.C: Apply Dense layer to the hidden state output of the decoder LSTM decoder_X = output_layer(decoder_X) #(m,vocab_tar_size) out = decoder_X #trick to add a dimension of 1 to tensor decoder_X = RepeatVector(1)(decoder_X) decoder_X = Lambda(lambda x: K.argmax(x))(decoder_X) #sampling #shape--(m,1) so that it can fit embedding layer outputs.append(out) model = Model([X0, s0, c0, decoder_X0], outputs) return model
from keras import backend as K from keras.callbacks import LearningRateScheduler as LRS import numpy as np import random import sys import glob import pickle import re WIDTH = 2029 MAXLEN = 31 INPUTLEN = 1000 inputs = Input(shape=(INPUTLEN, )) #enc = Dense(2024, activation='linear')( inputs ) #enc = Dense(1024, activation='tanh')( enc ) repeat = RepeatVector(31)(inputs) generated = Bi(GRU(256, return_sequences=True))(repeat) generated = TD(Dense(2049, activation='relu'))(generated) generated = TD(Dense(2049, activation='softmax'))(generated) #generator = Model( inputs, generated ) #generated = Lambda( lambda x:x*2.0 )(generated) generator = Model(inputs, generated) generator.compile(optimizer=Adam(), loss='categorical_crossentropy') def train_base(): for ge, name in enumerate(glob.glob('utils/dataset_*.pkl')[:1]): dataset = pickle.loads(open(name, 'rb').read()) xs, ys = dataset
X.append(data[in_start:in_end, :]) y.append(data[in_end:out_end, 0]) # move along one time step in_start += 1 # print(np.array(X).shape) train_x, train_y = np.array(X), np.array(y) (750, 10, 5) # define parameters epochs, batch_size = 50, 60 n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[2], train_y.shape[1] # reshape output into [samples, timesteps, features] train_y = train_y.reshape((train_y.shape[0], train_y.shape[1], 1)) # define model model = Sequential() model.add(LSTM(200, activation='relu', input_shape=(n_timesteps, n_features))) model.add(RepeatVector(n_outputs)) model.add(LSTM(200, activation='relu', return_sequences=True)) model.add(TimeDistributed(Dense(100, activation='relu'))) model.add(TimeDistributed(Dense(1))) model.compile(loss='mse', optimizer='adam') # fit network model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size) history = [x for x in train] predictions = list() for i in range(len(test)): # flatten data data = np.array(history) data = data.reshape((data.shape[0]*data.shape[1], data.shape[2])) # retrieve last observations for input data input_x = data[-n_input:, :]
z_mean = Dense(latent_dim)(h) z_log_var = Dense(latent_dim)(h) def sampling(args): z_mean, z_log_var = args epsilon = K.random_normal(shape=(batch_size, latent_dim), mean=0., stddev=epsilon_std) return z_mean + K.exp(z_log_var / 2) * epsilon # note that "output_shape" isn't necessary with the TensorFlow backend z = Lambda(sampling, output_shape=(latent_dim, ))([z_mean, z_log_var]) # we instantiate these layers separately so as to reuse them later repeated_context = RepeatVector(max_len) decoder_h = LSTM(intermediate_dim, return_sequences=True, recurrent_dropout=0.2) decoder_mean = Dense( NB_WORDS, activation='linear' ) #softmax is applied in the seq2seqloss by tf #TimeDistributed() h_decoded = decoder_h(repeated_context(z)) x_decoded_mean = decoder_mean(h_decoded) # placeholder loss def zero_loss(y_true, y_pred): return K.zeros_like(y_pred)
trainX=encodedData[:round(len(encodedData)*0.9)] testX=encodedData[round(len(encodedData)*0.9):] trainY=oneHotLabels[:round(len(oneHotLabels)*0.9)] testY=oneHotLabels[round(len(oneHotLabels)*0.9):] print(trainX.shape) print(testX.shape) print(trainY.shape) print(testY.shape) exists = os.path.isfile("Task2Model.h5") if not exists: N_NERURONS=256 model= Sequential() model.add(Embedding(len(encodedData),N_NERURONS,input_length=src_timesteps,mask_zero=True)) model.add(LSTM(N_NERURONS, input_shape=(trainY.shape[1],trainY.shape[2]),return_sequences=False)) model.add(RepeatVector(trainY.shape[1])) model.add(LSTM(N_NERURONS//2, return_sequences=True)) model.add(TimeDistributed(Dense(TARGET_SIZE,activation="softmax"))) model.compile(optimizer="adam",loss="categorical_crossentropy",metrics=["accuracy"]) model.fit(trainX,trainY,validation_data=(testX,testY),epochs=30,batch_size=50) model.save("Task2Model.h5") model.save_weights("Task2Model_weights.h5") else: model = load_model("Task2Model.h5") model.load_weights("Task2Model_weights.h5") # SourceFileList = list(open("wwhitman-clean-processed.txt", "r")) SourceFileList =list(open("poem.txt", "r")) source_tokenizer = helper.create_tokenizer(SourceFileList) source_encoded_sequence = helper.encode_sequences(source_tokenizer,trainX.shape[1],SourceFileList)
Xs.append(X.iloc[i:(i + time_steps)].values) ys.append(y.iloc[i + time_steps]) return np.array(Xs), np.array(ys) X_train, y_train = create_sequences(train[['Close']], train['Close']) X_test, y_test = create_sequences(test[['Close']], test['Close']) print(f'Training shape: {X_train.shape}') print(f'Testing shape: {X_test.shape}') model = Sequential() model.add(LSTM(128, input_shape=(X_train.shape[1], X_train.shape[2]))) model.add(Dropout(rate=0.2)) model.add(RepeatVector(X_train.shape[1])) model.add(LSTM(128, return_sequences=True)) model.add(Dropout(rate=0.2)) model.add(TimeDistributed(Dense(X_train.shape[2]))) model.compile(optimizer='adam', loss='mae') model.summary() history = model.fit(X_train, y_train, epochs=100, batch_size=32, validation_split=0.1, callbacks=[ keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, mode='min')
print("Yoh.shape:", Yoh.shape) index = 0 print("Source date:", dataset[index][0]) print("Target date:", dataset[index][1]) print() print("Source after preprocessing (indices):", X[index]) print("Target after preprocessing (indices):", Y[index]) print() print("Source after preprocessing (one-hot):", Xoh[index]) print("Target after preprocessing (one-hot):", Yoh[index]) # Defined shared layers as global variables repeator = RepeatVector(Tx) concatenator = Concatenate(axis=-1) densor1 = Dense(10, activation = "tanh") densor2 = Dense(1, activation = "relu") activator = Activation(softmax, name='attention_weights') # We are using a custom softmax(axis = 1) loaded in this notebook dotor = Dot(axes = 1) # GRADED FUNCTION: one_step_attention def one_step_attention(a, s_prev): """ Performs one step of attention: Outputs a context vector computed as a dot product of the attention weights "alphas" and the hidden states "a" of the Bi-LSTM.
def make_model_1d(l2_lambda, clip_lenth, dimension): input_holder = Input(shape=(clip_lenth, dimension)) x = Conv1D(filters=8, kernel_size=15, padding='same', activation='relu', input_shape=(clip_lenth, dimension), kernel_regularizer=l2(l2_lambda))(input_holder) x = BatchNormalization()(x) x = Conv1D(filters=8, kernel_size=10, padding='same', activation='relu', input_shape=(clip_lenth, dimension), kernel_regularizer=l2(l2_lambda))(x) x = BatchNormalization()(x) x = MaxPooling1D(2, padding='same')(x) x = Conv1D(filters=16, kernel_size=5, padding='same', activation='relu', input_shape=(clip_lenth, dimension), kernel_regularizer=l2(l2_lambda))(x) x = BatchNormalization()(x) x = Conv1D(filters=16, kernel_size=5, padding='same', activation='relu', input_shape=(clip_lenth, dimension), kernel_regularizer=l2(l2_lambda))(x) x = BatchNormalization()(x) #####_______________________________________________________________________ u = GlobalMaxPooling1D()(x) u_broadcast = RepeatVector(x.shape[1])(u) def op(inputs): x, y = inputs return K.pow((x - y), 2) Z = Lambda(op)([u_broadcast, x]) v = GlobalMaxPooling1D()(Z) x = concatenate([u, v]) #####______________________________Multi task_________________________________________ # ['DP','BD','E','FS','A','RC'] number_of_N = 16 DP = 0.2 y1 = Dense( number_of_N, kernel_initializer='he_uniform', # kernel_regularizer=l2(l2_lambda) )(x) y1 = BatchNormalization()(y1) y1 = Activation('relu')(y1) y1 = Dropout(DP)( y1) # add some dropout for regularization after conv layers y1 = Dense(1, activation='sigmoid', kernel_initializer='glorot_uniform', name='DP' # kernel_regularizer=l2(l2_lambda) )(y1) y2 = Dense( number_of_N, kernel_initializer='he_uniform', # kernel_regularizer=l2(l2_lambda) )(x) y2 = BatchNormalization()(y2) y2 = Activation('relu')(y2) y2 = Dropout(DP)( y2) # add some dropout for regularization after conv layers y2 = Dense(1, activation='sigmoid', kernel_initializer='glorot_uniform', name='BD' # kernel_regularizer=l2(l2_lambda) )(y2) y3 = Dense( number_of_N, kernel_initializer='he_uniform', # kernel_regularizer=l2(l2_lambda) )(x) y3 = BatchNormalization()(y3) y3 = Activation('relu')(y3) y3 = Dropout(DP)( y3) # add some dropout for regularization after conv layers y3 = Dense(1, activation='sigmoid', kernel_initializer='glorot_uniform', name='E' # kernel_regularizer=l2(l2_lambda) )(y3) y4 = Dense( number_of_N, kernel_initializer='he_uniform', # kernel_regularizer=l2(l2_lambda) )(x) y4 = BatchNormalization()(y4) y4 = Activation('relu')(y4) y4 = Dropout(DP)( y4) # add some dropout for regularization after conv layers y4 = Dense(1, activation='sigmoid', kernel_initializer='glorot_uniform', name='FS' # kernel_regularizer=l2(l2_lambda) )(y4) y5 = Dense( number_of_N, kernel_initializer='he_uniform', # kernel_regularizer=l2(l2_lambda) )(x) y5 = BatchNormalization()(y5) y5 = Activation('relu')(y5) y5 = Dropout(DP)( y5) # add some dropout for regularization after conv layers y5 = Dense(1, activation='sigmoid', kernel_initializer='glorot_uniform', name='A' # kernel_regularizer=l2(l2_lambda) )(y5) y6 = Dense( number_of_N, kernel_initializer='he_uniform', # kernel_regularizer=l2(l2_lambda) )(x) y6 = BatchNormalization()(y6) y6 = Activation('relu')(y6) y6 = Dropout(DP)( y6) # add some dropout for regularization after conv layers y6 = Dense(1, activation='sigmoid', kernel_initializer='glorot_uniform', name='RC' # kernel_regularizer=l2(l2_lambda) )(y6) #####______________________________Multi task_________________________________________ model = Model(inputs=input_holder, outputs=[y1, y2, y3, y4, y5, y6]) # ['DP','BD','E','FS','A','RC'] losses = { "DP": "mean_squared_error", "BD": "mean_squared_error", "E": "mean_squared_error", "FS": "mean_squared_error", "A": "mean_squared_error", "RC": "mean_squared_error", } model.compile( #loss='mean_squared_error', # 'categorical_crossentropy' 'mean_squared_error' 'mean_absolute_percentage_error' loss=losses, optimizer='adam') # 'adadelta' 'rmsprop' # model.summary() return model
def create_lstm_vae(input_dim, timesteps, batch_size, intermediate_dim, latent_dim, epsilon_std=1.): """ Creates an LSTM Variational Autoencoder (VAE). Returns VAE, Encoder, Generator. # Arguments input_dim: int. timesteps: int, input timestep dimension. batch_size: int. intermediate_dim: int, output shape of LSTM. latent_dim: int, latent z-layer shape. epsilon_std: float, z-layer sigma. # References - [Building Autoencoders in Keras](https://blog.keras.io/building-autoencoders-in-keras.html) - [Generating sentences from a continuous space](https://arxiv.org/abs/1511.06349) """ x = Input(shape=( timesteps, input_dim, )) # LSTM encoding h1 = LSTM(intermediate_dim + 20, return_sequences=True, activation='relu')(x) h = LSTM(intermediate_dim, activation='relu')(h1) # VAE Z layer z_mean = Dense(latent_dim, activation='relu')(h) z_log_sigma = Dense(latent_dim, activation='relu')(h) def sampling(args): z_mean, z_log_sigma = args epsilon = K.random_normal(shape=(batch_size, latent_dim), mean=0., stddev=epsilon_std) return Add()([z_mean, Multiply()([z_log_sigma, epsilon])]) # note that "output_shape" isn't necessary with the TensorFlow backend # so you could write `Lambda(sampling)([z_mean, z_log_sigma])` z = Lambda(sampling, output_shape=(latent_dim, ))([z_mean, z_log_sigma]) # decoded LSTM layer decoder_h = LSTM(intermediate_dim, return_sequences=True, activation='relu') decoder_h1 = LSTM(intermediate_dim + 20, return_sequences=True, activation='relu') decoder_mean = LSTM(input_dim, return_sequences=True, activation='tanh') h_decoded = RepeatVector(timesteps)(z) h_decoded = decoder_h(h_decoded) # decoded layer h1_decoded = decoder_h1(h_decoded) x_decoded_mean = decoder_mean(h1_decoded) # end-to-end autoencoder vae = Model(x, x_decoded_mean) # encoder, from inputs to latent space encoder = Model(x, z_mean) # generator, from latent space to reconstructed inputs decoder_input = Input(shape=(latent_dim, )) _h_decoded = RepeatVector(timesteps)(decoder_input) _h_decoded = decoder_h(_h_decoded) _h_decoded = decoder_h1(_h_decoded) _x_decoded_mean = decoder_mean(_h_decoded) generator = Model(decoder_input, _x_decoded_mean) def vae_loss(x, x_decoded_mean): xent_loss = keras.losses.logcosh(x, x_decoded_mean) #xent_loss = objectives.mse(x, x_decoded_mean) kl_loss = -0.5 * K.mean(1 + z_log_sigma - K.square(z_mean) - K.exp(z_log_sigma)) loss = xent_loss + kl_loss return loss #sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) vae.compile(optimizer='rmsprop', loss=vae_loss) # The standard was rmsprop # generator.compile(optimizer='rmsprop', loss=vae_loss) return vae, encoder, generator
print('X.shape = {}'.format(X.shape)) print('Xq.shape = {}'.format(Xq.shape)) print('Y.shape = {}'.format(Y.shape)) print('tX.shape = {}'.format(tX.shape)) print('tXq.shape = {}'.format(tXq.shape)) print('tY.shape = {}'.format(tY.shape)) print('story_maxlen, query_maxlen = {}, {}'.format(story_maxlen, query_maxlen)) print('Build model...') print(vocab_size, vocab_answer_size) sentrnn = Sequential() sentrnn.add(Embedding(vocab_size, EMBED_HIDDEN_SIZE, input_length=story_maxlen)) sentrnn.add(Dropout(0.3)) sentrnn.add(RNN(EMBED_HIDDEN_SIZE, return_sequences=False)) sentrnn.add(RepeatVector(story_maxlen)) qrnn = Sequential() qrnn.add(Embedding(vocab_size, EMBED_HIDDEN_SIZE, input_length=query_maxlen)) qrnn.add(Dropout(0.3)) qrnn.add(RNN(EMBED_HIDDEN_SIZE, return_sequences=False)) qrnn.add(RepeatVector(story_maxlen)) model = Sequential() model.add(Merge([sentrnn, qrnn], mode='sum')) model.add(RNN(EMBED_HIDDEN_SIZE, return_sequences=False)) model.add(Dropout(0.3)) model.add(Dense(vocab_answer_size, activation='softmax')) # loaded_model = load_model('my_model.h5')
def repeat_vector(layer, layer_in, layerId, tensor=True): out = {layerId: RepeatVector(layer['params']['n'])} if tensor: out[layerId] = out[layerId](*layer_in) return out
# Shuffle (x, y) in unison as the later parts of x will almost all be larger # digits. indices = np.arange(len(y)) np.random.shuffle(indices) x = x[indices] y = y[indices] # Explicitly set apart 10% for validation data that we never train over. split_at = len(x) - len(x) // 10 (x_train, x_val) = x[:split_at], x[split_at:] (y_train, y_val) = y[:split_at], y[split_at:] model = Sequential() model.add(LSTM(config.hidden_size, input_shape=(maxlen, len(chars)))) model.add(RepeatVector(config.digits + 1)) model.add(LSTM(config.hidden_size, return_sequences=True)) model.add(TimeDistributed(Dense(len(chars), activation='softmax'))) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary() model.fit(x_train, y_train, batch_size=config.batch_size, epochs=100, validation_data=(x_val, y_val), callbacks=[WandbCallback(), log_table_callback]) # Train the model each generation and show predictions against the validation # dataset.
n_features = timeseries.shape[1] timeseries timesteps = 3 X, y = temporalize(X = timeseries, y = np.zeros(len(timeseries)), lookback = timesteps) n_features = 2 X = np.array(X) X = X.reshape(X.shape[0], timesteps, n_features) # define model model = Sequential() model.add(LSTM(128, activation='relu', input_shape=(timesteps,n_features), return_sequences=True)) model.add(LSTM(128, activation='relu', return_sequences=True)) model.add(LSTM(64, activation='relu', return_sequences=False)) model.add(RepeatVector(timesteps)) model.add(LSTM(64, activation='relu', return_sequences=True)) model.add(LSTM(64, activation='relu', return_sequences=True)) model.add(LSTM(128, activation='relu', return_sequences=True)) model.add(TimeDistributed(Dense(n_features))) model.compile(optimizer='adam', loss='mse') model.summary() # fit model model.fit(X, X, epochs=300, batch_size=5, verbose=0) # demonstrate reconstruction yhat = model.predict(X, verbose=0) print('---Predicted---') print(np.round(yhat,3)) print('---Actual---') print(np.round(X, 3))