def Attention(): # https://github.com/farizrahman4u/seq2seq or https://github.com/datalogue/keras-attention import seq2seq from seq2seq.models import AttentionSeq2Seq input_q = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='float64') q = embedding()(input_q) q = AttentionSeq2Seq(input_dim=EMBEDDING_DIM, input_length=MAX_SEQUENCE_LENGTH, output_length=MAX_SEQUENCE_LENGTH, output_dim=QA_EMBED_SIZE, depth=1)(q) q = Flatten()(q) q = Dropout(DROPOUT_RATE)(q) input_a = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='float64') a = embedding()(input_a) a = AttentionSeq2Seq(input_dim=EMBEDDING_DIM, input_length=MAX_SEQUENCE_LENGTH, output_length=MAX_SEQUENCE_LENGTH, output_dim=QA_EMBED_SIZE, depth=1)(a) a = Flatten()(a) a = Dropout(DROPOUT_RATE)(a) merged = concatenate([q, a]) merged = Dense(512)(merged) merged = BatchNormalization()(merged) merged = Activation('relu')(merged) merged = Dropout(DROPOUT_RATE)(merged) merged = Dense(1, activation="sigmoid")(merged) model = Model([input_q, input_a], [merged]) return model
def test_AttentionSeq2Seq(): x = np.random.random((samples, input_length, input_dim)) y = np.random.random((samples, output_length, output_dim)) models = [] models += [ AttentionSeq2Seq(output_dim=output_dim, hidden_dim=hidden_dim, output_length=output_length, input_shape=(input_length, input_dim)) ] models += [ AttentionSeq2Seq(output_dim=output_dim, hidden_dim=hidden_dim, output_length=output_length, input_shape=(input_length, input_dim), depth=2) ] models += [ AttentionSeq2Seq(output_dim=output_dim, hidden_dim=hidden_dim, output_length=output_length, input_shape=(input_length, input_dim), depth=3) ] for model in models: model.compile(loss='mse', optimizer='sgd') model.fit(x, y, epochs=1)
def create_model(self, params=None): params_ = AttentionSeq2Seq.default_params().copy() params_.update({"source.reverse": True}) params_.update(params or {}) return AttentionSeq2Seq(source_vocab_info=self.vocab_info, target_vocab_info=self.vocab_info, params=params_)
def create_model(self, mode, params=None): params_ = AttentionSeq2Seq.default_params().copy() params_.update(TEST_PARAMS) params_.update({ "source.reverse": True, "vocab_source": self.vocab_file.name, "vocab_target": self.vocab_file.name, }) params_.update(params or {}) return AttentionSeq2Seq(params=params_, mode=mode)
def default_params(): params = AttentionSeq2Seq.default_params() params["discriminator_units"] = 256 params["discriminator_loss_multiplier"] = 10.0 params["discriminator_reverse_grad"] = False params["discriminator_mix_context"] = False return params
def main(): encoder_input_data, decoder_input_data, decoder_target_data = get_data() encoder_input_data = np.array( [convert_chord_indices_to_embeddings(c) for c in encoder_input_data]) decoder_input_data = to_categorical(decoder_input_data, num_classes=130) decoder_target_data = to_categorical(decoder_target_data, num_classes=130) # model = AttentionSeq2Seq(input_dim=32, hidden_dim=64, output_length=600, output_dim=130) model = AttentionSeq2Seq(input_dim=32, input_length=600, hidden_dim=64, output_length=600, output_dim=130) if not os.path.exists('s2s_attention.h5'): optimizer = Adam(clipnorm=1.0) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['categorical_accuracy']) model.fit(encoder_input_data, decoder_input_data, batch_size=32, epochs=2) model.save_weights('s2s_simple.h5') res = model.predict(np.expand_dims(encoder_input_data[15], axis=0)) print(np.argmax(res, axis=-1)) else: model.load_weights("s2s_fariz_attention.h5") output = model.predict(np.expand_dims(encoder_input_data[20], axis=0)) print(np.argmax(output, axis=-1))
def create_imgText(image_shape, max_caption_len, vocab_size): image_model = Sequential() # image_shape : C,W,H # input: 100x100 images with 3 channels -> (3, 100, 100) tensors. # this applies 32 convolution filters of size 3x3 each. image_model.add(Convolution2D(32, (3, 3), padding='valid', input_shape=image_shape)) image_model.add(BatchNormalization()) image_model.add(Activation('relu')) image_model.add(Convolution2D(32, (3, 3))) image_model.add(BatchNormalization()) image_model.add(Activation('relu')) image_model.add(MaxPooling2D(pool_size=(2, 2))) image_model.add(Dropout(0.25)) image_model.add(Convolution2D(64, (3, 3), padding='valid')) image_model.add(BatchNormalization()) image_model.add(Activation('relu')) image_model.add(Convolution2D(64, (3, 3))) image_model.add(BatchNormalization()) image_model.add(Activation('relu')) image_model.add(MaxPooling2D(pool_size=(2, 2))) image_model.add(Dropout(0.25)) image_model.add(Flatten()) # Note: Keras does automatic shape inference. image_model.add(Dense(128)) image_model.add(RepeatVector(1)) model = AttentionSeq2Seq(input_dim=128, input_length=1, hidden_dim=128, output_length=max_caption_len, output_dim=vocab_size) # model = Seq2Seq(input_dim=128, input_length=1, hidden_dim=128, output_length=max_caption_len, # output_dim=128, peek=True) image_model.add(model) image_model.add(TimeDistributed(Dense(vocab_size))) image_model.add(Activation('softmax')) return image_model
def create_model(in_encoder_vocabulary, in_decoder_vocabulary, in_embedding_matrix, in_input_length, in_output_length, in_config, mode=Mode.TRAIN): effective_vocabulary_size, embedding_size = in_embedding_matrix.shape embedding_layer = Embedding(effective_vocabulary_size, embedding_size, weights=[in_embedding_matrix], input_length=in_input_length, trainable=True, name='emb') seq2seq_model = AttentionSeq2Seq(bidirectional=False, input_dim=embedding_size, output_dim=len(in_decoder_vocabulary), hidden_dim=in_config['layer_size'], output_length=in_output_length, depth=in_config['max_layers'], dropout=0.0 if mode == Mode.TEST else 0.2) model = Sequential() model.add(embedding_layer) model.add(seq2seq_model) model.add(Activation('softmax')) sgd = SGD(lr=in_config['learning_rate'], decay=in_config['learning_rate_decay'], clipvalue=in_config['gradient_clip_value']) model.compile(loss='categorical_crossentropy', optimizer=sgd) return model
def train(self, train_names, train_codes, hyperparams, pct_train=0.8, lr=0.01, num_epoch=100): if self.model_name == 'SimpleSeq2Seq' or self.model_name == 'AttentionSeq2Seq': train_name, train_code, val_name, val_code, naming_data, hyperparams['n_tokens'] = trainModel.split_data(train_names, train_codes, hyperparams['output_length'], hyperparams['input_length'], pct_train) hyperparams['is_embedding'] = False train_name = trainModel.one_hot_name(train_name, hyperparams['n_tokens']) required_params = ['output_dim', 'output_length', 'input_length', 'is_embedding', 'n_tokens'] for param in required_params: assert param in hyperparams, (param) if self.model_name == 'SimpleSeq2Seq': model = SimpleSeq2Seq(**hyperparams) elif self.model_name == 'AttentionSeq2Seq': model = AttentionSeq2Seq(**hyperparams) elif self.model_name == 'Seq2Seq': model = Seq2Seq(**hyperparams) else: raise TypeError my_adam = optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08) model.compile(optimizer=my_adam, loss='categorical_crossentropy') print ('fit...') model.fit(train_code, train_name, epochs=num_epoch) print ('predict...') predict_probs = model.predict(val_code) predict_idx = np.argmax(predict_probs, axis=2) print('evaluate...') exact_match, _ = trainModel.exact_match(naming_data, predict_idx, val_name) precision, recall, f1, _, _ = trainModel.evaluate_tokens(naming_data, predict_idx, val_name) return model, exact_match, precision, recall, f1, naming_data
def build_model(input_dim, input_length, hidden_dim, output_length, out_dim, depth): model = AttentionSeq2Seq(input_dim=input_dim, input_length=input_length, hidden_dim=hidden_dim, output_length=output_length, output_dim=out_dim, depth=depth) model.compile(loss='logcosh', optimizer='adam', metrics=["mae", "mse"]) return model
def attention_s2s_lib(): model = AttentionSeq2Seq(output_dim=output_dim, hidden_dim=hidden_dim, output_length=output_steps, input_shape=(input_steps, input_dim), depth=(encoder_stack, decoder_stack), bidirectional=True) sgd = optimizers.SGD(lr=0.0001, decay=1e-6, momentum=0.9, nesterov=False) model.compile(loss='mse', optimizer=sgd) return model
def create_model(self): self.model = AttentionSeq2Seq( self.output_dim, self.output_length, input_dim=self.input_dim, # input_length=self.input_length, # hidden_dim=self.input_length, depth=self.depth) self.model.compile(loss=self.loss, optimizer=self.optimizer) return self.model
def create_model(self, mode, params=None): params_ = AttentionSeq2Seq.default_params().copy() params_.update(TEST_PARAMS) params_.update({ "source.reverse": True, "vocab_source": self.vocab_file.name, "vocab_target": self.vocab_file.name, }) params_.update(params or {}) return AttentionSeq2Seq(params=params_, mode=mode)
def attention_seq2seq_model(input_dim, hidden_dim, output_length, output_dim, input_length): model = Sequential() model_seq = AttentionSeq2Seq(input_dim=input_dim, hidden_dim=hidden_dim, output_length=output_length, output_dim=output_dim, input_length=input_length) model.add(model_seq) model.add(TimeDistributed(Dense(output_dim))) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') #model.compile(loss='categorical_crossentropy', optimizer='rmsprop') return model
def build_model(): model = AttentionSeq2Seq(input_dim=INPUT_DIM, input_length=INPUT_MAX_LENGTH, hidden_dim=HIDDEN_DIM, output_length=OUTPUT_MAX_LENGTH, output_dim=OUTPUT_DIM, depth=DEPTH, batch_size=BATCH_SIZE) model.compile(loss='mse', optimizer='sgd', sample_weight_mode="temporal") model.summary() return model
def create_att_model(dic_size): model = AttentionSeq2Seq(input_dim=word_size, input_length=encoder_seq_length, hidden_dim=lstm_hidden_dim, output_length=decoder_seq_length, output_dim=word_size) f_model = Sequential() f_model.add(Embedding(input_dim=dic_size, output_dim=word_size)) f_model.add(model) f_model.add(Dense(dic_size, activation='softmax')) f_model.summary() return f_model
def attention_model_recurrent_shop(n_timesteps_in, n_timesteps_out, n_features): model = AttentionSeq2Seq(input_dim=n_features, input_length=n_timesteps_in, hidden_dim=100, output_length=n_timesteps_out, output_dim=n_features, depth=1) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc']) return model
def create_attention_model(X_vocab_len, X_max_len, y_vocab_len, y_max_len, hidden_dim, layer_num, learning_rate, dropout): model = Sequential() model.add(AttentionSeq2Seq(output_dim=y_vocab_len, hidden_dim=hidden_dim, output_length=y_max_len, input_shape=(X_max_len, X_vocab_len), bidirectional=False, depth=layer_num, dropout=dropout)) model.add(TimeDistributed(Dense(y_vocab_len, activation='softmax'))) opt = RMSprop(lr=learning_rate) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) return model
def test_model(source_path, target_path, vocab_path): tf.logging.set_verbosity(tf.logging.INFO) batch_size = 2 # Build model graph mode = tf.contrib.learn.ModeKeys.TRAIN params_ = AttentionSeq2Seq.default_params().copy() params_.update({ "vocab_source": vocab_path, "vocab_target": vocab_path, }) model = AttentionSeq2Seq(params=params_, mode=mode) tf.logging.info(vocab_path) input_pipeline_ = input_pipeline.ParallelTextInputPipeline(params={ "source_files": [source_path], "target_files": [target_path] }, mode=mode) input_fn = training_utils.create_input_fn(pipeline=input_pipeline_, batch_size=batch_size) features, labels = input_fn() fetches = model(features, labels, None) fetches = [_ for _ in fetches if _ is not None] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) sess.run(tf.tables_initializer()) with tf.contrib.slim.queues.QueueRunners(sess): fetches_ = sess.run(fetches) return model, fetches_
def create_attention_model(model_filename, params_filename): input_token_index,target_token_index,\ input_characters,target_characters,\ max_encoder_seq_length,num_encoder_tokens,\ max_decoder_seq_length,num_decoder_tokens=get_parameters_from_file(params_filename) model = AttentionSeq2Seq(input_dim=num_encoder_tokens, input_length=max_encoder_seq_length, hidden_dim=latent_dim, output_length=max_decoder_seq_length, output_dim=num_decoder_tokens, depth=LSTM_LAYERS, dropout=DROPOUT) model.compile(loss='mse', optimizer='rmsprop', metrics=['acc']) model.save(model_filename)
def default_params(): """call in configurable class, return default params """ params = AttentionSeq2Seq.default_params().copy() params.update({ "pointer_gen": True, "coverage": True, "embedding.share": True, "attention.class": "AttentionLayerBahdanau", "attention.params": {}, # Arbitrary attention layer parameters "bridge.class": "seq2seq.models.bridges.ZeroBridge", "encoder.class": "seq2seq.encoders.BidirectionalRNNEncoder", "encoder.params": {}, # Arbitrary parameters for the encoder "decoder.class": "seq2seq.decoders.CopyGenDecoder", "decoder.params": {} # Arbitrary parameters for the decoder }) return params
def create_model(): model = Sequential() # model.add(LSTM(128,input_shape = (5,660), return_sequences = True, activation='tanh')) # model.add(LSTM(512, return_sequences=True, dropout=0.2, activation='tanh')) # model.add(LSTM(256, return_sequences=True, activation='tanh')) # model.add(LSTM(128, return_sequences=True, dropout=0.5, activation='tanh')) # model.add(LSTM(64, return_sequences=True, dropout=0.2, activation='tanh')) # model.add(LSTM(128, return_sequences=False, activation='tanh')) # model.add(GRU(128, input_shape = (5,660),return_sequences=True, activation='relu')) # model.add(GRU(128, input_shape = (30,770), return_sequences=True, activation='tanh')) # model.add(GRU(256, input_shape = (30,770), return_sequences=False, activation='tanh')) # model.add(GRU(128, input_shape = (6,1246), return_seqences=False, activation='relu')) model = AttentionSeq2Seq(input_dim=660, input_length=5 * 3, hidden_dim=64, output_length=1, output_dim=64, depth=(3, 3), bidirectional=False) # model.add(Dropout(0.8)) # model.add(Flatten()) # model.add(Dense(128, activation='relu')) # model.add(Dense(64, activation='relu')) # model.add(Dense(5, activation='softmax')) a = model.inputs d0 = BatchNormalization()(model.outputs[-1]) # drop = Dropout(0.5)(d0) d1 = Dense(5, activation='tanh')(d0) d2 = Dense(5, activation='softmax')(d1) model = Model(inputs=a, outputs=d2) sgd = SGD(lr=0.000001, decay=1e-8, momentum=0.9, nesterov=True) adam = Adam(lr=0.00005, clipnorm=4) rmsprop = RMSprop(lr=0.0007, rho=0.9, epsilon=None, decay=0.0) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) print model.summary() return model
def main_train(): # x = np.random.random((samples, input_length, input_dim)) # y = np.random.random((samples, output_length, output_dim)) checkpoint = ModelCheckpoint('model_best_weights.h5', monitor='loss', verbose=1, save_best_only=True, mode='min', period=10) model = AttentionSeq2Seq(output_dim=n_dim, hidden_dim=hidden_dim, output_length=output_length, input_shape=(input_length, n_dim), depth=4) model.compile(loss='mse', optimizer='sgd') model.fit_generator(generator=nasdaq_generator, use_multiprocessing=True, workers=5, epochs=100000, callbacks=[checkpoint])
def compile_model(input_length, input_depth, input_dim, hidden_dim, output_length, output_depth, output_dim, peek, attention, loss, optimizer): """ Returns a compiled seq2seq model Arguments: input_length - Batch size in which to partition the elements input_depth - the number of layers in encoder input_dim - the number of features for each word hidden_dim - number of hidden units output_length - (= maximum_output_length) The maximum number of words in output output_depth - the number of layers in decoder output_dim - the number of features in word embedding's output peek - (binray) add the peek feature attention - (binary) use attention model loss - (string) the loss function, one of keras options optimizer - (string) the optimizer function, one of keras options """ # Only pass peek as an argument if using an attention model model_fn = lambda **args: \ AttentionSeq2Seq(**args) \ if attention \ else Seq2Seq(peek = peek, **args) model = model_fn( input_length=input_length, input_dim=input_dim, hidden_dim=hidden_dim, output_length=output_length, output_dim=output_dim, depth=(input_depth, output_depth), ) model.compile(loss=loss, optimizer=optimizer) if logging.getLogger().getEffectiveLevel() == logging.DEBUG: model.summary() return model
vector = vector[len(vector) - length::] else: for _ in range(length - len(vector)): vector.append(model_wv["。"]) return vector def vec2line(vector): text = "" for v in vector: text += model_wv.most_similar([v], [], 1)[0][0] return text model = AttentionSeq2Seq(input_dim=200, input_length=20, output_length=20, output_dim=200) model.compile(loss='mse', optimizer='Adam') model.load_weights("./data/er.hdf5") def talk(text): vector = np.asarray(line2vec(text, length=20)[::-1]) vector = vector.reshape(1, 20, 200) predict = model.predict(vector, batch_size=132) return vec2line(predict.reshape(20, 200)) if __name__ == "__main__": while True: print(talk(input(">")))
def __init__(self, model_name, x_max_len, x_vocab_len, y_max_len, y_vocab_len, hidden_dim, layer_num, learning_rate, dropout, embedding_dim=0): self.model_name = model_name self.X_max_len = x_max_len self.X_vocab_len = x_vocab_len self.y_max_len = y_max_len self.y_vocab_len = y_vocab_len self.embedding_dim = embedding_dim self.hidden_dim = hidden_dim self.layer_num = layer_num self.learning_rate = learning_rate self.dropout = dropout print('[INFO] Compiling model...') self.model = Sequential() if self.embedding_dim > 0: self.model.add( Embedding(input_length=self.X_max_len, input_dim=self.X_vocab_len, output_dim=embedding_dim, mask_zero=True)) self.model.add( AttentionSeq2Seq(input_length=self.X_max_len, input_dim=embedding_dim, output_length=self.y_max_len, output_dim=self.y_vocab_len, hidden_dim=hidden_dim, bidirectional=False, depth=layer_num, dropout=dropout)) else: self.model.add( AttentionSeq2Seq(input_length=self.X_max_len, input_dim=self.X_vocab_len, output_length=self.y_max_len, output_dim=self.y_vocab_len, hidden_dim=hidden_dim, bidirectional=True, depth=layer_num, dropout=dropout)) self.model.add( TimeDistributed(Dense(units=self.y_vocab_len, activation='softmax'))) opt = RMSprop(lr=learning_rate) self.model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
# In[15]: # attention seq2seq inp = Input((maxlen, )) x = Embedding(vocab_size, dim_en_vec, input_length=maxlen, weights=[en_embs], trainable=False)(inp) x = AttentionSeq2Seq(input_dim=dim_en_vec, input_length=maxlen, hidden_dim=128, output_length=maxlen, output_dim=128, depth=3, bidirectional=True, unroll=False, stateful=False, dropout=0.1)(x) x = TimeDistributed(Dense(dim_fr_vec))(x) x = TimeDistributed(Dense(vocab_size, weights=fr_wgts))(x) x = Activation('softmax')(x) # In[16]: model = Model(inp, x) model.compile('adam', 'sparse_categorical_crossentropy') # In[17]:
def create_model(self): return AttentionSeq2Seq(source_vocab_info=self.vocab_info, target_vocab_info=self.vocab_info, params=AttentionSeq2Seq.default_params())
# THEANO_FLAGS=mode=FAST_RUN,device=cuda0,floatX=float32 python test.py fold1 fl1 1 fl=sys.argv[1] print fl frame_length = int(fl[1]) TIME_STEPS = int(round(308/frame_length)-1) #308 is the image's max height NUM = 64 num_dim = 50*frame_length*2 model = Sequential() model.add(TimeDistributed(Dense(64, activation='relu'), batch_input_shape=(None, TIME_STEPS, num_dim))) model.add(TimeDistributed(Dense(64, activation='relu'))) model.add(AttentionSeq2Seq(input_dim=64, input_length=TIME_STEPS, hidden_dim=64, output_length=25, output_dim=50, depth=1)) model.add(TimeDistributed(Activation('softmax'))) model.compile(loss='sparse_categorical_crossentropy', optimizer='Adam', metrics=['accuracy']) print sys.argv[2] load_weights(model, 'model_%s/epoch_%s.h5'%(fl,sys.argv[2])) text_lst = "$abcdefghijklmnopqrstuvwxyzABCDEGHIJKLMNOPQRSTUVZ#" with open('tree', 'rb') as f: tree = pickle.load(f) # find the next possible character def search_key(string): cur = tree string = '$'+string
def main(): filename = "data/messages.htm" tokenized_pairs, mappings = get_data(filename, verbose=False, load_prior=True) x, y = pad_pairs(tokenized_pairs, SEQUENCE_LENGTH, trim_under=True) input_length = SEQUENCE_LENGTH output_length = SEQUENCE_LENGTH input_dim = 1 output_dim = 1 if LOAD_PRIOR_MODEL: print("Loading model...") model = keras.models.load_model("data/model.h5") else: print("Building model...") model = Sequential() ''' seqtoseq = SimpleSeq2Seq( output_dim = 1, output_length = SEQUENCE_LENGTH, batch_input_shape=(BATCH_SIZE,SEQUENCE_LENGTH,1), depth=3 ) ''' seqtoseq = AttentionSeq2Seq(output_dim=1, output_length=SEQUENCE_LENGTH, batch_input_shape=(BATCH_SIZE, SEQUENCE_LENGTH, 1), depth=1) model.add(seqtoseq) print("Compiling model...") model.compile(loss='mse', optimizer='sgd') print("Fitting model...") model.fit(x, y, batch_size=BATCH_SIZE, nb_epoch=1) ''' for i in range(int(len(x)/BATCH_SIZE)): print("Dataset "+(str(i))) cur_x = np.empty([BATCH_SIZE,1,SEQUENCE_LENGTH]) cur_y = np.empty([BATCH_SIZE,1,SEQUENCE_LENGTH]) for j in range(BATCH_SIZE): cur_x[j][0] = x[i*BATCH_SIZE+j][0] cur_y[j][0] = y[i*BATCH_SIZE+j][0] model.fit(cur_x,cur_y,batch_size=BATCH_SIZE,nb_epoch=5) ''' #model.fit(x,y,batch_size=BATCH_SIZE,nb_epoch=5) #model.fit(x,y,batch_size=len(x),nb_epoch=1,show_accuracy=True,verbose=1) print("Saving model...") model.save("data/model.h5") while True: query = input("You: ") if query == "exit": break query = convert_sentence_to_ids(query, mappings) response = model.predict(pad(query)) print(response) print("\nDone.")
print(x_train.shape) print(y_train.shape) print('Validation Data:') print(x_val.shape) print(y_val.shape) # Try replacing GRU, or SimpleRNN. HIDDEN_SIZE = 128 BATCH_SIZE = 128 LAYERS = 3 model = AttentionSeq2Seq(input_dim=MAX_NUM_WORDS, input_length=MAXLEN, hidden_dim=128, output_length=MAXLEN - 5, output_dim=MAX_NUM_WORDS, depth=4) model.compile(loss='mse', optimizer='rmsprop') print(model.summary()) model.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=EPOCHS, validation_data=(x_val, y_val)) model.save('models/seq2seq.h5') model_l = load_model('models/seq2seq.h5') #test model
def Seq2SeqModel(self): """ Function can input the data and train the model seq2seq. :return: """ preprocess = Preprocess() # X_train, y_train, X_test, y_test, X_validation, y_validation = np.array() print("1. Generate database without Completion time") print("2. Generate database with Completion time") print("3. Training without Completion time") print("4. Training with Completion time") print("Your choice: ") choice = input() if choice == '1': preprocess.preprcessingData("../database/base.txt", size) return None elif choice == '2': preprocess.preprcessingDatawithC("../database/base.txt", size) return None elif choice == '3': input_dim = 2 X_train, y_train, X_test, y_test, X_validation, y_validation = preprocess.divideData(size, num_instance) elif choice == '4': input_dim = 4 X_train, y_train, X_test, y_test, X_validation, y_validation = preprocess.divideDatawithC(size, num_instance) # test with one instance # X_train = [] # y_train = [] # X_validation = [] # y_validation = [] # # with open('test2.csv') as data: # reader = csv.reader(data) # dataSet = list(reader) # trainning_set = dataSet[:32] # #print(type(trainning_set)) # validation_set = dataSet[32:44] # #print(validation_set) # # for line in trainning_set: # ptimes_list, solved_list = preprocess.saveLinewithC(line) # # ptimes_list = preprocessing.scale(ptimes_list) # X_train.append(ptimes_list) # y_train.append(solved_list) # # X_train = np.asarray(X_train) # # print(X_train) # y_train = np.asarray(y_train) # y_train = np.reshape(y_train, (len(y_train), 100, 1)) # # for l in validation_set: # ptimes_lis, solved_lis = preprocess.saveLinewithC(l) # X_validation.append(ptimes_lis) # y_validation.append(solved_lis) # # #print(X_validation) # # X_validation = np.asarray(X_validation) # #print(X_validation) # y_validation = np.asarray(y_validation) # y_validation = np.reshape(y_validation, (len(y_validation), 100, 1)) # print(X_validation) # print(y_validation) # print(X_train[0]) # print(y_train[0]) # X_train = np.random.random((samples, input_length, input_dim)) # y_train = np.random.random((samples, output_length, 3)) # y_train = np.reshape(y_train, (len(y_train), 100, 1)) # y_train = np.argmax(y_train, axis=2) # print(X_train[1]) # print(y_train[1]) model = AttentionSeq2Seq(output_dim=output_dim, hidden_dim=hidden_dim, output_length=output_length, input_shape=(input_length, input_dim), depth=1) # print(model.summary()) adam = optimizers.adam(decay=1e-6) seq2seqnn = SeqSeqNN() # model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy', WinAcc, OutWinAcc]) # model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', WinAcc, OutWinAcc]) # model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy', WinAcc, OutWinAcc]) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', seq2seqnn.WinAcc,seq2seqnn.WinAcc,seq2seqnn.OutWinAcc,]) YY = [] for y in y_train: YY.append(to_categorical(y)) YY = np.asarray(YY) YYv = [] for yv in y_validation: YYv.append(to_categorical(yv)) YYv = np.asarray(YYv) YYt = [] for yt in y_test: YYt.append(to_categorical(yt)) YYt = np.asarray(YYt) # checkpoint filepath = "best_weights.h5" checkpoint = ModelCheckpoint(filepath, monitor='val_WinAcc', save_best_only=True, verbose=0, mode='auto') callbacks_list = [checkpoint] # history = model.fit(X_train, YY, validation_data=(X_validation, YYv), epochs=2, batch_size=64, # callbacks=callbacks_list) maxfeatures = np.max(np.max(X_train, axis=1), axis=0) meanfeatures = np.mean(np.mean(X_train, axis=1), axis=0) stdfeatures = np.mean(np.std(X_train, axis=1), axis=0) # print(maxfeatures.shape) # print(maxfeatures) # print(meanfeatures) # print(stdfeatures) maxfeatures = 1 / maxfeatures # X_train=np.multiply(maxfeatures,X_train) # X_train=(X_train-meanfeatures)/stdfeatures maxfeatures = np.max(np.max(X_train, axis=1), axis=0) # print(maxfeatures) # input() # history = model.fit(X_train, YY, epochs=50000, validation_data=(X_validation, YYv), batch_size=64, # callbacks=callbacks_list) history = model.fit(X_train, YY, validation_data=(X_validation, YYv), epochs=5000, batch_size=64, callbacks=callbacks_list) # predict_test = model.predict(X_test) # print(predict_test) # predict = K.argmax(predict_test, axis=2) # print(K.get_value(predict).shape) # print(predict_test[10]) # print("------") # print(y_test.shape) # list all data in history print(history.history.keys()) # summarize history for accuracy # plt.plot(history.history['acc']) # summarize history for loss plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.title('model loss') plt.ylabel('loss') plt.xlabel('epoch') plt.legend(['train', 'test'], loc='upper left') plt.show() plt.plot(history.history['WinAcc']) plt.plot(history.history['val_WinAcc']) plt.title('model WinAcc') plt.ylabel('WinAcc') plt.xlabel('epoch') plt.legend(['train', 'test'], loc='upper left') plt.show() plt.plot(history.history['OutWinAcc']) plt.plot(history.history['val_OutWinAcc']) plt.title('model OutWinAcc') plt.ylabel('OutWinAcc') plt.xlabel('epoch') plt.legend(['train', 'test'], loc='upper left') plt.show()