def Attention():
# https://github.com/farizrahman4u/seq2seq or https://github.com/datalogue/keras-attention
import seq2seq
from seq2seq.models import AttentionSeq2Seq
input_q = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='float64')
q = embedding()(input_q)
q = AttentionSeq2Seq(input_dim=EMBEDDING_DIM, input_length=MAX_SEQUENCE_LENGTH,
output_length=MAX_SEQUENCE_LENGTH, output_dim=QA_EMBED_SIZE,
depth=1)(q)
q = Flatten()(q)
q = Dropout(DROPOUT_RATE)(q)
input_a = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='float64')
a = embedding()(input_a)
a = AttentionSeq2Seq(input_dim=EMBEDDING_DIM, input_length=MAX_SEQUENCE_LENGTH,
output_length=MAX_SEQUENCE_LENGTH, output_dim=QA_EMBED_SIZE,
depth=1)(a)
a = Flatten()(a)
a = Dropout(DROPOUT_RATE)(a)
merged = concatenate([q, a])
merged = Dense(512)(merged)
merged = BatchNormalization()(merged)
merged = Activation('relu')(merged)
merged = Dropout(DROPOUT_RATE)(merged)
merged = Dense(1, activation="sigmoid")(merged)
model = Model([input_q, input_a], [merged])
return model
def test_AttentionSeq2Seq():
x = np.random.random((samples, input_length, input_dim))
y = np.random.random((samples, output_length, output_dim))
models = []
models += [
AttentionSeq2Seq(output_dim=output_dim,
hidden_dim=hidden_dim,
output_length=output_length,
input_shape=(input_length, input_dim))
]
models += [
AttentionSeq2Seq(output_dim=output_dim,
hidden_dim=hidden_dim,
output_length=output_length,
input_shape=(input_length, input_dim),
depth=2)
]
models += [
AttentionSeq2Seq(output_dim=output_dim,
hidden_dim=hidden_dim,
output_length=output_length,
input_shape=(input_length, input_dim),
depth=3)
]
for model in models:
model.compile(loss='mse', optimizer='sgd')
model.fit(x, y, epochs=1)
def create_model(self, params=None):
params_ = AttentionSeq2Seq.default_params().copy()
params_.update({"source.reverse": True})
params_.update(params or {})
return AttentionSeq2Seq(source_vocab_info=self.vocab_info,
target_vocab_info=self.vocab_info,
params=params_)
def create_model(self, mode, params=None):
params_ = AttentionSeq2Seq.default_params().copy()
params_.update(TEST_PARAMS)
params_.update({
"source.reverse": True,
"vocab_source": self.vocab_file.name,
"vocab_target": self.vocab_file.name,
})
params_.update(params or {})
return AttentionSeq2Seq(params=params_, mode=mode)
def default_params(): params = AttentionSeq2Seq.default_params() params["discriminator_units"] = 256 params["discriminator_loss_multiplier"] = 10.0 params["discriminator_reverse_grad"] = False params["discriminator_mix_context"] = False return params
def main():
encoder_input_data, decoder_input_data, decoder_target_data = get_data()
encoder_input_data = np.array(
[convert_chord_indices_to_embeddings(c) for c in encoder_input_data])
decoder_input_data = to_categorical(decoder_input_data, num_classes=130)
decoder_target_data = to_categorical(decoder_target_data, num_classes=130)
# model = AttentionSeq2Seq(input_dim=32, hidden_dim=64, output_length=600, output_dim=130)
model = AttentionSeq2Seq(input_dim=32,
input_length=600,
hidden_dim=64,
output_length=600,
output_dim=130)
if not os.path.exists('s2s_attention.h5'):
optimizer = Adam(clipnorm=1.0)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['categorical_accuracy'])
model.fit(encoder_input_data,
decoder_input_data,
batch_size=32,
epochs=2)
model.save_weights('s2s_simple.h5')
res = model.predict(np.expand_dims(encoder_input_data[15], axis=0))
print(np.argmax(res, axis=-1))
else:
model.load_weights("s2s_fariz_attention.h5")
output = model.predict(np.expand_dims(encoder_input_data[20], axis=0))
print(np.argmax(output, axis=-1))
def create_imgText(image_shape, max_caption_len, vocab_size):
image_model = Sequential()
# image_shape : C,W,H
# input: 100x100 images with 3 channels -> (3, 100, 100) tensors.
# this applies 32 convolution filters of size 3x3 each.
image_model.add(Convolution2D(32, (3, 3), padding='valid', input_shape=image_shape))
image_model.add(BatchNormalization())
image_model.add(Activation('relu'))
image_model.add(Convolution2D(32, (3, 3)))
image_model.add(BatchNormalization())
image_model.add(Activation('relu'))
image_model.add(MaxPooling2D(pool_size=(2, 2)))
image_model.add(Dropout(0.25))
image_model.add(Convolution2D(64, (3, 3), padding='valid'))
image_model.add(BatchNormalization())
image_model.add(Activation('relu'))
image_model.add(Convolution2D(64, (3, 3)))
image_model.add(BatchNormalization())
image_model.add(Activation('relu'))
image_model.add(MaxPooling2D(pool_size=(2, 2)))
image_model.add(Dropout(0.25))
image_model.add(Flatten())
# Note: Keras does automatic shape inference.
image_model.add(Dense(128))
image_model.add(RepeatVector(1))
model = AttentionSeq2Seq(input_dim=128, input_length=1, hidden_dim=128, output_length=max_caption_len,
output_dim=vocab_size)
# model = Seq2Seq(input_dim=128, input_length=1, hidden_dim=128, output_length=max_caption_len,
# output_dim=128, peek=True)
image_model.add(model)
image_model.add(TimeDistributed(Dense(vocab_size)))
image_model.add(Activation('softmax'))
return image_model
def create_model(in_encoder_vocabulary,
in_decoder_vocabulary,
in_embedding_matrix,
in_input_length,
in_output_length,
in_config,
mode=Mode.TRAIN):
effective_vocabulary_size, embedding_size = in_embedding_matrix.shape
embedding_layer = Embedding(effective_vocabulary_size,
embedding_size,
weights=[in_embedding_matrix],
input_length=in_input_length,
trainable=True,
name='emb')
seq2seq_model = AttentionSeq2Seq(bidirectional=False,
input_dim=embedding_size,
output_dim=len(in_decoder_vocabulary),
hidden_dim=in_config['layer_size'],
output_length=in_output_length,
depth=in_config['max_layers'],
dropout=0.0 if mode == Mode.TEST else 0.2)
model = Sequential()
model.add(embedding_layer)
model.add(seq2seq_model)
model.add(Activation('softmax'))
sgd = SGD(lr=in_config['learning_rate'],
decay=in_config['learning_rate_decay'],
clipvalue=in_config['gradient_clip_value'])
model.compile(loss='categorical_crossentropy', optimizer=sgd)
return model
def train(self, train_names, train_codes, hyperparams, pct_train=0.8, lr=0.01, num_epoch=100):
if self.model_name == 'SimpleSeq2Seq' or self.model_name == 'AttentionSeq2Seq':
train_name, train_code, val_name, val_code, naming_data, hyperparams['n_tokens'] = trainModel.split_data(train_names, train_codes, hyperparams['output_length'], hyperparams['input_length'], pct_train)
hyperparams['is_embedding'] = False
train_name = trainModel.one_hot_name(train_name, hyperparams['n_tokens'])
required_params = ['output_dim', 'output_length', 'input_length', 'is_embedding', 'n_tokens']
for param in required_params:
assert param in hyperparams, (param)
if self.model_name == 'SimpleSeq2Seq':
model = SimpleSeq2Seq(**hyperparams)
elif self.model_name == 'AttentionSeq2Seq':
model = AttentionSeq2Seq(**hyperparams)
elif self.model_name == 'Seq2Seq':
model = Seq2Seq(**hyperparams)
else:
raise TypeError
my_adam = optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
model.compile(optimizer=my_adam, loss='categorical_crossentropy')
print ('fit...')
model.fit(train_code, train_name, epochs=num_epoch)
print ('predict...')
predict_probs = model.predict(val_code)
predict_idx = np.argmax(predict_probs, axis=2)
print('evaluate...')
exact_match, _ = trainModel.exact_match(naming_data, predict_idx, val_name)
precision, recall, f1, _, _ = trainModel.evaluate_tokens(naming_data, predict_idx, val_name)
return model, exact_match, precision, recall, f1, naming_data
def build_model(input_dim, input_length, hidden_dim, output_length, out_dim,
depth):
model = AttentionSeq2Seq(input_dim=input_dim,
input_length=input_length,
hidden_dim=hidden_dim,
output_length=output_length,
output_dim=out_dim,
depth=depth)
model.compile(loss='logcosh', optimizer='adam', metrics=["mae", "mse"])
return model
def attention_s2s_lib():
model = AttentionSeq2Seq(output_dim=output_dim,
hidden_dim=hidden_dim,
output_length=output_steps,
input_shape=(input_steps, input_dim),
depth=(encoder_stack, decoder_stack),
bidirectional=True)
sgd = optimizers.SGD(lr=0.0001, decay=1e-6, momentum=0.9, nesterov=False)
model.compile(loss='mse', optimizer=sgd)
return model
def create_model(self):
self.model = AttentionSeq2Seq(
self.output_dim,
self.output_length,
input_dim=self.input_dim,
# input_length=self.input_length,
# hidden_dim=self.input_length,
depth=self.depth)
self.model.compile(loss=self.loss, optimizer=self.optimizer)
return self.model
def create_model(self, mode, params=None):
params_ = AttentionSeq2Seq.default_params().copy()
params_.update(TEST_PARAMS)
params_.update({
"source.reverse": True,
"vocab_source": self.vocab_file.name,
"vocab_target": self.vocab_file.name,
})
params_.update(params or {})
return AttentionSeq2Seq(params=params_, mode=mode)
def attention_seq2seq_model(input_dim, hidden_dim, output_length, output_dim, input_length):
model = Sequential()
model_seq = AttentionSeq2Seq(input_dim=input_dim, hidden_dim=hidden_dim,
output_length=output_length, output_dim=output_dim,
input_length=input_length)
model.add(model_seq)
model.add(TimeDistributed(Dense(output_dim)))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
#model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
return model
def build_model():
model = AttentionSeq2Seq(input_dim=INPUT_DIM,
input_length=INPUT_MAX_LENGTH,
hidden_dim=HIDDEN_DIM,
output_length=OUTPUT_MAX_LENGTH,
output_dim=OUTPUT_DIM,
depth=DEPTH,
batch_size=BATCH_SIZE)
model.compile(loss='mse', optimizer='sgd', sample_weight_mode="temporal")
model.summary()
return model
def create_att_model(dic_size):
model = AttentionSeq2Seq(input_dim=word_size,
input_length=encoder_seq_length,
hidden_dim=lstm_hidden_dim,
output_length=decoder_seq_length,
output_dim=word_size)
f_model = Sequential()
f_model.add(Embedding(input_dim=dic_size, output_dim=word_size))
f_model.add(model)
f_model.add(Dense(dic_size, activation='softmax'))
f_model.summary()
return f_model
def attention_model_recurrent_shop(n_timesteps_in, n_timesteps_out,
n_features):
model = AttentionSeq2Seq(input_dim=n_features,
input_length=n_timesteps_in,
hidden_dim=100,
output_length=n_timesteps_out,
output_dim=n_features,
depth=1)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['acc'])
return model
def create_attention_model(X_vocab_len, X_max_len, y_vocab_len, y_max_len, hidden_dim, layer_num, learning_rate, dropout):
model = Sequential()
model.add(AttentionSeq2Seq(output_dim=y_vocab_len,
hidden_dim=hidden_dim,
output_length=y_max_len,
input_shape=(X_max_len, X_vocab_len),
bidirectional=False,
depth=layer_num,
dropout=dropout))
model.add(TimeDistributed(Dense(y_vocab_len, activation='softmax')))
opt = RMSprop(lr=learning_rate)
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
return model
def test_model(source_path, target_path, vocab_path):
tf.logging.set_verbosity(tf.logging.INFO)
batch_size = 2
# Build model graph
mode = tf.contrib.learn.ModeKeys.TRAIN
params_ = AttentionSeq2Seq.default_params().copy()
params_.update({
"vocab_source": vocab_path,
"vocab_target": vocab_path,
})
model = AttentionSeq2Seq(params=params_, mode=mode)
tf.logging.info(vocab_path)
input_pipeline_ = input_pipeline.ParallelTextInputPipeline(params={
"source_files": [source_path],
"target_files": [target_path]
},
mode=mode)
input_fn = training_utils.create_input_fn(pipeline=input_pipeline_,
batch_size=batch_size)
features, labels = input_fn()
fetches = model(features, labels, None)
fetches = [_ for _ in fetches if _ is not None]
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
sess.run(tf.tables_initializer())
with tf.contrib.slim.queues.QueueRunners(sess):
fetches_ = sess.run(fetches)
return model, fetches_
def create_attention_model(model_filename, params_filename):
input_token_index,target_token_index,\
input_characters,target_characters,\
max_encoder_seq_length,num_encoder_tokens,\
max_decoder_seq_length,num_decoder_tokens=get_parameters_from_file(params_filename)
model = AttentionSeq2Seq(input_dim=num_encoder_tokens,
input_length=max_encoder_seq_length,
hidden_dim=latent_dim,
output_length=max_decoder_seq_length,
output_dim=num_decoder_tokens,
depth=LSTM_LAYERS,
dropout=DROPOUT)
model.compile(loss='mse', optimizer='rmsprop', metrics=['acc'])
model.save(model_filename)
def default_params():
"""call in configurable class, return default params
"""
params = AttentionSeq2Seq.default_params().copy()
params.update({
"pointer_gen": True,
"coverage": True,
"embedding.share": True,
"attention.class": "AttentionLayerBahdanau",
"attention.params": {}, # Arbitrary attention layer parameters
"bridge.class": "seq2seq.models.bridges.ZeroBridge",
"encoder.class": "seq2seq.encoders.BidirectionalRNNEncoder",
"encoder.params": {}, # Arbitrary parameters for the encoder
"decoder.class": "seq2seq.decoders.CopyGenDecoder",
"decoder.params": {} # Arbitrary parameters for the decoder
})
return params
def create_model():
model = Sequential()
# model.add(LSTM(128,input_shape = (5,660), return_sequences = True, activation='tanh'))
# model.add(LSTM(512, return_sequences=True, dropout=0.2, activation='tanh'))
# model.add(LSTM(256, return_sequences=True, activation='tanh'))
# model.add(LSTM(128, return_sequences=True, dropout=0.5, activation='tanh'))
# model.add(LSTM(64, return_sequences=True, dropout=0.2, activation='tanh'))
# model.add(LSTM(128, return_sequences=False, activation='tanh'))
# model.add(GRU(128, input_shape = (5,660),return_sequences=True, activation='relu'))
# model.add(GRU(128, input_shape = (30,770), return_sequences=True, activation='tanh'))
# model.add(GRU(256, input_shape = (30,770), return_sequences=False, activation='tanh'))
# model.add(GRU(128, input_shape = (6,1246), return_seqences=False, activation='relu'))
model = AttentionSeq2Seq(input_dim=660,
input_length=5 * 3,
hidden_dim=64,
output_length=1,
output_dim=64,
depth=(3, 3),
bidirectional=False)
# model.add(Dropout(0.8))
# model.add(Flatten())
# model.add(Dense(128, activation='relu'))
# model.add(Dense(64, activation='relu'))
# model.add(Dense(5, activation='softmax'))
a = model.inputs
d0 = BatchNormalization()(model.outputs[-1])
# drop = Dropout(0.5)(d0)
d1 = Dense(5, activation='tanh')(d0)
d2 = Dense(5, activation='softmax')(d1)
model = Model(inputs=a, outputs=d2)
sgd = SGD(lr=0.000001, decay=1e-8, momentum=0.9, nesterov=True)
adam = Adam(lr=0.00005, clipnorm=4)
rmsprop = RMSprop(lr=0.0007, rho=0.9, epsilon=None, decay=0.0)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
print model.summary()
return model
def main_train():
# x = np.random.random((samples, input_length, input_dim))
# y = np.random.random((samples, output_length, output_dim))
checkpoint = ModelCheckpoint('model_best_weights.h5',
monitor='loss',
verbose=1,
save_best_only=True,
mode='min',
period=10)
model = AttentionSeq2Seq(output_dim=n_dim,
hidden_dim=hidden_dim,
output_length=output_length,
input_shape=(input_length, n_dim),
depth=4)
model.compile(loss='mse', optimizer='sgd')
model.fit_generator(generator=nasdaq_generator,
use_multiprocessing=True,
workers=5,
epochs=100000,
callbacks=[checkpoint])
def compile_model(input_length, input_depth, input_dim, hidden_dim,
output_length, output_depth, output_dim, peek, attention,
loss, optimizer):
"""
Returns a compiled seq2seq model
Arguments:
input_length - Batch size in which to partition the elements
input_depth - the number of layers in encoder
input_dim - the number of features for each word
hidden_dim - number of hidden units
output_length - (= maximum_output_length) The maximum number of words in output
output_depth - the number of layers in decoder
output_dim - the number of features in word embedding's output
peek - (binray) add the peek feature
attention - (binary) use attention model
loss - (string) the loss function, one of keras options
optimizer - (string) the optimizer function, one of keras options
"""
# Only pass peek as an argument if using an attention model
model_fn = lambda **args: \
AttentionSeq2Seq(**args) \
if attention \
else Seq2Seq(peek = peek, **args)
model = model_fn(
input_length=input_length,
input_dim=input_dim,
hidden_dim=hidden_dim,
output_length=output_length,
output_dim=output_dim,
depth=(input_depth, output_depth),
)
model.compile(loss=loss, optimizer=optimizer)
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
model.summary()
return model
vector = vector[len(vector) - length::]
else:
for _ in range(length - len(vector)):
vector.append(model_wv["。"])
return vector
def vec2line(vector):
text = ""
for v in vector:
text += model_wv.most_similar([v], [], 1)[0][0]
return text
model = AttentionSeq2Seq(input_dim=200,
input_length=20,
output_length=20,
output_dim=200)
model.compile(loss='mse', optimizer='Adam')
model.load_weights("./data/er.hdf5")
def talk(text):
vector = np.asarray(line2vec(text, length=20)[::-1])
vector = vector.reshape(1, 20, 200)
predict = model.predict(vector, batch_size=132)
return vec2line(predict.reshape(20, 200))
if __name__ == "__main__":
while True:
print(talk(input(">")))
def __init__(self,
model_name,
x_max_len,
x_vocab_len,
y_max_len,
y_vocab_len,
hidden_dim,
layer_num,
learning_rate,
dropout,
embedding_dim=0):
self.model_name = model_name
self.X_max_len = x_max_len
self.X_vocab_len = x_vocab_len
self.y_max_len = y_max_len
self.y_vocab_len = y_vocab_len
self.embedding_dim = embedding_dim
self.hidden_dim = hidden_dim
self.layer_num = layer_num
self.learning_rate = learning_rate
self.dropout = dropout
print('[INFO] Compiling model...')
self.model = Sequential()
if self.embedding_dim > 0:
self.model.add(
Embedding(input_length=self.X_max_len,
input_dim=self.X_vocab_len,
output_dim=embedding_dim,
mask_zero=True))
self.model.add(
AttentionSeq2Seq(input_length=self.X_max_len,
input_dim=embedding_dim,
output_length=self.y_max_len,
output_dim=self.y_vocab_len,
hidden_dim=hidden_dim,
bidirectional=False,
depth=layer_num,
dropout=dropout))
else:
self.model.add(
AttentionSeq2Seq(input_length=self.X_max_len,
input_dim=self.X_vocab_len,
output_length=self.y_max_len,
output_dim=self.y_vocab_len,
hidden_dim=hidden_dim,
bidirectional=True,
depth=layer_num,
dropout=dropout))
self.model.add(
TimeDistributed(Dense(units=self.y_vocab_len,
activation='softmax')))
opt = RMSprop(lr=learning_rate)
self.model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
# In[15]:
# attention seq2seq
inp = Input((maxlen, ))
x = Embedding(vocab_size,
dim_en_vec,
input_length=maxlen,
weights=[en_embs],
trainable=False)(inp)
x = AttentionSeq2Seq(input_dim=dim_en_vec,
input_length=maxlen,
hidden_dim=128,
output_length=maxlen,
output_dim=128,
depth=3,
bidirectional=True,
unroll=False,
stateful=False,
dropout=0.1)(x)
x = TimeDistributed(Dense(dim_fr_vec))(x)
x = TimeDistributed(Dense(vocab_size, weights=fr_wgts))(x)
x = Activation('softmax')(x)
# In[16]:
model = Model(inp, x)
model.compile('adam', 'sparse_categorical_crossentropy')
# In[17]:
def create_model(self):
return AttentionSeq2Seq(source_vocab_info=self.vocab_info,
target_vocab_info=self.vocab_info,
params=AttentionSeq2Seq.default_params())
# THEANO_FLAGS=mode=FAST_RUN,device=cuda0,floatX=float32 python test.py fold1 fl1 1
fl=sys.argv[1]
print fl
frame_length = int(fl[1])
TIME_STEPS = int(round(308/frame_length)-1) #308 is the image's max height
NUM = 64
num_dim = 50*frame_length*2
model = Sequential()
model.add(TimeDistributed(Dense(64, activation='relu'), batch_input_shape=(None, TIME_STEPS, num_dim)))
model.add(TimeDistributed(Dense(64, activation='relu')))
model.add(AttentionSeq2Seq(input_dim=64, input_length=TIME_STEPS, hidden_dim=64, output_length=25, output_dim=50, depth=1))
model.add(TimeDistributed(Activation('softmax')))
model.compile(loss='sparse_categorical_crossentropy', optimizer='Adam', metrics=['accuracy'])
print sys.argv[2]
load_weights(model, 'model_%s/epoch_%s.h5'%(fl,sys.argv[2]))
text_lst = "$abcdefghijklmnopqrstuvwxyzABCDEGHIJKLMNOPQRSTUVZ#"
with open('tree', 'rb') as f:
tree = pickle.load(f)
# find the next possible character
def search_key(string):
cur = tree
string = '$'+string
def main():
filename = "data/messages.htm"
tokenized_pairs, mappings = get_data(filename,
verbose=False,
load_prior=True)
x, y = pad_pairs(tokenized_pairs, SEQUENCE_LENGTH, trim_under=True)
input_length = SEQUENCE_LENGTH
output_length = SEQUENCE_LENGTH
input_dim = 1
output_dim = 1
if LOAD_PRIOR_MODEL:
print("Loading model...")
model = keras.models.load_model("data/model.h5")
else:
print("Building model...")
model = Sequential()
'''
seqtoseq = SimpleSeq2Seq(
output_dim = 1,
output_length = SEQUENCE_LENGTH,
batch_input_shape=(BATCH_SIZE,SEQUENCE_LENGTH,1),
depth=3
)
'''
seqtoseq = AttentionSeq2Seq(output_dim=1,
output_length=SEQUENCE_LENGTH,
batch_input_shape=(BATCH_SIZE,
SEQUENCE_LENGTH, 1),
depth=1)
model.add(seqtoseq)
print("Compiling model...")
model.compile(loss='mse', optimizer='sgd')
print("Fitting model...")
model.fit(x, y, batch_size=BATCH_SIZE, nb_epoch=1)
'''
for i in range(int(len(x)/BATCH_SIZE)):
print("Dataset "+(str(i)))
cur_x = np.empty([BATCH_SIZE,1,SEQUENCE_LENGTH])
cur_y = np.empty([BATCH_SIZE,1,SEQUENCE_LENGTH])
for j in range(BATCH_SIZE):
cur_x[j][0] = x[i*BATCH_SIZE+j][0]
cur_y[j][0] = y[i*BATCH_SIZE+j][0]
model.fit(cur_x,cur_y,batch_size=BATCH_SIZE,nb_epoch=5)
'''
#model.fit(x,y,batch_size=BATCH_SIZE,nb_epoch=5)
#model.fit(x,y,batch_size=len(x),nb_epoch=1,show_accuracy=True,verbose=1)
print("Saving model...")
model.save("data/model.h5")
while True:
query = input("You: ")
if query == "exit":
break
query = convert_sentence_to_ids(query, mappings)
response = model.predict(pad(query))
print(response)
print("\nDone.")
print(x_train.shape)
print(y_train.shape)
print('Validation Data:')
print(x_val.shape)
print(y_val.shape)
# Try replacing GRU, or SimpleRNN.
HIDDEN_SIZE = 128
BATCH_SIZE = 128
LAYERS = 3
model = AttentionSeq2Seq(input_dim=MAX_NUM_WORDS,
input_length=MAXLEN,
hidden_dim=128,
output_length=MAXLEN - 5,
output_dim=MAX_NUM_WORDS,
depth=4)
model.compile(loss='mse', optimizer='rmsprop')
print(model.summary())
model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
validation_data=(x_val, y_val))
model.save('models/seq2seq.h5')
model_l = load_model('models/seq2seq.h5')
#test model
def Seq2SeqModel(self):
"""
Function can input the data and train the model seq2seq.
:return:
"""
preprocess = Preprocess()
# X_train, y_train, X_test, y_test, X_validation, y_validation = np.array()
print("1. Generate database without Completion time")
print("2. Generate database with Completion time")
print("3. Training without Completion time")
print("4. Training with Completion time")
print("Your choice: ")
choice = input()
if choice == '1':
preprocess.preprcessingData("../database/base.txt", size)
return None
elif choice == '2':
preprocess.preprcessingDatawithC("../database/base.txt", size)
return None
elif choice == '3':
input_dim = 2
X_train, y_train, X_test, y_test, X_validation, y_validation = preprocess.divideData(size, num_instance)
elif choice == '4':
input_dim = 4
X_train, y_train, X_test, y_test, X_validation, y_validation = preprocess.divideDatawithC(size,
num_instance)
# test with one instance
# X_train = []
# y_train = []
# X_validation = []
# y_validation = []
#
# with open('test2.csv') as data:
# reader = csv.reader(data)
# dataSet = list(reader)
# trainning_set = dataSet[:32]
# #print(type(trainning_set))
# validation_set = dataSet[32:44]
# #print(validation_set)
#
# for line in trainning_set:
# ptimes_list, solved_list = preprocess.saveLinewithC(line)
# # ptimes_list = preprocessing.scale(ptimes_list)
# X_train.append(ptimes_list)
# y_train.append(solved_list)
#
# X_train = np.asarray(X_train)
# # print(X_train)
# y_train = np.asarray(y_train)
# y_train = np.reshape(y_train, (len(y_train), 100, 1))
#
# for l in validation_set:
# ptimes_lis, solved_lis = preprocess.saveLinewithC(l)
# X_validation.append(ptimes_lis)
# y_validation.append(solved_lis)
#
# #print(X_validation)
#
# X_validation = np.asarray(X_validation)
# #print(X_validation)
# y_validation = np.asarray(y_validation)
# y_validation = np.reshape(y_validation, (len(y_validation), 100, 1))
# print(X_validation)
# print(y_validation)
# print(X_train[0])
# print(y_train[0])
# X_train = np.random.random((samples, input_length, input_dim))
# y_train = np.random.random((samples, output_length, 3))
# y_train = np.reshape(y_train, (len(y_train), 100, 1))
# y_train = np.argmax(y_train, axis=2)
# print(X_train[1])
# print(y_train[1])
model = AttentionSeq2Seq(output_dim=output_dim, hidden_dim=hidden_dim, output_length=output_length,
input_shape=(input_length, input_dim), depth=1)
# print(model.summary())
adam = optimizers.adam(decay=1e-6)
seq2seqnn = SeqSeqNN()
# model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy', WinAcc, OutWinAcc])
# model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', WinAcc, OutWinAcc])
# model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy', WinAcc, OutWinAcc])
model.compile(loss='categorical_crossentropy', optimizer='adam',
metrics=['accuracy', seq2seqnn.WinAcc,seq2seqnn.WinAcc,seq2seqnn.OutWinAcc,])
YY = []
for y in y_train:
YY.append(to_categorical(y))
YY = np.asarray(YY)
YYv = []
for yv in y_validation:
YYv.append(to_categorical(yv))
YYv = np.asarray(YYv)
YYt = []
for yt in y_test:
YYt.append(to_categorical(yt))
YYt = np.asarray(YYt)
# checkpoint
filepath = "best_weights.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_WinAcc', save_best_only=True, verbose=0, mode='auto')
callbacks_list = [checkpoint]
# history = model.fit(X_train, YY, validation_data=(X_validation, YYv), epochs=2, batch_size=64,
# callbacks=callbacks_list)
maxfeatures = np.max(np.max(X_train, axis=1), axis=0)
meanfeatures = np.mean(np.mean(X_train, axis=1), axis=0)
stdfeatures = np.mean(np.std(X_train, axis=1), axis=0)
# print(maxfeatures.shape)
# print(maxfeatures)
# print(meanfeatures)
# print(stdfeatures)
maxfeatures = 1 / maxfeatures
# X_train=np.multiply(maxfeatures,X_train)
# X_train=(X_train-meanfeatures)/stdfeatures
maxfeatures = np.max(np.max(X_train, axis=1), axis=0)
# print(maxfeatures)
# input()
# history = model.fit(X_train, YY, epochs=50000, validation_data=(X_validation, YYv), batch_size=64,
# callbacks=callbacks_list)
history = model.fit(X_train, YY, validation_data=(X_validation, YYv), epochs=5000, batch_size=64,
callbacks=callbacks_list)
# predict_test = model.predict(X_test)
# print(predict_test)
# predict = K.argmax(predict_test, axis=2)
# print(K.get_value(predict).shape)
# print(predict_test[10])
# print("------")
# print(y_test.shape)
# list all data in history
print(history.history.keys())
# summarize history for accuracy
# plt.plot(history.history['acc'])
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
plt.plot(history.history['WinAcc'])
plt.plot(history.history['val_WinAcc'])
plt.title('model WinAcc')
plt.ylabel('WinAcc')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
plt.plot(history.history['OutWinAcc'])
plt.plot(history.history['val_OutWinAcc'])
plt.title('model OutWinAcc')
plt.ylabel('OutWinAcc')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
