def test_model(self):
model_fname = "cnn_model2_weights.h5"
read_train_data = prepare_data(self.vec_path, self.train_data_path,
self.word_vec_size, self.max_q_length,
self.max_option_length,
self.max_opt_count, self.max_sent_para,
self.max_words_sent)
read_val_data = prepare_data(self.vec_path, self.val_data_path,
self.word_vec_size, self.max_q_length,
self.max_option_length,
self.max_opt_count, self.max_sent_para,
self.max_words_sent)
read_test_data = prepare_data(self.vec_path, self.test_data_path,
self.word_vec_size, self.max_q_length,
self.max_option_length,
self.max_opt_count, self.max_sent_para,
self.max_words_sent)
model = sciq_model(self.word_vec_size, self.max_q_length,
self.max_option_length, self.max_opt_count,
self.max_sent_para, self.max_words_sent)
# select model
train_model = model.get_cnn_model3(
) # model.get_cnn_model2(), model.get_cnn_model4()
train_model.load_weights(os.path.join(self.models_path, model_fname))
s1 = train_model.evaluate_generator(read_val_data.read_all_vectors(),
steps=self.validation_steps)
s2 = train_model.evaluate_generator(read_train_data.read_all_vectors(),
steps=self.steps_per_epoch)
s3 = train_model.evaluate_generator(read_test_data.read_all_vectors(),
steps=self.test_steps)
print(s1)
print(s2)
print(s3)
def train_model(self):
#model_load_weights_fname = "cnn_model2_initial_weights.h5"
model_fname = "cnn_model2_weights.h5"
read_train_data = prepare_data(self.word2vec_path,
self.train_data_path,
self.word_vec_size, self.max_q_length,
self.max_option_length,
self.max_opt_count, self.max_sent_para,
self.max_words_sent)
read_val_data = prepare_data(self.word_vec_size, self.val_data_path,
self.word_vec_size, self.max_q_length,
self.max_option_length,
self.max_opt_count, self.max_sent_para,
self.max_words_sent)
read_test_data = prepare_data(self.word2vec_path, self.test_data_path,
self.word_vec_size, self.max_q_length,
self.max_option_length,
self.max_opt_count, self.max_sent_para,
self.max_words_sent)
model = sciq_model(self.word_vec_size, self.max_q_length,
self.max_option_length, self.max_opt_count,
self.max_sent_para, self.max_words_sent)
train_model = model.get_cnn_model2()
#train_model.load_weights(os.path.join(self.models_path,model_load_weights_fname))
checkpointer = ModelCheckpoint(filepath=os.path.join(
self.models_path, model_fname),
verbose=1,
save_best_only=True,
save_weights_only=True)
train_model.fit_generator(
read_train_data.read_all_vectors(),
steps_per_epoch=self.steps_per_epoch,
epochs=self.nb_epoch,
validation_data=read_val_data.read_all_vectors(),
callbacks=[checkpointer],
validation_steps=self.validation_steps,
verbose=1)
#train_model.save_weights(os.path.join(self.models_path,model_fname))
train_model.load_weights(os.path.join(self.models_path, model_fname))
s1 = train_model.evaluate_generator(read_val_data.read_all_vectors(),
steps=self.validation_steps)
s2 = train_model.evaluate_generator(read_train_data.read_all_vectors(),
steps=self.steps_per_epoch)
#s3 = train_model.evaluate_generator(read_test_data.read_all_vectors(),steps=self.test_steps) # For testing on test set
print(s1)
print(s2)
#print(s3)
return train_model, read_val_data
def train_ndq_model(self):
#model_load_weights_fname = "cnn_model2_initial_weights.h5"
model_fname = "cnn_model2_weights.h5"
read_train_data = prepare_data(self.word2vec_path,self.train_data_path,False,self.word_vec_size,self.max_q_length,self.max_option_length,self.max_opt_count,self.max_sent_para,self.max_words_sent)
read_val_data = prepare_data(self.word2vec_path, self.val_data_path,False, self.word_vec_size, self.max_q_length,
self.max_option_length, self.max_opt_count, self.max_sent_para, self.max_words_sent)
read_test_data = prepare_data(self.word2vec_path, self.test_data_path,True, self.word_vec_size, self.max_q_length,
self.max_option_length, self.max_opt_count, self.max_sent_para, self.max_words_sent)
model = tqa_model(self.word_vec_size,self.max_q_length,self.max_option_length,self.max_opt_count,self.max_sent_para,self.max_words_sent)
train_model = model.get_cnn_model2()
#train_model.load_weights(os.path.join(self.models_path,model_load_weights_fname))
train_model.fit_generator(read_train_data.read_all_vectors_for_ndq(),steps_per_epoch=self.steps_per_epoch_ndq,epochs = self.nb_epoch,validation_data=read_val_data.read_all_vectors_for_ndq(),validation_steps=self.validation_steps_ndq,verbose=1)
#train_model.save_weights(os.path.join(self.models_path,model_fname))
s1 = train_model.evaluate_generator(read_val_data.read_all_vectors_for_ndq(),steps=self.validation_steps_ndq)
s2 = train_model.evaluate_generator(read_train_data.read_all_vectors_for_ndq(),steps=self.steps_per_epoch_ndq)
print(s1)
print(s2)
return train_model, read_test_data
def gen_data(pred_period=20, lower_bound="2011-01-01", start="2014-01-01"):
db_type = "sqlite3"
conn = dbop.connect_db(db_type)
cursor = conn.cursor()
df_all, cols_future = prepare_data(cursor,
pred_period=pred_period,
start=lower_bound)
data_period = (df_all.index >= start)
df_all = df_all[data_period]
df_all = df_all[df_all["amt"] != 0]
return df_all, cols_future
def main():
parser = argparse.ArgumentParser(description="sentence Hi_CNN model")
parser.add_argument('--train_flag',
action='store_true',
help='Train or eval')
parser.add_argument('--fine_tune',
action='store_true',
help='Fine tune word embeddings')
parser.add_argument('--embedding',
type=str,
default='word2vec',
help='Word embedding type, word2vec, senna or glove')
parser.add_argument('--embedding_dict',
type=str,
default='glove/glove.6B.50d.txt',
help='Pretrained embedding path')
parser.add_argument(
'--embedding_dim',
type=int,
default=50,
help='Only useful when embedding is randomly initialised')
parser.add_argument(
'--char_embedd_dim',
type=int,
default=30,
help='char embedding dimension if using char embedding')
parser.add_argument('--use_char',
action='store_false',
help='Whether use char embedding or not')
parser.add_argument('--num_epochs',
type=int,
default=50,
help='number of epochs for training')
parser.add_argument('--batch_size',
type=int,
default=1,
help='Number of texts in each batch')
parser.add_argument("-v",
"--vocab-size",
dest="vocab_size",
type=int,
metavar='<int>',
default=4000,
help="Vocab size (default=4000)")
parser.add_argument('--nbfilters',
type=int,
default=100,
help='Num of filters in conv layer')
parser.add_argument('--char_nbfilters',
type=int,
default=20,
help='Num of char filters in conv layer')
parser.add_argument('--filter1_len',
type=int,
default=5,
help='filter length in 1st conv layer')
parser.add_argument(
'--filter2_len',
type=int,
default=3,
help='filter length in 2nd conv layer or char conv layer')
parser.add_argument('--rnn_type',
type=str,
default='LSTM',
help='Recurrent type')
parser.add_argument('--lstm_units',
type=int,
default=100,
help='Num of hidden units in recurrent layer')
# parser.add_argument('--project_hiddensize', type=int, default=100, help='num of units in projection layer')
parser.add_argument(
'--optimizer',
choices=['sgd', 'momentum', 'nesterov', 'adagrad', 'rmsprop'],
help='updating algorithm',
default='rmsprop')
parser.add_argument('--learning_rate',
type=float,
default=0.001,
help='Initial learning rate')
parser.add_argument('--dropout',
type=float,
default=0.5,
help='Dropout rate for layers')
parser.add_argument('--oov',
choices=['random', 'embedding'],
default='embedding',
help="Embedding for oov word")
parser.add_argument('--l2_value', type=float, help='l2 regularizer value')
parser.add_argument('--checkpoint_path',
type=str,
help='checkpoint directory',
default='checkpoints')
parser.add_argument('--train',
type=str,
help='train file',
default='data/fold_0/train.tsv'
) # "data/word-level/*.trpreprocess_asap.pyain"
parser.add_argument('--dev',
type=str,
help='dev file',
default='data/fold_0/dev.tsv')
parser.add_argument('--test',
type=str,
help='test file',
default='data/fold_0/test.tsv')
parser.add_argument('--prompt_id',
type=int,
default=1,
help='prompt id of essay set')
parser.add_argument(
'--init_bias',
action='store_true',
help='init the last layer bias with average score of training data')
parser.add_argument('--mode', type=str, choices=['mot', 'att', 'merged'], default='att', \
help='Mean-over-Time pooling or attention-pooling, or two pooling merged')
args = parser.parse_args()
args.use_char = False
train_flag = args.train_flag
fine_tune = args.fine_tune
USE_CHAR = args.use_char
batch_size = args.batch_size
checkpoint_dir = args.checkpoint_path
num_epochs = args.num_epochs
modelname = "attn-%s.prompt%s.%sfilters.bs%s.hdf5" % (
args.mode, args.prompt_id, args.nbfilters, batch_size)
imgname = "attn-%s.prompt%s.%sfilters.bs%s.png" % (
args.mode, args.prompt_id, args.nbfilters, batch_size)
if USE_CHAR:
modelname = 'char_' + modelname
imgname = 'char_' + imgname
modelpath = os.path.join(checkpoint_dir, modelname)
imgpath = os.path.join(checkpoint_dir, imgname)
datapaths = [args.train, args.dev, args.test]
embedding_path = args.embedding_dict
oov = args.oov
embedding = args.embedding
embedd_dim = args.embedding_dim
prompt_id = args.prompt_id
# debug mode
# debug = True
# if debug:
# nn_model = build_concat_model(args, args.vocab_size, 71, 20, embedd_dim, None, True)
if args.use_char:
(X_train, C_train, Y_train, mask_train), (X_dev, C_dev, Y_dev, mask_dev), (X_test, C_test, Y_test, mask_test), \
vocab, vocab_size, char_vocab, char_vocabsize, embed_table, overal_maxlen, overal_maxnum, maxcharlen, init_mean_value = data_prepare.prepare_data(datapaths, \
embedding_path, embedding, embedd_dim, prompt_id, args.vocab_size, tokenize_text=True, \
to_lower=True, sort_by_len=False, vocab_path=None, score_index=6)
else:
(X_train, Y_train, mask_train), (X_dev, Y_dev, mask_dev), (X_test, Y_test, mask_test), \
vocab, vocab_size, embed_table, overal_maxlen, overal_maxnum, init_mean_value = data_prepare.prepare_sentence_data(datapaths, \
embedding_path, embedding, embedd_dim, prompt_id, args.vocab_size, tokenize_text=True, \
to_lower=True, sort_by_len=False, vocab_path=None, score_index=6)
# print type(embed_table)
if embed_table is not None:
embedd_dim = embed_table.shape[1]
embed_table = [embed_table]
max_sentnum = overal_maxnum
max_sentlen = overal_maxlen
# print embed_table
# print X_train[0, 0:10, :]
# print Y_train[0:10]
# print C_train[0, 0, 0, :], C_train[0, 0, 1, :], C_train[0, 0, -1, :]
X_train = X_train.reshape(
(X_train.shape[0], X_train.shape[1] * X_train.shape[2]))
X_dev = X_dev.reshape((X_dev.shape[0], X_dev.shape[1] * X_dev.shape[2]))
X_test = X_test.reshape(
(X_test.shape[0], X_test.shape[1] * X_test.shape[2]))
logger.info("X_train shape: %s" % str(X_train.shape))
if not args.use_char:
C_train, C_dev, C_test = None, None, None
char_vocabsize = 0
maxcharlen = 0
else:
C_train = C_train.reshape(
(C_train.shape[0],
C_train.shape[1] * C_train.shape[2] * C_train.shape[3]))
C_dev = C_dev.reshape(
(C_dev.shape[0], C_dev.shape[1] * C_dev.shape[2] * C_dev.shape[3]))
C_test = C_test.reshape(
(C_test.shape[0],
C_test.shape[1] * C_test.shape[2] * C_test.shape[3]))
logger.info("C_train shape: %s" % str(C_train.shape))
model = build_hrcnn_model(args, vocab_size, char_vocabsize + 1,
max_sentnum, max_sentlen, maxcharlen, embedd_dim,
embed_table, True, init_mean_value)
evl = Evaluator(args.prompt_id, args.use_char, checkpoint_dir, modelname,
X_train, X_dev, X_test, C_train, C_dev, C_test, Y_train,
Y_dev, Y_test)
# Initial evaluation
logger.info("Initial evaluation: ")
evl.evaluate(model, -1, print_info=True)
logger.info("Train model")
for ii in range(args.num_epochs):
logger.info('Epoch %s/%s' % (str(ii + 1), args.num_epochs))
start_time = time.time()
if args.use_char:
model.fit([X_train, C_train],
Y_train,
batch_size=args.batch_size,
epochs=1,
verbose=0,
shuffle=True)
else:
model.fit(X_train,
Y_train,
batch_size=args.batch_size,
epochs=1,
verbose=0,
shuffle=True)
tt_time = time.time() - start_time
logger.info("Training one epoch in %.3f s" % tt_time)
evl.evaluate(model, ii + 1)
evl.print_info()
evl.print_final_info()
# PLOT_EVERY epoch毎にロスをプロット
if epoch % plot_every == 0:
plot_loss_avg = plot_loss_total / current_batch_size
print("[plot_loss_avg]: {}".format(plot_loss_avg))
plot_losses.append(plot_loss_avg)
plot_loss_total = 0
current_batch_size = 0
# TODO: validationもやる
savePlot(plot_losses, epochs)
torch.save(decoder, 'result/decoder.pt')
if __name__ == '__main__':
input_lang, output_lang, data_lists = data_prepare.prepare_data()
""" print("-------------input_lang-------------")
print("index2word: {}".format(input_lang.index2word))
print("------------output_lang-------------")
print("index2word: {}".format(output_lang.index2word))
print("-------------最終的な教師データ-------------")
print("data_lists: {}".format(data_lists)) """
decoder = DecoderRNN(HIDDEN_SIZE, output_lang.n_words, EMBED_SIZE,
DeviceSingleton.get_instance()).to(
DeviceSingleton.get_instance())
# トレーニング
train(data_lists=data_lists,
p_train_dataset=P_TRAIN_DATASET,
p_validation_dataset=P_VALIDATION_DATASET,
def main():
parser = argparse.ArgumentParser(description="sentence Hi_CNN model")
parser.add_argument('--train_flag', action='store_true', help='Train or eval')
parser.add_argument('--fine_tune', action='store_true', help='Fine tune word embeddings')
parser.add_argument('--embedding', type=str, default='glove', help='Word embedding type, word2vec, senna or glove')
parser.add_argument('--embedding_dict', type=str, default='embeddings/glove.6B.50d.txt', help='Pretrained embedding path')
parser.add_argument('--embedding_dim', type=int, default=64, help='Only useful when embedding is randomly initialised')
parser.add_argument('--num_epochs', type=int, default=50, help='number of epochs for training')
parser.add_argument('--batch_size', type=int, default=10, help='Number of texts in each batch')
parser.add_argument("-v", "--vocab-size", dest="vocab_size", type=int, metavar='<int>', default=4000, help="Vocab size (default=4000)")
parser.add_argument('--nbfilters', type=int, default=100, help='Num of filters in conv layer')
parser.add_argument('--filter1_len', type=int, default=5, help='filter length in 1st conv layer')
parser.add_argument('--filter2_len', type=int, default=3, help='filter length in 2nd conv layer or char conv layer')
parser.add_argument('--rnn_type', type=str, default='LSTM', help='Recurrent type')
parser.add_argument('--lstm_units', type=int, default=100, help='Num of hidden units in recurrent layer')
parser.add_argument('--optimizer', choices=['sgd', 'momentum', 'nesterov', 'adagrad', 'rmsprop'], help='updating algorithm', default='rmsprop')
parser.add_argument('--learning_rate', type=float, default=0.001, help='Initial learning rate')
parser.add_argument('--dropout', type=float, default=0.5, help='Dropout rate for layers')
parser.add_argument('--oov', choices=['random', 'embedding'], default='embedding', help="Embedding for oov word")
parser.add_argument('--l2_value', type=float, help='l2 regularizer value')
parser.add_argument('--checkpoint_path', type=str, default='checkpoints', help='checkpoint directory')
parser.add_argument('--train', type=str, help='train file', default='data/reformed/train.tsv')
parser.add_argument('--dev', type=str, help='dev file', default='data/reformed/valid.tsv')
parser.add_argument('--test', type=str, help='test file', default='data/reformed/test.tsv')
parser.add_argument('--prompt_id', type=int, default=1, help='prompt id of dev essay set')
parser.add_argument('--init_bias', action='store_true', help='init the last layer bias with average score of training data')
parser.add_argument('--mode', type=str, choices=['mot', 'att', 'merged'], default='att', \
help='Mean-over-Time pooling or attention-pooling, or two pooling merged')
args = parser.parse_args()
train_flag = args.train_flag
fine_tune = args.fine_tune
batch_size = args.batch_size
checkpoint_dir = args.checkpoint_path
num_epochs = args.num_epochs
datapaths = [args.train, args.dev, args.test]
embedding_path = args.embedding_dict
oov = args.oov
embedding = args.embedding
embedd_dim = args.embedding_dim
prompt_id = args.prompt_id
(X_train, Y_train, mask_train), (X_dev, Y_dev, mask_dev), (X_test, Y_test, mask_test), \
vocab, vocab_size, embed_table, overal_maxlen, overal_maxnum, init_mean_value = data_prepare.prepare_data(\
datapaths, embedding_path, embedding, embedd_dim, prompt_id, args.vocab_size, tokenize_text=True, to_lower=True, sort_by_len=False, vocab_path=None, score_index=3)
if embed_table is not None:
embedd_dim = embed_table.shape[1]
embed_table = [embed_table]
max_sentnum = overal_maxnum
max_sentlen = overal_maxlen
X_train = X_train.reshape((X_train.shape[0], X_train.shape[1]*X_train.shape[2]))
X_dev = X_dev.reshape((X_dev.shape[0], X_dev.shape[1]*X_dev.shape[2]))
X_test = X_test.reshape((X_test.shape[0], X_test.shape[1]*X_test.shape[2]))
logger.info("X_train shape: %s" % str(X_train.shape))
model = build_cnn_lstm_attention(args, vocab_size, max_sentnum, max_sentlen, embedd_dim, embed_table, True, init_mean_value)
# TODO
modelname="test"
evl = Evaluator(args.prompt_id, checkpoint_dir, modelname, X_train, X_dev, X_test, Y_train, Y_dev, Y_test)
logger.info("Initial evaluation: ")
evl.evaluate(model, -1, print_info=True)
logger.info("Train model")
for ii in range(args.num_epochs):
logger.info('Epoch %s/%s' % (str(ii+1), args.num_epochs))
start_time = time.time()
model.fit(X_train, Y_train, batch_size=args.batch_size, nb_epoch=1, verbose=0, shuffle=True)
tt_time = time.time() - start_time
logger.info("Training one epoch in %.3f s" % tt_time)
evl.evaluate(model, ii+1)
evl.print_info()
evl.print_final_info()