def main():
# generate dataset
lines = generate_dataset(100)
# create vocab
vocab = Vocab()
for line in lines:
vocab.add_words(line.split(" "))
# transform into big numerical matrix of sentences:
numerical_lines = []
for line in lines:
numerical_lines.append(vocab(line))
numerical_lines, numerical_lengths = pad_into_matrix(numerical_lines)
# construct model & theano functions:
model = Model(
input_size=10,
hidden_size=10,
vocab_size=len(vocab),
stack_size=1, # make this bigger, but makes compilation slow
celltype=RNN # use RNN or LSTM
)
model.stop_on(vocab.word2index["."])
# train:
for i in range(10000):
error = model.update_fun(numerical_lines, numerical_lengths)
if i % 100 == 0:
print("epoch %(epoch)d, error=%(error).2f" % ({
"epoch": i,
"error": error
}))
if i % 500 == 0:
print(vocab(model.greedy_fun(vocab.word2index["the"])))
return X
def predict_lstm(X_test, y_test, predictions):
predictions = predictions.flatten().clip(0, 1)
mae = round(mean_absolute_error(y_test, predictions), 6)
print('\n{} VAL MAE: {}'.format("LSTM", mae))
return predictions
configs = json.load(open('config.json', 'r'))
lstm_path = "good_lstm.h5"
model_lstm = Model(configs)
model_lstm.load_h5_model('{}{}'.format(models_path, lstm_path))
# A bit ridicolous but whatever
X = scale(X)
train = DataSet(X=X, y=y)
data = DataLoader(None, train)
Y = y
#del X, y, train
X, y = data.get_test_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
predictions_lstm = model_lstm.predict_point_by_point(X)
def split_data(X, y):
X_tr, X_test, y_tr, y_test = train_test_split(X, y)
train = DataSet(X=X_tr, y=y_tr)
test = DataSet(X=X_test, y=y_test)
return train, test
train, test = split_data(X_train, y_train)
data = DataLoader(train, test)
configs = json.load(open('config.json', 'r'))
input_dim = train.X.shape[1]
model = Model(configs)
model.build_model(input_dim)
x, y = data.get_train_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
# in-memory training
model.train(x,
y,
epochs=configs['training']['epochs'],
batch_size=configs['training']['batch_size'])
x_test, y_test = data.get_test_data(seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise'])
#predictions1 = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
print('начал работать')
cursor.execute(
"select posts,code from publication where language=%s and point is null",
('en', ))
data = cursor.fetchall()
print('загрузил данные -- ', len(data))
# Максимальное количество слов
num_words = 5000
# Максимальная длина новости
max_news_len = 50
# Количество классов новостей
nb_classes = 5
# Создание нейронной сети
model_lstm = Model().lstm()
model_lstm.load_weights('best_model_lstm.h5')
starts = time.time()
with open('tokenizer.pickle', 'rb') as handle:
tokenizer = pickle.load(handle)
async def posts_inst(datas):
for i in datas:
text = i['posts']
text = reg.sub('', text)
text = text.lower()
data_text = text.split(' ')
list_text = []
for word in data_text:
if word not in en_stops and len(word) > 1:
# -*- coding: UTF-8 -*-
from lstm_model import Model
from trec.dataset import DataSet
if __name__ == '__main__':
result_file = 'trec_result.txt'
print 'loading data...'
data = DataSet('./trec/trec-train.txt', './trec/trec-test.txt')
print('building model...')
model = Model('LSTM for trec', data)
print('training with testing...')
for i in xrange(1, 400):
model.train(result_file)
from lstm_model import LAE as Model, LAE, edit_loss
import tensorflow as tf
import numpy as np
import permutations as permute
from load import *
import json
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.utils import to_categorical
if __name__ == "__main__":
names_file = "../../data/publix_cleaned.json"
nn = Model(depth=0, hidden_size=512, input_size=39)
nn.nn.summary()
nn.compile("rmsprop", "categorical_crossentropy", metrics=["accuracy"])
# output = np.zeros((64, 68, 39))
# output1 = np.zeros((64, 69, 39))
# print(nn.nn.predict([output, output1]).shape)
# input()
training_gen = normal_generator(names_file, batch_size=64)
# training_gen = word_generator(names_file, batch_size=128)
# (batch, _), _
# (batch, _), _ = next(training_gen)
# for b in batch:
# word_len = (len(b) - list(reversed(b.tolist())).index(0))
# if word_len == 1:
def scale(X):
# normalize the dataset
scaler = preprocessing.MinMaxScaler(feature_range=(0, 1))
#scaler = load("scaler_lstm.joblib")
X = scaler.fit_transform(X)
return X
submit = {}
models_path = "classifiers/"
configs = json.load(open('config.json', 'r'))
lstm_path = "good_lstm.h5"
model_lstm = Model(configs)
model_lstm.load_h5_model('{}{}'.format(models_path, lstm_path))
for file in glob.glob(path_to_test + '*.csv'):
seg = os.path.basename(file).replace('feats_1500_', '')
feats = pd.read_csv(file, header=[0])
feats = feats.drop(columns=['Unnamed: 0'])
"""
X = scale(feats)
train = DataSet(X=X, y=X[:,0])
data = DataLoader(None, train)
X, _ = data.get_test_data(
seq_len=configs['data']['sequence_length'],
normalise=configs['data']['normalise']
save_fig=True)
selected_targets = ['Adj Close']
history_points = 28
pred_points = 1
X_train, X_test, y_train, y_test, scaler_y, y_test_date = train_test_split(
data=df_input_data,
train_test_split_ratio=0.9,
features=selected_features,
targets=selected_targets,
history_points=history_points,
pred_points=pred_points)
y_train = y_train.reshape(y_train.shape[0], -1)
y_test = y_test.reshape(y_test.shape[0], -1)
if TRAIN_MODEL:
model = Model(input_time_steps=history_points,
input_dim=len(selected_features),
output_time_steps=pred_points,
output_dim=1)
model.build_model()
model.train(x=X_train,
y=y_train,
epochs=20,
batch_size=32,
save_dir="saved_models")
else:
model = load_model(SAVE_MODEL_FILE_PATH)
if isinstance(model, Sequential):
print(model.summary())
y_pred = model.predict(X_test)
y_pred = scaler_y.inverse_transform(y_pred.reshape(-1, 1))
y_test = scaler_y.inverse_transform(y_test.reshape(-1, 1))
def test(config):
questionid = [] #对应的顺序questionid
new_alternatives = [] #更改之后的答案顺序
new_answer_index = [] #真正答案的下标
with open(config.test_file, "r") as fh:
lines = fh.readlines()
for line in tqdm(lines):
new_alternative = []
source = json.loads(line) # 加载训练数据集
questionid.append(source['query_id'])
alter_answer_origanl = [
ele.strip()
for ele in source['alternatives'].strip().split("|")
]
min_word = 'dfsdfdsafdsaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
for temp_alter in alter_answer_origanl:
if len(temp_alter) < len(min_word):
min_word = temp_alter
new_alternative.append(min_word)
for temp_alter in alter_answer_origanl:
if temp_alter.replace(min_word,
'') == '不' or temp_alter.replace(
min_word, '') == '没':
new_alternative.append(temp_alter)
break
for temp_alter in alter_answer_origanl:
if temp_alter not in new_alternative:
new_alternative.append(temp_alter)
new_alternative = np.array(new_alternative)
new_alternatives.append(new_alternative)
with open(config.word2idx_file, "r") as fh:
word2idx_dict = json.load(fh)
index_word = dict(zip(word2idx_dict.values(), word2idx_dict.keys()))
with open(config.word_emb_file, "rb") as fh:
word_mat = np.array(pickle.load(fh),
dtype=np.float32) # 加载对应的单词以及词向量,加载字符以及对应的词向量
with open(config.char_emb_file, "rb") as fh:
char_mat = np.array(pickle.load(fh), dtype=np.float32) # 加载字符向量对应的
# with open(config.test_eval_file, "r") as fh:
# eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
print("Loading model1111111...")
# parser = get_record_parser(config,is_test=True)#加载对应的训练数据特征,与存储特征数据的build_features对应，
# test_batch = get_dataset(config.test_record_file, parser, config)
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config,
test_batch,
word_mat=word_mat,
char_mat=char_mat,
filter_sizes=[3, 4, 5],
trainable=False,
embedding_size=300,
num_filters=128)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
# sess.run(tf.assign(model.is_train, tf.constant(True, dtype=tf.bool)))
for step in tqdm(range(total // config.batch_size)):
ture_pre = sess.run(model.predictions) #qa_id 预测对应的类别下标
new_answer_index.extend(ture_pre) #不改变顺序
print(len(new_answer_index))
with open(config.answer_file, "w") as fh:
normal = 0
for ele_k in range(len(questionid)):
alter1 = new_alternatives[ele_k]
if new_answer_index[ele_k] >= len(alter1):
ture_answer = alter1[0]
else:
normal += 1
ture_answer = alter1[new_answer_index[ele_k]]
ture_queid = str(questionid[ele_k])
fh.write(ture_queid)
fh.write('\t')
fh.write(ture_answer)
fh.write('\n')
print('the normal quiestion is:', normal)
def train(config):
with open(config.word_emb_file, "rb") as fh:
print(fh)
word_mat = np.array(pickle.load(fh),
dtype=np.float32) #加载对应的单词以及词向量,加载字符以及对应的词向量
with open(config.char_emb_file, "rb") as fh:
char_mat = np.array(pickle.load(fh), dtype=np.float32) #加载字符向量对应的
# with open(config.train_eval_file, "r") as fh:
# train_eval_file = json.load(fh) #"context": context, "spans": spans, "answers": answer_texts, "uuid": qa["id"]
# #span 对应的段落单词字符开始结束位置 context:全文内容 ,answer 实际词汇
# with open(config.dev_eval_file, "r") as fh:
# dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
# dev_total = meta["total"]
#
print("Building model1111111...")
parser = get_record_parser(config) #加载对应的训练数据特征,与存储特征数据的build_features对应，
train_dataset = get_batch_dataset(config.train_record_file, parser,
config) #返回已经batch好的训练数据集，用iterator进行迭代
dev_dataset = get_dataset(config.dev_record_file, parser,
config) #对这个数据进行同样的处理
# #把不同的数据dataset的数据feed进模型，首先需要建立iterator handle，既iterator placeholder
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes) #
train_iterator = train_dataset.make_one_shot_iterator(
) #首先提取数据，先对数据构建迭代器iterator
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config,
iterator,
word_mat=word_mat,
char_mat=char_mat,
filter_sizes=[3, 4, 5],
embedding_size=300,
num_filters=128)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.7)
config_sess = tf.ConfigProto(gpu_options=gpu_options)
loss_save = 0
patience = 0
lr = config.init_lr
with tf.Session(config=config_sess) as sess:
fig = plt.figure()
ax = fig.add_subplot(211)
ax2 = fig.add_subplot(212)
plt.grid(True)
plt.ion()
writer = tf.summary.FileWriter(config.log_dir)
ckpt = tf.train.get_checkpoint_state(config.save_dir)
if ckpt is not None:
print(ckpt.model_checkpoint_path)
model.saver.restore(sess, ckpt.model_checkpoint_path)
else:
sess.run(tf.global_variables_initializer())
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
for _ in tqdm(range(1, config.num_steps + 1)):
try:
global_step = sess.run(model.global_step) + 1
logits, loss, train_op, accuracy = sess.run(
[
model.predictions, model.loss, model.train_op,
model.accuracy
],
feed_dict={handle: train_handle})
# print(logits)
# print(np.array(real_y),"hhhhhhhhhhhhhhhhhhhhhhhh")
if global_step % 10 == 0:
ax.scatter(global_step, loss, c='b', marker='.')
plt.pause(0.001)
print("the loss is:", loss)
print('the accuracy is', accuracy)
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss)
])
writer.add_summary(loss_sum, global_step)
if global_step % config.checkpoint == 0:
# sess.run(tf.assign(model.is_train,tf.constant(False, dtype=tf.bool)))
dev_loss = 0
for k in range(500):
dev_loss += sess.run([model.accuracy],
feed_dict={handle: dev_handle})[0]
dev_loss = dev_loss / 500
ax2.scatter(global_step, dev_loss, c='b', marker='.')
plt.pause(0.001)
# _, summ = evaluate_batch(
# model1111111, config.val_num_batches, train_eval_file, sess, "train", handle, train_handle)
# for s in summ:
# writer.add_summary(s, global_step)
# metrics, summ = evaluate_batch(
# model1111111, dev_total // config.batch_size + 1, dev_eval_file, sess, "dev", handle, dev_handle)
# sess.run(tf.assign(model.is_train,tf.constant(True, dtype=tf.bool)))
# dev_loss = metrics["loss"]
if dev_loss > loss_save:
print(
dev_loss, loss_save,
'222222222222222222222222222222222222222222222222222222222222222'
)
loss_file = os.path.join(config.save_dir,
'loss_file.txt')
with open(loss_file, 'w') as fi:
fi.write(str(dev_loss))
fi.write('\t')
fi.write(str(loss_save))
fi.write('\n')
loss_save = dev_loss
filename = os.path.join(
config.save_dir,
"model_{}.ckpt".format(global_step))
model.saver.save(sess, filename)
figure_path = os.path.join(config.save_dir, 'img.png')
print(figure_path,
"ttttttttttttttttttttttttttttttttttt")
plt.savefig(figure_path)
# sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32)))
# for s in summ:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss_dev",
simple_value=dev_loss)
])
writer.add_summary(loss_sum, global_step)
writer.flush()
# filename = os.path.join(config.save_dir, "model_{}.ckpt".format(global_step))
# model.saver.save(sess, filename)
except Exception as e:
print(e)
'max_grad_norm':
int(hparams['max_grad_norm']), #maximum gradient norm during training
'pre_pool_size': int(hparams['pre_pool_size']),
'pre_pool_stride': int(hparams['pre_pool_stride']),
'batch_size': batch_size,
'learning_rate': float(hparams['learning_rate']),
'num_val': num_val,
'num_classes': num_classes,
'weight_vec': weight_vec
}
epochs = np.floor(batch_size * max_iterations / N)
print('Train %.0f samples in approximately %d epochs' % (N, epochs))
#Instantiate a model
model = Model(model_config)
saver = model.saver
"""Session time"""
sess = tf.Session() #Depending on your use, do not forget to close the session
writer = tf.summary.FileWriter(run_dir, sess.graph) #writer for Tensorboard
if os.path.isfile(model_path + ".index"):
saver.restore(sess, model_path)
print("Load Model")
else:
sess.run(model.init_op)
cost_train_ma = -np.log(
1 / float(num_classes) + 1e-9) #Moving average training cost
acc_train_ma = 0.0
try:
for i in range(max_iterations):
from lstm_model import LAE as Model import numpy as np nn = Model() print(nn) # result = nn.nn.predict(np.zeros((1, 101, 37))) # print(result.shape)