def main(): # generate dataset lines = generate_dataset(100) # create vocab vocab = Vocab() for line in lines: vocab.add_words(line.split(" ")) # transform into big numerical matrix of sentences: numerical_lines = [] for line in lines: numerical_lines.append(vocab(line)) numerical_lines, numerical_lengths = pad_into_matrix(numerical_lines) # construct model & theano functions: model = Model( input_size=10, hidden_size=10, vocab_size=len(vocab), stack_size=1, # make this bigger, but makes compilation slow celltype=RNN # use RNN or LSTM ) model.stop_on(vocab.word2index["."]) # train: for i in range(10000): error = model.update_fun(numerical_lines, numerical_lengths) if i % 100 == 0: print("epoch %(epoch)d, error=%(error).2f" % ({ "epoch": i, "error": error })) if i % 500 == 0: print(vocab(model.greedy_fun(vocab.word2index["the"])))
return X def predict_lstm(X_test, y_test, predictions): predictions = predictions.flatten().clip(0, 1) mae = round(mean_absolute_error(y_test, predictions), 6) print('\n{} VAL MAE: {}'.format("LSTM", mae)) return predictions configs = json.load(open('config.json', 'r')) lstm_path = "good_lstm.h5" model_lstm = Model(configs) model_lstm.load_h5_model('{}{}'.format(models_path, lstm_path)) # A bit ridicolous but whatever X = scale(X) train = DataSet(X=X, y=y) data = DataLoader(None, train) Y = y #del X, y, train X, y = data.get_test_data(seq_len=configs['data']['sequence_length'], normalise=configs['data']['normalise']) predictions_lstm = model_lstm.predict_point_by_point(X)
def split_data(X, y): X_tr, X_test, y_tr, y_test = train_test_split(X, y) train = DataSet(X=X_tr, y=y_tr) test = DataSet(X=X_test, y=y_test) return train, test train, test = split_data(X_train, y_train) data = DataLoader(train, test) configs = json.load(open('config.json', 'r')) input_dim = train.X.shape[1] model = Model(configs) model.build_model(input_dim) x, y = data.get_train_data(seq_len=configs['data']['sequence_length'], normalise=configs['data']['normalise']) # in-memory training model.train(x, y, epochs=configs['training']['epochs'], batch_size=configs['training']['batch_size']) x_test, y_test = data.get_test_data(seq_len=configs['data']['sequence_length'], normalise=configs['data']['normalise']) #predictions1 = model.predict_sequences_multiple(x_test, configs['data']['sequence_length'], configs['data']['sequence_length'])
print('начал работать') cursor.execute( "select posts,code from publication where language=%s and point is null", ('en', )) data = cursor.fetchall() print('загрузил данные -- ', len(data)) # Максимальное количество слов num_words = 5000 # Максимальная длина новости max_news_len = 50 # Количество классов новостей nb_classes = 5 # Создание нейронной сети model_lstm = Model().lstm() model_lstm.load_weights('best_model_lstm.h5') starts = time.time() with open('tokenizer.pickle', 'rb') as handle: tokenizer = pickle.load(handle) async def posts_inst(datas): for i in datas: text = i['posts'] text = reg.sub('', text) text = text.lower() data_text = text.split(' ') list_text = [] for word in data_text: if word not in en_stops and len(word) > 1:
# -*- coding: UTF-8 -*- from lstm_model import Model from trec.dataset import DataSet if __name__ == '__main__': result_file = 'trec_result.txt' print 'loading data...' data = DataSet('./trec/trec-train.txt', './trec/trec-test.txt') print('building model...') model = Model('LSTM for trec', data) print('training with testing...') for i in xrange(1, 400): model.train(result_file)
from lstm_model import LAE as Model, LAE, edit_loss import tensorflow as tf import numpy as np import permutations as permute from load import * import json from tensorflow.keras.preprocessing.sequence import pad_sequences from tensorflow.keras.utils import to_categorical if __name__ == "__main__": names_file = "../../data/publix_cleaned.json" nn = Model(depth=0, hidden_size=512, input_size=39) nn.nn.summary() nn.compile("rmsprop", "categorical_crossentropy", metrics=["accuracy"]) # output = np.zeros((64, 68, 39)) # output1 = np.zeros((64, 69, 39)) # print(nn.nn.predict([output, output1]).shape) # input() training_gen = normal_generator(names_file, batch_size=64) # training_gen = word_generator(names_file, batch_size=128) # (batch, _), _ # (batch, _), _ = next(training_gen) # for b in batch: # word_len = (len(b) - list(reversed(b.tolist())).index(0)) # if word_len == 1:
def scale(X): # normalize the dataset scaler = preprocessing.MinMaxScaler(feature_range=(0, 1)) #scaler = load("scaler_lstm.joblib") X = scaler.fit_transform(X) return X submit = {} models_path = "classifiers/" configs = json.load(open('config.json', 'r')) lstm_path = "good_lstm.h5" model_lstm = Model(configs) model_lstm.load_h5_model('{}{}'.format(models_path, lstm_path)) for file in glob.glob(path_to_test + '*.csv'): seg = os.path.basename(file).replace('feats_1500_', '') feats = pd.read_csv(file, header=[0]) feats = feats.drop(columns=['Unnamed: 0']) """ X = scale(feats) train = DataSet(X=X, y=X[:,0]) data = DataLoader(None, train) X, _ = data.get_test_data( seq_len=configs['data']['sequence_length'], normalise=configs['data']['normalise']
save_fig=True) selected_targets = ['Adj Close'] history_points = 28 pred_points = 1 X_train, X_test, y_train, y_test, scaler_y, y_test_date = train_test_split( data=df_input_data, train_test_split_ratio=0.9, features=selected_features, targets=selected_targets, history_points=history_points, pred_points=pred_points) y_train = y_train.reshape(y_train.shape[0], -1) y_test = y_test.reshape(y_test.shape[0], -1) if TRAIN_MODEL: model = Model(input_time_steps=history_points, input_dim=len(selected_features), output_time_steps=pred_points, output_dim=1) model.build_model() model.train(x=X_train, y=y_train, epochs=20, batch_size=32, save_dir="saved_models") else: model = load_model(SAVE_MODEL_FILE_PATH) if isinstance(model, Sequential): print(model.summary()) y_pred = model.predict(X_test) y_pred = scaler_y.inverse_transform(y_pred.reshape(-1, 1)) y_test = scaler_y.inverse_transform(y_test.reshape(-1, 1))
def test(config): questionid = [] #对应的顺序questionid new_alternatives = [] #更改之后的答案顺序 new_answer_index = [] #真正答案的下标 with open(config.test_file, "r") as fh: lines = fh.readlines() for line in tqdm(lines): new_alternative = [] source = json.loads(line) # 加载训练数据集 questionid.append(source['query_id']) alter_answer_origanl = [ ele.strip() for ele in source['alternatives'].strip().split("|") ] min_word = 'dfsdfdsafdsaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' for temp_alter in alter_answer_origanl: if len(temp_alter) < len(min_word): min_word = temp_alter new_alternative.append(min_word) for temp_alter in alter_answer_origanl: if temp_alter.replace(min_word, '') == '不' or temp_alter.replace( min_word, '') == '没': new_alternative.append(temp_alter) break for temp_alter in alter_answer_origanl: if temp_alter not in new_alternative: new_alternative.append(temp_alter) new_alternative = np.array(new_alternative) new_alternatives.append(new_alternative) with open(config.word2idx_file, "r") as fh: word2idx_dict = json.load(fh) index_word = dict(zip(word2idx_dict.values(), word2idx_dict.keys())) with open(config.word_emb_file, "rb") as fh: word_mat = np.array(pickle.load(fh), dtype=np.float32) # 加载对应的单词以及词向量,加载字符以及对应的词向量 with open(config.char_emb_file, "rb") as fh: char_mat = np.array(pickle.load(fh), dtype=np.float32) # 加载字符向量对应的 # with open(config.test_eval_file, "r") as fh: # eval_file = json.load(fh) with open(config.test_meta, "r") as fh: meta = json.load(fh) total = meta["total"] print("Loading model1111111...") # parser = get_record_parser(config,is_test=True)#加载对应的训练数据特征,与存储特征数据的build_features对应, # test_batch = get_dataset(config.test_record_file, parser, config) test_batch = get_dataset(config.test_record_file, get_record_parser(config, is_test=True), config).make_one_shot_iterator() model = Model(config, test_batch, word_mat=word_mat, char_mat=char_mat, filter_sizes=[3, 4, 5], trainable=False, embedding_size=300, num_filters=128) sess_config = tf.ConfigProto(allow_soft_placement=True) sess_config.gpu_options.allow_growth = True with tf.Session(config=sess_config) as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint(config.save_dir)) # sess.run(tf.assign(model.is_train, tf.constant(True, dtype=tf.bool))) for step in tqdm(range(total // config.batch_size)): ture_pre = sess.run(model.predictions) #qa_id 预测对应的类别下标 new_answer_index.extend(ture_pre) #不改变顺序 print(len(new_answer_index)) with open(config.answer_file, "w") as fh: normal = 0 for ele_k in range(len(questionid)): alter1 = new_alternatives[ele_k] if new_answer_index[ele_k] >= len(alter1): ture_answer = alter1[0] else: normal += 1 ture_answer = alter1[new_answer_index[ele_k]] ture_queid = str(questionid[ele_k]) fh.write(ture_queid) fh.write('\t') fh.write(ture_answer) fh.write('\n') print('the normal quiestion is:', normal)
def train(config): with open(config.word_emb_file, "rb") as fh: print(fh) word_mat = np.array(pickle.load(fh), dtype=np.float32) #加载对应的单词以及词向量,加载字符以及对应的词向量 with open(config.char_emb_file, "rb") as fh: char_mat = np.array(pickle.load(fh), dtype=np.float32) #加载字符向量对应的 # with open(config.train_eval_file, "r") as fh: # train_eval_file = json.load(fh) #"context": context, "spans": spans, "answers": answer_texts, "uuid": qa["id"] # #span 对应的段落单词字符开始结束位置 context:全文内容 ,answer 实际词汇 # with open(config.dev_eval_file, "r") as fh: # dev_eval_file = json.load(fh) with open(config.dev_meta, "r") as fh: meta = json.load(fh) # dev_total = meta["total"] # print("Building model1111111...") parser = get_record_parser(config) #加载对应的训练数据特征,与存储特征数据的build_features对应, train_dataset = get_batch_dataset(config.train_record_file, parser, config) #返回已经batch好的训练数据集,用iterator进行迭代 dev_dataset = get_dataset(config.dev_record_file, parser, config) #对这个数据进行同样的处理 # #把不同的数据dataset的数据feed进模型,首先需要建立iterator handle,既iterator placeholder handle = tf.placeholder(tf.string, shape=[]) iterator = tf.data.Iterator.from_string_handle( handle, train_dataset.output_types, train_dataset.output_shapes) # train_iterator = train_dataset.make_one_shot_iterator( ) #首先提取数据,先对数据构建迭代器iterator dev_iterator = dev_dataset.make_one_shot_iterator() model = Model(config, iterator, word_mat=word_mat, char_mat=char_mat, filter_sizes=[3, 4, 5], embedding_size=300, num_filters=128) sess_config = tf.ConfigProto(allow_soft_placement=True) sess_config.gpu_options.allow_growth = True gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.7) config_sess = tf.ConfigProto(gpu_options=gpu_options) loss_save = 0 patience = 0 lr = config.init_lr with tf.Session(config=config_sess) as sess: fig = plt.figure() ax = fig.add_subplot(211) ax2 = fig.add_subplot(212) plt.grid(True) plt.ion() writer = tf.summary.FileWriter(config.log_dir) ckpt = tf.train.get_checkpoint_state(config.save_dir) if ckpt is not None: print(ckpt.model_checkpoint_path) model.saver.restore(sess, ckpt.model_checkpoint_path) else: sess.run(tf.global_variables_initializer()) train_handle = sess.run(train_iterator.string_handle()) dev_handle = sess.run(dev_iterator.string_handle()) for _ in tqdm(range(1, config.num_steps + 1)): try: global_step = sess.run(model.global_step) + 1 logits, loss, train_op, accuracy = sess.run( [ model.predictions, model.loss, model.train_op, model.accuracy ], feed_dict={handle: train_handle}) # print(logits) # print(np.array(real_y),"hhhhhhhhhhhhhhhhhhhhhhhh") if global_step % 10 == 0: ax.scatter(global_step, loss, c='b', marker='.') plt.pause(0.001) print("the loss is:", loss) print('the accuracy is', accuracy) if global_step % config.period == 0: loss_sum = tf.Summary(value=[ tf.Summary.Value(tag="model/loss", simple_value=loss) ]) writer.add_summary(loss_sum, global_step) if global_step % config.checkpoint == 0: # sess.run(tf.assign(model.is_train,tf.constant(False, dtype=tf.bool))) dev_loss = 0 for k in range(500): dev_loss += sess.run([model.accuracy], feed_dict={handle: dev_handle})[0] dev_loss = dev_loss / 500 ax2.scatter(global_step, dev_loss, c='b', marker='.') plt.pause(0.001) # _, summ = evaluate_batch( # model1111111, config.val_num_batches, train_eval_file, sess, "train", handle, train_handle) # for s in summ: # writer.add_summary(s, global_step) # metrics, summ = evaluate_batch( # model1111111, dev_total // config.batch_size + 1, dev_eval_file, sess, "dev", handle, dev_handle) # sess.run(tf.assign(model.is_train,tf.constant(True, dtype=tf.bool))) # dev_loss = metrics["loss"] if dev_loss > loss_save: print( dev_loss, loss_save, '222222222222222222222222222222222222222222222222222222222222222' ) loss_file = os.path.join(config.save_dir, 'loss_file.txt') with open(loss_file, 'w') as fi: fi.write(str(dev_loss)) fi.write('\t') fi.write(str(loss_save)) fi.write('\n') loss_save = dev_loss filename = os.path.join( config.save_dir, "model_{}.ckpt".format(global_step)) model.saver.save(sess, filename) figure_path = os.path.join(config.save_dir, 'img.png') print(figure_path, "ttttttttttttttttttttttttttttttttttt") plt.savefig(figure_path) # sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32))) # for s in summ: loss_sum = tf.Summary(value=[ tf.Summary.Value(tag="model/loss_dev", simple_value=dev_loss) ]) writer.add_summary(loss_sum, global_step) writer.flush() # filename = os.path.join(config.save_dir, "model_{}.ckpt".format(global_step)) # model.saver.save(sess, filename) except Exception as e: print(e)
'max_grad_norm': int(hparams['max_grad_norm']), #maximum gradient norm during training 'pre_pool_size': int(hparams['pre_pool_size']), 'pre_pool_stride': int(hparams['pre_pool_stride']), 'batch_size': batch_size, 'learning_rate': float(hparams['learning_rate']), 'num_val': num_val, 'num_classes': num_classes, 'weight_vec': weight_vec } epochs = np.floor(batch_size * max_iterations / N) print('Train %.0f samples in approximately %d epochs' % (N, epochs)) #Instantiate a model model = Model(model_config) saver = model.saver """Session time""" sess = tf.Session() #Depending on your use, do not forget to close the session writer = tf.summary.FileWriter(run_dir, sess.graph) #writer for Tensorboard if os.path.isfile(model_path + ".index"): saver.restore(sess, model_path) print("Load Model") else: sess.run(model.init_op) cost_train_ma = -np.log( 1 / float(num_classes) + 1e-9) #Moving average training cost acc_train_ma = 0.0 try: for i in range(max_iterations):
from lstm_model import LAE as Model import numpy as np nn = Model() print(nn) # result = nn.nn.predict(np.zeros((1, 101, 37))) # print(result.shape)