def main(): config = Config() print('Prepare data for train and dev ... ') train_dev_split(config.original_file, config.train_file, config.dev_file, config.vocab_file, config.split_ratio) print('Prepare data sucessfully!') lstm_config = Config() blstm_model = LSTM_Dynamic(lstm_config) with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: if lstm_config.if_test: print('Start load dev data ...') (X_test, y_test) = load_data(lstm_config.dev_file) print('Loading sucess!') if len(X_test) < lstm_config.batch_size: for i in range(0, lstm_config.batch_size - len(X_test)): X_test.append([0]) y_test.append([0]) seq_len_test = list(map(lambda x: len(x), X_test)) print('Target to special model to test') test_model = os.path.join(lstm_config.model_path, "models_epoch38") print('Start do predicting...') blstm_model.test(sess, test_model, X_test, y_test, seq_len_test, lstm_config.vocab_file, lstm_config.model_path + 'result/')
def inference_main(contents): ''' :param contents: 一维列表,每行是一个字符串 :return: ''' config = Config() config.batch_size = 10000 config.if_inference = True print('Prepare data for train and dev ... ') train_dev_split(config.original_file, config.train_file, config.dev_file, config.vocab_file, config.split_ratio) print('Prepare data sucessfully!') blstm_model = LSTM_Dynamic(config) gpu_options =tf.GPUOptions(per_process_gpu_memory_fraction=0.5, allow_growth=True) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,allow_soft_placement=True)) as sess: if config.if_inference: x_infer = prepare_modelinputs(contents,y_flag=False) if len(x_infer) < config.batch_size: for i in range(0, config.batch_size - len(x_infer)): x_infer.append([0]) print('Target to special model to test') test_model = os.path.join(config.model_path, "models_epoch39") print('Start do predicting...','\n'*10) saver = tf.train.Saver(tf.trainable_variables()) saver.restore(sess, test_model) y_pred = infer(blstm_model, sess, x_infer, list(map(len, x_infer))) return y_pred
def main(): config = Config() print('Prepare data for train and dev ... ') train_dev_split(config.original_file, config.train_file, config.dev_file, config.vocab_file, config.split_ratio) print('Prepare data sucessfully!') lstm_config = Config() rnn_model = LSTM_Dynamic(lstm_config) gpu_options = tf.GPUOptions(allow_growth=True) gpu_options =tf.GPUOptions(per_process_gpu_memory_fraction=0.5, allow_growth=True) ##每个gpu占用0.8 的显存 config=tf.ConfigProto(gpu_options=gpu_options,allow_soft_placement=True) with tf.Session(config=config) as sess: if lstm_config.if_train: init=tf.global_variables_initializer() sess.run(init) (X_train, y_train) = load_data(lstm_config.train_file) if len(X_train) < lstm_config.batch_size: for i in range(0, lstm_config.batch_size - len(X_train)): X_train.append([0]) y_train.append([0]) seq_len_train = list(map(lambda x: len(x), X_train)) rnn_model.train_epoch(sess, lstm_config.train_file, X_train, y_train, seq_len_train, lstm_config.model_path) print('Success for preparing data')
def main(test_model): config = Config() config.batch_size = 1024 config.hidden_size = 64 config.vocab_size = 26 config.embed_size = 320 config.max_epochs = 100 config.label_kinds = 2 config.if_train = True config.if_test = True config.is_biLSTM = True config.max_seqlen = 20 config.original_file = '../data/most_frequent_words_label.txt' config.train_file = '../data/most_frequent_words_label_train.txt' config.dev_file = '../data/most_frequent_words_label_dev' config.vocab_file = '../data/vocab.txt' config.model_path = 'models/Transformer_softmax_lstm/' config.split_ratio = 0.8 print('Prepare data for train and dev ... ') train_dev_split(config.original_file, config.train_file, config.dev_file, config.vocab_file, config.split_ratio) print('Prepare data sucessfully!') model = Transformer(config) gpu_options = tf.GPUOptions(allow_growth=True) gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=0.5, allow_growth=True ) ##每个gpu占用0.8 的显存 tf_config = tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True) with tf.Session(config=tf_config) as sess: if config.if_test: init = tf.global_variables_initializer() sess.run(init) (X_test, y_test) = load_data(config.dev_file) if len(X_test) < config.batch_size: for i in range(0, config.batch_size - len(X_test)): X_test.append([0]) y_test.append([0]) seq_len_test = list(map(lambda x: len(x), X_test)) print('Target to special model to test') print('Start do predicting...') model.test(sess, test_model, X_test, y_test, seq_len_test, config.vocab_file, config.model_path + 'result/') print('Success for preparing data')
def main(): config = Config() config.if_train = True print('Prepare data for train and dev ... ') train_dev_split(config.original_file, config.train_file, config.dev_file, config.vocab_file, config.split_ratio) print('Prepare data sucessfully!') rnn_model = LSTM_Dynamic(config) gpu_options = tf.GPUOptions(allow_growth=True) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5, allow_growth=True) ##每个gpu占用0.8 的显存 tf_config=tf.ConfigProto(gpu_options=gpu_options,allow_soft_placement=True) with tf.Session(config=tf_config) as sess: if config.if_train: model_input_path = 'model_input' + str(config.embed_size) + os.path.basename(config.train_file) if not os.path.exists(model_input_path): prepare_and_write_modelinputs(config.train_file, model_input_path) train_epoch(rnn_model, sess, model_input_path, config.model_path)