def main(_): if not os.path.exists(ckpt_path + 'checkpoint'): print('there is not saved model, please check the ckpt path') exit() print('Loading model...') W_embedding = np.load(embedding_path) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: model = network.TextCNN(W_embedding, settings) model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path)) print('Local predicting...') local_predict(sess, model) print('Test predicting...') predict(sess, model)
def main(_): #引入模型-先判断需要引入的路径存不存在 if not os.path.exists(ckpt_path + 'checkpoint'): print('there is no saved model, exit the program') exit() print('loading the model') W_embedding = np.load(embedding_path) config = tf.ConfigProto() config.gpu_options.allow_growth = True #建立Session,feed数据,run with tf.Session(config=config) as sess: model = network.TextCNN(W_embedding, settings) model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path)) print('Local predicting...') local_predict(sess, model) print('Test predicting') predict(sess, model)
def main(_): if not os.path.exists(ckpt_path + 'checkpoint'): print('there is not saved model, please check the ckpt path') exit() print('Loading model...') W_embedding = np.load(embedding_path) config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 0.50 config.gpu_options.allocator_type = 'BFC' with tf.Session(config=config) as sess: model = network.TextCNN(W_embedding, settings) model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path)) #print('dev predicting...') #predict_dev(sess, model) print('test predicting...') predict(sess, model)
def main(_): if not os.path.exists(ckpt_path + 'checkpoint'): print('there is not saved model, please check the ckpt path') exit() print('Loading model...') W_embedding = np.load(embedding_path) config = tf.ConfigProto() config.gpu_options.allow_growth = True log_path = scores_path + settings.model_name + '/' if not os.path.exists(log_path): os.makedirs(log_path) logger = get_logger(log_path + 'predict.log') with tf.Session(config=config) as sess: model = network.TextCNN(W_embedding, settings) # ckpt_path: /ckpt/wd_BiGRU/checkpoint model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path)) print('Valid predicting...') predict_valid(sess, model, logger) print('Test predicting...') predict(sess, model, logger)
def main(): if not os.path.exists(ckpt_path + 'checkpoint'): print('there is not saved model, please check the ckpt path') exit() print('Loading model...') W_embedding = np.load(embedding_path) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: model = network.TextCNN(W_embedding, settings) model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path)) #print('Local predicting...') #print ('valid batches:%d'%n_va_batches) #local_predict(sess, model) print('Test predicting...') print('test batches:%d' % n_tr_batches) results = predict(sess, model) sub_path_name = sub_path + model_name + str( strftime("%m%d%H%M")) + '.csv' id_list = get_id_list(id_list_path) # id_list=id_list[:len(results)] write_submission(sub_path_name, id_list, results, sr_id2title)
def main(_): global ckpt_path global last_score12 if not os.path.exists(ckpt_path): os.makedirs(ckpt_path) if not os.path.exists(summary_path): os.makedirs(summary_path) elif not FLAGS.is_retrain: shutil.rmtree(summary_path) os.makedirs(summary_path) if not os.path.exists(summary_path): os.makedirs(summary_path) if not os.path.exists(log_path): os.makedirs(log_path) print('1.Loading data...') W_embedding = np.load(embedding_path) print('training sample_num = %d' % n_tr_batches) print('valid sample_num = %d' % n_va_batches) logger = get_logger(log_path + FLAGS.log_file_train) print('2.Building model...') config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: model = network.TextCNN(W_embedding, settings) with tf.variable_scope('training_ops') as vs: learning_rate = tf.train.exponential_decay(FLAGS.lr, model.global_step, FLAGS.decay_step, FLAGS.decay_rate, staircase=True) with tf.variable_scope('Optimizer1'): tvars1 = tf.trainable_variables() grads1 = tf.gradients(model.loss, tvars1) optimizer1 = tf.train.AdamOptimizer( learning_rate=learning_rate) train_op1 = optimizer1.apply_gradients( zip(grads1, tvars1), global_step=model.global_step) with tf.variable_scope('Optimizer2'): tvars2 = [ tvar for tvar in tvars1 if 'embedding' not in tvar.name ] grads2 = tf.gradients(model.loss, tvars2) optimizer2 = tf.train.AdamOptimizer( learning_rate=learning_rate) train_op2 = optimizer2.apply_gradients( zip(grads2, tvars2), global_step=model.global_step) update_op = tf.group(*model.update_emas) merged = tf.summary.merge_all() # summary train_writer = tf.summary.FileWriter(summary_path + 'train', sess.graph) test_writer = tf.summary.FileWriter(summary_path + 'test') training_ops = [ v for v in tf.global_variables() if v.name.startswith(vs.name + '/') ] if os.path.exists(ckpt_path + "checkpoint"): print("Restoring Variables from Checkpoint...") model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path)) f1_micro, f1_macro, score12 = valid_epoch(data_valid_path, sess, model) print('f1_micro=%g, f1_macro=%g, score12=%g' % (f1_micro, f1_macro, score12)) sess.run(tf.variables_initializer(training_ops)) train_op2 = train_op1 else: print('Initializing Variables...') sess.run(tf.global_variables_initializer()) print('3.Begin training...') print('max_epoch=%d, max_max_epoch=%d' % (FLAGS.max_epoch, FLAGS.max_max_epoch)) logger.info('max_epoch={}, max_max_epoch={}'.format( FLAGS.max_epoch, FLAGS.max_max_epoch)) train_op = train_op2 for epoch in range(FLAGS.max_max_epoch): print('\nepoch: ', epoch) logger.info('epoch:{}'.format(epoch)) global_step = sess.run(model.global_step) print('Global step %d, lr=%g' % (global_step, sess.run(learning_rate))) if epoch == FLAGS.max_epoch: train_op = train_op1 train_fetches = [merged, model.loss, train_op, update_op] valid_fetches = [merged, model.loss] train_epoch(data_train_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer, logger) # 最后再做一次验证 f1_micro, f1_macro, score12 = valid_epoch(data_valid_path, sess, model) print('END:Global_step=%d: f1_micro=%g, f1_macro=%g, score12=%g' % (sess.run(model.global_step), f1_micro, f1_macro, score12)) logger.info( 'END:Global_step={}: f1_micro={}, f1_macro={}, score12={}'.format( sess.run(model.global_step), f1_micro, f1_macro, score12)) if score12 > last_score12: saving_path = model.saver.save(sess, model_path, sess.run(model.global_step) + 1) print('saved new model to %s ' % saving_path) logger.info('saved new model to {}'.format(saving_path))
def main(_): global ckpt_path global last_f1 if not os.path.exists(ckpt_path): os.makedirs(ckpt_path) if not os.path.exists(summary_path): os.makedirs(summary_path) elif not FLAGS.is_retrain: # 重新训练本模型,删除以前的 summary shutil.rmtree(summary_path) os.makedirs(summary_path) if not os.path.exists(summary_path): os.makedirs(summary_path) print('1.Loading data...') W_embedding = np.load(embedding_path) print('training sample_num = %d' % n_tr_batches) print('valid sample_num = %d' % n_va_batches) # Initial or restore the model print('2.Building model...') config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: model = network.TextCNN(W_embedding, settings) with tf.variable_scope('training_ops') as vs: learning_rate = tf.train.exponential_decay(FLAGS.lr, model.global_step, FLAGS.decay_step, FLAGS.decay_rate, staircase=True) # two optimizer: op1, update embedding; op2, do not update embedding. with tf.variable_scope('Optimizer1'): tvars1 = tf.trainable_variables() grads1 = tf.gradients(model.loss, tvars1) optimizer1 = tf.train.AdamOptimizer(learning_rate=learning_rate) train_op1 = optimizer1.apply_gradients(zip(grads1, tvars1), global_step=model.global_step) with tf.variable_scope('Optimizer2'): tvars2 = [tvar for tvar in tvars1 if 'embedding' not in tvar.name] grads2 = tf.gradients(model.loss, tvars2) optimizer2 = tf.train.AdamOptimizer(learning_rate=learning_rate) train_op2 = optimizer2.apply_gradients(zip(grads2, tvars2), global_step=model.global_step) update_op = tf.group(*model.update_emas) merged = tf.summary.merge_all() # summary train_writer = tf.summary.FileWriter(summary_path + 'train', sess.graph) test_writer = tf.summary.FileWriter(summary_path + 'test') training_ops = [v for v in tf.global_variables() if v.name.startswith(vs.name+'/')] # 如果已经保存过模型,导入上次的模型 if os.path.exists(ckpt_path + "checkpoint"): print("Restoring Variables from Checkpoint...") model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path)) last_valid_cost, precision, recall, last_f1 = valid_epoch(data_valid_path, sess, model) print(' valid cost=%g; p=%g, r=%g, f1=%g' % (last_valid_cost, precision, recall, last_f1)) sess.run(tf.variables_initializer(training_ops)) train_op2 = train_op1 else: print('Initializing Variables...') sess.run(tf.global_variables_initializer()) print('3.Begin training...') print('max_epoch=%d, max_max_epoch=%d' % (FLAGS.max_epoch, FLAGS.max_max_epoch)) train_op = train_op2 for epoch in xrange(FLAGS.max_max_epoch): global_step = sess.run(model.global_step) print('Global step %d, lr=%g' % (global_step, sess.run(learning_rate))) if epoch == FLAGS.max_epoch: # update the embedding train_op = train_op1 train_fetches = [merged, model.loss, train_op, update_op] valid_fetches = [merged, model.loss] train_epoch(data_train_path, sess, model, train_fetches, valid_fetches, train_writer, test_writer) # 最后再做一次验证 valid_cost, precision, recall, f1 = valid_epoch(data_valid_path, sess, model) print('END.Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g' % ( sess.run(model.global_step), valid_cost, precision, recall, f1)) if f1 > last_f1: # save the better model saving_path = model.saver.save(sess, model_path, sess.run(model.global_step)+1) print('saved new model to %s ' % saving_path)
def main(): global ckpt_path global last_f1 #把不存在的ckpt,summary建立起来;重新训练时删除原来的summary if not os.path.exists(ckpt_path): os.makedirs(ckpt_path) if not os.path.exists(summary_path): os.makedirs(summary_path) elif not FLAGS.is_retain: shutil.rmtree(summary_path) os.makedirs(summary_path) #引入数据--已经训练好的embedding,穿插打印些提示语言 print('1.lodaing embedding...') W_embedding=np.load(embedding_path) print('training sample_num = %d' % n_tr_batches)#23360 print('valid sample_num = %d' % n_va_batches)#782 #构建模型, print('2.building model...') config=tf.ConfigProto() config.gpu_options.allow_growth=True #打开Session,模型图必须在Session中构建,对Sesssion中的config进行设置 with tf.Session(config=config) as sess: #构建模型图 model=network.TextCNN(W_embedding,settings) with tf.variable_scope('training_ops') as vs: learning_rate=tf.train.exponential_decay(lr,model.global_step,FLAGS.decay_step,FLAGS.decay_rate ,staircase=True) #two optimizer: Embedding not updated, and updated with tf.variable_scope('Optimizer1'): tvars1=tf.trainable_variables() grads1=tf.gradients(model.loss,tvars1) optimizer1=tf.train.AdamOptimizer(learing_rate=learning_rate) train_op1=optimizer1.apply_gradients(zip(grads1,tvars1),global_step=model.global_step) with tf.varialbe_scope('Optimizer2'): tvars2=[tvar for tvar in tvars1 if 'embedding' not in tvar.name] grads2=tf.gradients(model.loss,tvars2) optimizer2=tf.train.AdamOptimizer(learning_rate=learning_rate) train_op2=optimizer2.apply_gradients(zip(grads2,tvars2),global_step=model.global_step) update_op=tf.group(*model.update_emas) merged=tf.summary.merge_all() train_writer=tf.summary.FileWriter(summary_path+'train', sess.graph) test_writer=tf.summary.FileWriter(summary_path+'test',sess.graph) training_ops=[v for v in tf.global_variables() if v.name.starwit(vs.name+'/')] #若保存过模型,导入 if os.path.exists(ckpt_path+'checkpoint'): print('restoring model from chekcpoint...') model.saver.restore(sess,tf.train.latest_checkpoint(ckpt_path)) last_valid_cost,precision, recall, last_f1=valid_epoch(data_valid_path,sess,model) print('valid_cost=%g, p=%g, r=%g, f1=%g'%(last_valid_cost,precision, recall, last_f1)) sess.run(tf.variables_initializer(training_ops)) train_op2=train_op1 else: print('Initializing variables...') sess.run(tf.global_variables_initializer()) print('3. Begin training...') print('max_epoch=%d, max_max_epoch=%d'%(FLAGS.max_epoch,FLAGS.max_max_epoch)) for epoch in range(FLAGS.max_max_epoch): global_step=sess.run(model.global_step) print('Global_step=%d,lr=%g' %(global_step,sess.run(learning_rate))) if epoch==FLAGS.max_epoch: train_op=train_op1 else: train_op=train_op2 train_fetches=[merged,model.loss,train_op,update_op] valid_fetches=[merged,model.loss] train_epoch(data_train_path,sess,model,train_fetches,valid_fetches,train_writer,test_writer) #最后再做一次印证 valid_cost,precision,recall,f1=valid_epoch(data_valid_path,sess, model) print('END.Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g' % ( sess.run(model.global_step), valid_cost, precision, recall, f1)) if f1>last_f1:#save the better one saving_path=model.save(sess, model_path,sess.run(model.global_step)+1) print('save new model to s%'%saving_path)
if not os.path.exists(ckpt_path + 'checkpoint'): print('there is not saved model, please check the ckpt path') exit() #print('加载知识点节点id与名称......') df_train = pd.read_csv('../../raw_data/all_knowledge_set.txt', sep='\t', usecols=[0, 1], names=['topic_id', 'topic_name'], dtype={'topic_id': object}) dict_topic_id2name = dict(zip(df_train.topic_id, df_train.topic_name.values)) print('加载词向量......') W_embedding = np.load(embedding_path) print('定义模型结构......') model = network.TextCNN(W_embedding, settings) config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) print('初始化模型参数......') model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path)) app = Flask(__name__) @app.route('/', methods=['POST', 'GET']) def get_text_input(): content = request.args.get('title') if not content: return '参数有误,正确格式 http://127.0.0.1:5002/?title=输入文本'
def main(_): global ckpt_path global last_f1 if not os.path.exists(ckpt_path): os.makedirs(ckpt_path) firsttime = datetime.datetime.now() print('1.Loading data...') W_embedding = np.load(embedding_path) print('training sample_num = %d' % n_tr_batches) print('valid sample_num = %d' % n_va_batches) # Initial or restore the model print('2.Building model...') config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: model = network.TextCNN(W_embedding, settings) with tf.variable_scope('training_ops') as vs: update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): global_step = model.global_step optimizer = tf.train.MomentumOptimizer(FLAGS.lr, 0.9) train_op = tf.contrib.layers.optimize_loss( loss=model.loss, global_step=global_step, clip_gradients=4.0, learning_rate=FLAGS.lr, optimizer=optimizer, update_ops=update_ops) training_ops = [ v for v in tf.global_variables() if v.name.startswith(vs.name + '/') ] dataset = Dataset(cross_validation=3) train_sample = Dataset(valid=True, cross_validation=3) # 如果已经保存过模型,导入上次的模型 if os.path.exists(ckpt_path + "checkpoint"): print("Restoring Variables from Checkpoint...") model.saver.restore(sess, tf.train.latest_checkpoint(ckpt_path)) last_valid_cost, precision, recall, last_f1 = valid_epoch( dataset, sess, model) print(' valid cost=%g; p=%g, r=%g, f1=%g' % (last_valid_cost, precision, recall, last_f1)) sess.run(tf.variables_initializer(training_ops)) else: print('Initializing Variables...') sess.run(tf.global_variables_initializer()) print('3.Begin training...') for epoch in range(FLAGS.max_max_epoch): print( 'epoch %d ********************************************************' % epoch) starttime = datetime.datetime.now() train_fetches = [train_op] valid_fetches = [model.loss] train_epoch(dataset, sess, model, train_fetches, valid_fetches, '', '', train_sample) print('epoch %d cost time:%s' % (epoch, datetime.datetime.now() - starttime)) print('total cost time:%s' % (datetime.datetime.now() - firsttime)) # 最后再做一次验证 valid_cost, precision, recall, f1 = valid_epoch(dataset, sess, model) print('END.Global_step=%d: valid cost=%g; p=%g, r=%g, f1=%g' % (sess.run(model.global_step), valid_cost, precision, recall, f1)) if f1 > last_f1: # save the better model saving_path = model.saver.save(sess, model_path + str(f1) + '_', sess.run(model.global_step)) print('saved new model to %s ' % saving_path)