tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices") FLAGS = tf.flags.FLAGS FLAGS._parse_flags() print("\nParameters:") for attr, value in sorted(FLAGS.__flags.items()): print("{}={}".format(attr.upper(), value)) print("") # Data Preparation # ================================================== # Load data print("Loading data...") raw, label_map = insurance_qa_data_helpers.read_raw(FLAGS.train_file)#get true query vs answer max_seq_len = FLAGS.max_seq_len if FLAGS.embedding_type == "char_embedding": vocab = insurance_qa_data_helpers.build_vocab_char(FLAGS.domain_train_file) elif FLAGS.embedding_type == "subword_embedding": vocab = insurance_qa_data_helpers.build_vocab_subword(FLAGS.domain_train_file) else: vocab = insurance_qa_data_helpers.build_vocab(FLAGS.domain_train_file)#{word:id} alist, avec_list = insurance_qa_data_helpers.read_alist(FLAGS.train_file, vocab, max_seq_len, FLAGS.embedding_type)#raw_label_list,id_label_list testList = insurance_qa_data_helpers.load_test(FLAGS.dev_file, alist) print "vocab size:",len(vocab) print('seq_len', max_seq_len) print("Load one...")
# print(("{}={}".format(attr.upper(), value))) # print(("")) timeStamp = time.strftime("%Y%m%d%H%M%S", time.localtime(int(time.time()))) print(("Loading data...")) vocab = insurance_qa_data_helpers.build_vocab( ) # '/train',/test1',/test2',/dev' # with open("insuranceQA/vocab","w") as out_op: # for v in vocab: # result=v+"\t"+str(vocab[v])+"\n" # out_op.write(result) # embeddings =insurance_qa_data_helpers.load_vectors(vocab) alist = insurance_qa_data_helpers.read_alist( ) # [...] ('insuranceQA/train') A 全部的answer数据 raw = insurance_qa_data_helpers.read_raw( ) # [...] ('insuranceQA/train') Q,A 正的数据 test1List = insurance_qa_data_helpers.loadTestSet("test1") test2List = insurance_qa_data_helpers.loadTestSet("test2") devList = insurance_qa_data_helpers.loadTestSet("dev") testSet = [("test1", test1List), ("test2", test2List), ("dev", devList)] print("Load done...") log_precision = 'log/test1.gan_precision' + timeStamp loss_precision = 'log/test1.gan_loss' + timeStamp from functools import wraps def log_time_delta(func): @wraps(func)
FLAGS = tf.flags.FLAGS FLAGS._parse_flags() print("\nParameters:") for attr, value in sorted(FLAGS.__flags.items()): print("{}={}".format(attr.upper(), value)) print("") # Data Preparatopn # ================================================== # Load data print("Loading data...") vocab = insurance_qa_data_helpers.build_vocab() alist = insurance_qa_data_helpers.read_alist() raw = insurance_qa_data_helpers.read_raw() x_train_1, x_train_2, x_train_3 = insurance_qa_data_helpers.load_data_6(vocab, alist, raw, FLAGS.batch_size) testList, vectors = insurance_qa_data_helpers.load_test_and_vectors() vectors = '' print('x_train_1', np.shape(x_train_1)) print("Load done...") val_file = '../../insuranceQA/test1' precision = '../../insuranceQA/test1.gan'+timeStamp #x_val, y_val = data_deepqa.load_data_val() # Training # ================================================== def train_step(sess,cnn,x_batch_1, x_batch_2, x_batch_3): """ A single training step
FLAGS = tf.flags.FLAGS FLAGS._parse_flags() print("\nParameters:") for attr, value in sorted(FLAGS.__flags.items()): print("{}={}".format(attr.upper(), value)) print("") # Data Preparatopn # ================================================== # Load data print("Loading data...") vocab = insurance_qa_data_helpers.build_vocab() alist = insurance_qa_data_helpers.read_alist() raw = insurance_qa_data_helpers.read_raw() x_train_1, x_train_2, x_train_3 = insurance_qa_data_helpers.load_data_6(vocab, alist, raw, FLAGS.batch_size) testList, vectors = insurance_qa_data_helpers.load_test_and_vectors() vectors = '' print('x_train_1', np.shape(x_train_1)) print("Load done...") val_file = '/export/jw/cnn/insuranceQA/test1' precision = '/export/jw/cnn/insuranceQA/test1.acc' #x_val, y_val = data_deepqa.load_data_val() # Training # ================================================== with tf.Graph().as_default(): with tf.device("/gpu:1"):