print("")
# Data Preparation
# ==================================================
# Load data
print("Loading data...")
raw, label_map = insurance_qa_data_helpers.read_raw(FLAGS.train_file)#get true query vs answer
max_seq_len = FLAGS.max_seq_len
if FLAGS.embedding_type == "char_embedding":
vocab = insurance_qa_data_helpers.build_vocab_char(FLAGS.domain_train_file)
elif FLAGS.embedding_type == "subword_embedding":
vocab = insurance_qa_data_helpers.build_vocab_subword(FLAGS.domain_train_file)
else:
vocab = insurance_qa_data_helpers.build_vocab(FLAGS.domain_train_file)#{word:id}
alist, avec_list = insurance_qa_data_helpers.read_alist(FLAGS.train_file, vocab, max_seq_len, FLAGS.embedding_type)#raw_label_list,id_label_list
testList = insurance_qa_data_helpers.load_test(FLAGS.dev_file, alist)
print "vocab size:",len(vocab)
print('seq_len', max_seq_len)
print("Load one...")
# Training
# ==================================================
with tf.Graph().as_default():
with tf.device("/gpu:1"):
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices")
FLAGS = tf.flags.FLAGS
FLAGS._parse_flags()
print("\nParameters:")
for attr, value in sorted(FLAGS.__flags.items()):
print("{}={}".format(attr.upper(), value))
print("")
# Data Preparatopn
# ==================================================
# Load data
print("Loading data...")
vocab = insurance_qa_data_helpers.build_vocab()
alist = insurance_qa_data_helpers.read_alist()
raw = insurance_qa_data_helpers.read_raw()
x_train_1, x_train_2, x_train_3 = insurance_qa_data_helpers.load_data_6(vocab, alist, raw, FLAGS.batch_size)
testList, vectors = insurance_qa_data_helpers.load_test_and_vectors()
vectors = ''
print('x_train_1', np.shape(x_train_1))
print("Load done...")
val_file = '../../insuranceQA/test1'
precision = '../../insuranceQA/test1.gan'+timeStamp
#x_val, y_val = data_deepqa.load_data_val()
# Training
# ==================================================
def train_step(sess,cnn,x_batch_1, x_batch_2, x_batch_3):
def get_data(self):
self.vocab, self.train_y, self.train_x_1, self.train_x_2 = insurance_qa_data_helpers.build_vocab(
) # index of question'word and answer'words
self.test_x1, self.test_x2 = insurance_qa_data_helpers.build_vocab_test(
)
# Misc Parameters
tf.flags.DEFINE_boolean("allow_soft_placement", True,
"Allow device soft device placement")
tf.flags.DEFINE_boolean("log_device_placement", False,
"Log placement of ops on devices")
FLAGS = tf.flags.FLAGS
# FLAGS._parse_flags()
# print(("\nParameters:"))
# for attr, value in sorted(FLAGS.__flags.items()):
# print(("{}={}".format(attr.upper(), value)))
# print((""))
timeStamp = time.strftime("%Y%m%d%H%M%S", time.localtime(int(time.time())))
print(("Loading data..."))
vocab = insurance_qa_data_helpers.build_vocab(
) # '/train',/test1',/test2',/dev'
# with open("insuranceQA/vocab","w") as out_op:
# for v in vocab:
# result=v+"\t"+str(vocab[v])+"\n"
# out_op.write(result)
# embeddings =insurance_qa_data_helpers.load_vectors(vocab)
alist = insurance_qa_data_helpers.read_alist(
) # [...] ('insuranceQA/train') A 全部的answer数据
raw = insurance_qa_data_helpers.read_raw(
) # [...] ('insuranceQA/train') Q,A 正的数据
test1List = insurance_qa_data_helpers.loadTestSet("test1")
test2List = insurance_qa_data_helpers.loadTestSet("test2")
devList = insurance_qa_data_helpers.loadTestSet("dev")
tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices")
FLAGS = tf.flags.FLAGS
FLAGS._parse_flags()
print("\nParameters:")
for attr, value in sorted(FLAGS.__flags.items()):
print("{}={}".format(attr.upper(), value))
print("")
# Data Preparatopn
# ==================================================
# Load data
print("Loading data...")
vocab = insurance_qa_data_helpers.build_vocab()
alist = insurance_qa_data_helpers.read_alist()
raw = insurance_qa_data_helpers.read_raw()
x_train_1, x_train_2, x_train_3 = insurance_qa_data_helpers.load_data_6(vocab, alist, raw, FLAGS.batch_size)
testList, vectors = insurance_qa_data_helpers.load_test_and_vectors()
vectors = ''
print('x_train_1', np.shape(x_train_1))
print("Load done...")
val_file = '/export/jw/cnn/insuranceQA/test1'
precision = '/export/jw/cnn/insuranceQA/test1.acc'
#x_val, y_val = data_deepqa.load_data_val()
# Training
# ==================================================
