stats = [FLAGS.embed_status, FLAGS.gate_status, ADV_STATUS]
posfix = ['Y' if x else 'N' for x in stats]
posfix.append(MODEL_TYPE)
if ADV_STATUS:
posfix.append(str(FLAGS.adv_weight))
#Load data
train_data_iterator = []
dev_data_iterator = []
test_data_iterator = []
dev_df = []
test_df = []
print("Loading data...")
for i in range(FLAGS.num_corpus):
train_data_iterator.append(
data_helpers.BucketedDataIterator(pd.read_csv(TRAIN_FILE[i])))
dev_df.append(pd.read_csv(DEV_FILE[i]))
dev_data_iterator.append(data_helpers.BucketedDataIterator(dev_df[i]))
test_df.append(pd.read_csv(TEST_FILE[i]))
test_data_iterator.append(data_helpers.BucketedDataIterator(test_df[i]))
logger.info('-' * 50)
# Training
# ==================================================
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_conf.gpu_options.allow_growth = True
TEST_FILE.append('')
if i in [2, 3, 7, 19]:
DROP_OUT.append(0.7)
elif i in [6, 8, 14, 17]:
DROP_OUT.append(0.6)
else:
DROP_OUT.append(0.80)
# BUCKETS_NUM.append(max(5, 8 - i // 3))
BUCKETS_NUM.append(1)
print("Loading data...")
# 加载数据。保存到对应列表
for i in range(FLAGS.num_corpus):
# task data 0
train_data_iterator.append(
data_helpers.BucketedDataIterator(pd.read_csv(TRAIN_FILE[i]),
num_buckets=BUCKETS_NUM[i]))
# task data 1
dev_df.append(pd.read_csv(DEV_FILE[i]))
# task data 2
dev_data_iterator.append(data_helpers.BucketedDataIterator(dev_df[i]))
# task data 3
# test_df.append(pd.read_csv(TEST_FILE[i]))
# task data 4
# test_data_iterator.append(data_helpers.BucketedDataIterator(test_df[i]))
logger.info('-' * 50)
shared_train_stop_step = [FLAGS.num_epochs] * FLAGS.num_corpus
def Load_pkbs(use_shared_ckp_idx=-1):
logger.addHandler(hdlr)
logger.setLevel(logging.INFO)
# Load data
print("Loading data...")
if BI_GRAM is False:
train_file = 'data_' + TASK_NAME + '/train_uni.csv'
dev_file = 'data_' + TASK_NAME + '/dev_uni.csv'
test_file = 'data_' + TASK_NAME + '/test_uni.csv'
else:
train_file = 'data_' + TASK_NAME + '/train.csv'
dev_file = 'data_' + TASK_NAME + '/dev.csv'
test_file = 'data_' + TASK_NAME + '/test.csv'
train_df = pd.read_csv(train_file)
train_data_iterator = data_helpers.BucketedDataIterator(train_df)
dev_df = pd.read_csv(dev_file)
dev_data_iterator = data_helpers.BucketedDataIterator(dev_df)
test_df = pd.read_csv(test_file)
test_data_iterator = data_helpers.BucketedDataIterator(test_df)
# Training
# ==================================================
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_conf.gpu_options.allow_growth = True
bi_embedding = VOCABS.bi_vectors
da_idx = Data_index(VOCABS, TAGS)
da_idx.process_all_data()
# model names
tf.flags.DEFINE_string("model_name", "cws_"+TASK_NAME, "model name")
FLAGS = tf.flags.FLAGS
FLAGS._parse_flags()
# Load data
print("Loading data...")
test_file = TEST_DATA_BI
test_df = pd.read_csv(test_file)
test_data_iterator = data_helpers.BucketedDataIterator(test_df)
final_score = None
num_of_test = 0
for WHO in TEST_WHO:
num_of_test += 1
with tf.Graph().as_default():
config = tf.ConfigProto(device_count = {'GPU': 0})
sess = tf.Session(config=config)
with sess.as_default():