def _deal_fact_line(self, index, line, oov_token2id):
line = line.strip()
cur_fact_ids = []
cur_fact_ids_exp = []
fact_tar = None
fact_tar_loss = None
elems = line.split('\t')
no_fact_tag = False
# if there is no fact, add pad sequence
if elems[1] == config.NO_FACT:
cur_fact_ids.append(self.pad_fact_seqs)
cur_fact_ids_exp.append(self.pad_fact_seqs)
no_fact_tag = True
else:
for index, text in enumerate(elems[1:]):
seq, seq_exp = self._line2ids(text, self.max_fact_len, oov_token2id)
new_seq = helper_fn.pad_with_pad(seq, self.max_fact_len, self.pad_id)
cur_fact_ids.append(new_seq)
new_seq_exp = helper_fn.pad_with_pad(seq_exp, self.max_fact_len, self.pad_id)
cur_fact_ids_exp.append(new_seq_exp)
if index == 0:
seq, seq_exp = self._line2ids(text, self.tar_len, oov_token2id)
fact_tar = helper_fn.pad_with_start(seq, self.max_tar_len, self.start_id, self.pad_id)
fact_tar_loss = helper_fn.pad_with_end(seq, self.max_tar_len, self.start_id, self.pad_id)
# pad fact number
cur_fact_ids = cur_fact_ids[:self.args.fact_number]
cur_fact_ids_exp = cur_fact_ids_exp[:self.args.fact_number]
cur_fact_ids = cur_fact_ids + [self.pad_fact_seqs] * (self.args.fact_number - len(cur_fact_ids))
cur_fact_ids_exp = cur_fact_ids_exp + [self.pad_fact_seqs] * (self.args.fact_number - len(cur_fact_ids_exp))
return no_fact_tag, cur_fact_ids, cur_fact_ids_exp, fact_tar, fact_tar_loss
def __init__(self, args):
self.args = args
self.__set_file_path()
# get global token and ids
self.src_token_ids, self.src_id_tokens, self.src_vocab_size = self.__read_global_ids(self.src_global_token_path)
self.tar_token_ids, self.tar_id_tokens, self.tar_vocab_size = self.__read_global_ids(self.tar_global_token_path)
self.train_sample_num = 0
self.valid_sample_num = 0
self.test_sample_num = 0
# just set the max fact and conv length the same as source
self.max_src_len = args.src_seq_length
self.max_tar_len = args.tar_seq_length
if not args.fact_seq_length:
self.max_fact_len = args.src_seq_length
else:
self.max_fact_len = args.fact_seq_length
self.max_conv_len = args.conv_seq_length
self.src_len = self.max_src_len - config.src_reserved_pos
self.tar_len = self.max_tar_len - config.tar_reserved_pos
self.pad_fact_seqs = helper_fn.pad_with_pad([self.pad_id], self.max_fact_len, self.pad_id)
self.pad_conv_seqs = helper_fn.pad_with_pad([self.pad_id], self.max_conv_len, self.pad_id)
self.__get_sample_numbers()
def _deal_convs(f):
convs_ids = []
for index, line in enumerate(f):
line = line.strip()
conv_ids = []
elems = line.split('\t')
# if there is no context, add pad sequence
if elems[1] == config.NO_CONTEXT:
conv_ids.append(pad_conv_seqs)
else:
for text in elems[1:]:
seq = [
self.src_token_ids.get(token, self.unk_id)
for token in text.split()
]
#seq = [self.src_token_ids.get(token, self.pad_id) for token in text.split()]
seq = seq[:max_conv_len]
new_seq = helper_fn.pad_with_pad(
seq, max_conv_len, self.pad_id)
conv_ids.append(new_seq)
conv_ids = conv_ids[:self.args.conv_number]
conv_ids = conv_ids + [pad_conv_seqs] * (
self.args.conv_number - len(conv_ids))
convs_ids.append(conv_ids)
if ((index + 1) % self.args.batch_size == 0):
res = np.asarray(convs_ids)
convs_ids = []
yield res
if len(convs_ids) != 0:
res = np.asarray(convs_ids)
convs_ids = []
yield res
def _deal_facts(f):
facts_ids = []
for index, line in enumerate(f):
line = line.strip()
fact_ids = []
elems = line.split('\t')
# if there is no fact, add pad sequence
if elems[1] == config.NO_FACT:
fact_ids.append(pad_fact_seqs)
else:
for text in elems[1:]:
seq = [
self.src_token_ids.get(token, self.unk_id)
for token in text.split()
]
#seq = [self.src_token_ids.get(token, self.pad_id) for token in text.split()]
seq = seq[:max_fact_len]
new_seq = helper_fn.pad_with_pad(
seq, max_fact_len, self.pad_id)
fact_ids.append(new_seq)
# pad fact number
fact_ids = fact_ids[:self.args.fact_number]
fact_ids = fact_ids + [pad_fact_seqs] * (
self.args.fact_number - len(fact_ids))
facts_ids.append(fact_ids)
if ((index + 1) % self.args.batch_size == 0):
res = np.asarray(facts_ids)
facts_ids = []
yield res
if len(facts_ids) != 0:
res = np.asarray(facts_ids)
facts_ids = []
yield res
def _deal_conv_line(self, index, line):
line = line.strip()
cur_conv_ids = []
elems = line.split('\t')
# if there is no conv, add pad sequence
if elems[1] == config.NO_CONTEXT:
cur_conv_ids.append(self.pad_conv_seqs)
else:
for text in elems[1:]:
seq, _ = self._line2ids(text, self.max_conv_len)
new_seq = helper_fn.pad_with_pad(seq, self.max_conv_len, self.pad_id)
cur_conv_ids.append(new_seq)
cur_conv_ids = cur_conv_ids[:self.args.conv_number]
cur_conv_ids = cur_conv_ids + [self.pad_conv_seqs] * (self.args.conv_number - len(cur_conv_ids))
return cur_conv_ids
def read_file(self,
file_type,
max_src_len,
max_tar_len,
max_fact_len=30,
max_conv_len=30,
get_fact=False,
get_conv=False,
get_one_hot=False):
'''
:param file_type: This is supposed to be: train, valid, or test
:param max_src_len: This is maximem source (question) length
:param max_tar_len: This is maximem target (anwser) length
:param max_fact_len: This is maximem fact (external knowledge) length, should be the same with source
:param max_conv_len: This is maximem conversation (context) length
:param get_fact: This is a boolean value to indicate whether read fact file
:param get_conv: This is a boolean value to indicate whether read conv file
'''
assert (max_src_len > 0)
assert (max_tar_len > 0)
assert (max_fact_len > 0)
assert (max_conv_len > 0)
assert file_type == 'train' or file_type == 'valid' or file_type == 'test'
print('current file type: %s' % file_type)
src_len = max_src_len - config.src_reserved_pos
tar_len = max_tar_len - config.tar_reserved_pos
if file_type == 'train':
qa_path = self.train_set_path
conv_path = self.train_conv_path
fact_path = self.train_sent_fact_path
elif file_type == 'valid':
qa_path = self.valid_set_path
conv_path = self.valid_conv_path
fact_path = self.valid_sent_fact_path
elif file_type == 'test':
qa_path = self.test_set_path
conv_path = self.test_conv_path
fact_path = self.test_sent_fact_path
# read source and target
print(qa_path)
f = open(qa_path)
indexes = []
source_ids = []
target_ids = []
target_loss_ids = [
] # Use to calculate loss. Only END sign, dont have START sign
for line in f:
elems = line.strip().split('\t')
if len(elems) < 3:
raise ValueError(
'Exceptd input to be 3 dimension, but received %d' %
len(elems))
indexes.append(int(elems[0].strip()))
text = elems[1].strip()
seq = [
self.src_token_ids.get(token, self.unk_id)
for token in text.split()
]
seq = seq[:src_len]
new_seq = helper_fn.pad_with_start_end(seq, max_src_len,
self.start_id, self.end_id,
self.pad_id)
source_ids.append(new_seq)
text = elems[2].strip()
seq = [
self.tar_token_ids.get(token, self.unk_id)
for token in text.split()
]
seq = seq[:tar_len]
new_seq = helper_fn.pad_with_start(seq, max_tar_len, self.start_id,
self.pad_id)
target_ids.append(new_seq)
new_seq = helper_fn.pad_with_end(seq, max_tar_len, self.end_id,
self.pad_id)
target_loss_ids.append(new_seq)
f.close()
if get_one_hot == True:
target_one_hot = np.zeros(
(len(target_ids), len(target_ids[0]), self.vocab_size),
dtype='int32')
for i, target in enumerate(target_ids):
for t, term_idx in enumerate(target):
if t > 0:
intaa = 0
target_one_hot[i, t - 1, term_idx] = 1
target_loss_ids = target_one_hot
pad_seqs = helper_fn.pad_with_pad([self.pad_id], max_fact_len,
self.pad_id)
facts_ids = []
if get_fact == True:
print(fact_path)
with open(fact_path) as f:
for index, line in enumerate(f):
line = line.strip()
fact_ids = []
elems = line.split('\t')
# if there is no fact, add pad sequence
if elems[1] == config.NO_FACT:
fact_ids.append(pad_seqs)
else:
for text in elems[1:]:
seq = [
self.src_token_ids.get(token, self.unk_id)
for token in text.split()
]
seq = seq[:max_fact_len]
new_seq = helper_fn.pad_with_pad(
seq, max_fact_len, self.pad_id)
fact_ids.append(new_seq)
facts_ids.append(fact_ids)
# keep facts to be the same number. If there is no so many fact, use pad_id as fact to pad it.
facts_ids_tmp = []
for facts in facts_ids:
facts = facts[:self.args.fact_number]
facts = facts + [pad_seqs
] * (self.args.fact_number - len(facts))
facts_ids_tmp.append(facts)
facts_ids = facts_ids_tmp
#pad_convs = [self.pad_id] * max_conv_len
pad_seqs = helper_fn.pad_with_pad([self.pad_id], max_conv_len,
self.pad_id)
convs_ids = []
if get_conv == True:
print(conv_path)
with open(conv_path) as f:
for index, line in enumerate(f):
line = line.strip()
conv_ids = []
elems = line.split('\t')
# if there is no context, add pad sequence
if elems[1] == config.NO_CONTEXT:
conv_ids.append(pad_seqs)
else:
for text in elems[1:]:
seq = [
self.src_token_ids.get(token, self.unk_id)
for token in text.split()
]
seq = seq[:max_conv_len]
new_seq = helper_fn.pad_with_pad(
seq, max_conv_len, self.pad_id)
conv_ids.append(new_seq)
convs_ids.append(conv_ids)
# keep conv to be the same number. If there is no so many conv, use pad_id as conv to pad it.
convs_ids_tmp = []
for convs in convs_ids:
convs = convs[:self.args.conv_number]
convs = convs + [pad_seqs
] * (self.args.conv_number - len(convs))
convs_ids_tmp.append(convs)
convs_ids = convs_ids_tmp
assert (len(source_ids) == len(indexes))
assert (len(source_ids) == len(target_ids))
if get_fact == True:
assert (len(source_ids) == len(facts_ids))
if get_conv == True:
assert (len(source_ids) == len(convs_ids))
## [[[ if for Zeyang to output ordered index, not shuffiling.
#if get_fact == True and get_conv == True:
# indexes, source_ids, target_ids, convs_ids, facts_ids = shuffle(indexes, source_ids, target_ids, convs_ids, facts_ids)
#elif get_fact == True:
# indexes, source_ids, target_ids, facts_ids = shuffle(indexes, source_ids, target_ids, facts_ids)
#else:
# indexes, source_ids, target_ids = shuffle(indexes, source_ids, target_ids)
## ]]]
return indexes, source_ids, target_ids, target_loss_ids, convs_ids, facts_ids
def data_generator(self,
file_type,
model_type,
max_src_len,
max_tar_len,
max_fact_len=30,
max_conv_len=30,
get_fact=False,
get_conv=False):
'''
:param file_type: This is supposed to be: train, valid, or test
:param max_src_len: This is maximem source (question) length
:param max_tar_len: This is maximem target (anwser) length
:param max_fact_len: This is maximem fact (external knowledge) length, which should be the same with source
:param max_conv_len: This is maximem conversation (context) length
:param get_fact: This is a boolean value to indicate whether read fact file
:param get_conv: This is a boolean value to indicate whether read conv file
'''
print('This is in data generator...')
assert (max_src_len > 0)
assert (max_tar_len > 0)
assert (max_fact_len > 0)
assert (max_conv_len > 0)
assert file_type == 'train' or file_type == 'valid' or file_type == 'test'
src_len = max_src_len - config.src_reserved_pos
tar_len = max_tar_len - config.tar_reserved_pos
if file_type == 'train':
qa_path = self.train_set_path
conv_path = self.train_conv_path
fact_path = self.train_sent_fact_path
elif file_type == 'valid':
qa_path = self.valid_set_path
conv_path = self.valid_conv_path
fact_path = self.valid_sent_fact_path
elif file_type == 'test':
qa_path = self.test_set_path
conv_path = self.test_conv_path
fact_path = self.test_sent_fact_path
def _deal_qa(f):
source_ids = []
target_ids = []
target_loss_ids = [
] # Use to calculate loss. Only END sign, dont have START sign
for index, line in enumerate(f):
elems = line.strip().split('\t')
text = elems[1].strip()
seq = [
self.src_token_ids.get(token, self.unk_id)
for token in text.split()
]
#seq = [self.src_token_ids.get(token, self.pad_id) for token in text.split()]
seq = seq[:src_len]
new_seq = helper_fn.pad_with_start_end(seq, max_src_len,
self.start_id,
self.end_id,
self.pad_id)
source_ids.append(new_seq)
text = elems[2].strip()
seq = [
self.tar_token_ids.get(token, self.unk_id)
for token in text.split()
]
#seq = [self.tar_token_ids.get(token, self.pad_id) for token in text.split()]
seq = seq[:tar_len]
new_seq = helper_fn.pad_with_start(seq, max_tar_len,
self.start_id, self.pad_id)
target_ids.append(new_seq)
new_seq = helper_fn.pad_with_end(seq, max_tar_len, self.end_id,
self.pad_id)
target_loss_ids.append(new_seq)
if ((index + 1) % self.args.batch_size == 0):
res1 = np.asarray(source_ids)
res2 = np.asarray(target_ids)
res3 = np.asarray(target_loss_ids)
res3 = np.reshape(res3, (res3.shape[0], res3.shape[1], 1))
source_ids = []
target_ids = []
target_loss_ids = []
yield res1, res2, res3
if len(source_ids) != 0:
res1 = np.asarray(source_ids)
res2 = np.asarray(target_ids)
res3 = np.asarray(target_loss_ids)
res3 = np.reshape(res3, (res3.shape[0], res3.shape[1], 1))
source_ids = []
target_ids = []
target_loss_ids = []
yield res1, res2, res3
pad_fact_seqs = helper_fn.pad_with_pad([self.pad_id], max_fact_len,
self.pad_id)
def _deal_facts(f):
facts_ids = []
for index, line in enumerate(f):
line = line.strip()
fact_ids = []
elems = line.split('\t')
# if there is no fact, add pad sequence
if elems[1] == config.NO_FACT:
fact_ids.append(pad_fact_seqs)
else:
for text in elems[1:]:
seq = [
self.src_token_ids.get(token, self.unk_id)
for token in text.split()
]
#seq = [self.src_token_ids.get(token, self.pad_id) for token in text.split()]
seq = seq[:max_fact_len]
new_seq = helper_fn.pad_with_pad(
seq, max_fact_len, self.pad_id)
fact_ids.append(new_seq)
# pad fact number
fact_ids = fact_ids[:self.args.fact_number]
fact_ids = fact_ids + [pad_fact_seqs] * (
self.args.fact_number - len(fact_ids))
facts_ids.append(fact_ids)
if ((index + 1) % self.args.batch_size == 0):
res = np.asarray(facts_ids)
facts_ids = []
yield res
if len(facts_ids) != 0:
res = np.asarray(facts_ids)
facts_ids = []
yield res
pad_conv_seqs = helper_fn.pad_with_pad([self.pad_id], max_conv_len,
self.pad_id)
def _deal_convs(f):
convs_ids = []
for index, line in enumerate(f):
line = line.strip()
conv_ids = []
elems = line.split('\t')
# if there is no context, add pad sequence
if elems[1] == config.NO_CONTEXT:
conv_ids.append(pad_conv_seqs)
else:
for text in elems[1:]:
seq = [
self.src_token_ids.get(token, self.unk_id)
for token in text.split()
]
#seq = [self.src_token_ids.get(token, self.pad_id) for token in text.split()]
seq = seq[:max_conv_len]
new_seq = helper_fn.pad_with_pad(
seq, max_conv_len, self.pad_id)
conv_ids.append(new_seq)
conv_ids = conv_ids[:self.args.conv_number]
conv_ids = conv_ids + [pad_conv_seqs] * (
self.args.conv_number - len(conv_ids))
convs_ids.append(conv_ids)
if ((index + 1) % self.args.batch_size == 0):
res = np.asarray(convs_ids)
convs_ids = []
yield res
if len(convs_ids) != 0:
res = np.asarray(convs_ids)
convs_ids = []
yield res
def _check_and_shuffle(source_ids, target_ids, target_loss_ids,
convs_ids, facts_ids):
assert (len(source_ids) == len(target_ids))
assert (len(source_ids) == len(target_loss_ids))
if get_fact == True:
assert (len(source_ids) == len(facts_ids))
if get_conv == True:
assert (len(source_ids) == len(convs_ids))
if get_fact == True and get_conv == True:
source_ids, target_ids, target_loss_ids, convs_ids, facts_ids = \
shuffle(source_ids, target_ids, target_loss_ids, convs_ids, facts_ids)
elif get_fact == True:
source_ids, target_ids, target_loss_ids, facts_ids = shuffle(
source_ids, target_ids, target_loss_ids, facts_ids)
else:
source_ids, target_ids, target_loss_ids = shuffle(
source_ids, target_ids, target_loss_ids)
return (source_ids, target_ids, target_loss_ids, convs_ids,
facts_ids)
def _fit_model(res_src, res_tar, res_tar_loss, res_conv, res_fact):
'''
Please carefully choose the output type to fit with your model's inputs
'''
no_fact_list = [
'pg_only_ques_transformer', 'universal_transformer',
'transformer', 'seq2seq', 'copy_mechanism'
]
fact_list = [
'decex_transformer', 'vspg_transformer', 'spg_transformer',
'pg_transformer', 'memnn'
]
conv_fact_list = [
'transformer_conv_fact', 'universal_transformer_conv_fact'
]
if model_type in no_fact_list:
# only return question and answer as inputs
return ([res_src, res_tar], res_tar_loss)
elif model_type in fact_list:
# return question, answer and facts as inputs
return ([res_src, res_tar, res_fact], res_tar_loss)
elif model_type in conv_fact_list:
# return question, answer, context and facts as inputs
return ([res_src, res_tar, res_conv, res_fact], res_tar_loss)
else:
raise ValueError('The input model type: %s is not available. ' \
'Please chech the file: data_reader.py line: _fit_model' % model_type)
while True:
source_ids, target_ids, target_loss_ids, convs_ids, facts_ids = None, None, None, None, None
print(qa_path)
f_qa = open(qa_path)
res_src, res_tar, res_tar_loss, res_fact, res_conv = None, None, None, None, None
if get_fact == True and get_conv == True:
f_fact = open(fact_path)
f_conv = open(conv_path)
for ((source_ids, target_ids, target_loss_ids), facts_ids, convs_ids) in \
zip(_deal_qa(f_qa), _deal_facts(f_fact), _deal_convs(f_conv)):
res_src, res_tar, res_tar_loss, res_conv, res_fact = \
_check_and_shuffle(source_ids, target_ids, target_loss_ids, convs_ids, facts_ids)
yield _fit_model(res_src, res_tar, res_tar_loss, res_conv,
res_fact)
elif get_fact == True:
f_fact = open(fact_path)
for ((source_ids, target_ids, target_loss_ids), facts_ids) in \
zip(_deal_qa(f_qa), _deal_facts(f_fact)):
res_src, res_tar, res_tar_loss, res_conv, res_fact = \
_check_and_shuffle(source_ids, target_ids, target_loss_ids, convs_ids, facts_ids)
yield _fit_model(res_src, res_tar, res_tar_loss, res_conv,
res_fact)
else:
for (source_ids, target_ids,
target_loss_ids) in _deal_qa(f_qa):
res_src, res_tar, res_tar_loss, res_conv, res_fact = \
_check_and_shuffle(source_ids, target_ids, target_loss_ids, convs_ids, facts_ids)
yield _fit_model(res_src, res_tar, res_tar_loss, res_conv,
res_fact)