def _create_examples(self, set_type):
"""Creates examples for the training and dev sets."""
examples = []
with open(os.path.join(self.data_dir,
'{}_xs_multitags'.format(set_type)),
'r',
encoding='utf8') as f:
txts = f.read().splitlines()
with open(os.path.join(self.data_dir,
'{}_ys_multitags'.format(set_type)),
'r',
encoding='utf8') as f:
labels = f.read().splitlines()
for (i, n) in enumerate(zip(txts, labels)):
txt, label = n
guid = "%s-%s" % (set_type, i)
text_a = txt.split(' | ')[0]
text_a = ' '.join(list(text_a))
label = label.split(' | ')
label = [' '.join(list(l)) for l in label]
examples.append(
InputExample(guid=guid,
text_a=text_a,
text_b=None,
label=label))
return examples
def _create_examples(self, set_type):
examples = []
with open(os.path.join(self.data_dir,
'{}_xs_converted_tag.txt'.format(set_type)),
'r',
encoding='utf8') as f:
txts = f.read().splitlines()
with open(os.path.join(self.data_dir,
'{}_ys_converted_tag.txt'.format(set_type)),
'r',
encoding='utf8') as f:
labels = f.read().splitlines()
for (i, n) in enumerate(zip(txts, labels)):
txt, label = n
guid = "%s-%s" % (set_type, i)
text_a, text_b = txt.split(' | ')
text_a = ' '.join(list(text_a))
label = label.split(' | ')[0]
label = ' '.join(list(label))
examples.append(
InputExample(guid=guid,
text_a=text_a,
text_b=text_b,
label=label))
return examples
def _create_examples(self, set_type):
"""Creates examples for the training and dev sets."""
examples = []
with open(os.path.join(self.data_dir,
'{}_xs_multitags'.format(set_type)),
'r',
encoding='utf8') as f:
comments = f.read().splitlines()
with open(os.path.join(self.data_dir,
'{}_ys_multitags'.format(set_type)),
'r',
encoding='utf8') as f:
phrases = f.read().splitlines()
for (i, n) in enumerate(zip(comments, phrases)):
comment, phrase = n
guid = "%s-%s" % (set_type, i)
txt_split = comment.split(' | ')
text_a = txt_split[0]
text_a = ' '.join(list(text_a))
tags = txt_split[1:]
phrase_list = phrase.split(' | ')
phrase_list = [' '.join(list(n)) for n in phrase_list]
examples.append(
InputExample(guid=guid,
text_a=text_a,
text_b=phrase_list,
label=tags))
return examples
def get_prediction_examples(self, input_file):
with open(input_file, 'r', encoding='utf8') as f:
phrases = f.read().splitlines()
examples = [
InputExample(guid=str(i), text_a=phrase, text_b=None, label=[])
for i, phrase in enumerate(phrases)
]
return examples
def _create_examples(self, file):
examples = []
DELIMITER=' ||| '
set_type=os.path.basename(file)
with open(file, 'r', encoding='utf8') as f:
datas = f.read().splitlines()
for (i, n) in enumerate(datas):
content,intents,seq = n.split(DELIMITER)
guid = "%s-%s" % (set_type, i)
intents = intents.split('\t')
examples.append(
InputExample(guid=guid, text_a=content, text_b=seq, label=intents))
return examples
def _create_examples(self, set_type):
examples = []
with open(os.path.join(self.data_dir,
'{}_phrases_tags'.format(set_type)),
'r',
encoding='utf8') as f:
tags = f.read().splitlines()
with open(os.path.join(self.data_dir, '{}_phrases'.format(set_type)),
'r',
encoding='utf8') as f:
phrases = f.read().splitlines()
for (i, n) in enumerate(zip(tags, phrases)):
tag, phrase = n
guid = "%s-%s" % (set_type, i)
tag = tag.split(' | ')
examples.append(
InputExample(guid=guid, text_a=phrase, text_b=None, label=tag))
return examples