def _get_single_story_features(self, story_headline, articles):
"""Converts a list of articles to a tensorflow Example."""
def get_text_snippet(article):
if article.text_b:
return " [SEP] ".join([article.text_a, article.text_b])
else:
return article.text_a
story_features = collections.OrderedDict()
story_headline_feature = classifier_data_lib.convert_single_example(
ex_index=self.ex_index,
example=classifier_data_lib.InputExample(guid=self.guid,
text_a=story_headline,
label=self.label),
label_list=[self.label],
max_seq_length=self.len_title,
tokenizer=self.tokenizer)
if self.include_text_snippet_in_example:
story_headline_feature.label_id = story_headline
self._add_feature_with_suffix(feature=story_headline_feature,
suffix="a",
story_features=story_features)
for (article_index, article) in enumerate(articles):
if article_index == self.max_num_articles:
break
article_feature = classifier_data_lib.convert_single_example(
ex_index=self.ex_index,
example=article,
label_list=[self.label],
max_seq_length=self.len_passage,
tokenizer=self.tokenizer)
if self.include_text_snippet_in_example:
article_feature.label_id = get_text_snippet(article)
suffix = chr(ord("b") + article_index)
self._add_feature_with_suffix(feature=article_feature,
suffix=suffix,
story_features=story_features)
# Adds empty features as placeholder.
for article_index in range(len(articles), self.max_num_articles):
suffix = chr(ord("b") + article_index)
empty_article = classifier_data_lib.InputExample(guid=self.guid,
text_a="",
label=self.label)
empty_feature = classifier_data_lib.convert_single_example(
ex_index=self.ex_index,
example=empty_article,
label_list=[self.label],
max_seq_length=self.len_passage,
tokenizer=self.tokenizer)
if self.include_text_snippet_in_example:
empty_feature.label_id = ""
self._add_feature_with_suffix(feature=empty_feature,
suffix=suffix,
story_features=story_features)
return story_features
def __to_feature(self, text, label):
example = classifier_data_lib.InputExample(guid=None,
text_a=text.numpy(),
text_b=None,
label=label.numpy())
feature = classifier_data_lib.convert_single_example(
0, example, self.label_list, self.max_seq_length, self.tokenizer)
return feature.input_ids, feature.input_mask, feature.segment_ids, feature.label_id
def to_feature(text,
label,
label_list=label_list,
max_seq_length=max_seq_length,
tokenizer=tokenizer):
example = classifier_data_lib.InputExample(guid=None,
text_a=text.numpy(),
text_b=None,
label=label.numpy())
feature = classifier_data_lib.convert_single_example(
0, example, label_list, max_seq_length, tokenizer)
return (feature.input_ids, feature.input_mask, feature.segment_ids,
feature.label_id)