def _create_unsupervised_distillation_examples(self, lines,
distillation_prob):
examples = []
cnt = 0
for (i, line) in enumerate(lines):
content = line
guid = int(content["ID"])
text_a = content["sentence1"]
text_b = content["sentence2"]
label = content["gold_label"]
if isinstance(text_a, str) and isinstance(text_b, str):
text_a = tokenization.convert_to_unicode(text_a)
text_b = tokenization.convert_to_unicode(text_b)
input_labels = [label]
examples.append(
data_distillation_feature_classifier.InputExample(
guid=guid,
text_a=text_a,
text_b=text_b,
label=input_labels,
label_probs=distillation_prob[cnt],
label_ratio=0.0,
distillation_ratio=1.0))
cnt += 1
assert cnt == len(distillation_prob)
return examples
def _create_examples(self, lines, LABEL_SPLITTER="__label__"):
re_pattern = u"({}{})".format(LABEL_SPLITTER, "\d+")
examples = []
for (i, line) in enumerate(lines):
try:
guid = i
element_list = re.split(re_pattern, line)
text_a = clean(element_list[-1])
input_labels = clean(element_list[1]).split(LABEL_SPLITTER)[-1]
text_a = tokenization.convert_to_unicode(text_a)
input_labels = [
label.strip() for label in input_labels
if label.strip() in list(self.label2id.keys())
]
examples.append(
data_distillation_feature_classifier.InputExample(
guid=guid,
text_a=text_a,
text_b=None,
label=input_labels,
label_probs=[1.0 / len(self.label2id)] *
len(self.label2id),
label_ratio=1.0,
distillation_ratio=0.0))
except:
print(line, i)
return examples
def _create_supervised_distillation_examples(self,
lines,
distillation_dict_lst,
LABEL_SPLITTER="__label__"):
re_pattern = u"({}{})".format(LABEL_SPLITTER, "\d+")
label_pattern = "(?<={})(\d+)".format(LABEL_SPLITTER)
examples = []
cnt = 0
for (i, line) in enumerate(lines):
try:
guid = i
element_list = re.split(re_pattern, line)
text_a = clean(element_list[-1])
input_labels = []
for l in re.finditer(label_pattern, line):
input_labels.append(l.group())
text_a = tokenization.convert_to_unicode(text_a)
input_labels = [
label.strip() for label in input_labels
if label.strip() in list(self.label2id.keys())
]
if len(input_labels) == 1:
assert int(input_labels[0]
) == distillation_dict_lst[cnt]["label_id"]
examples.append(
data_distillation_feature_classifier.InputExample(
guid=guid,
text_a=text_a,
text_b=None,
label=input_labels,
label_probs=distillation_dict_lst[cnt]["prob"],
label_ratio=1.0,
distillation_ratio=1.0,
feature=distillation_dict_lst[cnt]["feature"]))
cnt += 1
except:
print(line, i)
continue
assert cnt == len(distillation_dict_lst)
return examples
def _create_examples(self, data, lang="zh"):
examples = []
for index in range(len(data)):
content = data[index]
guid = int(content["ID"])
text_a = content["sentence1"]
text_b = content["sentence2"]
label = content["gold_label"]
if isinstance(text_a, str) and isinstance(text_b, str):
examples.append(
data_distillation_feature_classifier.InputExample(
guid=guid,
text_a=clean(text_a),
text_b=clean(text_b),
label=[label],
label_probs=[1.0 / len(self.label2id)] *
len(self.label2id),
label_ratio=1.0,
distillation_ratio=0.0))
return examples