def _get_label_mapping(self, provided_split=None, provided_sentences=None):
if self._label_mapping is not None:
return self._label_mapping
if tf.io.gfile.exists(self._label_mapping_path):
self._label_mapping = utils.load_pickle(self._label_mapping_path)
return self._label_mapping
utils.log("Writing label mapping for task", self.name)
tag_counts = collections.Counter()
train_tags = set()
for split in ["train", "dev", "eval"]:
if not tf.io.gfile.exists(
os.path.join(self.config.raw_data_dir(self.name),
split + ".json")):
continue
if split == provided_split:
split_sentences = provided_sentences
else:
split_sentences, _id = self._get_labeled_sentences(split)
for _w, tags, _t in split_sentences:
for tag in tags:
tag_counts[tag] += 1
if provided_split == "train":
train_tags.add(tag)
labels = sorted(tag_counts.keys())
label_mapping = {label: i for i, label in enumerate(labels)}
utils.write_pickle(label_mapping, self._label_mapping_path)
self._label_mapping = label_mapping
return label_mapping
def eval_bagging_best_score(data_dir, split, task_name, selected_idx):
if task_name == 'ccks42ee':
model_name_part = 'electra_ensemble'
else:
model_name_part = 'electra_ensemble2'
all_models = []
for batch_size in [24, 32]:
for max_seq_length in [384, 480, 512]:
for epoch in [2, 3]:
model_name = "{}_{}_{}_{}".format(model_name_part, batch_size,
max_seq_length, epoch)
all_models.append(model_name)
models = [all_models[int(x)] for x in selected_idx.split('-')]
all_nbest = []
all_odds = []
for dire in [os.path.join(data_dir, d) for d in models]:
all_nbest.append(
utils.load_pickle(
(os.path.join(dire, 'models', 'electra_large', 'results',
'{}_qa'.format(task_name),
'{}_{}_all_nbest.pkl'.format(task_name,
split)))))
all_odds.append(
utils.load_json(
(os.path.join(dire, 'models', 'electra_large', 'results',
'{}_qa'.format(task_name),
'{}_{}_null_odds.json'.format(task_name,
split)))))
qid_answers = collections.OrderedDict()
qid_questions = collections.OrderedDict()
dataset = utils.load_json(
(os.path.join(data_dir, model_name_part, 'finetuning_data', task_name,
'{}.json'.format(split))))['data']
for article in dataset:
for paragraph in article["paragraphs"]:
for qa in paragraph['qas']:
_qid = qa['id']
qid_answers[_qid] = qa['answers'] if 'answers' in qa else ''
qid_questions[_qid] = qa['question']
all_nbest = filter_short_ans(all_nbest)
output_dir = os.path.join(data_dir, 'electra_best', 'models',
'electra_large', 'results', 'ccks42bagging',
task_name)
vote1(dataset, all_nbest, all_odds, qid_answers, split, output_dir)
vote2(dataset, all_nbest, all_odds, qid_answers, split, output_dir)
vote3(dataset, all_nbest, all_odds, qid_answers, qid_questions, models,
split, output_dir)
def _get_label_mapping(self, provided_split=None, provided_sentences=None):
# import pdb; pdb.set_trace() # IBO
if self._label_mapping is not None:
return self._label_mapping
if tf.io.gfile.exists(self._label_mapping_path):
self._label_mapping = utils.load_pickle(self._label_mapping_path)
return self._label_mapping
utils.log("Writing label mapping for task", self.name)
tag_counts = collections.Counter()
train_tags = set()
for split in ["train", "dev", "test"]:
if not tf.io.gfile.exists(
os.path.join(self.config.raw_data_dir(self.name),
split + ".txt")):
continue
if split == provided_split:
split_sentences = provided_sentences
else:
split_sentences = self._get_labeled_sentences(split)
for _, tags in split_sentences:
if not self._is_token_level:
span_labels = tagging_utils.get_span_labels(tags)
tags = tagging_utils.get_tags(span_labels, len(tags),
LABEL_ENCODING)
for tag in tags:
tag_counts[tag] += 1
if provided_split == "train":
train_tags.add(tag)
if self.name == "ccg":
infrequent_tags = []
for tag in tag_counts:
if tag not in train_tags:
infrequent_tags.append(tag)
label_mapping = {
label: i
for i, label in enumerate(
sorted(
filter(lambda t: t not in infrequent_tags,
tag_counts.keys())))
}
n = len(label_mapping)
for tag in infrequent_tags:
label_mapping[tag] = n
else:
labels = sorted(tag_counts.keys())
label_mapping = {label: i for i, label in enumerate(labels)}
utils.write_pickle(label_mapping, self._label_mapping_path)
self._label_mapping = label_mapping
return label_mapping
for max_seq_length in [384, 480, 512]:
for epoch in [2, 3]:
model_name = "{}_{}_{}_{}".format(model_name_part, batch_size,
max_seq_length, epoch)
full_models.append(model_name)
models = full_models
models_predictions = collections.OrderedDict()
for d in models:
dire = os.path.join(data_dir, d)
try:
prediction = collections.OrderedDict()
prediction['eval_all_nbest'] = filter_short_ans(
utils.load_pickle(
(os.path.join(dire, 'models', 'electra_large', 'results',
'{}_qa'.format(task_name),
'{}_{}_all_nbest.pkl'.format(task_name,
split)))))
prediction['squad_null_odds'] = utils.load_json(
(os.path.join(dire, 'models', 'electra_large', 'results',
'{}_qa'.format(task_name),
'{}_{}_null_odds.json'.format(task_name, split))))
models_predictions[d] = prediction
except:
utils.log(
"Error at loading all_nbest.pkl & null_odds.json for model {}".
format(d))
continue
dataset = \
utils.load_json((os.path.join(data_dir, model_name_part, 'finetuning_data', task_name, '{}.json'.format(split))))[
