def pred_word_to_list(pred_word, special_symb_ls):
index_special_ls = []
pred_word = [pred_word]
ind_pred_word = 0
counter = 0
while True:
counter += 1
index_special_ls = []
_pred_word = pred_word[ind_pred_word]
# Looking for all special character (we only look at the first one found)
for special_symb in special_symb_ls:
index_special_ls.append(_pred_word.find(special_symb))
indexes = np.argsort(index_special_ls)
index_special_char = -1
# Getting the index and the character of the first special character if nothing we get -1
for ind, a in enumerate(indexes):
if index_special_ls[a] >= 0:
special_symb = special_symb_ls[a]
index_special_char = index_special_ls[a]
break
if ind > len(indexes):
index_special_char = -1
special_symb = ""
break
# if found a special character
if (index_special_char) >= 0:
starting_seq = [_pred_word[:index_special_char]
] if index_special_char > 0 else []
middle = [
_pred_word[index_special_char:index_special_char +
len(special_symb)]
]
end_seq = [_pred_word[index_special_char + len(special_symb):]]
if len(end_seq[0].strip()) == 0:
end_seq = []
_pred_word_ls = starting_seq + middle + end_seq
pred_word[ind_pred_word] = _pred_word_ls[0]
if len(_pred_word_ls) > 0:
pred_word.extend(_pred_word_ls[1:])
ind_pred_word += 1
pdb.set_trace()
if len(starting_seq) > 0:
ind_pred_word += 1
else:
ind_pred_word += 1
pdb.set_trace()
if ind_pred_word >= len(pred_word):
break
new_word = []
# transform the way we splitted in list of characters (including special ones)
for word in pred_word:
if word in special_symb_ls:
new_word.append(word)
else:
new_word.extend(list(word))
return new_word
def __init__(
self, config,
**kwargs): # tasks, num_labels_per_task, mask_id, encoder_class):
super(BertMultiTask, self).__init__(config)
# encoder_class only BertModel or RobertaModel
# some arguments specific to BertMultiTask could be passed in config or in kwargs
encoder_class = kwargs["encoder"]
tasks = kwargs["tasks"] if "tasks" in kwargs else config.tasks
num_labels_per_task = kwargs[
"num_labels_per_task"] if "num_labels_per_task" in kwargs else config.tasks
mask_id = kwargs["mask_id"] if "mask_id" in kwargs else config.tasks
config.dropout_classifier = kwargs.get("dropout_classifier", 0.1)
if "parsing" in tasks:
config.graph_head_hidden_size_mlp_arc = kwargs.get(
"graph_head_hidden_size_mlp_arc", 500)
config.graph_head_hidden_size_mlp_rel = kwargs.get(
"graph_head_hidden_size_mlp_rel", 200)
pdb.set_trace()
self.encoder = encoder_class(config)
print("BertMultitask instantiated with {} encoder".format(
self.encoder.__class__.__name__))
self.config = config
assert isinstance(num_labels_per_task, dict)
assert isinstance(tasks, list) and len(
tasks) >= 1, "config.tasks should be a list of len >=1"
self.head = nn.ModuleDict()
self.mask_index_bert = mask_id
self.tasks = tasks #
self.tasks_available = tasks # all tasks available in the model (not only the one we want to use at a given run (self.tasks))
self.task_parameters = TASKS_PARAMETER
self.layer_wise_attention = None
self.labels_supported = [
label for task in tasks
for label in self.task_parameters[task]["label"]
]
self.sanity_checking_num_labels_per_task(num_labels_per_task, tasks,
self.task_parameters)
self.num_labels_dic = num_labels_per_task
for task in TASKS_PARAMETER:
if task in tasks:
num_label = get_key_name_num_label(task, self.task_parameters)
if not self.task_parameters[task]["num_labels_mandatory"]:
# in this case we need to define and load MLM head of the model
self.head[task] = eval(self.task_parameters[task]["head"])(
config
) #, self.encoder.embeddings.word_embeddings.weight)
else:
self.head[task] = eval(self.task_parameters[task]["head"])(
config, num_labels=self.num_labels_dic[num_label])
else:
# we define empty heads for downstream use
self.head[task] = None
def get_loss(loss_func,
label,
num_label_dic,
labels,
logits_dict,
task,
logit_label,
head_label=None):
if label not in ["heads", "types"]:
try:
loss = loss_func(
logits_dict[logit_label].view(-1,
num_label_dic[logit_label]),
labels.view(-1))
except Exception as e:
print(e)
pdb.set_trace()
print("ERROR task {} num_label {} , labels {} ".format(
task, num_label_dic, labels.view(-1)))
raise (e)
elif label == "heads":
# trying alternative way for loss
loss = CrossEntropyLoss(
ignore_index=LABEL_PARAMETER[label]["pad_value"],
reduction="sum")(logits_dict[logit_label].view(
-1, logits_dict[logit_label].size(2)), labels.view(-1))
# other possibilities is to do log softmax then L1 loss (lead to other results)
elif label == "types":
assert head_label is not None, "ERROR head_label should be passed"
# gold label after removing 0 gold
gold = labels[head_label != LABEL_PARAMETER["heads"]["pad_value"]]
# pred logits (after removing -1) on the gold heads
pred = logits_dict["parsing-types"][(
head_label != LABEL_PARAMETER["heads"]["pad_value"]
).nonzero()[:, 0], (
head_label != LABEL_PARAMETER["heads"]["pad_value"]
).nonzero()[:, 1], head_label[
head_label != LABEL_PARAMETER["heads"]["pad_value"]]]
# remark : in the way it's coded for paring : the padding is already removed (so ignore index is null)
loss = loss_func(pred, gold)
return loss
def preprocess_batch_string_for_bert(batch,
start_token,
end_token,
rp_space=False):
"""
adding starting and ending token in raw sentences
:param batch:
:return:
"""
for i in range(len(batch)):
try:
batch[i][0] = start_token
except:
pdb.set_trace()
batch[i][-1] = end_token
if rp_space:
batch[i] = rp_space_func(batch[i])
batch[i] = " ".join(batch[i])
return batch
def sanity_test_parsing_label(labels, output_tokens_tensor_new,
input_alignement_with_raw, cumulate_shift):
for sent in range(labels.size(0)):
ind_max = len(cumulate_shift[sent]) - 1
for _ in range(5):
ind = np.random.choice(range(ind_max))
# the new label must be equal to the old one at the corresponding position + 1 + the number of non-first-bpe-token (original indexing of the label)
if output_tokens_tensor_new[sent][ind] not in [
ROOT_HEADS_INDEX + 1, END_HEADS_INDEX,
PAD_ID_LOSS_STANDART
]:
try:
assert output_tokens_tensor_new[sent][ind] == labels[sent, int(input_alignement_with_raw[sent][ind])]+CLS_ADJUST+cumulate_shift[sent][labels[sent, int(input_alignement_with_raw[sent][ind])]], \
"ERROR sent {} ind word {} " \
"new {} and old {} cumulted {} ".format(sent, ind, output_tokens_tensor_new[sent][ind],
labels[sent, input_alignement_with_raw[sent][ind]], cumulate_shift[sent][ind])
except AssertionError as e:
print(e)
pdb.set_trace()
def main(args, dict_path, model_dir):
encoder = BERT_MODEL_DIC[args.bert_model]["encoder"]
vocab_size = BERT_MODEL_DIC[args.bert_model]["vocab_size"]
voc_tokenizer = BERT_MODEL_DIC[args.bert_model]["vocab"]
tokenizer = eval(BERT_MODEL_DIC[args.bert_model]["tokenizer"])
random.seed(args.seed)
if args.model_id_pref is None:
run_id = str(uuid4())[:4]
else:
run_id = args.model_id_pref + "1"
if args.init_args_dir is None:
dict_path += "/" + run_id
os.mkdir(dict_path)
tokenizer = tokenizer.from_pretrained(voc_tokenizer,
do_lower_case=args.case == "lower",
shuffle_bpe_embedding=False)
mask_id = tokenizer.encode([
"[MASK]"
])[0] if args.bert_model == "bert_base_multilingual_cased" else None
_dev_path = args.dev_path if args.dev_path is not None else args.train_path
word_dictionary, word_norm_dictionary, char_dictionary, pos_dictionary, \
xpos_dictionary, type_dictionary = \
conllu_data.load_dict(dict_path=dict_path,
train_path=args.train_path if args.init_args_dir is None else None,
dev_path=args.dev_path if args.init_args_dir is None else None,
test_path=args.test_paths if args.init_args_dir is None else None,
word_embed_dict={},
dry_run=False,
expand_vocab=False,
word_normalization=True,
force_new_dic=False,
tasks=args.tasks,
pos_specific_data_set=None,
#pos_specific_data_set=args.train_path[1] if len(args.tasks) > 1 and len(
# args.train_path) > 1 and "pos" in args.tasks else None,
case=args.case,
# if not normalize pos or parsing in tasks we don't need dictionary
do_not_fill_dictionaries=len(set(["normalize", "pos", "parsing"]) & set(
[task for tasks in args.tasks for task in tasks])) == 0,
add_start_char=True if args.init_args_dir is None else None,
verbose=1)
num_labels_per_task, task_to_label_dictionary = get_vocab_size_and_dictionary_per_task(
[task for tasks in args.tasks for task in tasks],
vocab_bert_wordpieces_len=vocab_size,
pos_dictionary=pos_dictionary,
type_dictionary=type_dictionary,
task_parameters=TASKS_PARAMETER)
model = make_bert_multitask(
args=args,
pretrained_model_dir=model_dir,
init_args_dir=args.init_args_dir,
tasks=[task for tasks in args.tasks for task in tasks],
mask_id=mask_id,
encoder=encoder,
num_labels_per_task=num_labels_per_task)
def get_n_params(model):
pp = 0
for p in list(model.parameters()):
nn = 1
for s in list(p.size()):
nn = nn * s
pp += nn
return pp
param = get_n_params(model)
model_parameters = filter(lambda p: p.requires_grad, model.parameters())
params = sum([np.prod(p.size()) for p in model_parameters])
pdb.set_trace()
data = ["I am here", "How are you"]
model.eval()
n_obs = args.n_sent
max_len = args.max_seq_len
lang_ls = args.raw_text_code
data_all, y_all = load_lang_ls(DATA_UD_RAW, lang_ls=lang_ls)
reg = linear_model.LogisticRegression()
X_train = OrderedDict()
X_test = OrderedDict()
y_train = OrderedDict()
y_test = OrderedDict()
# just to get the keyw
layer_head_att = get_hidden_representation(data,
model,
tokenizer,
max_len=max_len,
output_dic=False,
pad_below_max_len=True)
layer_head_att = layer_head_att[0]
report_ls = []
accuracy_dic = OrderedDict()
sampling = args.sampling
for ind, layer_head in enumerate(list(layer_head_att.keys())):
report = OrderedDict()
accuracy_ls = []
layer_head = list(
layer_head_att.keys())[len(list(layer_head_att.keys())) - ind - 1]
for _ in range(sampling):
sample_ind = random.sample(population=range(len(data_all)),
k=n_obs)
sample_ind_test = random.sample(population=range(len(data_all)),
k=n_obs)
data = data_all[sample_ind]
y = y_all[sample_ind]
all = get_hidden_representation(data,
model,
tokenizer,
max_len=max_len,
output_dic=False,
pad_below_max_len=True)
layer_head_att = all[0]
#pdb.set_trace()
def reshape_x(z):
return np.array(z.view(z.size(0) * z.size(1), -1))
def reshape_y(z, n_seq):
'multiply each element n_seq times'
new_z = []
for _z in z:
#for _ in range(n_seq):
new_z.extend([_z for _ in range(n_seq)])
return np.array(new_z)
#return np.array(z.view(z.size(0), -1).transpose(1, 0))
#X_train[layer_head] = np.array(layer_head_att[layer_head].view(layer_head_att[layer_head].size(0), -1).transpose(1,0))
X_train[layer_head] = reshape_x(layer_head_att[layer_head])
y_train[layer_head] = reshape_y(y, max_len)
#db.set_trace()
#y_train[layer_head] = y
reg.fit(X=X_train[layer_head], y=y_train[layer_head])
# test
data_test = data_all[sample_ind_test]
layer_head_att_test = get_hidden_representation(
data_test,
model,
tokenizer,
max_len=max_len,
output_dic=False,
pad_below_max_len=True)
X_test[layer_head] = reshape_x(layer_head_att_test[layer_head])
y_test[layer_head] = reshape_y(y_all[sample_ind_test], max_len)
y_pred = reg.predict(X_test[layer_head])
Accuracy = np.sum(
(y_test[layer_head] == y_pred)) / len(y_test[layer_head])
accuracy_ls.append(Accuracy)
accuracy_dic[layer_head] = np.mean(accuracy_ls)
layer = layer_head.split("-")[0]
if layer not in accuracy_dic:
accuracy_dic[layer] = []
accuracy_dic[layer].append(np.mean(accuracy_ls))
print(
f"Regression {layer_head} Accuracy test {np.mean(accuracy_ls)} on {n_obs * max_len}"
f" word sample from {len(lang_ls)} languages task {args.tasks} args {'/'.join(args.init_args_dir.split('/')[-2:]) if args.init_args_dir is not None else None} "
f"bert {args.bert_model} random init {args.random_init} std {np.std(accuracy_ls)} sampling {len(accuracy_ls)}=={sampling}"
)
#report["model_type"] = args.bert_model if args.init_args_dir is None else args.tasks[0][0]+"-tune"
#report["accuracy"] = np.mean(accuracy_ls)
#report["sampling"] = len(accuracy_ls)
#report["std"] = np.std(accuracy_ls)
#report["n_sent"] = n_obs
#report["n_obs"] = n_obs*max_len
report = report_template(
metric_val="accuracy",
subsample=",".join(lang_ls),
info_score_val=sampling,
score_val=np.mean(accuracy_ls),
n_sents=n_obs,
avg_per_sent=np.std(accuracy_ls),
n_tokens_score=n_obs * max_len,
model_full_name_val=run_id,
task="attention_analysis",
evaluation_script_val="exact_match",
model_args_dir=args.init_args_dir
if args.init_args_dir is not None else args.random_init,
token_type="word",
report_path_val=None,
data_val=layer_head)
report_ls.append(report)
# break
for key in accuracy_dic:
print(
f"Summary {key} {np.mean(accuracy_dic[key])} model word sample from {len(lang_ls)} languages task {args.tasks} args {'/'.join(args.init_args_dir.split('/')[-2:]) if args.init_args_dir is not None else None} "
f"bert {args.bert_model} random init {args.random_init} std {np.std(accuracy_ls)} sampling {len(accuracy_ls)}=={sampling}"
)
if args.report_dir is None:
report_dir = PROJECT_PATH + f"/../../analysis/attention_analysis/report/{run_id}-report"
os.mkdir(report_dir)
else:
report_dir = args.report_dir
assert os.path.isdir(report_dir)
with open(report_dir + "/report.json", "w") as f:
json.dump(report_ls, f)
overall_report = args.overall_report_dir + "/" + args.overall_label + "-grid-report.json"
with open(overall_report, "r") as g:
report_all = json.load(g)
report_all.extend(report_ls)
with open(overall_report, "w") as file:
json.dump(report_all, file)
print("{} {} ".format(REPORT_FLAG_DIR_STR, overall_report))
def forward(self,
input_ids_dict,
token_type_ids=None,
attention_mask=None,
labels=None,
head_masks=None):
if labels is None:
labels = OrderedDict()
sequence_output_dict = OrderedDict()
logits_dict = OrderedDict()
loss_dict = OrderedDict()
# sanity check the labels : they should all be in
for label, value in labels.items():
assert label in self.labels_supported, "label {} in {} not supported".format(
label, self.labels_supported)
# task_wise layer attention
printout_allocated_gpu_memory(1, " foward starting ")
for input_name, input_tensors in input_ids_dict.items():
# not able to output all layers anymore
sequence_output = self.encoder(
input_tensors,
token_type_ids=None,
attention_mask=attention_mask[input_name])
try:
assert len(
sequence_output
) == 2 + int(self.config.output_attentions) + int(
self.config.output_hidden_states
), f"ERROR should be {2+int(self.config.output_attentions)+int(self.config.output_hidden_states_per_head)+int(self.config.output_hidden_states)} : check that you're not outputing also all hidden states"
# add to remove : int(self.config.output_hidden_states_per_head)
except Exception as e:
#print(f"Exception output sequence {e}")
pdb.set_trace()
assert len(sequence_output) == 2
logits = sequence_output[0]
sequence_output_dict[input_name] = logits
printout_allocated_gpu_memory(1, " forward pass bert")
for task in self.tasks:
# we don't use mask for parsing heads (cf. test performed below : the -1 already ignore the heads we don't want)
# NB : head_masks for parsing only applies to heads not types
head_masks_task = None # head_masks.get(task, None) if task != "parsing" else None
# NB : head_mask means masks specific the the module heads (nothing related to parsing !! )
assert self.task_parameters[task]["input"] in sequence_output_dict, \
"ERROR input {} of task {} was not found in input_ids_dict {}" \
" and therefore not in sequence_output_dict {} ".format(self.task_parameters[task]["input"],
task, input_ids_dict.keys(),
sequence_output_dict.keys())
if not self.head[
task].__class__.__name__ == BertOnlyMLMHead.__name__: #isinstance(self.head[task], BertOnlyMLMHead):
logits_dict[task] = self.head[task](
sequence_output_dict[self.task_parameters[task]["input"]],
head_mask=head_masks_task)
else:
logits_dict[task] = self.head[task](
sequence_output_dict[self.task_parameters[task]["input"]])
# test performed : (logits_dict[task][0][1,2,:20]==float('-inf'))==(labels["parsing_heads"][1,:20]==-1)
# handle several labels at output (e.g parsing)
printout_allocated_gpu_memory(1,
" foward pass head {}".format(task))
logits_dict = self.rename_multi_modal_task_logits(
labels=self.task_parameters[task]["label"],
task=task,
logits_dict=logits_dict,
task_parameters=self.task_parameters)
printout_allocated_gpu_memory(1, "after renaming")
for logit_label in logits_dict:
label = re.match("(.*)-(.*)", logit_label)
assert label is not None, "ERROR logit_label {}".format(
logit_label)
label = label.group(2)
if label in self.task_parameters[task]["label"]:
_labels = None
if self.task_parameters[task]["input"] == "input_masked":
_labels = labels.get(label)
if _labels is not None:
_labels = _labels.clone()
_labels[input_ids_dict["input_masked"] != self.
mask_index_bert] = PAD_ID_LOSS_STANDART
else:
_labels = labels.get(label)
printout_allocated_gpu_memory(
1, " get label head {}".format(logit_label))
if _labels is not None:
#print("LABEL label {} {}".format(label, _labels))
loss_dict[logit_label] = self.get_loss(
loss_func=self.task_parameters[task]["loss"],
label=label,
num_label_dic=self.num_labels_dic,
labels=_labels,
logits_dict=logits_dict,
task=task,
logit_label=logit_label,
head_label=labels["heads"]
if label == "types" else None)
printout_allocated_gpu_memory(1,
" get loss {}".format(task))
printout_allocated_gpu_memory(
1, " puting to cpu {}".format(logit_label))
# thrid output is for potential attention weights
output = (
logits_dict,
loss_dict,
)
output = output + sequence_output[2:]
return output
def write_conll_multitask(format, dir_pred, dir_original, src_text_ls,
pred_per_task, tasks, task_parameters, cp_paste=False, gold=False,
all_indexes=None, sep_token=None, cls_token=None,
ind_batch=0, new_file=False, cut_sent=False, verbose=0):
assert format in ["conll"]
max_len_word = None
writing_top = 1
# assert each task is predicting as many sample per batch
pred_task_len_former = -1
task_former = ""
# assertion on number of samples predicted
for task_label in pred_per_task:
pred_task_len = len(pred_per_task[task_label]) if gold else len(pred_per_task[task_label][writing_top-1])
_task = re.match("(.*)-(.*)", task_label)
if _task is not None: # , "ERROR writer could not match {}".format(task_label)
task = _task.group(1)
else:
task = task_label
if pred_task_len_former > 0:
assert pred_task_len == pred_task_len_former, \
"ERROR {} and {} task ".format(task_former, task_label)
if not gold:
assert pred_task_len == len(src_text_ls[task_parameters[task]["input"]]), "ERROR src len {} and pred len {} ".format(len(src_text_ls[task_parameters[task]["input"]]),pred_task_len)
# we check also other input length
if src_text_ls.get("input_masked") is not None:
assert pred_task_len == len(src_text_ls["input_masked"])
if src_text_ls.get("wordpieces_inputs_words") is not None:
assert pred_task_len == len(src_text_ls["wordpieces_inputs_words"]), "ERROR mismatch source " \
"wordpieces_inputs_words {} " \
"and prediction {} ".format(src_text_ls, pred_per_task[task_label])
if src_text_ls.get("wordpieces_inputs_raw_tokens") is not None:
assert pred_task_len == len(src_text_ls["wordpieces_inputs_raw_tokens"]), \
"ERROR mismatch source wordpieces_inputs_" \
"raw_tokens {} and prediction {} ".format(src_text_ls, pred_per_task[task_label])
try:
assert pred_task_len == all_indexes.shape[0], "ERROR mismatch index {} and all_indexes {} : pred {}".format(pred_task_len, all_indexes.shape[0], pred_per_task[task_label])
except:
pdb.set_trace()
pred_task_len_former = pred_task_len
task_former = task_label
if format == "conll":
mode_write = "w" if new_file else "a"
if new_file:
printing("CREATING NEW FILE (io_/dat/normalized_writer) : {} ", var=[dir_pred], verbose=verbose,
verbose_level=1)
pos_label = "pos-pos" if not gold else "pos"
types_label = "parsing-types" if not gold else "types"
heads_label = "parsing-heads" if not gold else "heads"
n_masks_mwe_label = "n_masks_mwe-n_masks_mwe" if not gold else "n_masks_mwe"
mwe_detection_label = "mwe_detection-mwe_detection" if not gold else "mwe_detection"
with open(dir_pred, mode_write) as norm_file:
with open(dir_original, mode_write) as original:
len_original = 0
for ind_sent in range(all_indexes.shape[0]):
pred_sent = OrderedDict()
# NB : length assertion for each input-output (correcting if possible)
# TODO standartize !! INCONSITENCIES WHEN GOLD TRUE AND GOLD FALSE, IF GOLD : pred_per_task is indexed by labels (no relation 1-1 to task and src ! )
for task_label_or_gold_label in pred_per_task:
#task, _, label_processed = get_task_name_based_on_logit_label(task_label, label_processed)
if gold:
pred_sent[task_label_or_gold_label] = pred_per_task[task_label_or_gold_label][ind_sent]
else:
pred_sent[task_label_or_gold_label] = pred_per_task[task_label_or_gold_label][writing_top-1][ind_sent]
try:
# TODO : standartize (the first if is needed because we handle at the same time gold data indexed by label and prediction labelled by task+label
if gold:
try:
src = src_text_ls[LABEL_PARAMETER[task_label_or_gold_label]["default_input"]][ind_sent]
except Exception as e:
src = src_text_ls["input_masked"][ind_sent]
else:
_task = re.match("(.*)-(.*)", task_label_or_gold_label)
assert _task is not None#, "ERROR writer could not match {}".format(task_label)
_label = _task.group(2)
_task = _task.group(1)
src = src_text_ls[TASKS_PARAMETER[_task]["input"]][ind_sent]
assert len(src) == len(pred_sent[task_label_or_gold_label]),"WARNING : (writer) task {} original_sent len {} {} \n predicted sent len {} {}".format(task_label_or_gold_label, len(src), src,len(pred_sent[task_label_or_gold_label]), pred_sent[task_label_or_gold_label])
except AssertionError as e:
print(e)
pdb.set_trace()
if len(src) > len(pred_sent[task_label_or_gold_label]):
pred_sent[task_label_or_gold_label].extend(["UNK" for _ in range(len(src)-len(pred_sent[task_label_or_gold_label]))])
print("WARNING (writer) : original larger than prediction : so appending UNK token for writing")
else:
print("WARNING (writer) : original smaller than prediction for ")
norm_file.write("#\n")
original.write("#\n")
norm_file.write("#sent_id = {} \n".format(ind_sent+ind_batch+1))
original.write("#sent_id = {} \n".format(ind_sent+ind_batch+1))
ind_adjust = 0
#for ind, original_token in enumerate(original_sent):
last_mwe_index = -1
adjust_mwe = 0
for ind in all_indexes[ind_sent, :]:
# WE REMOVE SPECIAL TOKENS ONLY IF THEY APPEAR AT THE BEGINING OR AT THE END
# on the source token !! (it tells us when we stop) (we nevern want to use gold information)
if "-" in ind and ind != "-1":
matching_mwe_ind = re.match("([0-9]+)-([0-9]+)", str(ind))
assert matching_mwe_ind is not None, "ERROR ind is {} : could not found mwe index".format(ind)
last_mwe_index = int(matching_mwe_ind.group(2))
ind_mwe = int(matching_mwe_ind.group(1))
original_token = src_text_ls["wordpieces_inputs_raw_tokens"][ind_sent][ind_mwe] if mwe_detection_label in pred_per_task or "wordpieces_inputs_words" in pred_per_task or n_masks_mwe_label in pred_per_task else "NOT_NEEDED"
adjust_mwe += (last_mwe_index-ind_mwe)
#assert ind_adjust == 0, "ERROR not supported"
mwe_meta = "Norm={}|mwe_detection={}|n_masks_mwe={}".format("_", pred_sent[mwe_detection_label][ind_mwe] if mwe_detection_label in pred_per_task else "_",
pred_sent[n_masks_mwe_label][ind_mwe] if n_masks_mwe_label in pred_per_task else "_")
norm_file.write("{index}\t{original}\t_\t{pos}\t_\t_\t{dep}\t_\t{types}\t{norm}\n".format(index=ind, original=original_token, pos="_", types="_", dep="_", norm=mwe_meta))
original.write("{}\t{}\t_\t_\t_\t_\t_\t_\t{}\t_\n".format(ind, original_token, "_"))
continue
else:
ind = int(ind)
try:
if "normalize" in [task for _tasks in tasks for task in _tasks]:
original_token = src_text_ls["wordpiece_words_src_aligned_with_norm"][ind_sent][ind]
original_pretokenized_field = "wordpiece_words_src_aligned_with_norm"
else:
original_token = src_text_ls["wordpieces_inputs_words"][ind_sent][ind]
original_pretokenized_field = "wordpieces_inputs_words"
except Exception as e:
original_token = src_text_ls["input_masked"][ind_sent][ind]
original_pretokenized_field = "input_masked"
# asserting that we have everything together on the source side
if ind > last_mwe_index:
if src_text_ls.get("wordpieces_inputs_raw_tokens") is not None:
try:
assert src_text_ls[original_pretokenized_field][ind_sent][ind] == src_text_ls["wordpieces_inputs_raw_tokens"][ind_sent][ind-adjust_mwe], \
"ERROR sequence {} on non-mwe tokens : raw and tokenized " \
"should be same but are raw {} tokenized {}".format(original_pretokenized_field, src_text_ls["wordpieces_inputs_raw_tokens"][ind_sent][ind],
src_text_ls[original_pretokenized_field][ind_sent][ind+adjust_mwe])
except:
print("WARNING sanity checking input failed (nomalized_writer) (might be due to dropout) {}".format(e))
max_len_word = max(len(original_token), len_original)
#if original_token in SPECIAL_TOKEN_LS and (ind+1 == len(original_sent) or ind == 0):
if (original_token in SPECIAL_TOKEN_LS or original_token in [cls_token, sep_token]):
# ind 0 is skipped because it corresponds to CLS
ind_adjust = 1
continue
pos = pred_sent[pos_label][ind] if pos_label in pred_per_task else "_"
types = pred_sent[types_label][ind] if types_label in pred_per_task else "_"
heads = pred_sent[heads_label][ind] if heads_label in pred_per_task else ind - 1
tenth_col = "Norm={}|mwe_detection={}|n_masks_mwe={}".format(pred_sent["normalize"][ind] if "normalize" in pred_per_task else "_",
pred_sent[mwe_detection_label][ind-adjust_mwe] if mwe_detection_label in pred_per_task else "_",
pred_sent[n_masks_mwe_label][ind-adjust_mwe] if n_masks_mwe_label in pred_per_task else "_")
norm_file.write("{index}\t{original}\t_\t{pos}\t_\t_\t{dep}\t_\t{types}\t{norm}\n".format(index=ind, original=original_token, pos=pos, types=types, dep=heads, norm=tenth_col))
original.write("{}\t{}\t_\t_\t_\t_\t_\t_\t{}\t_\n".format(ind, original_token, ind-1))
if cut_sent:
if ind > 50:
break
print("CUTTING SENT index {}>50 ".format(ind))
norm_file.write("\n")
original.write("\n")
printing("WRITING predicted batch of {} original and {} normalized", var=[dir_original, dir_pred], verbose=verbose, verbose_level=2)
assert max_len_word is not None, "ERROR : something went wrong in the writer"
return max_len_word
def overall_word_level_metric_measure(task_label,
pred_label,
gold_sent_ls_dict,
pred_sent_ls_topk_dict,
topk,
metric="exact_match",
samples=None,
src_detokenized=None,
reference_word_dic=None,
compute_intersection_score=True,
mask_token=None,
cls_token=None,
sep_token=None,
agg_func_ls=None):
"""
'metric' based on a word level comparison of (pred,gold) : e.g : exact_match , edit
'agg_func' based on a aggregation func to get the overall batch score : e.g : sum
:param metric:
:param agg_func:
:return batch : score, number of token measured
"""
if samples is None:
samples = ["all"]
if agg_func_ls is None:
agg_func_ls = ["sum"]
if task_label == "parsing_types":
assert "parsing_heads" in gold_sent_ls_dict and "parsing_heads" in pred_sent_ls_topk_dict, \
"ERROR : to compute the score of parsing_types : parsing_heads is required "
assert isinstance(samples, list)
assert len(set(samples) & set(AVAILABLE_EVALUATION_SAMPLE_FILTER)) > 0, \
"ERROR : one of the samples in {} not supported {}".format(samples, AVAILABLE_EVALUATION_SAMPLE_FILTER)
assert isinstance(agg_func_ls, list)
pred_sent_ls_topk = pred_sent_ls_topk_dict[pred_label]
gold_sent_ls = gold_sent_ls_dict[task_label]
assert len(
pred_sent_ls_topk
) == topk, "ERROR topk not consistent with prediction list ".format(
len(pred_sent_ls_topk), topk)
overall_score_ls_sent = []
intersected_samples = []
if compute_intersection_score:
intersected_samples, sample_to_intersesct = get_intersection_score(
samples)
overall_filter_ls = {
sample: []
for sample in samples + intersected_samples
}
skipping_sent = 0
for gold_ind_sent, gold_sent in enumerate(gold_sent_ls):
try:
assert len(gold_sent) == len(
pred_sent_ls_topk[0][gold_ind_sent]
), "ERROR : gold_sent : {} len {} and pred_sent_ls_topk[0][gold_ind_sent] {} len {} ".format(
gold_sent, len(gold_sent), pred_sent_ls_topk[0][gold_ind_sent],
len(pred_sent_ls_topk[0][gold_ind_sent]))
# WARNING : this might not be true in POS mode for some cases (when mask bpe is used)
except Exception as e:
raise (e)
print("ERROR (scoring/report) on task {} ".format(task_label), e)
if len(gold_sent) > len(pred_sent_ls_topk[0][gold_ind_sent]):
counter = 0
n_to_solve = len(gold_sent) - len(
pred_sent_ls_topk[0][gold_ind_sent])
for ind in range(n_to_solve):
counter += gold_sent[-n_to_solve:][ind] == "_PAD_POS"
if n_to_solve == counter:
gold_sent = gold_sent[:-n_to_solve]
src_detokenized[gold_ind_sent] = src_detokenized[
gold_ind_sent][:-n_to_solve]
print(
"WARNING : we handled mismatch between pred len/src and gold len by cutting it based on "
"GOLD padding (SHOULD BE RAISED IN TASK POS)")
# NB : this should be handle properly :
# the detokenization has a problem when dropout_bpe_mask is not null
else:
if pred_sent_ls_topk[0][gold_ind_sent][-1] == "[SEP]":
pred_sent_ls_topk[0][gold_ind_sent] = pred_sent_ls_topk[
0][gold_ind_sent] + [
"[SEP]" for _ in range(
len(gold_sent) -
len(pred_sent_ls_topk[0][gold_ind_sent]))
]
print("APPENDING pred_sent_ls_topk[0] {} with {} ".
format(
len(gold_sent) -
len(pred_sent_ls_topk[0][gold_ind_sent]),
pred_sent_ls_topk[0][gold_ind_sent]))
assert len(gold_sent) == len(
pred_sent_ls_topk[0][gold_ind_sent])
else:
print(
Exception(
"ERROR {} : could not handled mismatch between pred {} len/src {} "
"and gold len by cutting it based on GOLD padding (SHOULD BE RAISED IN TASK POS)"
.format(e, gold_sent,
pred_sent_ls_topk[0][gold_ind_sent])))
skipping_sent += len(gold_sent_ls)
overall_score_ls_sent = [[0]]
break
else:
skipping_sent += len(gold_sent_ls)
overall_score_ls_sent = [[0]]
break
if src_detokenized is not None and samples[0] != "all" and len(
samples) > 1:
# otherise we don't need src_detokenized
assert len(gold_sent) == len(
src_detokenized[gold_ind_sent]
), "ERROR src_detokenized {} and gold_sent_ls for sent {} have different length ".format(
gold_sent, src_detokenized[gold_ind_sent])
score_sent = []
filter_sent = {_sample: [] for _sample in samples}
for ind_word in range(len(gold_sent)):
gold_token = gold_sent[ind_word]
topk_word_pred = [
pred_sent_ls_topk[top][gold_ind_sent][ind_word]
for top in range(topk)
]
# handling with LAS specificity ; a types is correct iif its label is correct and its head is correct
if task_label == "types":
gold_token_to_score = gold_token \
if gold_sent_ls_dict["heads"][gold_ind_sent][ind_word] == pred_sent_ls_topk_dict["parsing-heads"][0][gold_ind_sent][ind_word] else "ZERO-ING-SCORE-TYPES-AS-HEADS-IS-UNCORRECT"
else:
gold_token_to_score = gold_token
score_sent.append(
word_level_scoring(metric=metric,
gold=gold_token_to_score,
topk_pred=topk_word_pred,
topk=topk))
for ind_sample, _sample in enumerate(samples):
try:
src = src_detokenized[gold_ind_sent][
ind_word] if _sample != "all" and not _sample.startswith(
"n_masks") else None
except Exception as e:
print(
"ERROR (scoring/report) handling src {} index ({},{}) "
.format(src_detokenized, gold_ind_sent, ind_word), e)
pdb.set_trace()
filter_sent[_sample].append(
word_level_filter(sample=_sample,
gold=gold_token,
topk_pred=topk_word_pred,
topk=topk,
src=src,
is_mwe=gold_sent_ls_dict[task_label]
[gold_ind_sent][ind_word],
word_reference_dic_ls=reference_word_dic,
mask_token=mask_token,
cls_token=cls_token,
sep_token=sep_token))
if compute_intersection_score:
for ind_sample, _sample in enumerate(sample_to_intersesct):
for ind_sample_2 in range(ind_sample):
inter = _sample + "-n-" + sample_to_intersesct[
ind_sample_2]
if filter_sent.get(inter, None) is None:
filter_sent[inter] = []
filter_sent[inter].append(
word_level_filter(
sample=_sample,
sample_2=sample_to_intersesct[ind_sample_2],
gold=gold_token,
topk_pred=topk_word_pred,
topk=topk,
src=src_detokenized[gold_ind_sent][ind_word],
word_reference_dic_ls=reference_word_dic,
is_mwe=gold_sent_ls_dict[task_label]
[gold_ind_sent][ind_word],
cls_token=cls_token,
sep_token=sep_token,
mask_token=mask_token))
if compute_intersection_score:
for _sample in samples + intersected_samples:
overall_filter_ls[_sample].append(filter_sent[_sample])
else:
for _sample in samples:
overall_filter_ls[_sample].append(filter_sent[_sample])
overall_score_ls_sent.append(score_sent)
result = {agg_func: {} for agg_func in agg_func_ls}
for agg_func in agg_func_ls:
for sample in samples + intersected_samples:
try:
result[agg_func][sample] = {
"score":
agg_func_batch_score(
overall_ls_sent_score=overall_score_ls_sent,
agg_func=agg_func,
overall_filter=overall_filter_ls[sample]),
"agg_func":
agg_func,
"metric":
"exact_match",
"n_tokens":
agg_func_batch_score(
overall_ls_sent_score=overall_score_ls_sent,
overall_filter=overall_filter_ls[sample],
agg_func="n_tokens"),
"n_sents":
agg_func_batch_score(
overall_ls_sent_score=overall_score_ls_sent,
overall_filter=overall_filter_ls[sample],
agg_func="n_sents")
}
except Exception as e:
print(e)
pdb.set_trace()
return result, skipping_sent, samples + intersected_samples
def from_pretrained(cls, pretrained_model_name_or_path, *model_args,
**kwargs):
r"""Instantiate a pretrained pytorch model from a pre-trained model configuration.
The model is set in evaluation mode by default using ``model.eval()`` (Dropout modules are deactivated)
To train the model, you should first set it back in training mode with ``model.train()``
The warning ``Weights from XXX not initialized from pretrained model`` means that the weights of XXX do not come pre-trained with the rest of the model.
It is up to you to train those weights with a downstream fine-tuning task.
The warning ``Weights from XXX not used in YYY`` means that the layer XXX is not used by YYY, therefore those weights are discarded.
Parameters:
pretrained_model_name_or_path: either:
- a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``.
- a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``.
- a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
- None if you are both providing the configuration and state dictionary (resp. with keyword arguments ``config`` and ``state_dict``)
model_args: (`optional`) Sequence of positional arguments:
All remaning positional arguments will be passed to the underlying model's ``__init__`` method
config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`:
Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when:
- the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or
- the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory.
- the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory.
state_dict: (`optional`) dict:
an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file.
This option can be used if you want to create a model from a pretrained configuration but load your own weights.
In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option.
cache_dir: (`optional`) string:
Path to a directory in which a downloaded pre-trained model
configuration should be cached if the standard cache should not be used.
force_download: (`optional`) boolean, default False:
Force to (re-)download the model weights and configuration files and override the cached versions if they exists.
proxies: (`optional`) dict, default None:
A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.
The proxies are used on each request.
output_loading_info: (`optional`) boolean:
Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages.
kwargs: (`optional`) Remaining dictionary of keyword arguments:
Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded:
- If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done)
- If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function.
Examples::
model = BertModel.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache.
model = BertModel.from_pretrained('./test/saved_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')`
model = BertModel.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading
assert model.config.output_attention == True
# Loading from a TF checkpoint file instead of a PyTorch model (slower)
config = BertConfig.from_json_file('./tf_model/my_tf_model_config.json')
model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_tf=True, config=config)
"""
config = kwargs.pop('config', None)
state_dict = kwargs.pop('state_dict', None)
cache_dir = kwargs.pop('cache_dir', None)
from_tf = kwargs.pop('from_tf', False)
force_download = kwargs.pop('force_download', False)
proxies = kwargs.pop('proxies', None)
output_loading_info = kwargs.pop('output_loading_info', False)
random_init = kwargs.pop("random_init", False)
kwargs_config = kwargs.copy()
mapping_keys_state_dic = kwargs.pop("mapping_keys_state_dic", None)
kwargs_config.pop("mapping_keys_state_dic", None)
pdb.set_trace()
if config is None:
config, model_kwargs = cls.config_class.from_pretrained(
pretrained_model_name_or_path,
*model_args,
cache_dir=cache_dir,
return_unused_kwargs=True,
force_download=force_download,
**kwargs_config)
else:
model_kwargs = kwargs
# Load model
if pretrained_model_name_or_path is not None:
if pretrained_model_name_or_path in cls.pretrained_model_archive_map:
archive_file = cls.pretrained_model_archive_map[
pretrained_model_name_or_path]
elif os.path.isdir(pretrained_model_name_or_path):
if from_tf and os.path.isfile(
os.path.join(pretrained_model_name_or_path,
TF_WEIGHTS_NAME + ".index")):
# Load from a TF 1.0 checkpoint
archive_file = os.path.join(pretrained_model_name_or_path,
TF_WEIGHTS_NAME + ".index")
elif from_tf and os.path.isfile(
os.path.join(pretrained_model_name_or_path,
TF2_WEIGHTS_NAME)):
# Load from a TF 2.0 checkpoint
archive_file = os.path.join(pretrained_model_name_or_path,
TF2_WEIGHTS_NAME)
elif os.path.isfile(
os.path.join(pretrained_model_name_or_path,
WEIGHTS_NAME)):
# Load from a PyTorch checkpoint
archive_file = os.path.join(pretrained_model_name_or_path,
WEIGHTS_NAME)
else:
raise EnvironmentError(
"Error no file named {} found in directory {} or `from_tf` set to False"
.format([
WEIGHTS_NAME, TF2_WEIGHTS_NAME,
TF_WEIGHTS_NAME + ".index"
], pretrained_model_name_or_path))
elif os.path.isfile(pretrained_model_name_or_path):
archive_file = pretrained_model_name_or_path
else:
assert from_tf, "Error finding file {}, no file or TF 1.X checkpoint found".format(
pretrained_model_name_or_path)
archive_file = pretrained_model_name_or_path + ".index"
# redirect to the cache, if necessary
try:
resolved_archive_file = cached_path(
archive_file,
cache_dir=cache_dir,
force_download=force_download,
proxies=proxies)
except EnvironmentError:
if pretrained_model_name_or_path in cls.pretrained_model_archive_map:
msg = "Couldn't reach server at '{}' to download pretrained weights.".format(
archive_file)
else:
msg = "Model name '{}' was not found in model name list ({}). " \
"We assumed '{}' was a path or url to model weight files named one of {} but " \
"couldn't find any such file at this path or url.".format(
pretrained_model_name_or_path,
', '.join(cls.pretrained_model_archive_map.keys()),
archive_file,
[WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME])
raise EnvironmentError(msg)
if resolved_archive_file == archive_file:
logger.info("loading weights file {}".format(archive_file))
else:
logger.info("loading weights file {} from cache at {}".format(
archive_file, resolved_archive_file))
else:
resolved_archive_file = None
# Instantiate model.
model = cls(config, *model_args, **model_kwargs)
if state_dict is None and not from_tf:
state_dict = torch.load(resolved_archive_file, map_location='cpu')
missing_keys = []
unexpected_keys = []
error_msgs = []
if from_tf:
if resolved_archive_file.endswith('.index'):
# Load from a TensorFlow 1.X checkpoint - provided by original authors
model = cls.load_tf_weights(
model, config,
resolved_archive_file[:-6]) # Remove the '.index'
else:
# Load from our TensorFlow 2.0 checkpoints
try:
from transformers import load_tf2_checkpoint_in_pytorch_model
model = load_tf2_checkpoint_in_pytorch_model(
model, resolved_archive_file, allow_missing_keys=True)
except ImportError as e:
logger.error(
"Loading a TensorFlow model in PyTorch, requires both PyTorch and TensorFlow to be installed. Please see "
"https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions."
)
raise e
else:
# Convert old format to new format if needed from a PyTorch state_dict
old_keys = []
new_keys = []
for key in state_dict.keys():
new_key = None
if 'gamma' in key:
new_key = key.replace('gamma', 'weight')
if 'beta' in key:
new_key = key.replace('beta', 'bias')
if new_key:
old_keys.append(key)
new_keys.append(new_key)
for old_key, new_key in zip(old_keys, new_keys):
state_dict[new_key] = state_dict.pop(old_key)
# copy state_dict so _load_from_state_dict can modify it
metadata = getattr(state_dict, '_metadata', None)
state_dict = state_dict.copy()
if metadata is not None:
state_dict._metadata = metadata
#assert mapping_keys_state_dic is not None, "ERROR did not found mapping dicts for {} ".format(pretrained_model_name_or_path)
#mapping_keys_state_dic = {"roberta": "encoder", "lm_head": "head.mlm"}
if mapping_keys_state_dic is not None:
assert isinstance(mapping_keys_state_dic, dict), "ERROR "
print(
"INFO : from loading from pretrained method (assuming loading original google model : "
"need to rename some keys {})".format(
mapping_keys_state_dic))
state_dict = cls.adapt_state_dic_to_multitask(
state_dict, keys_mapping=mapping_keys_state_dic)
def load(module, prefix=''):
##pdb.set_trace()
#local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {})
local_metadata = {"version": 1}
#print("MODEL loading with {} prefix , local metadata {} , missing keys {} unexpected keys {} ".format(prefix, local_metadata, missing_keys, unexpected_keys))
if not prefix.startswith("head") or prefix.startswith(
"head.mlm"):
assert len(
missing_keys
) == 0, "ERROR {} missing keys in state_dict {}".format(
prefix, missing_keys)
else:
if len(missing_keys) == 0:
print(
"WARNING {} missing keys in state_dict {}".format(
prefix, missing_keys))
module._load_from_state_dict(state_dict, prefix,
local_metadata, True,
missing_keys, unexpected_keys,
error_msgs)
for name, child in module._modules.items():
#load_params_only_ls = kwargs.get("load_params_only_ls ")
not_load_params_ls = kwargs.get(
"not_load_params_ls") if kwargs.get(
"not_load_params_ls") is not None else []
assert isinstance(
not_load_params_ls, list
), f"Argument error not_load_params_ls should be a list but is {not_load_params_ls}"
matching_not_load = []
for pattern in not_load_params_ls:
matching = re.match(pattern, prefix + name)
if matching is not None:
matching_not_load.append(matching)
if len(matching_not_load) > 0:
# means there is at least one patter in not load pattern that matched --> so should load
print("MATCH not loading : {} parameters {} ".format(
prefix + name, not_load_params_ls))
if child is not None and len(matching_not_load) == 0:
print("MODEL loading : child {} full {} ".format(
name, prefix + name + '.'))
load(child, prefix + name + '.')
else:
print(
"MODEL not loading : child {} matching_not_load {} "
.format(child, matching_not_load))
# Make sure we are able to load base models as well as derived models (with heads)
start_prefix = ''
model_to_load = model
if not hasattr(model, cls.base_model_prefix) and any(
s.startswith(cls.base_model_prefix)
for s in state_dict.keys()):
start_prefix = cls.base_model_prefix + '.'
if hasattr(model, cls.base_model_prefix) and not any(
s.startswith(cls.base_model_prefix)
for s in state_dict.keys()):
model_to_load = getattr(model, cls.base_model_prefix)
if not random_init:
load(model_to_load, prefix=start_prefix)
else:
print("WARNING : RANDOM INTIALIZATION OF BERTMULTITASK")
if len(missing_keys) > 0:
logger.info(
"Weights of {} not initialized from pretrained model: {}".
format(model.__class__.__name__, missing_keys))
if len(unexpected_keys) > 0:
logger.info(
"Weights from pretrained model not used in {}: {}".format(
model.__class__.__name__, unexpected_keys))
if len(error_msgs) > 0:
raise RuntimeError(
'Error(s) in loading state_dict for {}:\n\t{}'.format(
model.__class__.__name__, "\n\t".join(error_msgs)))
if hasattr(model, 'tie_weights'):
model.tie_weights(
) # make sure word embedding weights are still tied
# Set model in evaluation mode to desactivate DropOut modules by default
model.eval()
if output_loading_info:
loading_info = {
"missing_keys": missing_keys,
"unexpected_keys": unexpected_keys,
"error_msgs": error_msgs
}
return model, loading_info
return model
def readers_load(datasets,
tasks,
word_dictionary,
word_dictionary_norm,
char_dictionary,
pos_dictionary,
xpos_dictionary,
type_dictionary,
bert_tokenizer,
word_decoder=False,
must_get_norm=True,
bucket=True,
input_level_ls=None,
run_mode="train",
add_start_char=1,
add_end_char=1,
symbolic_end=True,
symbolic_root=True,
verbose=1):
readers = {}
simultanuous_training = False #depreciated
assert "all" not in tasks, "ERROR not supported yet (pb for simultanuous training..) "
if not "all" in tasks and not simultanuous_training:
try:
assert len(tasks) == len(datasets), "ERROR : as simultanuous_training is {} : " \
"we need 1 dataset per task but have only {} for task {} ".format(simultanuous_training, datasets, tasks)
except Exception as e:
pdb.set_trace()
datasets = [datasets[0] for _ in tasks]
# SHOULD NOT DO THAT !!
print("WARNING : duplicating readers", e)
elif not simultanuous_training:
assert len(
tasks) == 1, "ERROR : if all should have only all nothing else"
printing("TRAINING : MultiTask Iterator wit task 'all' ",
verbose_level=1,
verbose=verbose)
elif simultanuous_training:
printing(
"TRAINING : Training simultaneously tasks provided in {} (should have all required labels in datasets)",
verbose_level=1,
verbose=verbose)
raise (Exception("Not supported yet --> should handle the loop "))
for simul_task, data in zip(tasks, datasets):
normalization_in_reader = does_one_task_require_normalization(
simul_task)
# 1 reader per simultaneously trained task
readers[",".join(simul_task)] = conllu_data.read_data_to_variable(
data,
word_dictionary,
char_dictionary,
pos_dictionary,
xpos_dictionary,
type_dictionary,
word_decoder=word_decoder,
symbolic_end=symbolic_end,
symbolic_root=symbolic_root,
dry_run=0,
normalization=normalization_in_reader,
bucket=bucket,
add_start_char=add_start_char,
add_end_char=add_end_char,
tasks=simul_task,
max_char_len=None,
must_get_norm=must_get_norm,
bert_tokenizer=bert_tokenizer,
input_level_ls=input_level_ls,
run_mode=run_mode,
word_norm_dictionary=word_dictionary_norm,
pad_id=bert_tokenizer.convert_tokens_to_ids(
bert_tokenizer.pad_token),
verbose=verbose)
return readers
xpos_dictionary=xpos_dictionary,
type_dictionary=type_dictionary,
use_gpu=None,
norm_not_norm=True,
word_decoder=word_decoder,
bucket=False,
add_start_char=1,
add_end_char=1,
symbolic_end=True,
symbolic_root=True,
verbose=1)
iterator_multi = data_gen_multi_task_sampling_batch(
tasks=tasks,
readers=readers,
batch_size=1,
word_dictionary=word_dictionary,
char_dictionary=char_dictionary,
pos_dictionary=pos_dictionary,
word_dictionary_norm=word_dictionary_norm,
print_raw=True,
get_batch_mode=False,
verbose=1)
while True:
try:
batch = iterator_multi.__next__()
pdb.set_trace()
except StopIteration as e:
print(Exception(e))
break