def run_experiments(
relations,
data_path_pre,
data_path_post,
input_param={
"lm":
"bert",
"label":
"bert_large",
"models_names": ["bert"],
"bert_model_name":
"bert-large-cased",
"bert_model_dir":
"pre-trained_language_models/bert/cased_L-24_H-1024_A-16",
},
use_negated_probes=False,
):
model = None
pp = pprint.PrettyPrinter(width=41, compact=True)
all_Precision1 = []
type_Precision1 = defaultdict(list)
type_count = defaultdict(list)
results_file = open("last_results.csv", "w+")
for relation in relations:
pp.pprint(relation)
PARAMETERS = {
"dataset_filename":
"{}{}{}".format(data_path_pre, relation["relation"],
data_path_post),
"common_vocab_filename":
"pre-trained_language_models/common_vocab_lowercased.txt",
"template":
"",
"bert_vocab_name":
"vocab.txt",
"batch_size":
32,
"logdir":
"output",
"full_logdir":
"output/results/{}/{}".format(input_param["label"],
relation["relation"]),
"lowercase":
True,
"max_sentence_length":
100,
"threads":
-1,
"interactive":
False,
"use_negated_probes":
use_negated_probes,
}
if "template" in relation:
PARAMETERS["template"] = relation["template"]
if use_negated_probes:
PARAMETERS["template_negated"] = relation["template_negated"]
PARAMETERS.update(input_param)
print(PARAMETERS)
args = argparse.Namespace(**PARAMETERS)
# see if file exists
try:
data = load_file(args.dataset_filename)
except Exception as e:
print("Relation {} excluded.".format(relation["relation"]))
print("Exception: {}".format(e))
continue
if model is None:
[model_type_name] = args.models_names
model = build_model_by_name(model_type_name, args)
Precision1 = run_evaluation(args, shuffle_data=False, model=model)
print("P@1 : {}".format(Precision1), flush=True)
all_Precision1.append(Precision1)
results_file.write("{},{}\n".format(relation["relation"],
round(Precision1 * 100, 2)))
results_file.flush()
if "type" in relation:
type_Precision1[relation["type"]].append(Precision1)
data = load_file(PARAMETERS["dataset_filename"])
type_count[relation["type"]].append(len(data))
mean_p1 = statistics.mean(all_Precision1)
print("@@@ {} - mean P@1: {}".format(input_param["label"], mean_p1))
results_file.close()
for t, l in type_Precision1.items():
print(
"@@@ ",
input_param["label"],
t,
statistics.mean(l),
sum(type_count[t]),
len(type_count[t]),
flush=True,
)
return mean_p1, all_Precision1
def run_experiments(
data_path_pre,
data_path_post,
input_param={
"lm":
"bert",
"label":
"bert_large",
"models_names": ["bert"],
"bert_model_name":
"bert-large-cased",
"bert_model_dir":
"pre-trained_language_models/bert/cased_L-24_H-1024_A-16",
},
use_negated_probes=False,
):
model = None
pp = pprint.PrettyPrinter(width=41, compact=True)
all_Precision1 = []
type_Precision1 = defaultdict(list)
type_count = defaultdict(list)
for i in range(1):
PARAMETERS = {
"dataset_filename":
"reproduction/data/TREx_filter/different_queries.jsonl",
"fact_pair_filename":
"reproduction/data/TREx_filter/different_queries_facts.jsonl",
"common_vocab_filename":
"pre-trained_language_models/common_vocab_cased.txt",
"template":
"",
"bert_vocab_name":
"vocab.txt",
"batch_size":
10,
"logdir":
"output",
"full_logdir":
"output/results/{}/{}".format(input_param["label"],
"different_queries"),
"lowercase":
False,
"max_sentence_length":
100,
"threads":
-1,
"interactive":
False,
"use_negated_probes":
use_negated_probes,
}
PARAMETERS.update(input_param)
args = argparse.Namespace(**PARAMETERS)
if model is None:
[model_type_name] = args.models_names
model = build_model_by_name(model_type_name, args)
Precision1 = run_evaluation(args, shuffle_data=False, model=model)
print("P@1 : {}".format(Precision1), flush=True)
all_Precision1.append(Precision1)
mean_p1 = statistics.mean(all_Precision1)
print("@@@ {} - mean P@1: {}".format(input_param["label"], mean_p1))
for t, l in type_Precision1.items():
print(
"@@@ ",
input_param["label"],
t,
statistics.mean(l),
sum(type_count[t]),
len(type_count[t]),
flush=True,
)
return mean_p1, all_Precision1
def run_experiments(
relations,
data_path_pre,
data_path_post,
input_param={
"lm":
"bert",
"label":
"bert_large",
"models_names": ["bert"],
"bert_model_name":
"bert-large-cased",
"bert_model_dir":
"pre-trained_language_models/bert/cased_L-24_H-1024_A-16",
},
use_negated_probes=False,
):
model = None
pp = pprint.PrettyPrinter(width=41, compact=True)
all_Precision1 = []
all_Precision10 = []
type_Precision1 = defaultdict(list)
type_count = defaultdict(list)
# Append to results_file
results_file = open("last_results.csv", "a", encoding='utf-8')
results_file.write('\n')
for relation in relations:
pp.pprint(relation)
PARAMETERS = {
"dataset_filename":
"{}{}{}".format(data_path_pre, relation["relation"],
data_path_post),
"common_vocab_filename":
"pre-trained_language_models/common_vocab_cased.txt",
"template":
"",
"bert_vocab_name":
"vocab.txt",
"batch_size":
32,
"logdir":
"output",
"full_logdir":
"output/results/{}/{}".format(input_param["label"],
relation["relation"]),
"lowercase":
False,
"max_sentence_length":
100,
"threads":
-1,
"interactive":
False,
"use_negated_probes":
use_negated_probes,
}
if "template" in relation:
PARAMETERS["template"] = relation["template"]
if use_negated_probes:
PARAMETERS["template_negated"] = relation["template_negated"]
PARAMETERS.update(input_param)
print(PARAMETERS)
args = argparse.Namespace(**PARAMETERS)
relation_name = relation["relation"]
if relation_name == "test":
relation_name = data_path_pre.replace("/", "") + "_test"
# see if file exists
try:
data = load_file(args.dataset_filename)
except Exception as e:
print("Relation {} excluded.".format(relation_name))
print("Exception: {}".format(e))
continue
if model is None:
[model_type_name] = args.models_names
model = build_model_by_name(model_type_name, args)
Precision1, Precision10 = run_evaluation(args,
shuffle_data=False,
model=model)
print("P@1 : {}".format(Precision1), flush=True)
all_Precision1.append(Precision1)
all_Precision10.append(Precision10)
results_file.write("[{}] {}: {}, P10 = {}, P1 = {}\n".format(
datetime.now(), input_param["label"], relation_name,
round(Precision10 * 100, 2), round(Precision1 * 100, 2)))
results_file.flush()
if "type" in relation:
type_Precision1[relation["type"]].append(Precision1)
data = load_file(PARAMETERS["dataset_filename"])
type_count[relation["type"]].append(len(data))
mean_p1 = statistics.mean(all_Precision1)
mean_p10 = statistics.mean(all_Precision10)
summaryP1 = "@@@ {} - mean P@10 = {}, mean P@1 = {}".format(
input_param["label"], round(mean_p10 * 100, 2),
round(mean_p1 * 100, 2))
print(summaryP1)
results_file.write(f'{summaryP1}\n')
results_file.flush()
for t, l in type_Precision1.items():
prec1item = f'@@@ Label={input_param["label"]}, type={t}, samples={sum(type_count[t])}, relations={len(type_count[t])}, mean prec1={round(statistics.mean(l) * 100, 2)}\n'
print(prec1item, flush=True)
results_file.write(prec1item)
results_file.flush()
results_file.close()
return mean_p1, all_Precision1
def run_experiments(
relations,
data_path_pre,
data_path_post,
refine_template,
get_objs,
batch_size,
dynamic=None,
use_prob=False,
bt_obj=None,
temp_model=None,
save=None,
load=None,
feature_dir=None,
enforce_prob=True,
num_feat=1,
temperature=0.0,
use_model2=False,
lowercase=False,
upper_entity=False,
input_param={
"lm":
"bert",
"label":
"bert_large",
"models_names": ["bert"],
"bert_model_name":
"bert-large-cased",
"bert_model_dir":
"pre-trained_language_models/bert/cased_L-24_H-1024_A-16",
},
):
model, model2 = None, None
pp = pprint.PrettyPrinter(width=41, compact=True)
all_Precision1 = []
type_Precision1 = defaultdict(list)
type_count = defaultdict(list)
print('use lowercase: {}, use upper entity: {}'.format(
lowercase, upper_entity))
results_file = open("last_results.csv", "w+")
if refine_template:
refine_temp_fout = open(refine_template, 'w')
new_relations = []
templates_set = set()
rel2numtemp = {}
for relation in relations: # collect templates
if 'template' in relation:
if type(relation['template']) is not list:
relation['template'] = [relation['template']]
rel2numtemp[relation['relation']] = len(relation['template'])
if temp_model is not None:
if temp_model.startswith('mixture'):
method = temp_model.split('_')[1]
if method == 'optimize': # (extract feature) + optimize
temp_model = TempModel(rel2numtemp,
enforce_prob=enforce_prob,
num_feat=num_feat)
temp_model.train()
optimizer = optim.Adam(temp_model.parameters(), lr=1e-1)
temp_model = (temp_model, (optimizer, temperature))
elif method == 'precompute': # extract feature
temp_model = (None, 'precompute')
elif method == 'predict': # predict
temp_model = TempModel(
rel2numtemp, enforce_prob=enforce_prob,
num_feat=num_feat) # TODO: number of feature
if load is not None:
temp_model.load_state_dict(torch.load(load))
temp_model.eval()
temp_model = (temp_model, None)
else:
raise NotImplementedError
else:
raise NotImplementedError
for relation in relations:
pp.pprint(relation)
PARAMETERS = {
"relation":
relation["relation"],
"dataset_filename":
"{}/{}{}".format(data_path_pre, relation["relation"],
data_path_post),
"common_vocab_filename":
"pre-trained_language_models/common_vocab_cased.txt",
"template":
"",
"bert_vocab_name":
"vocab.txt",
"batch_size":
batch_size,
"logdir":
"output",
"full_logdir":
"output/results/{}/{}".format(input_param["label"],
relation["relation"]),
"lowercase":
lowercase,
"upper_entity":
upper_entity,
"max_sentence_length":
100,
"threads":
-1,
"interactive":
False,
}
dev_param = deepcopy(PARAMETERS)
dev_param['dataset_filename'] = '{}/{}{}'.format(
data_path_pre + '_dev', relation['relation'], data_path_post)
bert_large_param = deepcopy(PARAMETERS)
if 'template' in relation:
PARAMETERS['template'] = relation['template']
dev_param['template'] = relation['template']
bert_large_param['template'] = relation['template']
PARAMETERS.update(input_param)
dev_param.update(input_param)
bert_large_param.update(
LM_BERT_LARGE
) # this is used to optimize the weights for bert-base and bert-large at the same time
print(PARAMETERS)
args = argparse.Namespace(**PARAMETERS)
dev_args = argparse.Namespace(**dev_param)
bert_large_args = argparse.Namespace(**bert_large_param)
# see if file exists
try:
data = load_file(args.dataset_filename)
except Exception as e:
print("Relation {} excluded.".format(relation["relation"]))
print("Exception: {}".format(e))
continue
if model is None:
[model_type_name] = args.models_names
model = build_model_by_name(model_type_name, args)
if use_model2:
model2 = build_model_by_name(bert_large_args.models_names[0],
bert_large_args)
if temp_model is not None:
if temp_model[1] == 'precompute':
features = run_evaluation(
args,
shuffle_data=False,
model=model,
refine_template=bool(refine_template),
get_objs=get_objs,
dynamic=dynamic,
use_prob=use_prob,
bt_obj=bt_obj,
temp_model=temp_model)
print('save features for {}'.format(relation['relation']))
torch.save(features,
os.path.join(save, relation['relation'] + '.pt'))
continue
elif temp_model[1] is not None: # train temp model
if feature_dir is None:
loss = run_evaluation(
args,
shuffle_data=False,
model=model,
model2=model2,
refine_template=bool(refine_template),
get_objs=get_objs,
dynamic=dynamic,
use_prob=use_prob,
bt_obj=bt_obj,
temp_model=temp_model)
else:
temp_model_, (optimizer, temperature) = temp_model
temp_model_.cuda()
min_loss = 1e10
es = 0
for e in range(500):
# SHAPE: (num_sample, num_temp)
feature = torch.load(
os.path.join(feature_dir,
args.relation + '.pt')).cuda()
#dev_feature = torch.load(os.path.join(feature_dir + '_dev', args.relation + '.pt')).cuda()
#feature = torch.cat([feature, dev_feature], 0)
#weight = feature.mean(0)
#temp_model[0].set_weight(args.relation, weight)
optimizer.zero_grad()
loss = temp_model_(args.relation, feature)
if os.path.exists(feature_dir +
'__dev'): # TODO: debug
dev_feature = torch.load(
os.path.join(feature_dir + '_dev',
args.relation + '.pt')).cuda()
dev_loss = temp_model_(args.relation, dev_feature)
else:
dev_loss = loss
loss.backward()
optimizer.step()
if dev_loss - min_loss < -1e-3:
min_loss = dev_loss
es = 0
else:
es += 1
if es >= 10:
print('early stop')
break
continue
Precision1 = run_evaluation(args,
shuffle_data=False,
model=model,
refine_template=bool(refine_template),
get_objs=get_objs,
dynamic=dynamic,
use_prob=use_prob,
bt_obj=bt_obj,
temp_model=temp_model)
if get_objs:
return
if refine_template and Precision1 is not None:
if Precision1 in templates_set:
continue
templates_set.add(Precision1)
new_relation = deepcopy(relation)
new_relation['old_template'] = new_relation['template']
new_relation['template'] = Precision1
new_relations.append(new_relation)
refine_temp_fout.write(json.dumps(new_relation) + '\n')
refine_temp_fout.flush()
continue
print("P@1 : {}".format(Precision1), flush=True)
all_Precision1.append(Precision1)
results_file.write("{},{}\n".format(relation["relation"],
round(Precision1 * 100, 2)))
results_file.flush()
if "type" in relation:
type_Precision1[relation["type"]].append(Precision1)
data = load_file(PARAMETERS["dataset_filename"])
type_count[relation["type"]].append(len(data))
if refine_template:
refine_temp_fout.close()
return
if temp_model is not None:
if save is not None and temp_model[0] is not None:
torch.save(temp_model[0].state_dict(), save)
return
mean_p1 = statistics.mean(all_Precision1)
print("@@@ {} - mean P@1: {}".format(input_param["label"], mean_p1))
results_file.close()
for t, l in type_Precision1.items():
print(
"@@@ ",
input_param["label"],
t,
statistics.mean(l),
sum(type_count[t]),
len(type_count[t]),
flush=True,
)
return mean_p1, all_Precision1
def run_experiments(
relations,
data_path_pre,
data_path_post,
input_param={
"lm":
"bert",
"label":
"bert_large",
"models_names": ["bert"],
"bert_model_name":
"bert-large-cased",
"bert_model_dir":
"pre-trained_language_models/bert/cased_L-24_H-1024_A-16",
},
use_negated_probes=False,
):
model = None
pp = pprint.PrettyPrinter(width=41, compact=True)
all_Precision1 = []
type_Precision1 = defaultdict(list)
type_count = defaultdict(list)
results_file = open("last_results.csv", "w+")
uid_list_all, mask_feature_list_all, answers_list_all = [], [], []
all_correct_uuids = []
total_modified_correct, total_unmodified_correct = 0, 0
total_modified_num, total_unmodified_num = 0, 0
for relation in relations:
# if "type" not in relation or relation["type"] != "1-1":
# continue
pp.pprint(relation)
PARAMETERS = {
"dataset_filename":
"{}{}{}".format(data_path_pre, relation["relation"],
data_path_post),
"common_vocab_filename":
'pre-trained_language_models/bert/cased_L-12_H-768_A-12/vocab.txt', #"pre-trained_language_models/common_vocab_cased.txt",
"template":
"",
"bert_vocab_name":
"vocab.txt",
"batch_size":
32,
"logdir":
"output",
"full_logdir":
"output/results/{}/{}".format(input_param["label"],
relation["relation"]),
"lowercase":
False,
"max_sentence_length":
512, # change to 512 later
"threads":
2,
"interactive":
False,
"use_negated_probes":
use_negated_probes,
"return_features":
False,
"uuid_list": []
}
if "template" in relation:
PARAMETERS["template"] = relation["template"]
if use_negated_probes:
PARAMETERS["template_negated"] = relation["template_negated"]
PARAMETERS.update(input_param)
print(PARAMETERS)
args = argparse.Namespace(**PARAMETERS)
# see if file exists
try:
data = load_file(args.dataset_filename)
except Exception as e:
print("Relation {} excluded.".format(relation["relation"]))
print("Exception: {}".format(e))
continue
if model is None:
[model_type_name] = args.models_names
model = build_model_by_name(model_type_name, args)
if getattr(args, 'output_feature_path', ''):
# Get the features for kNN-LM. Ignore this part if only obtaining the correct-predicted questions.
Precision1, total_unmodified, Precision1_modified, total_modified, uid_list, mask_feature_list, answers_list = run_evaluation(
args, shuffle_data=False, model=model)
if len(uid_list) > 0:
uid_list_all.extend(uid_list)
mask_feature_tensor = torch.cat(mask_feature_list, dim=0)
mask_feature_list_all.append(mask_feature_tensor)
answers_list_all.extend(answers_list)
else:
Precision1, total_unmodified, Precision1_modified, total_modified, correct_uuids = run_evaluation(
args, shuffle_data=False, model=model)
all_correct_uuids.extend(correct_uuids)
total_modified_correct += Precision1_modified
total_unmodified_correct += Precision1
total_modified_num += total_modified
total_unmodified_num += total_unmodified
print("P@1 : {}".format(Precision1), flush=True)
all_Precision1.append(Precision1)
results_file.write("{},{}\n".format(relation["relation"],
round(Precision1 * 100, 2)))
results_file.flush()
if "type" in relation:
type_Precision1[relation["type"]].append(Precision1)
data = load_file(PARAMETERS["dataset_filename"])
type_count[relation["type"]].append(len(data))
mean_p1 = statistics.mean(all_Precision1)
print("@@@ {} - mean P@1: {}".format(input_param["label"], mean_p1))
print("Unmodified acc: {}, modified acc: {}".format(
total_unmodified_correct / float(total_unmodified_num),
0 if total_modified_num == 0 else total_modified_correct /
float(total_modified_num)))
results_file.close()
for t, l in type_Precision1.items():
print(
"@@@ ",
input_param["label"],
t,
statistics.mean(l),
sum(type_count[t]),
len(type_count[t]),
flush=True,
)
if len(uid_list_all) > 0:
out_dict = {
'mask_features': torch.cat(mask_feature_list_all, dim=0),
'uuids': uid_list_all,
'obj_labels': answers_list_all
}
torch.save(out_dict, 'datastore/ds_change32.pt')
if len(all_correct_uuids) > 0:
if not os.path.exists('modification'):
os.makedirs('modification')
json.dump(all_correct_uuids,
open('modification/correct_uuids.json', 'w'))
return mean_p1, all_Precision1