def test_evaluate_performance_default_protocol_with_filter():
wn18 = load_wn18()
X_filter = np.concatenate((wn18['train'], wn18['valid'], wn18['test']))
model = TransE(batches_count=10, seed=0, epochs=1,
k=50, eta=10, verbose=True,
embedding_model_params={'normalize_ent_emb':False, 'norm':1},
loss = 'self_adversarial', loss_params={'margin':1, 'alpha':0.5},
optimizer='adam',
optimizer_params={'lr':0.0005})
model.fit(wn18['train'])
from ampligraph.evaluation import evaluate_performance
ranks_sep = []
from ampligraph.evaluation import hits_at_n_score, mrr_score, mr_score
ranks = evaluate_performance(wn18['test'][::100], model, X_filter, verbose=True, corrupt_side='o',
use_default_protocol=False)
ranks_sep.extend(ranks)
from ampligraph.evaluation import evaluate_performance
from ampligraph.evaluation import hits_at_n_score, mrr_score, mr_score
ranks = evaluate_performance(wn18['test'][::100], model, X_filter, verbose=True, corrupt_side='s',
use_default_protocol=False)
ranks_sep.extend(ranks)
print('----------EVAL WITH FILTER-----------------')
print('----------Subj and obj corrupted separately-----------------')
mr_sep = mr_score(ranks_sep)
print('MAR:', mr_sep)
print('Mrr:', mrr_score(ranks_sep))
print('hits10:', hits_at_n_score(ranks_sep, 10))
print('hits3:', hits_at_n_score(ranks_sep, 3))
print('hits1:', hits_at_n_score(ranks_sep, 1))
from ampligraph.evaluation import evaluate_performance
from ampligraph.evaluation import hits_at_n_score, mrr_score, mr_score
ranks = evaluate_performance(wn18['test'][::100], model, X_filter, verbose=True, corrupt_side='s+o',
use_default_protocol=True)
print('----------corrupted with default protocol-----------------')
mr_joint = mr_score(ranks)
mrr_joint = mrr_score(ranks)
print('MAR:', mr_joint)
print('Mrr:', mrr_joint)
print('hits10:', hits_at_n_score(ranks, 10))
print('hits3:', hits_at_n_score(ranks, 3))
print('hits1:', hits_at_n_score(ranks, 1))
np.testing.assert_equal(mr_sep, mr_joint)
assert(mrr_joint is not np.Inf)
def kge(triples, kge_name, verbose):
# Train test split
t_size = math.ceil(len(triples)*0.2)
X_train, X_test = train_test_split_no_unseen(triples, test_size=t_size)
# Select kge_name
if kge_name == 'complex':
# ComplEx model
model = ComplEx(batches_count=50,
epochs=300,
k=100,
eta=20,
optimizer='adam',
optimizer_params={'lr':1e-4},
loss='multiclass_nll',
regularizer='LP',
regularizer_params={'p':3, 'lambda':1e-5},
seed=0,
verbose=verbose)
else:
sys.exit('Given kge_name is not valid.')
model.fit(X_train)
#Embedding evaluation
if verbose:
filter_triples = np.concatenate((X_train, X_test))
ranks = evaluate_performance(X_test,
model=model,
filter_triples=filter_triples,
use_default_protocol=True,
verbose=True)
mrr = mrr_score(ranks)
print("MRR: %.2f" % (mrr))
mr = mr_score(ranks)
print("MR: %.2f" % (mr))
hits_10 = hits_at_n_score(ranks, n=10)
print("Hits@10: %.2f" % (hits_10))
hits_3 = hits_at_n_score(ranks, n=3)
print("Hits@3: %.2f" % (hits_3))
hits_1 = hits_at_n_score(ranks, n=1)
print("Hits@1: %.2f" % (hits_1))
print('''
- Ampligraph example -
MRR: 0.25
MR: 4927.33
Hits@10: 0.35
Hits@3: 0.28
Hits@1: 0.19
''')
return model
def kge(triples, kge_name, epochs, batch_size, learning_rate, seed, verbose):
kge_name = parsed_args.kge
kge_model_savepath = f'./temp/ampligraph.model'
if not os.path.isfile(kge_model_savepath):
#Embedding evaluation
if verbose:
# Train test split
t_size = math.ceil(len(triples) * 0.2)
X_train, X_test = train_test_split_no_unseen(triples,
test_size=t_size)
eval_model = select_kge(kge_name, batch_size, epochs, seed,
verbose)
eval_model.fit(X_train)
filter_triples = np.concatenate((X_train, X_test))
ranks = evaluate_performance(X_test,
model=eval_model,
filter_triples=filter_triples,
use_default_protocol=True,
verbose=True)
mrr = mrr_score(ranks)
print("MRR: %.2f" % (mrr))
mr = mr_score(ranks)
print("MR: %.2f" % (mr))
hits_10 = hits_at_n_score(ranks, n=10)
print("Hits@10: %.2f" % (hits_10))
hits_3 = hits_at_n_score(ranks, n=3)
print("Hits@3: %.2f" % (hits_3))
hits_1 = hits_at_n_score(ranks, n=1)
print("Hits@1: %.2f" % (hits_1))
print('''
- Ampligraph example -
MRR: 0.25
MR: 4927.33
Hits@10: 0.35
Hits@3: 0.28
Hits@1: 0.19
''')
model = select_kge(kge_name, batch_size, epochs, seed, verbose)
print('Training...')
model.fit(np.array(triples))
save_model(model, model_name_path=kge_model_savepath)
else:
model = restore_model(model_name_path=kge_model_savepath)
return model
def run_single_exp(config, dataset, model):
hyperparams = config["hyperparams"][dataset][model]
if hyperparams is None:
print("dataset {0}...model {1} \
experiment is not conducted yet..." \
.format(dataset, config["model_name_map"][model]))
return {"hyperparams": ".??"}
print("dataset {0}...model {1}...\
hyperparameter:...{2}" \
.format(dataset,
config["model_name_map"][model],
hyperparams))
es_code = "{0}_{1}".format(dataset, model)
load_func = getattr(ampligraph.datasets,
config["load_function_map"][dataset])
X = load_func()
# logging.debug("Loaded...{0}...".format(dataset))
# load model
model_class = getattr(ampligraph.latent_features,
config["model_name_map"][model])
model = model_class(**hyperparams)
# Fit the model on training and validation set
# The entire dataset will be used to filter out false positives statements
# created by the corruption procedure:
filter = np.concatenate((X['train'], X['valid'], X['test']))
if es_code in config["no_early_stopping"]:
logging.debug("Fit without early stopping...")
model.fit(X["train"])
else:
logging.debug("Fit with early stopping...")
model.fit(
X["train"], True, {
'x_valid': X['valid'][::10],
'criteria': 'mrr',
'x_filter': filter,
'stop_interval': 2,
'burn_in': 0,
'check_interval': 100
})
# Run the evaluation procedure on the test set. Will create filtered rankings.
# To disable filtering: filter_triples=None
ranks = evaluate_performance(X['test'], model, filter, verbose=False)
# compute and print metrics:
mr = mr_score(ranks)
mrr = mrr_score(ranks)
hits_1 = hits_at_n_score(ranks, n=1)
hits_3 = hits_at_n_score(ranks, n=3)
hits_10 = hits_at_n_score(ranks, n=10)
return {
"mr": mr,
"mrr": mrr,
"H@1": hits_1,
"H@3": hits_3,
"H@10": hits_10,
"hyperparams": hyperparams
}
emg_triple = (row["data_id"]+"EMG: "+str(row["EMG"]), "isEMGDataIn", row["data_id"])
eda_triple = (row["data_id"]+"EDA: "+str(row["EDA"]), "isEDADataIn", row["data_id"])
temp_triple = (row["data_id"]+"TEMP: "+str(row["TEMP"]), "isTEMPDataIn", row["data_id"])
resp_triple = (row["data_id"]+"RESP: "+str(row["RESP"]), "isRESPDataIn", row["data_id"])
suj_triple = (row["data_id"], "isProducedBy", row["subject_id"])
triples.extend((ecg_triple, emg_triple, eda_triple, temp_triple,
resp_triple, suj_triple))
X_train, X_valid = train_test_split_no_unseen(np.array(triples), test_size=10000)
#Evaluar modelo
filter_triples = np.concatenate((X_train, X_valid))
ranks = evaluate_performance(X_valid, model=model, filter_triples=filter_triples, use_default_protocol=True, verbose=True, filter_unseen=True)
mr = mr_score(ranks)
mrr = mrr_score(ranks)
print("MRR: %.2f" % (mrr))
print("MR: %.2f" % (mr))
hits_10 = hits_at_n_score(ranks, n=10)
print("Hits@10: %.2f" % (hits_10))
hits_3 = hits_at_n_score(ranks, n=3)
print("Hits@3: %.2f" % (hits_3))
hits_1 = hits_at_n_score(ranks, n=1)
print("Hits@1: %.2f" % (hits_1))
hits_100 = hits_at_n_score(ranks, n=100)
print("Hits@100: %.2f" % (hits_100))
hits_1000 = hits_at_n_score(ranks, n=1000)
print("Hits@1000: %.2f" % (hits_1000))
