def test_evaluate_performance_default_protocol_with_filter(): wn18 = load_wn18() X_filter = np.concatenate((wn18['train'], wn18['valid'], wn18['test'])) model = TransE(batches_count=10, seed=0, epochs=1, k=50, eta=10, verbose=True, embedding_model_params={'normalize_ent_emb':False, 'norm':1}, loss = 'self_adversarial', loss_params={'margin':1, 'alpha':0.5}, optimizer='adam', optimizer_params={'lr':0.0005}) model.fit(wn18['train']) from ampligraph.evaluation import evaluate_performance ranks_sep = [] from ampligraph.evaluation import hits_at_n_score, mrr_score, mr_score ranks = evaluate_performance(wn18['test'][::100], model, X_filter, verbose=True, corrupt_side='o', use_default_protocol=False) ranks_sep.extend(ranks) from ampligraph.evaluation import evaluate_performance from ampligraph.evaluation import hits_at_n_score, mrr_score, mr_score ranks = evaluate_performance(wn18['test'][::100], model, X_filter, verbose=True, corrupt_side='s', use_default_protocol=False) ranks_sep.extend(ranks) print('----------EVAL WITH FILTER-----------------') print('----------Subj and obj corrupted separately-----------------') mr_sep = mr_score(ranks_sep) print('MAR:', mr_sep) print('Mrr:', mrr_score(ranks_sep)) print('hits10:', hits_at_n_score(ranks_sep, 10)) print('hits3:', hits_at_n_score(ranks_sep, 3)) print('hits1:', hits_at_n_score(ranks_sep, 1)) from ampligraph.evaluation import evaluate_performance from ampligraph.evaluation import hits_at_n_score, mrr_score, mr_score ranks = evaluate_performance(wn18['test'][::100], model, X_filter, verbose=True, corrupt_side='s+o', use_default_protocol=True) print('----------corrupted with default protocol-----------------') mr_joint = mr_score(ranks) mrr_joint = mrr_score(ranks) print('MAR:', mr_joint) print('Mrr:', mrr_joint) print('hits10:', hits_at_n_score(ranks, 10)) print('hits3:', hits_at_n_score(ranks, 3)) print('hits1:', hits_at_n_score(ranks, 1)) np.testing.assert_equal(mr_sep, mr_joint) assert(mrr_joint is not np.Inf)
def kge(triples, kge_name, verbose): # Train test split t_size = math.ceil(len(triples)*0.2) X_train, X_test = train_test_split_no_unseen(triples, test_size=t_size) # Select kge_name if kge_name == 'complex': # ComplEx model model = ComplEx(batches_count=50, epochs=300, k=100, eta=20, optimizer='adam', optimizer_params={'lr':1e-4}, loss='multiclass_nll', regularizer='LP', regularizer_params={'p':3, 'lambda':1e-5}, seed=0, verbose=verbose) else: sys.exit('Given kge_name is not valid.') model.fit(X_train) #Embedding evaluation if verbose: filter_triples = np.concatenate((X_train, X_test)) ranks = evaluate_performance(X_test, model=model, filter_triples=filter_triples, use_default_protocol=True, verbose=True) mrr = mrr_score(ranks) print("MRR: %.2f" % (mrr)) mr = mr_score(ranks) print("MR: %.2f" % (mr)) hits_10 = hits_at_n_score(ranks, n=10) print("Hits@10: %.2f" % (hits_10)) hits_3 = hits_at_n_score(ranks, n=3) print("Hits@3: %.2f" % (hits_3)) hits_1 = hits_at_n_score(ranks, n=1) print("Hits@1: %.2f" % (hits_1)) print(''' - Ampligraph example - MRR: 0.25 MR: 4927.33 Hits@10: 0.35 Hits@3: 0.28 Hits@1: 0.19 ''') return model
def kge(triples, kge_name, epochs, batch_size, learning_rate, seed, verbose): kge_name = parsed_args.kge kge_model_savepath = f'./temp/ampligraph.model' if not os.path.isfile(kge_model_savepath): #Embedding evaluation if verbose: # Train test split t_size = math.ceil(len(triples) * 0.2) X_train, X_test = train_test_split_no_unseen(triples, test_size=t_size) eval_model = select_kge(kge_name, batch_size, epochs, seed, verbose) eval_model.fit(X_train) filter_triples = np.concatenate((X_train, X_test)) ranks = evaluate_performance(X_test, model=eval_model, filter_triples=filter_triples, use_default_protocol=True, verbose=True) mrr = mrr_score(ranks) print("MRR: %.2f" % (mrr)) mr = mr_score(ranks) print("MR: %.2f" % (mr)) hits_10 = hits_at_n_score(ranks, n=10) print("Hits@10: %.2f" % (hits_10)) hits_3 = hits_at_n_score(ranks, n=3) print("Hits@3: %.2f" % (hits_3)) hits_1 = hits_at_n_score(ranks, n=1) print("Hits@1: %.2f" % (hits_1)) print(''' - Ampligraph example - MRR: 0.25 MR: 4927.33 Hits@10: 0.35 Hits@3: 0.28 Hits@1: 0.19 ''') model = select_kge(kge_name, batch_size, epochs, seed, verbose) print('Training...') model.fit(np.array(triples)) save_model(model, model_name_path=kge_model_savepath) else: model = restore_model(model_name_path=kge_model_savepath) return model
def run_single_exp(config, dataset, model): hyperparams = config["hyperparams"][dataset][model] if hyperparams is None: print("dataset {0}...model {1} \ experiment is not conducted yet..." \ .format(dataset, config["model_name_map"][model])) return {"hyperparams": ".??"} print("dataset {0}...model {1}...\ hyperparameter:...{2}" \ .format(dataset, config["model_name_map"][model], hyperparams)) es_code = "{0}_{1}".format(dataset, model) load_func = getattr(ampligraph.datasets, config["load_function_map"][dataset]) X = load_func() # logging.debug("Loaded...{0}...".format(dataset)) # load model model_class = getattr(ampligraph.latent_features, config["model_name_map"][model]) model = model_class(**hyperparams) # Fit the model on training and validation set # The entire dataset will be used to filter out false positives statements # created by the corruption procedure: filter = np.concatenate((X['train'], X['valid'], X['test'])) if es_code in config["no_early_stopping"]: logging.debug("Fit without early stopping...") model.fit(X["train"]) else: logging.debug("Fit with early stopping...") model.fit( X["train"], True, { 'x_valid': X['valid'][::10], 'criteria': 'mrr', 'x_filter': filter, 'stop_interval': 2, 'burn_in': 0, 'check_interval': 100 }) # Run the evaluation procedure on the test set. Will create filtered rankings. # To disable filtering: filter_triples=None ranks = evaluate_performance(X['test'], model, filter, verbose=False) # compute and print metrics: mr = mr_score(ranks) mrr = mrr_score(ranks) hits_1 = hits_at_n_score(ranks, n=1) hits_3 = hits_at_n_score(ranks, n=3) hits_10 = hits_at_n_score(ranks, n=10) return { "mr": mr, "mrr": mrr, "H@1": hits_1, "H@3": hits_3, "H@10": hits_10, "hyperparams": hyperparams }
emg_triple = (row["data_id"]+"EMG: "+str(row["EMG"]), "isEMGDataIn", row["data_id"]) eda_triple = (row["data_id"]+"EDA: "+str(row["EDA"]), "isEDADataIn", row["data_id"]) temp_triple = (row["data_id"]+"TEMP: "+str(row["TEMP"]), "isTEMPDataIn", row["data_id"]) resp_triple = (row["data_id"]+"RESP: "+str(row["RESP"]), "isRESPDataIn", row["data_id"]) suj_triple = (row["data_id"], "isProducedBy", row["subject_id"]) triples.extend((ecg_triple, emg_triple, eda_triple, temp_triple, resp_triple, suj_triple)) X_train, X_valid = train_test_split_no_unseen(np.array(triples), test_size=10000) #Evaluar modelo filter_triples = np.concatenate((X_train, X_valid)) ranks = evaluate_performance(X_valid, model=model, filter_triples=filter_triples, use_default_protocol=True, verbose=True, filter_unseen=True) mr = mr_score(ranks) mrr = mrr_score(ranks) print("MRR: %.2f" % (mrr)) print("MR: %.2f" % (mr)) hits_10 = hits_at_n_score(ranks, n=10) print("Hits@10: %.2f" % (hits_10)) hits_3 = hits_at_n_score(ranks, n=3) print("Hits@3: %.2f" % (hits_3)) hits_1 = hits_at_n_score(ranks, n=1) print("Hits@1: %.2f" % (hits_1)) hits_100 = hits_at_n_score(ranks, n=100) print("Hits@100: %.2f" % (hits_100)) hits_1000 = hits_at_n_score(ranks, n=1000) print("Hits@1000: %.2f" % (hits_1000))