def kge(triples, kge_name, verbose): # Train test split t_size = math.ceil(len(triples)*0.2) X_train, X_test = train_test_split_no_unseen(triples, test_size=t_size) # Select kge_name if kge_name == 'complex': # ComplEx model model = ComplEx(batches_count=50, epochs=300, k=100, eta=20, optimizer='adam', optimizer_params={'lr':1e-4}, loss='multiclass_nll', regularizer='LP', regularizer_params={'p':3, 'lambda':1e-5}, seed=0, verbose=verbose) else: sys.exit('Given kge_name is not valid.') model.fit(X_train) #Embedding evaluation if verbose: filter_triples = np.concatenate((X_train, X_test)) ranks = evaluate_performance(X_test, model=model, filter_triples=filter_triples, use_default_protocol=True, verbose=True) mrr = mrr_score(ranks) print("MRR: %.2f" % (mrr)) mr = mr_score(ranks) print("MR: %.2f" % (mr)) hits_10 = hits_at_n_score(ranks, n=10) print("Hits@10: %.2f" % (hits_10)) hits_3 = hits_at_n_score(ranks, n=3) print("Hits@3: %.2f" % (hits_3)) hits_1 = hits_at_n_score(ranks, n=1) print("Hits@1: %.2f" % (hits_1)) print(''' - Ampligraph example - MRR: 0.25 MR: 4927.33 Hits@10: 0.35 Hits@3: 0.28 Hits@1: 0.19 ''') return model
def test_evaluate_RandomBaseline(): model = RandomBaseline(seed=0) X = load_wn18() model.fit(X["train"]) ranks = evaluate_performance(X["test"], model=model, use_default_protocol=False, corrupt_side='s+o', verbose=False) hits10 = hits_at_n_score(ranks, n=10) hits1 = hits_at_n_score(ranks, n=1) assert (hits10 == 0.0002 and hits1 == 0.0)
def kge(triples, kge_name, epochs, batch_size, learning_rate, seed, verbose): kge_name = parsed_args.kge kge_model_savepath = f'./temp/ampligraph.model' if not os.path.isfile(kge_model_savepath): #Embedding evaluation if verbose: # Train test split t_size = math.ceil(len(triples) * 0.2) X_train, X_test = train_test_split_no_unseen(triples, test_size=t_size) eval_model = select_kge(kge_name, batch_size, epochs, seed, verbose) eval_model.fit(X_train) filter_triples = np.concatenate((X_train, X_test)) ranks = evaluate_performance(X_test, model=eval_model, filter_triples=filter_triples, use_default_protocol=True, verbose=True) mrr = mrr_score(ranks) print("MRR: %.2f" % (mrr)) mr = mr_score(ranks) print("MR: %.2f" % (mr)) hits_10 = hits_at_n_score(ranks, n=10) print("Hits@10: %.2f" % (hits_10)) hits_3 = hits_at_n_score(ranks, n=3) print("Hits@3: %.2f" % (hits_3)) hits_1 = hits_at_n_score(ranks, n=1) print("Hits@1: %.2f" % (hits_1)) print(''' - Ampligraph example - MRR: 0.25 MR: 4927.33 Hits@10: 0.35 Hits@3: 0.28 Hits@1: 0.19 ''') model = select_kge(kge_name, batch_size, epochs, seed, verbose) print('Training...') model.fit(np.array(triples)) save_model(model, model_name_path=kge_model_savepath) else: model = restore_model(model_name_path=kge_model_savepath) return model
def compute_metrics(model, train_samples, test_samples): ranks = evaluate_performance( test_samples, model=model, filter_triples=train_samples, # Corruption strategy filter defined above use_default_protocol= True, # corrupt subj and obj separately while evaluating verbose=True) return { 'MRR': mrr_score(ranks), 'Hits@10': hits_at_n_score(ranks, n=10), 'Hits@3': hits_at_n_score(ranks, n=3), 'Hits@1': hits_at_n_score(ranks, n=1) }
def test_evaluate_performance_TransE(): X = load_wn18() model = TransE(batches_count=10, seed=0, epochs=100, k=100, eta=5, optimizer_params={'lr': 0.1}, loss='pairwise', loss_params={'margin': 5}, optimizer='adagrad') model.fit(np.concatenate((X['train'], X['valid']))) filter_triples = np.concatenate((X['train'], X['valid'], X['test'])) ranks = evaluate_performance(X['test'][:200], model=model, filter_triples=filter_triples, verbose=True) # ranks = evaluate_performance(X['test'][:200], model=model) mrr = mrr_score(ranks) hits_10 = hits_at_n_score(ranks, n=10) print("ranks: %s" % ranks) print("MRR: %f" % mrr) print("Hits@10: %f" % hits_10)
def test_evaluate_performance_so_side_corruptions_without_filter(): X = load_wn18() model = ComplEx(batches_count=10, seed=0, epochs=5, k=200, eta=10, loss='nll', regularizer=None, optimizer='adam', optimizer_params={'lr': 0.01}, verbose=True) model.fit(X['train']) X_filter = np.concatenate((X['train'], X['valid'], X['test'])) ranks = evaluate_performance(X['test'][::20], model, X_filter, verbose=True, use_default_protocol=False, corrupt_side='s+o') mrr = mrr_score(ranks) hits_10 = hits_at_n_score(ranks, n=10) print("ranks: %s" % ranks) print("MRR: %f" % mrr) print("Hits@10: %f" % hits_10) assert (mrr is not np.Inf)
def evaluate_model(X_test, model, positives_filter): ranks = evaluate_performance( X_test, model=model, filter_triples= positives_filter, # Corruption strategy filter defined above use_default_protocol= True, # corrupt subj and obj separately while evaluating verbose=True) mrr = mrr_score(ranks) print("MRR: %.2f" % (mrr)) hits_10 = hits_at_n_score(ranks, n=10) print("Hits@10: %.2f" % (hits_10)) hits_3 = hits_at_n_score(ranks, n=3) print("Hits@3: %.2f" % (hits_3)) hits_1 = hits_at_n_score(ranks, n=1) print("Hits@1: %.2f" % (hits_1))
def test_evaluate_RandomBaseline(): model = RandomBaseline(seed=0) X = load_wn18() model.fit(X["train"]) ranks = evaluate_performance(X["test"], model=model, corrupt_side='s+o', verbose=False) hits10 = hits_at_n_score(ranks, n=10) hits1 = hits_at_n_score(ranks, n=1) assert ranks.shape == (len(X['test']), ) assert hits10 < 0.01 and hits1 == 0.0 ranks = evaluate_performance(X["test"], model=model, corrupt_side='s,o', verbose=False) hits10 = hits_at_n_score(ranks, n=10) hits1 = hits_at_n_score(ranks, n=1) assert ranks.shape == (len(X['test']), 2) assert hits10 < 0.01 and hits1 == 0.0 ranks_filtered = evaluate_performance(X["test"], filter_triples=np.concatenate( (X['train'], X['valid'], X['test'])), model=model, corrupt_side='s,o', verbose=False) hits10 = hits_at_n_score(ranks_filtered, n=10) hits1 = hits_at_n_score(ranks_filtered, n=1) assert ranks_filtered.shape == (len(X['test']), 2) assert hits10 < 0.01 and hits1 == 0.0 assert np.all(ranks_filtered <= ranks) assert np.any(ranks_filtered != ranks)
def test_evaluate_performance_nll_complex(): X = load_wn18() model = ComplEx(batches_count=10, seed=0, epochs=10, k=150, optimizer_params={'lr': 0.1}, eta=10, loss='nll', optimizer='adagrad', verbose=True) model.fit(np.concatenate((X['train'], X['valid']))) filter = np.concatenate((X['train'], X['valid'], X['test'])) ranks = evaluate_performance(X['test'][:200], model=model, filter_triples=filter, verbose=True) mrr = mrr_score(ranks) hits_10 = hits_at_n_score(ranks, n=10) print("ranks: %s" % ranks) print("MRR: %f" % mrr) print("Hits@10: %f" % hits_10)
def test_evaluate_performance_default_protocol_with_filter(): wn18 = load_wn18() X_filter = np.concatenate((wn18['train'], wn18['valid'], wn18['test'])) model = TransE(batches_count=10, seed=0, epochs=1, k=50, eta=10, verbose=True, embedding_model_params={'normalize_ent_emb':False, 'norm':1}, loss = 'self_adversarial', loss_params={'margin':1, 'alpha':0.5}, optimizer='adam', optimizer_params={'lr':0.0005}) model.fit(wn18['train']) from ampligraph.evaluation import evaluate_performance ranks_sep = [] from ampligraph.evaluation import hits_at_n_score, mrr_score, mr_score ranks = evaluate_performance(wn18['test'][::100], model, X_filter, verbose=True, corrupt_side='o', use_default_protocol=False) ranks_sep.extend(ranks) from ampligraph.evaluation import evaluate_performance from ampligraph.evaluation import hits_at_n_score, mrr_score, mr_score ranks = evaluate_performance(wn18['test'][::100], model, X_filter, verbose=True, corrupt_side='s', use_default_protocol=False) ranks_sep.extend(ranks) print('----------EVAL WITH FILTER-----------------') print('----------Subj and obj corrupted separately-----------------') mr_sep = mr_score(ranks_sep) print('MAR:', mr_sep) print('Mrr:', mrr_score(ranks_sep)) print('hits10:', hits_at_n_score(ranks_sep, 10)) print('hits3:', hits_at_n_score(ranks_sep, 3)) print('hits1:', hits_at_n_score(ranks_sep, 1)) from ampligraph.evaluation import evaluate_performance from ampligraph.evaluation import hits_at_n_score, mrr_score, mr_score ranks = evaluate_performance(wn18['test'][::100], model, X_filter, verbose=True, corrupt_side='s+o', use_default_protocol=True) print('----------corrupted with default protocol-----------------') mr_joint = mr_score(ranks) mrr_joint = mrr_score(ranks) print('MAR:', mr_joint) print('Mrr:', mrr_joint) print('hits10:', hits_at_n_score(ranks, 10)) print('hits3:', hits_at_n_score(ranks, 3)) print('hits1:', hits_at_n_score(ranks, 1)) np.testing.assert_equal(mr_sep, mr_joint) assert(mrr_joint is not np.Inf)
else: print('The model is not fit! Did you skip a step?') from ampligraph.evaluation import evaluate_performance ranks = evaluate_performance(X_test, model=model, filter_triples=positives_filter, # Corruption strategy filter defined above use_default_protocol=True, # corrupt subj and obj separately while evaluating verbose=True) from ampligraph.evaluation import mr_score, mrr_score, hits_at_n_score mrr = mrr_score(ranks) print("MRR: %.2f" % (mrr)) hits_10 = hits_at_n_score(ranks, n=10) print("Hits@10: %.2f" % (hits_10)) hits_3 = hits_at_n_score(ranks, n=3) print("Hits@3: %.2f" % (hits_3)) hits_1 = hits_at_n_score(ranks, n=1) print("Hits@1: %.2f" % (hits_1)) data = pd.read_csv('triplet.csv') data.drop(data[data['name'] == 'no pc_item'].index, inplace=True) data.drop(data[data['prop'] == 'no price'].index, inplace=True) print(data.head()) import itertools pcItem = data['name'].unique() pcItem_embeddings = dict(zip(pcItem,model.get_embeddings(pcItem)))
def run_single_exp(config, dataset, model): hyperparams = config["hyperparams"][dataset][model] if hyperparams is None: print("dataset {0}...model {1} \ experiment is not conducted yet..." \ .format(dataset, config["model_name_map"][model])) return {"hyperparams": ".??"} print("dataset {0}...model {1}...\ hyperparameter:...{2}" \ .format(dataset, config["model_name_map"][model], hyperparams)) es_code = "{0}_{1}".format(dataset, model) load_func = getattr(ampligraph.datasets, config["load_function_map"][dataset]) X = load_func() # logging.debug("Loaded...{0}...".format(dataset)) # load model model_class = getattr(ampligraph.latent_features, config["model_name_map"][model]) model = model_class(**hyperparams) # Fit the model on training and validation set # The entire dataset will be used to filter out false positives statements # created by the corruption procedure: filter = np.concatenate((X['train'], X['valid'], X['test'])) if es_code in config["no_early_stopping"]: logging.debug("Fit without early stopping...") model.fit(X["train"]) else: logging.debug("Fit with early stopping...") model.fit( X["train"], True, { 'x_valid': X['valid'][::10], 'criteria': 'mrr', 'x_filter': filter, 'stop_interval': 2, 'burn_in': 0, 'check_interval': 100 }) # Run the evaluation procedure on the test set. Will create filtered rankings. # To disable filtering: filter_triples=None ranks = evaluate_performance(X['test'], model, filter, verbose=False) # compute and print metrics: mr = mr_score(ranks) mrr = mrr_score(ranks) hits_1 = hits_at_n_score(ranks, n=1) hits_3 = hits_at_n_score(ranks, n=3) hits_10 = hits_at_n_score(ranks, n=10) return { "mr": mr, "mrr": mrr, "H@1": hits_1, "H@3": hits_3, "H@10": hits_10, "hyperparams": hyperparams }
optimizer="adam", optimizer_params={"lr": 0.01}) model.fit(X['train']) y_pred = model.predict(X['test'][:5, ]) from scipy.special import expit print(expit(y_pred)) ranks = evaluate_performance(X['test'][:10], model=model) print(ranks) mrr = mrr_score(ranks) hits_10 = hits_at_n_score(ranks, n=10) print("MRR: %f, Hits@10: %f" % (mrr, hits_10)) import matplotlib.pyplot as plt from sklearn.manifold import TSNE embs = model.get_embeddings(embs_labels, type='entity') embs_2d = TSNE(n_components=2).fit_transform(embs) fig, ax = plt.subplots() ax.scatter(embs_2d[:, 0], embs_2d[:, 1]) for i, lab in enumerate(embs_labels): ax.annotate(lab, (embs_2d[i, 0], embs_2d[i, 1])) plt.show(fig)
def main(): # load Wordnet18 dataset: # X = load_wn18() X = load_fb15k_237() modify_flag = False # Initialize a ComplEx neural embedding model with pairwise loss function: # The model will be trained for 300 epochs. model = ComplEx( batches_count=10, seed=0, epochs=30, k=150, eta=10, # Use adam optimizer with learning rate 1e-3 optimizer='adam', optimizer_params={'lr': 1e-3}, # Use pairwise loss with margin 0.5 loss='pairwise', loss_params={'margin': 0.5}, # Use L2 regularizer with regularizer weight 1e-5 regularizer='LP', regularizer_params={ 'p': 2, 'lambda': 1e-5 }, # Enable stdout messages (set to false if you don't want to display) verbose=True) #, modify_flag = modify_flag) if False: # ground truth params (have not tried yet) # k: 350; epochs: 4000; eta: 30; loss: self_adversarial; loss_params: alpha: 1; margin: 0.5; optimizer: adam; optimizer_params: lr: 0.0001; seed: 0; batches_count: 50 model = ComplEx( batches_count=50, seed=0, epochs=4000, k=350, eta=30, # Use adam optimizer with learning rate 1e-3 optimizer='adam', optimizer_params={'lr': 1e-4}, # Use pairwise loss with margin 0.5 loss='self_adversarial', loss_params={ 'margin': 0.5, 'alpha': 1 }, # Use L2 regularizer with regularizer weight 1e-5 regularizer='LP', regularizer_params={ 'p': 2, 'lambda': 1e-5 }, # Enable stdout messages (set to false if you don't want to display) verbose=True, modify_flag=modify_flag) # For evaluation, we can use a filter which would be used to filter out # positives statements created by the corruption procedure. # Here we define the filter set by concatenating all the positives filter = np.concatenate((X['train'], X['valid'], X['test'])) # Fit the model on training and validation set model.fit(X['train'], early_stopping=True, early_stopping_params= \ { 'x_valid': X['valid'], # validation set 'criteria': 'hits10', # Uses hits10 criteria for early stopping 'burn_in': 100, # early stopping kicks in after 100 epochs 'check_interval': 20, # validates every 20th epoch 'stop_interval': 5, # stops if 5 successive validation checks are bad. 'x_filter': filter, # Use filter for filtering out positives 'corruption_entities': 'all', # corrupt using all entities 'corrupt_side': 's+o' # corrupt subject and object (but not at once) } ) # Run the evaluation procedure on the test set (with filtering). # To disable filtering: filter_triples=None # Usually, we corrupt subject and object sides separately and compute ranks # restore model sys.exit() # import the inspect_checkpoint library sys.exit() ranks = evaluate_performance( X['test'], model=model, filter_triples=filter, use_default_protocol= True, # corrupt subj and obj separately while evaluating verbose=True) # compute and print metrics: mrr = mrr_score(ranks) hits_10 = hits_at_n_score(ranks, n=10) print("MRR: %f, Hits@10: %f" % (mrr, hits_10))