def train(rels,
lang=LANG,
epochs=VALUE_DEFAULT_EPOCHS,
epochs_load=0,
size=VALUE_DEFAULT_SIZE,
negative=VALUE_DEFAULT_NEGATIVE,
memo=VALUE_DEFAULT_MEMO,
burnin=None,
reg=None,
resume=False):
try:
if resume:
filename = make_filename_model(lang, epochs_load, size, negative, memo, burnin, reg)
model = PoincareModel.load(filename)
print("resume {}".format(filename))
else:
print("first training")
raise ValueError()
except:
if resume:
print("file not found")
model = PoincareModel(rels, burn_in=0, regularization_coeff=0, negative=negative, size=size)
model.train(epochs=epochs, print_every=1500)
model.save(make_filename_model(lang, epochs+epochs_load, size, negative, memo, burnin, reg))
return model
def test_persistence_separate_file(self):
"""Tests whether the model is saved and loaded correctly when the arrays are stored separately."""
model = PoincareModel(self.data, burn_in=0, negative=3)
model.train(epochs=1)
model.save(testfile(), sep_limit=1)
loaded = PoincareModel.load(testfile())
self.models_equal(model, loaded)
def test_persistence_separate_file(self):
"""Tests whether the model is saved and loaded correctly when the arrays are stored separately."""
model = PoincareModel(self.data, burn_in=0, negative=3)
model.train(epochs=1)
model.save(testfile(), sep_limit=1)
loaded = PoincareModel.load(testfile())
self.models_equal(model, loaded)
def poincare(relations_file, key_epochs):
# read relation file
with open(relations_file) as f:
output_all = json.load(f)
relations = [(x[0], x[3]) for x in output_all if x[3] != ""]
# ######### poincare embedding ##########
# train
print(datetime.datetime.now(), "---poincare embedding Start", location())
# os.environ['PYTHONHASHSEED'] = '0'
# If you want to get the same result for the same input, set the PoincareModel () argument to workers = 1 and lock seed.
model = PoincareModel(train_data=relations,
size=2,
negative=8,
workers=1,
seed=1)
model.train(epochs=key_epochs)
print(datetime.datetime.now(), "---poincare embedding End", location())
# Create dictionary {keys: value} = {term: 2D coordinate values}
vec = {}
for word in model.kv.vocab.keys():
vec[word] = model.kv.get_vector(word)
print(datetime.datetime.now(), "---model.kv.vocab.keys End", location())
# plot
'''
poincare_map = gensim.viz.poincare.poincare_2d_visualization(model=model,
tree=relations,
figure_title="tutorial",
show_node_labels=model.kv.vocab.keys())
offline.plot(poincare_map)
'''
return vec, model
def test_wrong_gradients_raises_assertion(self):
"""Tests that discrepancy in gradients raises an error."""
model = PoincareModel(self.data, negative=3)
model._loss_grad = Mock(return_value=np.zeros((2 + model.negative,
model.size)))
with self.assertRaises(AssertionError):
model.train(epochs=1, batch_size=1, check_gradients_every=1)
def test_burn_in_only_done_once(self):
"""Tests that burn-in does not happen when train is called a second time."""
model = PoincareModel(self.data, negative=3, burn_in=1)
model.train(epochs=0)
original_vectors = np.copy(model.kv.syn0)
model.train(epochs=0)
self.assertTrue(np.allclose(model.kv.syn0, original_vectors))
def test_burn_in_only_done_once(self):
"""Tests that burn-in does not happen when train is called a second time."""
model = PoincareModel(self.data, negative=3, burn_in=1)
model.train(epochs=0)
original_vectors = np.copy(model.kv.syn0)
model.train(epochs=0)
self.assertTrue(np.allclose(model.kv.syn0, original_vectors))
def test_gradients_check(self):
"""Tests that the model is trained successfully with gradients check enabled."""
model = PoincareModel(self.data, negative=3)
try:
model.train(epochs=1, batch_size=1, check_gradients_every=1)
except Exception as e:
self.fail('Exception %s raised unexpectedly while training with gradient checking' % repr(e))
def test_persistence(self):
"""Tests whether the model is saved and loaded correctly."""
model = PoincareModel(self.data, burn_in=0, negative=3)
model.train(epochs=1)
model.save(testfile())
loaded = PoincareModel.load(testfile())
self.models_equal(model, loaded)
def test_persistence(self):
"""Tests whether the model is saved and loaded correctly."""
model = PoincareModel(self.data, burn_in=0, negative=3)
model.train(epochs=1)
model.save(testfile())
loaded = PoincareModel.load(testfile())
self.models_equal(model, loaded)
def test_gradients_check(self):
"""Tests that the model is trained successfully with gradients check enabled."""
model = PoincareModel(self.data, negative=3)
try:
model.train(epochs=1, batch_size=1, check_gradients_every=1)
except Exception as e:
self.fail('Exception %s raised unexpectedly while training with gradient checking' % repr(e))
def train_embeddings(
input_path, # path to input edge relations
delimiter, # input file delim
output_path, # path to output embedding vectors
size=2, # embed dimension
alpha=0.1, # learning rate
burn_in=10, # burn in train rounds
burn_in_alpha=0.01, # burn in learning rate
workers=1, # number of training threads used
negative=10, # negative sample size
epochs=100, # training rounds
print_every=500, # print train info
batch_size=10): # num samples in batch
# load file with edge relations between entities
relations = PoincareRelations(file_path=input_path, delimiter=delimiter)
# train model
model = PoincareModel(train_data=relations,
size=size,
alpha=alpha,
burn_in=burn_in,
burn_in_alpha=burn_in_alpha,
workers=workers,
negative=negative)
model.train(epochs=epochs, print_every=print_every, batch_size=batch_size)
# save output vectors
model.kv.save_word2vec_format(output_path)
return
def train_run(args):
# create experiment name from args
# create log folder, params folder from exp name
# Start logging in exp log folder
# save trained model in exp params folder
exp_name = 'HB'+'time' + str(datetime.now()) + '_EXP' + str(args.train_dir) + \
'_prbt' + str(args.prob_threshold) + '_reg' + str(args.reg_coef) + \
'_dim' + str(args.embed_dim) + '_lr' + str(args.learning_rate) + \
'_neg' + str(args.negs) + '_epoc' + str(args.epochs) + '_burnin' + str(args.burn_in)
exp_name = exp_name.replace(":", "-")
exp_name = exp_name.replace("/", "-")
exp_name = exp_name.replace(" ", "-")
print(exp_name)
# Training Logs Folder
exp_log_folder = args.log_folder + exp_name + '/'
if not os.path.exists(exp_log_folder):
os.makedirs(exp_log_folder)
logging_file = exp_log_folder + 'logging.txt'
logging.basicConfig(filename=logging_file, level=logging.INFO)
# Model saving folder
exp_params_folder = args.params_folder + exp_name + '/'
if not os.path.exists(exp_params_folder):
os.makedirs(exp_params_folder)
training_file = args.train_dir + args.trn_file
trn_dataset = data_loader.get_data_list(training_file, args.prob_threshold)
print("Number of training examples: ", len(trn_dataset))
# Create the model definition
model = PoincareModel(train_data=trn_dataset,
size=args.embed_dim,
alpha=args.learning_rate,
negative=args.negs,
regularization_coeff=args.reg_coef,
burn_in=args.burn_in,
burn_in_alpha=args.burn_in_alpha,
init_range=args.init_range,
seed=args.random_seed)
# Start the model training
model.train(epochs=args.epochs,
batch_size=args.batch_size,
print_every=args.print_every)
# Save the model
model_save_name = exp_params_folder + 'gensim_model.params'
model.save(model_save_name)
# Save the arguments in the params folder
args_fname = exp_params_folder + 'args_model.pkl'
with open(args_fname, "wb") as f:
pickle.dump(args, f)
return
def test_reproducible(self):
"""Tests that vectors are same for two independent models trained with the same seed."""
model_1 = PoincareModel(self.data_large, seed=1, negative=3, burn_in=1)
model_1.train(epochs=2)
model_2 = PoincareModel(self.data_large, seed=1, negative=3, burn_in=1)
model_2.train(epochs=2)
self.assertTrue(np.allclose(model_1.kv.syn0, model_2.kv.syn0))
def test_reproducible(self):
"""Tests that vectors are same for two independent models trained with the same seed."""
model_1 = PoincareModel(self.data_large, seed=1, negative=3, burn_in=1)
model_1.train(epochs=2)
model_2 = PoincareModel(self.data_large, seed=1, negative=3, burn_in=1)
model_2.train(epochs=2)
self.assertTrue(np.allclose(model_1.kv.syn0, model_2.kv.syn0))
def get_poincare_model(relations, emb_size, num_threads=1):
print('Learning Poincare embeddings with %d relations' % len(relations))
model = PoincareModel(relations, size=emb_size, negative=2)
t_start = datetime.now()
model.train(epochs=50)
t_end = datetime.now()
print('Training time: %s' % (t_end - t_start))
return model
def embedding(namespace, emb_fname):
graph = go_graph.copy()
for n, attr in go_graph._node.items():
if attr['namespace'] != namespace:
graph.remove_node(n)
model = PoincareModel(train_data=graph.edges(), size=dim)
model.train(epochs=num_epochs, print_every=500)
model.kv.save(emb_fname)
return model.kv
def test_vector_dtype(self):
"""Tests whether vectors have the correct dtype before and after training."""
model = PoincareModel(self.data_large,
dtype=np.float32,
burn_in=0,
negative=3)
self.assertEqual(model.kv.syn0.dtype, np.float32)
model.train(epochs=1)
self.assertEqual(model.kv.syn0.dtype, np.float32)
def test_train_after_load(self):
"""Tests whether the model can be trained correctly after loading from disk."""
model = PoincareModel(self.data, burn_in=0, negative=3)
model.train(epochs=1)
model.save(testfile())
loaded = PoincareModel.load(testfile())
model.train(epochs=1)
loaded.train(epochs=1)
self.models_equal(model, loaded)
def test_train_after_load(self):
"""Tests whether the model can be trained correctly after loading from disk."""
model = PoincareModel(self.data, burn_in=0, negative=3)
model.train(epochs=1)
model.save(testfile())
loaded = PoincareModel.load(testfile())
model.train(epochs=1)
loaded.train(epochs=1)
self.models_equal(model, loaded)
def poincare_disk_model(relations, dimension = 2, workers = 1, negative_sample = 2, batch_number = 10):
#for i in range(100):
# print(relations[i])
print("poincare ball model initialization")
model = PoincareModel(relations, negative = negative_sample, size = dimension, workers = workers)
print("start poincare ball model training")
#batch = int(len(relations)/batch_number)
#print("batch size: ",batch)
model.train(epochs = 50, print_every=1000, batch_size = 100000)
return model
def hello_world():
relations = [('math', 'science'), ('cs', 'science'), ('ml', 'cs'),
('db', 'cs'), ('linalg', 'math')]
model = PoincareModel(relations, size=8, negative=2)
model.train(epochs=50)
# Poincare distance between two entities
print(model.kv.distance('ml', 'db'))
# Compute absolute position in hierarchy of input node or vector.
# Values range between 0 and 1. A lower value indicates the input
# node or vector is higher in the hierarchy.
print(model.kv.norm('ml'))
print(model.kv.norm('ml'))
# Get the vectors
print(model.kv.get_vector('ml'))
model.save('test_embeddings.bin')
model.kv.save_word2vec_format('test_embeddings.w2v')
return
def test_poincare(self):
doc_count = 100
cits, _ = self.get_citation_graph(doc_count)
poincare_model = PoincareModel(
cits,
size=300,
alpha=0.1,
negative=10,
workers=1,
epsilon=1e-05,
regularization_coeff=1.0,
burn_in=10,
burn_in_alpha=0.01,
init_range=(-0.001, 0.001),
)
poincare_model.train(epochs=2, )
print(poincare_model.kv.vector_size)
def poincare_train(hypertouple_dataset, size=2, burn_in=0, epochs = 5, print_freq = 100):
"""
Train a poincare embedding
Args:
hypertouple_dataset (list): The hypertouple dataset to feed for training
size (int): size of model
burn_in (int): Burnin identifier
epochs (int): Number of epochs to train
print_freq (int): Update frequency number
Returns:
poincare_model (model object) : The trained Poincare Model
"""
poincare_model = None
try:
#poincare_model = PoincareModel(train_data = hypertouple_dataset)
poincare_model = PoincareModel(train_data=hypertouple_dataset, size = size, burn_in = burn_in)
poincare_model.train(epochs=epochs, print_every = print_freq)
except Exception as e:
print(e)
return poincare_model
def test_training_multiple(self):
"""Tests that calling train multiple times results in different vectors."""
model = PoincareModel(self.data_large, burn_in=0, negative=3)
model.train(epochs=2)
old_vectors = np.copy(model.kv.syn0)
model.train(epochs=1)
self.assertFalse(np.allclose(old_vectors, model.kv.syn0))
old_vectors = np.copy(model.kv.syn0)
model.train(epochs=0)
self.assertTrue(np.allclose(old_vectors, model.kv.syn0))
def test_training_multiple(self):
"""Tests that calling train multiple times results in different vectors."""
model = PoincareModel(self.data_large, burn_in=0, negative=3)
model.train(epochs=2)
old_vectors = np.copy(model.kv.syn0)
model.train(epochs=1)
self.assertFalse(np.allclose(old_vectors, model.kv.syn0))
old_vectors = np.copy(model.kv.syn0)
model.train(epochs=0)
self.assertTrue(np.allclose(old_vectors, model.kv.syn0))
def run(mode, language):
if mode == "train_poincare_custom":
gold_s,_ = read_all_data(domain = "science", language = language)
gold_e,_ = read_all_data(domain = "environment", language = language)
gold_f,_ = read_all_data(domain = "food", language = language)
vocabulary = set([relation[0].lower() for relation in gold_s] + [relation[1].lower() for relation in gold_s])
vocabulary = vocabulary | set([relation[0].lower() for relation in gold_f] + [relation[1].lower() for relation in gold_f])
vocabulary = vocabulary | set([relation[0].lower() for relation in gold_e] + [relation[1].lower() for relation in gold_e])
relations ="data/" + language + "/poincare_common_and_domains_" + language + ".tsv"
assert len(open(relations, 'r').readlines()) > 10, "Not enough relations to train embeddings. Aborting ..."
poincare_rel = PoincareRelations(relations)
dim = 50
model = PoincareModel(poincare_rel, size = dim)
print("Starting Training...")
model.train(epochs=400)
model.save("embeddings/poincare_common_and_domains_5_3_" + language + "_" + str(dim))
if mode == 'train_poincare_wordnet':
assert language == 'EN', "Wordnet consists only of English nouns"
gold_s,_ = read_all_data(domain = "science")
gold_e,_ = read_all_data(domain = "environment")
gold_f,_ = read_all_data(domain = "food")
vocabulary = set([relation[0].lower() for relation in gold_s] + [relation[1].lower() for relation in gold_s])
vocabulary = vocabulary | set([relation[0].lower() for relation in gold_f] + [relation[1].lower() for relation in gold_f])
vocabulary = vocabulary | set([relation[0].lower() for relation in gold_e] + [relation[1].lower() for relation in gold_e])
preprocess_wordnet('data/EN/noun_closure.tsv', vocabulary)
poincare_rel = PoincareRelations('data/EN/noun_closure_filtered.tsv')
dim = 50
model = PoincareModel(poincare_rel, size = dim)
print("Starting Training...")
model.train(epochs=400)
model.save("embeddings/wordnet_filtered" + "_" + str(dim))
if mode == "train_word2vec":
gold_s,relations_s = read_all_data("science")
gold_e,relations_e = read_all_data("environment")
gold_f,relations_f = read_all_data("food")
vocabulary = set([relation[0].lower() for relation in gold_s] + [relation[1].lower() for relation in gold_s])
vocabulary = vocabulary | set([relation[0].lower() for relation in gold_f] + [relation[1].lower() for relation in gold_f])
vocabulary = vocabulary | set([relation[0].lower() for relation in gold_e] + [relation[1].lower() for relation in gold_e])
documents = []
documents = list(read_input("/data/EN/wikipedia_utf8_filtered_20pageviews.csv",vocabulary))
model = gensim.models.Word2Vec(documents, size= 300, window = 10, min_count = 2, workers = 10)
model.train(documents, total_examples=len(documents), epochs=30)
print("Finished building word2vec model")
model.save("embeddings/own_embeddings_w2v")
def normalize_diacritics_text(text_string):
return unicodedata.normalize("NFC", text_string)
path = "/path/to/training_set.txt" # poincare_dict.txt
path_lexical = "path/to/lexical.txt" # example file_lexical_fon.txt following the HyperLex Format
path_validation = "/file/to/validation_set.txt" # validation set poincare_embedding_validation.txt
relations_ = load_doc(path)
# parameters
size = 15 # dimension of the embedding space
c = 15 # constant of negative curvature
epochs = 2000 # number of training epochs
# define the model
model = PoincareModel(relations_, size=size, negative=c)
model.train(epochs)
# save the model
model.save('/path/to/model')
# save model embedding
model.kv.save_word2vec_format("/path/to/embedding")
# load the model and the embedding
model = PoincareModel.load("/path/to/model")
model.kv.load_word2vec_format("/path/to/embedding")
all_relations = set(relations_)
# add different classes to the labels to add them to the graph
labels = list(set([_[0] for _ in relations_])) + ["girl_name", "boy_name", "mixed_name", "body_part", "benin_city"]
title = "Title Figure"
import pandas as pd
from gensim.models.poincare import PoincareModel, PoincareRelations, LexicalEntailmentEvaluation
from gensim.models.poincare import *
from gensim.test.utils import datapath
from gensim.models.poincare import PoincareKeyedVectors
data_dir = '../../../Documents/kg_embeddings_data/data/'
relations = []
edgelist = pd.read_csv(data_dir + 'edgelist_has_indication.txt',
header=None,
sep=' ')
edgelist_data = edgelist.values
data = pd.read_csv(data_dir + 'mapping_RDFGraph.txt', header=None, sep='\t')
data = data.values
mapping_dict = dict(zip(data[:, 1], data[:, 0]))
for item in edgelist_data:
it1 = mapping_dict[item[0]].split('/')[-1]
it2 = mapping_dict[item[1]].split('/')[-1]
relations.append((it1, it2))
model = PoincareModel(relations)
model.train(
epochs=50, batch_size=200
) # according to paper, good representations could be learned after only 20 epochs
model.kv.save_word2vec_format(
data_dir + 'poincare/embeddings_has_indication_poincare_minibatch_200.txt')
pdb.set_trace()
def compute_doc_vecs(experiment,
data_dir='./data',
workers=None,
override=False,
dense_vector_size=300,
sparse_vector_size=500000,
gpu=None):
"""
Examples:
python cli.py compute_doc_vecs wikisource --override=1 --gpu 0
python cli.py compute_doc_vecs ocb --override=1 --gpu 1
:param data_dir: Path to data (for input and output)
:param experiment: Experiment name (ocb or wikisource)
:param workers: Number of workers
:param override: Override existing output
:param dense_vector_size: Size of dense document vectors (avg word2vec, graph embeddings, ...)
:param sparse_vector_size: Size of sparse document vectors (TF-IDF)
:param cuda_device: Use CUDA device for Transformer models
:return:
"""
env = get_env()
data_dir = Path(data_dir)
logger.info(f'Experiment: {experiment}')
exp = Experiment(name=experiment, env=env, data_dir=data_dir)
exp.load_data()
exp.filter_docs()
models_dir = exp.models_dir
common_kwargs = exp.get_common_kwargs()
if not workers:
workers = env['workers']
logger.info(f'Using {workers} workers')
if gpu:
logger.info(f'Using CUDA device={gpu}')
os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu)
# TF-IDF
out_fp = models_dir / 'tfidf.pickle'
if override or not os.path.exists(out_fp):
rs = TfIdfRecSys(vector_size=sparse_vector_size, **common_kwargs)
rs.train(exp.texts)
rs.save_to_disk(out_fp, override=override)
# Doc2Vec
out_fp = models_dir / 'doc2vec.w2v.txt'
if override or not os.path.exists(out_fp):
rs = Doc2VecRecSys(**common_kwargs, vector_size=dense_vector_size)
rs.train(exp.texts)
rs.save_word2vec_format(out_fp, override=override)
out_fp = models_dir / 'doc2vec_512.w2v.txt'
if override or not os.path.exists(out_fp):
rs = Doc2VecRecSys(**common_kwargs, vector_size=dense_vector_size)
rs.train(exp.get_limited_texts(512))
rs.save_word2vec_format(out_fp, override=override)
out_fp = models_dir / 'doc2vec_4096.w2v.txt'
if override or not os.path.exists(out_fp):
rs = Doc2VecRecSys(**common_kwargs, vector_size=dense_vector_size)
rs.train(exp.get_limited_texts(4096))
rs.save_word2vec_format(out_fp, override=override)
# Avg GloVe
out_fp = models_dir / 'avg_glove.w2v.txt'
if override or not os.path.exists(out_fp):
rs = WeightedAvgWordVectorsRecSys(w2v_model=exp.get_w2v_model('glove'),
**common_kwargs)
rs.train(exp.texts)
rs.save_word2vec_format(out_fp, override=override)
# With custom GloVe embeddings
out_fp = models_dir / 'avg_glove_custom.w2v.txt'
if override or not os.path.exists(out_fp):
rs = WeightedAvgWordVectorsRecSys(
w2v_model=exp.get_w2v_model('glove_custom'), **common_kwargs)
rs.train(exp.texts)
rs.save_word2vec_format(out_fp, override=override)
out_fp = models_dir / 'avg_fasttext.w2v.txt'
if override or not os.path.exists(out_fp):
rs = WeightedAvgWordVectorsRecSys(
w2v_model=exp.get_w2v_model('fasttext'), **common_kwargs)
rs.train(exp.texts)
rs.save_word2vec_format(out_fp, override=override)
out_fp = models_dir / 'avg_fasttext_custom.w2v.txt'
if override or not os.path.exists(out_fp):
rs = WeightedAvgWordVectorsRecSys(
w2v_model=exp.get_w2v_model('fasttext_custom'), **common_kwargs)
rs.train(exp.texts)
rs.save_word2vec_format(out_fp, override=override)
out_fp = models_dir / 'avg_fasttext_custom_512.w2v.txt'
if override or not os.path.exists(out_fp):
rs = WeightedAvgWordVectorsRecSys(
w2v_model=exp.get_w2v_model('fasttext_custom'), **common_kwargs)
rs.train(exp.get_limited_texts(512))
rs.save_word2vec_format(out_fp, override=override)
out_fp = models_dir / 'avg_fasttext_custom_4096.w2v.txt'
if override or not os.path.exists(out_fp):
rs = WeightedAvgWordVectorsRecSys(
w2v_model=exp.get_w2v_model('fasttext_custom'), **common_kwargs)
rs.train(exp.get_limited_texts(4096))
rs.save_word2vec_format(out_fp, override=override)
# Transformers
# BERT standard pooled
out_fp = models_dir / 'bert-base-cased.w2v.txt'
if override or not os.path.exists(out_fp):
rs = TransformerRecSys(model_name_or_path=env['bert_dir'] +
'/bert-base-cased',
**common_kwargs)
rs.train(exp.texts)
rs.save_word2vec_format(models_dir / 'bert-base-cased.w2v.txt',
override=override)
# All "MEAN" transformers
for tf_name in [
'bert-base-cased', 'bert-large-cased', 'roberta-base',
'roberta-large', 'legal-bert'
]:
out_fp = models_dir / f'{tf_name}_mean.w2v.txt'
if override or not os.path.exists(out_fp):
rs = TransformerRecSys(model_name_or_path=env['bert_dir'] + '/' +
tf_name,
pooling_strategy='reduce_mean',
**common_kwargs)
rs.train(exp.texts)
rs.save_word2vec_format(out_fp, override=override)
# Long former
if transformers.__version__ == '2.0.0':
from longformer.longformer import Longformer
from transformers import RobertaTokenizer
out_fp = models_dir / 'longformer-base-4096-mean.w2v.txt'
if override or not os.path.exists(out_fp):
lf_lm = Longformer.from_pretrained(env['bert_dir'] +
'/longformer-base-4096')
lf_tokenizer = RobertaTokenizer.from_pretrained(env['bert_dir'] +
'/roberta-base')
lf_tokenizer.max_len = lf_lm.config.max_position_embeddings
rs = TransformerRecSys(language_model=lf_lm,
tokenizer=lf_tokenizer,
max_length=4096,
pooling_strategy='reduce_mean',
**common_kwargs)
rs.train(exp.texts)
rs.save_word2vec_format(out_fp, override=override)
out_fp = models_dir / 'longformer-large-4096-mean.w2v.txt'
if override or not os.path.exists(out_fp):
lf_lm = Longformer.from_pretrained(env['bert_dir'] +
'/longformer-large-4096')
lf_tokenizer = RobertaTokenizer.from_pretrained(env['bert_dir'] +
'/roberta-large')
lf_tokenizer.max_len = lf_lm.config.max_position_embeddings
rs = TransformerRecSys(language_model=lf_lm,
tokenizer=lf_tokenizer,
max_length=4096,
pooling_strategy='reduce_mean',
**common_kwargs)
rs.train(exp.texts)
rs.save_word2vec_format(out_fp, override=override)
else:
# Wait for https://github.com/allenai/longformer/pull/14
logger.warning('Cannot run LongFormer with transformers!=2.0.0')
# Sentence transformer
if LooseVersion(transformers.__version__) >= LooseVersion('2.8.0'):
# See https://github.com/UKPLab/sentence-transformers/blob/master/requirements.txt#L1
st_models = [
'bert-base-nli-mean-tokens',
'bert-large-nli-mean-tokens',
'roberta-base-nli-mean-tokens',
'roberta-large-nli-mean-tokens',
'bert-base-nli-stsb-mean-tokens',
'bert-large-nli-stsb-mean-tokens',
'roberta-base-nli-stsb-mean-tokens',
'roberta-large-nli-stsb-mean-tokens',
]
st_dir = env['datasets_dir'] + '/sentence_transformers/'
for st_model_name in st_models:
out_fp = models_dir / f's{st_model_name}.w2v.txt'
if override or not os.path.exists(out_fp):
rs = SentenceTransformerRecSys(model_name_or_path=st_dir +
st_model_name,
**common_kwargs)
rs.train(exp.texts)
rs.save_word2vec_format(out_fp, override=override)
# break
else:
logger.warning(
'Cannot run sentence-transformers with transformers==%s' %
transformers.__version__)
# Citation
# DeepWalk
out_fp = models_dir / 'deepwalk.pickle'
if override or not os.path.exists(out_fp):
rs = GraphEmbeddingRecSys(include_seeds=exp.get_included_seeds(),
graph_model_cls='karateclub.DeepWalk',
graph_model_kwargs=dict(
dimensions=dense_vector_size,
workers=workers),
**common_kwargs)
rs.train(exp.cits)
rs.save_to_disk(out_fp, override=override)
# Diff2Vec
"""
out_fp = models_dir / 'diff2vec.pickle'
if override or not os.path.exists(out_fp):
diff2vec = GraphEmbeddingRecSys(
include_seeds=exp.get_included_seeds(),
graph_model_cls='karateclub.Diff2Vec',
graph_model_kwargs=dict(dimensions=dense_vector_size, workers=workers),
**common_kwargs
)
diff2vec.train(exp.cits)
diff2vec.save_to_disk(out_fp, override=override)
"""
# Walklets
out_fp = models_dir / 'walklets.pickle'
if override or not os.path.exists(out_fp):
walklets_window_size = 5 # or 3
walklets_dim = int(dense_vector_size /
walklets_window_size) # must be int
rs = GraphEmbeddingRecSys(include_seeds=exp.get_included_seeds(),
graph_model_cls='karateclub.Walklets',
graph_model_kwargs=dict(
dimensions=walklets_dim,
window_size=walklets_window_size,
workers=workers),
**common_kwargs)
rs.train(exp.cits)
rs.save_to_disk(out_fp, override=override)
# Node2Vec
out_fp = models_dir / 'node2vec.pickle'
if override or not os.path.exists(out_fp):
rs = GraphEmbeddingRecSys(include_seeds=exp.get_included_seeds(),
graph_model_cls='node2vec.Node2Vec',
graph_model_kwargs=dict(
dimensions=dense_vector_size,
workers=workers),
**common_kwargs)
rs.train(exp.cits)
rs.save_to_disk(out_fp, override=override)
# NodeSketch
"""
out_fp = models_dir / 'nodesketch.pickle'
if override or not os.path.exists(out_fp):
nodesketch = GraphEmbeddingRecSys(
include_seeds=exp.get_included_seeds(),
graph_model_cls='karateclub.NodeSketch',
graph_model_kwargs=dict(dimensions=dense_vector_size),
**common_kwargs
)
nodesketch.train(exp.cits)
nodesketch.save_to_disk(out_fp, override=override)
"""
# BoostNE
out_fp = models_dir / 'boostne.pickle'
if override or not os.path.exists(out_fp):
boostne_iters = 9 # 14
boostne_dim = 30 # 20
assert boostne_dim * (boostne_iters + 1) == dense_vector_size
boostne = GraphEmbeddingRecSys(
include_seeds=exp.get_included_seeds(),
# vector_size=dense_vector_size,
graph_model_cls='karateclub.BoostNE',
graph_model_kwargs=dict(
dimensions=boostne_dim, # 8
order=2, # 2
iterations=boostne_iters, # 16
alpha=0.01,
),
# Take only embedding from last boosting
# node_embedding_slice=slice(dense_vector_size * boostne_iters, dense_vector_size * (boostne_iters + 1)),
**common_kwargs)
boostne.train(exp.cits)
boostne.save_to_disk(out_fp, override=override)
# Poincare
from gensim.models.poincare import PoincareModel
out_fp = models_dir / 'poincare.w2v.txt'
if override or not os.path.exists(out_fp):
poincare_model = PoincareModel(
exp.cits,
size=300,
alpha=0.1,
negative=10,
workers=1,
epsilon=1e-05,
regularization_coeff=1.0,
burn_in=10,
burn_in_alpha=0.01,
init_range=(-0.001, 0.001),
)
poincare_model.train(epochs=50, )
# init empty model
poincare = KeyedVectors(vector_size=poincare_model.kv.vector_size)
# ignore items not part of gold standard
for doc_id in list(poincare_model.kv.vocab.keys()):
if doc_id in exp.get_included_seeds():
poincare.add(doc_id, poincare_model.kv.get_vector(doc_id))
poincare.save_word2vec_format(out_fp)
logger.info('Done')
# writer = csv.writer(outfile, delimiter='\t')
# writer.writerows(hyp)
with open('polyFileEdgesRand.tsv', 'w+') as outfile:
writer = csv.writer(outfile, delimiter='\t')
writer.writerows(poly)
with open('polyTrainFileRand.tsv', 'w+') as outfile:
writer = csv.writer(outfile, delimiter='\t')
writer.writerows(polyTrain)
with open('polyTestFileRand.tsv', 'w+') as outfile:
writer = csv.writer(outfile, delimiter='\t')
writer.writerows(polyTest)
# with open('hypTestFile.tsv', 'w+') as outfile:
# writer = csv.writer(outfile, delimiter='\t')
# writer.writerows(hypTest)
#
# with open('hypTrainFile.tsv', 'w+') as outfile:
# writer = csv.writer(outfile, delimiter='\t')
# writer.writerows(hypTrain)
#file_path = datapath('randFileEdges.tsv')
print("POLY TO HYP")
model = PoincareModel(PoincareRelations("randFileEdges.tsv"), negative=2)
model.train(epochs=100)
#print(model.kv.most_similar('pitch.n.02', topn=10))
test = LinkPredictionEvaluation("polyTrainFileRand.tsv", "polyTestFileRand.tsv", model.kv)
print(test.evaluate())
recon = ReconstructionEvaluation("polyFileEdgesRand.tsv", model.kv)
print(recon.evaluate())
import json
from gensim.models.poincare import PoincareModel, PoincareRelations
from gensim.test.utils import datapath
# leaves and total nodes
num = 1840
total = 8801
file_path = datapath('path of the original dataset for poincare')
model = PoincareModel(PoincareRelations(file_path,delimiter=','), negative=2, size=32)
model.train(epochs=10000,print_every=10)
child2ParentDict = {}
for each in range(total):
r = model.kv.closest_parent(str(each))
if r is None:
print(each)
else:
child2ParentDict[each] = r
parent2ChildDict = {}
for child in child2ParentDict:
parent = child2ParentDict[child]
if(parent in parent2ChildDict):
parent2ChildDict[parent].append(child)
else:
def test_wrong_gradients_raises_assertion(self):
"""Tests that discrepancy in gradients raises an error."""
model = PoincareModel(self.data, negative=3)
model._loss_grad = Mock(return_value=np.zeros((2 + model.negative, model.size)))
with self.assertRaises(AssertionError):
model.train(epochs=1, batch_size=1, check_gradients_every=1)
from gensim.models.poincare import PoincareModel, PoincareRelations
from gensim.test.utils import datapath
from utils import Parameters
import pandas as pd
data_path = datapath("D:/PyCharm/PyCharm_Project/paper/data/type_relation.tsv")
type_embedding_path = "data/type_embedding"
model = PoincareModel(train_data=PoincareRelations(data_path, encoding="gbk"),
size=Parameters.type_embedding_dim,
negative=3)
model.train(epochs=50, print_every=5)
print(model.kv.word_vec("川菜"))
model.save(type_embedding_path)
# poincareModel = PoincareModel.load("data/type_embedding")
# print(poincareModel.kv.word_vec('东北菜'))
def test_vector_dtype(self):
"""Tests whether vectors have the correct dtype before and after training."""
model = PoincareModel(self.data_large, dtype=np.float32, burn_in=0, negative=3)
self.assertEqual(model.kv.syn0.dtype, np.float32)
model.train(epochs=1)
self.assertEqual(model.kv.syn0.dtype, np.float32)
def test_training(self):
"""Tests that vectors are different before and after training."""
model = PoincareModel(self.data_large, burn_in=0, negative=3)
old_vectors = np.copy(model.kv.syn0)
model.train(epochs=2)
self.assertFalse(np.allclose(old_vectors, model.kv.syn0))
def test_error_if_negative_more_than_population(self):
"""Tests error is rased if number of negatives to sample is more than remaining nodes."""
model = PoincareModel(self.data, negative=5)
with self.assertRaises(ValueError):
model.train(epochs=1)
def run(mode,
embedding,
embedding_name,
experiment_name=None,
log=False,
trial=False):
if embedding == "fasttext":
#model = gensim.models.KeyedVectors.load_word2vec_format('wiki-news-300d-1M-subword.vec', binary=False)
model = gensim.models.FastText.load_fasttext_format('wiki.en.bin')
#model = gensim.models.FastText.load_fasttext_format('crawl-300d-2M.vec')
elif embedding == "wiki2M":
#model = gensim.models.FastText.load_fasttext_format('crawl-300d-2M.vec','vec')
model = gensim.models.KeyedVectors.load_word2vec_format(
'embeddings/crawl-300d-2M.vec', binary=False)
#model.save("crawl-300d-2M.bin")
elif embedding == "wiki1M_subword":
model = gensim.models.KeyedVectors.load_word2vec_format(
'embeddings/wiki-news-300d-1M-subword.vec', binary=False)
elif embedding == "own_w2v":
model = gensim.models.KeyedVectors.load(
'embeddings/own_embeddings_w2v')
elif embedding == "quick":
model = gensim.models.KeyedVectors.load_word2vec_format(
'embeddings/crawl-300d-2M.vec', binary=False, limit=50000)
elif embedding == "poincare":
model = PoincareModel.load(
'embeddings/poincare_common_domains02_5_3_50')
print(len(model.kv.vocab))
words = [
"computer_science", "biology", "physics", "science", "virology",
"life_science", "chemistry", "earth_science", "algebra",
"economics", "optics"
"immunology"
]
for word in words:
print("Current word: ", word)
if word in model.kv.vocab:
try:
print("Closest Parent: ", model.kv.closest_parent(word))
print("Closest Child ", model.kv.closest_child(word))
print("Descendants: ", model.kv.descendants(word))
print("Ancestors: ", model.kv.ancestors(word))
print("Hierarchy diff to Science: ",
model.kv.difference_in_hierarchy(word, "science"))
print('\n')
except:
continue
else:
print("Word not in Vocab")
if mode == "visualize_embedding_poincare":
relations = set([])
filename_in = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"data/isas_1000.tsv")
with open(filename_in, 'r') as f:
reader = csv.reader(f, delimiter='\t')
for i, line in enumerate(reader):
relations.add((line[0], line[1]))
plot = poincare_2d_visualization(model, relations, experiment_name)
py.image.save_as(plot, "vis/" + experiment_name + '.png')
print("Starting visualization")
#visualize_taxonomy(vectors, names)
#todo own file for train
if mode == "visualize_embedding":
gold, relations = read_all_data()
vectors = []
names = []
for relation in (
[relation1[1].replace(" ", "_") for relation1 in relations] +
[relation2[2].replace(" ", "_") for relation2 in relations]):
if relation not in names:
if relation not in model.wv:
print(relation)
continue
vectors.append(model.wv[relation])
names.append(relation)
visualize_taxonomy(vectors, names, experiment_name)
if mode == 'train_poincare':
# gold,relations = read_all_data()
# freq_science = [3,5]
# for entry_science in freq_science:
# relations = './data/' + domain +'_crawl_' + str(entry_science) +'.tsv'
# #relations = './data/science_crawl_merge_10_3_02.tsv'
# poincare_rel = PoincareRelations(relations)
# dim = 50
# model = PoincareModel(poincare_rel, size = dim)
# print("Starting Training...")
# model.train(epochs=400)
# model.save("embeddings/embeddings_" + domain + "_crawl_poincare_" + str(entry_science) + "_" + str(dim))
# #model.save("embeddings/embeddings_science_crawl_merge_poincare_10_3_50_02")
# break
relations = './data/poincare_common_domains.tsv'
#relations = './data/science_crawl_merge_10_3_02.tsv'
poincare_rel = PoincareRelations(relations)
dim = 50
model = PoincareModel(poincare_rel, size=dim)
print("Starting Training...")
model.train(epochs=400)
model.save("embeddings/poincare_common_domains_5_3" + "_" + str(dim))
if mode == "train_word2vec":
gold_s, relations_s = read_all_data("science")
gold_e, relations_e = read_all_data("environment")
gold_f, relations_f = read_all_data("food")
vocabulary = set([relation[2] for relation in gold_s] +
[relation[1] for relation in gold_s])
vocabulary = vocabulary | set([relation[2] for relation in gold_f] +
[relation[1] for relation in gold_f])
vocabulary = vocabulary | set([relation[2] for relation in gold_e] +
[relation[1] for relation in gold_e])
documents = list(
read_input(
"/srv/data/5aly/data_text/wikipedia_utf8_filtered_20pageviews.csv",
vocabulary))
model = gensim.models.Word2Vec(size=300,
window=5,
min_count=5,
workers=30)
model.build_vocab(documents)
#model.train(documents, total_examples = len(documents), epochs=10)
model.train(documents, total_examples=model.corpus_count, epochs=30)
model.save("embeddings/own_embeddings_w2v_all")
elif mode == "analysis":
gold, relations = read_all_data()
voc_rel = set([relation[1] for relation in relations] +
[relation[2] for relation in relations])
voc_gold = set([relation[1] for relation in gold] +
[relation[2] for relation in gold])
print("Vokabeln in Gold: " + str(len(voc_gold)) +
"Vokabeln in Taxonomy: " + str(len(voc_rel)))
def train(): from gensim.models.poincare import PoincareModel,PoincareRelations relations = PoincareRelations(file_path="../data/word_relation.csv", delimiter=',') model = PoincareModel(relations, negative=10,size=5) model.train(epochs=500) return model
def test_burn_in(self):
"""Tests that vectors are different after burn-in."""
model = PoincareModel(self.data, burn_in=1, negative=3)
original_vectors = np.copy(model.kv.syn0)
model.train(epochs=0)
self.assertFalse(np.allclose(model.kv.syn0, original_vectors))
('West Romance', 'Spanish'), ('West Romance', 'Portguese'),
('West Romance', 'Galician'), ('West Romance', 'Catalan'),
('West Romance', 'Provencal'), ('West Romance', 'Romansh'),
('Celtic', 'Breton'), ('Celtic', 'Welsh'), ('Celtic', 'Irish'),
('Baltic', 'Latvian'), ('Baltic', 'Lituanian'), ('Slavic', 'Russian'),
('Slavic', 'Polish'), ('Slavic', 'Ukrainian'), ('Slavic', 'Bulgagian'),
('Slavic', 'Czech'), ('Slavic', 'Slovakian'), ('Slavic', 'Croatian'),
('Slavic', 'Serbian'), ('Finno-Ugric', 'Finnish'),
('Finno-Ugric', 'Estonian'), ('Finno-Ugric', 'Hungarian'),
('Non-Indo-European', 'Uralic'), ('Romance', 'East-Romance'),
('Romance', 'West Romance'), ('Germanic', 'West Germanic'),
('Germanic', 'North Germanic'), ('Anglo-Norman', 'English'),
('Anglo-Norman', 'French'), ('West Germanic', 'Scots'),
('West Romance', 'Anglo-Norman')]
model = PoincareModel(data, negative=2, size=2)
model.train(epochs=50)
print("Distance between English and French: ",
model.kv.distance('English', 'French'))
print("Distance between English and German: ",
model.kv.distance('English', 'German'))
print("Distance between English and Frisian: ",
model.kv.distance('English', 'Frisian'))
print("Distance between English and Welsh: ",
model.kv.distance('English', 'Welsh'))
print("Distance between English and Russian: ",
model.kv.distance('English', 'Russian'))
print("Distance between English and Hungarian: ",
model.kv.distance('English', 'Hungarian'))
print("Distance between English and Albanian: ",
model.kv.distance('English', 'Albanian'))
print("Distance between English and Greek: ",
def test_error_if_negative_more_than_population(self):
"""Tests error is rased if number of negatives to sample is more than remaining nodes."""
model = PoincareModel(self.data, negative=5)
with self.assertRaises(ValueError):
model.train(epochs=1)
