def __init__(self,
dataframe: List[Dict[str, str]],
teacher_model: LanguageTransformer,
combinations: List[Tuple[str, str]],
batch_size: int = 8,
name=''):
self.combinations = combinations
self.name = name
self.batch_size = batch_size
if name:
name = "_" + name
self.csv_file = "mse_evaluation" + name + "_results.csv"
self.csv_headers = ["epoch", "steps"]
self.data = {}
logging.info("Compute teacher embeddings")
all_source_sentences = set()
for src_lang, trg_lang in self.combinations:
src_sentences = []
trg_sentences = []
for row in dataframe:
if row[src_lang].strip() != "" and row[trg_lang].strip() != "":
all_source_sentences.add(row[src_lang])
src_sentences.append(row[src_lang])
trg_sentences.append(row[trg_lang])
self.data[(src_lang, trg_lang)] = (src_sentences, trg_sentences)
self.csv_headers.append("{}-{}".format(src_lang, trg_lang))
all_source_sentences = list(all_source_sentences)
all_src_embeddings = teacher_model.encode(all_source_sentences,
batch_size=self.batch_size)
self.teacher_embeddings = {
sent: emb
for sent, emb in zip(all_source_sentences, all_src_embeddings)
}
if not os.path.exists(sts_dataset_path):
util.http_get('https://sbert.net/datasets/stsbenchmark.tsv.gz', sts_dataset_path)
# Read the dataset
model_name = 'bert-base-nli-mean-tokens'
train_batch_size = 16
num_epochs = 4
model_save_path = 'output/training_stsbenchmark_continue_training-'+model_name+'-'+datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
# Load a pre-trained sentence transformer model
model = LanguageTransformer(model_name)
# Convert the dataset to a DataLoader ready for training
logging.info("Read STSbenchmark train dataset")
train_samples = []
dev_samples = []
test_samples = []
with gzip.open(sts_dataset_path, 'rt', encoding='utf8') as fIn:
reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
for row in reader:
score = float(row['score']) / 5.0 # Normalize score to range 0 ... 1
inp_example = InputExample(texts=[row['sentence1'], row['sentence2']], label=score)
if row['split'] == 'dev':
dev_samples.append(inp_example)
from language_bert.readers import STSBenchmarkDataReader
import logging
import torch
#Limit torch to 4 threads, as this example runs on the CPU
torch.set_num_threads(4)
#### Just some code to print debug information to stdout
logging.basicConfig(format='%(asctime)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO,
handlers=[LoggingHandler()])
#### /print debug information to stdout
#1) Point the transformer model to the BERT / RoBERTa etc. model you would like to use. Ensure that output_hidden_states is true
word_embedding_model = models.Transformer(
'bert-base-uncased', model_args={'output_hidden_states': True})
#2) Add WKPooling
pooling_model = models.WKPooling(
word_embedding_model.get_word_embedding_dimension())
#3) Create a sentence transformer model to glue both models together
model = LanguageTransformer(modules=[word_embedding_model, pooling_model])
sts_reader = STSBenchmarkDataReader('../datasets/stsbenchmark')
evaluator = EmbeddingSimilarityEvaluator.from_input_examples(
sts_reader.get_examples("sts-test.csv"))
model.evaluate(evaluator)
"""
This examples clusters different sentences that come from the same wikipedia article.
It uses the 'wikipedia-sections' model, a model that was trained to differentiate if two sentences from the
same article come from the same section or from different sections in that article.
"""
from language_bert import LanguageTransformer
from sklearn.cluster import AgglomerativeClustering
embedder = LanguageTransformer('bert-base-wikipedia-sections-mean-tokens')
#Sentences and sections are from Wikipeda.
#Source: https://en.wikipedia.org/wiki/Bushnell,_Illinois
corpus = [
("Bushnell is located at 40°33′6″N 90°30′29″W (40.551667, -90.507921).",
"Geography"),
("According to the 2010 census, Bushnell has a total area of 2.138 square miles (5.54 km2), of which 2.13 square miles (5.52 km2) (or 99.63%) is land and 0.008 square miles (0.02 km2) (or 0.37%) is water.",
"Geography"),
("The town was founded in 1854 when the Northern Cross Railroad built a line through the area.",
"History"),
("Nehemiah Bushnell was the President of the Railroad, and townspeople honored him by naming their community after him. ",
"History"),
("Bushnell was also served by the Toledo, Peoria and Western Railway, now the Keokuk Junction Railway.",
"History"),
("As of the census[6] of 2000, there were 3,221 people, 1,323 households, and 889 families residing in the city. ",
"Demographics"),
("The population density was 1,573.9 people per square mile (606.7/km²).",
"Demographics"),
("There were 1,446 housing units at an average density of 706.6 per square mile (272.3/km²).",
"Demographics"),
("From 1991 to 2012, Bushnell was home to one of the largest Christian Music and Arts festivals in the world, known as the Cornerstone Festival.",
model_name = 'bert-base-uncased'
logging.info("Model %s will be created from based model %s" %
(model_save_path, model_name))
word_embedding_model = models.Transformer(model_name)
cnn = models.CNN(in_word_embedding_dimension=word_embedding_model.
get_word_embedding_dimension())
# Apply mean pooling to get one fixed sized sentence vector
pooling_model = models.Pooling(cnn.get_word_embedding_dimension(),
pooling_mode_mean_tokens=True,
pooling_mode_cls_token=False,
pooling_mode_max_tokens=False)
model = LanguageTransformer(modules=[word_embedding_model, cnn, pooling_model],
device='cpu')
# -----------PREPARE TRAIN DATASET -------------------
# create the training dataset
logging.info("Read train dataset")
train_dataset = SentencesDataset(train_samples, model=model)
train_dataloader = DataLoader(train_dataset,
shuffle=True,
batch_size=train_batch_size)
train_loss = losses.MutualInformationLoss(
model=model,
sentence_embedding_dimension=model.get_sentence_embedding_dimension())
# -------- PREPARE Dev evalution DATASET ------------------
#Read dataset and use it as development set
logging.info("Read dev dataset")
from language_bert import LanguageTransformer, LoggingHandler
import numpy as np
import logging
#### Just some code to print debug information to stdout
np.set_printoptions(threshold=100)
logging.basicConfig(format='%(asctime)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO,
handlers=[LoggingHandler()])
#### /print debug information to stdout
# Load pre-trained Sentence Transformer Model (based on DistilBERT). It will be downloaded automatically
model = LanguageTransformer('distilbert-base-nli-stsb-mean-tokens')
# Embed a list of sentences
sentences = [
'This framework generates embeddings for each input sentence',
'Sentences are passed as a list of string.',
'The quick brown fox jumps over the lazy dog.'
]
sentence_embeddings = model.encode(sentences)
# The result is a list of sentence embeddings as numpy arrays
for sentence, embedding in zip(sentences, sentence_embeddings):
print("Sentence:", sentence)
print("Embedding:", embedding)
print("")
"""
This is a simple application for sentence embeddings: clustering
Sentences are mapped to sentence embeddings and then k-mean clustering is applied.
"""
from language_bert import LanguageTransformer
from sklearn.cluster import KMeans
embedder = LanguageTransformer('bert-base-nli-mean-tokens')
# Corpus with example sentences
corpus = [
'A man is eating food.', 'A man is eating a piece of bread.',
'A man is eating pasta.', 'The girl is carrying a baby.',
'The baby is carried by the woman', 'A man is riding a horse.',
'A man is riding a white horse on an enclosed ground.',
'A monkey is playing drums.',
'Someone in a gorilla costume is playing a set of drums.',
'A cheetah is running behind its prey.',
'A cheetah chases prey on across a field.'
]
corpus_embeddings = embedder.encode(corpus)
# Perform kmean clustering
num_clusters = 5
clustering_model = KMeans(n_clusters=num_clusters)
clustering_model.fit(corpus_embeddings)
cluster_assignment = clustering_model.labels_
clustered_sentences = [[] for i in range(num_clusters)]
for sentence_id, cluster_id in enumerate(cluster_assignment):
#Limit torch to 4 threads
torch.set_num_threads(4)
#### Just some code to print debug information to stdout
logging.basicConfig(format='%(asctime)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO,
handlers=[LoggingHandler()])
#### /print debug information to stdout
model_name = '../training/nli/output/training_nli_bert-base-uncased-2021-01-10_14-44-13'
# Load a named sentence model (based on BERT). This will download the model from our server.
# Alternatively, you can also pass a filepath to LanguageTransformer()
model = LanguageTransformer(model_name)
sts_corpus = "../datasets/stsbenchmark/"
target_eval_files = set(
['sts', 'sts12', 'sts13', 'sts14', 'sts15', 'sts16', 'sick-r'])
evaluators = [
] #evaluators has a list of different evaluator classes we call periodically
sts_reader = STSBenchmarkDataReader(
os.path.join(script_folder_path, sts_corpus))
for target in target_eval_files:
output_filename_eval = os.path.join(script_folder_path,
sts_corpus + target + "-test.csv")
evaluators.append(
EmbeddingSimilarityEvaluator.from_input_examples(
sts_reader.get_examples(output_filename_eval), name=target))
As embeddings model, we use the SBERT model 'distilbert-multilingual-nli-stsb-quora-ranking',
that it aligned for 100 languages. I.e., you can type in a question in various languages and it will
return the closest questions in the corpus (questions in the corpus are mainly in English).
"""
from language_bert import LanguageTransformer, util
import os
import csv
import pickle
import time
import torch
from annoy import AnnoyIndex
if __name__ == '__main__':
model_name = 'distilbert-multilingual-nli-stsb-quora-ranking'
model = LanguageTransformer(model_name)
url = "http://qim.fs.quoracdn.net/quora_duplicate_questions.tsv"
dataset_path = "quora_duplicate_questions.tsv"
max_corpus_size = 100000
n_trees = 256 #Number of trees used for Annoy. More trees => better recall, worse run-time
embedding_size = 768 #Size of embeddings
top_k_hits = 10 #Output k hits
annoy_index_path = 'quora-embeddings-{}-size-{}-annoy_index-trees-{}.ann'.format(
model_name.replace('/', '_'), max_corpus_size, n_trees)
embedding_cache_path = 'quora-embeddings-{}-size-{}.pkl'.format(
model_name.replace('/', '_'), max_corpus_size)
#Check if embedding cache path exists
sentences in parallel. This gives a near linear speed-up
when encoding large text collections.
"""
from language_bert import LanguageTransformer, LoggingHandler
import logging
logging.basicConfig(format='%(asctime)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO,
handlers=[LoggingHandler()])
#Important, you need to shield your code with if __name__. Otherwise, CUDA runs into issues when spawning new processes.
if __name__ == '__main__':
#Create a large list of 100k sentences
sentences = ["This is sentence {}".format(i) for i in range(100000)]
#Define the model
model = LanguageTransformer('distilbert-base-nli-stsb-mean-tokens')
#Start the multi-process pool on all available CUDA devices
pool = model.start_multi_process_pool()
#Compute the embeddings using the multi-process pool
emb = model.encode_multi_process(sentences, pool)
print("Embeddings computed. Shape:", emb.shape)
#Optional: Stop the proccesses in the pool
model.stop_multi_process_pool(pool)
from language_bert import LanguageTransformer, evaluation, LoggingHandler
import sys
import gzip
import os
import logging
logging.basicConfig(format='%(asctime)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO,
handlers=[LoggingHandler()])
model_name = sys.argv[1]
filepaths = sys.argv[2:]
inference_batch_size = 32
model = LanguageTransformer(model_name)
for filepath in filepaths:
src_sentences = []
trg_sentences = []
with gzip.open(filepath, 'rt',
encoding='utf8') if filepath.endswith('.gz') else open(
filepath, 'r', encoding='utf8') as fIn:
for line in fIn:
splits = line.strip().split('\t')
if len(splits) >= 2:
src_sentences.append(splits[0])
trg_sentences.append(splits[1])
logging.info(
os.path.basename(filepath) + ": " + str(len(src_sentences)) +
that it aligned for 100 languages. I.e., you can type in a question in various languages and it will
return the closest questions in the corpus (questions in the corpus are mainly in English).
"""
from language_bert import LanguageTransformer, util
import os
from elasticsearch import Elasticsearch
import csv
import time
import tqdm.autonotebook
from elasticsearch import helpers
if __name__ == '__main__':
es = Elasticsearch()
model = LanguageTransformer('distilbert-multilingual-nli-stsb-quora-ranking')
url = "http://qim.fs.quoracdn.net/quora_duplicate_questions.tsv"
dataset_path = "quora_duplicate_questions.tsv"
max_corpus_size = 100000
#Download dataset if needed
if not os.path.exists(dataset_path):
print("Download dataset")
util.http_get(url, dataset_path)
#Get all unique sentences from the file
all_questions = {}
with open(dataset_path, encoding='utf8') as fIn:
reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_MINIMAL)
for row in reader:
"""
This is a simple application for sentence embeddings: semantic search
We have a corpus with various sentences. Then, for a given query sentence,
we want to find the most similar sentence in this corpus.
This script outputs for various queries the top 5 most similar sentences in the corpus.
"""
from language_bert import LanguageTransformer, util
import torch
embedder = LanguageTransformer('distilbert-base-nli-stsb-mean-tokens')
# Corpus with example sentences
corpus = [
'A man is eating food.', 'A man is eating a piece of bread.',
'The girl is carrying a baby.', 'A man is riding a horse.',
'A woman is playing violin.', 'Two men pushed carts through the woods.',
'A man is riding a white horse on an enclosed ground.',
'A monkey is playing drums.', 'A cheetah is running behind its prey.'
]
corpus_embeddings = embedder.encode(corpus, convert_to_tensor=True)
# Query sentences:
queries = [
'A man is eating pasta.',
'Someone in a gorilla costume is playing a set of drums.',
'A cheetah chases prey on across a field.'
]
# Find the closest 5 sentences of the corpus for each query sentence based on cosine similarity
from language_bert import LanguageTransformer, util
import sys
import os
import time
import torch
import gzip
import csv
#Limit torch to 4 threads
torch.set_num_threads(4)
model_name = sys.argv[1] if len(sys.argv) > 1 else 'bert-base-nli-mean-tokens'
# Load a named sentence model (based on BERT). This will download the model from our server.
# Alternatively, you can also pass a filepath to LanguageTransformer()
model = LanguageTransformer(model_name)
nli_dataset_path = 'datasets/AllNLI.tsv.gz'
sentences = set()
max_sentences = 100000
#Download datasets if needed
if not os.path.exists(nli_dataset_path):
util.http_get('https://sbert.net/datasets/AllNLI.tsv.gz', nli_dataset_path)
with gzip.open(nli_dataset_path, 'rt', encoding='utf8') as fIn:
reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
for row in reader:
sentences.add(row['sentence1'])
if len(sentences) >= max_sentences:
break