def addTranslation(alljobs,
outputFile,
method="google",
fallback=True,
maxWrite=-1):
if method == "google":
from googletrans import Translator
modelGoogle = Translator()
if method == "easynmt" or fallback:
from easynmt import EasyNMT
modelNMT = EasyNMT('opus-mt')
for i, entry in enumerate(alljobs[:maxWrite]):
entry["CLEANED_JOBS"] = textCleaner(entry["CONTENT"])
# Seems to be a maximum number of characters allowed for translation
# Split in half at a sentence boundary
sentences = entry["CLEANED_JOBS"].split(". ")
chunks = [". ".join(sentences)]
entry["TRANSLATED_JOBS"] = ""
if len(entry["CLEANED_JOBS"]) > 5000:
mid = int(len(sentences) / 2)
chunks = [". ".join(sentences[:mid]), ". ".join(sentences[mid:])]
for chunk in chunks:
if method == "google":
try:
translate = modelGoogle.translate(chunk, dest="en")
translate = translate.text
except (TypeError, AttributeError, IndexError,
ReadTimeout) as e:
print("Failed to translate entry", i)
print("Error was:", e)
if fallback:
print("Trying instead with EasyNMT")
translate = modelNMT.translate(chunk, target_lang="en")
else:
print("Leaving untranslated!")
translate = entry["CLEANED_JOBS"]
else:
translate = modelNMT.translate(chunk, target_lang="en")
entry["TRANSLATED_JOBS"] += translate
entry["TOKENIZED_JOBS"] = sentenceSplitter(entry["TRANSLATED_JOBS"])
if outputFile:
with open(outputFile, "w") as f:
json.dump(alljobs[:maxWrite], f)
return alljobs
def load_model(model_name='m2m_100_418M'):
"""
EasyNMT model to load
:param model_name: name of the model to load - List of supported model visit: https://github.com/UKPLab/EasyNMT#available-models
:return Machine translation model
"""
return EasyNMT(model_name)
class SentenceSimilarity_translationBased(SentenceSimilarity_abstract):
def __init__(self):
self.vectorizer = Vectorizer()
self.translationModel = EasyNMT('opus-mt')
self.targetLanguage = "en"
# this function computes the similarity between two sentences, the more similar the two snetends are the lower the
# computed score is
def compute_SentenceToSentence_similarity(self, sentenceA, sentenceB):
sourceLanguageA = self.translationModel.language_detection(sentenceA)
translationsA = self.translationModel.translate(
[sentenceA],
source_lang=sourceLanguageA,
target_lang=self.targetLanguage)
sourceLanguageB = self.translationModel.language_detection(sentenceB)
translationsB = self.translationModel.translate(
[sentenceB],
source_lang=sourceLanguageB,
target_lang=self.targetLanguage)
sentences = [translationsA[0], translationsB[0]]
self.vectorizer.bert(sentences)
vectors = self.vectorizer.vectors
embeddingOf_sentenceA = vectors[0]
embeddingOf_sentenceB = vectors[1]
print("\nsentenceA \"" + sentenceA + "\" --- sourceLanguageA=" +
sourceLanguageA + " --- translation = " + translationsA[0])
print("sentenceB \"" + sentenceB + "\" --- sourceLanguageB=" +
sourceLanguageB + " --- translation = " + translationsB[0])
distance = spatial.distance.cosine(embeddingOf_sentenceA,
embeddingOf_sentenceB)
return distance
def translateParagraph(chunk, method, fallback=True):
if method == "google":
from googletrans import Translator
modelGoogle = Translator()
if method == "easynmt" or fallback:
from easynmt import EasyNMT
modelNMT = EasyNMT('opus-mt')
if method == "google":
try:
translate = modelGoogle.translate(chunk, dest="en")
translate = translate.text
except (TypeError, AttributeError, IndexError, ReadTimeout) as e:
print("Failed to translate entry")
print("Error was:", e)
if fallback:
print("Trying instead with EasyNMT")
translate = modelNMT.translate(chunk, target_lang="en")
else:
print("Leaving untranslated!")
translate = chunk
else:
translate = modelNMT.translate(chunk, target_lang="en")
return translate
First documented in the 13th century and at the crossing of two important historic trade routes,[11] Berlin became the capital of the Margraviate of Brandenburg (1417–1701), the Kingdom of Prussia (1701–1918), the German Empire (1871–1918), the Weimar Republic (1919–1933), and the Third Reich (1933–1945).[12] Berlin in the 1920s was the third-largest municipality in the world.[13] After World War II and its subsequent occupation by the victorious countries, the city was divided; West Berlin became a de facto West German exclave, surrounded by the Berlin Wall (1961–1989) and East German territory.[14] East Berlin was declared capital of East Germany, while Bonn became the West German capital. Following German reunification in 1990, Berlin once again became the capital of all of Germany.
Berlin is a world city of culture, politics, media and science.[15][16][17][18] Its economy is based on high-tech firms and the service sector, encompassing a diverse range of creative industries, research facilities, media corporations and convention venues.[19][20] Berlin serves as a continental hub for air and rail traffic and has a highly complex public transportation network. The metropolis is a popular tourist destination.[21] Significant industries also include IT, pharmaceuticals, biomedical engineering, clean tech, biotechnology, construction and electronics.""",
"""This is the second document. With some test sentences.
This document contains also a list, which we want to translate.
This is a list:
- Berlin is a city in Germany
- Paris is the capital of France
- London is a large city in England
This is my final sentence.""",
"""Mon document final est rédigé en français. Nous voulons donc vérifier si l'étape de détection de la langue fonctionne."""
]
from easynmt import EasyNMT
target_lang = 'de' # We want to translate the sentences to German (de)
model = EasyNMT('opus-mt')
translations = model.translate(documents,
target_lang=target_lang,
batch_size=8,
beam_size=3)
for doc in translations:
print(doc)
print("\n======\n")
import csv
import sys
import time
import logging
if __name__ == '__main__':
logging.basicConfig(
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
level=logging.INFO,
stream=sys.stdout,
)
model = EasyNMT(sys.argv[1])
nli_dataset_path = 'AllNLI.tsv.gz'
sentences = set()
snli_max_sentences = 2000
mnli_max_sentences = 2000
snli = 0
mnli = 0
# Download datasets if needed
if not os.path.exists(nli_dataset_path):
util.http_get('https://sbert.net/datasets/AllNLI.tsv.gz', nli_dataset_path)
with gzip.open(nli_dataset_path, 'rt', encoding='utf8') as fIn:
reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
IS_BACKEND = os.getenv('ROLE', 'FRONT') == 'BACKEND'
BACKEND_URL = os.getenv('BACKEND_URL', 'http://*****:*****@app.get("/translate")
async def translate(target_lang: str,
text: List[str] = Query([]),
source_lang: Optional[str] = '',
beam_size: Optional[int] = 5,
perform_sentence_splitting: Optional[bool] = True):
"""
Translation
:param text: Text that should be translated
:param target_lang: Target language
:param source_lang: Language of text (optional)
:param beam_size: Beam size
:param perform_sentence_splitting: Split longer documents into individual sentences for translation
import shutil
import os
import re
from werkzeug.utils import secure_filename
from flask import Flask, flash, request, redirect, send_file, render_template, url_for
from nltk.tokenize import word_tokenize
from easynmt import EasyNMT
import nltk
import pprint
import secrets
import pyconll as pc
import torch
import gc
# model = EasyNMT('m2m_100_1.2B')
model = EasyNMT('opus-mt')
nltk.download('punkt')
torch.cuda.init()
"""
sentences = ['Dies ist ein Satz in Deutsch.', # This is a German sentence
'这是一个中文句子', # This is a chinese sentence
'Esta es una oración en español.'] # This is a spanish sentence
print(model.translate(sentences, target_lang='en', batch_size=1))
"""
def to_conllu(sentence):
lines = sentence.split('\n')
ok = ""
import csv
import sys
import time
import logging
if __name__ == '__main__':
logging.basicConfig(
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
level=logging.INFO,
stream=sys.stdout,
)
model = EasyNMT(sys.argv[1])
nli_dataset_path = 'AllNLI.tsv.gz'
sentences = set()
snli_max_sentences = 2000
mnli_max_sentences = 2000
snli = 0
mnli = 0
# Download datasets if needed
if not os.path.exists(nli_dataset_path):
util.http_get('https://sbert.net/datasets/AllNLI.tsv.gz', nli_dataset_path)
with gzip.open(nli_dataset_path, 'rt', encoding='utf8') as fIn:
reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
def __init__(self):
self.vectorizer = Vectorizer()
self.translationModel = EasyNMT('opus-mt')
self.targetLanguage = "en"
"""
This example tests the translation with all available models.
"""
from easynmt import EasyNMT
available_models = ['opus-mt', 'mbart50_m2m', 'm2m_100_418M', 'm2m_100_1.2B']
for model_name in available_models:
print("\n\nLoad model:", model_name)
model = EasyNMT(model_name)
sentences = [
'In dieser Liste definieren wir mehrere Sätze.',
'Jeder dieser Sätze wird dann in die Zielsprache übersetzt.',
'Puede especificar en esta lista la oración en varios idiomas.',
'El sistema detectará automáticamente el idioma y utilizará el modelo correcto.'
]
translations = model.translate(sentences, target_lang='en')
print("Translations:")
for sent, trans in zip(sentences, translations):
print(sent)
print("=>", trans, "\n")
tar_filepath)
with tarfile.open(tar_filepath, "r:gz") as tar:
tar.extractall(path=data_folder)
with open(queries_filepath, 'r', encoding='utf8') as fIn:
for line in fIn:
qid, query = line.strip().split("\t")
if qid in train_queries:
train_queries[qid] = query.strip()
qids = [qid for qid in train_queries if train_queries[qid] is not None]
queries = [train_queries[qid] for qid in qids]
#Define our translation model
translation_model = EasyNMT('opus-mt')
print("Start translation of {} queries.".format(len(queries)))
print("This can take a while. But you can stop this script at any point")
with open(output_filename,
'a' if os.path.exists(output_filename) else 'w',
encoding='utf8') as fOut:
for qid, query, translated_query in zip(
qids, queries,
translation_model.translate_stream(
queries,
source_lang='en',
target_lang=target_lang,
beam_size=2,
perform_sentence_splitting=False,
"""
This script measures the translation speed.
Usage:
python translation_speed.py model_name
"""
import os
from easynmt import util, EasyNMT
import gzip
import csv
import sys
import time
model = EasyNMT(sys.argv[1])
nli_dataset_path = 'AllNLI.tsv.gz'
sentences = set()
snli_max_sentences = 1000
mnli_max_sentences = 1000
snli = 0
mnli = 0
#Download datasets if needed
if not os.path.exists(nli_dataset_path):
util.http_get('https://sbert.net/datasets/AllNLI.tsv.gz', nli_dataset_path)
with gzip.open(nli_dataset_path, 'rt', encoding='utf8') as fIn:
reader = csv.DictReader(fIn, delimiter='\t', quoting=csv.QUOTE_NONE)
for row in reader:
"""
This example shows how EasyNMT to stream translations.
Streaming translations can be useful when you want to translate a large set of sentences / documents.
The method chunks the data, translates it, and returns the results.
This can be useful if you want to write it e.g. to a file.
"""
from easynmt import EasyNMT
#First, we create a large set of sentences:
sentences = ['This is sentence ' + str(i) for i in range(10000)]
target_lang = 'de' # We want to translate the sentences to German (de)
model = EasyNMT('opus-mt')
#The method model.translate_stream chunks the data into sets of size chunk_size
#It then translate these documents/sentences and yields the result
for translation in model.translate_stream(sentences,
show_progress_bar=False,
chunk_size=16,
target_lang=target_lang):
print(translation)