class Translator(object):
def __init__(self):
self.cache = {}
self.handler = YandexTranslate(os.environ['YA_TRANSLATE_KEY'])
self.dict = {}
self.loaddict()
def loaddict(self):
try:
dictfile = open(local_storage,'rt')
except:
open(local_storage,'wt')
self.loaddict()
return
for line in dictfile:
try:
word, translated_word = map(lambda x: x.strip('\n'),line.split('<===>'))
self.dict.update({word:translated_word})
except:
continue
def savedict(self):
with open(local_storage,'wt') as dictfile:
for word,translated_word in self.dict.items():
dictfile.write("{}<===>{}\n".format(word,translated_word))
def add_translated_word(self,word,translated_word):
self.dict[word] = translated_word
self.savedict()
def translate(self,text):
if text not in self.cache.keys():
print ("Query local dict ...")
if text not in self.dict.keys():
print ("Query API...")
translated_text = self.handler.translate(text,'ru-en')
if translated_text['code'] != 200:
return
translated_word = translated_text['text'][0]
if len(text.split(' ')) > 1:
return translated_word
self.add_translated_word(text, translated_word)
self.savedict()
self.cache.update({text:self.dict[text]})
else:
self.cache.update({text:self.dict[text]})
else:
print ("From cache...")
return self.cache[text]
def translate_russian_to_ukrainian(self,text):
translated_text = self.handler.translate(text, 'ru-uk')['text'][0]
return translated_text
def translate_english_to_ukrainian(self,text):
translated_text = self.handler.translate(text, 'en-uk')['text'][0]
return translated_text
def translate_corpus(api_key, translation_pair,
path_in, path_out, resume_from=0,
max_sent=float('inf')):
"""
Translate a corpus using Yandex API
api_key: API key
translation_pair: FROM_LANGUAGE-TO_LANGUAGE, e.g.: "en-es"
path_in: corpus to be translated
path_out: path where the translation will be saved
resume_from: number of the line on the input corpus from which to start translating;
this allows for translations to be resumed in case of failure;
NOTE: if this is set the path_out will be open to append
"""
translator = YandexTranslate(api_key)
fails = []
if resume_from > 0:
mode = "a"
else:
mode = "w"
print "open in mode ", mode
with open(path_out, mode) as fod:
with open(path_in) as fid:
if resume_from > 0:
print "skipping the first %d lines" % resume_from
#skip the first lines
for _ in xrange(resume_from): next(fid)
for i, l in enumerate(fid):
if i > max_sent:
break
elif not i%100:
sys.stdout.write("\r> %d" % i)
sys.stdout.flush()
#request translation
try:
tr = translator.translate(l,translation_pair)
#grab the tex
txt = tr['text'][0].encode("utf-8")
except YandexTranslateException as e:
#check response codes
if e.message == 'ERR_KEY_INVALID':
print "\n[ABORTED: Invalid API key]\n"
break
elif e.message == 'ERR_KEY_BLOCKED':
print "\n[ABORTED: Blocked API key]\n"
break
elif e.message == 'ERR_DAILY_CHAR_LIMIT_EXCEEDED':
print "\n[ABORTED: Exceeded daily limit]\n"
break
elif e.message == 'ERR_LANG_NOT_SUPPORTED':
print "\n[ABORTED: Translation direction not supported]\n"
break
elif e.message == 'ERR_UNPROCESSABLE_TEXT' or tr['code'] == 'ERR_TEXT_TOO_LONG':
#413 - exceeded maximum text size
#422 - text cannot be translated
txt = "FAIL: ", e.message
fails.append(l)
fod.write(txt)
print "\ntranslated corpus @ %s " % path_out
print "fails", fails
def translate(text,source,dest):
# print("\n\n {0} {1} {2} \n\n".format(text,source,dest))
translate = YandexTranslate('trnsl.1.1.20190515T114153Z.33301c8ad79f95bd.69a1158dfac270a48213caaf9b6102a3115b8fe1')
if source =='Eng':
lang_source='en'
if source =='Ru':
lang_source='ru'
if dest =='Eng':
lang_dest='en'
if dest =='Ru':
lang_dest='ru'
if source == 'Fra':
lang_source='fr'
if dest =='Fra':
lang_dest='fr'
try:
check = translate.detect(text)
except:
print("Check Exception on text: {}".format(text))
return text
# print("\n\n check: {0} \n\n".format(check))
if not check.find(lang_dest):
empty = lang_dest + '-' + lang_source
#print("\n\n empty: {0} \n\n".format(empty))
rez=translate.translate(text, empty)
s = rez['text'][0].decode('utf8')
# print(u"\n\n rez: {0} \n\n".format(s))
return s
return text
def translate_new_words(self):
if not os.path.isfile(self.yandex_translate_key_file):
print('put your api ket in {}\n'.format(
self.yandex_translate_key_file))
sys.exit(1)
with open(self.yandex_translate_key_file) as f:
api_key = f.read()
words_translation = {}
print('translate new words')
try:
YT = YandexTranslate(api_key)
for w in sorted(self.new_words):
words_translation[w] = ', '.join(
YT.translate(w, self.args.translate)['text'])
except YandexTranslateException as ex:
print(ex)
else:
tr_text = ''
for w, tr in words_translation.items():
tr_text += '{} = {}\n'.format(w, tr)
tr_file = self.new_words_translation_file.format(
self.args.translate.replace('-', '_'))
with open(tr_file, 'wt', encoding='utf-8') as f:
f.write(tr_text)
async def callback(data):
match = reg.match(data['body'])
convo = data['convo']
if match and (convo == from_convo or (from_convo is None)):
query = (match.group(1) or '').strip()
try:
if pass_data:
res = await func(data, query)
else:
res = await func(query)
except Exception as e:
import traceback
traceback.print_exc()
lang = data.get('country', 'US').split('-')[0].lower()
if lang == 'us':
lang = 'en'
if lang not in error_cache:
translator = YandexTranslate(config.yandex_translate_key)
error_cache[lang] = translator.translate(config.error_message, 'en-{}'.format(lang))['text'][0]
with open('errors.json', 'w') as f:
json.dump(error_cache, f)
res = error_cache[lang]
if not res:
return
elif type(res) in (str,):
body, file = res, None
else:
body, file = res
if not file and post_avatar:
file = os.path.join(config.avatar_folder, random.choice(os.listdir(config.avatar_folder)))
body = u'>>{}\n{}'.format(data['count'], body)
await post(body, config.bot_name, to_convo if to_convo else data['convo'], config.bot_trip, file)
return match
def test_blocked_key(self):
translate = YandexTranslate(
"trnsl.1.1.20130723T112255Z.cfcd2b1ebae9f"
"ff1.86f3d1de3621e69b8c432fcdd6803bb87ef0e963")
with self.assertRaises(YandexTranslateException,
msg="ERR_KEY_BLOCKED"):
languages = translate.detect("Hello!")
def __init__(self, file1, file2, lang1, lang2):
self.es = Elasticsearch()
self.translate = YandexTranslate(
'trnsl.1.1.20161120T092429Z.26536300f1fab524.bb5fcc81494303125cb46f52681dfdcf652477e0'
)
self.file1 = file1
self.file2 = file2
self.file1_lang = lang1
self.file2_lang = lang2
self.paragraphLinks = []
self.withoutWhiteChars = re.compile(r'[\n\t \r]+')
self.nonAlphaNumeric = re.compile(r'^\W+$')
self.endOfSetnence = re.compile(r'([\.\?\!:])')
self.sentences_pair = []
self.getNameWords = re.compile(r'([A-ZА-Я]\w+)')
self.onlyLetters = re.compile(r'[^a-zA-Zа-яА-Я ]+')
self.MIN_RATE = 75
self.synced_sentences = []
self.nonsynced_sentences = {'first': [], 'second': [], 'keywords': []}
def get_lyrics_translation():
translate = YandexTranslate('')
con = mdb.connect(ADDRESS, USERNAME, PASSWORD, SCHEMA)
con.set_character_set('utf8')
with con:
cur = con.cursor()
cur.execute('SET CHARACTER SET utf8')
cur.execute('SET character_set_connection=utf8')
dict_of_lyrics = dict()
try:
cur.execute("SELECT song_id, text, src_lang FROM lyrics WHERE text != '' AND translation IS NULL")
if cur.rowcount == 0:
exit(0)
for i in range(cur.rowcount):
row = cur.fetchone()
song_id = row[0]
text = row[1]
src_language = row[2]
if src_language == '':
src_language = 'en'
message = translate.translate(text, src_language + '-he')
if message.get('code') == 200:
translation = message.get('text')
dict_of_lyrics.update({song_id: translation})
else:
print message.get('code')
except Exception as e:
print "error occurred: "
print e.message
print "size of dict: " + str(len(dict_of_lyrics))
insert_into_lyrics_translation(cur, dict_of_lyrics)
class YandexTranslateTest(unittest.TestCase):
def setUp(self):
self.translate = YandexTranslate(YANDEX_API_KEY)
def test_langs(self):
languages = self.translate.langs
self.assertEqual(languages, set(
[
u"el", u"en", u"ca", u"it",
u"hy", u"cs", u"et", u"az",
u"es", u"ru", u"nl", u"pt",
u"no", u"tr", u"lv", u"lt",
u"ro", u"pl", u"be", u"fr",
u"bg", u"hr", u"de", u"da",
u"fi", u"hu", u"sr", u"sq",
u"sv", u"mk", u"sk", u"uk",
u"sl"
]
))
def test_lang_detection(self):
language = self.translate.detect("Je m'appele Malika")
self.assertEqual(language, 'fr')
def test_translation(self):
result = self.translate.translate(u"Hello", 'en-es')
self.assertEqual(result["text"][0], u"Hola")
self.assertEqual(result["code"], 200)
def test_opposite_translation(self):
result = self.translate.translate(u"Hola", 'es-en')
self.assertEqual(result["text"][0], u"Hello")
self.assertEqual(result["code"], 200)
def translate_corpus(api_key,
translation_pair,
path_in,
path_out,
max_sent=float('inf')):
translator = YandexTranslate(api_key)
fails = []
with open(path_out, "w") as fod:
with open(path_in) as fid:
for i, l in enumerate(fid):
if i > max_sent:
break
elif not i % 1000:
sys.stdout.write("\r> %d" % i)
sys.stdout.flush()
try:
tr = translator.translate(
l, translation_pair)['text'][0].strip("\n")
except:
fails.append(l)
fod.write(tr.encode("utf-8") + "\n")
print "\ntranslated corpus @ %s " % path_out
print "fails"
print fails
def __init__(self, parent=None, api_key=None):
self.parent = parent
self.api_key = api_key
if not YandexTranslate:
raise GnrException(
'Missing YandexTranslate. hint: pip install yandex.translate')
self.translator = YandexTranslate(self.api_key)
def __init__(self):
self.__api_key = 'trnsl.1.1.20170419T213309Z.70ddc54f4fe2d025.fa23b8e218345631f11e83f28520ea417e7e44f9'
self.__ya_translator = YandexTranslate(self.__api_key)
self.__con = sqlite3.connect("translator/dictionary.db")
self.__cur = self.__con.cursor()
def translate_yandex(text, src="auto", dst="en"):
if src != "auto":
lang = "{src}-{dst}".format(src=src, dst=dst)
else:
lang = dst
langdex = YandexTranslate(yandex_key)
result = langdex.translate(text, lang)
assert result['code'] == 200
return result['text'][0]
def translate(text):
trans = YandexTranslate('trnsl.1.1.20190421T135944Z.7292abf1150a9315.88e1b4e89ec715ff8de021cc3eaf0ef1cae0259b')
if not russian(text):
resp = ''.join(trans.translate(text, 'en-ru')['text'])
else:
resp = ''.join(trans.translate(text, 'ru-en')['text'])
if resp:
return resp
return False
def translate(text: str) -> str:
"""
:text: --> The text in arabic
:return: --> return a translation of :text: in english
"""
DEBUG = False
if text == "":
return ""
self = FatabyyanoFactCheckingSiteExtractor
yandexAPI = self.YANDEX_API_KEY
yandex = YandexTranslate(yandexAPI)
responses = []
try:
response = yandex.translate(text, 'ar-en')
if DEBUG:
print("response : " + response, end='\n\n')
responses = [response]
text_too_long = False
except YandexTranslateException as e:
if e.args == 'ERR_TEXT_TOO_LONG' or 'ERR_TEXT_TOO_LONG' in e.args:
text_too_long = True
else:
print(text, end='\n\n')
print("Erreur API Yandex\nCode d'erreur : " + str(e.args))
sys.exit(1)
text_list = [text]
while text_too_long:
if DEBUG:
print("cutting : " + str(text_list), end='\n\n')
text_too_long = False
try:
text_list = self.cut_str(text_list)
except ValueError:
print("Erreur ")
sys.exit(1)
responses = []
for t in text_list:
try:
responses.append(yandex.translate(t, 'ar-en'))
except YandexTranslateException:
text_too_long = True
continue
text_list = []
for r in responses:
if int(r['code'] != 200):
print("Erreur lors de la traduction\nCode de l'erreur : " +
r['code'])
sys.exit(1)
else:
text_list.append(r['text'][0])
return self.concat_str(text_list)
def __init__(self):
print('Iniciando Classifier')
self.vocabulary = []
self.modules = {}
self.input_vector = []
self.output_vector = []
self.config = Config()
self.translator = YandexTranslate(self.config.get_token_yandex())
self.dao = Mongo_DAO('localhost', 27017, 'classifier')
async def ytranslate(self, ctx, language: str, *, text: str):
"""Translate text via yandex
Language may be just "ru" (target language to translate)
or "en-ru" (original text's language - target language)"""
text = chat.escape(text, formatting=True)
apikeys = await self.bot.db.api_tokens.get_raw("yandex", default={"translate": None})
try:
translate = YandexTranslate(apikeys["translate"])
response = translate.translate(text, language)
except YandexTranslateException as e:
if str(e) == "ERR_LANG_NOT_SUPPORTED":
await ctx.send(chat.error("An error has been occurred: Language {} is not supported"
.format(chat.inline(language))))
elif str(e) == "ERR_TEXT_TOO_LONG":
# Discord will return BAD REQUEST (400) sooner than this happen, but whatever...
await ctx.send(chat.error("An error has been occurred: Text that you provided is too big to "
"translate"))
elif str(e) == "ERR_KEY_INVALID":
await ctx.send(chat.error("This command requires Yandex.Translate API key\n"
"You can set it via {}ytapikey".format(ctx.prefix)))
elif str(e) == "ERR_KEY_BLOCKED":
await ctx.send(chat.error("API key is blocked. You need to get new api key or unlock current."))
elif str(e) == "ERR_DAILY_REQ_LIMIT_EXCEEDED":
await ctx.send(chat.error("Daily requests limit reached. Try again later."))
elif str(e) == "ERR_DAILY_CHAR_LIMIT_EXCEEDED":
await ctx.send(chat.error("Daily char limit is exceeded. Try again later."))
elif str(e) == "ERR_UNPROCESSABLE_TEXT":
await ctx.send(chat.error("An error has been occurred: Text is unprocessable by translation server"))
elif str(e) == "ERR_SERVICE_NOT_AVAIBLE":
await ctx.send(chat.error("An error has been occurred: Service Unavailable. Try again later"))
else:
await ctx.send(chat.error("An error has been occurred: {}".format(e)))
return
input_lang = None
output_lang = None
if len(language) == 2:
try:
input_lang = translate.detect(text=text)
except YandexTranslateException as e:
if str(e) == "ERR_LANG_NOT_SUPPORTED":
await ctx.send(chat.error("This language is not supported"))
else:
await ctx.send(chat.error("Unable to detect language: {}".format(e)))
return
output_lang = language
elif len(language) == 5:
input_lang = language[:2]
output_lang = language[3:]
if response["code"] == 200:
await ctx.send("**[{}] Input:** {}".format(input_lang.upper(), chat.box(text)))
await ctx.send("**[{}] Translation:** {}".format(output_lang.upper(), chat.box(response["text"][0])))
else:
# According to yandex.translate source code this cannot happen too, but whatever...
await ctx.send("An error has been occurred. Translation server returned code {}"
.format(chat.inline(response["code"])))
def translate(text):
"""
:text: --> The text in arabic
:return: --> return a translation of :text: in english
"""
translate = YandexTranslate(
'trnsl.1.1.20200311T210815Z.796b747ff14857c9.2e7b856527d2689a26379ff56769f5b3c087f55b'
)
return translate.translate(text, 'ar-en')['text'][0]
def translate(text):
'''Translates the message from one language to another'''
## Mr. Scott's secret key for Yandex API
translate = YandexTranslate('') #Yandex key
finalText, languageTo = getLanguage(text)
## detect the language of the text
language = translate.detect(finalText)
print('Translating from',language,'to',languageTo)
## translate to english
translated = 'Translate:', translate.translate(finalText, language+'-'+languageTo)
return (translated[1]['text'][0])
def __yandex_translate__(self, text, lang_to, lang_from, formatt):
translator = YandexTranslate(self.yandex_api_key)
try:
if not lang_from:
lang_from = self.detect_lang(text)
lang = "{}-{}".format(lang_from, lang_to)
translated = translator.translate(text, lang, format=formatt)
return translated['text'][0]
except YandexTranslateException:
return text
class plugin_thread(QtCore.QThread):
finished = QtCore.pyqtSignal(object, object)
def __init__(self, options, source, callback, parent=None):
QtCore.QThread.__init__(self, parent)
self.options = options
self.source = source
#Insert your api keys here
self.mstranslate_key = ''
self.yandex_key = ''
self.options = options
self.finished.connect(callback)
self.aborted = False
def run(self):
response = ''
try:
if self.source == "gtranslateweb":
user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'
host = 'https://translate.google.com/m'
headers = {
'User-Agent': user_agent,
}
params = {
'sl': self.options['source_language'],
'tl': self.options['target_language'],
'q': self.options['source_text']
}
html = requests.get(host, params=params, headers=headers)
soup = BeautifulSoup(html.text, "lxml")
for div in soup.find_all('div'):
if div.has_attr('class'):
if div['class'][0] == 't0':
response = div.string
elif self.source == "mstranslator":
self.translator = Translator(self.mstranslate_key)
response = self.translator.translate(
self.options['source_text'],
lang_from=self.options['source_language'],
lang_to=self.options['target_language'])
elif self.source == "yandex":
self.translator = YandexTranslate(self.yandex_key)
response = self.translator.translate(
self.options['source_text'],
self.options['source_language'] + "-" +
self.options['target_language'])['text'][0]
except Exception as e:
response = '[Error] ' + str(e)
if not self.aborted:
self.finished.emit(self.source, response)
class ya_translate(object):
def __init__(self):
self.translate = YandexTranslate('trnsl.1.1.20170417T120755Z.ace07b3e54a7011b.23ef0f7b86f04dff506c4b86bdb4eb547901af1b')
def get_english_words(self,eng_word):
tmp_translate_word=self.translate.translate(eng_word,"ru")
translate_word=tmp_translate_word['text'][0]
if self.translate.detect(translate_word) in 'ru':
return translate_word
else:
return
def translate_language(sourcetext, translationfrom, translationTo):
translate = YandexTranslate(
'trnsl.1.1.20171108T161528Z.11fd5cf559085f7e.68f45fa90a6197a56fc8353410bec46f7386e419'
)
print('Languages:', translate.langs)
print('Translate directions:', translate.directions)
print('Detect language:', translate.detect(sourcetext))
translated_text = translate.translate(
sourcetext, translationfrom + '-' + translationTo)
arr = translated_text['text']
return arr
print(arr)
def translate(text, source_language, dest_language):
# Yandex doesn't require requests (implied?)
# Yandex doesn't require wrapping key in auth
# Yandex returns a dict of lists, the translation is found in ['text'][0]
if 'TRANSLATOR_KEY' not in current_app.config or \
not current_app.config['TRANSLATOR_KEY']:
return _('Error: the translation service is not configured.')
t = YandexTranslate(current_app.config['TRANSLATOR_KEY'])
r = t.translate(text, source_language + '-' + dest_language)
if r['code'] != 200:
return "Error: the translation service failed. %r" % ( r['code'] )
return r['text'][0]
def translate(text, source_language, dest_language):
# Yandex doesn't require requests (implied?)
# Yandex doesn't require wrapping key in auth
# Yandex returns a dict of lists, the translation is found in ['text'][0]
if ("YANDEX_TRANSLATOR_KEY" not in current_app.config
or not current_app.config["YANDEX_TRANSLATOR_KEY"]):
return _("Error: the translation service is not configured.")
t = YandexTranslate(current_app.config["YANDEX_TRANSLATOR_KEY"])
r = t.translate(text, source_language + "-" + dest_language)
if r["code"] != 200:
return _("Error: the translation service failed. %r" % (r["code"]))
return r["text"][0]
def yandex_translate():
translated_text = ""
if len(request.form['text']) > 0:
translator = YandexTranslate(
"trnsl.1.1.20170124T023923Z.db368f9b9c7a7f06.26bd8be0cf53946d203c786ae83d24c73ea5954d"
)
translated_text = translator.translate(request.form['text'],
request.form['dest'])['text'][0]
return jsonify({
'translation': translated_text,
'original_text': request.form['text']
})
def to_english(message):
# Detect language
english_set = set(w.lower()
for w in nltk.corpus.words.words())
text_set = set( w.lower() for w in
re.sub( '\W', ' ', message).split()
if w[0].isalpha() )
unusual = text_set.difference(english_set)
if len(unusual)*3 > len(text_set):
print("Translating...")
translate = YandexTranslate('trnsl.1.1.20170406T225123Z.cb036d7ebd624db9.8835c34234d938773f407ac77a601e10fc1e1771')
message = translate.translate(message, 'en')['text'][0]
return message
def mainTestInteractive(self, sess):
""" Try predicting the sentences that the user will enter in the console
Args:
sess: The current running session
"""
# TODO: If verbose mode, also show similar sentences from the training set with the same words (include in mainTest also)
# TODO: Also show the top 10 most likely predictions for each predicted output (when verbose mode)
# TODO: Log the questions asked for latter re-use (merge with test/samples.txt)
#print('Testing: Launch interactive mode:')
translate = YandexTranslate('trnsl.1.1.20180403T155201Z.58ec00a2b669a2cc.8e269003229f8e698616dad576839b9cf5335af8')
#print('')
#print('Welcome to the interactive mode, here you can ask to Deep Q&A the sentence you want. Don\'t have high '
# 'expectation. Type \'exit\' or just press ENTER to quit the program. Have fun.')
while True:
question = input(self.SENTENCES_PREFIX[0])
traduccion = translate.translate (question, 'en')
texto = str (traduccion['text'])
texto = texto.replace ("['", "")
texto = texto.replace ("[", "")
texto = texto.replace ("']", "")
texto = texto.replace ("]", "")
question=texto
if question == '' or question == 'exit':
break
questionSeq = [] # Will be contain the question as seen by the encoder
answer = self.singlePredict(question, questionSeq)
if not answer:
print('Oye baby, tu pregunta esta muy compleja, ¿podrías intentar algo más simple?')
continue # Back to the beginning, try again
respuesta=self.textData.sequence2str(answer, clean=True)
traduccion=translate.translate(respuesta, 'es')
texto=str(traduccion['text'])
texto=texto.replace("['", "")
texto=texto.replace("']", "")
print('{}{}'.format(self.SENTENCES_PREFIX[1],texto ))
if self.args.verbose:
print(self.textData.batchSeq2str(questionSeq, clean=True, reverse=True))
print(self.textData.sequence2str(answer))
print()
def __init__(self, yandex_key, src_lang=None, dest_lang=None, src_po_file=None):
"""
Конструктор
:param yandex_key: ключ для доступа к API яндекс-переводчика
:param src_lang: исходный язык
:param dest_lang: язык назначения
:param src_po_file: исходный po-файл
:return:
"""
self.yandex_key = yandex_key
self.src_lang = src_lang
self.dest_lang = dest_lang
self.yandex_translate = YandexTranslate(self.yandex_key)
if src_po_file is not None:
self.open_po_fle(src_po_file)
def __goodKeyYandex(self, key):
success = True
trans = YandexTranslate(key)
try:
trans.translate("hola", "en")["text"][0]
except:
success = False
return success
def main():
update_id = lastUpdate(getUpdates())['update_id']
while True:
if update_id == lastUpdate(getUpdates())['update_id']:
from yandex_translate import YandexTranslate
translate = YandexTranslate(
'trnsl.1.1.20181221T225719Z.e3a1a909c9a77883.918af8cc684b8477f891022c74cccc76ef2709b5'
)
c = (lastUpdate(getUpdates())['message']['text'])
w = translate.translate(c, 'ru-en')
trans = w['text']
sendMessage(getChatId(lastUpdate(getUpdates())), trans)
update_id += 1
sleep(1)
def get_translated_text_from_api(api, original_text, destination_language):
if api is 'google_translate':
translator = Translator()
return translator.translate(original_text, dest=destination_language).text
else:
translator = YandexTranslate(app.config['API_KEY'])
return translator.translate(original_text, destination_language)['text'][0]
def main(self):
log.info('run serve subcommand')
token = Core.config.telegram.api_token()
if type(token) is not str:
log.critical(
'Can not find "telegram:api_token" in config. Exiting.')
exit(1)
self._bot = Bot(token=Core.config.telegram.api_token())
dp = Dispatcher(self._bot)
self._processor = VolleyProcessor(
YandexTranslate(Core.config.yandex_translate.api_key()),
ChatScript(Core.config.chatscript.host(),
Core.config.chatscript.port(), 'botan'))
dp.register_message_handler(self.send_welcome,
commands=['start', 'help'])
dp.register_message_handler(self.everything)
start_polling(
dp,
skip_updates=True,
# on_startup=self.on_startup,
# on_shutdown=self.on_shutdown
)
class Main(GnrBaseService):
def __init__(self, parent=None,api_key=None):
self.parent = parent
self.api_key = api_key
if not YandexTranslate:
raise GnrException('Missing YandexTranslate. hint: pip install yandex.translate')
self.translator = YandexTranslate(self.api_key)
def translate(self,what=None, to_language=None,from_language=None,format=None):
if not what:
return
direction = [to_language] if not from_language else [from_language,to_language]
safedict = dict()
def cb(m):
safekey = '[NO_TR_%i]' %len(safedict)
safedict[safekey] = m.group(1)
return safekey
base_to_translate = SAFETRANSLATE.sub(cb,what)
print 'safedict',safedict
print 'base_to_translate',base_to_translate
result = self.translator.translate(base_to_translate, '-'.join(direction),format=format)
if result['code'] == 200:
txt = result['text'][0]
for k,v in safedict.items():
txt = txt.replace(k,'[tr-off]%s[tr-on]' %v)
return txt
else:
raise GnrException('Error in translation')
def write_down_result_tr(file1, file2, adj_root_tr, adj_root1, adj_root2, api_key):
"""
Запись в файл result_lines_selector.tsv строк в формате
" прилагательное существительное число вхождений" (отсортированно). С переводом.
"""
from yandex_translate import YandexTranslate
translate = YandexTranslate(api_key)
result, result_arr1, result_arr2 = common_tr(file1, file2)
f1 = codecs.open(file1, 'r', 'utf-8')
f2 = codecs.open(file2, 'r', 'utf-8')
dict1 = {line.split()[0]: line.split()[1] for line in f1} # сущ и частотность
dict2 = {line.split()[0]: line.split()[1] for line in f2} # сущ и частотность
f1.close()
f2.close()
trans = {}
for noun in result:
nounblob = translate.translate(noun, 'eng-ru')['text'][0].lower() # делаем перевод
nounblob = morph.parse(nounblob)[0].normal_form # делаем нормализацию
trans[noun] = nounblob
for noun1 in result_arr1:
nounblob1 = translate.translate(noun1, 'eng-ru')['text'][0].lower() # делаем перевод
nounblob1 = morph.parse(nounblob1)[0].normal_form # делаем перевод
trans[noun1] = nounblob1
for noun2 in result_arr2:
nounblob2 = translate.translate(noun2, 'eng-ru')['text'][0].lower() # делаем перевод
nounblob2 = morph.parse(nounblob2)[0].normal_form # делаем перевод
trans[noun2] = nounblob2
# Запись в файл result_lines_selector.tsv строк в формате " прилагательное существительное число вхождений" (отсортированно)
w = codecs.open('./english/results/' + adj_root_tr + '_result_0_translate_comparation.csv', 'w', 'utf-8')
w.write('noun' + ',' + str(adj_root1) + ',' + str(adj_root2) + ',' + 'translation' + '\r\n')
for noun in result:
nounblob = trans[noun]
w.write(noun + ',' + str(dict1[noun]) + ',' + str(dict2[noun]) + ',' + nounblob + '\r\n')
for noun in result_arr1:
nounblob = trans[noun]
w.write(noun + ',' + str(dict1[noun]) + ',' + "0" + ',' + nounblob + '\r\n')
for noun in result_arr2:
nounblob = trans[noun]
w.write(noun + ',' + "10" + ',' + str(dict2[noun]) + ',' + nounblob + '\r\n')
w.close()
def geocoder(request, lang=None):
if request.GET:
form = GeocoderForm(request.GET)
if form.is_valid():
q = form.cleaned_data['query']
if lang:
translator = YandexTranslate(settings.YANDEX_TRANSLATE_KEY)
result = translator.translate(q, lang)
q = result['text'][0] if result['code'] == 200 else q
response = JsonResponse(GeoObject.objects.filter(title__icontains=q))
else:
response = 'Not valid parameters'
else:
response = 'Not get request'
return HttpResponse(response)
class Yanslate:
def __init__( self, key ):
self.translate = YandexTranslate( key )
def detect( self, text ):
print('Detect language:', self.translate.detect( text ))
def list( self ):
print('Languages:', self.translate.langs)
def trans( self,text, lang ):
return ('Translate:', self.translate.translate(text, lang))
def __init__(self, key, service_api):
#creating object
self.apiservice = service_api
self.msg = "EMPTY"
self.msg_reply = "EMPTY"
if self.apiservice == 'google':
self.service = build('translate', 'v2',developerKey=key)
elif self.apiservice == 'yandex':
self.service = YandexTranslate(key)
def __init__(self):
self.engine = YandexTranslate(config.TRANSLATE_YANDEX_KEY)
BaseService.__init__(self)
def _yandex_connect(self, api_key): ''' connect to yandex server with personal api_key ''' self.translate = YandexTranslate(api_key)
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], "", ["extract-text", "generate-resource=", "display-html", "help"])
except getopt.GetoptError as err:
# print help information and exit:
print(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
(abc, b) = opts[0]
class myparser(HTMLParser):
def handle_data(self, data): # Override method of HTMLParser
if abc == '--extract-text':
data = data.lstrip(); # Strip all unwanted
data = data.rstrip(); # whitespace in HTML File.
if data != '':
hash_object = hashlib.md5(data.encode()) # Create an MD5 hash of the text segment.
f1 = open('example.properties', mode='a')
f1.write(hash_object.hexdigest() + "=" + data + "\n") # Write hash and data into file.
elif abc == '--display-html':
data = data.lstrip();
data = data.rstrip();
if data != '':
hash_object = hashlib.md5(data.encode())
rep = tran_dict[hash_object.hexdigest()] # Retrieve translated string corresponding to the hash of original data.
self.newdata = self.newdata.replace(data, rep) # Replace original text with translated text
for o, a in opts:
if o == "--extract-text":
data_dict = {}
f1 = open('example.properties', mode='w')
parser = myparser()
f = open('sample.html')
data = f.read()
parser.feed(data)
f1.close()
f.close()
elif o == "--generate-resource":
translate = YandexTranslate('trnsl.1.1.20160607T100954Z.0329b17b7e667944.3b9682b4988dfe500b80d3d23a867954d107ed6c')
lang = a
name = a + ".properties"
f = open('example.properties')
f1 = open('selected_language', 'w')
f2 = open(name, mode='w')
f1.write(name)
for line in f:
(key, val) = str(line).split('=');
f2 = open(name, mode='a');
res = translate.translate(val, a)
out = str(res['text'])
out = out.strip('[]') # Strip all unwanted
out = out.lstrip("'") # characters returned by
out = out.rstrip("'") # Yandex.Translate
out = out.strip("\\n")
f2.write(key + "=" + str(out) + "\n")
f.close()
f1.close()
f2.close()
elif o == "--display-html":
data_dict = {}
with open('selected_language') as f3:
name = f3.read()
with open('example.properties') as f4:
for line in f4:
(key, val) = line.split('=') # Create a dictionary from properties file.
data_dict[key] = val
tran_dict = {}
with open(name) as f5:
for line in f5:
(key, val) = line.split('=') # Create a dictionary from properties file.
tran_dict[key] = val
infile = open('sample.html', 'r');
olddata = infile.read()
newdata = olddata
parser = myparser();
parser.newdata = newdata;
parser.feed(olddata);
with open('sample.translated.html', 'w') as file:
file.write(parser.newdata)
infile.close()
elif o == "--help":
usage()
else:
assert False, "unhandled option"
def handle(self, *args, **options):
translate = YandexTranslate(settings.YANDEX_TRANSLATE_KEY)
objects = Page.objects.all()
for item in objects:
print('Translate:', translate.translate(item.text, 'ru-en'))
self.stdout.write('Finished translate')
class Yandex(BaseService):
def __init__(self, api_key):
self._api_key = api_key
self._service = YandexTranslate(self._api_key)
self._directions = self._directions()
def translate_cascade(self, initial_language,
cascade_steps, text):
""" 1. Check for the text if the service thinks it is the same language as the user has provided
2. Check if the services thinks steps are legit and there is no step that cannot be done
3. Translate cascadingly
:param initial_language: two letter string of the language user needs to start with
:param cascade_steps: user provided steps (usually excluding the initial language)
:param text: the text user wants to translate cascadingly
:return: a tuple of all translations and the final translation in the original language
"""
logging.debug(initial_language + " - " + text)
cascade_steps = self.steps_to_execute(initial_language, cascade_steps)
if not self.check_language(initial_language, text):
raise YandexTranslateException(501)
if not self.check_cascade_steps(initial_language, cascade_steps):
raise YandexTranslateException(501)
results = {}
for lang in cascade_steps[1:]:
try:
results[lang] = self._service.translate(text, lang).get('text', '')[0]
except YandexTranslateException:
return {}
text = results[lang]
logging.debug(lang + " - " + text)
result = results[initial_language]
return (results, result)
def get_language(self, text=""):
"""get the language detected by the service
:param text: the text user wants to translate cascadingly
:return: language detected
"""
return self._service.detect(text)
def check_language(self, initial_language, text):
"""check whether the user provided text is in the same langauge as the
initial langauge provided by the user
:param initial_language: two letter string of the language user needs to start with
:param text: the text user wants to translate cascadingly
:return: boolean whether a language is correct
"""
lang = self.get_language(text)
if lang == initial_language:
is_correct_language = True
else:
is_correct_language = False
return is_correct_language
def _directions(self):
"""Service's available translation directions
:return: list of the available translation directions (from-to)
"""
return self._service.directions
def is_translation_step_valid(self, from_lang, to_lang):
"""
If one translation step valid
:param from_lang: two letter string for lang
:param to_lang: two letter string for lang
:return: boolean if translation valid from_lang to to_lang
"""
lang_pair = from_lang + "-" + to_lang
logging.debug(lang_pair)
if lang_pair.strip() in self._directions:
valid = True
else:
valid = False
return valid
def check_cascade_steps(self, initial_language, cascade_steps):
"""check if steps provided by the user are allowed by the service
:param initial_language: two letter string of the language user needs to start with
:param cascade_steps: user provided steps (usually excluding the initial language)
:return: boolean of whether all the translation steps are doable
"""
cascade_steps = self.steps_to_execute(initial_language,
cascade_steps)
is_cascade_achievable = False
# checking a language with itself is not allowed
# the first item is the initial_language
for lang in cascade_steps[1:]:
if self.is_translation_step_valid(initial_language, lang):
is_cascade_achievable = True
initial_language = lang
else:
is_cascade_achievable = False
break
return is_cascade_achievable
def __init__( self, key ):
self.translate = YandexTranslate( key )
class translator(object):
def __init__(self, key, service_api):
#creating object
self.apiservice = service_api
self.msg = "EMPTY"
self.msg_reply = "EMPTY"
if self.apiservice == 'google':
self.service = build('translate', 'v2',developerKey=key)
elif self.apiservice == 'yandex':
self.service = YandexTranslate(key)
def svtoen(self, msg):
if self.apiservice == 'google':
self.msg_reply = self.service.translations().list(
source='sv',
target='en',
q=[msg]
).execute()
returnstr = self.msg_reply['translations'][0]['translatedText']
return returnstr
elif self.apiservice == 'yandex':
msg_reply = self.service.translate(msg, 'sv-en')
returnstr = msg_reply['text'][0]
return returnstr
def entosv(self, msg):
if self.apiservice == 'google':
self.msg_reply = self.service.translations().list(
source='en',
target='sv',
q=[msg]
).execute()
return_string = self.msg_reply['translations'][0]['translatedText']
return return_string
elif self.apiservice == 'yandex':
self.msg_reply = self.service.translate(msg, 'en-sv')
return_string = self.msg_reply['text'][0]
return return_string
def input_svtoen(self):
self.msg = raw_input("Översätt: ")
self.svtoen(self.msg)
def print_translated_message(self):
#print (self.msg_reply)
#for a in self.msg_reply:
print (self.msg_reply['translations'][0]['translatedText'])
#return_string = self.msg_reply['translations'][0]['translatedText']
#return return_string
#test = translator('api_key','yandex')
#testout = test.svtoen('hej')
#test.print_translated_message()
#test.entosv("who are you")
#print (testout)
"""
Test yandex
https://translate.yandex.com/developers/keys
"""
print "Testing Yandex"
from yandex_translate import YandexTranslate
translate = YandexTranslate('trnsl.1.1.20170808T200819Z.353052b379db306f.afbcedbaeec9d823cfac008f017161e8cb93791c')
input_text = "Hola, mundo!"
output_text = translate.translate('Hola, mundo!', 'es-en')['text'][0]
print "Translate:\n\t", output_text
def __init__(self):
self.cache = {}
self.handler = YandexTranslate(os.environ['YA_TRANSLATE_KEY'])
self.dict = {}
self.loaddict()
# coding=utf-8
import os
from Tkinter import *
from yandex_translate import YandexTranslate
import clipboard
# Copied string
Text = clipboard.paste()
# Yandex api
translate = YandexTranslate('Your Yandex Api Key')
Translated = ""
if Text is not None:
Translated = translate.translate(Text, 'tr')['text'][0]
message = """
%s
---
%s
""" % (Text, Translated)
# User interface
class Application(Frame):
def widgets(self):
self.LABEL = Label(self)
self.LABEL["text"] = message
self.LABEL["wraplength"] = 200
def __init__(self, api_key):
self._api_key = api_key
self._service = YandexTranslate(self._api_key)
self._directions = self._directions()
def setUp(self):
self.translate = YandexTranslate(
"trnsl.1.1.20130421T140201Z.323e508a" "33e9d84b.f1e0d9ca9bcd0a00b0ef71d82e6cf4158183d09e"
)
from yandex_translate import YandexTranslate
translate = YandexTranslate('trnsl.1.1.20160420T200137Z.602d58f64d6ac2cf.d8e7eca404d946f979b434e59caa856c178eb095')
"""
print('Languages: ', translate.langs)
print('Translate directions: ',translate.directions)
print('Detect language: ', translate.detect('hola'))
"""
text = 'sjdhb'
print('Translate: ', translate.translate(text, 'es-en'))
class YandexTranslateTest(unittest.TestCase):
def setUp(self):
self.translate = YandexTranslate(
"trnsl.1.1.20130421T140201Z.323e508a" "33e9d84b.f1e0d9ca9bcd0a00b0ef71d82e6cf4158183d09e"
)
def test_directions(self):
directions = self.translate.directions
self.assertGreater(len(directions), 1)
def test_langs(self):
languages = self.translate.langs
self.assertEqual(
languages,
set(
[
u"el",
u"en",
u"ca",
u"it",
u"hy",
u"cs",
u"et",
u"az",
u"es",
u"ru",
u"nl",
u"pt",
u"no",
u"tr",
u"lv",
u"lt",
u"ro",
u"pl",
u"be",
u"fr",
u"bg",
u"hr",
u"de",
u"da",
u"fi",
u"hu",
u"sr",
u"sq",
u"sv",
u"mk",
u"sk",
u"uk",
u"sl",
]
),
)
def test_blocked_key(self):
translate = YandexTranslate(
"trnsl.1.1.20130723T112255Z.cfcd2b1ebae9f" "ff1.86f3d1de3621e69b8c432fcdd6803bb87ef0e963"
)
with self.assertRaises(YandexTranslateException, msg="ERR_KEY_BLOCKED"):
languages = translate.detect("Hello!")
def test_detect_language(self):
language = self.translate.detect(text="Hello, world!")
self.assertEqual(language, "en")
def test_translate(self):
result = self.translate.translate(u"Hello!", "ru")
self.assertEqual(result["text"][0], u"Здравствуйте!")
self.assertEqual(result["code"], 200)
def test_translate_in_another_direction(self):
result = self.translate.translate(u"Здравствуйте", "en")
self.assertEqual(result["text"][0], u"Hello")
self.assertEqual(result["code"], 200)
def test_language_detection_error(self):
with self.assertRaises(YandexTranslateException, msg="ERR_LANG_NOT_SUPPORTED"):
self.translate.detect("なのです")
# Yandex.Translate tries to translate this as english-to-russian
# def test_translate_error(self):
# with self.assertRaises(YandexTranslateException,
# msg="ERR_LANG_NOT_SUPPORTED"):
# self.translate.translate("なのです", "ru")
def test_without_key(self):
with self.assertRaises(
YandexTranslateException,
msg="Please, provide key for " "Yandex.Translate API: " "https://translate.yandex.ru/apikeys",
):
translate = YandexTranslate()
def test_error_long_text(self):
with self.assertRaises(YandexTranslateException, msg="ERR_TEXT_TOO_LONG"):
self.translate.translate("hi! " * 4098, "ru")
def test_invalid_key(self):
with self.assertRaises(YandexTranslateException, msg="ERR_KEY_INVALID"):
translate = YandexTranslate("my-invalid-key")
language = translate.detect("Hello!")
import pytesseract
from PIL import Image
from yandex_translate import YandexTranslate
translate = YandexTranslate('trnsl.1.1.20160420T200137Z.602d58f64d6ac2cf.d8e7eca404d946f979b434e59caa856c178eb095')
"""
print('Languages: ', translate.langs)
print('Translate directions: ',translate.directions)
print('Detect language: ', translate.detect('hola'))
"""
print(pytesseract.image_to_string(Image.open('wifi.png')))
text = pytesseract.image_to_string(Image.open('wifi.png'))
print('Translate: ', translate.translate(text, 'en-es'))
def test_blocked_key(self):
translate = YandexTranslate(
"trnsl.1.1.20130723T112255Z.cfcd2b1ebae9f" "ff1.86f3d1de3621e69b8c432fcdd6803bb87ef0e963"
)
with self.assertRaises(YandexTranslateException, msg="ERR_KEY_BLOCKED"):
languages = translate.detect("Hello!")
from yandex_translate import YandexTranslate
import language_codes
if __name__ == "__main__":
translate = YandexTranslate('trnsl.1.1.20160421T051442Z.3f43d13c5d30c594.bf3c396e87421e06cac7eab7b3771b3bf20af85c')
print ":speak: What do you want to translate?"
print ":listen:"
message = raw_input()
print "Translating {}".format(message)
language = translate.detect(message)
# translate from detected language to english
print ":speak:You are speaking {}".format(language_codes.languages[language])
translation = translate.translate(message, '{}-en'.format(language))
print translation
print ":speak:The english translation is {}".format(translation["text"][0])
def test_invalid_key(self):
with self.assertRaises(YandexTranslateException, msg="ERR_KEY_INVALID"):
translate = YandexTranslate("my-invalid-key")
language = translate.detect("Hello!")
def __init__(self, parent=None,api_key=None):
self.parent = parent
self.api_key = api_key
if not YandexTranslate:
raise GnrException('Missing YandexTranslate. hint: pip install yandex.translate')
self.translator = YandexTranslate(self.api_key)
class TranslateService(BaseService):
#
# Public methods
def __init__(self):
self.engine = YandexTranslate(config.TRANSLATE_YANDEX_KEY)
BaseService.__init__(self)
def translate(self, text, direction):
"""
Translate text from foreign language on native.
This method tries to find translated word in own DB and if translate doesn't exists,
it sends request to external service
:param text: sentence to translate.
:param direction: Direction for translate. For example en-ru.
:return: translated text
"""
translation = self.get(text, direction)
if translation:
return translation
response = self.engine.translate(text, direction)
if response and response[u'code'] == 200:
return self.add(text, response)
raise TranslateError(response)
def get(self, text, direction):
"""
Try to find translation in DB
:param text: sentence to translate.
:param direction: Direction for translate. For example en-ru.
:return: translated text
"""
return self.db.Translation.find_one({'text': text, 'direction': direction})
def add(self, text, translation):
"""
Add to collection new translation
:param text: sentence to translate.
:param translation: response from translate.yandex.ru
:return: translated text
"""
ss = ServiceLocator.resolve(ServiceLocator.SESSIONS)
translation_entity = self.db.Translation()
translation_entity.direction = translation[u'lang']
translation_entity.text = text
translation_entity.variations = translation[u'text']
translation_entity.author = ss.get().login
try:
translation_entity.validate()
translation_entity.save()
return translation_entity
except Exception as ex:
error(u'Translate.add', ex)
return None
def detect(self, text):
"""
Detect language
:param text: text for which needs to detect language
:return: language
"""
return self.engine.detect(text)
class translate:
def __init__(self, input_dir, dest_dir):
global logger
logger = utils.start_logging('translate.log') # create log file
self.input_dir = input_dir
self.dest_dir = dest_dir
self.database = 'translate.db' # hardcode database name for now
try:
with open(self.database) as file:
pass # file exists and is readable, nothing else to do
except IOError as e:
# file does not exist OR no read permissions
# create database if not existing
self._create_database()
# read yandex api key from file
self._read_yandex_api_key('yandex_key') # TODO: hardcode for now
self._yandex_connect(self.api_key)
def _create_database(self):
'''
create database and add all files to it
'''
self._new_database()
self._create_list_of_files()
self._add_files_to_database()
self._close_connection()
def _create_list_of_files(self):
'''
create a list of files in a directory structure: self.files
list all files in the directory that is given as an argument to the
function, searches all subdirectories recursively
'''
self.files = []
for root, dirnames, filenames in os.walk(self.input_dir):
for filename in filenames:
self.files.append(os.path.join(root, filename))
def _new_database(self):
'''
create and connect to a new sqlite database.
raise an error if there already is a database in place, asking the
user to manually remove the database (for safety reasons)
'''
# TODO: remove next two lines after testing -> don't automatically remove
if os.path.exists(self.database):
os.remove(self.database)
if os.path.exists(self.database):
message = ('Database already exists, please remove manually: %s'
%self.database)
logger.error(message)
raise IOError(message)
else:
logger.info('Database not found, creating database %s' %self.database)
try:
self.connection = sqlite3.connect(
self.database,
detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
except:
message = 'Failed to create database: %s' %self.database
logger.error(message)
raise sqlite3.OperationalError(message) # re-raise error
self._create_dbstructure()
sqlite3.register_adapter(bool, int)
sqlite3.register_converter("BOOLEAN", lambda v: bool(int(v)))
# tuples
self.connection.row_factory = sqlite3.Row
def _connect_to_database(self):
'''
check if database exists and try to connect to the database
'''
#utils.check_file_exists(self.database) # check if database exists
try:
logger.debug('Connecting to database: %s' %self.database)
self.connection = sqlite3.connect(
self.database,
detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
self.cursor = self.connection.cursor()
except:
message = 'Database %s exists, but failed to connect' %self.database
logger.error(message)
raise
def _close_connection(self):
'''
close connection to database
'''
try:
logger.debug('Closing database connection')
self.connection.close()
except NameError:
logger.error(('Failed to close database connection, '
'variable does not exist'))
def _create_dbstructure(self):
'''
create database structure
'''
# self.connection.row_factory = sqlite3.Row # use dict instread of tuple
self.cursor = self.connection.cursor()
# create table
self.cursor.execute(
'''CREATE TABLE filelist
(filename TEXT, processed BOOLEAN, skipped BOOLEAN)''')
def _add_files_to_database(self):
'''
add filenames to database table
'''
list_files = numpy.transpose(
numpy.vstack((numpy.transpose(self.files),
numpy.transpose(numpy.zeros(len(self.files))),
numpy.transpose(numpy.zeros(len(self.files))))))
logger.info('Adding filenames to database')
# add all files to database
self.cursor.executemany('''INSERT INTO filelist VALUES (?,?,?)''',
(list_files))
# commit changes to database
self.connection.commit()
def _get_next_file(self):
'''
get all unprocessed files from database
'''
# connect to database
self._connect_to_database()
# query database for all unprocessed files
self.cursor.execute("SELECT filename FROM filelist WHERE processed=(?)",
(False,))
# pick a random unprocessed file
self.next_file = random.choice(self.cursor.fetchall())[0]
# close connection
self._close_connection
def _read_yandex_api_key(self, yandex_key):
'''
read yandex api key from file
'''
f = open(yandex_key, 'r')
self.api_key = f.readline().strip('\n')
f.close()
def _yandex_connect(self, api_key):
'''
connect to yandex server with personal api_key
'''
self.translate = YandexTranslate(api_key)
def read_next_file(self):
'''
read the next file to be processed
'''
f = open(self.next_file, 'r')
self.text = f.read()
f.close()
def _translate_text(self, dest_lang='nl'):
'''
translate email to different language, default destination language is nl
'''
try:
self.translated_text = self.translate.translate(self.text, dest_lang)
except YandexTranslateException as e:
if e[0]=='ERR_TEXT_TOO_LONG':
logger.info('Unable to translate e-mail using yandex, ERR_TEXT_TOO_LONG')
self._update_db_skipped()
raise yandex_skipped
else:
logger.error('Error using Yandex Translation service, exiting...')
raise
def _save_translated_text(self):
'''
save translated text
'''
dest_file = os.path.join(self.dest_dir, self.next_file.rsplit(
self.input_dir)[1].strip('/'))
dir_name = os.path.dirname(dest_file)
utils._create_directory(dir_name)
f = open(dest_file, 'w')
f.write(self.translated_text['text'][0].encode('utf-8'))
f.close()
def _modify_database(self):
'''
modify database after file is processed
'''
self._connect_to_database()
# update database processed
self.cursor.execute("UPDATE filelist SET processed=(?) WHERE filename=(?)",
(True, self.next_file,))
# commit changes to database
self.connection.commit()
# close connection
self._close_connection
def _update_db_skipped(self):
'''
modify database after file is processed
'''
self._connect_to_database()
# update database skipped
self.cursor.execute("UPDATE filelist SET skipped=(?) WHERE filename=(?)",
(True, self.next_file,))
# update database processed
self.cursor.execute("UPDATE filelist SET processed=(?) WHERE filename=(?)",
(True, self.next_file,))
# commit changes to database
self.connection.commit()
# close connection
self._close_connection
def _get_processed_files(self):
'''
get all unprocessed files from database
'''
# connect to database
self._connect_to_database()
# query database for all unprocessed files
self.cursor.execute("SELECT filename FROM filelist WHERE processed=(?)",
(True,))
# pick a random unprocessed file
self.processed = self.cursor.fetchall()
# close connection
self._close_connection
def process(self):
'''
process files
stops automatically when yandex translation service returns an error
yandex error codes are the following:
error_codes = {
401: "ERR_KEY_INVALID",
402: "ERR_KEY_BLOCKED",
403: "ERR_DAILY_REQ_LIMIT_EXCEEDED",
404: "ERR_DAILY_CHAR_LIMIT_EXCEEDED",
413: "ERR_TEXT_TOO_LONG",
422: "ERR_UNPROCESSABLE_TEXT",
501: "ERR_LANG_NOT_SUPPORTED",
503: "ERR_SERVICE_NOT_AVAIBLE",
}
'''
while True:
self._get_next_file()
if len(self.next_file)==0:
logger.info('No more files to process, exiting...')
sys.exit()
else:
logger.info('Processing file %s' %self.next_file)
self.read_next_file()
try:
self._translate_text()
except yandex_skipped:
continue
self._save_translated_text()
self._modify_database()
def __init__(self):
self.developer_key = getattr(settings, 'YANDEX_TRANSLATE_KEY', None)
self.yandex_translate_obj = YandexTranslate(self.developer_key)
