def _translate(self, query: TranslationQuery) -> TranslationResponse:
if not query.is_context_aware_request():
return TranslationResponse(
costs=TranslationCosts(
money=GoogleTranslator._cost_of_query(query.query)),
translations=[
self.make_translation(
translation=self._simple_translate(query.query))
])
google_query = f'{query.before_context}<{HTML_TAG}>{query.query}</{HTML_TAG}>{query.after_context}'
costs = TranslationCosts(
money=GoogleTranslator._cost_of_query(query.query))
translation = self._simple_translate(google_query)
translation_response = TranslationResponse(costs=costs)
try:
result = GoogleTranslator.parse_spanned_string(translation)
translation_response.add_translation(self.make_translation(result))
except ValueError:
pass
return translation_response
def _translate(self, query: TranslationQuery) -> TranslationResponse:
dict_code = self.language_codes_to_dict_code(self.source_language,
self.target_language)
query_params = {'q': query.query, 'format': 'xml'}
api_url = '%(base_url)s/dictionaries/%(dict_code)s/search/first?%(query_params)s' % {
'base_url': API_BASE_URL,
'dict_code': dict_code,
'query_params': urllib.parse.urlencode(query_params)
}
# Decode JSON response
json_response = requests.get(api_url,
headers=self._get_base_headers()).json()
# Make sure key is valid
if 'errorCode' in json_response:
raise Exception(json_response['errorMessage'])
# Construct XML object
xml_tree = ET.ElementTree(
ET.fromstring(json_response['entryContent'].encode('utf-8')))
return TranslationResponse(
translations=[xml_tree.iter('quote').next().text],
costs=TranslationCosts(money=0) # a (limited) free API
)
def _translate(self, query: TranslationQuery) -> TranslationResponse:
"""
:param query:
:return:
"""
# Construct url
api_url = GlosbeTranslator.build_url(query.query, self.source_language,
self.target_language)
# Send request
response = requests.get(api_url).json()['tuc']
# Extract the translations (thanks @SAMSUNG)
translations = []
try:
for translation in response[:query.max_translations]:
translations.append(
self.make_translation(translation['phrase']['text']))
except KeyError:
pass
return TranslationResponse(
translations=translations,
costs=TranslationCosts(money=0 # API is free
))
def _translate(self, query: TranslationQuery) -> TranslationResponse:
"""
:param query:
:return:
"""
# Construct url
api_url = GlosbeTranslator.build_url(query.query, self.source_language,
self.target_language)
# Send request
response = requests.get(api_url).json()['tuc']
# Extract the translations (thanks @SAMSUNG)
translations = []
for translation in response[0:query.max_translations]:
try:
translations.append(
self.make_translation(translation['phrase']['text']))
except KeyError:
pass
translation_count = len(translations)
if translation_count < query.max_translations:
for meaning in response[0]['meanings']:
try:
this_meaning = meaning['text']
this_meaning = this_meaning[0:this_meaning.find("; ""
)]
problematic = False
for sign in problematic_definition_signs:
if this_meaning.lower().find(sign) != -1:
problematic = True
print(
f"ignoring problematic translation: {this_meaning}"
)
break
if problematic:
continue
if len(this_meaning.split(" ")) < 10:
translations.append(
self.make_translation(this_meaning))
translation_count += 1
except KeyError:
pass
if translation_count == query.max_translations:
break
return TranslationResponse(
translations=translations,
costs=TranslationCosts(money=0 # API is free
))
def translate(self, query):
translations = [
self.make_translation(each) for each in self.wordTranslations[query.query]
]
return TranslationResponse(
translations=translations,
costs=TranslationCosts(money=0), # API is free for testing
)
def _translate(self, query: TranslationQuery) -> TranslationResponse:
"""
:param query:
:return:
"""
pos = self._get_pos_tag(query)
response = self.word_api.getDefinitions(query.query, partOfSpeech=pos)
if not response:
if self._query_is_uppercase(query):
query.query = query.query.lower()
return self._translate(query)
elif self._query_starts_with_quote(query):
# try to see whether the problem is due to 'quotes' around the word name
self._remove_quotes(query)
return self._translate(query)
if not response:
response = []
translations = []
quality = int(self.get_quality())
for d in response[:query.max_translations]:
quality -= 1
try:
definition = self.definition_without_example_and_without_see_synonims(d)
if self.not_too_long(definition):
translations.append(self.make_translation(definition, quality))
meta_defined_word = self.is_meta_definition(definition)
if meta_defined_word:
response2 = self.word_api.getDefinitions(meta_defined_word)
for d2 in response2[:query.max_translations]:
d2clean = self.definition_without_example_and_without_see_synonims(d2)
if self.not_too_long(d2clean):
translations.append(self.make_translation(meta_defined_word + ": " + d2clean, quality))
except Exception as e:
logger.info(f"Can't parse definition: {e}")
if not translations:
# if we don't know the translation, just parrot back the question
translations.append(self.make_translation(query.query, quality))
rez = TranslationResponse(
translations=translations,
costs=TranslationCosts(
money=0 # API is free
)
)
return rez
def get_next_results(data,
exclude_services=[],
exclude_results=[],
number_of_results=-1):
translator_data = {
"source_language": data["from_lang_code"],
"target_language": data["to_lang_code"],
"query": data["query"],
}
api_mux = None
if data["from_lang_code"] == data["to_lang_code"] == "en":
api_mux = api_mux_worddefs
else:
api_mux = api_mux_translators
if number_of_results == 1:
logger.debug("Getting only top result")
translator_results = api_mux.get_next_results(translator_data,
number_of_results=1)
else:
logger.debug("Getting all results")
translator_results = api_mux.get_next_results(
translator_data,
number_of_results=-1,
exclude_services=exclude_services)
zeeguu_core.log(f"Got results get_next_results: {translator_results}")
json_translator_results = [(x, y.to_json()) for x, y in translator_results]
logger.debug("get_next_results Zeeguu-API - Got results: %s" %
json_translator_results)
logger.debug("get_next_results - exclude_services %s" % exclude_services)
# Returning data: [('GoogleTranslateWithContext',
# <python_translators.translation_response.TranslationResponse>), ...]
translations = []
for service_name, translation in translator_results:
if translation is None:
continue
lower_translation = translation.translations[0]["translation"].lower()
if lower_translation in exclude_results:
# Translation already exists fetched by get_top_translation
continue
translations = merge_translations(translations,
translation.translations)
translations = filter_empty_translations(translations)
if not MULTI_LANG_TRANSLATOR_AB_TESTING:
# Disabling order by quality when A/B testing is enabled
translations = order_by_quality(translations, translator_data["query"])
zeeguu_core.log(f"Translations get_next_results: {translations}")
response = TranslationResponse(translations=translations)
zeeguu_core.log(f"Returning response get_next_results: {response}")
return response
def _translate(self, query: TranslationQuery) -> TranslationResponse:
"""
:param query:
:return:
"""
duplicate = self.make_translation(query.query + query.query)
translations = [duplicate]
return TranslationResponse(
translations=translations,
costs=TranslationCosts(money=0 # no costs for this
))
def _translate(self, query: TranslationQuery) -> TranslationResponse:
api_query = MicrosoftTranslator._build_raw_query(query)
translation = self.send_translation_request(api_query,
'application/json')
# Enclose in <s> tag to make it valid XML (<s> is arbitrarily chosen)
xml_object = ET.fromstring(f'<s>{translation}</s>')
parsed_translation = xml_object.find(HTML_TAG).text
parsed_translation = parsed_translation.strip()
return TranslationResponse(
translations=[self.make_translation(parsed_translation)],
costs=TranslationCosts(money=0))
def _translate(self, query: TranslationQuery) -> TranslationResponse:
"""
:param query:
:return:
"""
simple_reverse = self.make_translation(query.query[::-1])
title_reverse = self.make_translation(query.query[::-1].title())
upper_reverse = self.make_translation(query.query[::-1].upper())
translations = [simple_reverse, title_reverse,
upper_reverse][:query.max_translations]
return TranslationResponse(
translations=translations,
costs=TranslationCosts(money=0 # API is free
))
def _translate(self, query: TranslationQuery) -> TranslationResponse:
responses = [None] * len(self.translators)
threads = []
# Start a thread for each translator
for idx, translator in enumerate(self.translators):
translate_thread = threading.Thread(target=translate_worker,
args=(translator, query,
responses, idx))
translate_thread.start()
threads.append(translate_thread)
# Wait for all threads to complete
if query.budget_is_unconstrained():
join_threads(threads)
else:
join_threads(threads, timeout_ms=query.budget.time)
translations = []
money_costs = 0
# Process all the responses
for idx, resp in enumerate(responses):
if not threads[idx].is_alive():
if resp: #fixing issue #35 (https://github.com/zeeguu-ecosystem/Python-Translators/issues/35)
translations = merge_translations(translations,
resp.translations)
money_costs += resp.costs.money
# reorder translations such that the translation which is the same
# as the original word is not the first
translations = filter_empty_translations(translations)
translations = order_by_quality(translations, query)
return TranslationResponse(translations=translations,
costs=TranslationCosts(money=money_costs))
def _translate(self, query: TranslationQuery) -> TranslationResponse:
"""
:param query:
:return:
"""
# Construct url
api_url = GlosbeTranslator.build_url(query.query, self.source_language,
self.target_language)
# Send request
response = requests.get(api_url).json()
# Attempt request each 5 seconds
while response['result'] == 'error':
print('awaiting reset')
sleep(5)
#print('new_connection: ', self.tr.get('http://ipecho.net/plain').text)
response = requests.get(api_url).json()
response = response['tuc']
# Extract the translations
translations = []
try:
for translation in response[:query.max_translations]:
translations.append(
self.make_translation(translation['phrase']['text']))
except KeyError:
pass
return TranslationResponse(
translations=translations,
costs=TranslationCosts(money=0 # API is free
))
def _translate(self, query: TranslationQuery) -> TranslationResponse:
"""
:param query:
:return:
"""
pos = self._get_pos_tag(query)
try:
if pos:
response = self.word_api.getDefinitions(query.query,
partOfSpeech=pos)
else:
response = self.word_api.getDefinitions(query.query)
except HTTPError as e:
print(
f"error code returned: {e.code} when translating {query.query}"
)
if e.code == 429:
logger.info("Wordnik returned 429: 'Too Many Requests'!")
elif e.code == 404:
# Wordnik API will return 404 for Uppercase or quoted words
# in that case we adapt the query and retry
if self._query_is_uppercase(query):
query.query = query.query.lower()
print(f"trying to translate {query.query} instead")
return self._translate(query)
elif self._query_starts_with_quote(query):
# try to see whether the problem is due to 'quotes' around the word name
self._remove_quotes(query)
print(f"trying to translate {query.query} instead")
return self._translate(query)
if not response:
response = []
translations = []
quality = int(self.get_quality())
for d in response[:query.max_translations]:
quality -= 1
try:
definition = self.definition_without_example_and_without_see_synonims(
d)
if self.not_too_long(definition):
translations.append(
self.make_translation(definition, quality))
meta_defined_word = self.is_meta_definition(definition)
if meta_defined_word:
response2 = self.word_api.getDefinitions(meta_defined_word)
for d2 in response2[:query.max_translations]:
d2clean = self.definition_without_example_and_without_see_synonims(
d2)
if self.not_too_long(d2clean):
translations.append(
self.make_translation(
meta_defined_word + ": " + d2clean,
quality))
except Exception as e:
logger.info(f"Can't parse definition: {e}")
if not translations:
# if we don't know the translation, just parrot back the question
translations.append(self.make_translation(query.query, quality))
rez = TranslationResponse(
translations=translations,
costs=TranslationCosts(money=0 # API is free
))
return rez
def translate(self, query: TranslationQuery) -> TranslationResponse:
start_time = current_milli_time()
if self._should_reject_request(query):
return TranslationResponse(
translations=[],
costs=TranslationCosts(
time=current_milli_time() - start_time
)
)
if self.cache:
results = self.cache.fetch(
query=query,
source_language=self.source_language,
target_language=self.target_language)
if results:
time_passed = current_milli_time() - start_time
logger.info(format_dict_for_logging(dict(
EVENT='translation_result',
FROM_CACHE=True,
TIME_PASSED=time_passed,
TRANSLATOR_NAME=self.get_translator_name(),
# The query
QUERY='\'' + query.query + '\'',
BEFORE_CONTEXT='\'' + query.before_context + '\'',
PRE_AFTER_CONTEXT='\'' + query.after_context + '\'',
TRANSLATIONS=[r['translation'] for r in results],
QUALITIES=[r['quality'] for r in results],
SERVICE_NAMES=[r['service_name'] for r in results],
)))
self.time_expense_tracker.track(time_passed)
return TranslationResponse(
translations=results,
costs=TranslationCosts(
money=0,
time=current_milli_time() - start_time
)
)
before_pre_processing = copy.copy(query)
# Pre-processing
for query_processor in self.query_processors:
query = query_processor.process_query(query)
# try/catch added to fix issue #36 (https://github.com/zeeguu-ecosystem/Python-Translators/issues/36)
try:
translation_response = self._translate(query)
except Exception as e:
logger.info(f"Translator {self.get_translator_name()} failed in _translate()")
return TranslationResponse(
translations=[],
costs=TranslationCosts(
time=current_milli_time() - start_time
)
)
# Post-processing
for response_processor in self.response_processors:
translation_response = response_processor.process_response(translation_response)
time_passed = current_milli_time() - start_time
log_string = format_dict_for_logging(dict(
EVENT='translation_result',
FROM_CACHE=False,
TIME_PASSED=time_passed,
TRANSLATOR_NAME=self.get_translator_name(),
CONTEXT_PROCESSORS=list(map(lambda qp: qp.get_name(), self.query_processors)),
RESPONSE_PROCESSORS=list(map(lambda rp: rp.get_name(), self.response_processors)),
# Before pre-processing
PRE_QUERY='\'' + before_pre_processing.query + '\'',
PRE_BEFORE_CONTEXT='\'' + before_pre_processing.before_context + '\'',
PRE_AFTER_CONTEXT='\'' + before_pre_processing.after_context + '\'',
# After pre-processing
POST_QUERY='\'' + query.query + '\'',
POST_BEFORE_CONTEXT='\'' + query.before_context + '\'',
POST_AFTER_CONTEXT='\'' + query.after_context + '\'',
TRANSLATIONS=translation_response.get_raw_translations(),
QUALITIES=translation_response.get_raw_qualities(),
SERVICE_NAMES=translation_response.get_raw_service_names(),
))
logger.info(log_string)
# Store time costs in response
translation_response.costs.time = time_passed
self.time_expense_tracker.track(time_passed)
if self.cache:
self.cache.store(
query=before_pre_processing,
source_language=self.source_language,
target_language=self.target_language,
translations=translation_response.translations
)
return translation_response
def _translate(self, query: TranslationQuery) -> TranslationResponse:
time.sleep(3) # very unresponsive translator
return TranslationResponse(costs=TranslationCosts(time=3000,
money=0.001),
translations=[])