def _translate(self, query: TranslationQuery) -> TranslationResponse: if not query.is_context_aware_request(): return TranslationResponse( costs=TranslationCosts( money=GoogleTranslator._cost_of_query(query.query)), translations=[ self.make_translation( translation=self._simple_translate(query.query)) ]) google_query = f'{query.before_context}<{HTML_TAG}>{query.query}</{HTML_TAG}>{query.after_context}' costs = TranslationCosts( money=GoogleTranslator._cost_of_query(query.query)) translation = self._simple_translate(google_query) translation_response = TranslationResponse(costs=costs) try: result = GoogleTranslator.parse_spanned_string(translation) translation_response.add_translation(self.make_translation(result)) except ValueError: pass return translation_response
def _translate(self, query: TranslationQuery) -> TranslationResponse: dict_code = self.language_codes_to_dict_code(self.source_language, self.target_language) query_params = {'q': query.query, 'format': 'xml'} api_url = '%(base_url)s/dictionaries/%(dict_code)s/search/first?%(query_params)s' % { 'base_url': API_BASE_URL, 'dict_code': dict_code, 'query_params': urllib.parse.urlencode(query_params) } # Decode JSON response json_response = requests.get(api_url, headers=self._get_base_headers()).json() # Make sure key is valid if 'errorCode' in json_response: raise Exception(json_response['errorMessage']) # Construct XML object xml_tree = ET.ElementTree( ET.fromstring(json_response['entryContent'].encode('utf-8'))) return TranslationResponse( translations=[xml_tree.iter('quote').next().text], costs=TranslationCosts(money=0) # a (limited) free API )
def _translate(self, query: TranslationQuery) -> TranslationResponse: """ :param query: :return: """ # Construct url api_url = GlosbeTranslator.build_url(query.query, self.source_language, self.target_language) # Send request response = requests.get(api_url).json()['tuc'] # Extract the translations (thanks @SAMSUNG) translations = [] try: for translation in response[:query.max_translations]: translations.append( self.make_translation(translation['phrase']['text'])) except KeyError: pass return TranslationResponse( translations=translations, costs=TranslationCosts(money=0 # API is free ))
def _translate(self, query: TranslationQuery) -> TranslationResponse: """ :param query: :return: """ # Construct url api_url = GlosbeTranslator.build_url(query.query, self.source_language, self.target_language) # Send request response = requests.get(api_url).json()['tuc'] # Extract the translations (thanks @SAMSUNG) translations = [] for translation in response[0:query.max_translations]: try: translations.append( self.make_translation(translation['phrase']['text'])) except KeyError: pass translation_count = len(translations) if translation_count < query.max_translations: for meaning in response[0]['meanings']: try: this_meaning = meaning['text'] this_meaning = this_meaning[0:this_meaning.find("; "" )] problematic = False for sign in problematic_definition_signs: if this_meaning.lower().find(sign) != -1: problematic = True print( f"ignoring problematic translation: {this_meaning}" ) break if problematic: continue if len(this_meaning.split(" ")) < 10: translations.append( self.make_translation(this_meaning)) translation_count += 1 except KeyError: pass if translation_count == query.max_translations: break return TranslationResponse( translations=translations, costs=TranslationCosts(money=0 # API is free ))
def translate(self, query): translations = [ self.make_translation(each) for each in self.wordTranslations[query.query] ] return TranslationResponse( translations=translations, costs=TranslationCosts(money=0), # API is free for testing )
def _translate(self, query: TranslationQuery) -> TranslationResponse: """ :param query: :return: """ pos = self._get_pos_tag(query) response = self.word_api.getDefinitions(query.query, partOfSpeech=pos) if not response: if self._query_is_uppercase(query): query.query = query.query.lower() return self._translate(query) elif self._query_starts_with_quote(query): # try to see whether the problem is due to 'quotes' around the word name self._remove_quotes(query) return self._translate(query) if not response: response = [] translations = [] quality = int(self.get_quality()) for d in response[:query.max_translations]: quality -= 1 try: definition = self.definition_without_example_and_without_see_synonims(d) if self.not_too_long(definition): translations.append(self.make_translation(definition, quality)) meta_defined_word = self.is_meta_definition(definition) if meta_defined_word: response2 = self.word_api.getDefinitions(meta_defined_word) for d2 in response2[:query.max_translations]: d2clean = self.definition_without_example_and_without_see_synonims(d2) if self.not_too_long(d2clean): translations.append(self.make_translation(meta_defined_word + ": " + d2clean, quality)) except Exception as e: logger.info(f"Can't parse definition: {e}") if not translations: # if we don't know the translation, just parrot back the question translations.append(self.make_translation(query.query, quality)) rez = TranslationResponse( translations=translations, costs=TranslationCosts( money=0 # API is free ) ) return rez
def merge_responses(responses: [TranslationResponse]) -> TranslationResponse: new_translations = [] money_costs = 0 for response in responses: new_translations = merge_translations(new_translations, response.translations) money_costs += response.costs.money return TranslationResponse(translations=new_translations, costs=TranslationCosts(money=money_costs))
def _translate(self, query: TranslationQuery) -> TranslationResponse: """ :param query: :return: """ duplicate = self.make_translation(query.query + query.query) translations = [duplicate] return TranslationResponse( translations=translations, costs=TranslationCosts(money=0 # no costs for this ))
def _translate(self, query: TranslationQuery) -> TranslationResponse: api_query = MicrosoftTranslator._build_raw_query(query) translation = self.send_translation_request(api_query, 'application/json') # Enclose in <s> tag to make it valid XML (<s> is arbitrarily chosen) xml_object = ET.fromstring(f'<s>{translation}</s>') parsed_translation = xml_object.find(HTML_TAG).text parsed_translation = parsed_translation.strip() return TranslationResponse( translations=[self.make_translation(parsed_translation)], costs=TranslationCosts(money=0))
def _translate(self, query: TranslationQuery) -> TranslationResponse: """ :param query: :return: """ simple_reverse = self.make_translation(query.query[::-1]) title_reverse = self.make_translation(query.query[::-1].title()) upper_reverse = self.make_translation(query.query[::-1].upper()) translations = [simple_reverse, title_reverse, upper_reverse][:query.max_translations] return TranslationResponse( translations=translations, costs=TranslationCosts(money=0 # API is free ))
def _translate(self, query: TranslationQuery) -> TranslationResponse: responses = [None] * len(self.translators) threads = [] # Start a thread for each translator for idx, translator in enumerate(self.translators): translate_thread = threading.Thread(target=translate_worker, args=(translator, query, responses, idx)) translate_thread.start() threads.append(translate_thread) # Wait for all threads to complete if query.budget_is_unconstrained(): join_threads(threads) else: join_threads(threads, timeout_ms=query.budget.time) translations = [] money_costs = 0 # Process all the responses for idx, resp in enumerate(responses): if not threads[idx].is_alive(): if resp: #fixing issue #35 (https://github.com/zeeguu-ecosystem/Python-Translators/issues/35) translations = merge_translations(translations, resp.translations) money_costs += resp.costs.money # reorder translations such that the translation which is the same # as the original word is not the first translations = filter_empty_translations(translations) translations = order_by_quality(translations, query) return TranslationResponse(translations=translations, costs=TranslationCosts(money=money_costs))
def _translate(self, query: TranslationQuery) -> TranslationResponse: """ :param query: :return: """ # Construct url api_url = GlosbeTranslator.build_url(query.query, self.source_language, self.target_language) # Send request response = requests.get(api_url).json() # Attempt request each 5 seconds while response['result'] == 'error': print('awaiting reset') sleep(5) #print('new_connection: ', self.tr.get('http://ipecho.net/plain').text) response = requests.get(api_url).json() response = response['tuc'] # Extract the translations translations = [] try: for translation in response[:query.max_translations]: translations.append( self.make_translation(translation['phrase']['text'])) except KeyError: pass return TranslationResponse( translations=translations, costs=TranslationCosts(money=0 # API is free ))
def __init__(self, translations: [dict] = None, costs: TranslationCosts = None): self.translations = translations if translations else [] self.costs = costs if costs else TranslationCosts()
def _estimate_costs(self, query: TranslationQuery) -> TranslationCosts: return TranslationCosts(money=0)
def _translate(self, query: TranslationQuery) -> TranslationResponse: time.sleep(3) # very unresponsive translator return TranslationResponse(costs=TranslationCosts(time=3000, money=0.001), translations=[])
def _translate(self, query: TranslationQuery) -> TranslationResponse: """ :param query: :return: """ pos = self._get_pos_tag(query) try: if pos: response = self.word_api.getDefinitions(query.query, partOfSpeech=pos) else: response = self.word_api.getDefinitions(query.query) except HTTPError as e: print( f"error code returned: {e.code} when translating {query.query}" ) if e.code == 429: logger.info("Wordnik returned 429: 'Too Many Requests'!") elif e.code == 404: # Wordnik API will return 404 for Uppercase or quoted words # in that case we adapt the query and retry if self._query_is_uppercase(query): query.query = query.query.lower() print(f"trying to translate {query.query} instead") return self._translate(query) elif self._query_starts_with_quote(query): # try to see whether the problem is due to 'quotes' around the word name self._remove_quotes(query) print(f"trying to translate {query.query} instead") return self._translate(query) if not response: response = [] translations = [] quality = int(self.get_quality()) for d in response[:query.max_translations]: quality -= 1 try: definition = self.definition_without_example_and_without_see_synonims( d) if self.not_too_long(definition): translations.append( self.make_translation(definition, quality)) meta_defined_word = self.is_meta_definition(definition) if meta_defined_word: response2 = self.word_api.getDefinitions(meta_defined_word) for d2 in response2[:query.max_translations]: d2clean = self.definition_without_example_and_without_see_synonims( d2) if self.not_too_long(d2clean): translations.append( self.make_translation( meta_defined_word + ": " + d2clean, quality)) except Exception as e: logger.info(f"Can't parse definition: {e}") if not translations: # if we don't know the translation, just parrot back the question translations.append(self.make_translation(query.query, quality)) rez = TranslationResponse( translations=translations, costs=TranslationCosts(money=0 # API is free )) return rez
def translate(self, query: TranslationQuery) -> TranslationResponse: start_time = current_milli_time() if self._should_reject_request(query): return TranslationResponse( translations=[], costs=TranslationCosts( time=current_milli_time() - start_time ) ) if self.cache: results = self.cache.fetch( query=query, source_language=self.source_language, target_language=self.target_language) if results: time_passed = current_milli_time() - start_time logger.info(format_dict_for_logging(dict( EVENT='translation_result', FROM_CACHE=True, TIME_PASSED=time_passed, TRANSLATOR_NAME=self.get_translator_name(), # The query QUERY='\'' + query.query + '\'', BEFORE_CONTEXT='\'' + query.before_context + '\'', PRE_AFTER_CONTEXT='\'' + query.after_context + '\'', TRANSLATIONS=[r['translation'] for r in results], QUALITIES=[r['quality'] for r in results], SERVICE_NAMES=[r['service_name'] for r in results], ))) self.time_expense_tracker.track(time_passed) return TranslationResponse( translations=results, costs=TranslationCosts( money=0, time=current_milli_time() - start_time ) ) before_pre_processing = copy.copy(query) # Pre-processing for query_processor in self.query_processors: query = query_processor.process_query(query) # try/catch added to fix issue #36 (https://github.com/zeeguu-ecosystem/Python-Translators/issues/36) try: translation_response = self._translate(query) except Exception as e: logger.info(f"Translator {self.get_translator_name()} failed in _translate()") return TranslationResponse( translations=[], costs=TranslationCosts( time=current_milli_time() - start_time ) ) # Post-processing for response_processor in self.response_processors: translation_response = response_processor.process_response(translation_response) time_passed = current_milli_time() - start_time log_string = format_dict_for_logging(dict( EVENT='translation_result', FROM_CACHE=False, TIME_PASSED=time_passed, TRANSLATOR_NAME=self.get_translator_name(), CONTEXT_PROCESSORS=list(map(lambda qp: qp.get_name(), self.query_processors)), RESPONSE_PROCESSORS=list(map(lambda rp: rp.get_name(), self.response_processors)), # Before pre-processing PRE_QUERY='\'' + before_pre_processing.query + '\'', PRE_BEFORE_CONTEXT='\'' + before_pre_processing.before_context + '\'', PRE_AFTER_CONTEXT='\'' + before_pre_processing.after_context + '\'', # After pre-processing POST_QUERY='\'' + query.query + '\'', POST_BEFORE_CONTEXT='\'' + query.before_context + '\'', POST_AFTER_CONTEXT='\'' + query.after_context + '\'', TRANSLATIONS=translation_response.get_raw_translations(), QUALITIES=translation_response.get_raw_qualities(), SERVICE_NAMES=translation_response.get_raw_service_names(), )) logger.info(log_string) # Store time costs in response translation_response.costs.time = time_passed self.time_expense_tracker.track(time_passed) if self.cache: self.cache.store( query=before_pre_processing, source_language=self.source_language, target_language=self.target_language, translations=translation_response.translations ) return translation_response
def estimate_costs(self, query: TranslationQuery) -> TranslationCosts: costs = TranslationCosts() costs.money = self.compute_money_costs(query=query) costs.time = self.time_expense_tracker.mean(default=100) return costs