def install_pair_package(self, pair): logger.info('Installing pair package %s' % pair) package_file = self.download_pair_package(pair) package_dir = self.extract_pair_package(package_file) alias_dir = self.create_pair_package_alias(package_dir) self.import_modes(clear = False) return package_dir
def download_package(self, name, force=False): package = self.find_package(name) package_url = REPOSITORY_URL + '/' + package['Filename'] package_dir = join(self.cache_dir, name) package_file = join(package_dir, 'package') + '.deb' mkdir('-p', package_dir) # Don't download the file twice if force or not isfile(package_file): logger.info('Downloading package %s' % name) request.urlretrieve(package_url, package_file) return package_file
def print_done(self, str, quiet = False): logger.info(str) str = '\r%s...' % str self.print_flush(str) try: yield print('{0} \033[92mdone\033[0m'.format(str)) except Full: logger.error('Timeout reached (%ss).' % self.pool_timeout) print('{0} \033[91merror\033[0m'.format(str)) except Exception as error: logger.error(error, exc_info=True) print('{0} \033[91merror\033[0m'.format(str)) if not quiet: raise error sys.exit(1)
def print_done(self, str, quiet=False): logger.info(str) # Avoid conflicting with a high log level if self.stdout_loglevel > 20: str = '\r%s...' % str self.print_flush(str) try: yield print('{0} \033[92mdone\033[0m'.format(str)) except Full: logger.error('Timeout reached (%ss).' % self.pool_timeout) print('{0} \033[91merror\033[0m'.format(str)) except Exception as error: logger.error(error, exc_info=True) print('{0} \033[91merror\033[0m'.format(str)) if not quiet: raise error sys.exit(1) else: yield
def __init__(self, source=None, target=None, intermediary=None, pack_dir=None): self.source = source self.target = target self.intermediary = intermediary # Create a temporary pack dir (if needed) self.pack_dir = abspath(pack_dir) # A class to download necessary pair package self.repository = ApertiumRepository(self.pack_dir) # Raise an exception if the language pair is unkown if not self.is_pair_available and self.has_pair: try: self.download_necessary_pairs() except StopIteration: raise Exception('The pair is not available') else: logger.info('Existing package(s) found for pair %s' % self.pair)
def translation_worker(queue): while True: try: es_translator, hit, index = queue.get(True) logger.info('Translating doc %s (%s)' % (index, hit.meta.id)) translated_hit = TranslatedHit(hit, es_translator.source_field, es_translator.target_field) translated_hit.add_translation(es_translator.apertium) logger.info('Translated doc %s (%s)' % (index, hit.meta.id)) # Skip on dry run if not es_translator.dry_run: # Create a new client to client = Elasticsearch(es_translator.url) translated_hit.save(client) logger.info('Saved translation for doc %s (%s)' % (index, hit.meta.id)) queue.task_done() except Exception as error: logger.warning('Unable to translate doc %s (%s)' % (index, hit.meta.id)) logger.warning(error)
def download_necessary_pairs(self): logger.info('Downloading necessary package(s) for %s' % self.pair) if self.any_pair_variant_in_packages: self.download_pair() else: self.download_intermediary_pairs()