def install_pair_package(self, pair):
logger.info('Installing pair package %s' % pair)
package_file = self.download_pair_package(pair)
package_dir = self.extract_pair_package(package_file)
alias_dir = self.create_pair_package_alias(package_dir)
self.import_modes(clear = False)
return package_dir
def download_package(self, name, force=False):
package = self.find_package(name)
package_url = REPOSITORY_URL + '/' + package['Filename']
package_dir = join(self.cache_dir, name)
package_file = join(package_dir, 'package') + '.deb'
mkdir('-p', package_dir)
# Don't download the file twice
if force or not isfile(package_file):
logger.info('Downloading package %s' % name)
request.urlretrieve(package_url, package_file)
return package_file
def print_done(self, str, quiet = False):
logger.info(str)
str = '\r%s...' % str
self.print_flush(str)
try:
yield
print('{0} \033[92mdone\033[0m'.format(str))
except Full:
logger.error('Timeout reached (%ss).' % self.pool_timeout)
print('{0} \033[91merror\033[0m'.format(str))
except Exception as error:
logger.error(error, exc_info=True)
print('{0} \033[91merror\033[0m'.format(str))
if not quiet: raise error
sys.exit(1)
def print_done(self, str, quiet=False):
logger.info(str)
# Avoid conflicting with a high log level
if self.stdout_loglevel > 20:
str = '\r%s...' % str
self.print_flush(str)
try:
yield
print('{0} \033[92mdone\033[0m'.format(str))
except Full:
logger.error('Timeout reached (%ss).' % self.pool_timeout)
print('{0} \033[91merror\033[0m'.format(str))
except Exception as error:
logger.error(error, exc_info=True)
print('{0} \033[91merror\033[0m'.format(str))
if not quiet: raise error
sys.exit(1)
else:
yield
def __init__(self,
source=None,
target=None,
intermediary=None,
pack_dir=None):
self.source = source
self.target = target
self.intermediary = intermediary
# Create a temporary pack dir (if needed)
self.pack_dir = abspath(pack_dir)
# A class to download necessary pair package
self.repository = ApertiumRepository(self.pack_dir)
# Raise an exception if the language pair is unkown
if not self.is_pair_available and self.has_pair:
try:
self.download_necessary_pairs()
except StopIteration:
raise Exception('The pair is not available')
else:
logger.info('Existing package(s) found for pair %s' % self.pair)
def translation_worker(queue):
while True:
try:
es_translator, hit, index = queue.get(True)
logger.info('Translating doc %s (%s)' % (index, hit.meta.id))
translated_hit = TranslatedHit(hit, es_translator.source_field, es_translator.target_field)
translated_hit.add_translation(es_translator.apertium)
logger.info('Translated doc %s (%s)' % (index, hit.meta.id))
# Skip on dry run
if not es_translator.dry_run:
# Create a new client to
client = Elasticsearch(es_translator.url)
translated_hit.save(client)
logger.info('Saved translation for doc %s (%s)' % (index, hit.meta.id))
queue.task_done()
except Exception as error:
logger.warning('Unable to translate doc %s (%s)' % (index, hit.meta.id))
logger.warning(error)
def download_necessary_pairs(self):
logger.info('Downloading necessary package(s) for %s' % self.pair)
if self.any_pair_variant_in_packages:
self.download_pair()
else:
self.download_intermediary_pairs()