def run_daily_activities(self):
# If gengosystem is disabled, we don't want to do anything.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
balance = gengo_api.get_balance()
threshold = settings.GENGO_ACCOUNT_BALANCE_THRESHOLD
if threshold < balance < (2 * threshold):
mail_admins(
subject='Warning: Gengo account balance {0} < {1}'.format(
balance, 2 * threshold),
message=wrap_with_paragraphs(
'Dear mom,'
'\n\n'
'Translations are the fab. Running low on funds. Send '
'more money when you get a chance.'
'\n\n'
'Love,'
'\n\n'
'Fjord McGengo'
)
)
def run_daily_activities(self):
# If gengosystem is disabled, we don't want to do anything.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
balance = gengo_api.get_balance()
threshold = settings.GENGO_ACCOUNT_BALANCE_THRESHOLD
if threshold < balance < (2 * threshold):
mail_admins(
subject='Warning: Gengo account balance {0} < {1}'.format(
balance, 2 * threshold),
message=wrap_with_paragraphs(
'Dear mom,'
'\n\n'
'Translations are the fab. Running low on funds. Send '
'more money when you get a chance.'
'\n\n'
'Love,'
'\n\n'
'Fjord McGengo'
)
)
def pull_translations(self):
# If gengosystem is disabled, we just return immediately. We
# can backfill later.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
# Get all the orders that are in progress
orders = GengoOrder.objects.filter(status=STATUS_IN_PROGRESS)
for order in orders:
# Get the list of all completed jobs
completed = gengo_api.completed_jobs_for_order(order.order_id)
# If there are no completed jobs, then we don't need to
# bother doing any additional processing for this order
if not completed:
continue
# For each complete job we haven't seen before, pull it
# from the db, save the translated text and update all the
# bookkeeping.
for comp in completed:
id_ = GengoJob.unique_id_to_id(comp['custom_data'])
job = GengoJob.objects.get(pk=id_)
if job.status == STATUS_COMPLETE:
continue
instance = job.content_object
setattr(instance, job.dst_field, comp['body_tgt'])
instance.save()
job.mark_complete()
# Check to see if there are still outstanding jobs for
# this order. If there aren't, close the order out.
outstanding = (GengoJob.objects
.filter(order=order, status=STATUS_IN_PROGRESS)
.count())
if outstanding == 0:
order.mark_complete()
def pull_translations(self):
# If gengosystem is disabled, we just return immediately. We
# can backfill later.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
# Get all the orders that are in progress
orders = GengoOrder.objects.filter(status=STATUS_IN_PROGRESS)
for order in orders:
# Get the list of all completed jobs
completed = gengo_api.completed_jobs_for_order(order.order_id)
# If there are no completed jobs, then we don't need to
# bother doing any additional processing for this order
if not completed:
continue
# For each complete job we haven't seen before, pull it
# from the db, save the translated text and update all the
# bookkeeping.
for comp in completed:
id_ = GengoJob.unique_id_to_id(comp['custom_data'])
job = GengoJob.objects.get(pk=id_)
if job.status == STATUS_COMPLETE:
continue
instance = job.content_object
setattr(instance, job.dst_field, comp['body_tgt'])
instance.save()
job.mark_complete()
# Check to see if there are still outstanding jobs for
# this order. If there aren't, close the order out.
outstanding = (GengoJob.objects
.filter(order=order, status=STATUS_IN_PROGRESS)
.count())
if outstanding == 0:
order.mark_complete()
def gengo_translator_view(request):
gengo_machiners = Product.objects.filter(translation_system='gengo_machine')
balance = None
configured = False
if settings.GENGO_PUBLIC_KEY and settings.GENGO_PRIVATE_KEY:
gengo_api = FjordGengo()
balance = gengo_api.get_balance()
configured = True
return render(request, 'admin/gengo_translator_view.html', {
'configured': configured,
'settings': settings,
'gengo_machiners': gengo_machiners,
'balance': balance
})
def gengo_translator_view(request):
gengo_machiners = Product.objects.filter(
translation_system='gengo_machine')
balance = None
configured = False
if settings.GENGO_PUBLIC_KEY and settings.GENGO_PRIVATE_KEY:
gengo_api = FjordGengo()
balance = gengo_api.get_balance()
configured = True
return render(
request, 'admin/gengo_translator_view.html', {
'configured': configured,
'settings': settings,
'gengo_machiners': gengo_machiners,
'balance': balance
})
def run_daily_activities(self):
# If gengosystem is disabled, we don't want to do anything.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
balance = gengo_api.get_balance()
threshold = settings.GENGO_ACCOUNT_BALANCE_THRESHOLD
if threshold < balance < (2 * threshold):
recipients = get_recipients('gengo_balance')
subject = 'Warning: Gengo account balance {0} < {1}'.format(
balance, 2 * threshold)
body = wrap_with_paragraphs(dedent("""\
Dear mom,
Translations are the fab. Running low on funds. Send
more money when you get a chance.
Love,
Fjord McGengo
"""))
if recipients:
send_mail(
subject=subject,
message=body,
recipient_list=recipients,
from_email=settings.SERVER_EMAIL
)
else:
log.info('No recipients for "%s"\n%s\n%s' % (
'gengo_balance', subject, body))
def run_daily_activities(self):
# If gengosystem is disabled, we don't want to do anything.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
balance = gengo_api.get_balance()
threshold = settings.GENGO_ACCOUNT_BALANCE_THRESHOLD
if threshold < balance < (2 * threshold):
recipients = get_recipients('gengo_balance')
subject = 'Warning: Gengo account balance {0} < {1}'.format(
balance, 2 * threshold)
body = wrap_with_paragraphs(
dedent("""\
Dear mom,
Translations are the fab. Running low on funds. Send
more money when you get a chance.
Love,
Fjord McGengo
"""))
if recipients:
send_mail(subject=subject,
message=body,
recipient_list=recipients,
from_email=settings.SERVER_EMAIL)
else:
log.info('No recipients for "%s"\n%s\n%s' %
('gengo_balance', subject, body))
def gengo_translator_view(request):
"""Covers Gengo-specific translation system status"""
products = Product.objects.all()
balance = None
configured = False
gengo_languages = None
missing_prod_locales = None
if settings.GENGO_PUBLIC_KEY and settings.GENGO_PRIVATE_KEY:
gengo_api = FjordGengo()
balance = gengo_api.get_balance()
configured = True
# Figure out the list of languages Gengo supports and relate them
# to PROD locales.
languages = sorted(gengo_api.get_languages(raw=True)['response'],
key=lambda item: item['lc'])
gengo_languages = []
PROD_LANG = settings.PROD_LANGUAGES
for lang in languages:
lang_lc = lang['lc']
prod_langs = [item for item in PROD_LANG
if locale_equals_language(item, lang_lc)]
prod_langs = ' '.join(prod_langs)
gengo_languages.append(
(lang_lc, lang['language'], prod_langs)
)
# Figure out the list of PROD locales that don't have a Gengo
# supported language. (Yes, this conflates locale with
# language which isn't great, but is good enough for now.)
languages = gengo_api.get_languages()
missing_prod_locales = []
for prod_lang in PROD_LANG:
langs = [item for item in languages
if locale_equals_language(prod_lang, item)]
if langs:
continue
missing_prod_locales.append(prod_lang)
# How many orders have we created/completed in the last week
# day-by-day?
seven_days = datetime.now() - timedelta(days=7)
orders = GengoOrder.objects.filter(created__gte=seven_days)
created_by_day = {}
for order in orders:
dt = order.created.strftime('%Y-%m-%d')
created_by_day.setdefault(dt, []).append(order)
orders = GengoOrder.objects.filter(completed__gte=seven_days)
completed_by_day = {}
for order in orders:
dt = order.completed.strftime('%Y-%m-%d')
completed_by_day.setdefault(dt, []).append(order)
# Get date labels in YYYY-mm-dd form for the last 7 days
days = [
(datetime.now() - timedelta(days=i)).strftime('%Y-%m-%d')
for i in range(7)
]
seven_days_of_orders = []
for day in sorted(days):
seven_days_of_orders.append(
(day,
len(created_by_day.get(day, [])),
len(completed_by_day.get(day, [])))
)
outstanding = [
{
'id': order.id,
'order_id': order.order_id,
'created': order.created,
'total_jobs': order.gengojob_set.all().count(),
'completed_jobs': order.completed_jobs().count(),
'outstanding_jobs': order.outstanding_jobs().count(),
}
for order in GengoOrder.objects.filter(completed__isnull=True)]
return render(request, 'admin/gengo_translator_view.html', {
'title': 'Translations - Gengo Maintenance',
'configured': configured,
'settings': settings,
'products': products,
'outstanding': outstanding,
'seven_days_of_orders': seven_days_of_orders,
'balance': balance,
'gengo_languages': gengo_languages,
'missing_prod_locales': missing_prod_locales,
})
def push_translations(self):
# If gengosystem is disabled, we just return immediately. We
# can backfill later.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
if self.gengo_watch_balance:
balance = gengo_api.get_balance()
threshold = settings.GENGO_ACCOUNT_BALANCE_THRESHOLD
# statsd the balance so we can track it with graphite.
statsd.gauge('translation.gengo.balance', balance)
if not self.balance_good_to_continue(balance, threshold):
# If we don't have enough balance, stop.
return
# Create language buckets for the jobs for this translator.
# We bucket by language because this makes it easier for a
# single Gengo translator to translate all the jobs in an
# order.
jobs = GengoJob.objects.filter(
tier=self.gengo_tier, status=STATUS_CREATED)
lang_buckets = {}
for job in jobs:
lang_buckets.setdefault(job.src_lang, []).append(job)
# For each bucket, assemble an order and post it.
for lang, jobs in lang_buckets.items():
batch = []
for job in jobs:
batch.append({
'id': job.id,
'lc_src': job.src_lang,
'lc_dst': job.dst_lang,
'tier': self.gengo_tier,
'text': getattr(job.content_object, job.src_field),
'unique_id': job.unique_id
})
try:
resp = gengo_api.translate_bulk(batch)
except GengoError as exc:
self.log_error(
instance=None, action='push-translations',
msg=unicode(exc),
metadata={
'batch': batch
})
continue
# We should have an `order_id` at this point, so we create a
# GengoOrder with it.
order = GengoOrder(order_id=resp['order_id'])
order.save()
order.log('created', metadata={'response': resp})
# Update all the jobs in the order.
for job in jobs:
job.assign_to_order(order)
if self.gengo_watch_balance:
# Update the balance and see if we're below the threshold.
balance = balance - float(resp['credits_used'])
if not self.balance_good_to_continue(balance, threshold):
# If we don't have enough balance, stop.
return
def translate(self, instance, src_lang, src_field, dst_lang, dst_field):
# If gengosystem is disabled, we just return immediately. We
# can backfill later.
if not waffle.switch_is_active('gengosystem'):
return
text = getattr(instance, src_field)
metadata = {
'tier': self.gengo_tier,
'locale': instance.locale,
'length': len(text),
'body': text[:50].encode('utf-8')
}
gengo_api = FjordGengo()
# Guess the language. If we can't guess the language, then we
# don't create a GengoJob.
try:
lc_src = gengo_api.guess_language(text)
if lc_src not in gengo_api.get_languages():
raise GengoUnsupportedLanguage(
'unsupported language: {0}'.format(lc_src))
except GengoUnknownLanguage as exc:
# FIXME: This might be an indicator that this response is
# spam. At some point p, we can write code to account for
# that.
self.log_error(instance, action='guess-language', msg=unicode(exc),
metadata=metadata)
statsd.incr('translation.{0}.unknown'.format(self.name))
return
except GengoUnsupportedLanguage as exc:
# FIXME: This is a similar boat to GengoUnknownLanguage
# where for now, we're just going to ignore it because I'm
# not sure what to do about it and I'd like more data.
self.log_error(instance, action='translate', msg=unicode(exc),
metadata=metadata)
statsd.incr('translation.{0}.unsupported'.format(self.name))
return
# If the locale doesn't equal the guessed language, then
# that's interesting since the user is writing feedback in a
# language other than what the ui is showing. We want to log
# that for metrics purposes.
if not locale_equals_language(instance.locale, lc_src):
self.log_error(
instance,
action='guess-language',
msg='locale "{0}" != guessed language "{1}"'.format(
instance.locale, lc_src),
metadata=metadata)
# If the source language is English, we just copy it over and
# we're done.
if locale_equals_language(dst_lang, lc_src):
setattr(instance, dst_field, text)
instance.save()
self.log_info(
instance, action='translate',
msg=u'lc_src == dst_lang, so we copy src to dst',
metadata=metadata)
return
if ((self.gengo_check_supported_machine_lc_dst
and lc_src in GENGO_UNSUPPORTED_MACHINE_LC_SRC)):
return
# If src/dst isn't a supported pair, log an issue for metrics
# purposes and move on.
if ((self.gengo_check_supported_language_pair
and (lc_src, dst_lang) not in gengo_api.get_language_pairs())):
self.log_error(
instance, action='translate',
msg=u'(lc_src {0}, dst_lang {1}) not supported'.format(
lc_src, dst_lang),
metadata=metadata)
return
job = GengoJob(
tier=self.gengo_tier,
content_object=instance,
src_lang=lc_src,
src_field=src_field,
dst_lang=dst_lang,
dst_field=dst_field
)
job.save()
def push_translations(self):
# If gengosystem is disabled, we just return immediately. We
# can backfill later.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
if self.gengo_watch_balance:
balance = gengo_api.get_balance()
threshold = settings.GENGO_ACCOUNT_BALANCE_THRESHOLD
# statsd the balance so we can track it with graphite.
statsd.gauge('translation.gengo.balance', balance)
if not self.balance_good_to_continue(balance, threshold):
# If we don't have enough balance, stop.
return
# Create language buckets for the jobs for this translator.
# We bucket by language because this makes it easier for a
# single Gengo translator to translate all the jobs in an
# order.
jobs = GengoJob.objects.filter(tier=self.gengo_tier,
status=STATUS_CREATED)
lang_buckets = {}
for job in jobs:
lang_buckets.setdefault(job.src_lang, []).append(job)
# For each bucket, assemble an order and post it.
for lang, jobs in lang_buckets.items():
batch = []
for job in jobs:
batch.append({
'id': job.id,
'lc_src': job.src_lang,
'lc_dst': job.dst_lang,
'tier': self.gengo_tier,
'text': getattr(job.content_object, job.src_field),
'unique_id': job.unique_id
})
try:
resp = gengo_api.translate_bulk(batch)
except GengoError as exc:
self.log_error(instance=None,
action='push-translations',
msg=unicode(exc),
metadata={'batch': batch})
continue
# We should have an `order_id` at this point, so we create a
# GengoOrder with it.
order = GengoOrder(order_id=resp['order_id'])
order.save()
order.log('created', metadata={'response': resp})
# Update all the jobs in the order.
for job in jobs:
job.assign_to_order(order)
if self.gengo_watch_balance:
# Update the balance and see if we're below the threshold.
balance = balance - float(resp['credits_used'])
if not self.balance_good_to_continue(balance, threshold):
# If we don't have enough balance, stop.
return
def translate(self, instance, src_lang, src_field, dst_lang, dst_field):
# If gengosystem is disabled, we just return immediately. We
# can backfill later.
if not waffle.switch_is_active('gengosystem'):
return
text = getattr(instance, src_field)
metadata = {
'tier': self.gengo_tier,
'locale': instance.locale,
'length': len(text),
'body': text[:50].encode('utf-8')
}
gengo_api = FjordGengo()
# Guess the language. If we can't guess the language, then we
# don't create a GengoJob.
try:
lc_src = gengo_api.guess_language(text)
if lc_src not in gengo_api.get_languages():
raise GengoUnsupportedLanguage(
'unsupported language: {0}'.format(lc_src))
except GengoUnknownLanguage as exc:
# FIXME: This might be an indicator that this response is
# spam. At some point p, we can write code to account for
# that.
self.log_error(instance,
action='guess-language',
msg=unicode(exc),
metadata=metadata)
statsd.incr('translation.{0}.unknown'.format(self.name))
return
except GengoUnsupportedLanguage as exc:
# FIXME: This is a similar boat to GengoUnknownLanguage
# where for now, we're just going to ignore it because I'm
# not sure what to do about it and I'd like more data.
self.log_error(instance,
action='translate',
msg=unicode(exc),
metadata=metadata)
statsd.incr('translation.{0}.unsupported'.format(self.name))
return
# If the locale doesn't equal the guessed language, then
# that's interesting since the user is writing feedback in a
# language other than what the ui is showing. We want to log
# that for metrics purposes.
if not locale_equals_language(instance.locale, lc_src):
self.log_error(instance,
action='guess-language',
msg='locale "{0}" != guessed language "{1}"'.format(
instance.locale, lc_src),
metadata=metadata)
# If the source language is English, we just copy it over and
# we're done.
if locale_equals_language(dst_lang, lc_src):
setattr(instance, dst_field, text)
instance.save()
self.log_info(instance,
action='translate',
msg=u'lc_src == dst_lang, so we copy src to dst',
metadata=metadata)
return
if ((self.gengo_check_supported_machine_lc_dst
and lc_src in GENGO_UNSUPPORTED_MACHINE_LC_SRC)):
return
# If src/dst isn't a supported pair, log an issue for metrics
# purposes and move on.
if ((self.gengo_check_supported_language_pair
and (lc_src, dst_lang) not in gengo_api.get_language_pairs())):
self.log_error(
instance,
action='translate',
msg=u'(lc_src {0}, dst_lang {1}) not supported'.format(
lc_src, dst_lang),
metadata=metadata)
return
job = GengoJob(tier=self.gengo_tier,
content_object=instance,
src_lang=lc_src,
src_field=src_field,
dst_lang=dst_lang,
dst_field=dst_field)
job.save()
def gengo_translator_view(request):
"""Covers Gengo-specific translation system status"""
products = Product.objects.all()
balance = None
configured = False
gengo_languages = None
missing_prod_locales = None
outstanding = []
seven_days_of_orders = []
if settings.GENGO_PUBLIC_KEY and settings.GENGO_PRIVATE_KEY:
gengo_api = FjordGengo()
balance = gengo_api.get_balance()
configured = True
# Figure out the list of languages Gengo supports and relate them
# to PROD locales.
languages = sorted(gengo_api.get_languages(raw=True)['response'],
key=lambda item: item['lc'])
gengo_languages = []
PROD_LANG = settings.PROD_LANGUAGES
for lang in languages:
lang_lc = lang['lc']
prod_langs = [
item for item in PROD_LANG
if locale_equals_language(item, lang_lc)
]
prod_langs = ' '.join(prod_langs)
gengo_languages.append((lang_lc, lang['language'], prod_langs))
# Figure out the list of PROD locales that don't have a Gengo
# supported language. (Yes, this conflates locale with
# language which isn't great, but is good enough for now.)
languages = gengo_api.get_languages()
missing_prod_locales = []
for prod_lang in PROD_LANG:
langs = [
item for item in languages
if locale_equals_language(prod_lang, item)
]
if langs:
continue
missing_prod_locales.append(prod_lang)
# How many orders have we created/completed in the last week
# day-by-day?
seven_days = datetime.now() - timedelta(days=7)
orders = GengoOrder.objects.filter(created__gte=seven_days)
created_by_day = {}
for order in orders:
dt = order.created.strftime('%Y-%m-%d')
created_by_day.setdefault(dt, []).append(order)
orders = GengoOrder.objects.filter(completed__gte=seven_days)
completed_by_day = {}
for order in orders:
dt = order.completed.strftime('%Y-%m-%d')
completed_by_day.setdefault(dt, []).append(order)
# Get date labels in YYYY-mm-dd form for the last 7 days
days = [(datetime.now() - timedelta(days=i)).strftime('%Y-%m-%d')
for i in range(7)]
for day in sorted(days):
seven_days_of_orders.append((day, len(created_by_day.get(day, [])),
len(completed_by_day.get(day, []))))
outstanding = [{
'id': order.id,
'order_id': order.order_id,
'created': order.created,
'total_jobs': order.gengojob_set.all().count(),
'completed_jobs': order.completed_jobs().count(),
'outstanding_jobs': order.outstanding_jobs().count(),
} for order in GengoOrder.objects.filter(completed__isnull=True)]
return render(
request, 'admin/gengo_translator_view.html', {
'title': 'Translations - Gengo Maintenance',
'configured': configured,
'settings': settings,
'products': products,
'outstanding': outstanding,
'seven_days_of_orders': seven_days_of_orders,
'balance': balance,
'gengo_languages': gengo_languages,
'missing_prod_locales': missing_prod_locales,
})