def gengo_translator_view(request):
"""Covers Gengo-specific translation system status"""
products = Product.objects.all()
balance = None
configured = False
gengo_languages = None
missing_prod_locales = None
if settings.GENGO_PUBLIC_KEY and settings.GENGO_PRIVATE_KEY:
gengo_api = FjordGengo()
balance = gengo_api.get_balance()
configured = True
# Figure out the list of languages Gengo supports and relate them
# to PROD locales.
languages = sorted(gengo_api.get_languages(raw=True)['response'],
key=lambda item: item['lc'])
gengo_languages = []
PROD_LANG = settings.PROD_LANGUAGES
for lang in languages:
lang_lc = lang['lc']
prod_langs = [item for item in PROD_LANG
if locale_equals_language(item, lang_lc)]
prod_langs = ' '.join(prod_langs)
gengo_languages.append(
(lang_lc, lang['language'], prod_langs)
)
# Figure out the list of PROD locales that don't have a Gengo
# supported language. (Yes, this conflates locale with
# language which isn't great, but is good enough for now.)
languages = gengo_api.get_languages()
missing_prod_locales = []
for prod_lang in PROD_LANG:
langs = [item for item in languages
if locale_equals_language(prod_lang, item)]
if langs:
continue
missing_prod_locales.append(prod_lang)
# How many orders have we created/completed in the last week
# day-by-day?
seven_days = datetime.now() - timedelta(days=7)
orders = GengoOrder.objects.filter(created__gte=seven_days)
created_by_day = {}
for order in orders:
dt = order.created.strftime('%Y-%m-%d')
created_by_day.setdefault(dt, []).append(order)
orders = GengoOrder.objects.filter(completed__gte=seven_days)
completed_by_day = {}
for order in orders:
dt = order.completed.strftime('%Y-%m-%d')
completed_by_day.setdefault(dt, []).append(order)
# Get date labels in YYYY-mm-dd form for the last 7 days
days = [
(datetime.now() - timedelta(days=i)).strftime('%Y-%m-%d')
for i in range(7)
]
seven_days_of_orders = []
for day in sorted(days):
seven_days_of_orders.append(
(day,
len(created_by_day.get(day, [])),
len(completed_by_day.get(day, [])))
)
outstanding = [
{
'id': order.id,
'order_id': order.order_id,
'created': order.created,
'total_jobs': order.gengojob_set.all().count(),
'completed_jobs': order.completed_jobs().count(),
'outstanding_jobs': order.outstanding_jobs().count(),
}
for order in GengoOrder.objects.filter(completed__isnull=True)]
return render(request, 'admin/gengo_translator_view.html', {
'title': 'Translations - Gengo Maintenance',
'configured': configured,
'settings': settings,
'products': products,
'outstanding': outstanding,
'seven_days_of_orders': seven_days_of_orders,
'balance': balance,
'gengo_languages': gengo_languages,
'missing_prod_locales': missing_prod_locales,
})
def translate(self, instance, src_lang, src_field, dst_lang, dst_field):
# If gengosystem is disabled, we just return immediately. We
# can backfill later.
if not waffle.switch_is_active('gengosystem'):
return
text = getattr(instance, src_field)
metadata = {
'tier': self.gengo_tier,
'locale': instance.locale,
'length': len(text),
'body': text[:50].encode('utf-8')
}
gengo_api = FjordGengo()
# Guess the language. If we can't guess the language, then we
# don't create a GengoJob.
try:
lc_src = gengo_api.guess_language(text)
if lc_src not in gengo_api.get_languages():
raise GengoUnsupportedLanguage(
'unsupported language: {0}'.format(lc_src))
except GengoUnknownLanguage as exc:
# FIXME: This might be an indicator that this response is
# spam. At some point p, we can write code to account for
# that.
self.log_error(instance,
action='guess-language',
msg=unicode(exc),
metadata=metadata)
statsd.incr('translation.{0}.unknown'.format(self.name))
return
except GengoUnsupportedLanguage as exc:
# FIXME: This is a similar boat to GengoUnknownLanguage
# where for now, we're just going to ignore it because I'm
# not sure what to do about it and I'd like more data.
self.log_error(instance,
action='translate',
msg=unicode(exc),
metadata=metadata)
statsd.incr('translation.{0}.unsupported'.format(self.name))
return
# If the locale doesn't equal the guessed language, then
# that's interesting since the user is writing feedback in a
# language other than what the ui is showing. We want to log
# that for metrics purposes.
if not locale_equals_language(instance.locale, lc_src):
self.log_error(instance,
action='guess-language',
msg='locale "{0}" != guessed language "{1}"'.format(
instance.locale, lc_src),
metadata=metadata)
# If the source language is English, we just copy it over and
# we're done.
if locale_equals_language(dst_lang, lc_src):
setattr(instance, dst_field, text)
instance.save()
self.log_info(instance,
action='translate',
msg=u'lc_src == dst_lang, so we copy src to dst',
metadata=metadata)
return
if ((self.gengo_check_supported_machine_lc_dst
and lc_src in GENGO_UNSUPPORTED_MACHINE_LC_SRC)):
return
# If src/dst isn't a supported pair, log an issue for metrics
# purposes and move on.
if ((self.gengo_check_supported_language_pair
and (lc_src, dst_lang) not in gengo_api.get_language_pairs())):
self.log_error(
instance,
action='translate',
msg=u'(lc_src {0}, dst_lang {1}) not supported'.format(
lc_src, dst_lang),
metadata=metadata)
return
job = GengoJob(tier=self.gengo_tier,
content_object=instance,
src_lang=lc_src,
src_field=src_field,
dst_lang=dst_lang,
dst_field=dst_field)
job.save()
def translate(self, instance, src_lang, src_field, dst_lang, dst_field):
# If gengosystem is disabled, we just return immediately. We
# can backfill later.
if not waffle.switch_is_active('gengosystem'):
return
text = getattr(instance, src_field)
metadata = {
'tier': self.gengo_tier,
'locale': instance.locale,
'length': len(text),
'body': text[:50].encode('utf-8')
}
gengo_api = FjordGengo()
# Guess the language. If we can't guess the language, then we
# don't create a GengoJob.
try:
lc_src = gengo_api.guess_language(text)
if lc_src not in gengo_api.get_languages():
raise GengoUnsupportedLanguage(
'unsupported language: {0}'.format(lc_src))
except GengoUnknownLanguage as exc:
# FIXME: This might be an indicator that this response is
# spam. At some point p, we can write code to account for
# that.
self.log_error(instance, action='guess-language', msg=unicode(exc),
metadata=metadata)
statsd.incr('translation.{0}.unknown'.format(self.name))
return
except GengoUnsupportedLanguage as exc:
# FIXME: This is a similar boat to GengoUnknownLanguage
# where for now, we're just going to ignore it because I'm
# not sure what to do about it and I'd like more data.
self.log_error(instance, action='translate', msg=unicode(exc),
metadata=metadata)
statsd.incr('translation.{0}.unsupported'.format(self.name))
return
# If the locale doesn't equal the guessed language, then
# that's interesting since the user is writing feedback in a
# language other than what the ui is showing. We want to log
# that for metrics purposes.
if not locale_equals_language(instance.locale, lc_src):
self.log_error(
instance,
action='guess-language',
msg='locale "{0}" != guessed language "{1}"'.format(
instance.locale, lc_src),
metadata=metadata)
# If the source language is English, we just copy it over and
# we're done.
if locale_equals_language(dst_lang, lc_src):
setattr(instance, dst_field, text)
instance.save()
self.log_info(
instance, action='translate',
msg=u'lc_src == dst_lang, so we copy src to dst',
metadata=metadata)
return
if ((self.gengo_check_supported_machine_lc_dst
and lc_src in GENGO_UNSUPPORTED_MACHINE_LC_SRC)):
return
# If src/dst isn't a supported pair, log an issue for metrics
# purposes and move on.
if ((self.gengo_check_supported_language_pair
and (lc_src, dst_lang) not in gengo_api.get_language_pairs())):
self.log_error(
instance, action='translate',
msg=u'(lc_src {0}, dst_lang {1}) not supported'.format(
lc_src, dst_lang),
metadata=metadata)
return
job = GengoJob(
tier=self.gengo_tier,
content_object=instance,
src_lang=lc_src,
src_field=src_field,
dst_lang=dst_lang,
dst_field=dst_field
)
job.save()
def gengo_translator_view(request):
"""Covers Gengo-specific translation system status"""
products = Product.objects.all()
balance = None
configured = False
gengo_languages = None
missing_prod_locales = None
outstanding = []
seven_days_of_orders = []
if settings.GENGO_PUBLIC_KEY and settings.GENGO_PRIVATE_KEY:
gengo_api = FjordGengo()
balance = gengo_api.get_balance()
configured = True
# Figure out the list of languages Gengo supports and relate them
# to PROD locales.
languages = sorted(gengo_api.get_languages(raw=True)['response'],
key=lambda item: item['lc'])
gengo_languages = []
PROD_LANG = settings.PROD_LANGUAGES
for lang in languages:
lang_lc = lang['lc']
prod_langs = [
item for item in PROD_LANG
if locale_equals_language(item, lang_lc)
]
prod_langs = ' '.join(prod_langs)
gengo_languages.append((lang_lc, lang['language'], prod_langs))
# Figure out the list of PROD locales that don't have a Gengo
# supported language. (Yes, this conflates locale with
# language which isn't great, but is good enough for now.)
languages = gengo_api.get_languages()
missing_prod_locales = []
for prod_lang in PROD_LANG:
langs = [
item for item in languages
if locale_equals_language(prod_lang, item)
]
if langs:
continue
missing_prod_locales.append(prod_lang)
# How many orders have we created/completed in the last week
# day-by-day?
seven_days = datetime.now() - timedelta(days=7)
orders = GengoOrder.objects.filter(created__gte=seven_days)
created_by_day = {}
for order in orders:
dt = order.created.strftime('%Y-%m-%d')
created_by_day.setdefault(dt, []).append(order)
orders = GengoOrder.objects.filter(completed__gte=seven_days)
completed_by_day = {}
for order in orders:
dt = order.completed.strftime('%Y-%m-%d')
completed_by_day.setdefault(dt, []).append(order)
# Get date labels in YYYY-mm-dd form for the last 7 days
days = [(datetime.now() - timedelta(days=i)).strftime('%Y-%m-%d')
for i in range(7)]
for day in sorted(days):
seven_days_of_orders.append((day, len(created_by_day.get(day, [])),
len(completed_by_day.get(day, []))))
outstanding = [{
'id': order.id,
'order_id': order.order_id,
'created': order.created,
'total_jobs': order.gengojob_set.all().count(),
'completed_jobs': order.completed_jobs().count(),
'outstanding_jobs': order.outstanding_jobs().count(),
} for order in GengoOrder.objects.filter(completed__isnull=True)]
return render(
request, 'admin/gengo_translator_view.html', {
'title': 'Translations - Gengo Maintenance',
'configured': configured,
'settings': settings,
'products': products,
'outstanding': outstanding,
'seven_days_of_orders': seven_days_of_orders,
'balance': balance,
'gengo_languages': gengo_languages,
'missing_prod_locales': missing_prod_locales,
})