def run_daily_activities(self):
# If gengosystem is disabled, we don't want to do anything.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
balance = gengo_api.get_balance()
threshold = settings.GENGO_ACCOUNT_BALANCE_THRESHOLD
if threshold < balance < (2 * threshold):
mail_admins(
subject='Warning: Gengo account balance {0} < {1}'.format(
balance, 2 * threshold),
message=wrap_with_paragraphs(
'Dear mom,'
'\n\n'
'Translations are the fab. Running low on funds. Send '
'more money when you get a chance.'
'\n\n'
'Love,'
'\n\n'
'Fjord McGengo'
)
)
def run_daily_activities(self):
# If gengosystem is disabled, we don't want to do anything.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
balance = gengo_api.get_balance()
threshold = settings.GENGO_ACCOUNT_BALANCE_THRESHOLD
if threshold < balance < (2 * threshold):
mail_admins(
subject='Warning: Gengo account balance {0} < {1}'.format(
balance, 2 * threshold),
message=wrap_with_paragraphs(
'Dear mom,'
'\n\n'
'Translations are the fab. Running low on funds. Send '
'more money when you get a chance.'
'\n\n'
'Love,'
'\n\n'
'Fjord McGengo'
)
)
def gengo_translator_view(request):
gengo_machiners = Product.objects.filter(translation_system='gengo_machine')
balance = None
configured = False
if settings.GENGO_PUBLIC_KEY and settings.GENGO_PRIVATE_KEY:
gengo_api = FjordGengo()
balance = gengo_api.get_balance()
configured = True
return render(request, 'admin/gengo_translator_view.html', {
'configured': configured,
'settings': settings,
'gengo_machiners': gengo_machiners,
'balance': balance
})
def gengo_translator_view(request):
gengo_machiners = Product.objects.filter(
translation_system='gengo_machine')
balance = None
configured = False
if settings.GENGO_PUBLIC_KEY and settings.GENGO_PRIVATE_KEY:
gengo_api = FjordGengo()
balance = gengo_api.get_balance()
configured = True
return render(
request, 'admin/gengo_translator_view.html', {
'configured': configured,
'settings': settings,
'gengo_machiners': gengo_machiners,
'balance': balance
})
def run_daily_activities(self):
# If gengosystem is disabled, we don't want to do anything.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
balance = gengo_api.get_balance()
threshold = settings.GENGO_ACCOUNT_BALANCE_THRESHOLD
if threshold < balance < (2 * threshold):
recipients = get_recipients('gengo_balance')
subject = 'Warning: Gengo account balance {0} < {1}'.format(
balance, 2 * threshold)
body = wrap_with_paragraphs(dedent("""\
Dear mom,
Translations are the fab. Running low on funds. Send
more money when you get a chance.
Love,
Fjord McGengo
"""))
if recipients:
send_mail(
subject=subject,
message=body,
recipient_list=recipients,
from_email=settings.SERVER_EMAIL
)
else:
log.info('No recipients for "%s"\n%s\n%s' % (
'gengo_balance', subject, body))
def run_daily_activities(self):
# If gengosystem is disabled, we don't want to do anything.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
balance = gengo_api.get_balance()
threshold = settings.GENGO_ACCOUNT_BALANCE_THRESHOLD
if threshold < balance < (2 * threshold):
recipients = get_recipients('gengo_balance')
subject = 'Warning: Gengo account balance {0} < {1}'.format(
balance, 2 * threshold)
body = wrap_with_paragraphs(
dedent("""\
Dear mom,
Translations are the fab. Running low on funds. Send
more money when you get a chance.
Love,
Fjord McGengo
"""))
if recipients:
send_mail(subject=subject,
message=body,
recipient_list=recipients,
from_email=settings.SERVER_EMAIL)
else:
log.info('No recipients for "%s"\n%s\n%s' %
('gengo_balance', subject, body))
def gengo_translator_view(request):
"""Covers Gengo-specific translation system status"""
products = Product.objects.all()
balance = None
configured = False
gengo_languages = None
missing_prod_locales = None
if settings.GENGO_PUBLIC_KEY and settings.GENGO_PRIVATE_KEY:
gengo_api = FjordGengo()
balance = gengo_api.get_balance()
configured = True
# Figure out the list of languages Gengo supports and relate them
# to PROD locales.
languages = sorted(gengo_api.get_languages(raw=True)['response'],
key=lambda item: item['lc'])
gengo_languages = []
PROD_LANG = settings.PROD_LANGUAGES
for lang in languages:
lang_lc = lang['lc']
prod_langs = [item for item in PROD_LANG
if locale_equals_language(item, lang_lc)]
prod_langs = ' '.join(prod_langs)
gengo_languages.append(
(lang_lc, lang['language'], prod_langs)
)
# Figure out the list of PROD locales that don't have a Gengo
# supported language. (Yes, this conflates locale with
# language which isn't great, but is good enough for now.)
languages = gengo_api.get_languages()
missing_prod_locales = []
for prod_lang in PROD_LANG:
langs = [item for item in languages
if locale_equals_language(prod_lang, item)]
if langs:
continue
missing_prod_locales.append(prod_lang)
# How many orders have we created/completed in the last week
# day-by-day?
seven_days = datetime.now() - timedelta(days=7)
orders = GengoOrder.objects.filter(created__gte=seven_days)
created_by_day = {}
for order in orders:
dt = order.created.strftime('%Y-%m-%d')
created_by_day.setdefault(dt, []).append(order)
orders = GengoOrder.objects.filter(completed__gte=seven_days)
completed_by_day = {}
for order in orders:
dt = order.completed.strftime('%Y-%m-%d')
completed_by_day.setdefault(dt, []).append(order)
# Get date labels in YYYY-mm-dd form for the last 7 days
days = [
(datetime.now() - timedelta(days=i)).strftime('%Y-%m-%d')
for i in range(7)
]
seven_days_of_orders = []
for day in sorted(days):
seven_days_of_orders.append(
(day,
len(created_by_day.get(day, [])),
len(completed_by_day.get(day, [])))
)
outstanding = [
{
'id': order.id,
'order_id': order.order_id,
'created': order.created,
'total_jobs': order.gengojob_set.all().count(),
'completed_jobs': order.completed_jobs().count(),
'outstanding_jobs': order.outstanding_jobs().count(),
}
for order in GengoOrder.objects.filter(completed__isnull=True)]
return render(request, 'admin/gengo_translator_view.html', {
'title': 'Translations - Gengo Maintenance',
'configured': configured,
'settings': settings,
'products': products,
'outstanding': outstanding,
'seven_days_of_orders': seven_days_of_orders,
'balance': balance,
'gengo_languages': gengo_languages,
'missing_prod_locales': missing_prod_locales,
})
def push_translations(self):
# If gengosystem is disabled, we just return immediately. We
# can backfill later.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
if self.gengo_watch_balance:
balance = gengo_api.get_balance()
threshold = settings.GENGO_ACCOUNT_BALANCE_THRESHOLD
# statsd the balance so we can track it with graphite.
statsd.gauge('translation.gengo.balance', balance)
if not self.balance_good_to_continue(balance, threshold):
# If we don't have enough balance, stop.
return
# Create language buckets for the jobs for this translator.
# We bucket by language because this makes it easier for a
# single Gengo translator to translate all the jobs in an
# order.
jobs = GengoJob.objects.filter(
tier=self.gengo_tier, status=STATUS_CREATED)
lang_buckets = {}
for job in jobs:
lang_buckets.setdefault(job.src_lang, []).append(job)
# For each bucket, assemble an order and post it.
for lang, jobs in lang_buckets.items():
batch = []
for job in jobs:
batch.append({
'id': job.id,
'lc_src': job.src_lang,
'lc_dst': job.dst_lang,
'tier': self.gengo_tier,
'text': getattr(job.content_object, job.src_field),
'unique_id': job.unique_id
})
try:
resp = gengo_api.translate_bulk(batch)
except GengoError as exc:
self.log_error(
instance=None, action='push-translations',
msg=unicode(exc),
metadata={
'batch': batch
})
continue
# We should have an `order_id` at this point, so we create a
# GengoOrder with it.
order = GengoOrder(order_id=resp['order_id'])
order.save()
order.log('created', metadata={'response': resp})
# Update all the jobs in the order.
for job in jobs:
job.assign_to_order(order)
if self.gengo_watch_balance:
# Update the balance and see if we're below the threshold.
balance = balance - float(resp['credits_used'])
if not self.balance_good_to_continue(balance, threshold):
# If we don't have enough balance, stop.
return
def push_translations(self):
# If gengosystem is disabled, we just return immediately. We
# can backfill later.
if not waffle.switch_is_active('gengosystem'):
return
gengo_api = FjordGengo()
if not gengo_api.is_configured():
# If Gengo isn't configured, then we drop out here rather
# than raise a GengoConfig error.
return
if self.gengo_watch_balance:
balance = gengo_api.get_balance()
threshold = settings.GENGO_ACCOUNT_BALANCE_THRESHOLD
# statsd the balance so we can track it with graphite.
statsd.gauge('translation.gengo.balance', balance)
if not self.balance_good_to_continue(balance, threshold):
# If we don't have enough balance, stop.
return
# Create language buckets for the jobs for this translator.
# We bucket by language because this makes it easier for a
# single Gengo translator to translate all the jobs in an
# order.
jobs = GengoJob.objects.filter(tier=self.gengo_tier,
status=STATUS_CREATED)
lang_buckets = {}
for job in jobs:
lang_buckets.setdefault(job.src_lang, []).append(job)
# For each bucket, assemble an order and post it.
for lang, jobs in lang_buckets.items():
batch = []
for job in jobs:
batch.append({
'id': job.id,
'lc_src': job.src_lang,
'lc_dst': job.dst_lang,
'tier': self.gengo_tier,
'text': getattr(job.content_object, job.src_field),
'unique_id': job.unique_id
})
try:
resp = gengo_api.translate_bulk(batch)
except GengoError as exc:
self.log_error(instance=None,
action='push-translations',
msg=unicode(exc),
metadata={'batch': batch})
continue
# We should have an `order_id` at this point, so we create a
# GengoOrder with it.
order = GengoOrder(order_id=resp['order_id'])
order.save()
order.log('created', metadata={'response': resp})
# Update all the jobs in the order.
for job in jobs:
job.assign_to_order(order)
if self.gengo_watch_balance:
# Update the balance and see if we're below the threshold.
balance = balance - float(resp['credits_used'])
if not self.balance_good_to_continue(balance, threshold):
# If we don't have enough balance, stop.
return
def gengo_translator_view(request):
"""Covers Gengo-specific translation system status"""
products = Product.objects.all()
balance = None
configured = False
gengo_languages = None
missing_prod_locales = None
outstanding = []
seven_days_of_orders = []
if settings.GENGO_PUBLIC_KEY and settings.GENGO_PRIVATE_KEY:
gengo_api = FjordGengo()
balance = gengo_api.get_balance()
configured = True
# Figure out the list of languages Gengo supports and relate them
# to PROD locales.
languages = sorted(gengo_api.get_languages(raw=True)['response'],
key=lambda item: item['lc'])
gengo_languages = []
PROD_LANG = settings.PROD_LANGUAGES
for lang in languages:
lang_lc = lang['lc']
prod_langs = [
item for item in PROD_LANG
if locale_equals_language(item, lang_lc)
]
prod_langs = ' '.join(prod_langs)
gengo_languages.append((lang_lc, lang['language'], prod_langs))
# Figure out the list of PROD locales that don't have a Gengo
# supported language. (Yes, this conflates locale with
# language which isn't great, but is good enough for now.)
languages = gengo_api.get_languages()
missing_prod_locales = []
for prod_lang in PROD_LANG:
langs = [
item for item in languages
if locale_equals_language(prod_lang, item)
]
if langs:
continue
missing_prod_locales.append(prod_lang)
# How many orders have we created/completed in the last week
# day-by-day?
seven_days = datetime.now() - timedelta(days=7)
orders = GengoOrder.objects.filter(created__gte=seven_days)
created_by_day = {}
for order in orders:
dt = order.created.strftime('%Y-%m-%d')
created_by_day.setdefault(dt, []).append(order)
orders = GengoOrder.objects.filter(completed__gte=seven_days)
completed_by_day = {}
for order in orders:
dt = order.completed.strftime('%Y-%m-%d')
completed_by_day.setdefault(dt, []).append(order)
# Get date labels in YYYY-mm-dd form for the last 7 days
days = [(datetime.now() - timedelta(days=i)).strftime('%Y-%m-%d')
for i in range(7)]
for day in sorted(days):
seven_days_of_orders.append((day, len(created_by_day.get(day, [])),
len(completed_by_day.get(day, []))))
outstanding = [{
'id': order.id,
'order_id': order.order_id,
'created': order.created,
'total_jobs': order.gengojob_set.all().count(),
'completed_jobs': order.completed_jobs().count(),
'outstanding_jobs': order.outstanding_jobs().count(),
} for order in GengoOrder.objects.filter(completed__isnull=True)]
return render(
request, 'admin/gengo_translator_view.html', {
'title': 'Translations - Gengo Maintenance',
'configured': configured,
'settings': settings,
'products': products,
'outstanding': outstanding,
'seven_days_of_orders': seven_days_of_orders,
'balance': balance,
'gengo_languages': gengo_languages,
'missing_prod_locales': missing_prod_locales,
})