Пример #1
0
def gengo_translator_view(request):
    """Covers Gengo-specific translation system status"""
    products = Product.objects.all()
    balance = None
    configured = False
    gengo_languages = None
    missing_prod_locales = None

    if settings.GENGO_PUBLIC_KEY and settings.GENGO_PRIVATE_KEY:
        gengo_api = FjordGengo()
        balance = gengo_api.get_balance()
        configured = True

        # Figure out the list of languages Gengo supports and relate them
        # to PROD locales.
        languages = sorted(gengo_api.get_languages(raw=True)['response'],
                           key=lambda item: item['lc'])
        gengo_languages = []
        PROD_LANG = settings.PROD_LANGUAGES
        for lang in languages:
            lang_lc = lang['lc']
            prod_langs = [item for item in PROD_LANG
                          if locale_equals_language(item, lang_lc)]
            prod_langs = ' '.join(prod_langs)

            gengo_languages.append(
                (lang_lc, lang['language'], prod_langs)
            )

        # Figure out the list of PROD locales that don't have a Gengo
        # supported language. (Yes, this conflates locale with
        # language which isn't great, but is good enough for now.)
        languages = gengo_api.get_languages()
        missing_prod_locales = []
        for prod_lang in PROD_LANG:
            langs = [item for item in languages
                     if locale_equals_language(prod_lang, item)]
            if langs:
                continue
            missing_prod_locales.append(prod_lang)

        # How many orders have we created/completed in the last week
        # day-by-day?
        seven_days = datetime.now() - timedelta(days=7)

        orders = GengoOrder.objects.filter(created__gte=seven_days)
        created_by_day = {}
        for order in orders:
            dt = order.created.strftime('%Y-%m-%d')
            created_by_day.setdefault(dt, []).append(order)

        orders = GengoOrder.objects.filter(completed__gte=seven_days)
        completed_by_day = {}
        for order in orders:
            dt = order.completed.strftime('%Y-%m-%d')
            completed_by_day.setdefault(dt, []).append(order)

        # Get date labels in YYYY-mm-dd form for the last 7 days
        days = [
            (datetime.now() - timedelta(days=i)).strftime('%Y-%m-%d')
            for i in range(7)
        ]

        seven_days_of_orders = []
        for day in sorted(days):
            seven_days_of_orders.append(
                (day,
                 len(created_by_day.get(day, [])),
                 len(completed_by_day.get(day, [])))
            )

        outstanding = [
            {
                'id': order.id,
                'order_id': order.order_id,
                'created': order.created,
                'total_jobs': order.gengojob_set.all().count(),
                'completed_jobs': order.completed_jobs().count(),
                'outstanding_jobs': order.outstanding_jobs().count(),
            }
            for order in GengoOrder.objects.filter(completed__isnull=True)]

    return render(request, 'admin/gengo_translator_view.html', {
        'title': 'Translations - Gengo Maintenance',
        'configured': configured,
        'settings': settings,
        'products': products,
        'outstanding': outstanding,
        'seven_days_of_orders': seven_days_of_orders,
        'balance': balance,
        'gengo_languages': gengo_languages,
        'missing_prod_locales': missing_prod_locales,
    })
Пример #2
0
    def translate(self, instance, src_lang, src_field, dst_lang, dst_field):
        # If gengosystem is disabled, we just return immediately. We
        # can backfill later.
        if not waffle.switch_is_active('gengosystem'):
            return

        text = getattr(instance, src_field)
        metadata = {
            'tier': self.gengo_tier,
            'locale': instance.locale,
            'length': len(text),
            'body': text[:50].encode('utf-8')
        }

        gengo_api = FjordGengo()

        # Guess the language. If we can't guess the language, then we
        # don't create a GengoJob.
        try:
            lc_src = gengo_api.guess_language(text)
            if lc_src not in gengo_api.get_languages():
                raise GengoUnsupportedLanguage(
                    'unsupported language: {0}'.format(lc_src))

        except GengoUnknownLanguage as exc:
            # FIXME: This might be an indicator that this response is
            # spam. At some point p, we can write code to account for
            # that.
            self.log_error(instance,
                           action='guess-language',
                           msg=unicode(exc),
                           metadata=metadata)
            statsd.incr('translation.{0}.unknown'.format(self.name))
            return

        except GengoUnsupportedLanguage as exc:
            # FIXME: This is a similar boat to GengoUnknownLanguage
            # where for now, we're just going to ignore it because I'm
            # not sure what to do about it and I'd like more data.
            self.log_error(instance,
                           action='translate',
                           msg=unicode(exc),
                           metadata=metadata)
            statsd.incr('translation.{0}.unsupported'.format(self.name))
            return

        # If the locale doesn't equal the guessed language, then
        # that's interesting since the user is writing feedback in a
        # language other than what the ui is showing. We want to log
        # that for metrics purposes.
        if not locale_equals_language(instance.locale, lc_src):
            self.log_error(instance,
                           action='guess-language',
                           msg='locale "{0}" != guessed language "{1}"'.format(
                               instance.locale, lc_src),
                           metadata=metadata)

        # If the source language is English, we just copy it over and
        # we're done.
        if locale_equals_language(dst_lang, lc_src):
            setattr(instance, dst_field, text)
            instance.save()
            self.log_info(instance,
                          action='translate',
                          msg=u'lc_src == dst_lang, so we copy src to dst',
                          metadata=metadata)
            return

        if ((self.gengo_check_supported_machine_lc_dst
             and lc_src in GENGO_UNSUPPORTED_MACHINE_LC_SRC)):
            return

        # If src/dst isn't a supported pair, log an issue for metrics
        # purposes and move on.
        if ((self.gengo_check_supported_language_pair
             and (lc_src, dst_lang) not in gengo_api.get_language_pairs())):
            self.log_error(
                instance,
                action='translate',
                msg=u'(lc_src {0}, dst_lang {1}) not supported'.format(
                    lc_src, dst_lang),
                metadata=metadata)
            return

        job = GengoJob(tier=self.gengo_tier,
                       content_object=instance,
                       src_lang=lc_src,
                       src_field=src_field,
                       dst_lang=dst_lang,
                       dst_field=dst_field)
        job.save()
Пример #3
0
    def translate(self, instance, src_lang, src_field, dst_lang, dst_field):
        # If gengosystem is disabled, we just return immediately. We
        # can backfill later.
        if not waffle.switch_is_active('gengosystem'):
            return

        text = getattr(instance, src_field)
        metadata = {
            'tier': self.gengo_tier,
            'locale': instance.locale,
            'length': len(text),
            'body': text[:50].encode('utf-8')
        }

        gengo_api = FjordGengo()

        # Guess the language. If we can't guess the language, then we
        # don't create a GengoJob.
        try:
            lc_src = gengo_api.guess_language(text)
            if lc_src not in gengo_api.get_languages():
                raise GengoUnsupportedLanguage(
                    'unsupported language: {0}'.format(lc_src))

        except GengoUnknownLanguage as exc:
            # FIXME: This might be an indicator that this response is
            # spam. At some point p, we can write code to account for
            # that.
            self.log_error(instance, action='guess-language', msg=unicode(exc),
                           metadata=metadata)
            statsd.incr('translation.{0}.unknown'.format(self.name))
            return

        except GengoUnsupportedLanguage as exc:
            # FIXME: This is a similar boat to GengoUnknownLanguage
            # where for now, we're just going to ignore it because I'm
            # not sure what to do about it and I'd like more data.
            self.log_error(instance, action='translate', msg=unicode(exc),
                           metadata=metadata)
            statsd.incr('translation.{0}.unsupported'.format(self.name))
            return

        # If the locale doesn't equal the guessed language, then
        # that's interesting since the user is writing feedback in a
        # language other than what the ui is showing. We want to log
        # that for metrics purposes.
        if not locale_equals_language(instance.locale, lc_src):
            self.log_error(
                instance,
                action='guess-language',
                msg='locale "{0}" != guessed language "{1}"'.format(
                    instance.locale, lc_src),
                metadata=metadata)

        # If the source language is English, we just copy it over and
        # we're done.
        if locale_equals_language(dst_lang, lc_src):
            setattr(instance, dst_field, text)
            instance.save()
            self.log_info(
                instance, action='translate',
                msg=u'lc_src == dst_lang, so we copy src to dst',
                metadata=metadata)
            return

        if ((self.gengo_check_supported_machine_lc_dst
             and lc_src in GENGO_UNSUPPORTED_MACHINE_LC_SRC)):
            return

        # If src/dst isn't a supported pair, log an issue for metrics
        # purposes and move on.
        if ((self.gengo_check_supported_language_pair
             and (lc_src, dst_lang) not in gengo_api.get_language_pairs())):
            self.log_error(
                instance, action='translate',
                msg=u'(lc_src {0}, dst_lang {1}) not supported'.format(
                    lc_src, dst_lang),
                metadata=metadata)
            return

        job = GengoJob(
            tier=self.gengo_tier,
            content_object=instance,
            src_lang=lc_src,
            src_field=src_field,
            dst_lang=dst_lang,
            dst_field=dst_field
        )
        job.save()
Пример #4
0
def gengo_translator_view(request):
    """Covers Gengo-specific translation system status"""
    products = Product.objects.all()
    balance = None
    configured = False
    gengo_languages = None
    missing_prod_locales = None
    outstanding = []
    seven_days_of_orders = []

    if settings.GENGO_PUBLIC_KEY and settings.GENGO_PRIVATE_KEY:
        gengo_api = FjordGengo()
        balance = gengo_api.get_balance()
        configured = True

        # Figure out the list of languages Gengo supports and relate them
        # to PROD locales.
        languages = sorted(gengo_api.get_languages(raw=True)['response'],
                           key=lambda item: item['lc'])
        gengo_languages = []
        PROD_LANG = settings.PROD_LANGUAGES
        for lang in languages:
            lang_lc = lang['lc']
            prod_langs = [
                item for item in PROD_LANG
                if locale_equals_language(item, lang_lc)
            ]
            prod_langs = ' '.join(prod_langs)

            gengo_languages.append((lang_lc, lang['language'], prod_langs))

        # Figure out the list of PROD locales that don't have a Gengo
        # supported language. (Yes, this conflates locale with
        # language which isn't great, but is good enough for now.)
        languages = gengo_api.get_languages()
        missing_prod_locales = []
        for prod_lang in PROD_LANG:
            langs = [
                item for item in languages
                if locale_equals_language(prod_lang, item)
            ]
            if langs:
                continue
            missing_prod_locales.append(prod_lang)

        # How many orders have we created/completed in the last week
        # day-by-day?
        seven_days = datetime.now() - timedelta(days=7)

        orders = GengoOrder.objects.filter(created__gte=seven_days)
        created_by_day = {}
        for order in orders:
            dt = order.created.strftime('%Y-%m-%d')
            created_by_day.setdefault(dt, []).append(order)

        orders = GengoOrder.objects.filter(completed__gte=seven_days)
        completed_by_day = {}
        for order in orders:
            dt = order.completed.strftime('%Y-%m-%d')
            completed_by_day.setdefault(dt, []).append(order)

        # Get date labels in YYYY-mm-dd form for the last 7 days
        days = [(datetime.now() - timedelta(days=i)).strftime('%Y-%m-%d')
                for i in range(7)]

        for day in sorted(days):
            seven_days_of_orders.append((day, len(created_by_day.get(day, [])),
                                         len(completed_by_day.get(day, []))))

        outstanding = [{
            'id': order.id,
            'order_id': order.order_id,
            'created': order.created,
            'total_jobs': order.gengojob_set.all().count(),
            'completed_jobs': order.completed_jobs().count(),
            'outstanding_jobs': order.outstanding_jobs().count(),
        } for order in GengoOrder.objects.filter(completed__isnull=True)]

    return render(
        request, 'admin/gengo_translator_view.html', {
            'title': 'Translations - Gengo Maintenance',
            'configured': configured,
            'settings': settings,
            'products': products,
            'outstanding': outstanding,
            'seven_days_of_orders': seven_days_of_orders,
            'balance': balance,
            'gengo_languages': gengo_languages,
            'missing_prod_locales': missing_prod_locales,
        })