def bulk_check_translations(self): """ Run checks on all changed translations from supported resources :return: primary keys of translations without warnings and errors. """ changed_pks = {t.pk for t in self.changed_translations} bulk_run_checks( Translation.objects .for_checks() .filter(pk__in=changed_pks) ) valid_translations = set( Translation.objects .filter( pk__in=changed_pks, errors__isnull=True, ) .values_list('pk', flat=True) .order_by('pk') ) return valid_translations
def test_bulk_run_checks( translation_compare_locales_warning, translation_compare_locales_error, translation_pontoon_error, ): warnings, errors = bulk_run_checks([ translation_compare_locales_warning, translation_pontoon_error, translation_compare_locales_error, ]) (cl_warning, ) = warnings p_error, cl_error = errors assert cl_warning.pk is not None assert cl_warning.library == "cl" assert cl_warning.message == "unknown escape sequence, \\q" assert cl_warning.translation == translation_compare_locales_warning assert cl_error.pk is not None assert cl_error.library == "cl" assert cl_error.message == "Found single %" assert cl_error.translation == translation_compare_locales_error assert p_error.pk is not None assert p_error.library == "p" assert p_error.message == "Empty translations are not allowed" assert p_error.translation == translation_pontoon_error
def test_bulk_run_checks( translation_compare_locales_warning, translation_compare_locales_error, translation_pontoon_error, ): warnings, errors = bulk_run_checks([ translation_compare_locales_warning, translation_pontoon_error, translation_compare_locales_error, ]) cl_warning, = warnings p_error, cl_error = errors assert cl_warning.pk is not None assert cl_warning.library == 'cl' assert cl_warning.message == 'unknown escape sequence, \q' assert cl_warning.translation == translation_compare_locales_warning assert cl_error.pk is not None assert cl_error.library == 'cl' assert cl_error.message == 'Found single %' assert cl_error.translation == translation_compare_locales_error assert p_error.pk is not None assert p_error.library == 'p' assert p_error.message == 'Empty translations are not allowed' assert p_error.translation == translation_pontoon_error
def translation_dtd_unapproved(): translation = TranslationFactory.create( string='Test Translation', approved=False, entity__key='test', entity__resource__format='dtd', entity__resource__path='test.dtd', ) bulk_run_checks([translation]) ProjectLocaleFactory.create( project=translation.entity.resource.project, locale=translation.locale, ) yield translation
def translation_dtd_unapproved(): translation = TranslationFactory.create( string="Test Translation", active=True, approved=False, entity__key="test", entity__resource__format="dtd", entity__resource__path="test.dtd", ) bulk_run_checks([translation]) ProjectLocaleFactory.create( project=translation.entity.resource.project, locale=translation.locale, ) yield translation
def test_bulk_run_checks_no_translations(): """ Don't generate any warnings and errors without translations """ assert bulk_run_checks([]) is None assert Warning.objects.count() == 0 assert Error.objects.count() == 0
def translation_dtd_invalid_unapproved(): # Provide invalid characters in translation to cause checks to fail translation = TranslationFactory.create( string='!@#$""\'', approved=False, entity__key='test', entity__resource__format='dtd', entity__resource__path='test.dtd', ) bulk_run_checks([translation]) ProjectLocaleFactory.create( project=translation.entity.resource.project, locale=translation.locale, ) yield translation
def check_translations(self, translations_pks): """ Run checks on translations :arg list[int] translations_pks: list of primary keys for translations that should be processed """ with transaction.atomic(): translations = (Translation.objects.for_checks().filter( pk__in=translations_pks)) warnings, errors = bulk_run_checks(translations) log.info( "Task: {}, Processed items: {}, Warnings: {}, Errors: {}".format( self.request.id, len(translations), len(warnings), len(errors), ))
def check_translations(self, translations_pks): """ Run checks on translations :arg list[int] translations_pks: list of primary keys for translations that should be processed """ start_time = time.time() with transaction.atomic(): translations = get_translations(pk__in=translations_pks) warnings, errors = bulk_run_checks(translations) log.info("Task[{}]: Processed items: {}, Warnings({}) Errors({}) in {}".format( self.request.id, len(translations), len(warnings), len(errors), time.time() - start_time ))
def check_translations(self, translations_pks): """ Run checks on translations :arg list[int] translations_pks: list of primary keys for translations that should be processed """ with transaction.atomic(): translations = ( Translation.objects .for_checks() .filter(pk__in=translations_pks) ) warnings, errors = bulk_run_checks(translations) log.info("Task: {}, Processed items: {}, Warnings: {}, Errors: {}".format( self.request.id, len(translations), len(warnings), len(errors), ))
def pretranslate(self, project_pk, locales=None, entities=None): """ Identifies strings without any translations and any suggestions. Engages TheAlgorithm (bug 1552796) to gather pretranslations. Stores pretranslations as suggestions (approved=False) to DB. :arg project_pk: the pk of the project to be pretranslated :arg Queryset locales: the locales for the project to be pretranslated :arg Queryset entites: the entities for the project to be pretranslated :returns: None """ project = Project.objects.get(pk=project_pk) log.info("Fetching pretranslations for project {} started".format( project.name)) if locales: locales = project.locales.filter(pk__in=locales) else: locales = project.locales locales = (locales.filter(project_locale__readonly=False).distinct(). prefetch_project_locale(project)) if not entities: entities = Entity.objects.filter( resource__project=project, obsolete=False, ).prefetch_related("resource") # get available TranslatedResource pairs tr_pairs = (TranslatedResource.objects.filter( resource__project=project, locale__in=locales, ).annotate(locale_resource=Concat( "locale_id", V("-"), "resource_id", output_field=CharField())).values_list("locale_resource", flat=True).distinct()) # Fetch all distinct locale-entity pairs for which translation exists translated_entities = (Translation.objects.filter( locale__in=locales, entity__in=entities, ).annotate(locale_entity=Concat( "locale_id", V("-"), "entity_id", output_field=CharField())).values_list("locale_entity", flat=True).distinct()) translated_entities = list(translated_entities) translations = [] # To keep track of changed TranslatedResources and their latest_translation tr_dict = {} tr_filter = [] index = -1 for locale in locales: log.info("Fetching pretranslations for locale {} started".format( locale.code)) for entity in entities: locale_entity = "{}-{}".format(locale.id, entity.id) locale_resource = "{}-{}".format(locale.id, entity.resource.id) if locale_entity in translated_entities or locale_resource not in tr_pairs: continue strings = get_translations(entity, locale) if not strings: continue for string, plural_form, user in strings: t = Translation( entity=entity, locale=locale, string=string, user=user, approved=False, fuzzy=True, active=True, plural_form=plural_form, ) index += 1 translations.append(t) if locale_resource not in tr_dict: tr_dict[locale_resource] = index # Add query for fetching respective TranslatedResource. tr_filter.append( Q(locale__id=locale.id) & Q(resource__id=entity.resource.id)) # Update the latest translation index tr_dict[locale_resource] = index log.info("Fetching pretranslations for locale {} done".format( locale.code)) if len(translations) == 0: return translations = Translation.objects.bulk_create(translations) # Run checks on all translations translation_pks = {translation.pk for translation in translations} bulk_run_checks( Translation.objects.for_checks().filter(pk__in=translation_pks)) # Mark translations as changed changed_entities = {} existing = ChangedEntityLocale.objects.values_list("entity", "locale").distinct() for t in translations: key = (t.entity.pk, t.locale.pk) # Remove duplicate changes to prevent unique constraint violation if key not in existing: changed_entities[key] = ChangedEntityLocale(entity=t.entity, locale=t.locale) ChangedEntityLocale.objects.bulk_create(changed_entities.values()) # Update latest activity and stats for changed instances. update_changed_instances(tr_filter, tr_dict, translations) log.info("Fetching pretranslations for project {} done".format( project.name))