def run_checks(self, same_state=True, same_content=True): """Update checks for this unit.""" was_change = False has_checks = None if self.translation.is_template: checks_to_run = {} else: checks_to_run = CHECKS.data src = self.get_source_plurals() tgt = self.get_target_plurals() content_hash = self.content_hash project = self.translation.component.project language = self.translation.language old_checks = set(self.checks(True)) create = [] # Run all target checks for check, check_obj in checks_to_run.items(): if self.is_batch_update and check_obj.batch_update: old_checks.discard(check) continue if check_obj.check_target(src, tgt, self): if check in old_checks: # We already have this check old_checks.remove(check) else: # Create new check create.append(Check( content_hash=content_hash, project=project, language=language, ignore=False, check=check, )) was_change = True has_checks = True if create: Check.objects.bulk_create_ignore(create) # Delete no longer failing checks if old_checks: was_change = True Check.objects.filter( content_hash=content_hash, project=project, language=language, check__in=old_checks ).delete() # Update failing checks flag if not self.is_batch_update and (was_change or not same_content): self.update_has_failing_check(was_change, has_checks)
def run_checks(self, unit=None, project=None, batch=False): """Update checks for this unit.""" if unit is None: try: unit = self.units[0] except IndexError: return content_hash = unit.content_hash src = unit.get_source_plurals() if project is None: project = self.component.project # Fetch old checks if self.component.checks_cache is not None: old_checks = self.component.checks_cache.get((content_hash, None), []) else: old_checks = set( Check.objects.filter(content_hash=content_hash, project=project, language=None).values_list('check', flat=True)) create = [] # Run all source checks for check, check_obj in CHECKS.items(): if batch and check_obj.batch_update: if check in old_checks: # Do not remove batch checks in batch processing old_checks.remove(check) continue if check_obj.source and check_obj.check_source(src, unit): if check in old_checks: # We already have this check old_checks.remove(check) else: # Create new check create.append( Check(content_hash=content_hash, project=project, language=None, ignore=False, check=check)) if create: Check.objects.bulk_create_ignore(create) # Remove stale checks if old_checks: Check.objects.filter(content_hash=content_hash, project=project, language=None, check__in=old_checks).delete()
def run_checks(self, same_state=True, same_content=True): """Update checks for this unit.""" was_change = False has_checks = None is_source = self.translation.is_source src = self.get_source_plurals() tgt = self.get_target_plurals() old_checks = set(self.check_set.values_list('check', flat=True)) create = [] # Run all checks for check, check_obj in CHECKS.items(): # Do not remove batch checks in batch processing if self.is_batch_update and check_obj.batch_update: old_checks.discard(check) continue # Does the check fire? if ( not is_source and check_obj.target and check_obj.check_target(src, tgt, self) ) or (is_source and check_obj.source and check_obj.check_source(src, self)): if check in old_checks: # We already have this check old_checks.remove(check) else: # Create new check create.append( Check( unit=self, ignore=False, check=check, ) ) was_change = True has_checks = True if create: Check.objects.bulk_create_ignore(create) # Delete no longer failing checks if old_checks: was_change = True Check.objects.filter( unit=self, check__in=old_checks, ).delete() # Update failing checks flag if not self.is_batch_update and (was_change or not same_content): self.update_has_failing_check(was_change, has_checks)
def test_description_nolocation(self): unit = Unit( source="%d %s", target="%s %d", extra_flags="python-format", ) check = Check(unit=unit) self.assertEqual( self.check.get_description(check), "Following format strings are wrongly ordered: %d, %s", )
def test_description(self): unit = Unit(source="string $URL$", target="string") unit.__dict__["all_flags"] = Flags("placeholders:$URL$") check = Check(unit=unit) self.assertHTMLEqual( self.check.get_description(check), """ Following format strings are missing: <span class="hlcheck" data-value="$URL$">$URL$</span> """, )
def test_description(self): unit = Unit( source="{0}''s brush is {1} centimeters tall", target="{0}'s brush is {1} centimeters tall", extra_flags="java-messageformat", ) check = Check(unit=unit) self.assertEqual( self.check.get_description(check), "You need to pair up an apostrophe with another one.", )
def test_description(self): unit = Unit( source="%(count)d", target="%(languages)d", extra_flags="python-format", ) check = Check(unit=unit) self.assertEqual( self.check.get_description(check), "Following format strings are missing: %(count)d<br />" "Following format strings are extra: %(languages)d", )
def test_description(self): unit = Unit( source="${foo}", target="${bar}", extra_flags="es-format", ) check = Check(unit=unit) self.assertEqual( self.check.get_description(check), "Following format strings are missing: ${foo}<br />" "Following format strings are extra: ${bar}", )
def get_checks(self): # Build fake unit to run checks fake_unit = copy(self.unit) fake_unit.target = self.target fake_unit.state = STATE_TRANSLATED source = fake_unit.get_source_plurals() target = fake_unit.get_target_plurals() result = [] for check, check_obj in CHECKS.target.items(): if check_obj.check_target(source, target, fake_unit): result.append(Check(unit=fake_unit, dismissed=False, check=check)) return result
def run_checks(self, same_state=True, same_content=True): """Update checks for this unit.""" was_change = False has_checks = None src = self.get_source_plurals() tgt = self.get_target_plurals() old_checks = set(self.check_set.values_list("check", flat=True)) create = [] if self.translation.is_source: checks = CHECKS.source.items() meth = "check_source" args = src, self else: checks = CHECKS.target.items() meth = "check_target" args = src, tgt, self # Run all checks for check, check_obj in checks: # Do not remove batch checks in batch processing if self.is_batch_update and check_obj.batch_update: old_checks.discard(check) continue # Does the check fire? if getattr(check_obj, meth)(*args): if check in old_checks: # We already have this check old_checks.remove(check) else: # Create new check create.append(Check(unit=self, ignore=False, check=check)) was_change = True has_checks = True if create: Check.objects.bulk_create(create, batch_size=500, ignore_conflicts=True) # Delete no longer failing checks if old_checks: was_change = True Check.objects.filter(unit=self, check__in=old_checks).delete() # Update failing checks flag if not self.is_batch_update and (was_change or not same_content): self.update_has_failing_check(was_change, has_checks)
def perform_batch(self, component): from weblate.checks.models import Check from weblate.trans.models import Component handled = set() create = [] components = {} for unit in self.check_component(component): # Handle ignore flags if self.should_skip(unit): continue handled.add(unit.pk) # Check is already there if self.check_id in unit.all_checks_names: continue create.append(Check(unit=unit, dismissed=False, name=self.check_id)) components[ unit.translation.component.id] = unit.translation.component Check.objects.bulk_create(create, batch_size=500, ignore_conflicts=True) # Delete stale checks stale_checks = Check.objects.exclude(unit_id__in=handled) if self.batch_project_wide and component.allow_translation_propagation: stale_checks = stale_checks.filter( unit__translation__component__project=component.project, unit__translation__component__allow_translation_propagation= True, name=self.check_id, ) for current in Component.objects.filter( pk__in=stale_checks.values_list( "unit__translation__component", flat=True)): components[current.pk] = current stale_checks.delete() else: stale_checks = stale_checks.filter( unit__translation__component=component, name=self.check_id, ) if stale_checks.delete()[0]: components[component.id] = component # Invalidate stats in case there were changes for current in components.values(): current.invalidate_cache()
def run_checks(self, same_content=True): """Update checks for this unit.""" run_propagate = False src = self.get_source_plurals() tgt = self.get_target_plurals() old_checks = self.all_checks_names create = [] if self.translation.is_source: checks = CHECKS.source.items() meth = "check_source" args = src, self else: checks = CHECKS.target.items() meth = "check_target" args = src, tgt, self # Run all checks for check, check_obj in checks: # Does the check fire? if getattr(check_obj, meth)(*args): if check in old_checks: # We already have this check old_checks.remove(check) else: # Create new check create.append(Check(unit=self, ignore=False, check=check)) run_propagate |= check_obj.propagates if create: Check.objects.bulk_create(create, batch_size=500, ignore_conflicts=True) # Propagate checks which need it (for example consistency) if run_propagate: for unit in self.same_source_units: unit.run_checks() # Trigger source checks on target check update (multiple failing checks) if not self.translation.is_source: self.source_info.is_batch_update = self.is_batch_update self.source_info.run_checks() # Delete no longer failing checks if old_checks: Check.objects.filter(unit=self, check__in=old_checks).delete() # This is always preset as it is used in top of this method del self.__dict__["all_checks"]
def test_description(self): unit = Unit( source="string", target="I have two two lemons lemons", translation=Translation( language=Language("cs"), component=Component(source_language=Language("en")), ), ) check = Check(unit=unit) self.assertEqual( self.check.get_description(check), "Text contains the same word twice in a row: lemons, two", )
def run_batch_checks(self, attr_name): """Run batch executed checks.""" from weblate.trans.models import Unit create = [] meth_name = 'check_{}_project'.format(attr_name) for check, check_obj in CHECKS.items(): if not getattr(check_obj, attr_name) or not check_obj.batch_update: continue self.log_info('running batch check: %s', check) # List of triggered checks data = getattr(check_obj, meth_name)(self) # Fetch existing check instances existing = set( Check.objects.filter( unit__translation__component__project=self, check=check).values_list('unit__content_hash', 'unit__translation__language_id')) # Create new check instances for item in data: if 'translation__language' not in item: item['translation__language'] = self.source_language.id key = (item['content_hash'], item['translation__language']) if key in existing: existing.discard(key) else: units = Unit.objects.filter( translation__component__project=self, translation__language_id=item['translation__language'], content_hash=item['content_hash'], ) for unit in units: create.append( Check(unit=unit, check=check, ignore=False)) # Remove stale instances if existing: query = functools.reduce( lambda q, value: q | (Q(unit__content_hash=value[0]) & Q(unit__translation__language_id=value[1])), existing, Q(), ) Check.objects.filter(check=check).filter(query).delete() # Create new checks if create: Check.objects.bulk_create(create, batch_size=500, ignore_conflicts=True)
def test_regexp(self): unit = Unit(source="string $URL$", target="string $FOO$") unit.__dict__["all_flags"] = Flags( r"""placeholders:r"(\$)([^$]*)(\$)" """) check = Check(unit=unit) self.assertHTMLEqual( self.check.get_description(check), """ Following format strings are missing: <span class="hlcheck" data-value="$URL$">$URL$</span> <br /> Following format strings are extra: <span class="hlcheck" data-value="$FOO$">$FOO$</span> """, )
def test_description(self): unit = Unit( source="${foo}", target="${bar}", extra_flags="es-format", ) check = Check(unit=unit) self.assertHTMLEqual( self.check.get_description(check), """ Following format strings are missing: <span class="hlcheck" data-value="${foo}">${foo}</span> <br /> Following format strings are extra: <span class="hlcheck" data-value="${bar}">${bar}</span> """, )
def run_batch_checks(self, attr_name): """Run batch executed checks""" create = [] meth_name = 'check_{}_project'.format(attr_name) for check, check_obj in CHECKS.items(): if not getattr(check_obj, attr_name) or not check_obj.batch_update: continue self.log_info('running batch check: %s', check) # List of triggered checks data = getattr(check_obj, meth_name)(self) # Fetch existing check instances existing = set( Check.objects.filter( project=self, check=check ).values_list( 'content_hash', 'language_id' ) ) # Create new check instances for item in data: if 'translation__language' not in item: item['translation__language'] = None key = (item['content_hash'], item['translation__language']) if key in existing: existing.discard(key) else: create.append(Check( content_hash=item['content_hash'], project=self, language_id=item['translation__language'], check=check, ignore=False, )) # Remove stale instances if existing: query = functools.reduce( lambda q, value: q | (Q(content_hash=value[0]) & Q(language_id=value[1])), existing, Q() ) Check.objects.filter(check=check).filter(query).delete() # Create new checks if create: Check.objects.bulk_create_ignore(create)
def test_description(self): unit = Unit( source="%(count)d", target="%(languages)d", extra_flags="python-format", ) check = Check(unit=unit) self.assertHTMLEqual( self.check.get_description(check), """ Following format strings are missing: <span class="hlcheck" data-value="%(count)d">%(count)d</span> <br /> Following format strings are extra: <span class="hlcheck" data-value="%(languages)d">%(languages)d</span> """, )
def test_description(self): unit = Unit( source="{0}''s brush is {1} centimeters tall", target="{0}'s brush is {1} centimeters tall", extra_flags="java-messageformat", translation=Translation( component=Component( file_format="auto", source_language=Language("en"), ), language=Language("cs"), ), ) check = Check(unit=unit) self.assertEqual( self.check.get_description(check), "You need to pair up an apostrophe with another one.", )
def test_whitespace(self): unit = Unit(source="string {URL} ", target="string {URL}") unit.__dict__["all_flags"] = Flags(r"""placeholders:r"\s?{\w+}\s?" """) check = Check(unit=unit) self.assertHTMLEqual( self.check.get_description(check), """ Following format strings are missing: <span class="hlcheck" data-value=" {URL} "> <span class="hlspace"><span class="space-space"><span class="sr-only"> </span></span></span> {URL} <span class="hlspace"><span class="space-space"><span class="sr-only"> </span></span></span> </span> <br /> Following format strings are extra: <span class="hlcheck" data-value=" {URL}"> <span class="hlspace"><span class="space-space"><span class="sr-only"> </span></span></span> {URL} </span> """, )
def run_checks(self, propagate: Optional[bool] = None): # noqa: C901 """Update checks for this unit.""" needs_propagate = bool(propagate) src = self.get_source_plurals() tgt = self.get_target_plurals() old_checks = self.all_checks_names create = [] if self.translation.component.is_glossary: # We might eventually run some checks on glossary checks = {} meth = "check_source" args = src, self elif self.is_source: checks = CHECKS.source meth = "check_source" args = src, self else: if self.readonly: checks = {} else: checks = CHECKS.target meth = "check_target" args = src, tgt, self # Run all checks for check, check_obj in checks.items(): # Does the check fire? if getattr(check_obj, meth)(*args): if check in old_checks: # We already have this check old_checks.remove(check) # Propagation is handled in # weblate.checks.models.remove_complimentary_checks else: # Create new check create.append( Check(unit=self, dismissed=False, check=check)) needs_propagate |= check_obj.propagates if create: Check.objects.bulk_create(create, batch_size=500, ignore_conflicts=True) # Propagate checks which need it (for example consistency) if (needs_propagate and propagate is not False) or propagate is True: for unit in self.same_source_units: try: # Ensure we get a fresh copy of checks # It might be modified meanwhile by propagating to other units unit.clear_checks_cache() unit.run_checks(False) except Unit.DoesNotExist: # This can happen in some corner cases like changing # source language of a project - the source language is # changed first and then components are updated. But # not all are yet updated and this spans across them. continue # Delete no longer failing checks if old_checks: Check.objects.filter(unit=self, check__in=old_checks).delete() propagated_old_checks = [ name for name in old_checks if CHECKS[name].propagates ] if propagated_old_checks: Check.objects.filter(unit__in=self.same_source_units, check__in=propagated_old_checks).delete() for other in self.same_source_units: other.translation.invalidate_cache() other.clear_checks_cache() # Trigger source checks on target check update (multiple failing checks) if (create or old_checks) and not self.is_source: if self.is_batch_update: self.translation.component.updated_sources[ self.source_unit.id] = self.source_unit else: self.source_unit.run_checks() # This is always preset as it is used in top of this method self.clear_checks_cache() if not self.is_batch_update: self.translation.invalidate_cache()
def test_get_description(self): self.test_untranslated() check = Check(unit=self.get_unit()) self.assertEqual( self.check.get_description(check), 'Last translation was "Nazdar svete!\n".' )
def run_checks(self): """Update checks for this unit.""" run_propagate = False src = self.get_source_plurals() tgt = self.get_target_plurals() # Ensure we get a fresh copy of checks # It might be modified meanwhile by propagating to other units if "all_checks" in self.__dict__: del self.__dict__["all_checks"] old_checks = self.all_checks_names create = [] if self.translation.is_source: checks = CHECKS.source meth = "check_source" args = src, self else: checks = CHECKS.target meth = "check_target" args = src, tgt, self # Run all checks for check, check_obj in checks.items(): # Does the check fire? if getattr(check_obj, meth)(*args): if check in old_checks: # We already have this check old_checks.remove(check) else: # Create new check create.append( Check(unit=self, dismissed=False, check=check)) run_propagate |= check_obj.propagates if create: Check.objects.bulk_create(create, batch_size=500, ignore_conflicts=True) # Propagate checks which need it (for example consistency) if run_propagate: for unit in self.same_source_units: try: unit.run_checks() except Unit.DoesNotExist: # This can happen in some corner cases like changing # source language of a project - the source language is # changed first and then components are updated. But # not all are yet updated and this spans across them. continue # Trigger source checks on target check update (multiple failing checks) if not self.translation.is_source: self.source_info.is_batch_update = self.is_batch_update self.source_info.run_checks() # Delete no longer failing checks if old_checks: Check.objects.filter(unit=self, check__in=old_checks).delete() # This is always preset as it is used in top of this method del self.__dict__["all_checks"]