def run_checks(self, same_state=True, same_content=True):
"""Update checks for this unit."""
was_change = False
has_checks = None
if self.translation.is_template:
checks_to_run = {}
else:
checks_to_run = CHECKS.data
src = self.get_source_plurals()
tgt = self.get_target_plurals()
content_hash = self.content_hash
project = self.translation.component.project
language = self.translation.language
old_checks = set(self.checks(True))
create = []
# Run all target checks
for check, check_obj in checks_to_run.items():
if self.is_batch_update and check_obj.batch_update:
old_checks.discard(check)
continue
if check_obj.check_target(src, tgt, self):
if check in old_checks:
# We already have this check
old_checks.remove(check)
else:
# Create new check
create.append(Check(
content_hash=content_hash,
project=project,
language=language,
ignore=False,
check=check,
))
was_change = True
has_checks = True
if create:
Check.objects.bulk_create_ignore(create)
# Delete no longer failing checks
if old_checks:
was_change = True
Check.objects.filter(
content_hash=content_hash,
project=project,
language=language,
check__in=old_checks
).delete()
# Update failing checks flag
if not self.is_batch_update and (was_change or not same_content):
self.update_has_failing_check(was_change, has_checks)
def run_checks(self, unit=None, project=None, batch=False):
"""Update checks for this unit."""
if unit is None:
try:
unit = self.units[0]
except IndexError:
return
content_hash = unit.content_hash
src = unit.get_source_plurals()
if project is None:
project = self.component.project
# Fetch old checks
if self.component.checks_cache is not None:
old_checks = self.component.checks_cache.get((content_hash, None),
[])
else:
old_checks = set(
Check.objects.filter(content_hash=content_hash,
project=project,
language=None).values_list('check',
flat=True))
create = []
# Run all source checks
for check, check_obj in CHECKS.items():
if batch and check_obj.batch_update:
if check in old_checks:
# Do not remove batch checks in batch processing
old_checks.remove(check)
continue
if check_obj.source and check_obj.check_source(src, unit):
if check in old_checks:
# We already have this check
old_checks.remove(check)
else:
# Create new check
create.append(
Check(content_hash=content_hash,
project=project,
language=None,
ignore=False,
check=check))
if create:
Check.objects.bulk_create_ignore(create)
# Remove stale checks
if old_checks:
Check.objects.filter(content_hash=content_hash,
project=project,
language=None,
check__in=old_checks).delete()
def run_checks(self, same_state=True, same_content=True):
"""Update checks for this unit."""
was_change = False
has_checks = None
is_source = self.translation.is_source
src = self.get_source_plurals()
tgt = self.get_target_plurals()
old_checks = set(self.check_set.values_list('check', flat=True))
create = []
# Run all checks
for check, check_obj in CHECKS.items():
# Do not remove batch checks in batch processing
if self.is_batch_update and check_obj.batch_update:
old_checks.discard(check)
continue
# Does the check fire?
if (
not is_source
and check_obj.target
and check_obj.check_target(src, tgt, self)
) or (is_source and check_obj.source and check_obj.check_source(src, self)):
if check in old_checks:
# We already have this check
old_checks.remove(check)
else:
# Create new check
create.append(
Check(
unit=self,
ignore=False,
check=check,
)
)
was_change = True
has_checks = True
if create:
Check.objects.bulk_create_ignore(create)
# Delete no longer failing checks
if old_checks:
was_change = True
Check.objects.filter(
unit=self,
check__in=old_checks,
).delete()
# Update failing checks flag
if not self.is_batch_update and (was_change or not same_content):
self.update_has_failing_check(was_change, has_checks)
def test_description_nolocation(self):
unit = Unit(
source="%d %s",
target="%s %d",
extra_flags="python-format",
)
check = Check(unit=unit)
self.assertEqual(
self.check.get_description(check),
"Following format strings are wrongly ordered: %d, %s",
)
def test_description(self):
unit = Unit(source="string $URL$", target="string")
unit.__dict__["all_flags"] = Flags("placeholders:$URL$")
check = Check(unit=unit)
self.assertHTMLEqual(
self.check.get_description(check),
"""
Following format strings are missing:
<span class="hlcheck" data-value="$URL$">$URL$</span>
""",
)
def test_description(self):
unit = Unit(
source="{0}''s brush is {1} centimeters tall",
target="{0}'s brush is {1} centimeters tall",
extra_flags="java-messageformat",
)
check = Check(unit=unit)
self.assertEqual(
self.check.get_description(check),
"You need to pair up an apostrophe with another one.",
)
def test_description(self):
unit = Unit(
source="%(count)d",
target="%(languages)d",
extra_flags="python-format",
)
check = Check(unit=unit)
self.assertEqual(
self.check.get_description(check),
"Following format strings are missing: %(count)d<br />"
"Following format strings are extra: %(languages)d",
)
def test_description(self):
unit = Unit(
source="${foo}",
target="${bar}",
extra_flags="es-format",
)
check = Check(unit=unit)
self.assertEqual(
self.check.get_description(check),
"Following format strings are missing: ${foo}<br />"
"Following format strings are extra: ${bar}",
)
def get_checks(self):
# Build fake unit to run checks
fake_unit = copy(self.unit)
fake_unit.target = self.target
fake_unit.state = STATE_TRANSLATED
source = fake_unit.get_source_plurals()
target = fake_unit.get_target_plurals()
result = []
for check, check_obj in CHECKS.target.items():
if check_obj.check_target(source, target, fake_unit):
result.append(Check(unit=fake_unit, dismissed=False, check=check))
return result
def run_checks(self, same_state=True, same_content=True):
"""Update checks for this unit."""
was_change = False
has_checks = None
src = self.get_source_plurals()
tgt = self.get_target_plurals()
old_checks = set(self.check_set.values_list("check", flat=True))
create = []
if self.translation.is_source:
checks = CHECKS.source.items()
meth = "check_source"
args = src, self
else:
checks = CHECKS.target.items()
meth = "check_target"
args = src, tgt, self
# Run all checks
for check, check_obj in checks:
# Do not remove batch checks in batch processing
if self.is_batch_update and check_obj.batch_update:
old_checks.discard(check)
continue
# Does the check fire?
if getattr(check_obj, meth)(*args):
if check in old_checks:
# We already have this check
old_checks.remove(check)
else:
# Create new check
create.append(Check(unit=self, ignore=False, check=check))
was_change = True
has_checks = True
if create:
Check.objects.bulk_create(create,
batch_size=500,
ignore_conflicts=True)
# Delete no longer failing checks
if old_checks:
was_change = True
Check.objects.filter(unit=self, check__in=old_checks).delete()
# Update failing checks flag
if not self.is_batch_update and (was_change or not same_content):
self.update_has_failing_check(was_change, has_checks)
def perform_batch(self, component):
from weblate.checks.models import Check
from weblate.trans.models import Component
handled = set()
create = []
components = {}
for unit in self.check_component(component):
# Handle ignore flags
if self.should_skip(unit):
continue
handled.add(unit.pk)
# Check is already there
if self.check_id in unit.all_checks_names:
continue
create.append(Check(unit=unit, dismissed=False,
name=self.check_id))
components[
unit.translation.component.id] = unit.translation.component
Check.objects.bulk_create(create,
batch_size=500,
ignore_conflicts=True)
# Delete stale checks
stale_checks = Check.objects.exclude(unit_id__in=handled)
if self.batch_project_wide and component.allow_translation_propagation:
stale_checks = stale_checks.filter(
unit__translation__component__project=component.project,
unit__translation__component__allow_translation_propagation=
True,
name=self.check_id,
)
for current in Component.objects.filter(
pk__in=stale_checks.values_list(
"unit__translation__component", flat=True)):
components[current.pk] = current
stale_checks.delete()
else:
stale_checks = stale_checks.filter(
unit__translation__component=component,
name=self.check_id,
)
if stale_checks.delete()[0]:
components[component.id] = component
# Invalidate stats in case there were changes
for current in components.values():
current.invalidate_cache()
def run_checks(self, same_content=True):
"""Update checks for this unit."""
run_propagate = False
src = self.get_source_plurals()
tgt = self.get_target_plurals()
old_checks = self.all_checks_names
create = []
if self.translation.is_source:
checks = CHECKS.source.items()
meth = "check_source"
args = src, self
else:
checks = CHECKS.target.items()
meth = "check_target"
args = src, tgt, self
# Run all checks
for check, check_obj in checks:
# Does the check fire?
if getattr(check_obj, meth)(*args):
if check in old_checks:
# We already have this check
old_checks.remove(check)
else:
# Create new check
create.append(Check(unit=self, ignore=False, check=check))
run_propagate |= check_obj.propagates
if create:
Check.objects.bulk_create(create,
batch_size=500,
ignore_conflicts=True)
# Propagate checks which need it (for example consistency)
if run_propagate:
for unit in self.same_source_units:
unit.run_checks()
# Trigger source checks on target check update (multiple failing checks)
if not self.translation.is_source:
self.source_info.is_batch_update = self.is_batch_update
self.source_info.run_checks()
# Delete no longer failing checks
if old_checks:
Check.objects.filter(unit=self, check__in=old_checks).delete()
# This is always preset as it is used in top of this method
del self.__dict__["all_checks"]
def test_description(self):
unit = Unit(
source="string",
target="I have two two lemons lemons",
translation=Translation(
language=Language("cs"),
component=Component(source_language=Language("en")),
),
)
check = Check(unit=unit)
self.assertEqual(
self.check.get_description(check),
"Text contains the same word twice in a row: lemons, two",
)
def run_batch_checks(self, attr_name):
"""Run batch executed checks."""
from weblate.trans.models import Unit
create = []
meth_name = 'check_{}_project'.format(attr_name)
for check, check_obj in CHECKS.items():
if not getattr(check_obj, attr_name) or not check_obj.batch_update:
continue
self.log_info('running batch check: %s', check)
# List of triggered checks
data = getattr(check_obj, meth_name)(self)
# Fetch existing check instances
existing = set(
Check.objects.filter(
unit__translation__component__project=self,
check=check).values_list('unit__content_hash',
'unit__translation__language_id'))
# Create new check instances
for item in data:
if 'translation__language' not in item:
item['translation__language'] = self.source_language.id
key = (item['content_hash'], item['translation__language'])
if key in existing:
existing.discard(key)
else:
units = Unit.objects.filter(
translation__component__project=self,
translation__language_id=item['translation__language'],
content_hash=item['content_hash'],
)
for unit in units:
create.append(
Check(unit=unit, check=check, ignore=False))
# Remove stale instances
if existing:
query = functools.reduce(
lambda q, value: q
| (Q(unit__content_hash=value[0])
& Q(unit__translation__language_id=value[1])),
existing,
Q(),
)
Check.objects.filter(check=check).filter(query).delete()
# Create new checks
if create:
Check.objects.bulk_create(create,
batch_size=500,
ignore_conflicts=True)
def test_regexp(self):
unit = Unit(source="string $URL$", target="string $FOO$")
unit.__dict__["all_flags"] = Flags(
r"""placeholders:r"(\$)([^$]*)(\$)" """)
check = Check(unit=unit)
self.assertHTMLEqual(
self.check.get_description(check),
"""
Following format strings are missing:
<span class="hlcheck" data-value="$URL$">$URL$</span>
<br />
Following format strings are extra:
<span class="hlcheck" data-value="$FOO$">$FOO$</span>
""",
)
def test_description(self):
unit = Unit(
source="${foo}",
target="${bar}",
extra_flags="es-format",
)
check = Check(unit=unit)
self.assertHTMLEqual(
self.check.get_description(check),
"""
Following format strings are missing:
<span class="hlcheck" data-value="${foo}">${foo}</span>
<br />
Following format strings are extra:
<span class="hlcheck" data-value="${bar}">${bar}</span>
""",
)
def run_batch_checks(self, attr_name):
"""Run batch executed checks"""
create = []
meth_name = 'check_{}_project'.format(attr_name)
for check, check_obj in CHECKS.items():
if not getattr(check_obj, attr_name) or not check_obj.batch_update:
continue
self.log_info('running batch check: %s', check)
# List of triggered checks
data = getattr(check_obj, meth_name)(self)
# Fetch existing check instances
existing = set(
Check.objects.filter(
project=self,
check=check
).values_list(
'content_hash', 'language_id'
)
)
# Create new check instances
for item in data:
if 'translation__language' not in item:
item['translation__language'] = None
key = (item['content_hash'], item['translation__language'])
if key in existing:
existing.discard(key)
else:
create.append(Check(
content_hash=item['content_hash'],
project=self,
language_id=item['translation__language'],
check=check,
ignore=False,
))
# Remove stale instances
if existing:
query = functools.reduce(
lambda q, value:
q | (Q(content_hash=value[0]) & Q(language_id=value[1])),
existing,
Q()
)
Check.objects.filter(check=check).filter(query).delete()
# Create new checks
if create:
Check.objects.bulk_create_ignore(create)
def test_description(self):
unit = Unit(
source="%(count)d",
target="%(languages)d",
extra_flags="python-format",
)
check = Check(unit=unit)
self.assertHTMLEqual(
self.check.get_description(check),
"""
Following format strings are missing:
<span class="hlcheck" data-value="%(count)d">%(count)d</span>
<br />
Following format strings are extra:
<span class="hlcheck" data-value="%(languages)d">%(languages)d</span>
""",
)
def test_description(self):
unit = Unit(
source="{0}''s brush is {1} centimeters tall",
target="{0}'s brush is {1} centimeters tall",
extra_flags="java-messageformat",
translation=Translation(
component=Component(
file_format="auto",
source_language=Language("en"),
),
language=Language("cs"),
),
)
check = Check(unit=unit)
self.assertEqual(
self.check.get_description(check),
"You need to pair up an apostrophe with another one.",
)
def test_whitespace(self):
unit = Unit(source="string {URL} ", target="string {URL}")
unit.__dict__["all_flags"] = Flags(r"""placeholders:r"\s?{\w+}\s?" """)
check = Check(unit=unit)
self.assertHTMLEqual(
self.check.get_description(check),
"""
Following format strings are missing:
<span class="hlcheck" data-value=" {URL} ">
<span class="hlspace"><span class="space-space"><span class="sr-only">
</span></span></span>
{URL}
<span class="hlspace"><span class="space-space"><span class="sr-only">
</span></span></span>
</span>
<br />
Following format strings are extra:
<span class="hlcheck" data-value=" {URL}">
<span class="hlspace"><span class="space-space"><span class="sr-only">
</span></span></span>
{URL}
</span>
""",
)
def run_checks(self, propagate: Optional[bool] = None): # noqa: C901
"""Update checks for this unit."""
needs_propagate = bool(propagate)
src = self.get_source_plurals()
tgt = self.get_target_plurals()
old_checks = self.all_checks_names
create = []
if self.translation.component.is_glossary:
# We might eventually run some checks on glossary
checks = {}
meth = "check_source"
args = src, self
elif self.is_source:
checks = CHECKS.source
meth = "check_source"
args = src, self
else:
if self.readonly:
checks = {}
else:
checks = CHECKS.target
meth = "check_target"
args = src, tgt, self
# Run all checks
for check, check_obj in checks.items():
# Does the check fire?
if getattr(check_obj, meth)(*args):
if check in old_checks:
# We already have this check
old_checks.remove(check)
# Propagation is handled in
# weblate.checks.models.remove_complimentary_checks
else:
# Create new check
create.append(
Check(unit=self, dismissed=False, check=check))
needs_propagate |= check_obj.propagates
if create:
Check.objects.bulk_create(create,
batch_size=500,
ignore_conflicts=True)
# Propagate checks which need it (for example consistency)
if (needs_propagate and propagate is not False) or propagate is True:
for unit in self.same_source_units:
try:
# Ensure we get a fresh copy of checks
# It might be modified meanwhile by propagating to other units
unit.clear_checks_cache()
unit.run_checks(False)
except Unit.DoesNotExist:
# This can happen in some corner cases like changing
# source language of a project - the source language is
# changed first and then components are updated. But
# not all are yet updated and this spans across them.
continue
# Delete no longer failing checks
if old_checks:
Check.objects.filter(unit=self, check__in=old_checks).delete()
propagated_old_checks = [
name for name in old_checks if CHECKS[name].propagates
]
if propagated_old_checks:
Check.objects.filter(unit__in=self.same_source_units,
check__in=propagated_old_checks).delete()
for other in self.same_source_units:
other.translation.invalidate_cache()
other.clear_checks_cache()
# Trigger source checks on target check update (multiple failing checks)
if (create or old_checks) and not self.is_source:
if self.is_batch_update:
self.translation.component.updated_sources[
self.source_unit.id] = self.source_unit
else:
self.source_unit.run_checks()
# This is always preset as it is used in top of this method
self.clear_checks_cache()
if not self.is_batch_update:
self.translation.invalidate_cache()
def test_get_description(self):
self.test_untranslated()
check = Check(unit=self.get_unit())
self.assertEqual(
self.check.get_description(check), 'Last translation was "Nazdar svete!\n".'
)
def run_checks(self):
"""Update checks for this unit."""
run_propagate = False
src = self.get_source_plurals()
tgt = self.get_target_plurals()
# Ensure we get a fresh copy of checks
# It might be modified meanwhile by propagating to other units
if "all_checks" in self.__dict__:
del self.__dict__["all_checks"]
old_checks = self.all_checks_names
create = []
if self.translation.is_source:
checks = CHECKS.source
meth = "check_source"
args = src, self
else:
checks = CHECKS.target
meth = "check_target"
args = src, tgt, self
# Run all checks
for check, check_obj in checks.items():
# Does the check fire?
if getattr(check_obj, meth)(*args):
if check in old_checks:
# We already have this check
old_checks.remove(check)
else:
# Create new check
create.append(
Check(unit=self, dismissed=False, check=check))
run_propagate |= check_obj.propagates
if create:
Check.objects.bulk_create(create,
batch_size=500,
ignore_conflicts=True)
# Propagate checks which need it (for example consistency)
if run_propagate:
for unit in self.same_source_units:
try:
unit.run_checks()
except Unit.DoesNotExist:
# This can happen in some corner cases like changing
# source language of a project - the source language is
# changed first and then components are updated. But
# not all are yet updated and this spans across them.
continue
# Trigger source checks on target check update (multiple failing checks)
if not self.translation.is_source:
self.source_info.is_batch_update = self.is_batch_update
self.source_info.run_checks()
# Delete no longer failing checks
if old_checks:
Check.objects.filter(unit=self, check__in=old_checks).delete()
# This is always preset as it is used in top of this method
del self.__dict__["all_checks"]
