def sync_unit(self, dbunits, updated, id_hash, unit, pos):
try:
newunit = dbunits[id_hash]
is_new = False
except KeyError:
newunit = Unit(translation=self, id_hash=id_hash, state=-1)
# Avoid fetching empty list of checks from the database
newunit.all_checks = []
is_new = True
newunit.update_from_unit(unit, pos, is_new)
# Check if unit is worth notification:
# - new and untranslated
# - newly not translated
# - newly fuzzy
# - source string changed
if newunit.state < STATE_TRANSLATED and (
newunit.state != newunit.old_unit.state
or is_new
or newunit.source != newunit.old_unit.source
):
self.was_new += 1
# Store current unit ID
updated[id_hash] = newunit
def sync_unit(self, dbunits, updated, id_hash, unit, pos):
try:
newunit = dbunits[id_hash]
is_new = False
except KeyError:
newunit = Unit(translation=self, id_hash=id_hash, state=-1)
is_new = True
newunit.update_from_unit(unit, pos, is_new)
# Check if unit is worth notification:
# - new and untranslated
# - newly not translated
# - newly fuzzy
# - source string changed
self.was_new = self.was_new or (
newunit.state < STATE_TRANSLATED
and (
newunit.state != newunit.old_unit.state
or is_new
or newunit.source != newunit.old_unit.source
)
)
# Store current unit ID
updated[id_hash] = newunit
def test_whitespace(self):
unit = Unit(source=u'Foo\n')
fix = SameBookendingWhitespace()
self.assertEqual(fix.fix_target(['Bar'], unit), ([u'Bar\n'], True))
self.assertEqual(fix.fix_target(['Bar\n'], unit), ([u'Bar\n'], False))
unit = Unit(source=u' ')
self.assertEqual(fix.fix_target([' '], unit), ([' '], False))
def add_unit( # noqa: C901
self,
request,
context: str,
source: Union[str, List[str]],
target: Optional[Union[str, List[str]]] = None,
extra_flags: str = "",
auto_context: bool = False,
is_batch_update: bool = False,
):
user = request.user if request else None
component = self.component
if self.is_source:
translations = [self]
translations.extend(component.translation_set.exclude(id=self.id))
else:
translations = [component.source_translation, self]
has_template = component.has_template()
source_unit = None
result = None
# Automatic context
suffix = 0
base = context
while self.unit_set.filter(context=context, source=source).exists():
suffix += 1
context = f"{base}{suffix}"
for translation in translations:
is_source = translation.is_source
kwargs = {}
if has_template:
kwargs["pending"] = is_source
else:
kwargs["pending"] = not is_source
if kwargs["pending"]:
kwargs["details"] = {"add_unit": True}
if is_source:
current_target = source
kwargs["extra_flags"] = extra_flags
else:
current_target = target
if current_target is None:
current_target = ""
if isinstance(current_target, list):
current_target = join_plural(current_target)
if isinstance(source, list):
source = join_plural(source)
if has_template:
id_hash = calculate_hash(context)
else:
id_hash = calculate_hash(source, context)
# When adding to a target the source string can already exist
unit = None
if not self.is_source and is_source:
try:
unit = translation.unit_set.get(id_hash=id_hash)
flags = Flags(unit.extra_flags)
flags.merge(extra_flags)
new_flags = flags.format()
if unit.extra_flags != new_flags:
unit.save(update_fields=["extra_flags"], same_content=True)
except Unit.DoesNotExist:
pass
if unit is None:
unit = Unit(
translation=translation,
context=context,
source=source,
target=current_target,
state=STATE_TRANSLATED if bool(current_target) else STATE_EMPTY,
source_unit=source_unit,
id_hash=id_hash,
position=0,
**kwargs,
)
unit.is_batch_update = is_batch_update
unit.save(force_insert=True)
Change.objects.create(
unit=unit,
action=Change.ACTION_NEW_UNIT,
target=source,
user=user,
author=user,
)
# The source language is always first in the translations array
if source_unit is None:
source_unit = unit
if translation == self:
result = unit
if not is_batch_update:
component.update_variants()
component.sync_terminology()
return result
def check_sync(self, force=False, request=None, change=None):
"""Check whether database is in sync with git and possibly updates."""
if change is None:
change = Change.ACTION_UPDATE
if request is None:
user = None
else:
user = request.user
# Check if we're not already up to date
if not self.revision:
self.reason = 'new file'
elif self.revision != self.get_git_blob_hash():
self.reason = 'content changed'
elif force:
self.reason = 'check forced'
else:
self.reason = ''
return
self.log_info('processing %s, %s', self.filename, self.reason)
# List of updated units (used for cleanup and duplicates detection)
updated = {}
try:
store = self.store
# Store plural
plural = store.get_plural(self.language)
if plural != self.plural:
self.plural = plural
self.save(update_fields=['plural'])
# Was there change?
self.was_new = False
# Position of current unit
pos = 0
# Select all current units for update
dbunits = {
unit.id_hash: unit
for unit in self.unit_set.select_for_update()
}
for unit in store.content_units:
id_hash = unit.id_hash
# Update position
pos += 1
# Check for possible duplicate units
if id_hash in updated:
newunit = updated[id_hash]
self.log_warning(
'duplicate string to translate: %s (%s)',
newunit,
repr(newunit.source),
)
Change.objects.create(
unit=newunit,
action=Change.ACTION_DUPLICATE_STRING,
user=user,
author=user,
)
self.component.trigger_alert(
'DuplicateString',
language_code=self.language.code,
source=newunit.source,
unit_pk=newunit.pk,
)
continue
try:
newunit = dbunits[id_hash]
is_new = False
except KeyError:
newunit = Unit(translation=self, id_hash=id_hash, state=-1)
is_new = True
newunit.update_from_unit(unit, pos, is_new)
# Check if unit is worth notification:
# - new and untranslated
# - newly not translated
# - newly fuzzy
# - source string changed
self.was_new = self.was_new or (
newunit.state < STATE_TRANSLATED and
(newunit.state != newunit.old_unit.state or is_new
or newunit.source != newunit.old_unit.source))
# Store current unit ID
updated[id_hash] = newunit
except FileParseError as error:
self.log_warning('skipping update due to parse error: %s', error)
return
# Delete stale units
stale = set(dbunits) - set(updated)
if stale:
self.unit_set.filter(id_hash__in=stale).delete()
self.component.needs_cleanup = True
# We should also do cleanup on source strings tracking objects
# Update revision and stats
self.store_hash()
# Store change entry
Change.objects.create(translation=self,
action=change,
user=user,
author=user)
# Invalidate keys cache
transaction.on_commit(self.invalidate_keys)
def check_sync(self, force=False, request=None, change=None):
"""Check whether database is in sync with git and possibly updates"""
if change is None:
change = Change.ACTION_UPDATE
if request is None:
user = None
else:
user = request.user
# Check if we're not already up to date
if not self.revision:
reason = 'new file'
elif self.revision != self.get_git_blob_hash():
reason = 'content changed'
elif force:
reason = 'check forced'
else:
return
self.notify_new_string = False
self.log_info('processing %s, %s', self.filename, reason)
# List of created units (used for cleanup and duplicates detection)
created = {}
try:
store = self.store
except FileParseError as error:
self.log_warning('skipping update due to parse error: %s', error)
return
# Store plural
plural = store.get_plural(self.language)
if plural != self.plural:
self.plural = plural
self.save(update_fields=['plural'])
# Was there change?
was_new = False
# Position of current unit
pos = 0
# Select all current units for update
dbunits = {
unit.id_hash: unit for unit in self.unit_set.select_for_update()
}
for unit in store.all_units:
if not unit.is_translatable():
continue
id_hash = unit.id_hash
# Update position
pos += 1
# Check for possible duplicate units
if id_hash in created:
newunit = created[id_hash]
self.log_warning(
'duplicate string to translate: %s (%s)',
newunit,
repr(newunit.source)
)
Change.objects.create(
unit=newunit,
action=Change.ACTION_DUPLICATE_STRING,
user=user,
author=user
)
self.component.trigger_alert(
'DuplicateString',
language_code=self.language.code,
source=newunit.source,
unit_pk=newunit.pk,
)
continue
try:
newunit = dbunits[id_hash]
is_new = False
except KeyError:
newunit = Unit(
translation=self,
id_hash=id_hash,
content_hash=unit.content_hash,
source=unit.source,
context=unit.context
)
is_new = True
newunit.update_from_unit(unit, pos, is_new)
# Check if unit is worth notification:
# - new and untranslated
# - newly not translated
# - newly fuzzy
was_new = (
was_new or
(
newunit.state < STATE_TRANSLATED and
(newunit.state != newunit.old_unit.state or is_new)
)
)
# Store current unit ID
created[id_hash] = newunit
# Following query can get huge, so we should find better way
# to delete stale units, probably sort of garbage collection
# We should also do cleanup on source strings tracking objects
# Delete stale units
if self.unit_set.exclude(id_hash__in=created.keys()).delete()[0]:
self.component.needs_cleanup = True
# Update revision and stats
self.store_hash()
# Store change entry
Change.objects.create(
translation=self,
action=change,
user=user,
author=user
)
# Notify subscribed users
self.notify_new_string = was_new
def test_fix_target(self):
unit = Unit(source=u'Foo…')
fixed, fixups = fix_target(['Bar...'], unit)
self.assertEqual(fixed, [u'Bar…'])
self.assertEqual(len(fixups), 1)
self.assertEqual(unicode(fixups[0]), u'Trailing ellipsis')
def test_no_zerospace(self):
unit = Unit(source=u'Foo')
fix = RemoveZeroSpace()
self.assertEqual(fix.fix_target(['Bar'], unit), ([u'Bar'], False))
self.assertEqual(fix.fix_target([u'Bar\u200b'], unit),
([u'Bar'], True))
def test_no_ellipsis(self):
unit = Unit(source=u'Foo...')
fix = ReplaceTrailingDotsWithEllipsis()
self.assertEqual(fix.fix_target(['Bar...'], unit),
([u'Bar...'], False))
self.assertEqual(fix.fix_target([u'Bar…'], unit), ([u'Bar…'], False))
def check_sync(self, force=False, request=None, change=None):
"""Check whether database is in sync with git and possibly updates"""
if change is None:
change = Change.ACTION_UPDATE
if request is None:
user = None
else:
user = request.user
# Check if we're not already up to date
if not self.revision:
reason = 'new file'
elif self.revision != self.get_git_blob_hash():
reason = 'content changed'
elif force:
reason = 'check forced'
else:
return
self.log_info('processing %s, %s', self.filename, reason)
# List of created units (used for cleanup and duplicates detection)
created = {}
try:
store = self.store
except FileParseError as error:
self.log_warning('skipping update due to parse error: %s', error)
return
# Store plural
plural = store.get_plural(self.language)
if plural != self.plural:
self.plural = plural
self.save(update_fields=['plural'])
# Was there change?
self.was_new = False
# Position of current unit
pos = 0
# Select all current units for update
dbunits = {
unit.id_hash: unit for unit in self.unit_set.select_for_update()
}
for unit in store.all_units:
if not unit.is_translatable():
continue
id_hash = unit.id_hash
# Update position
pos += 1
# Check for possible duplicate units
if id_hash in created:
newunit = created[id_hash]
self.log_warning(
'duplicate string to translate: %s (%s)',
newunit,
repr(newunit.source)
)
Change.objects.create(
unit=newunit,
action=Change.ACTION_DUPLICATE_STRING,
user=user,
author=user
)
self.component.trigger_alert(
'DuplicateString',
language_code=self.language.code,
source=newunit.source,
unit_pk=newunit.pk,
)
continue
try:
newunit = dbunits[id_hash]
is_new = False
except KeyError:
newunit = Unit(
translation=self,
id_hash=id_hash,
state=-1,
)
is_new = True
newunit.update_from_unit(unit, pos, is_new)
# Check if unit is worth notification:
# - new and untranslated
# - newly not translated
# - newly fuzzy
self.was_new = (
self.was_new or
(
newunit.state < STATE_TRANSLATED and
(newunit.state != newunit.old_unit.state or is_new)
)
)
# Store current unit ID
created[id_hash] = newunit
# Following query can get huge, so we should find better way
# to delete stale units, probably sort of garbage collection
# We should also do cleanup on source strings tracking objects
# Delete stale units
if self.unit_set.exclude(id_hash__in=created.keys()).delete()[0]:
self.component.needs_cleanup = True
# Update revision and stats
self.store_hash()
# Store change entry
Change.objects.create(
translation=self,
action=change,
user=user,
author=user
)
