def sync_unit(self, dbunits, updated, id_hash, unit, pos): try: newunit = dbunits[id_hash] is_new = False except KeyError: newunit = Unit(translation=self, id_hash=id_hash, state=-1) # Avoid fetching empty list of checks from the database newunit.all_checks = [] is_new = True newunit.update_from_unit(unit, pos, is_new) # Check if unit is worth notification: # - new and untranslated # - newly not translated # - newly fuzzy # - source string changed if newunit.state < STATE_TRANSLATED and ( newunit.state != newunit.old_unit.state or is_new or newunit.source != newunit.old_unit.source ): self.was_new += 1 # Store current unit ID updated[id_hash] = newunit
def sync_unit(self, dbunits, updated, id_hash, unit, pos): try: newunit = dbunits[id_hash] is_new = False except KeyError: newunit = Unit(translation=self, id_hash=id_hash, state=-1) is_new = True newunit.update_from_unit(unit, pos, is_new) # Check if unit is worth notification: # - new and untranslated # - newly not translated # - newly fuzzy # - source string changed self.was_new = self.was_new or ( newunit.state < STATE_TRANSLATED and ( newunit.state != newunit.old_unit.state or is_new or newunit.source != newunit.old_unit.source ) ) # Store current unit ID updated[id_hash] = newunit
def test_whitespace(self): unit = Unit(source=u'Foo\n') fix = SameBookendingWhitespace() self.assertEqual(fix.fix_target(['Bar'], unit), ([u'Bar\n'], True)) self.assertEqual(fix.fix_target(['Bar\n'], unit), ([u'Bar\n'], False)) unit = Unit(source=u' ') self.assertEqual(fix.fix_target([' '], unit), ([' '], False))
def add_unit( # noqa: C901 self, request, context: str, source: Union[str, List[str]], target: Optional[Union[str, List[str]]] = None, extra_flags: str = "", auto_context: bool = False, is_batch_update: bool = False, ): user = request.user if request else None component = self.component if self.is_source: translations = [self] translations.extend(component.translation_set.exclude(id=self.id)) else: translations = [component.source_translation, self] has_template = component.has_template() source_unit = None result = None # Automatic context suffix = 0 base = context while self.unit_set.filter(context=context, source=source).exists(): suffix += 1 context = f"{base}{suffix}" for translation in translations: is_source = translation.is_source kwargs = {} if has_template: kwargs["pending"] = is_source else: kwargs["pending"] = not is_source if kwargs["pending"]: kwargs["details"] = {"add_unit": True} if is_source: current_target = source kwargs["extra_flags"] = extra_flags else: current_target = target if current_target is None: current_target = "" if isinstance(current_target, list): current_target = join_plural(current_target) if isinstance(source, list): source = join_plural(source) if has_template: id_hash = calculate_hash(context) else: id_hash = calculate_hash(source, context) # When adding to a target the source string can already exist unit = None if not self.is_source and is_source: try: unit = translation.unit_set.get(id_hash=id_hash) flags = Flags(unit.extra_flags) flags.merge(extra_flags) new_flags = flags.format() if unit.extra_flags != new_flags: unit.save(update_fields=["extra_flags"], same_content=True) except Unit.DoesNotExist: pass if unit is None: unit = Unit( translation=translation, context=context, source=source, target=current_target, state=STATE_TRANSLATED if bool(current_target) else STATE_EMPTY, source_unit=source_unit, id_hash=id_hash, position=0, **kwargs, ) unit.is_batch_update = is_batch_update unit.save(force_insert=True) Change.objects.create( unit=unit, action=Change.ACTION_NEW_UNIT, target=source, user=user, author=user, ) # The source language is always first in the translations array if source_unit is None: source_unit = unit if translation == self: result = unit if not is_batch_update: component.update_variants() component.sync_terminology() return result
def check_sync(self, force=False, request=None, change=None): """Check whether database is in sync with git and possibly updates.""" if change is None: change = Change.ACTION_UPDATE if request is None: user = None else: user = request.user # Check if we're not already up to date if not self.revision: self.reason = 'new file' elif self.revision != self.get_git_blob_hash(): self.reason = 'content changed' elif force: self.reason = 'check forced' else: self.reason = '' return self.log_info('processing %s, %s', self.filename, self.reason) # List of updated units (used for cleanup and duplicates detection) updated = {} try: store = self.store # Store plural plural = store.get_plural(self.language) if plural != self.plural: self.plural = plural self.save(update_fields=['plural']) # Was there change? self.was_new = False # Position of current unit pos = 0 # Select all current units for update dbunits = { unit.id_hash: unit for unit in self.unit_set.select_for_update() } for unit in store.content_units: id_hash = unit.id_hash # Update position pos += 1 # Check for possible duplicate units if id_hash in updated: newunit = updated[id_hash] self.log_warning( 'duplicate string to translate: %s (%s)', newunit, repr(newunit.source), ) Change.objects.create( unit=newunit, action=Change.ACTION_DUPLICATE_STRING, user=user, author=user, ) self.component.trigger_alert( 'DuplicateString', language_code=self.language.code, source=newunit.source, unit_pk=newunit.pk, ) continue try: newunit = dbunits[id_hash] is_new = False except KeyError: newunit = Unit(translation=self, id_hash=id_hash, state=-1) is_new = True newunit.update_from_unit(unit, pos, is_new) # Check if unit is worth notification: # - new and untranslated # - newly not translated # - newly fuzzy # - source string changed self.was_new = self.was_new or ( newunit.state < STATE_TRANSLATED and (newunit.state != newunit.old_unit.state or is_new or newunit.source != newunit.old_unit.source)) # Store current unit ID updated[id_hash] = newunit except FileParseError as error: self.log_warning('skipping update due to parse error: %s', error) return # Delete stale units stale = set(dbunits) - set(updated) if stale: self.unit_set.filter(id_hash__in=stale).delete() self.component.needs_cleanup = True # We should also do cleanup on source strings tracking objects # Update revision and stats self.store_hash() # Store change entry Change.objects.create(translation=self, action=change, user=user, author=user) # Invalidate keys cache transaction.on_commit(self.invalidate_keys)
def check_sync(self, force=False, request=None, change=None): """Check whether database is in sync with git and possibly updates""" if change is None: change = Change.ACTION_UPDATE if request is None: user = None else: user = request.user # Check if we're not already up to date if not self.revision: reason = 'new file' elif self.revision != self.get_git_blob_hash(): reason = 'content changed' elif force: reason = 'check forced' else: return self.notify_new_string = False self.log_info('processing %s, %s', self.filename, reason) # List of created units (used for cleanup and duplicates detection) created = {} try: store = self.store except FileParseError as error: self.log_warning('skipping update due to parse error: %s', error) return # Store plural plural = store.get_plural(self.language) if plural != self.plural: self.plural = plural self.save(update_fields=['plural']) # Was there change? was_new = False # Position of current unit pos = 0 # Select all current units for update dbunits = { unit.id_hash: unit for unit in self.unit_set.select_for_update() } for unit in store.all_units: if not unit.is_translatable(): continue id_hash = unit.id_hash # Update position pos += 1 # Check for possible duplicate units if id_hash in created: newunit = created[id_hash] self.log_warning( 'duplicate string to translate: %s (%s)', newunit, repr(newunit.source) ) Change.objects.create( unit=newunit, action=Change.ACTION_DUPLICATE_STRING, user=user, author=user ) self.component.trigger_alert( 'DuplicateString', language_code=self.language.code, source=newunit.source, unit_pk=newunit.pk, ) continue try: newunit = dbunits[id_hash] is_new = False except KeyError: newunit = Unit( translation=self, id_hash=id_hash, content_hash=unit.content_hash, source=unit.source, context=unit.context ) is_new = True newunit.update_from_unit(unit, pos, is_new) # Check if unit is worth notification: # - new and untranslated # - newly not translated # - newly fuzzy was_new = ( was_new or ( newunit.state < STATE_TRANSLATED and (newunit.state != newunit.old_unit.state or is_new) ) ) # Store current unit ID created[id_hash] = newunit # Following query can get huge, so we should find better way # to delete stale units, probably sort of garbage collection # We should also do cleanup on source strings tracking objects # Delete stale units if self.unit_set.exclude(id_hash__in=created.keys()).delete()[0]: self.component.needs_cleanup = True # Update revision and stats self.store_hash() # Store change entry Change.objects.create( translation=self, action=change, user=user, author=user ) # Notify subscribed users self.notify_new_string = was_new
def test_fix_target(self): unit = Unit(source=u'Foo…') fixed, fixups = fix_target(['Bar...'], unit) self.assertEqual(fixed, [u'Bar…']) self.assertEqual(len(fixups), 1) self.assertEqual(unicode(fixups[0]), u'Trailing ellipsis')
def test_no_zerospace(self): unit = Unit(source=u'Foo') fix = RemoveZeroSpace() self.assertEqual(fix.fix_target(['Bar'], unit), ([u'Bar'], False)) self.assertEqual(fix.fix_target([u'Bar\u200b'], unit), ([u'Bar'], True))
def test_no_ellipsis(self): unit = Unit(source=u'Foo...') fix = ReplaceTrailingDotsWithEllipsis() self.assertEqual(fix.fix_target(['Bar...'], unit), ([u'Bar...'], False)) self.assertEqual(fix.fix_target([u'Bar…'], unit), ([u'Bar…'], False))
def check_sync(self, force=False, request=None, change=None): """Check whether database is in sync with git and possibly updates""" if change is None: change = Change.ACTION_UPDATE if request is None: user = None else: user = request.user # Check if we're not already up to date if not self.revision: reason = 'new file' elif self.revision != self.get_git_blob_hash(): reason = 'content changed' elif force: reason = 'check forced' else: return self.log_info('processing %s, %s', self.filename, reason) # List of created units (used for cleanup and duplicates detection) created = {} try: store = self.store except FileParseError as error: self.log_warning('skipping update due to parse error: %s', error) return # Store plural plural = store.get_plural(self.language) if plural != self.plural: self.plural = plural self.save(update_fields=['plural']) # Was there change? self.was_new = False # Position of current unit pos = 0 # Select all current units for update dbunits = { unit.id_hash: unit for unit in self.unit_set.select_for_update() } for unit in store.all_units: if not unit.is_translatable(): continue id_hash = unit.id_hash # Update position pos += 1 # Check for possible duplicate units if id_hash in created: newunit = created[id_hash] self.log_warning( 'duplicate string to translate: %s (%s)', newunit, repr(newunit.source) ) Change.objects.create( unit=newunit, action=Change.ACTION_DUPLICATE_STRING, user=user, author=user ) self.component.trigger_alert( 'DuplicateString', language_code=self.language.code, source=newunit.source, unit_pk=newunit.pk, ) continue try: newunit = dbunits[id_hash] is_new = False except KeyError: newunit = Unit( translation=self, id_hash=id_hash, state=-1, ) is_new = True newunit.update_from_unit(unit, pos, is_new) # Check if unit is worth notification: # - new and untranslated # - newly not translated # - newly fuzzy self.was_new = ( self.was_new or ( newunit.state < STATE_TRANSLATED and (newunit.state != newunit.old_unit.state or is_new) ) ) # Store current unit ID created[id_hash] = newunit # Following query can get huge, so we should find better way # to delete stale units, probably sort of garbage collection # We should also do cleanup on source strings tracking objects # Delete stale units if self.unit_set.exclude(id_hash__in=created.keys()).delete()[0]: self.component.needs_cleanup = True # Update revision and stats self.store_hash() # Store change entry Change.objects.create( translation=self, action=change, user=user, author=user )