def __init__(self, *args, **kwargs): """Constructor to initialize some cache properties.""" super().__init__(*args, **kwargs) self.stats = TranslationStats(self) self.addon_commit_files = [] self.was_new = 0 self.reason = ""
def __init__(self, *args, **kwargs): """Constructor to initialize some cache properties.""" super(Translation, self).__init__(*args, **kwargs) self.stats = TranslationStats(self) self.addon_commit_files = [] self.notify_new_string = False self.commit_template = ''
def __init__(self, *args, **kwargs): """Constructor to initialize some cache properties.""" super(Translation, self).__init__(*args, **kwargs) self.stats = TranslationStats(self) self.addon_commit_files = [] self.commit_template = '' self.was_new = False
class Translation(models.Model, URLMixin, LoggerMixin): component = models.ForeignKey('Component', on_delete=models.deletion.CASCADE) language = models.ForeignKey(Language, on_delete=models.deletion.CASCADE) plural = models.ForeignKey(Plural, on_delete=models.deletion.CASCADE) revision = models.CharField(max_length=100, default='', blank=True) filename = models.CharField(max_length=200) language_code = models.CharField(max_length=20, default='', blank=True) objects = TranslationManager.from_queryset(TranslationQuerySet)() is_lockable = False _reverse_url_name = 'translation' class Meta(object): ordering = ['language__name'] app_label = 'trans' unique_together = ('component', 'language') def __init__(self, *args, **kwargs): """Constructor to initialize some cache properties.""" super(Translation, self).__init__(*args, **kwargs) self.stats = TranslationStats(self) self.addon_commit_files = [] self.notify_new_string = False self.commit_template = '' @cached_property def log_prefix(self): return '/'.join(( self.component.project.slug, self.component.slug, self.language.code, )) @cached_property def is_template(self): """Check whether this is template translation This means that translations should be propagated as sources to others. """ return self.filename == self.component.template def clean(self): """Validate that filename exists and can be opened using translate-toolkit. """ if not os.path.exists(self.get_filename()): raise ValidationError( _('Filename %s not found in repository! To add new ' 'translation, add language file into repository.') % self.filename) try: self.load_store() except Exception as error: raise ValidationError( _('Failed to parse file %(file)s: %(error)s') % { 'file': self.filename, 'error': str(error) }) def get_reverse_url_kwargs(self): """Return kwargs for URL reversing.""" return { 'project': self.component.project.slug, 'component': self.component.slug, 'lang': self.language.code } def get_widgets_url(self): """Return absolute URL for widgets.""" return get_site_url('{0}?lang={1}&component={2}'.format( reverse('widgets', kwargs={ 'project': self.component.project.slug, }), self.language.code, self.component.slug, )) def get_share_url(self): """Return absolute URL usable for sharing.""" return get_site_url( reverse('engage', kwargs={ 'project': self.component.project.slug, 'lang': self.language.code })) def get_translate_url(self): return reverse('translate', kwargs=self.get_reverse_url_kwargs()) def __str__(self): return '{0} - {1}'.format(force_text(self.component), force_text(self.language)) def get_filename(self): """Return absolute filename.""" return os.path.join(self.component.full_path, self.filename) def load_store(self): """Load translate-toolkit storage from disk.""" store = self.component.file_format_cls.parse( self.get_filename(), self.component.template_store, language_code=self.language_code) store_post_load.send(sender=self.__class__, translation=self, store=store) return store @cached_property def store(self): """Return translate-toolkit storage object for a translation.""" try: return self.load_store() except ParseError: raise except Exception as exc: self.component.handle_parse_error(exc, self) def check_sync(self, force=False, request=None, change=None): """Check whether database is in sync with git and possibly updates""" if change is None: change = Change.ACTION_UPDATE if request is None: user = None else: user = request.user # Check if we're not already up to date if not self.revision: reason = 'new file' elif self.revision != self.get_git_blob_hash(): reason = 'hash has changed' elif force: reason = 'check forced' else: return self.notify_new_string = False self.log_info('processing %s, %s', self.filename, reason) # List of created units (used for cleanup and duplicates detection) created_units = set() # Store plural plural = self.store.get_plural(self.language) if plural != self.plural: self.plural = plural self.save(update_fields=['plural']) # Was there change? was_new = False # Position of current unit pos = 1 # Select all current units for update self.unit_set.select_for_update() for unit in self.store.all_units(): if not unit.is_translatable(): continue newunit, is_new = Unit.objects.update_from_unit(self, unit, pos) # Check if unit is worth notification: # - new and untranslated # - newly not translated # - newly fuzzy was_new = (was_new or (newunit.state < STATE_TRANSLATED and (newunit.state != newunit.old_unit.state or is_new))) # Update position pos += 1 # Check for possible duplicate units if newunit.id in created_units: self.log_error('duplicate string to translate: %s (%s)', newunit, repr(newunit.source)) Change.objects.create(unit=newunit, action=Change.ACTION_DUPLICATE_STRING, user=user, author=user) # Store current unit ID created_units.add(newunit.id) # Following query can get huge, so we should find better way # to delete stale units, probably sort of garbage collection # We should also do cleanup on source strings tracking objects # Delete stale units if self.unit_set.exclude(id__in=created_units).delete()[0]: self.component.needs_cleanup = True # Update revision and stats self.store_hash() # Store change entry Change.objects.create(translation=self, action=change, user=user, author=user) # Notify subscribed users self.notify_new_string = was_new def get_last_remote_commit(self): return self.component.get_last_remote_commit() def do_update(self, request=None, method=None): return self.component.do_update(request, method=method) def do_push(self, request=None): return self.component.do_push(request) def do_reset(self, request=None): return self.component.do_reset(request) def do_cleanup(self, request=None): return self.component.do_cleanup(request) def can_push(self): return self.component.can_push() def get_git_blob_hash(self): """Return current VCS blob hash for file.""" ret = self.component.repository.get_object_hash(self.get_filename()) if not self.component.has_template(): return ret return ','.join([ ret, self.component.repository.get_object_hash(self.component.template) ]) def store_hash(self): """Store current hash in database.""" self.revision = self.get_git_blob_hash() self.save(update_fields=['revision']) def get_last_author(self, email=False): """Return last autor of change done in Weblate.""" if not self.stats.last_author: return None from weblate.auth.models import User return User.objects.get( pk=self.stats.last_author).get_author_name(email) def commit_pending(self, reason, request, skip_push=False): """Commit any pending changes.""" if not self.unit_set.filter(pending=True).exists(): return False self.log_info('committing pending changes (%s)', reason) with self.component.repository.lock: while True: # Find oldest change break loop if there is none left try: unit = self.unit_set.filter( pending=True, change__action__in=Change.ACTIONS_CONTENT, change__user__isnull=False, ).annotate(Max('change__timestamp')).order_by( 'change__timestamp__max')[0] except IndexError: break # Can not use get as there can be more with same timestamp change = unit.change_set.content().filter( timestamp=unit.change__timestamp__max)[0] author_name = change.author.get_author_name() # Flush pending units for this author self.update_units(author_name, change.author.id) # Commit changes self.git_commit(request, author_name, change.timestamp, skip_push=skip_push) # Update stats (the translated flag might have changed) self.invalidate_cache() return True def get_commit_message(self, author): """Format commit message based on project configuration.""" if self.commit_template == 'add': template = self.component.add_message self.commit_template = '' elif self.commit_template == 'delete': template = self.component.delete_message self.commit_template = '' else: template = self.component.commit_message msg = render_template(template, self, author=author) return msg def __git_commit(self, author, timestamp): """Commit translation to git.""" # Format commit message msg = self.get_commit_message(author) # Pre commit hook vcs_pre_commit.send(sender=self.__class__, translation=self, author=author) # Create list of files to commit files = [self.filename] # Do actual commit self.component.repository.commit(msg, author, timestamp, files + self.addon_commit_files) self.addon_commit_files = [] # Post commit hook vcs_post_commit.send(sender=self.__class__, translation=self) # Store updated hash self.store_hash() def repo_needs_commit(self): """Check whether there are some not committed changes.""" return (self.unit_set.filter(pending=True).exists() or self.component.repository.needs_commit(self.filename)) def repo_needs_merge(self): return self.component.repo_needs_merge() def repo_needs_push(self): return self.component.repo_needs_push() def git_commit(self, request, author, timestamp, skip_push=False, force_new=False): """Wrapper for commiting translation to git.""" repository = self.component.repository with repository.lock: # Is there something for commit? if not force_new and not repository.needs_commit(self.filename): return False # Do actual commit with git lock self.log_info('commiting %s as %s', self.filename, author) Change.objects.create( action=Change.ACTION_COMMIT, translation=self, user=request.user if request else None, ) self.__git_commit(author, timestamp) # Push if we should if not skip_push: self.component.push_if_needed(request) return True @transaction.atomic def update_units(self, author_name, author_id): """Update backend file and unit.""" updated = False for unit in self.unit_set.filter(pending=True).select_for_update(): # Skip changes by other authors unit_change = unit.change_set.content().order_by('-timestamp')[0] if unit_change.author_id != author_id: continue pounit, add = self.store.find_unit(unit.context, unit.get_source_plurals()[0]) unit.pending = False # Bail out if we have not found anything if pounit is None or pounit.is_obsolete(): self.log_error('message %s disappeared!', unit) unit.save(update_fields=['pending'], same_content=True) continue # Check for changes if ((not add or unit.target == '') and unit.target == pounit.get_target() and unit.approved == pounit.is_approved(unit.approved) and unit.fuzzy == pounit.is_fuzzy()): unit.save(update_fields=['pending'], same_content=True) continue updated = True # Optionally add unit to translation file. # This has be done prior setting tatget as some formats # generate content based on target language. if add: self.store.add_unit(pounit.unit) # Store translations if unit.is_plural(): pounit.set_target(unit.get_target_plurals()) else: pounit.set_target(unit.target) # Update fuzzy/approved flag pounit.mark_fuzzy(unit.state == STATE_FUZZY) pounit.mark_approved(unit.state == STATE_APPROVED) # Update comments as they might have been changed (eg, fuzzy flag # removed) state = unit.get_unit_state(pounit, False) flags = pounit.get_flags() if state != unit.state or flags != unit.flags: unit.state = state unit.flags = flags unit.save(update_fields=['state', 'flags', 'pending'], same_content=True) # Did we do any updates? if not updated: return # Update po file header now = timezone.now() if not timezone.is_aware(now): now = timezone.make_aware(now, timezone.utc) # Prepare headers to update headers = { 'add': True, 'last_translator': author_name, 'plural_forms': self.plural.plural_form, 'language': self.language_code, 'PO_Revision_Date': now.strftime('%Y-%m-%d %H:%M%z'), } # Optionally store language team with link to website if self.component.project.set_translation_team: headers['language_team'] = '{0} <{1}>'.format( self.language.name, get_site_url(self.get_absolute_url())) # Optionally store email for reporting bugs in source report_source_bugs = self.component.report_source_bugs if report_source_bugs: headers['report_msgid_bugs_to'] = report_source_bugs # Update genric headers self.store.update_header(**headers) # save translation changes self.store.save() def get_source_checks(self): """Return list of failing source checks on current component.""" result = TranslationChecklist() choices = dict(get_filter_choice(True)) result.add(self.stats, choices, 'all', 'success') # All checks result.add_if(self.stats, choices, 'sourcechecks', 'danger') # Process specific checks for check in CHECKS: check_obj = CHECKS[check] if not check_obj.source: continue result.add_if( self.stats, choices, check_obj.url_id, check_obj.severity, ) # Grab comments result.add_if(self.stats, choices, 'sourcecomments', 'info') return result @cached_property def list_translation_checks(self): """Return list of failing checks on current translation.""" result = TranslationChecklist() choices = dict(get_filter_choice()) # All strings result.add(self.stats, choices, 'all', 'success') result.add_if(self.stats, choices, 'approved', 'success') # Count of translated strings result.add_if(self.stats, choices, 'translated', 'success') # To approve if self.component.project.enable_review: result.add_if(self.stats, choices, 'unapproved', 'warning') # Approved with suggestions result.add_if(self.stats, choices, 'approved_suggestions', 'danger') # Untranslated strings result.add_if(self.stats, choices, 'todo', 'danger') # Not translated strings result.add_if(self.stats, choices, 'nottranslated', 'danger') # Fuzzy strings result.add_if(self.stats, choices, 'fuzzy', 'danger') # Translations with suggestions result.add_if(self.stats, choices, 'suggestions', 'info') result.add_if(self.stats, choices, 'nosuggestions', 'info') # All checks result.add_if(self.stats, choices, 'allchecks', 'danger') # Process specific checks for check in CHECKS: check_obj = CHECKS[check] if not check_obj.target: continue result.add_if( self.stats, choices, check_obj.url_id, check_obj.severity, ) # Grab comments result.add_if(self.stats, choices, 'comments', 'info') return result def merge_translations(self, request, store2, overwrite, method, fuzzy): """Merge translation unit wise Needed for template based translations to add new strings. """ not_found = 0 skipped = 0 accepted = 0 add_fuzzy = (method == 'fuzzy') add_approve = (method == 'approve') for set_fuzzy, unit2 in store2.iterate_merge(fuzzy): try: unit = self.unit_set.get_unit(unit2) except Unit.DoesNotExist: not_found += 1 continue if ((unit.translated and not overwrite) or (not request.user.has_perm('unit.edit', unit))): skipped += 1 continue accepted += 1 # We intentionally avoid propagating: # - in most cases it's not desired # - it slows down import considerably # - it brings locking issues as import is # executed with lock held and linked repos # can't obtain the lock state = STATE_TRANSLATED if add_fuzzy or set_fuzzy: state = STATE_FUZZY elif add_approve: state = STATE_APPROVED unit.translate(request, split_plural(unit2.get_target()), state, change_action=Change.ACTION_UPLOAD, propagate=False) if accepted > 0: self.invalidate_cache() request.user.profile.refresh_from_db() request.user.profile.translated += accepted request.user.profile.save(update_fields=['translated']) return (not_found, skipped, accepted, store2.count_units()) def merge_suggestions(self, request, store, fuzzy): """Merge content of translate-toolkit store as a suggestions.""" not_found = 0 skipped = 0 accepted = 0 for dummy, unit in store.iterate_merge(fuzzy): # Grab database unit try: dbunit = self.unit_set.get_unit(unit) except Unit.DoesNotExist: not_found += 1 continue # Add suggestion if dbunit.target != unit.get_target(): Suggestion.objects.add(dbunit, unit.get_target(), request) accepted += 1 else: skipped += 1 # Update suggestion count if accepted > 0: self.invalidate_cache() return (not_found, skipped, accepted, store.count_units()) def merge_upload(self, request, fileobj, overwrite, author=None, method='translate', fuzzy=''): """Top level handler for file uploads.""" filecopy = fileobj.read() fileobj.close() # Strip possible UTF-8 BOM if filecopy[:3] == codecs.BOM_UTF8: filecopy = filecopy[3:] # Load backend file store = try_load(fileobj.name, filecopy, self.component.file_format_cls, self.component.template_store) # Optionally set authorship if author is None: author = request.user.get_author_name() # Check valid plural forms if hasattr(store.store, 'parseheader'): header = store.store.parseheader() try: number, equation = Plural.parse_formula(header['Plural-Forms']) if not self.plural.same_plural(number, equation): raise Exception('Plural forms do not match the language.') except (ValueError, KeyError): # Formula wrong or missing pass if method in ('translate', 'fuzzy', 'approve'): # Merge on units level with self.component.repository.lock: return self.merge_translations( request, store, overwrite, method, fuzzy, ) # Add as sugestions return self.merge_suggestions(request, store, fuzzy) def invalidate_cache(self, recurse=True): """Invalidate any cached stats.""" # Invalidate summary stats self.stats.invalidate() if recurse and self.component.allow_translation_propagation: related = Translation.objects.filter( component__project=self.component.project, component__allow_translation_propagation=True, language=self.language, ).exclude(pk=self.pk) for translation in related: translation.invalidate_cache(False) def get_export_url(self): """Return URL of exported git repository.""" return self.component.get_export_url() def get_stats(self): """Return stats dictionary""" return { 'code': self.language.code, 'name': self.language.name, 'total': self.stats.all, 'total_words': self.stats.all_words, 'last_change': self.stats.last_changed, 'last_author': self.get_last_author(), 'translated': self.stats.translated, 'translated_words': self.stats.translated_words, 'translated_percent': self.stats.translated_percent, 'fuzzy': self.stats.fuzzy, 'fuzzy_percent': self.stats.fuzzy_percent, 'failing': self.stats.allchecks, 'failing_percent': self.stats.allchecks_percent, 'url': self.get_share_url(), 'url_translate': get_site_url(self.get_absolute_url()), } def remove(self, user): """Remove translation from the VCS""" author = user.get_author_name() # Log self.log_info('removing %s as %s', self.filename, author) # Remove file from VCS self.commit_template = 'delete' with self.component.repository.lock: self.component.repository.remove( [self.filename], self.get_commit_message(author), author, ) # Delete from the database self.delete() # Record change Change.objects.create(component=self.component, action=Change.ACTION_REMOVE, target=self.filename, user=user, author=user) def new_unit(self, request, key, value): with self.component.repository.lock: self.commit_pending('new unit', request) Change.objects.create(translation=self, action=Change.ACTION_NEW_UNIT, target=value, user=request.user, author=request.user) self.store.new_unit(key, value) self.component.create_translations(request=request) self.__git_commit(request.user.get_author_name(), timezone.now()) self.component.push_if_needed(request) def get_display_filename(self): return self.filename.replace('/', '/\u200B')
class Translation( FastDeleteModelMixin, models.Model, URLMixin, LoggerMixin, CacheKeyMixin ): component = models.ForeignKey("Component", on_delete=models.deletion.CASCADE) language = models.ForeignKey(Language, on_delete=models.deletion.CASCADE) plural = models.ForeignKey(Plural, on_delete=models.deletion.CASCADE) revision = models.CharField(max_length=200, default="", blank=True) filename = models.CharField(max_length=FILENAME_LENGTH) language_code = models.CharField(max_length=50, default="", blank=True) check_flags = models.TextField( verbose_name="Translation flags", default="", validators=[validate_check_flags], blank=True, ) objects = TranslationManager.from_queryset(TranslationQuerySet)() is_lockable = False _reverse_url_name = "translation" class Meta: app_label = "trans" unique_together = ("component", "language") verbose_name = "translation" verbose_name_plural = "translations" def __str__(self): return f"{self.component} — {self.language}" def __init__(self, *args, **kwargs): """Constructor to initialize some cache properties.""" super().__init__(*args, **kwargs) self.stats = TranslationStats(self) self.addon_commit_files = [] self.was_new = 0 self.reason = "" def get_badges(self): if self.is_source: yield (_("source"), _("This translation is used for source strings.")) @cached_property def full_slug(self): return "/".join( (self.component.project.slug, self.component.slug, self.language.code) ) def log_hook(self, level, msg, *args): self.component.store_log(self.full_slug, msg, *args) @cached_property def is_template(self): """Check whether this is template translation. This means that translations should be propagated as sources to others. """ return self.filename == self.component.template @cached_property def is_source(self): """Check whether this is source strings. This means that translations should be propagated as sources to others. """ return self.language_id == self.component.source_language_id @cached_property def all_flags(self): """Return parsed list of flags.""" return Flags(self.component.all_flags, self.check_flags) @cached_property def is_readonly(self): return "read-only" in self.all_flags def clean(self): """Validate that filename exists and can be opened using translate-toolkit.""" if not os.path.exists(self.get_filename()): raise ValidationError( _( "Filename %s not found in repository! To add new " "translation, add language file into repository." ) % self.filename ) try: self.load_store() except Exception as error: raise ValidationError( _("Failed to parse file %(file)s: %(error)s") % {"file": self.filename, "error": str(error)} ) def notify_new(self, request): if self.was_new: # Create change after flags has been updated and cache # invalidated, otherwise we might be sending notification # with outdated values Change.objects.create( translation=self, action=Change.ACTION_NEW_STRING, user=request.user if request else None, author=request.user if request else None, details={"count": self.was_new}, ) self.was_new = 0 def get_reverse_url_kwargs(self): """Return kwargs for URL reversing.""" return { "project": self.component.project.slug, "component": self.component.slug, "lang": self.language.code, } def get_widgets_url(self): """Return absolute URL for widgets.""" return get_site_url( "{}?lang={}&component={}".format( reverse("widgets", kwargs={"project": self.component.project.slug}), self.language.code, self.component.slug, ) ) def get_share_url(self): """Return absolute URL usable for sharing.""" return get_site_url( reverse( "engage", kwargs={ "project": self.component.project.slug, "lang": self.language.code, }, ) ) def get_translate_url(self): return reverse("translate", kwargs=self.get_reverse_url_kwargs()) def get_filename(self): """Return absolute filename.""" if not self.filename: return None return os.path.join(self.component.full_path, self.filename) def load_store(self, fileobj=None, force_intermediate=False): """Load translate-toolkit storage from disk.""" # Use intermediate store as template for source translation if force_intermediate or (self.is_template and self.component.intermediate): template = self.component.intermediate_store else: template = self.component.template_store if fileobj is None: fileobj = self.get_filename() elif self.is_template: template = self.component.load_template_store(fileobj) fileobj.seek(0) store = self.component.file_format_cls.parse( fileobj, template, language_code=self.language_code, source_language=self.component.source_language.code, is_template=self.is_template, ) store_post_load.send(sender=self.__class__, translation=self, store=store) return store @cached_property def store(self): """Return translate-toolkit storage object for a translation.""" try: return self.load_store() except FileParseError: raise except Exception as exc: report_error(cause="Translation parse error") self.component.handle_parse_error(exc, self) def sync_unit(self, dbunits, updated, id_hash, unit, pos): try: newunit = dbunits[id_hash] is_new = False except KeyError: newunit = Unit(translation=self, id_hash=id_hash, state=-1) # Avoid fetching empty list of checks from the database newunit.all_checks = [] is_new = True newunit.update_from_unit(unit, pos, is_new) # Check if unit is worth notification: # - new and untranslated # - newly not translated # - newly fuzzy # - source string changed if newunit.state < STATE_TRANSLATED and ( newunit.state != newunit.old_unit.state or is_new or newunit.source != newunit.old_unit.source ): self.was_new += 1 # Store current unit ID updated[id_hash] = newunit def check_sync(self, force=False, request=None, change=None): # noqa: C901 """Check whether database is in sync with git and possibly updates.""" if change is None: change = Change.ACTION_UPDATE if request is None: user = None else: user = request.user # Check if we're not already up to date if not self.revision: self.reason = "new file" elif self.revision != self.get_git_blob_hash(): self.reason = "content changed" elif force: self.reason = "check forced" else: self.reason = "" return self.log_info("processing %s, %s", self.filename, self.reason) # List of updated units (used for cleanup and duplicates detection) updated = {} try: store = self.store translation_store = None # Store plural plural = store.get_plural(self.language) if plural != self.plural: self.plural = plural self.save(update_fields=["plural"]) # Was there change? self.was_new = 0 # Select all current units for update dbunits = { unit.id_hash: unit for unit in self.unit_set.prefetch_bulk().select_for_update() } # Process based on intermediate store if available if self.component.intermediate: translation_store = store store = self.load_store(force_intermediate=True) for pos, unit in enumerate(store.content_units): # Use translation store if exists and if it contains the string if translation_store is not None: try: translated_unit, created = translation_store.find_unit( unit.context ) if translated_unit and not created: unit = translated_unit else: # Patch unit to have matching source unit.source = translated_unit.source except UnitNotFound: pass id_hash = unit.id_hash # Check for possible duplicate units if id_hash in updated: newunit = updated[id_hash] self.log_warning( "duplicate string to translate: %s (%s)", newunit, repr(newunit.source), ) Change.objects.create( unit=newunit, action=Change.ACTION_DUPLICATE_STRING, user=user, author=user, ) self.component.trigger_alert( "DuplicateString", language_code=self.language.code, source=newunit.source, unit_pk=newunit.pk, ) continue self.sync_unit(dbunits, updated, id_hash, unit, pos + 1) except FileParseError as error: self.log_warning("skipping update due to parse error: %s", error) return # Delete stale units stale = set(dbunits) - set(updated) if stale: self.unit_set.filter(id_hash__in=stale).delete() self.component.needs_cleanup = True # We should also do cleanup on source strings tracking objects # Update revision and stats self.store_hash() # Store change entry Change.objects.create(translation=self, action=change, user=user, author=user) # Invalidate keys cache transaction.on_commit(self.invalidate_keys) self.log_info("updating completed") # Use up to date list as prefetch for source if self.is_source: self.component.preload_sources(updated) def do_update(self, request=None, method=None): return self.component.do_update(request, method=method) def do_push(self, request=None): return self.component.do_push(request) def do_reset(self, request=None): return self.component.do_reset(request) def do_cleanup(self, request=None): return self.component.do_cleanup(request) def do_file_sync(self, request=None): return self.component.do_file_sync(request) def can_push(self): return self.component.can_push() def get_git_blob_hash(self): """Return current VCS blob hash for file.""" get_object_hash = self.component.repository.get_object_hash # Include language file hashes = [get_object_hash(self.get_filename())] if self.component.has_template(): # Include template hashes.append(get_object_hash(self.component.template)) if self.component.intermediate: # Include intermediate language as it might add new strings hashes.append(get_object_hash(self.component.intermediate)) return ",".join(hashes) def store_hash(self): """Store current hash in database.""" self.revision = self.get_git_blob_hash() self.save(update_fields=["revision"]) def get_last_author(self, email=False): """Return last autor of change done in Weblate.""" if not self.stats.last_author: return None from weblate.auth.models import User return User.objects.get(pk=self.stats.last_author).get_author_name(email) @transaction.atomic def commit_pending(self, reason, user, skip_push=False, force=False, signals=True): """Commit any pending changes.""" if not force and not self.needs_commit(): return False # Commit template first if ( not self.is_source and self.component.has_template() and self.component.source_translation.needs_commit() ): self.component.source_translation.commit_pending( reason, user, skip_push=skip_push, force=force, signals=signals ) self.log_info("committing pending changes (%s)", reason) try: store = self.store except FileParseError as error: report_error(cause="Failed to parse file on commit") self.log_error("skipping commit due to error: %s", error) return False with self.component.repository.lock: units = ( self.unit_set.filter(pending=True) .prefetch_recent_content_changes() .select_for_update() ) for unit in units: # We reuse the queryset, so pending units might reappear here if not unit.pending: continue # Get last change metadata author, timestamp = unit.get_last_content_change() author_name = author.get_author_name() # Flush pending units for this author self.update_units(units, store, author_name, author.id) # Commit changes self.git_commit( user, author_name, timestamp, skip_push=skip_push, signals=signals ) # Update stats (the translated flag might have changed) self.invalidate_cache() return True def get_commit_message(self, author: str, template: str, **kwargs): """Format commit message based on project configuration.""" return render_template(template, translation=self, author=author, **kwargs) @property def count_pending_units(self): return self.unit_set.filter(pending=True).count() def needs_commit(self): """Check whether there are some not committed changes.""" return self.count_pending_units > 0 def repo_needs_merge(self): return self.component.repo_needs_merge() def repo_needs_push(self): return self.component.repo_needs_push() @cached_property def filenames(self): if not self.filename: return [] if self.component.file_format_cls.simple_filename: return [self.get_filename()] return self.store.get_filenames() def repo_needs_commit(self): return self.component.repository.needs_commit(self.filenames) def git_commit( self, user, author: str, timestamp: Optional[datetime] = None, skip_push=False, signals=True, template: Optional[str] = None, store_hash: bool = True, ): """Wrapper for committing translation to git.""" repository = self.component.repository if template is None: template = self.component.commit_message with repository.lock: # Pre commit hook vcs_pre_commit.send(sender=self.__class__, translation=self, author=author) # Do actual commit with git lock if not self.component.commit_files( template=template, author=author, timestamp=timestamp, skip_push=skip_push, signals=signals, files=self.filenames + self.addon_commit_files, extra_context={"translation": self}, ): self.log_info("committed %s as %s", self.filenames, author) Change.objects.create( action=Change.ACTION_COMMIT, translation=self, user=user ) # Store updated hash if store_hash: self.store_hash() self.addon_commit_files = [] return True def update_units(self, units, store, author_name, author_id): """Update backend file and unit.""" updated = False for unit in units: # We reuse the queryset, so pending units might reappear here if not unit.pending: continue # Skip changes by other authors change_author = unit.get_last_content_change()[0] if change_author.id != author_id: continue # Remove pending flag unit.pending = False if unit.details.get("add_unit"): pounit = store.new_unit( unit.context, unit.get_source_plurals(), unit.get_target_plurals() ) updated = True del unit.details["add_unit"] else: try: pounit, add = store.find_unit(unit.context, unit.source) except UnitNotFound: # Bail out if we have not found anything report_error(cause="String disappeared") self.log_error("disappeared string: %s", unit) continue updated = True # Optionally add unit to translation file. # This has be done prior setting tatget as some formats # generate content based on target language. if add: store.add_unit(pounit.unit) # Store translations if unit.is_plural: pounit.set_target(unit.get_target_plurals()) else: pounit.set_target(unit.target) # Update fuzzy/approved flag pounit.mark_fuzzy(unit.state == STATE_FUZZY) pounit.mark_approved(unit.state == STATE_APPROVED) # Update comments as they might have been changed by state changes state = unit.get_unit_state(pounit, "") flags = pounit.flags update_fields = ["pending", "details"] only_save = True if state != unit.state or flags != unit.flags: unit.state = state update_fields.append("state") unit.flags = flags update_fields.append("flags") only_save = False unit.save( update_fields=update_fields, same_content=True, only_save=only_save ) # Did we do any updates? if not updated: return # Update po file header now = timezone.now() if not timezone.is_aware(now): now = timezone.make_aware(now, timezone.utc) # Prepare headers to update headers = { "add": True, "last_translator": author_name, "plural_forms": self.plural.plural_form, "language": self.language_code, "PO_Revision_Date": now.strftime("%Y-%m-%d %H:%M%z"), } # Optionally store language team with link to website if self.component.project.set_language_team: headers["language_team"] = "{} <{}>".format( self.language.name, get_site_url(self.get_absolute_url()) ) # Optionally store email for reporting bugs in source report_source_bugs = self.component.report_source_bugs if report_source_bugs: headers["report_msgid_bugs_to"] = report_source_bugs # Update genric headers store.update_header(**headers) # save translation changes store.save() @cached_property def enable_review(self): project = self.component.project return project.source_review if self.is_source else project.translation_review @cached_property def list_translation_checks(self): """Return list of failing checks on current translation.""" result = TranslationChecklist() # All strings result.add(self.stats, "all", "") result.add_if(self.stats, "readonly", "default") if not self.is_readonly: if self.enable_review: result.add_if(self.stats, "approved", "info") # Count of translated strings result.add_if(self.stats, "translated", "success") # To approve if self.enable_review: result.add_if(self.stats, "unapproved", "dark") # Approved with suggestions result.add_if(self.stats, "approved_suggestions", "info") # Untranslated strings result.add_if(self.stats, "todo", "danger") # Not translated strings result.add_if(self.stats, "nottranslated", "danger") # Fuzzy strings result.add_if(self.stats, "fuzzy", "danger") # Translations with suggestions result.add_if(self.stats, "suggestions", "dark") result.add_if(self.stats, "nosuggestions", "dark") # All checks result.add_if(self.stats, "allchecks", "warning") # Translated strings with checks if not self.is_source: result.add_if(self.stats, "translated_checks", "warning") # Dismissed checks result.add_if(self.stats, "dismissed_checks", "warning") # Process specific checks for check in CHECKS: check_obj = CHECKS[check] result.add_if(self.stats, check_obj.url_id, "warning") # Grab comments result.add_if(self.stats, "comments", "dark") # Include labels labels = self.component.project.label_set.order_by("name") if labels: for label in labels: result.add_if( self.stats, f"label:{label.name}", f"label label-{label.color}", ) result.add_if(self.stats, "unlabeled", "") return result def merge_translations( self, request, store2, conflicts: str, method: str, fuzzy: str ): """Merge translation unit wise. Needed for template based translations to add new strings. """ not_found = 0 skipped = 0 accepted = 0 add_fuzzy = method == "fuzzy" add_approve = method == "approve" unit_set = self.unit_set.all() for set_fuzzy, unit2 in store2.iterate_merge(fuzzy): try: unit = unit_set.get_unit(unit2) except Unit.DoesNotExist: not_found += 1 continue state = STATE_TRANSLATED if add_fuzzy or set_fuzzy: state = STATE_FUZZY elif add_approve: state = STATE_APPROVED if ( (unit.translated and not conflicts) or (unit.approved and conflicts != "replace-approved") or unit.readonly or (not request.user.has_perm("unit.edit", unit)) or (unit.target == unit2.target and unit.state == state) ): skipped += 1 continue accepted += 1 # We intentionally avoid propagating: # - in most cases it's not desired # - it slows down import considerably # - it brings locking issues as import is # executed with lock held and linked repos # can't obtain the lock unit.translate( request.user, split_plural(unit2.target), state, change_action=Change.ACTION_UPLOAD, propagate=False, ) if accepted > 0: self.invalidate_cache() request.user.profile.increase_count("translated", accepted) return (not_found, skipped, accepted, len(list(store2.content_units))) def merge_suggestions(self, request, store, fuzzy): """Merge content of translate-toolkit store as a suggestions.""" not_found = 0 skipped = 0 accepted = 0 unit_set = self.unit_set.all() for _unused, unit in store.iterate_merge(fuzzy): # Grab database unit try: dbunit = unit_set.get_unit(unit) except Unit.DoesNotExist: not_found += 1 continue # Add suggestion if dbunit.target != unit.target and not dbunit.readonly: if Suggestion.objects.add(dbunit, unit.target, request): accepted += 1 else: skipped += 1 else: skipped += 1 # Update suggestion count if accepted > 0: self.invalidate_cache() return (not_found, skipped, accepted, len(list(store.content_units))) def drop_store_cache(self): if "store" in self.__dict__: del self.__dict__["store"] if self.is_source: self.component.drop_template_store_cache() def handle_source(self, request, fileobj): """Replace source translations with uploaded one.""" component = self.component filenames = [] with component.repository.lock: # Commit pending changes component.commit_pending("source update", request.user) # Create actual file with the uploaded content temp = tempfile.NamedTemporaryFile( prefix="weblate-upload", dir=self.component.full_path, delete=False ) temp.write(fileobj.read()) temp.close() try: # Prepare msgmerge args, this is merely a copy from # weblate.addons.gettext.MsgmergeAddon and should be turned into # file format parameters args = ["--previous"] try: addon = component.addon_set.get(name="weblate.gettext.customize") addon_config = addon.configuration if addon_config["width"] != 77: args.append("--no-wrap") except ObjectDoesNotExist: pass try: addon = component.addon_set.get(name="weblate.gettext.msgmerge") addon_config = addon.configuration if not addon_config.get("fuzzy", True): args.append("--no-fuzzy-matching") if addon_config.get("previous", True): args.append("--previous") if addon_config.get("no_location", False): args.append("--no-location") except ObjectDoesNotExist: pass # Update translation files for translation in component.translation_set.exclude( language=component.source_language ): filename = translation.get_filename() component.file_format_cls.update_bilingual( filename, temp.name, args=args ) filenames.append(filename) finally: if os.path.exists(temp.name): if component.new_base: filename = component.get_new_base_filename() os.replace(temp.name, filename) filenames.append(filename) else: os.unlink(temp.name) # Commit changes previous_revision = self.component.repository.last_revision if component.commit_files( template=component.addon_message, files=filenames, author=request.user.get_author_name(), extra_context={"addon_name": "Source update"}, ): self.drop_store_cache() self.handle_store_change( request, request.user, previous_revision, change=Change.ACTION_REPLACE_UPLOAD, ) return (0, 0, self.unit_set.count(), self.unit_set.count()) def handle_replace(self, request, fileobj): """Replace file content with uploaded one.""" filecopy = fileobj.read() fileobj.close() fileobj = BytesIOMode(fileobj.name, filecopy) with self.component.repository.lock: if self.is_source: self.component.commit_pending("replace file", request.user) else: self.commit_pending("replace file", request.user) # This will throw an exception in case of error store2 = self.load_store(fileobj) store2.check_valid() # Actually replace file content self.store.save_atomic( self.store.storefile, lambda handle: handle.write(filecopy) ) # Commit to VCS previous_revision = self.component.repository.last_revision if self.git_commit( request.user, request.user.get_author_name(), store_hash=False ): # Drop store cache self.drop_store_cache() self.handle_store_change( request, request.user, previous_revision, change=Change.ACTION_REPLACE_UPLOAD, ) return (0, 0, self.unit_set.count(), len(list(store2.content_units))) def handle_add_upload(self, request, store, fuzzy: str = ""): skipped = 0 accepted = 0 existing = set(self.unit_set.values_list("context", "source")) for _set_fuzzy, unit in store.iterate_merge(fuzzy): if (unit.context, unit.source) in existing: skipped += 1 continue self.add_unit( request, unit.context, split_plural(unit.source), split_plural(unit.target), is_batch_update=True, ) accepted += 1 self.invalidate_cache() self.component.update_variants() self.component.sync_terminology() self.component.update_source_checks() self.component.run_batched_checks() return (0, skipped, accepted, len(list(store.content_units))) @transaction.atomic def merge_upload( self, request, fileobj: BinaryIO, conflicts: str, author_name: Optional[str] = None, author_email: Optional[str] = None, method: str = "translate", fuzzy: str = "", ): """Top level handler for file uploads.""" from weblate.accounts.models import AuditLog # Optionally set authorship orig_user = None if author_email: from weblate.auth.models import User orig_user = request.user request.user, created = User.objects.get_or_create( email=author_email, defaults={ "username": author_email, "full_name": author_name or author_email, }, ) if created: AuditLog.objects.create( request.user, request, "autocreated", ) try: if method == "replace": return self.handle_replace(request, fileobj) if method == "source": return self.handle_source(request, fileobj) filecopy = fileobj.read() fileobj.close() # Strip possible UTF-8 BOM if filecopy[:3] == codecs.BOM_UTF8: filecopy = filecopy[3:] # Load backend file store = try_load( fileobj.name, filecopy, self.component.file_format_cls, self.component.template_store, ) # Check valid plural forms if hasattr(store.store, "parseheader"): header = store.store.parseheader() try: number, formula = Plural.parse_plural_forms(header["Plural-Forms"]) if not self.plural.same_plural(number, formula): raise PluralFormsMismatch() except (ValueError, KeyError): # Formula wrong or missing pass if method in ("translate", "fuzzy", "approve"): # Merge on units level with self.component.repository.lock: return self.merge_translations( request, store, conflicts, method, fuzzy ) elif method == "add": return self.handle_add_upload(request, store, fuzzy=fuzzy) # Add as sugestions return self.merge_suggestions(request, store, fuzzy) finally: if orig_user: request.user = orig_user def invalidate_cache(self): """Invalidate any cached stats.""" # Invalidate summary stats transaction.on_commit(self.stats.invalidate) transaction.on_commit(self.component.invalidate_glossary_cache) @property def keys_cache_key(self): return f"translation-keys-{self.pk}" def invalidate_keys(self): cache.delete(self.keys_cache_key) def get_export_url(self): """Return URL of exported git repository.""" return self.component.get_export_url() def remove(self, user): """Remove translation from the VCS.""" author = user.get_author_name() # Log self.log_info("removing %s as %s", self.filenames, author) # Remove file from VCS if any(os.path.exists(name) for name in self.filenames): with self.component.repository.lock: self.component.repository.remove( self.filenames, self.get_commit_message( author, template=self.component.delete_message ), author, ) self.component.push_if_needed() # Delete from the database self.stats.invalidate() self.delete() # Record change Change.objects.create( component=self.component, action=Change.ACTION_REMOVE_TRANSLATION, target=self.filename, user=user, author=user, ) def handle_store_change(self, request, user, previous_revision: str, change=None): if self.is_source: self.component.create_translations(request=request) else: self.check_sync(request=request, change=change) self.invalidate_cache() # Trigger post-update signal self.component.trigger_post_update(previous_revision, False) def get_store_change_translations(self): component = self.component if not self.is_source or component.has_template(): return [self] return component.translation_set.exclude(id=self.id) @transaction.atomic def add_unit( # noqa: C901 self, request, context: str, source: Union[str, List[str]], target: Optional[Union[str, List[str]]] = None, extra_flags: str = "", auto_context: bool = False, is_batch_update: bool = False, ): user = request.user if request else None component = self.component if self.is_source: translations = [self] translations.extend(component.translation_set.exclude(id=self.id)) else: translations = [component.source_translation, self] has_template = component.has_template() source_unit = None result = None # Automatic context suffix = 0 base = context while self.unit_set.filter(context=context, source=source).exists(): suffix += 1 context = f"{base}{suffix}" for translation in translations: is_source = translation.is_source kwargs = {} if has_template: kwargs["pending"] = is_source else: kwargs["pending"] = not is_source if kwargs["pending"]: kwargs["details"] = {"add_unit": True} if is_source: current_target = source kwargs["extra_flags"] = extra_flags else: current_target = target if current_target is None: current_target = "" if isinstance(current_target, list): current_target = join_plural(current_target) if isinstance(source, list): source = join_plural(source) if has_template: id_hash = calculate_hash(context) else: id_hash = calculate_hash(source, context) # When adding to a target the source string can already exist unit = None if not self.is_source and is_source: try: unit = translation.unit_set.get(id_hash=id_hash) flags = Flags(unit.extra_flags) flags.merge(extra_flags) new_flags = flags.format() if unit.extra_flags != new_flags: unit.save(update_fields=["extra_flags"], same_content=True) except Unit.DoesNotExist: pass if unit is None: unit = Unit( translation=translation, context=context, source=source, target=current_target, state=STATE_TRANSLATED if bool(current_target) else STATE_EMPTY, source_unit=source_unit, id_hash=id_hash, position=0, **kwargs, ) unit.is_batch_update = is_batch_update unit.save(force_insert=True) Change.objects.create( unit=unit, action=Change.ACTION_NEW_UNIT, target=source, user=user, author=user, ) # The source language is always first in the translations array if source_unit is None: source_unit = unit if translation == self: result = unit if not is_batch_update: component.update_variants() component.sync_terminology() return result @transaction.atomic def delete_unit(self, request, unit): from weblate.auth.models import get_anonymous component = self.component user = request.user if request else get_anonymous() with component.repository.lock: component.commit_pending("delete unit", user) previous_revision = self.component.repository.last_revision for translation in self.get_store_change_translations(): try: pounit, add = translation.store.find_unit(unit.context, unit.source) except UnitNotFound: return if add: return extra_files = translation.store.remove_unit(pounit.unit) translation.addon_commit_files.extend(extra_files) translation.drop_store_cache() translation.git_commit(user, user.get_author_name(), store_hash=False) self.handle_store_change(request, user, previous_revision) @transaction.atomic def sync_terminology(self): if self.is_source: return for source in self.component.get_all_sources(): # Is the string a terminology if "terminology" not in source.all_flags: continue # Does it already exist if self.unit_set.filter(id_hash=source.id_hash).exists(): continue # Unit is already present self.add_unit(None, source.context, source.get_source_plurals(), "") def validate_new_unit_data( # noqa: C901 self, context: str, source: Union[str, List[str]], target: Optional[Union[str, List[str]]] = None, auto_context: bool = False, extra_flags: Optional[str] = None, ): extra = {} if isinstance(source, str): source = [source] if isinstance(target, str): target = [target] if not self.component.has_template(): extra["source"] = join_plural(source) if not auto_context and self.unit_set.filter(context=context, **extra).exists(): raise ValidationError(_("This string seems to already exist.")) # Avoid using source translations without a filename if not self.filename: try: translation = self.component.translation_set.exclude(pk=self.pk)[0] except IndexError: raise ValidationError( _("Failed adding string: %s") % _("No translation found.") ) translation.validate_new_unit_data( context, source, target, auto_context=auto_context, extra_flags=extra_flags, ) return # Always load a new copy of store store = self.load_store() old_units = len(store.all_units) # Add new unit store.new_unit(context, source, target, skip_build=True) # Serialize the content handle = BytesIOMode("", b"") # Catch serialization error try: store.save_content(handle) except Exception as error: raise ValidationError(_("Failed adding string: %s") % error) handle.seek(0) # Parse new file (check that it is valid) try: newstore = self.load_store(handle) except Exception as error: raise ValidationError(_("Failed adding string: %s") % error) # Verify there is a single unit added if len(newstore.all_units) != old_units + 1: raise ValidationError( _("Failed adding string: %s") % _("Failed to parse new string") ) # Find newly added unit (it can be on any position), but we assume # the storage has consistent ordering unit = None for pos, current in enumerate(newstore.all_units): if pos >= old_units or ( current.source != store.all_units[pos].source and current.context != store.all_units[pos].context ): unit = current break # Verify unit matches data if unit is None: raise ValidationError( _("Failed adding string: %s") % _("Failed to parse new string") ) created_source = split_plural(unit.source) if unit.context != context and ( self.component.has_template() or self.component.file_format_cls.set_context_bilingual ): raise ValidationError( {"context": _('Context would be created as "%s"') % unit.context} ) if created_source != source: raise ValidationError( {"source": _("Source would be created as %s") % created_source} )
class Translation(models.Model, URLMixin, LoggerMixin): component = models.ForeignKey("Component", on_delete=models.deletion.CASCADE) language = models.ForeignKey(Language, on_delete=models.deletion.CASCADE) plural = models.ForeignKey(Plural, on_delete=models.deletion.CASCADE) revision = models.CharField(max_length=200, default="", blank=True) filename = models.CharField(max_length=FILENAME_LENGTH) language_code = models.CharField(max_length=20, default="", blank=True) check_flags = models.TextField( verbose_name="Translation flags", default="", validators=[validate_check_flags], blank=True, ) objects = TranslationManager.from_queryset(TranslationQuerySet)() is_lockable = False _reverse_url_name = "translation" class Meta: app_label = "trans" unique_together = ("component", "language") verbose_name = "translation" verbose_name_plural = "translations" def __str__(self): return "{0} — {1}".format(self.component, self.language) def __init__(self, *args, **kwargs): """Constructor to initialize some cache properties.""" super().__init__(*args, **kwargs) self.stats = TranslationStats(self) self.addon_commit_files = [] self.commit_template = "" self.was_new = 0 self.reason = "" def get_badges(self): if self.is_source: yield (_("source"), _("This translation is used for source strings.")) @cached_property def full_slug(self): return "/".join((self.component.project.slug, self.component.slug, self.language.code)) def log_hook(self, level, msg, *args): self.component.store_log(self.full_slug, msg, *args) @cached_property def is_template(self): """Check whether this is template translation. This means that translations should be propagated as sources to others. """ return self.filename == self.component.template @cached_property def is_source(self): """Check whether this is source strings. This means that translations should be propagated as sources to others. """ return self.language_id == self.component.project.source_language_id @cached_property def all_flags(self): """Return parsed list of flags.""" return Flags(self.component.all_flags, self.check_flags) @cached_property def is_readonly(self): return "read-only" in self.all_flags def clean(self): """Validate that filename exists and can be opened using translate-toolkit.""" if not os.path.exists(self.get_filename()): raise ValidationError( _("Filename %s not found in repository! To add new " "translation, add language file into repository.") % self.filename) try: self.load_store() except Exception as error: raise ValidationError( _("Failed to parse file %(file)s: %(error)s") % { "file": self.filename, "error": str(error) }) def notify_new(self, request): if self.was_new: # Create change after flags has been updated and cache # invalidated, otherwise we might be sending notification # with outdated values Change.objects.create( translation=self, action=Change.ACTION_NEW_STRING, user=request.user if request else None, author=request.user if request else None, details={"count": self.was_new}, ) self.was_new = 0 def get_reverse_url_kwargs(self): """Return kwargs for URL reversing.""" return { "project": self.component.project.slug, "component": self.component.slug, "lang": self.language.code, } def get_widgets_url(self): """Return absolute URL for widgets.""" return get_site_url("{0}?lang={1}&component={2}".format( reverse("widgets", kwargs={"project": self.component.project.slug}), self.language.code, self.component.slug, )) def get_share_url(self): """Return absolute URL usable for sharing.""" return get_site_url( reverse( "engage", kwargs={ "project": self.component.project.slug, "lang": self.language.code, }, )) def get_translate_url(self): return reverse("translate", kwargs=self.get_reverse_url_kwargs()) def get_filename(self): """Return absolute filename.""" if not self.filename: return None return os.path.join(self.component.full_path, self.filename) def load_store(self, fileobj=None, force_intermediate=False): """Load translate-toolkit storage from disk.""" if fileobj is None: fileobj = self.get_filename() # Use intermediate store as template for source translation if force_intermediate or (self.is_template and self.component.intermediate): template = self.component.intermediate_store else: template = self.component.template_store store = self.component.file_format_cls.parse( fileobj, template, language_code=self.language_code, is_template=self.is_template, ) store_post_load.send(sender=self.__class__, translation=self, store=store) return store @cached_property def store(self): """Return translate-toolkit storage object for a translation.""" try: return self.load_store() except FileParseError: raise except Exception as exc: report_error(cause="Translation parse error") self.component.handle_parse_error(exc, self) def sync_unit(self, dbunits, updated, id_hash, unit, pos): try: newunit = dbunits[id_hash] is_new = False except KeyError: newunit = Unit(translation=self, id_hash=id_hash, state=-1) is_new = True newunit.update_from_unit(unit, pos, is_new) # Check if unit is worth notification: # - new and untranslated # - newly not translated # - newly fuzzy # - source string changed if newunit.state < STATE_TRANSLATED and ( newunit.state != newunit.old_unit.state or is_new or newunit.source != newunit.old_unit.source): self.was_new += 1 # Store current unit ID updated[id_hash] = newunit def check_sync(self, force=False, request=None, change=None): # noqa: C901 """Check whether database is in sync with git and possibly updates.""" if change is None: change = Change.ACTION_UPDATE if request is None: user = None else: user = request.user # Check if we're not already up to date if not self.revision: self.reason = "new file" elif self.revision != self.get_git_blob_hash(): self.reason = "content changed" elif force: self.reason = "check forced" else: self.reason = "" return self.log_info("processing %s, %s", self.filename, self.reason) # List of updated units (used for cleanup and duplicates detection) updated = {} # Position of current unit pos = 0 try: store = self.store translation_store = None # Store plural plural = store.get_plural(self.language) if plural != self.plural: self.plural = plural self.save(update_fields=["plural"]) # Was there change? self.was_new = 0 # Select all current units for update dbunits = { unit.id_hash: unit for unit in self.unit_set.select_for_update() } # Process based on intermediate store if available if self.component.intermediate: translation_store = store store = self.load_store(force_intermediate=True) for unit in store.content_units: # Use translation store if exists and if it contains the string if translation_store is not None: try: translated_unit, created = translation_store.find_unit( unit.context) if translated_unit and not created: unit = translated_unit else: # Patch unit to have matching source unit.source = translated_unit.source except UnitNotFound: pass id_hash = unit.id_hash # Check for possible duplicate units if id_hash in updated: newunit = updated[id_hash] self.log_warning( "duplicate string to translate: %s (%s)", newunit, repr(newunit.source), ) Change.objects.create( unit=newunit, action=Change.ACTION_DUPLICATE_STRING, user=user, author=user, ) self.component.trigger_alert( "DuplicateString", language_code=self.language.code, source=newunit.source, unit_pk=newunit.pk, ) continue # Update position pos += 1 self.sync_unit(dbunits, updated, id_hash, unit, pos) except FileParseError as error: self.log_warning("skipping update due to parse error: %s", error) return # Delete stale units stale = set(dbunits) - set(updated) if stale: self.unit_set.filter(id_hash__in=stale).delete() self.component.needs_cleanup = True # We should also do cleanup on source strings tracking objects # Update revision and stats self.store_hash() # Store change entry Change.objects.create(translation=self, action=change, user=user, author=user) # Invalidate keys cache transaction.on_commit(self.invalidate_keys) def do_update(self, request=None, method=None): return self.component.do_update(request, method=method) def do_push(self, request=None): return self.component.do_push(request) def do_reset(self, request=None): return self.component.do_reset(request) def do_cleanup(self, request=None): return self.component.do_cleanup(request) def can_push(self): return self.component.can_push() def get_git_blob_hash(self): """Return current VCS blob hash for file.""" get_object_hash = self.component.repository.get_object_hash # Include language file hashes = [get_object_hash(self.get_filename())] if self.component.has_template(): # Include template hashes.append(get_object_hash(self.component.template)) if self.component.intermediate: # Include intermediate language as it might add new strings hashes.append(get_object_hash(self.component.intermediate)) return ",".join(hashes) def store_hash(self): """Store current hash in database.""" self.revision = self.get_git_blob_hash() self.save(update_fields=["revision"]) def get_last_author(self, email=False): """Return last autor of change done in Weblate.""" if not self.stats.last_author: return None from weblate.auth.models import User return User.objects.get( pk=self.stats.last_author).get_author_name(email) def commit_pending(self, reason, user, skip_push=False, force=False, signals=True): """Commit any pending changes.""" if not force and not self.needs_commit(): return False self.log_info("committing pending changes (%s)", reason) try: store = self.store except FileParseError as error: report_error(cause="Failed to parse file on commit") self.log_error("skipping commit due to error: %s", error) return False with self.component.repository.lock, transaction.atomic(): while True: # Find oldest change break loop if there is none left try: unit = (self.unit_set.filter(pending=True).annotate( Max("change__timestamp")).order_by( "change__timestamp__max")[0]) except IndexError: break # Get last change metadata author, timestamp = unit.get_last_content_change() author_name = author.get_author_name() # Flush pending units for this author self.update_units(store, author_name, author.id) # Commit changes self.git_commit(user, author_name, timestamp, skip_push=skip_push, signals=signals) # Update stats (the translated flag might have changed) self.invalidate_cache() return True def get_commit_message(self, author, template=None, **kwargs): """Format commit message based on project configuration.""" if template is None: if self.commit_template == "add": template = self.component.add_message self.commit_template = "" elif self.commit_template == "delete": template = self.component.delete_message self.commit_template = "" else: template = self.component.commit_message return render_template(template, translation=self, author=author, **kwargs) def __git_commit(self, author, timestamp, signals=True): """Commit translation to git.""" # Format commit message msg = self.get_commit_message(author) # Pre commit hook vcs_pre_commit.send(sender=self.__class__, translation=self, author=author) # Create list of files to commit files = self.filenames # Do actual commit if self.repo_needs_commit(): self.component.repository.commit(msg, author, timestamp, files + self.addon_commit_files) self.addon_commit_files = [] # Post commit hook if signals: vcs_post_commit.send(sender=self.__class__, component=self.component, translation=self) # Store updated hash self.store_hash() def needs_commit(self): """Check whether there are some not committed changes.""" return self.unit_set.filter(pending=True).exists() def repo_needs_merge(self): return self.component.repo_needs_merge() def repo_needs_push(self): return self.component.repo_needs_push() @cached_property def filenames(self): if not self.filename: return [] if self.component.file_format_cls.simple_filename: return [self.get_filename()] return self.store.get_filenames() def repo_needs_commit(self): return self.component.repository.needs_commit(*self.filenames) def git_commit(self, user, author, timestamp, skip_push=False, signals=True): """Wrapper for committing translation to git.""" repository = self.component.repository with repository.lock: # Is there something for commit? if not self.repo_needs_commit(): return False # Do actual commit with git lock self.log_info("committing %s as %s", self.filenames, author) Change.objects.create(action=Change.ACTION_COMMIT, translation=self, user=user) self.__git_commit(author, timestamp, signals=signals) # Push if we should if not skip_push: self.component.push_if_needed() return True @transaction.atomic def update_units(self, store, author_name, author_id): """Update backend file and unit.""" updated = False for unit in self.unit_set.filter(pending=True).select_for_update(): # Skip changes by other authors change_author = unit.get_last_content_change()[0] if change_author.id != author_id: continue # Remove pending flag unit.pending = False try: pounit, add = store.find_unit(unit.context, unit.source) except UnitNotFound: # Bail out if we have not found anything report_error(cause="String disappeared") self.log_error("disappeared string: %s", unit) unit.save(update_fields=["pending"], same_content=True, same_state=True) continue # Check for changes if ((not add or unit.target == "") and unit.target == pounit.target and unit.approved == pounit.is_approved(unit.approved) and unit.fuzzy == pounit.is_fuzzy()): unit.save(update_fields=["pending"], same_content=True, same_state=True) continue updated = True # Optionally add unit to translation file. # This has be done prior setting tatget as some formats # generate content based on target language. if add: store.add_unit(pounit.unit) # Store translations if unit.is_plural(): pounit.set_target(unit.get_target_plurals()) else: pounit.set_target(unit.target) # Update fuzzy/approved flag pounit.mark_fuzzy(unit.state == STATE_FUZZY) pounit.mark_approved(unit.state == STATE_APPROVED) # Update comments as they might have been changed by state changes state = unit.get_unit_state(pounit, "") flags = pounit.flags same_state = True if state != unit.state or flags != unit.flags: unit.state = state unit.flags = flags same_state = False unit.save( update_fields=["state", "flags", "pending"], same_content=True, same_state=same_state, ) # Did we do any updates? if not updated: return # Update po file header now = timezone.now() if not timezone.is_aware(now): now = timezone.make_aware(now, timezone.utc) # Prepare headers to update headers = { "add": True, "last_translator": author_name, "plural_forms": self.plural.plural_form, "language": self.language_code, "PO_Revision_Date": now.strftime("%Y-%m-%d %H:%M%z"), } # Optionally store language team with link to website if self.component.project.set_language_team: headers["language_team"] = "{0} <{1}>".format( self.language.name, get_site_url(self.get_absolute_url())) # Optionally store email for reporting bugs in source report_source_bugs = self.component.report_source_bugs if report_source_bugs: headers["report_msgid_bugs_to"] = report_source_bugs # Update genric headers store.update_header(**headers) # save translation changes store.save() @cached_property def enable_review(self): project = self.component.project return project.source_review if self.is_source else project.translation_review @cached_property def list_translation_checks(self): """Return list of failing checks on current translation.""" result = TranslationChecklist() # All strings result.add(self.stats, "all", "success") if not self.is_readonly: if self.enable_review: result.add_if(self.stats, "approved", "success") # Count of translated strings result.add_if(self.stats, "translated", "success") # To approve if self.enable_review: result.add_if(self.stats, "unapproved", "warning") # Approved with suggestions result.add_if(self.stats, "approved_suggestions", "danger") # Untranslated strings result.add_if(self.stats, "todo", "danger") # Not translated strings result.add_if(self.stats, "nottranslated", "danger") # Fuzzy strings result.add_if(self.stats, "fuzzy", "danger") # Translations with suggestions result.add_if(self.stats, "suggestions", "info") result.add_if(self.stats, "nosuggestions", "info") # All checks result.add_if(self.stats, "allchecks", "danger") # Translated strings with checks if not self.is_source: result.add_if(self.stats, "translated_checks", "danger") # Process specific checks for check in CHECKS: check_obj = CHECKS[check] result.add_if(self.stats, check_obj.url_id, "warning") # Grab comments result.add_if(self.stats, "comments", "info") # Include labels labels = self.component.project.label_set.order_by("name") if labels: for label in labels: result.add_if(self.stats, "label:{}".format(label.name), "info") result.add_if(self.stats, "unlabeled", "info") return result def merge_translations(self, request, store2, overwrite, method, fuzzy): """Merge translation unit wise. Needed for template based translations to add new strings. """ not_found = 0 skipped = 0 accepted = 0 add_fuzzy = method == "fuzzy" add_approve = method == "approve" for set_fuzzy, unit2 in store2.iterate_merge(fuzzy): try: unit = self.unit_set.get_unit(unit2) except Unit.DoesNotExist: not_found += 1 continue state = STATE_TRANSLATED if add_fuzzy or set_fuzzy: state = STATE_FUZZY elif add_approve: state = STATE_APPROVED if ((unit.translated and not overwrite) or unit.readonly or (not request.user.has_perm("unit.edit", unit)) or (unit.target == unit2.target and unit.state == state)): skipped += 1 continue accepted += 1 # We intentionally avoid propagating: # - in most cases it's not desired # - it slows down import considerably # - it brings locking issues as import is # executed with lock held and linked repos # can't obtain the lock unit.translate( request.user, split_plural(unit2.target), state, change_action=Change.ACTION_UPLOAD, propagate=False, ) if accepted > 0: self.invalidate_cache() request.user.profile.refresh_from_db() request.user.profile.translated += accepted request.user.profile.save(update_fields=["translated"]) return (not_found, skipped, accepted, len(list(store2.content_units))) def merge_suggestions(self, request, store, fuzzy): """Merge content of translate-toolkit store as a suggestions.""" not_found = 0 skipped = 0 accepted = 0 for _unused, unit in store.iterate_merge(fuzzy): # Grab database unit try: dbunit = self.unit_set.get_unit(unit) except Unit.DoesNotExist: not_found += 1 continue # Add suggestion if dbunit.target != unit.target and not dbunit.readonly: if Suggestion.objects.add(dbunit, unit.target, request): accepted += 1 else: skipped += 1 else: skipped += 1 # Update suggestion count if accepted > 0: self.invalidate_cache() return (not_found, skipped, accepted, len(list(store.content_units))) def drop_store_cache(self): if "store" in self.__dict__: del self.__dict__["store"] def handle_source(self, request, fileobj): """Replace source translations with uploaded one.""" component = self.component filenames = [] with component.repository.lock: # Commit pending changes component.commit_pending("source update", request.user) # Create acutal file with the file temp = tempfile.NamedTemporaryFile(prefix="weblate-upload", dir=self.component.full_path, delete=False) temp.write(fileobj.read()) temp.close() try: # Update translation files for translation in component.translation_set.exclude( language=component.project.source_language): filename = translation.get_filename() component.file_format_cls.update_bilingual( filename, temp.name) filenames.append(filename) finally: if os.path.exists(temp.name): if component.new_base: filename = component.get_new_base_filename() os.replace(temp.name, filename) filenames.append(filename) else: os.unlink(temp.name) # Commit changes if component.repository.needs_commit(*filenames): component.repository.commit( self.get_commit_message( request.user.get_author_name(), template=component.addon_message, addon_name="Source update", ), files=filenames, ) component.create_translations(request=request, force=True) component.push_if_needed(None) return (0, 0, self.unit_set.count(), self.unit_set.count()) def handle_replace(self, request, fileobj): """Replace file content with uploaded one.""" filecopy = fileobj.read() fileobj.close() fileobj = BytesIOMode(fileobj.name, filecopy) with self.component.repository.lock: self.commit_pending("replace file", request.user) # This will throw an exception in case of error store2 = self.load_store(fileobj) store2.check_valid() # Actually replace file content self.store.save_atomic(self.store.storefile, lambda handle: handle.write(filecopy)) # Commit to VCS if self.repo_needs_commit(): self.__git_commit(request.user.get_author_name(), timezone.now()) # Drop store cache self.drop_store_cache() # Parse the file again if self.is_template: self.component.create_translations(request=request, force=True) else: self.check_sync( force=True, request=request, change=Change.ACTION_REPLACE_UPLOAD, ) self.invalidate_cache() return (0, 0, self.unit_set.count(), len(list(store2.content_units))) @transaction.atomic def merge_upload( self, request, fileobj, overwrite, author_name=None, author_email=None, method="translate", fuzzy="", ): """Top level handler for file uploads.""" # Optionally set authorship orig_user = None if author_email: from weblate.auth.models import User orig_user = request.user request.user = User.objects.get_or_create( email=author_email, defaults={ "username": author_email, "is_active": False, "full_name": author_name or author_email, }, )[0] try: if method == "replace": return self.handle_replace(request, fileobj) if method == "source": return self.handle_source(request, fileobj) filecopy = fileobj.read() fileobj.close() # Strip possible UTF-8 BOM if filecopy[:3] == codecs.BOM_UTF8: filecopy = filecopy[3:] # Load backend file store = try_load( fileobj.name, filecopy, self.component.file_format_cls, self.component.template_store, ) # Check valid plural forms if hasattr(store.store, "parseheader"): header = store.store.parseheader() try: number, formula = Plural.parse_plural_forms( header["Plural-Forms"]) if not self.plural.same_plural(number, formula): raise PluralFormsMismatch() except (ValueError, KeyError): # Formula wrong or missing pass if method in ("translate", "fuzzy", "approve"): # Merge on units level with self.component.repository.lock: return self.merge_translations(request, store, overwrite, method, fuzzy) # Add as sugestions return self.merge_suggestions(request, store, fuzzy) finally: if orig_user: request.user = orig_user def invalidate_cache(self, recurse=True): """Invalidate any cached stats.""" # Invalidate summary stats transaction.on_commit(lambda: self.stats.invalidate(recurse)) @property def keys_cache_key(self): return "translation-keys-{}".format(self.pk) def invalidate_keys(self): cache.delete(self.keys_cache_key) def get_export_url(self): """Return URL of exported git repository.""" return self.component.get_export_url() def get_stats(self): """Return stats dictionary.""" stats = self.stats return { "code": self.language.code, "name": self.language.name, "total": stats.all, "total_words": stats.all_words, "last_change": stats.last_changed, "last_author": self.get_last_author(), "recent_changes": stats.recent_changes, "translated": stats.translated, "translated_words": stats.translated_words, "translated_percent": stats.translated_percent, "translated_words_percent": stats.translated_words_percent, "translated_chars": stats.translated_chars, "translated_chars_percent": stats.translated_chars_percent, "total_chars": stats.all_chars, "fuzzy": stats.fuzzy, "fuzzy_percent": stats.fuzzy_percent, "failing": stats.allchecks, "failing_percent": stats.allchecks_percent, "url": self.get_share_url(), "url_translate": get_site_url(self.get_absolute_url()), } def remove(self, user): """Remove translation from the VCS.""" author = user.get_author_name() # Log self.log_info("removing %s as %s", self.filenames, author) # Remove file from VCS if any((os.path.exists(name) for name in self.filenames)): self.commit_template = "delete" with self.component.repository.lock: self.component.repository.remove( self.filenames, self.get_commit_message(author), author) # Delete from the database self.stats.invalidate() self.delete() # Record change Change.objects.create( component=self.component, action=Change.ACTION_REMOVE_TRANSLATION, target=self.filename, user=user, author=user, ) def new_unit(self, request, key, value): with self.component.repository.lock: self.commit_pending("new unit", request.user) Change.objects.create( translation=self, action=Change.ACTION_NEW_UNIT, target=value, user=request.user, author=request.user, ) self.store.new_unit(key, value) self.component.create_translations(request=request) self.__git_commit(request.user.get_author_name(), timezone.now()) self.component.push_if_needed()
class Translation(models.Model, URLMixin, LoggerMixin): component = models.ForeignKey('Component', on_delete=models.deletion.CASCADE) language = models.ForeignKey(Language, on_delete=models.deletion.CASCADE) plural = models.ForeignKey(Plural, on_delete=models.deletion.CASCADE) revision = models.CharField(max_length=100, default='', blank=True) filename = models.CharField(max_length=FILENAME_LENGTH) language_code = models.CharField(max_length=20, default='', blank=True) check_flags = models.TextField( verbose_name="Translation flags", default="", validators=[validate_check_flags], blank=True, ) objects = TranslationManager.from_queryset(TranslationQuerySet)() is_lockable = False _reverse_url_name = 'translation' class Meta: app_label = 'trans' unique_together = ('component', 'language') def __init__(self, *args, **kwargs): """Constructor to initialize some cache properties.""" super().__init__(*args, **kwargs) self.stats = TranslationStats(self) self.addon_commit_files = [] self.commit_template = '' self.was_new = False self.reason = '' def get_badges(self): if self.is_source: yield (_('source'), _('This translation is used for source strings.')) @cached_property def full_slug(self): return '/'.join((self.component.project.slug, self.component.slug, self.language.code)) def log_hook(self, level, msg, *args): self.component.store_log(self.full_slug, msg, *args) @cached_property def is_template(self): """Check whether this is template translation. This means that translations should be propagated as sources to others. """ return self.filename == self.component.template @cached_property def is_source(self): """Check whether this is source strings. This means that translations should be propagated as sources to others. """ return self.language == self.component.project.source_language @cached_property def all_flags(self): """Return parsed list of flags.""" return Flags(self.component.all_flags, self.check_flags) @cached_property def is_readonly(self): return 'read-only' in self.all_flags def clean(self): """Validate that filename exists and can be opened using translate-toolkit.""" if not os.path.exists(self.get_filename()): raise ValidationError( _('Filename %s not found in repository! To add new ' 'translation, add language file into repository.') % self.filename) try: self.load_store() except Exception as error: raise ValidationError( _('Failed to parse file %(file)s: %(error)s') % { 'file': self.filename, 'error': str(error) }) def notify_new(self, request): if self.was_new: # Create change after flags has been updated and cache # invalidated, otherwise we might be sending notification # with outdated values Change.objects.create( translation=self, action=Change.ACTION_NEW_STRING, user=request.user if request else None, author=request.user if request else None, ) self.was_new = False def get_reverse_url_kwargs(self): """Return kwargs for URL reversing.""" return { 'project': self.component.project.slug, 'component': self.component.slug, 'lang': self.language.code, } def get_widgets_url(self): """Return absolute URL for widgets.""" return get_site_url('{0}?lang={1}&component={2}'.format( reverse('widgets', kwargs={'project': self.component.project.slug}), self.language.code, self.component.slug, )) def get_share_url(self): """Return absolute URL usable for sharing.""" return get_site_url( reverse( 'engage', kwargs={ 'project': self.component.project.slug, 'lang': self.language.code, }, )) def get_translate_url(self): return reverse('translate', kwargs=self.get_reverse_url_kwargs()) def __str__(self): return '{0} — {1}'.format(self.component, self.language) def get_filename(self): """Return absolute filename.""" if not self.filename: return None return os.path.join(self.component.full_path, self.filename) def load_store(self, fileobj=None): """Load translate-toolkit storage from disk.""" if fileobj is None: fileobj = self.get_filename() store = self.component.file_format_cls.parse( fileobj, self.component.template_store, language_code=self.language_code, is_template=self.is_template, ) store_post_load.send(sender=self.__class__, translation=self, store=store) return store @cached_property def store(self): """Return translate-toolkit storage object for a translation.""" try: return self.load_store() except FileParseError: raise except Exception as exc: self.component.handle_parse_error(exc, self) def check_sync(self, force=False, request=None, change=None): """Check whether database is in sync with git and possibly updates.""" if change is None: change = Change.ACTION_UPDATE if request is None: user = None else: user = request.user # Check if we're not already up to date if not self.revision: self.reason = 'new file' elif self.revision != self.get_git_blob_hash(): self.reason = 'content changed' elif force: self.reason = 'check forced' else: self.reason = '' return self.log_info('processing %s, %s', self.filename, self.reason) # List of updated units (used for cleanup and duplicates detection) updated = {} try: store = self.store # Store plural plural = store.get_plural(self.language) if plural != self.plural: self.plural = plural self.save(update_fields=['plural']) # Was there change? self.was_new = False # Position of current unit pos = 0 # Select all current units for update dbunits = { unit.id_hash: unit for unit in self.unit_set.select_for_update() } for unit in store.content_units: id_hash = unit.id_hash # Update position pos += 1 # Check for possible duplicate units if id_hash in updated: newunit = updated[id_hash] self.log_warning( 'duplicate string to translate: %s (%s)', newunit, repr(newunit.source), ) Change.objects.create( unit=newunit, action=Change.ACTION_DUPLICATE_STRING, user=user, author=user, ) self.component.trigger_alert( 'DuplicateString', language_code=self.language.code, source=newunit.source, unit_pk=newunit.pk, ) continue try: newunit = dbunits[id_hash] is_new = False except KeyError: newunit = Unit(translation=self, id_hash=id_hash, state=-1) is_new = True newunit.update_from_unit(unit, pos, is_new) # Check if unit is worth notification: # - new and untranslated # - newly not translated # - newly fuzzy # - source string changed self.was_new = self.was_new or ( newunit.state < STATE_TRANSLATED and (newunit.state != newunit.old_unit.state or is_new or newunit.source != newunit.old_unit.source)) # Store current unit ID updated[id_hash] = newunit except FileParseError as error: self.log_warning('skipping update due to parse error: %s', error) return # Delete stale units stale = set(dbunits) - set(updated) if stale: self.unit_set.filter(id_hash__in=stale).delete() self.component.needs_cleanup = True # We should also do cleanup on source strings tracking objects # Update revision and stats self.store_hash() # Store change entry Change.objects.create(translation=self, action=change, user=user, author=user) # Invalidate keys cache transaction.on_commit(self.invalidate_keys) def do_update(self, request=None, method=None): return self.component.do_update(request, method=method) def do_push(self, request=None): return self.component.do_push(request) def do_reset(self, request=None): return self.component.do_reset(request) def do_cleanup(self, request=None): return self.component.do_cleanup(request) def can_push(self): return self.component.can_push() def get_git_blob_hash(self): """Return current VCS blob hash for file.""" ret = self.component.repository.get_object_hash(self.get_filename()) if not self.component.has_template(): return ret return ','.join([ ret, self.component.repository.get_object_hash(self.component.template) ]) def store_hash(self): """Store current hash in database.""" self.revision = self.get_git_blob_hash() self.save(update_fields=['revision']) def get_last_author(self, email=False): """Return last autor of change done in Weblate.""" if not self.stats.last_author: return None from weblate.auth.models import User return User.objects.get( pk=self.stats.last_author).get_author_name(email) def commit_pending(self, reason, user, skip_push=False, force=False, signals=True): """Commit any pending changes.""" if not force and not self.needs_commit(): return False self.log_info('committing pending changes (%s)', reason) with self.component.repository.lock, transaction.atomic(): while True: # Find oldest change break loop if there is none left try: unit = (self.unit_set.filter(pending=True).annotate( Max('change__timestamp')).order_by( 'change__timestamp__max')[0]) except IndexError: break # Get last change metadata author, timestamp = unit.get_last_content_change() author_name = author.get_author_name() # Flush pending units for this author self.update_units(author_name, author.id) # Commit changes self.git_commit(user, author_name, timestamp, skip_push=skip_push, signals=signals) # Update stats (the translated flag might have changed) self.invalidate_cache() return True def get_commit_message(self, author): """Format commit message based on project configuration.""" if self.commit_template == 'add': template = self.component.add_message self.commit_template = '' elif self.commit_template == 'delete': template = self.component.delete_message self.commit_template = '' else: template = self.component.commit_message return render_template(template, translation=self, author=author) def __git_commit(self, author, timestamp, signals=True): """Commit translation to git.""" # Format commit message msg = self.get_commit_message(author) # Pre commit hook vcs_pre_commit.send(sender=self.__class__, translation=self, author=author) # Create list of files to commit files = self.filenames # Do actual commit if self.repo_needs_commit(): self.component.repository.commit(msg, author, timestamp, files + self.addon_commit_files) self.addon_commit_files = [] # Post commit hook if signals: vcs_post_commit.send(sender=self.__class__, component=self.component, translation=self) # Store updated hash self.store_hash() def needs_commit(self): """Check whether there are some not committed changes.""" return self.unit_set.filter(pending=True).exists() def repo_needs_merge(self): return self.component.repo_needs_merge() def repo_needs_push(self): return self.component.repo_needs_push() @cached_property def filenames(self): if not self.filename: return [] if self.component.file_format_cls.simple_filename: return [self.get_filename()] return self.store.get_filenames() def repo_needs_commit(self): return self.component.repository.needs_commit(*self.filenames) def git_commit(self, user, author, timestamp, skip_push=False, signals=True): """Wrapper for committing translation to git.""" repository = self.component.repository with repository.lock: # Is there something for commit? if not self.repo_needs_commit(): return False # Do actual commit with git lock self.log_info('committing %s as %s', self.filenames, author) Change.objects.create(action=Change.ACTION_COMMIT, translation=self, user=user) self.__git_commit(author, timestamp, signals=signals) # Push if we should if not skip_push: self.component.push_if_needed() return True @transaction.atomic def update_units(self, author_name, author_id): """Update backend file and unit.""" updated = False for unit in self.unit_set.filter(pending=True).select_for_update(): # Skip changes by other authors change_author = unit.get_last_content_change()[0] if change_author.id != author_id: continue try: pounit, add = self.store.find_unit(unit.context, unit.source) except UnitNotFound as error: report_error(error, prefix='String disappeared') pounit = None unit.pending = False # Bail out if we have not found anything if pounit is None: self.log_error('disappeared string: %s', unit) unit.save(update_fields=['pending'], same_content=True) continue # Check for changes if ((not add or unit.target == '') and unit.target == pounit.target and unit.approved == pounit.is_approved(unit.approved) and unit.fuzzy == pounit.is_fuzzy()): unit.save(update_fields=['pending'], same_content=True) continue updated = True # Optionally add unit to translation file. # This has be done prior setting tatget as some formats # generate content based on target language. if add: self.store.add_unit(pounit.unit) # Store translations if unit.is_plural(): pounit.set_target(unit.get_target_plurals()) else: pounit.set_target(unit.target) # Update fuzzy/approved flag pounit.mark_fuzzy(unit.state == STATE_FUZZY) pounit.mark_approved(unit.state == STATE_APPROVED) # Update comments as they might have been changed (eg, fuzzy flag # removed) state = unit.get_unit_state(pounit, '') flags = pounit.flags if state != unit.state or flags != unit.flags: unit.state = state unit.flags = flags unit.save(update_fields=['state', 'flags', 'pending'], same_content=True) # Did we do any updates? if not updated: return # Update po file header now = timezone.now() if not timezone.is_aware(now): now = timezone.make_aware(now, timezone.utc) # Prepare headers to update headers = { 'add': True, 'last_translator': author_name, 'plural_forms': self.plural.plural_form, 'language': self.language_code, 'PO_Revision_Date': now.strftime('%Y-%m-%d %H:%M%z'), } # Optionally store language team with link to website if self.component.project.set_language_team: headers['language_team'] = '{0} <{1}>'.format( self.language.name, get_site_url(self.get_absolute_url())) # Optionally store email for reporting bugs in source report_source_bugs = self.component.report_source_bugs if report_source_bugs: headers['report_msgid_bugs_to'] = report_source_bugs # Update genric headers self.store.update_header(**headers) # save translation changes self.store.save() def get_source_checks(self): """Return list of failing source checks on current component.""" result = TranslationChecklist() result.add(self.stats, 'all', 'success') # All checks result.add_if(self.stats, 'allchecks', 'danger') # Process specific checks for check in CHECKS: check_obj = CHECKS[check] if not check_obj.source: continue result.add_if(self.stats, check_obj.url_id, check_obj.severity) # Grab comments result.add_if(self.stats, 'comments', 'info') return result def get_target_checks(self): """Return list of failing checks on current component.""" result = TranslationChecklist() # All strings result.add(self.stats, 'all', 'success') result.add_if(self.stats, 'approved', 'success') # Count of translated strings result.add_if(self.stats, 'translated', 'success') # To approve if self.component.project.enable_review: result.add_if(self.stats, 'unapproved', 'warning') # Approved with suggestions result.add_if(self.stats, 'approved_suggestions', 'danger') # Untranslated strings result.add_if(self.stats, 'todo', 'danger') # Not translated strings result.add_if(self.stats, 'nottranslated', 'danger') # Fuzzy strings result.add_if(self.stats, 'fuzzy', 'danger') # Translations with suggestions result.add_if(self.stats, 'suggestions', 'info') result.add_if(self.stats, 'nosuggestions', 'info') # All checks result.add_if(self.stats, 'allchecks', 'danger') # Process specific checks for check in CHECKS: check_obj = CHECKS[check] if not check_obj.target: continue result.add_if(self.stats, check_obj.url_id, 'warning') # Grab comments result.add_if(self.stats, 'comments', 'info') return result @cached_property def list_translation_checks(self): """Return list of failing checks on current translation.""" if self.is_source: result = self.get_source_checks() else: result = self.get_target_checks() # Include labels for label in self.component.project.label_set.all(): result.add_if(self.stats, 'label:{}'.format(label.name), 'info') return result def merge_translations(self, request, store2, overwrite, method, fuzzy): """Merge translation unit wise. Needed for template based translations to add new strings. """ not_found = 0 skipped = 0 accepted = 0 add_fuzzy = method == 'fuzzy' add_approve = method == 'approve' for set_fuzzy, unit2 in store2.iterate_merge(fuzzy): try: unit = self.unit_set.get_unit(unit2) except Unit.DoesNotExist: not_found += 1 continue state = STATE_TRANSLATED if add_fuzzy or set_fuzzy: state = STATE_FUZZY elif add_approve: state = STATE_APPROVED if ((unit.translated and not overwrite) or unit.readonly or (not request.user.has_perm('unit.edit', unit)) or (unit.target == unit2.target and unit.state == state)): skipped += 1 continue accepted += 1 # We intentionally avoid propagating: # - in most cases it's not desired # - it slows down import considerably # - it brings locking issues as import is # executed with lock held and linked repos # can't obtain the lock unit.translate( request.user, split_plural(unit2.target), state, change_action=Change.ACTION_UPLOAD, propagate=False, ) if accepted > 0: self.invalidate_cache() request.user.profile.refresh_from_db() request.user.profile.translated += accepted request.user.profile.save(update_fields=['translated']) return (not_found, skipped, accepted, len(list(store2.content_units))) def merge_suggestions(self, request, store, fuzzy): """Merge content of translate-toolkit store as a suggestions.""" not_found = 0 skipped = 0 accepted = 0 for _unused, unit in store.iterate_merge(fuzzy): # Grab database unit try: dbunit = self.unit_set.get_unit(unit) except Unit.DoesNotExist: not_found += 1 continue # Add suggestion if dbunit.target != unit.target and not dbunit.readonly: if Suggestion.objects.add(dbunit, unit.target, request): accepted += 1 else: skipped += 1 else: skipped += 1 # Update suggestion count if accepted > 0: self.invalidate_cache() return (not_found, skipped, accepted, len(list(store.content_units))) def drop_store_cache(self): if 'store' in self.__dict__: del self.__dict__['store'] def handle_replace(self, request, fileobj): """Replace file content with uploaded one.""" filecopy = fileobj.read() fileobj.close() fileobj = BytesIOMode(fileobj.name, filecopy) with self.component.repository.lock, transaction.atomic(): self.commit_pending('replace file', request.user) # This will throw an exception in case of error store2 = self.load_store(fileobj) store2.check_valid() # Actually replace file content self.store.save_atomic(self.store.storefile, lambda handle: handle.write(filecopy)) # Commit to VCS if self.repo_needs_commit(): self.__git_commit(request.user.get_author_name(), timezone.now()) # Drop store cache self.drop_store_cache() # Parse the file again if self.is_template: self.component.create_translations(request=request, force=True) else: self.check_sync(force=True, request=request, change=Change.ACTION_UPLOAD) self.invalidate_cache() return (0, 0, self.unit_set.count(), len(list(store2.content_units))) @transaction.atomic def merge_upload( self, request, fileobj, overwrite, author_name=None, author_email=None, method='translate', fuzzy='', ): """Top level handler for file uploads.""" # Optionally set authorship orig_user = None if author_email: from weblate.auth.models import User orig_user = request.user request.user = User.objects.get_or_create( email=author_email, defaults={ 'username': author_email, 'is_active': False, 'full_name': author_name or author_email, }, )[0] if method == 'replace': return self.handle_replace(request, fileobj) filecopy = fileobj.read() fileobj.close() # Strip possible UTF-8 BOM if filecopy[:3] == codecs.BOM_UTF8: filecopy = filecopy[3:] # Load backend file store = try_load( fileobj.name, filecopy, self.component.file_format_cls, self.component.template_store, ) # Check valid plural forms if hasattr(store.store, 'parseheader'): header = store.store.parseheader() try: number, equation = Plural.parse_formula(header['Plural-Forms']) if not self.plural.same_plural(number, equation): raise Exception( _('Plural forms do not match the language.')) except (ValueError, KeyError): # Formula wrong or missing pass try: if method in ('translate', 'fuzzy', 'approve'): # Merge on units level with self.component.repository.lock: return self.merge_translations(request, store, overwrite, method, fuzzy) # Add as sugestions return self.merge_suggestions(request, store, fuzzy) finally: if orig_user: request.user = orig_user def invalidate_cache(self): """Invalidate any cached stats.""" # Invalidate summary stats transaction.on_commit(self.stats.invalidate) @property def keys_cache_key(self): return 'translation-keys-{}'.format(self.pk) def invalidate_keys(self): cache.delete(self.keys_cache_key) def get_export_url(self): """Return URL of exported git repository.""" return self.component.get_export_url() def get_stats(self): """Return stats dictionary.""" return { 'code': self.language.code, 'name': self.language.name, 'total': self.stats.all, 'total_words': self.stats.all_words, 'last_change': self.stats.last_changed, 'last_author': self.get_last_author(), 'recent_changes': self.stats.recent_changes, 'translated': self.stats.translated, 'translated_words': self.stats.translated_words, 'translated_percent': self.stats.translated_percent, 'fuzzy': self.stats.fuzzy, 'fuzzy_percent': self.stats.fuzzy_percent, 'failing': self.stats.allchecks, 'failing_percent': self.stats.allchecks_percent, 'url': self.get_share_url(), 'url_translate': get_site_url(self.get_absolute_url()), } def remove(self, user): """Remove translation from the VCS.""" author = user.get_author_name() # Log self.log_info('removing %s as %s', self.filenames, author) # Remove file from VCS if any((os.path.exists(name) for name in self.filenames)): self.commit_template = 'delete' with self.component.repository.lock: self.component.repository.remove( self.filenames, self.get_commit_message(author), author) # Delete from the database self.stats.invalidate() self.delete() # Record change Change.objects.create( component=self.component, action=Change.ACTION_REMOVE_TRANSLATION, target=self.filename, user=user, author=user, ) def new_unit(self, request, key, value): with self.component.repository.lock: self.commit_pending('new unit', request.user) Change.objects.create( translation=self, action=Change.ACTION_NEW_UNIT, target=value, user=request.user, author=request.user, ) self.store.new_unit(key, value) self.component.create_translations(request=request) self.__git_commit(request.user.get_author_name(), timezone.now()) self.component.push_if_needed()
class Translation(models.Model, URLMixin, LoggerMixin): component = models.ForeignKey( 'Component', on_delete=models.deletion.CASCADE ) language = models.ForeignKey(Language, on_delete=models.deletion.CASCADE) plural = models.ForeignKey(Plural, on_delete=models.deletion.CASCADE) revision = models.CharField(max_length=100, default='', blank=True) filename = models.CharField(max_length=200) language_code = models.CharField(max_length=20, default='', blank=True) objects = TranslationManager.from_queryset(TranslationQuerySet)() is_lockable = False _reverse_url_name = 'translation' class Meta(object): ordering = ['language__name'] app_label = 'trans' unique_together = ('component', 'language') def __init__(self, *args, **kwargs): """Constructor to initialize some cache properties.""" super(Translation, self).__init__(*args, **kwargs) self.stats = TranslationStats(self) self.addon_commit_files = [] self.commit_template = '' self.was_new = False @cached_property def full_slug(self): return '/'.join(( self.component.project.slug, self.component.slug, self.language.code, )) def log_hook(self, level, msg, *args): self.component.store_log(self.full_slug, msg, *args) @cached_property def is_template(self): """Check whether this is template translation This means that translations should be propagated as sources to others. """ return self.filename == self.component.template def clean(self): """Validate that filename exists and can be opened using translate-toolkit. """ if not os.path.exists(self.get_filename()): raise ValidationError( _( 'Filename %s not found in repository! To add new ' 'translation, add language file into repository.' ) % self.filename ) try: self.load_store() except Exception as error: raise ValidationError( _('Failed to parse file %(file)s: %(error)s') % { 'file': self.filename, 'error': str(error) } ) def notify_new(self, request): if self.was_new: # Create change after flags has been updated and cache # invalidated, otherwise we might be sending notification # with outdated values Change.objects.create( translation=self, action=Change.ACTION_NEW_STRING, user=request.user if request else None, author=request.user if request else None, ) self.was_new = False def get_reverse_url_kwargs(self): """Return kwargs for URL reversing.""" return { 'project': self.component.project.slug, 'component': self.component.slug, 'lang': self.language.code } def get_widgets_url(self): """Return absolute URL for widgets.""" return get_site_url( '{0}?lang={1}&component={2}'.format( reverse( 'widgets', kwargs={ 'project': self.component.project.slug, } ), self.language.code, self.component.slug, ) ) def get_share_url(self): """Return absolute URL usable for sharing.""" return get_site_url( reverse( 'engage', kwargs={ 'project': self.component.project.slug, 'lang': self.language.code } ) ) def get_translate_url(self): return reverse('translate', kwargs=self.get_reverse_url_kwargs()) def __str__(self): return '{0} - {1}'.format( force_text(self.component), force_text(self.language) ) def get_filename(self): """Return absolute filename.""" return os.path.join(self.component.full_path, self.filename) def load_store(self): """Load translate-toolkit storage from disk.""" store = self.component.file_format_cls.parse( self.get_filename(), self.component.template_store, language_code=self.language_code, is_template=self.is_template, ) store_post_load.send( sender=self.__class__, translation=self, store=store ) return store @cached_property def store(self): """Return translate-toolkit storage object for a translation.""" try: return self.load_store() except FileParseError: raise except Exception as exc: self.component.handle_parse_error(exc, self) def check_sync(self, force=False, request=None, change=None): """Check whether database is in sync with git and possibly updates""" if change is None: change = Change.ACTION_UPDATE if request is None: user = None else: user = request.user # Check if we're not already up to date if not self.revision: reason = 'new file' elif self.revision != self.get_git_blob_hash(): reason = 'content changed' elif force: reason = 'check forced' else: return self.log_info('processing %s, %s', self.filename, reason) # List of created units (used for cleanup and duplicates detection) created = {} try: store = self.store except FileParseError as error: self.log_warning('skipping update due to parse error: %s', error) return # Store plural plural = store.get_plural(self.language) if plural != self.plural: self.plural = plural self.save(update_fields=['plural']) # Was there change? self.was_new = False # Position of current unit pos = 0 # Select all current units for update dbunits = { unit.id_hash: unit for unit in self.unit_set.select_for_update() } for unit in store.all_units: if not unit.is_translatable(): continue id_hash = unit.id_hash # Update position pos += 1 # Check for possible duplicate units if id_hash in created: newunit = created[id_hash] self.log_warning( 'duplicate string to translate: %s (%s)', newunit, repr(newunit.source) ) Change.objects.create( unit=newunit, action=Change.ACTION_DUPLICATE_STRING, user=user, author=user ) self.component.trigger_alert( 'DuplicateString', language_code=self.language.code, source=newunit.source, unit_pk=newunit.pk, ) continue try: newunit = dbunits[id_hash] is_new = False except KeyError: newunit = Unit( translation=self, id_hash=id_hash, state=-1, ) is_new = True newunit.update_from_unit(unit, pos, is_new) # Check if unit is worth notification: # - new and untranslated # - newly not translated # - newly fuzzy self.was_new = ( self.was_new or ( newunit.state < STATE_TRANSLATED and (newunit.state != newunit.old_unit.state or is_new) ) ) # Store current unit ID created[id_hash] = newunit # Following query can get huge, so we should find better way # to delete stale units, probably sort of garbage collection # We should also do cleanup on source strings tracking objects # Delete stale units if self.unit_set.exclude(id_hash__in=created.keys()).delete()[0]: self.component.needs_cleanup = True # Update revision and stats self.store_hash() # Store change entry Change.objects.create( translation=self, action=change, user=user, author=user ) def get_last_remote_commit(self): return self.component.get_last_remote_commit() def do_update(self, request=None, method=None): return self.component.do_update(request, method=method) def do_push(self, request=None): return self.component.do_push(request) def do_reset(self, request=None): return self.component.do_reset(request) def do_cleanup(self, request=None): return self.component.do_cleanup(request) def can_push(self): return self.component.can_push() def get_git_blob_hash(self): """Return current VCS blob hash for file.""" ret = self.component.repository.get_object_hash(self.get_filename()) if not self.component.has_template(): return ret return ','.join([ ret, self.component.repository.get_object_hash( self.component.template ) ]) def store_hash(self): """Store current hash in database.""" self.revision = self.get_git_blob_hash() self.save(update_fields=['revision']) def get_last_author(self, email=False): """Return last autor of change done in Weblate.""" if not self.stats.last_author: return None from weblate.auth.models import User return User.objects.get( pk=self.stats.last_author ).get_author_name(email) def commit_pending(self, reason, request, skip_push=False, force=False): """Commit any pending changes.""" if not force and not self.unit_set.filter(pending=True).exists(): return False self.log_info('committing pending changes (%s)', reason) with self.component.repository.lock: while True: # Find oldest change break loop if there is none left try: unit = self.unit_set.filter( pending=True ).annotate( Max('change__timestamp') ).order_by( 'change__timestamp__max' )[0] except IndexError: break # Get last change metadata author, timestamp = unit.get_last_content_change(request) author_name = author.get_author_name() # Flush pending units for this author self.update_units(author_name, author.id) # Commit changes self.git_commit( request, author_name, timestamp, skip_push=skip_push ) # Update stats (the translated flag might have changed) self.invalidate_cache() return True def get_commit_message(self, author): """Format commit message based on project configuration.""" if self.commit_template == 'add': template = self.component.add_message self.commit_template = '' elif self.commit_template == 'delete': template = self.component.delete_message self.commit_template = '' else: template = self.component.commit_message msg = render_template(template, translation=self, author=author) return msg def __git_commit(self, author, timestamp): """Commit translation to git.""" # Format commit message msg = self.get_commit_message(author) # Pre commit hook vcs_pre_commit.send( sender=self.__class__, translation=self, author=author ) # Create list of files to commit files = self.filenames # Do actual commit self.component.repository.commit( msg, author, timestamp, files + self.addon_commit_files ) self.addon_commit_files = [] # Post commit hook vcs_post_commit.send(sender=self.__class__, translation=self) # Store updated hash self.store_hash() def needs_commit(self): """Check whether there are some not committed changes.""" return self.unit_set.filter(pending=True).exists() def repo_needs_merge(self): return self.component.repo_needs_merge() def repo_needs_push(self): return self.component.repo_needs_push() @cached_property def filenames(self): if self.component.file_format_cls.simple_filename: return [self.get_filename()] return self.store.get_filenames() def repo_needs_commit(self): return self.component.repository.needs_commit(*self.filenames) def git_commit(self, request, author, timestamp, skip_push=False): """Wrapper for committing translation to git.""" repository = self.component.repository with repository.lock: # Is there something for commit? if not self.repo_needs_commit(): return False # Do actual commit with git lock self.log_info('committing %s as %s', self.filenames, author) Change.objects.create( action=Change.ACTION_COMMIT, translation=self, user=request.user if request else None, ) self.__git_commit(author, timestamp) # Push if we should if not skip_push: self.component.push_if_needed(request) return True @transaction.atomic def update_units(self, author_name, author_id): """Update backend file and unit.""" updated = False for unit in self.unit_set.filter(pending=True).select_for_update(): # Skip changes by other authors change_author = unit.get_last_content_change(None)[0] if change_author.id != author_id: continue pounit, add = self.store.find_unit(unit.context, unit.source) unit.pending = False # Bail out if we have not found anything if pounit is None or pounit.is_obsolete(): self.log_error('message %s disappeared!', unit) unit.save(update_fields=['pending'], same_content=True) continue # Check for changes if ((not add or unit.target == '') and unit.target == pounit.target and unit.approved == pounit.is_approved(unit.approved) and unit.fuzzy == pounit.is_fuzzy()): unit.save(update_fields=['pending'], same_content=True) continue updated = True # Optionally add unit to translation file. # This has be done prior setting tatget as some formats # generate content based on target language. if add: self.store.add_unit(pounit.unit) # Store translations if unit.is_plural(): pounit.set_target(unit.get_target_plurals()) else: pounit.set_target(unit.target) # Update fuzzy/approved flag pounit.mark_fuzzy(unit.state == STATE_FUZZY) pounit.mark_approved(unit.state == STATE_APPROVED) # Update comments as they might have been changed (eg, fuzzy flag # removed) state = unit.get_unit_state(pounit) flags = pounit.flags if state != unit.state or flags != unit.flags: unit.state = state unit.flags = flags unit.save( update_fields=['state', 'flags', 'pending'], same_content=True ) # Did we do any updates? if not updated: return # Update po file header now = timezone.now() if not timezone.is_aware(now): now = timezone.make_aware(now, timezone.utc) # Prepare headers to update headers = { 'add': True, 'last_translator': author_name, 'plural_forms': self.plural.plural_form, 'language': self.language_code, 'PO_Revision_Date': now.strftime('%Y-%m-%d %H:%M%z'), } # Optionally store language team with link to website if self.component.project.set_language_team: headers['language_team'] = '{0} <{1}>'.format( self.language.name, get_site_url(self.get_absolute_url()) ) # Optionally store email for reporting bugs in source report_source_bugs = self.component.report_source_bugs if report_source_bugs: headers['report_msgid_bugs_to'] = report_source_bugs # Update genric headers self.store.update_header( **headers ) # save translation changes self.store.save() def get_source_checks(self): """Return list of failing source checks on current component.""" result = TranslationChecklist() choices = dict(get_filter_choice(True)) result.add(self.stats, choices, 'all', 'success') # All checks result.add_if(self.stats, choices, 'sourcechecks', 'danger') # Process specific checks for check in CHECKS: check_obj = CHECKS[check] if not check_obj.source: continue result.add_if( self.stats, choices, check_obj.url_id, check_obj.severity, ) # Grab comments result.add_if(self.stats, choices, 'sourcecomments', 'info') return result @cached_property def list_translation_checks(self): """Return list of failing checks on current translation.""" result = TranslationChecklist() choices = dict(get_filter_choice()) # All strings result.add(self.stats, choices, 'all', 'success') result.add_if(self.stats, choices, 'approved', 'success') # Count of translated strings result.add_if(self.stats, choices, 'translated', 'success') # To approve if self.component.project.enable_review: result.add_if(self.stats, choices, 'unapproved', 'warning') # Approved with suggestions result.add_if(self.stats, choices, 'approved_suggestions', 'danger') # Untranslated strings result.add_if(self.stats, choices, 'todo', 'danger') # Not translated strings result.add_if(self.stats, choices, 'nottranslated', 'danger') # Fuzzy strings result.add_if(self.stats, choices, 'fuzzy', 'danger') # Translations with suggestions result.add_if(self.stats, choices, 'suggestions', 'info') result.add_if(self.stats, choices, 'nosuggestions', 'info') # All checks result.add_if(self.stats, choices, 'allchecks', 'danger') # Process specific checks for check in CHECKS: check_obj = CHECKS[check] if not check_obj.target: continue result.add_if( self.stats, choices, check_obj.url_id, check_obj.severity, ) # Grab comments result.add_if(self.stats, choices, 'comments', 'info') return result def merge_translations(self, request, store2, overwrite, method, fuzzy): """Merge translation unit wise Needed for template based translations to add new strings. """ not_found = 0 skipped = 0 accepted = 0 add_fuzzy = (method == 'fuzzy') add_approve = (method == 'approve') for set_fuzzy, unit2 in store2.iterate_merge(fuzzy): try: unit = self.unit_set.get_unit(unit2) except Unit.DoesNotExist: not_found += 1 continue if ((unit.translated and not overwrite) or (not request.user.has_perm('unit.edit', unit))): skipped += 1 continue accepted += 1 # We intentionally avoid propagating: # - in most cases it's not desired # - it slows down import considerably # - it brings locking issues as import is # executed with lock held and linked repos # can't obtain the lock state = STATE_TRANSLATED if add_fuzzy or set_fuzzy: state = STATE_FUZZY elif add_approve: state = STATE_APPROVED unit.translate( request, split_plural(unit2.target), state, change_action=Change.ACTION_UPLOAD, propagate=False ) if accepted > 0: self.invalidate_cache() request.user.profile.refresh_from_db() request.user.profile.translated += accepted request.user.profile.save(update_fields=['translated']) return (not_found, skipped, accepted, len(store2.all_units)) def merge_suggestions(self, request, store, fuzzy): """Merge content of translate-toolkit store as a suggestions.""" not_found = 0 skipped = 0 accepted = 0 for dummy, unit in store.iterate_merge(fuzzy): # Grab database unit try: dbunit = self.unit_set.get_unit(unit) except Unit.DoesNotExist: not_found += 1 continue # Add suggestion if dbunit.target != unit.target: if Suggestion.objects.add(dbunit, unit.target, request): accepted += 1 else: skipped += 1 else: skipped += 1 # Update suggestion count if accepted > 0: self.invalidate_cache() return (not_found, skipped, accepted, len(store.all_units)) def merge_upload(self, request, fileobj, overwrite, author_name=None, author_email=None, method='translate', fuzzy=''): """Top level handler for file uploads.""" filecopy = fileobj.read() fileobj.close() # Strip possible UTF-8 BOM if filecopy[:3] == codecs.BOM_UTF8: filecopy = filecopy[3:] # Load backend file store = try_load( fileobj.name, filecopy, self.component.file_format_cls, self.component.template_store ) # Check valid plural forms if hasattr(store.store, 'parseheader'): header = store.store.parseheader() try: number, equation = Plural.parse_formula(header['Plural-Forms']) if not self.plural.same_plural(number, equation): raise Exception('Plural forms do not match the language.') except (ValueError, KeyError): # Formula wrong or missing pass # Optionally set authorship orig_user = None if author_email: from weblate.auth.models import User orig_user = request.user request.user = User.objects.get_or_create( email=author_email, defaults={ 'username': author_email, 'is_active': False, 'full_name': author_name or author_email, } )[0] try: if method in ('translate', 'fuzzy', 'approve'): # Merge on units level with self.component.repository.lock: return self.merge_translations( request, store, overwrite, method, fuzzy, ) # Add as sugestions return self.merge_suggestions(request, store, fuzzy) finally: if orig_user: request.user = orig_user def invalidate_cache(self, recurse=True): """Invalidate any cached stats.""" # Invalidate summary stats self.stats.invalidate() if recurse and self.component.allow_translation_propagation: related = Translation.objects.filter( component__project=self.component.project, component__allow_translation_propagation=True, language=self.language, ).exclude( pk=self.pk ) for translation in related: translation.invalidate_cache(False) def get_export_url(self): """Return URL of exported git repository.""" return self.component.get_export_url() def get_stats(self): """Return stats dictionary""" return { 'code': self.language.code, 'name': self.language.name, 'total': self.stats.all, 'total_words': self.stats.all_words, 'last_change': self.stats.last_changed, 'last_author': self.get_last_author(), 'translated': self.stats.translated, 'translated_words': self.stats.translated_words, 'translated_percent': self.stats.translated_percent, 'fuzzy': self.stats.fuzzy, 'fuzzy_percent': self.stats.fuzzy_percent, 'failing': self.stats.allchecks, 'failing_percent': self.stats.allchecks_percent, 'url': self.get_share_url(), 'url_translate': get_site_url(self.get_absolute_url()), } def remove(self, user): """Remove translation from the VCS""" author = user.get_author_name() # Log self.log_info('removing %s as %s', self.filenames, author) # Remove file from VCS if any((os.path.exists(name) for name in self.filenames)): self.commit_template = 'delete' with self.component.repository.lock: self.component.repository.remove( self.filenames, self.get_commit_message(author), author, ) # Delete from the database self.stats.invalidate() self.delete() # Record change Change.objects.create( component=self.component, action=Change.ACTION_REMOVE_TRANSLATION, target=self.filename, user=user, author=user ) def new_unit(self, request, key, value): with self.component.repository.lock: self.commit_pending('new unit', request) Change.objects.create( translation=self, action=Change.ACTION_NEW_UNIT, target=value, user=request.user, author=request.user ) self.store.new_unit(key, value) self.component.create_translations(request=request) self.__git_commit( request.user.get_author_name(), timezone.now() ) self.component.push_if_needed(request)