class PootleSite(models.Model): """Model to store each specific Pootle site configuration. The configuration includes some data for install/upgrade mechanisms. """ site = models.OneToOneField(Site, editable=False) title = models.CharField( max_length=50, blank=False, default="Pootle Demo", verbose_name=_("Title"), help_text=_("The name for this Pootle server"), ) description = MarkupField( blank=True, default='', verbose_name=_("Description"), help_text=_( "The description and instructions shown on the about " "page. Allowed markup: %s", get_markup_filter_name()), ) objects = PootleSiteManager() class Meta: app_label = "pootle_app"
class Goal(TagBase): """Goal is a tag with a priority. Also it might be used to set shared goals across a translation project, for example a goal with all the files that must focus first their effor on all the translators (independently of the language they are translating to). It inherits from TagBase instead of Tag because that way it is possible to reduce the number of DB queries. """ description = MarkupField( verbose_name=_("Description"), blank=True, help_text=_('A description of this goal. This is useful to give more ' 'information or instructions. Allowed markup: %s', get_markup_filter_name()), ) # Priority goes from 1 to 10, being 1 the greater and 10 the lower. priority = models.IntegerField( verbose_name=_("Priority"), default=10, help_text=_("The priority for this goal."), ) # Tells if the goal is going to be shared across a project. This might be # seen as a 'virtual goal' because it doesn't apply to any real TP, but to # the templates one. project_goal = models.BooleanField( verbose_name=_("Project goal?"), default=False, help_text=_("Designates that this is a project goal (shared across " "all languages in the project)."), ) # Necessary for assigning and checking permissions. directory = models.OneToOneField( 'pootle_app.Directory', db_index=True, editable=False, ) CACHED_FUNCTIONS = ["get_raw_stats_for_path"] class Meta: ordering = ["priority"] ############################ Properties ################################### @property def pootle_path(self): return "/goals/" + self.slug + "/" @property def goal_name(self): """Return the goal name, i.e. the name without the 'goal:' prefix. If this is a project goal, then is appended a text indicating that. """ if self.project_goal: return "%s %s" % (self.name[5:], _("(Project goal)")) else: return self.name[5:] ############################ Methods ###################################### @classmethod def get_goals_for_path(cls, pootle_path): """Return the goals applied to the stores in this path. If this is not the 'templates' translation project for the project then also return the 'project goals' applied to the stores in the 'templates' translation project. :param pootle_path: A string with a valid pootle path. """ # Putting the next imports at the top of the file causes circular # import issues. from pootle_app.models.directory import Directory from pootle_store.models import Store directory = Directory.objects.get(pootle_path=pootle_path) stores_pks = directory.stores.values_list("pk", flat=True) criteria = { 'items_with_goal__content_type': ContentType.objects \ .get_for_model(Store), 'items_with_goal__object_id__in': stores_pks, } try: tp = directory.translation_project except: return [] if tp.is_template_project: # Return the 'project goals' applied to stores in this path. return cls.objects.filter(**criteria) \ .order_by('project_goal', 'priority').distinct() else: # Get the 'non-project goals' (aka regular goals) applied to stores # in this path. criteria['project_goal'] = False regular_goals = cls.objects.filter(**criteria).distinct() # Now get the 'project goals' applied to stores in the 'templates' # TP for this TP's project. template_tp = tp.project.get_template_translationproject() if template_tp is None: # If this project has no 'templates' TP. project_goals = cls.objects.none() else: tpl_dir_path = "/%s/%s" % (template_tp.language.code, pootle_path.split("/", 2)[-1]) try: tpl_dir = Directory.objects.get(pootle_path=tpl_dir_path) except Directory.DoesNotExist: project_goals = cls.objects.none() else: tpl_stores_pks = tpl_dir.stores.values_list('pk', flat=True) criteria.update({ 'project_goal': True, 'items_with_goal__object_id__in': tpl_stores_pks, }) project_goals = cls.objects.filter(**criteria).distinct() return list(chain(regular_goals, project_goals)) @classmethod def get_trail_for_path(cls, pootle_path): """Return a list with the trail for the given path. If the pootle path does not exist, then an empty list is returned. The trail is all the directories that correspond to the given pootle path, plus the Translation project where the given pootle path is. For example for the pootle path /ru/firefoxos/add-ons/dropbox/nvda.po the following trail is returned: * Translation project object for /ru/firefoxos/ * Directory object for /ru/firefoxos/add-ons/ * Directory object for /ru/firefoxos/add-ons/dropbox/ Note that no object for the store is included in the returned trail. :param pootle_path: A string with a valid pootle path. """ # Putting the next import at the top of the file causes circular import # issues. from pootle_store.models import Store try: path_obj = Store.objects.get(pootle_path=pootle_path) except Store.DoesNotExist: # Putting the next import at the top of the file causes circular # import issues. from pootle_app.models.directory import Directory try: path_obj = Directory.objects.get(pootle_path=pootle_path) except Directory.DoesNotExist: # If it is not possible to retrieve any path_obj for the # provided pootle_path, then abort. return [] if isinstance(path_obj, Store): path_dir = path_obj.parent else: # Else it is a directory. path_dir = path_obj # Note: Not including path_obj (if it is a store) in path_objs since we # still don't support including units in a goal. path_objs = chain([path_obj.translation_project], path_dir.trail()) return path_objs @classmethod def get_most_important_incomplete_for_path(cls, path_obj): """Return the most important incomplete goal for this path or None. If this is not the 'templates' translation project for the project then also considers the 'project goals' applied to the stores in the 'templates' translation project. The most important goal is the one with the lowest priority, or if more than a goal have the lower priority then the alphabetical order is taken in account. :param path_obj: A pootle path object. """ most_important = None for goal in cls.get_goals_for_path(path_obj.pootle_path): if (most_important is None or goal.priority < most_important.priority or (goal.priority == most_important.priority and goal.name < most_important.name)): if goal.get_incomplete_words_in_path(path_obj): most_important = goal return most_important @classmethod def flush_all_caches_in_tp(cls, translation_project): """Remove the cache for all the goals in the given translation project. :param translation_project: An instance of :class:`TranslationProject`. """ pootle_path = translation_project.pootle_path keys = set() for goal in cls.get_goals_for_path(pootle_path): for store in goal.get_stores_for_path(pootle_path): for path_obj in store.parent.trail(): for function_name in cls.CACHED_FUNCTIONS: keys.add(iri_to_uri(goal.pootle_path + ":" + path_obj.pootle_path + ":" + function_name)) for function_name in cls.CACHED_FUNCTIONS: keys.add(iri_to_uri(goal.pootle_path + ":" + pootle_path + ":" + function_name)) cache.delete_many(list(keys)) @classmethod def flush_all_caches_for_path(cls, pootle_path): """Remove the cache for all the goals in the given path and upper directories. The cache is deleted for the given path, for the directories between the given path and the translation project, and for the translation project itself. :param pootle_path: A string with a valid pootle path. """ # Get all the affected objects just once, to avoid querying the # database all the time if there are too many objects involved. affected_trail = cls.get_trail_for_path(pootle_path) if not affected_trail: return affected_goals = cls.get_goals_for_path(pootle_path) keys = [] for goal in affected_goals: for path_obj in affected_trail: for function_name in cls.CACHED_FUNCTIONS: keys.append(iri_to_uri(goal.pootle_path + ":" + path_obj.pootle_path + ":" + function_name)) cache.delete_many(keys) def save(self, *args, **kwargs): # Putting the next import at the top of the file causes circular import # issues. from pootle_app.models.directory import Directory self.directory = Directory.objects.goals.get_or_make_subdir(self.slug) super(Goal, self).save(*args, **kwargs) def delete(self, *args, **kwargs): directory = self.directory super(Goal, self).delete(*args, **kwargs) directory.delete() def get_translate_url_for_path(self, pootle_path, **kwargs): """Return this goal's translate URL for the given path. :param pootle_path: A string with a valid pootle path. """ lang, proj, dir_path, fn = split_pootle_path(pootle_path) return u''.join([ reverse('pootle-tp-translate', args=[lang, proj, dir_path, fn]), get_editor_filter(goal=self.slug, **kwargs), ]) def get_critical_url_for_path(self, pootle_path, **kwargs): """Return this goal's translate URL for critical checks failures in the given path. :param pootle_path: A string with a valid pootle path. """ critical = ','.join(get_qualitychecks_by_category(Category.CRITICAL)) return self.get_translate_url_for_path(pootle_path, check=critical) def get_drill_down_url_for_path(self, pootle_path): """Return this goal's drill down URL for the given path. :param pootle_path: A string with a valid pootle path. """ lang, proj, dir_path, filename = split_pootle_path(pootle_path) reverse_args = [lang, proj, self.slug, dir_path, filename] return reverse('pootle-tp-goal-drill-down', args=reverse_args) def get_stores_for_path(self, pootle_path): """Return the stores for this goal in the given pootle path. If this is a project goal then the corresponding stores in the path to that ones in the 'templates' TP for this goal are returned instead. :param pootle_path: A string with a valid pootle path. """ # Putting the next imports at the top of the file causes circular # import issues. from pootle_store.models import Store from pootle_translationproject.models import TranslationProject lang, proj, dir_path, filename = split_pootle_path(pootle_path) # Get the translation project for this pootle_path. try: tp = TranslationProject.objects.get(language__code=lang, project__code=proj) except TranslationProject.DoesNotExist: return Store.objects.none() if self.project_goal and not tp.is_template_project: # Get the stores for this goal that are in the 'templates' TP. templates_tp = tp.project.get_template_translationproject() if templates_tp is None: return Store.objects.none() else: path_in_templates = (templates_tp.pootle_path + dir_path + filename) lookups = { 'pootle_path__startswith': path_in_templates, 'goals__in': [self], } template_stores_in_goal = Store.objects.filter(**lookups) # Putting the next imports at the top of the file causes circular # import issues. if tp.file_style == 'gnu': from pootle_app.project_tree import ( get_translated_name_gnu as get_translated_name) else: from pootle_app.project_tree import get_translated_name # Get the pootle path for the corresponding stores in the given # TP for those stores in the 'templates' TP. criteria = { 'pootle_path__in': [get_translated_name(tp, store)[0] for store in template_stores_in_goal], } else: # This is a regular goal or the given TP is the 'templates' TP, so # just retrieve the goal stores on this TP. criteria = { 'pootle_path__startswith': pootle_path, 'goals__in': [self], } # Return the stores. return Store.objects.filter(**criteria) def get_children_for_path(self, pootle_path): """Return this goal stores and subdirectories in the given directory. The subdirectories returned are the ones that have any store for this goal just below them, or in any of its subdirectories. If this is a project goal then are returned instead: * The stores in the given directory that correspond to the goal stores in the corresponding directory in the 'templates' TP, * The subdirectories in the given directory that have stores that correspond to goal stores in the 'templates' TP. :param pootle_path: The pootle path for a :class:`Directory` instance. :return: Tuple with a stores list and a directories queryset. """ # Putting the next import at the top of the file causes circular import # issues. from pootle_app.models.directory import Directory stores_in_dir = [] subdir_paths = set() stores_for_path = self.get_stores_for_path(pootle_path) # Put apart the stores that are just below the directory from those # that are in subdirectories inside directory. for store in stores_for_path: trailing_path = store.pootle_path[len(pootle_path):] if "/" in trailing_path: # Store is in a subdirectory. subdir_name = trailing_path.split("/")[0] + "/" subdir_paths.add(pootle_path + subdir_name) else: # Store is in the directory. stores_in_dir.append(store) # Get the subdirectories that have stores for this goal. subdirs_in_dir = Directory.objects.filter(pootle_path__in=subdir_paths) # Return a tuple with stores and subdirectories in the given directory. return (stores_in_dir, subdirs_in_dir) def slugify(self, tag, i=None): return slugify_tag_name(tag) def get_incomplete_words_in_path(self, path_obj): """Return the number of incomplete words for this goal in the path. :param path_obj: A pootle path object. """ total = path_obj.get_total_wordcount() translated = path_obj.get_translated_wordcount() return total - translated
class AbstractPage(DirtyFieldsMixin, models.Model): active = models.BooleanField( _('Active'), help_text=_('Whether this page is active or not.'), ) virtual_path = models.CharField( _("Virtual Path"), max_length=100, default='', unique=True, help_text='/pages/', ) # TODO: make title and body localizable fields title = models.CharField(_("Title"), max_length=100) body = MarkupField( # Translators: Content that will be used to display this static page _("Display Content"), blank=True, help_text=_('Allowed markup: %s', get_markup_filter_name()), ) url = models.URLField( _("Redirect to URL"), blank=True, help_text=_('If set, any references to this page will redirect to this' ' URL'), ) # This will go away with bug 2830, but works fine for now. modified_on = models.DateTimeField( default=now, editable=False, auto_now_add=True, ) objects = PageManager() class Meta: abstract = True def __unicode__(self): return self.virtual_path def save(self): # Update the `modified_on` timestamp only when specific fields change. dirty_fields = self.get_dirty_fields() if any(field in dirty_fields for field in ('title', 'body', 'url')): self.modified_on = now() super(AbstractPage, self).save() def get_absolute_url(self): if self.url: return self.url return reverse('pootle-staticpages-display', args=[self.virtual_path]) @staticmethod def max_pk(): """Returns the sum of all the highest PKs for each submodel.""" return reduce( lambda x, y: x + y, [ int(p.objects.aggregate(Max('pk')).values()[0] or 0) for p in AbstractPage.__subclasses__() ], ) def clean(self): """Fail validation if: - URL and body are blank - Current virtual path exists in other page models """ if not self.url and not self.body: # Translators: 'URL' and 'content' refer to form fields. raise ValidationError(_('URL or content must be provided.')) pages = [ p.objects.filter( Q(virtual_path=self.virtual_path), ~Q(pk=self.pk), ).exists() for p in AbstractPage.__subclasses__() ] if True in pages: raise ValidationError(_(u'Virtual path already in use.'))
class VirtualFolder(models.Model): # any changes to the `name` field may require updating the schema # see migration 0003_case_sensitive_schema.py name = models.CharField(_('Name'), blank=False, max_length=70) # any changes to the `location` field may require updating the schema # see migration 0003_case_sensitive_schema.py location = models.CharField( _('Location'), blank=False, max_length=255, help_text=_('Root path where this virtual folder is applied.'), ) filter_rules = models.TextField( # Translators: This is a noun. _('Filter'), blank=False, help_text=_('Filtering rules that tell which stores this virtual ' 'folder comprises.'), ) priority = models.FloatField( _('Priority'), default=1, help_text=_('Number specifying importance. Greater priority means it ' 'is more important.'), ) is_public = models.BooleanField( _('Is public?'), default=True, help_text=_('Whether this virtual folder is public or not.'), ) description = MarkupField( _('Description'), blank=True, help_text=_( 'Use this to provide more information or instructions. ' 'Allowed markup: %s', get_markup_filter_display_name()), ) units = models.ManyToManyField( Unit, db_index=True, related_name='vfolders', ) class Meta(object): unique_together = ('name', 'location') @property def all_locations(self): """Return a list with all the locations this virtual folder applies. If the virtual folder location has no {LANG} nor {PROJ} placeholders then the list only contains its location. If any of the placeholders is present, then they get expanded to match all the existing languages and projects. """ if "{LANG}/{PROJ}" in self.location: locations = [] for lang in Language.objects.all(): temp = self.location.replace("{LANG}", lang.code) for proj in Project.objects.all(): locations.append(temp.replace("{PROJ}", proj.code)) return locations elif "{LANG}" in self.location: try: project = Project.objects.get(code=self.location.split("/")[2]) languages = project.languages.iterator() except Exception: languages = Language.objects.iterator() return [ self.location.replace("{LANG}", lang.code) for lang in languages ] elif "{PROJ}" in self.location: try: projects = Project.objects.filter( translationproject__language__code=self.location.split( "/")[1]).iterator() except Exception: projects = Project.objects.iterator() return [ self.location.replace("{PROJ}", proj.code) for proj in projects ] return [self.location] def __unicode__(self): return ": ".join([self.name, self.location]) def save(self, *args, **kwargs): # Force validation of fields. self.clean_fields() self.name = self.name.lower() if self.pk is None: projects = set() else: # If this is an already existing vfolder, keep a list of the # projects it was related to. projects = set( Project.objects.filter( translationproject__stores__unit__vfolders=self).distinct( ).values_list('code', flat=True)) super(VirtualFolder, self).save(*args, **kwargs) # Clean any existing relationship between units and this vfolder. self.units.clear() # Recreate relationships between this vfolder and units. vfolder_stores_set = set() for location in self.all_locations: for filename in self.filter_rules.split(","): vf_file = "".join([location, filename]) qs = Store.objects.live().filter(pootle_path=vf_file) if qs.exists(): self.units.add(*qs[0].units.all()) vfolder_stores_set.add(qs[0]) else: if not vf_file.endswith("/"): vf_file += "/" if Directory.objects.filter(pootle_path=vf_file).exists(): qs = Unit.objects.filter( state__gt=OBSOLETE, store__pootle_path__startswith=vf_file) self.units.add(*qs) vfolder_stores_set.update( Store.objects.filter( pootle_path__startswith=vf_file)) # For each store create all VirtualFolderTreeItem tree structure up to # its adjusted vfolder location. for store in vfolder_stores_set: try: VirtualFolderTreeItem.objects.get_or_create( directory=store.parent, vfolder=self, ) except ValidationError: # If there is some problem, e.g. a clash with a directory, # delete the virtual folder and all its related items, and # reraise the exception. self.delete() raise # Get the set of projects whose resources cache must be invalidated. # This includes the projects the projects it was previously related to # for the already existing vfolders. projects.update( Project.objects.filter( translationproject__stores__unit__vfolders=self).distinct(). values_list('code', flat=True)) # Send the signal. This is used to invalidate the cached resources for # all the related projects. vfolder_post_save.send(sender=self.__class__, instance=self, projects=list(projects)) def delete(self, *args, **kwargs): self.vf_treeitems.all().delete() super(VirtualFolder, self).delete(*args, **kwargs) def clean_fields(self): """Validate virtual folder fields.""" if self.priority <= 0: raise ValidationError(u'Priority must be greater than zero.') elif self.location == "/": raise ValidationError(u'The "/" location is not allowed. Use ' u'"/{LANG}/{PROJ}/" instead.') elif self.location.startswith("/projects/"): raise ValidationError(u'Locations starting with "/projects/" are ' u'not allowed. Use "/{LANG}/" instead.') if not self.filter_rules: raise ValidationError(u'Some filtering rule must be specified.') def get_adjusted_location(self, pootle_path): """Return the virtual folder location adjusted to the given path. The virtual folder location might have placeholders, which affect the actual filenames since those have to be concatenated to the virtual folder location. """ count = self.location.count("/") if pootle_path.count("/") < count: raise ValueError("%s is not applicable in %s" % (self, pootle_path)) pootle_path_parts = pootle_path.strip("/").split("/") location_parts = self.location.strip("/").split("/") try: if (location_parts[0] != pootle_path_parts[0] and location_parts[0] != "{LANG}"): raise ValueError("%s is not applicable in %s" % (self, pootle_path)) if (location_parts[1] != pootle_path_parts[1] and location_parts[1] != "{PROJ}"): raise ValueError("%s is not applicable in %s" % (self, pootle_path)) except IndexError: pass return "/".join(pootle_path.split("/")[:count]) def get_adjusted_pootle_path(self, pootle_path): """Adjust the given pootle path to this virtual folder. The provided pootle path is converted to a path that includes the virtual folder name in the right place. For example a virtual folder named vfolder8, with a location /{LANG}/firefox/browser/ in a path /af/firefox/browser/chrome/overrides/ gets converted to /af/firefox/browser/vfolder8/chrome/overrides/ """ count = self.location.count('/') if pootle_path.count('/') < count: # The provided pootle path is above the virtual folder location. path_parts = pootle_path.rstrip('/').split('/') pootle_path = '/'.join(path_parts + self.location.split('/')[len(path_parts):]) if count < 3: # If the virtual folder location is not long as a translation # project pootle path then the returned adjusted location is too # short, meaning that the returned translate URL will have the # virtual folder name as the project or language code. path_parts = pootle_path.split('/') return '/'.join(path_parts[:3] + [self.name] + path_parts[3:]) # If the virtual folder location is as long as a TP pootle path and # the provided pootle path isn't above the virtual folder location. lead = self.get_adjusted_location(pootle_path) trail = pootle_path.replace(lead, '').lstrip('/') return '/'.join([lead, self.name, trail])
class VirtualFolder(models.Model): name = models.CharField(_('Name'), blank=False, max_length=70) location = models.CharField( _('Location'), blank=False, max_length=255, help_text=_('Root path where this virtual folder is applied.'), ) filter_rules = models.TextField( # Translators: This is a noun. _('Filter'), blank=False, help_text=_('Filtering rules that tell which stores this virtual ' 'folder comprises.'), ) priority = models.FloatField( _('Priority'), default=1, help_text=_('Number specifying importance. Greater priority means it ' 'is more important.'), ) is_browsable = models.BooleanField( _('Is browsable?'), default=True, help_text=_('Whether this virtual folder is active or not.'), ) description = MarkupField( _('Description'), blank=True, help_text=_( 'Use this to provide more information or instructions. ' 'Allowed markup: %s', get_markup_filter_name()), ) units = models.ManyToManyField( Unit, db_index=True, related_name='vfolders', ) class Meta: unique_together = ('name', 'location') ordering = ['-priority', 'name'] @classmethod def get_matching_for(cls, pootle_path): """Return the matching virtual folders in the given pootle path. Not all the applicable virtual folders have matching filtering rules. This method further restricts the list of applicable virtual folders to retrieve only those with filtering rules that actually match. """ return VirtualFolder.objects.filter( units__store__pootle_path__startswith=pootle_path).distinct() def __unicode__(self): return ": ".join([self.name, self.location]) def save(self, *args, **kwargs): # Force validation of fields. self.clean_fields() self.name = self.name.lower() super(VirtualFolder, self).save(*args, **kwargs) # Clean any existing relationship between units and this vfolder. self.units.clear() # Recreate relationships between this vfolder and units. for location in self.get_all_pootle_paths(): for filename in self.filter_rules.split(","): vf_file = "".join([location, filename]) qs = Store.objects.live().filter(pootle_path=vf_file) if qs.exists(): self.units.add(*qs[0].units.all()) else: if not vf_file.endswith("/"): vf_file += "/" if Directory.objects.filter(pootle_path=vf_file).exists(): qs = Unit.objects.filter( store__pootle_path__startswith=vf_file) self.units.add(*qs) def clean_fields(self): """Validate virtual folder fields.""" if not self.priority > 0: raise ValidationError(u'Priority must be greater than zero.') elif self.location == "/": raise ValidationError(u'The "/" location is not allowed. Use ' u'"/{LANG}/{PROJ}/" instead.') def get_all_pootle_paths(self): """Return a list with all the locations this virtual folder applies. If the virtual folder location has no {LANG} nor {PROJ} placeholders then the list only contains its location. If any of the placeholders is present, then they get expanded to match all the existing languages and projects. """ # Locations like /project/<my_proj>/ are not handled correctly. So # rewrite them. if self.location.startswith("/projects/"): self.location = self.location.replace("/projects/", "/{LANG}/") if "{LANG}" in self.location and "{PROJ}" in self.location: locations = [] for lang in Language.objects.all(): temp = self.location.replace("{LANG}", lang.code) for proj in Project.objects.all(): locations.append(temp.replace("{PROJ}", proj.code)) return locations elif "{LANG}" in self.location: try: project = Project.objects.get(code=self.location.split("/")[2]) languages = project.languages.iterator() except: languages = Language.objects.iterator() return [ self.location.replace("{LANG}", lang.code) for lang in languages ] elif "{PROJ}" in self.location: try: projects = Project.objects.filter( translationproject__language__code=self.location.split( "/")[1]).iterator() except: projects = Project.objects.iterator() return [ self.location.replace("{PROJ}", proj.code) for proj in projects ] return [self.location]
class TranslationProject(models.Model): _non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE, settings.PARSE_POOL_CULL_FREQUENCY) objects = TranslationProjectManager() index_directory = ".translation_index" class Meta: unique_together = ('language', 'project') db_table = 'pootle_app_translationproject' description_help_text = _( 'A description of this translation project. ' 'This is useful to give more information or instructions. ' 'Allowed markup: %s', get_markup_filter_name()) description = MarkupField(blank=True, help_text=description_help_text) language = models.ForeignKey(Language, db_index=True) project = models.ForeignKey(Project, db_index=True) real_path = models.FilePathField(editable=False) directory = models.OneToOneField(Directory, db_index=True, editable=False) pootle_path = models.CharField(max_length=255, null=False, unique=True, db_index=True, editable=False) def natural_key(self): return (self.pootle_path, ) natural_key.dependencies = [ 'pootle_app.Directory', 'pootle_language.Language', 'pootle_project.Project' ] def __unicode__(self): return self.pootle_path def save(self, *args, **kwargs): created = self.id is None project_dir = self.project.get_real_path() from pootle_app.project_tree import get_translation_project_dir self.abs_real_path = get_translation_project_dir(self.language, project_dir, self.file_style, make_dirs=True) self.directory = self.language.directory \ .get_or_make_subdir(self.project.code) self.pootle_path = self.directory.pootle_path super(TranslationProject, self).save(*args, **kwargs) if created: self.scan_files() def delete(self, *args, **kwargs): directory = self.directory super(TranslationProject, self).delete(*args, **kwargs) directory.delete() deletefromcache(self, [ "getquickstats", "getcompletestats", "get_mtime", "get_suggestion_count" ]) def get_absolute_url(self): return l(self.pootle_path) fullname = property(lambda self: "%s [%s]" % (self.project.fullname, self.language.name)) def _get_abs_real_path(self): return absolute_real_path(self.real_path) def _set_abs_real_path(self, value): self.real_path = relative_real_path(value) abs_real_path = property(_get_abs_real_path, _set_abs_real_path) def _get_treestyle(self): return self.project.get_treestyle() file_style = property(_get_treestyle) def _get_checker(self): from translate.filters import checks checkerclasses = [ checks.projectcheckers.get(self.project.checkstyle, checks.StandardChecker), checks.StandardUnitChecker ] return checks.TeeChecker(checkerclasses=checkerclasses, excludefilters=['hassuggestion'], errorhandler=self.filtererrorhandler, languagecode=self.language.code) checker = property(_get_checker) def filtererrorhandler(self, functionname, str1, str2, e): logging.error(u"Error in filter %s: %r, %r, %s", functionname, str1, str2, e) return False def _get_non_db_state(self): if not hasattr(self, "_non_db_state"): try: self._non_db_state = self._non_db_state_cache[self.id] except KeyError: self._non_db_state = TranslationProjectNonDBState(self) self._non_db_state_cache[self.id] = \ TranslationProjectNonDBState(self) return self._non_db_state non_db_state = property(_get_non_db_state) def update(self): """Update all stores to reflect state on disk""" stores = self.stores.exclude(file='').filter(state__gte=PARSED) for store in stores.iterator(): store.update(update_translation=True, update_structure=True) def sync(self, conservative=True, skip_missing=False, modified_since=0): """Sync unsaved work on all stores to disk""" stores = self.stores.exclude(file='').filter(state__gte=PARSED) for store in stores.iterator(): store.sync(update_translation=True, update_structure=not conservative, conservative=conservative, create=False, skip_missing=skip_missing, modified_since=modified_since) @getfromcache def get_mtime(self): tp_units = Unit.objects.filter(store__translation_project=self) return max_column(tp_units, 'mtime', None) def require_units(self): """Makes sure all stores are parsed""" errors = 0 for store in self.stores.filter(state__lt=PARSED).iterator(): try: store.require_units() except IntegrityError: logging.info(u"Duplicate IDs in %s", store.abs_real_path) errors += 1 except ParseError, e: logging.info(u"Failed to parse %s\n%s", store.abs_real_path, e) errors += 1 except (IOError, OSError), e: logging.info(u"Can't access %s\n%s", store.abs_real_path, e) errors += 1
class Language(models.Model, TreeItem): code = models.CharField( max_length=50, null=False, unique=True, db_index=True, verbose_name=_("Code"), help_text=_('ISO 639 language code for the language, possibly ' 'followed by an underscore (_) and an ISO 3166 country ' 'code. <a href="http://www.w3.org/International/articles/' 'language-tags/">More information</a>'), ) fullname = models.CharField( max_length=255, null=False, verbose_name=_("Full Name"), ) description = MarkupField( blank=True, help_text=_( 'A description of this language. This is useful to give ' 'more information or instructions. Allowed markup: %s', get_markup_filter_name()), ) specialchars = models.CharField( max_length=255, blank=True, verbose_name=_("Special Characters"), help_text=_('Enter any special characters that users might find ' 'difficult to type'), ) nplurals = models.SmallIntegerField( default=0, choices=((0, _('Unknown')), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)), verbose_name=_("Number of Plurals"), help_text=_('For more information, visit <a href="' 'http://docs.translatehouse.org/projects/' 'localization-guide/en/latest/l10n/pluralforms.html">our ' 'page</a> on plural forms.'), ) pluralequation = models.CharField( max_length=255, blank=True, verbose_name=_("Plural Equation"), help_text=_('For more information, visit <a href="' 'http://docs.translatehouse.org/projects/' 'localization-guide/en/latest/l10n/pluralforms.html">our ' 'page</a> on plural forms.'), ) directory = models.OneToOneField( 'pootle_app.Directory', db_index=True, editable=False, ) objects = LanguageManager() live = LiveLanguageManager() class Meta: ordering = ['code'] db_table = 'pootle_app_language' def natural_key(self): return (self.code, ) natural_key.dependencies = ['pootle_app.Directory'] ############################ Properties ################################### @property def pootle_path(self): return '/%s/' % self.code @property def name(self): """Localized fullname for the language.""" return tr_lang(self.fullname) @property def direction(self): """Return the language direction.""" return language_dir(self.code) ############################ Methods ###################################### def __init__(self, *args, **kwargs): super(Language, self).__init__(*args, **kwargs) def __repr__(self): return u'<%s: %s>' % (self.__class__.__name__, self.fullname) def __unicode__(self): return u"%s - %s" % (self.name, self.code) def save(self, *args, **kwargs): # create corresponding directory object. from pootle_app.models.directory import Directory self.directory = Directory.objects.root.get_or_make_subdir(self.code) super(Language, self).save(*args, **kwargs) # FIXME: far from ideal, should cache at the manager level instead. cache.delete(CACHE_KEY) def delete(self, *args, **kwargs): directory = self.directory super(Language, self).delete(*args, **kwargs) directory.delete() # FIXME: far from ideal, should cache at the manager level instead. cache.delete(CACHE_KEY) def get_absolute_url(self): return reverse('pootle-language-overview', args=[self.code]) def get_translate_url(self, **kwargs): return u''.join([ reverse('pootle-language-translate', args=[self.code]), get_editor_filter(**kwargs), ]) ### TreeItem def get_children(self): return self.translationproject_set.all() def get_cachekey(self): return self.directory.pootle_path ### /TreeItem def translated_percentage(self): total = max(self.get_total_wordcount(), 1) translated = self.get_translated_wordcount() return int(100.0 * translated / total)
class Project(models.Model, TreeItem): code = models.CharField( max_length=255, null=False, unique=True, db_index=True, verbose_name=_('Code'), help_text=_('A short code for the project. This should only contain ' 'ASCII characters, numbers, and the underscore (_) ' 'character.'), ) fullname = models.CharField( max_length=255, null=False, verbose_name=_("Full Name"), ) description = MarkupField( blank=True, help_text=_('A description of this project. This is useful to give ' 'more information or instructions. Allowed markup: %s', get_markup_filter_name()), ) checker_choices = [('standard', 'standard')] checkers = list(checks.projectcheckers.keys()) checkers.sort() checker_choices.extend([(checker, checker) for checker in checkers]) checkstyle = models.CharField( max_length=50, default='standard', null=False, choices=checker_choices, verbose_name=_('Quality Checks'), ) localfiletype = models.CharField( max_length=50, default="po", choices=filetype_choices, verbose_name=_('File Type'), ) treestyle = models.CharField( max_length=20, default='auto', choices=( # TODO: check that the None is stored and handled correctly ('auto', _('Automatic detection (slower)')), ('gnu', _('GNU style: files named by language code')), ('nongnu', _('Non-GNU: Each language in its own directory')), ), verbose_name=_('Project Tree Style'), ) source_language = models.ForeignKey( 'pootle_language.Language', db_index=True, verbose_name=_('Source Language'), ) ignoredfiles = models.CharField( max_length=255, blank=True, null=False, default="", verbose_name=_('Ignore Files'), ) directory = models.OneToOneField( 'pootle_app.Directory', db_index=True, editable=False, ) report_email = models.EmailField( max_length=254, blank=True, verbose_name=_("Errors Report Email"), help_text=_('An email address where issues with the source text can ' 'be reported.'), ) disabled = models.BooleanField(verbose_name=_('Disabled'), default=False) objects = ProjectManager() class Meta: ordering = ['code'] db_table = 'pootle_app_project' ############################ Properties ################################### @property def name(self): return self.fullname @property def pootle_path(self): return "/projects/" + self.code + "/" @property def is_terminology(self): """Returns ``True`` if this project is a terminology project.""" return self.checkstyle == 'terminology' @property def is_monolingual(self): """Return ``True`` if this project is monolingual.""" return is_monolingual(self.get_file_class()) ############################ Cached properties ############################ @cached_property def languages(self): """Returns a list of active :cls:`~pootle_languages.models.Language` objects for this :cls:`~pootle_project.models.Project`. """ from pootle_language.models import Language # FIXME: we should better have a way to automatically cache models with # built-in invalidation -- did I hear django-cache-machine? return Language.objects.filter(Q(translationproject__project=self), ~Q(code='templates')) @cached_property def resources(self): """Returns a list of :cls:`~pootle_app.models.Directory` and :cls:`~pootle_store.models.Store` objects available for this :cls:`~pootle_project.models.Project` across all languages. """ from pootle_store.models import Store resources_path = ''.join(['/%/', self.code, '/%']) store_objs = Store.objects.extra( where=[ 'pootle_store_store.pootle_path LIKE %s', 'pootle_store_store.pootle_path NOT LIKE %s', ], params=[resources_path, '/templates/%'] ).select_related('parent').distinct() # Populate with stores and their parent directories, avoiding any # duplicates resources = [] for store in store_objs.iterator(): directory = store.parent if (not directory.is_translationproject() and all(directory.path != r.path for r in resources)): resources.append(directory) if all(store.path != r.path for r in resources): resources.append(store) resources.sort(key=get_path_sortkey) return resources ############################ Methods ###################################### @classmethod def for_username(self, username): """Returns a list of project codes available to `username`. Checks for `view` permissions in project directories, and if no explicit permissions are available, falls back to the root directory for that user. """ key = iri_to_uri('projects:accessible:%s' % username) user_projects = cache.get(key, None) if user_projects is None: logging.debug(u'Cache miss for %s', key) lookup_args = { 'directory__permission_sets__positive_permissions__codename': 'view', 'directory__permission_sets__profile__user__username': username, } user_projects = self.objects.cached().filter(**lookup_args) \ .values_list('code', flat=True) # No explicit permissions for projects, let's examine the root if not user_projects.count(): root_permissions = PermissionSet.objects.filter( directory__pootle_path='/', profile__user__username=username, positive_permissions__codename='view', ) if root_permissions.count(): user_projects = self.objects.cached() \ .values_list('code', flat=True) cache.set(key, user_projects, settings.OBJECT_CACHE_TIMEOUT) return user_projects @classmethod def accessible_by_user(self, user): """Returns a list of project codes accessible by `user`. First checks for `user`, and if no explicit `view` permissions have been found, falls back to `default` (if logged-in) and `nobody` users. """ user_projects = [] check_usernames = ['nobody'] if user.is_authenticated(): check_usernames = [user.username, 'default', 'nobody'] for username in check_usernames: user_projects = self.for_username(username) if user_projects: break return user_projects def __unicode__(self): return self.fullname def __init__(self, *args, **kwargs): super(Project, self).__init__(*args, **kwargs) def save(self, *args, **kwargs): # Create file system directory if needed project_path = self.get_real_path() if not os.path.exists(project_path): os.makedirs(project_path) from pootle_app.models.directory import Directory self.directory = Directory.objects.projects \ .get_or_make_subdir(self.code) super(Project, self).save(*args, **kwargs) # FIXME: far from ideal, should cache at the manager level instead cache.delete(CACHE_KEY) users_list = User.objects.values_list('username', flat=True) cache.delete_many(map(lambda x: 'projects:accessible:%s' % x, users_list)) def delete(self, *args, **kwargs): directory = self.directory # Just doing a plain delete will collect all related objects in memory # before deleting: translation projects, stores, units, quality checks, # pootle_store suggestions, pootle_app suggestions and submissions. # This can easily take down a process. If we do a translation project # at a time and force garbage collection, things stay much more # managable. import gc gc.collect() for tp in self.translationproject_set.iterator(): tp.delete() gc.collect() super(Project, self).delete(*args, **kwargs) directory.delete() # FIXME: far from ideal, should cache at the manager level instead cache.delete(CACHE_KEY) users_list = User.objects.values_list('username', flat=True) cache.delete_many(map(lambda x: 'projects:accessible:%s' % x, users_list)) def get_absolute_url(self): return reverse('pootle-project-overview', args=[self.code]) def get_translate_url(self, **kwargs): return u''.join([ reverse('pootle-project-translate', args=[self.code]), get_editor_filter(**kwargs), ]) def clean(self): if self.code in RESERVED_PROJECT_CODES: raise ValidationError( _('"%s" cannot be used as a project code' % (self.code,)) ) ### TreeItem def get_children(self): return self.translationproject_set.all() def get_cachekey(self): return self.directory.pootle_path def get_parents(self): from pootle_app.models.directory import Directory return [Directory.objects.projects] ### /TreeItem def translated_percentage(self): total = self.get_total_wordcount() translated = self.get_translated_wordcount() max_words = max(total, 1) return int(100.0 * translated / max_words) def get_real_path(self): return absolute_real_path(self.code) def is_accessible_by(self, user): """Returns `True` if the current project is accessible by `user`. """ if user.is_superuser: return True return self.code in Project.accessible_by_user(user) def get_template_filetype(self): if self.localfiletype == 'po': return 'pot' else: return self.localfiletype def get_file_class(self): """Returns the TranslationStore subclass required for parsing project files.""" return factory_classes[self.localfiletype] def file_belongs_to_project(self, filename, match_templates=True): """Tests if ``filename`` matches project filetype (ie. extension). If ``match_templates`` is ``True``, this will also check if the file matches the template filetype. """ template_ext = os.path.extsep + self.get_template_filetype() return (filename.endswith(os.path.extsep + self.localfiletype) or match_templates and filename.endswith(template_ext)) def _detect_treestyle(self): try: dirlisting = os.walk(self.get_real_path()) dirpath, dirnames, filenames = dirlisting.next() if not dirnames: # No subdirectories if filter(self.file_belongs_to_project, filenames): # Translation files found, assume gnu return "gnu" else: # There are subdirectories if filter(lambda dirname: dirname == 'templates' or langcode_re.match(dirname), dirnames): # Found language dirs assume nongnu return "nongnu" else: # No language subdirs found, look for any translation file for dirpath, dirnames, filenames in os.walk(self.get_real_path()): if filter(self.file_belongs_to_project, filenames): return "gnu" except: pass # Unsure return None def get_treestyle(self): """Returns the real treestyle, if :attr:`Project.treestyle` is set to ``auto`` it checks the project directory and tries to guess if it is gnu style or nongnu style. We are biased towards nongnu because it makes managing projects from the web easier. """ if self.treestyle != "auto": return self.treestyle else: detected = self._detect_treestyle() if detected is not None: return detected # When unsure return nongnu return "nongnu" def get_template_translationproject(self): """Returns the translation project that will be used as a template for this project. First it tries to retrieve the translation project that has the special 'templates' language within this project, otherwise it falls back to the source language set for current project. """ try: return self.translationproject_set.get(language__code='templates') except ObjectDoesNotExist: try: return self.translationproject_set \ .get(language=self.source_language_id) except ObjectDoesNotExist: pass
class Language(models.Model): objects = LanguageManager() live = LiveLanguageManager() class Meta: ordering = ['code'] db_table = 'pootle_app_language' code_help_text = _('ISO 639 language code for the language, possibly ' 'followed by an underscore (_) and an ISO 3166 country code. ' '<a href="http://www.w3.org/International/articles/language-tags/">' 'More information</a>') code = models.CharField(max_length=50, null=False, unique=True, db_index=True, verbose_name=_("Code"), help_text=code_help_text) fullname = models.CharField(max_length=255, null=False, verbose_name=_("Full Name")) description_help_text = _('A description of this language. ' 'This is useful to give more information or instructions. ' 'Allowed markup: %s', get_markup_filter_name()) description = MarkupField(blank=True, help_text=description_help_text) specialchars_help_text = _('Enter any special characters that users ' 'might find difficult to type') specialchars = models.CharField(max_length=255, blank=True, verbose_name=_("Special Characters"), help_text=specialchars_help_text) plurals_help_text = _('For more information, visit ' '<a href="http://translate.sourceforge.net/wiki/l10n/pluralforms">' 'our wiki page</a> on plural forms.') nplural_choices = ( (0, _('Unknown')), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6) ) nplurals = models.SmallIntegerField(default=0, choices=nplural_choices, verbose_name=_("Number of Plurals"), help_text=plurals_help_text) pluralequation = models.CharField(max_length=255, blank=True, verbose_name=_("Plural Equation"), help_text=plurals_help_text) directory = models.OneToOneField('pootle_app.Directory', db_index=True, editable=False) pootle_path = property(lambda self: '/%s/' % self.code) def natural_key(self): return (self.code,) natural_key.dependencies = ['pootle_app.Directory'] def save(self, *args, **kwargs): # create corresponding directory object from pootle_app.models.directory import Directory self.directory = Directory.objects.root.get_or_make_subdir(self.code) super(Language, self).save(*args, **kwargs) # FIXME: far from ideal, should cache at the manager level instead cache.delete(CACHE_KEY) cache.set(CACHE_KEY, Language.live.all(), 0) def delete(self, *args, **kwargs): directory = self.directory super(Language, self).delete(*args, **kwargs) directory.delete() # FIXME: far from ideal, should cache at the manager level instead cache.delete(CACHE_KEY) def __repr__(self): return u'<%s: %s>' % (self.__class__.__name__, self.fullname) def __unicode__(self): return u"%s - %s" % (self.name, self.code) @getfromcache def get_mtime(self): return max_column(Unit.objects.filter( store__translation_project__language=self), 'mtime', None) @getfromcache def getquickstats(self): return statssum(self.translationproject_set.iterator()) def get_absolute_url(self): return l(self.pootle_path) def localname(self): """localized fullname""" return tr_lang(self.fullname) name = property(localname) def get_direction(self): """returns language direction""" return language_dir(self.code) def translated_percentage(self): qs = self.getquickstats() word_count = max(qs['totalsourcewords'], 1) return int(100.0 * qs['translatedsourcewords'] / word_count)
class TranslationProject(models.Model, TreeItem): description = MarkupField( blank=True, help_text=_('A description of this translation project. This is ' 'useful to give more information or instructions. Allowed ' 'markup: %s', get_markup_filter_name()), ) language = models.ForeignKey(Language, db_index=True) project = models.ForeignKey(Project, db_index=True) real_path = models.FilePathField(editable=False) directory = models.OneToOneField(Directory, db_index=True, editable=False) pootle_path = models.CharField( max_length=255, null=False, unique=True, db_index=True, editable=False, ) disabled = models.BooleanField(default=False) tags = TaggableManager( blank=True, verbose_name=_("Tags"), help_text=_("A comma-separated list of tags."), ) goals = TaggableManager( blank=True, verbose_name=_("Goals"), through=ItemWithGoal, help_text=_("A comma-separated list of goals."), ) # Cached Unit values total_wordcount = models.PositiveIntegerField( default=0, null=True, editable=False, ) translated_wordcount = models.PositiveIntegerField( default=0, null=True, editable=False, ) fuzzy_wordcount = models.PositiveIntegerField( default=0, null=True, editable=False, ) suggestion_count = models.PositiveIntegerField( default=0, null=True, editable=False, ) failing_critical_count = models.PositiveIntegerField( default=0, null=True, editable=False, ) last_submission = models.OneToOneField( Submission, null=True, editable=False, ) last_unit = models.OneToOneField(Unit, null=True, editable=False) _non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE, settings.PARSE_POOL_CULL_FREQUENCY) index_directory = ".translation_index" objects = TranslationProjectManager() class Meta: unique_together = ('language', 'project') db_table = 'pootle_app_translationproject' ############################ Properties ################################### @property def tag_like_objects(self): """Return the tag like objects applied to this translation project. Tag like objects can be either tags or goals. """ return list(chain(self.tags.all().order_by("name"), self.goals.all().order_by("name"))) @property def name(self): # TODO: See if `self.fullname` can be removed return self.fullname @property def fullname(self): return "%s [%s]" % (self.project.fullname, self.language.name) @property def abs_real_path(self): return absolute_real_path(self.real_path) @abs_real_path.setter def abs_real_path(self, value): self.real_path = relative_real_path(value) @property def file_style(self): return self.project.get_treestyle() @property def checker(self): from translate.filters import checks checkerclasses = [checks.projectcheckers.get(self.project.checkstyle, checks.StandardChecker), checks.StandardUnitChecker] return checks.TeeChecker(checkerclasses=checkerclasses, excludefilters=excluded_filters, errorhandler=self.filtererrorhandler, languagecode=self.language.code) @property def non_db_state(self): if not hasattr(self, "_non_db_state"): try: self._non_db_state = self._non_db_state_cache[self.id] except KeyError: self._non_db_state = TranslationProjectNonDBState(self) self._non_db_state_cache[self.id] = \ TranslationProjectNonDBState(self) return self._non_db_state @property def units(self): self.require_units() # FIXME: we rely on implicit ordering defined in the model. We might # want to consider pootle_path as well return Unit.objects.filter(store__translation_project=self, state__gt=OBSOLETE).select_related('store') @property def is_terminology_project(self): return self.pootle_path.endswith('/terminology/') @property def is_template_project(self): return self == self.project.get_template_translationproject() @property def indexer(self): if (self.non_db_state.indexer is None and self.non_db_state._indexing_enabled): try: indexer = self.make_indexer() if not self.non_db_state._index_initialized: self.init_index(indexer) self.non_db_state._index_initialized = True self.non_db_state.indexer = indexer except Exception as e: logging.warning(u"Could not initialize indexer for %s in %s: " u"%s", self.project.code, self.language.code, str(e)) self.non_db_state._indexing_enabled = False return self.non_db_state.indexer @property def has_index(self): return (self.non_db_state._indexing_enabled and (self.non_db_state._index_initialized or self.indexer is not None)) ############################ Cached properties ############################ @cached_property def code(self): return u'-'.join([self.language.code, self.project.code]) @cached_property def all_goals(self): # Putting the next import at the top of the file causes circular # import issues. from pootle_tagging.models import Goal return Goal.get_goals_for_path(self.pootle_path) ############################ Methods ###################################### def __unicode__(self): return self.pootle_path def __init__(self, *args, **kwargs): super(TranslationProject, self).__init__(*args, **kwargs) def save(self, *args, **kwargs): created = self.id is None project_dir = self.project.get_real_path() if not self.disabled: from pootle_app.project_tree import get_translation_project_dir self.abs_real_path = get_translation_project_dir(self.language, project_dir, self.file_style, make_dirs=True) self.directory = self.language.directory \ .get_or_make_subdir(self.project.code) self.pootle_path = self.directory.pootle_path super(TranslationProject, self).save(*args, **kwargs) if created: self.scan_files() def delete(self, *args, **kwargs): directory = self.directory super(TranslationProject, self).delete(*args, **kwargs) #TODO: avoid an access to directory while flushing the cache directory.flush_cache() directory.delete() def get_absolute_url(self): lang, proj, dir, fn = split_pootle_path(self.pootle_path) return reverse('pootle-tp-overview', args=[lang, proj, dir, fn]) def get_translate_url(self, **kwargs): lang, proj, dir, fn = split_pootle_path(self.pootle_path) return u''.join([ reverse('pootle-tp-translate', args=[lang, proj, dir, fn]), get_editor_filter(**kwargs), ]) def filtererrorhandler(self, functionname, str1, str2, e): logging.error(u"Error in filter %s: %r, %r, %s", functionname, str1, str2, e) return False def is_accessible_by(self, user): """Returns `True` if the current translation project is accessible by `user`. """ if user.is_superuser: return True return self.project.code in Project.accessible_by_user(user) def update(self): """Update all stores to reflect state on disk.""" stores = self.stores.exclude(file='').filter(state__gte=PARSED) for store in stores.iterator(): store.update(update_translation=True, update_structure=True) def sync(self, conservative=True, skip_missing=False, modified_since=0): """Sync unsaved work on all stores to disk.""" stores = self.stores.exclude(file='').filter(state__gte=PARSED) for store in stores.iterator(): store.sync(update_translation=True, update_structure=not conservative, conservative=conservative, create=False, skip_missing=skip_missing, modified_since=modified_since) def get_mtime(self): return self.directory.get_mtime() def require_units(self): """Makes sure all stores are parsed""" errors = 0 for store in self.stores.filter(state__lt=PARSED).iterator(): try: store.require_units() except IntegrityError: logging.info(u"Duplicate IDs in %s", store.abs_real_path) errors += 1 except ParseError as e: logging.info(u"Failed to parse %s\n%s", store.abs_real_path, e) errors += 1 except (IOError, OSError) as e: logging.info(u"Can't access %s\n%s", store.abs_real_path, e) errors += 1 return errors ### TreeItem def get_children_for_stats(self, goal=None): if goal is None: return super(TranslationProject, self).get_children_for_stats() else: from itertools import chain stores, dirs = goal.get_children_for_path(self.pootle_path) return list(chain(stores, dirs)) def get_progeny(self, goal=None): if goal is None: return super(TranslationProject, self).get_progeny() else: return goal.get_stores_for_path(self.pootle_path) def get_self_stats(self, goal=None): if goal is None: return super(TranslationProject, self).get_self_stats() else: return { 'total': self.get_total_wordcount(goal), 'translated': self.get_translated_wordcount(goal), 'fuzzy': self.get_fuzzy_wordcount(goal), 'suggestions': self.get_suggestion_count(goal), 'critical': self.get_critical_error_unit_count(goal), 'lastupdated': self.get_last_updated(goal), 'lastaction': self.get_last_action(goal), } def get_children(self): return self.directory.get_children() def get_total_wordcount(self, goal=None): if goal is None: return self.total_wordcount else: return super(TranslationProject, self).get_total_wordcount(goal) def get_translated_wordcount(self, goal=None): if goal is None: return self.translated_wordcount else: return super(TranslationProject, self).get_translated_wordcount(goal) def get_fuzzy_wordcount(self, goal=None): if goal is None: return self.fuzzy_wordcount else: return super(TranslationProject, self).get_fuzzy_wordcount(goal) def get_suggestion_count(self, goal=None): if goal is None: return self.suggestion_count else: return super(TranslationProject, self).get_suggestion_count(goal) def get_critical_error_unit_count(self, goal=None): if goal is None: return self.failing_critical_count else: return super(TranslationProject, self).get_critical_error_unit_count(goal) def get_next_goal_count(self): # Putting the next import at the top of the file causes circular # import issues. from pootle_tagging.models import Goal goal = Goal.get_most_important_incomplete_for_path(self.directory) if goal is not None: return goal.get_incomplete_words_in_path(self.directory) return 0 def get_last_updated(self, goal=None): if self.last_unit is None: return {'id': 0, 'creation_time': 0, 'snippet': ''} creation_time = dateformat.format(self.last_unit.creation_time, 'U') return { 'id': self.last_unit.id, 'creation_time': int(creation_time), 'snippet': self.last_unit.get_last_updated_message() } def get_last_action(self, goal=None): try: if (self.last_submission is None or (self.last_submission is not None and self.last_submission.unit is None)): return {'id': 0, 'mtime': 0, 'snippet': ''} except Submission.DoesNotExist: return {'id': 0, 'mtime': 0, 'snippet': ''} mtime = dateformat.format(self.last_submission.creation_time, 'U') return { 'id': self.last_submission.unit.id, 'mtime': int(mtime), 'snippet': self.last_submission.get_submission_message() } def get_next_goal_url(self): # Putting the next import at the top of the file causes circular # import issues. from pootle_tagging.models import Goal goal = Goal.get_most_important_incomplete_for_path(self.directory) if goal is not None: return goal.get_translate_url_for_path(self.directory.pootle_path, state='incomplete') return '' def get_cachekey(self): return self.directory.pootle_path def get_parents(self): return [self.language, self.project] ### /TreeItem def update_against_templates(self, pootle_path=None): """Update translation project from templates.""" if self.is_template_project: return template_translation_project = self.project \ .get_template_translationproject() if (template_translation_project is None or template_translation_project == self): return monolingual = self.project.is_monolingual if not monolingual: self.sync() from pootle_app.project_tree import (convert_template, get_translated_name, get_translated_name_gnu) for store in template_translation_project.stores.iterator(): if self.file_style == 'gnu': new_pootle_path, new_path = get_translated_name_gnu(self, store) else: new_pootle_path, new_path = get_translated_name(self, store) if pootle_path is not None and new_pootle_path != pootle_path: continue convert_template(self, store, new_pootle_path, new_path, monolingual) all_files, new_files = self.scan_files(vcs_sync=False) project_path = self.project.get_real_path() if new_files and versioncontrol.hasversioning(project_path): message = ("New files added from %s based on templates" % get_site_title()) filestocommit = [f.file.name for f in new_files] success = True try: output = versioncontrol.add_files(project_path, filestocommit, message) except Exception: logging.exception(u"Failed to add files") success = False if pootle_path is None: from pootle_app.signals import post_template_update post_template_update.send(sender=self) def scan_files(self, vcs_sync=True): """Scan the file system and return a list of translation files. :param vcs_sync: boolean on whether or not to synchronise the PO directory with the VCS checkout. """ proj_ignore = [p.strip() for p in self.project.ignoredfiles.split(',')] ignored_files = set(proj_ignore) ext = os.extsep + self.project.localfiletype # Scan for pots if template project if self.is_template_project: ext = os.extsep + self.project.get_template_filetype() from pootle_app.project_tree import (add_files, match_template_filename, direct_language_match_filename) all_files = [] new_files = [] if self.file_style == 'gnu': if self.pootle_path.startswith('/templates/'): file_filter = lambda filename: match_template_filename( self.project, filename, ) else: file_filter = lambda filename: direct_language_match_filename( self.language.code, filename, ) else: file_filter = lambda filename: True if vcs_sync: from versioncontrol.utils import sync_from_vcs sync_from_vcs(ignored_files, ext, self.real_path, file_filter) all_files, new_files = add_files( self, ignored_files, ext, self.real_path, self.directory, file_filter, ) return all_files, new_files def update_file_from_version_control(self, store): store.sync(update_translation=True) filetoupdate = store.file.name # Keep a copy of working files in memory before updating working_copy = store.file.store try: logging.debug(u"Updating %s from version control", store.file.name) versioncontrol.update_file(filetoupdate) store.file._delete_store_cache() store.file._update_store_cache() except Exception: # Something wrong, file potentially modified, bail out # and replace with working copy logging.exception(u"Near fatal catastrophe, while updating %s " u"from version control", store.file.name) working_copy.save() raise versioncontrol.VersionControlError try: logging.debug(u"Parsing version control copy of %s into db", store.file.name) store.update(update_structure=True, update_translation=True) #FIXME: try to avoid merging if file was not updated logging.debug(u"Merging %s with version control update", store.file.name) store.mergefile(working_copy, None, allownewstrings=False, suggestions=True, notranslate=False, obsoletemissing=False) except Exception: logging.exception(u"Near fatal catastrophe, while merging %s with " u"version control copy", store.file.name) working_copy.save() store.update(update_structure=True, update_translation=True) raise def update_dir(self, request=None, directory=None): """Updates translation project's files from version control, retaining uncommitted translations. """ remote_stats = {} try: versioncontrol.update_dir(self.real_path) except IOError as e: logging.exception(u"Error during update of %s", self.real_path) if request: msg = _("Failed to update from version control: %(error)s", {"error": e}) messages.error(request, msg) return all_files, new_files = self.scan_files() new_file_set = set(new_files) # Go through all stores except any pootle-terminology.* ones if directory.is_translationproject(): stores = self.stores.exclude(file="") else: stores = directory.stores.exclude(file="") for store in stores.iterator(): if store in new_file_set: continue store.sync(update_translation=True) filetoupdate = store.file.name # keep a copy of working files in memory before updating working_copy = store.file.store versioncontrol.copy_to_podir(filetoupdate) store.file._delete_store_cache() store.file._update_store_cache() try: logging.debug(u"Parsing version control copy of %s into db", store.file.name) store.update(update_structure=True, update_translation=True) #FIXME: Try to avoid merging if file was not updated logging.debug(u"Merging %s with version control update", store.file.name) store.mergefile(working_copy, None, allownewstrings=False, suggestions=True, notranslate=False, obsoletemissing=False) except Exception: logging.exception(u"Near fatal catastrophe, while merging %s " "with version control copy", store.file.name) working_copy.save() store.update(update_structure=True, update_translation=True) raise if request: msg = \ _(u'Updated project <em>%(project)s</em> from version control', {'project': self.fullname}) messages.info(request, msg) from pootle_app.signals import post_vc_update post_vc_update.send(sender=self) def update_file(self, request, store): """Updates file from version control, retaining uncommitted translations""" try: self.update_file_from_version_control(store) # FIXME: This belongs to views msg = _(u'Updated file <em>%(filename)s</em> from version control', {'filename': store.file.name}) messages.info(request, msg) from pootle_app.signals import post_vc_update post_vc_update.send(sender=self) except versioncontrol.VersionControlError as e: # FIXME: This belongs to views msg = _(u"Failed to update <em>%(filename)s</em> from " u"version control: %(error)s", { 'filename': store.file.name, 'error': e, } ) messages.error(request, msg) self.scan_files() def commit_dir(self, user, directory, request=None): """Commits files under a directory to version control. This does not do permission checking. """ self.sync() total = directory.get_total_wordcount() translated = directory.get_translated_wordcount() fuzzy = directory.get_fuzzy_wordcount() author = user.username message = stats_message_raw("Commit from %s by user %s." % (get_site_title(), author), total, translated, fuzzy) # Try to append email as well, since some VCS does not allow omitting # it (ie. Git). if user.is_authenticated() and len(user.email): author += " <%s>" % user.email if directory.is_translationproject(): stores = list(self.stores.exclude(file="")) else: stores = list(directory.stores.exclude(file="")) filestocommit = [store.file.name for store in stores] success = True try: project_path = self.project.get_real_path() versioncontrol.add_files(project_path, filestocommit, message, author) # FIXME: This belongs to views if request is not None: msg = _("Committed all files under <em>%(path)s</em> to " "version control", {'path': directory.pootle_path}) messages.success(request, msg) except Exception as e: logging.exception(u"Failed to commit directory") # FIXME: This belongs to views if request is not None: msg = _("Failed to commit to version control: %(error)s", {'error': e}) messages.error(request, msg) success = False from pootle_app.signals import post_vc_commit post_vc_commit.send(sender=self, path_obj=directory, user=user, success=success) return success def commit_file(self, user, store, request=None): """Commits an individual file to version control. This does not do permission checking. """ from pootle_app.signals import post_vc_commit store.sync(update_structure=False, update_translation=True, conservative=True) total = store.get_total_wordcount() translated = store.get_translated_wordcount() fuzzy = store.get_fuzzy_wordcount() author = user.username message = stats_message_raw("Commit from %s by user %s." % \ (get_site_title(), author), total, translated, fuzzy) # Try to append email as well, since some VCS does not allow omitting # it (ie. Git). if user.is_authenticated() and len(user.email): author += " <%s>" % user.email filestocommit = [store.file.name] success = True for file in filestocommit: try: versioncontrol.commit_file(file, message=message, author=author) # FIXME: This belongs to views if request is not None: msg = _("Committed file <em>%(filename)s</em> to version " "control", {'filename': file}) messages.success(request, msg) except Exception as e: logging.exception(u"Failed to commit file") # FIXME: This belongs to views if request is not None: msg_params = { "filename": file, "error": e, } msg = _("Failed to commit <em>%(filename)s</em> to version " "control: %(error)s", msg_params) messages.error(request, msg) success = False post_vc_commit.send(sender=self, path_obj=store, user=user, success=success) return success ########################################################################### def get_archive(self, stores, path=None): """Returns an archive of the given files.""" import shutil import subprocess from pootle_misc import ptempfile as tempfile tempzipfile = None archivecontents = None try: # Using zip command line is fast # The temporary file below is opened and immediately closed for # security reasons fd, tempzipfile = tempfile.mkstemp(prefix='pootle', suffix='.zip') os.close(fd) archivecontents = open(tempzipfile, "wb") file_list = u" ".join( store.abs_real_path[len(self.abs_real_path)+1:] \ for store in stores.iterator() ) process = subprocess.Popen(['zip', '-r', '-', file_list], cwd=self.abs_real_path, stdout=archivecontents) result = process.wait() if result == 0: if path is not None: shutil.move(tempzipfile, path) return else: filedata = open(tempzipfile, "r").read() if filedata: return filedata else: raise Exception("failed to read temporary zip file") else: raise Exception("zip command returned error code: %d" % result) except Exception as e: # But if it doesn't work, we can do it from Python. logging.debug(e) logging.debug("falling back to zipfile module") if path is not None: if tempzipfile is None: fd, tempzipfile = tempfile.mkstemp(prefix='pootle', suffix='.zip') os.close(fd) archivecontents = open(tempzipfile, "wb") else: import cStringIO archivecontents = cStringIO.StringIO() import zipfile archive = zipfile.ZipFile(archivecontents, 'w', zipfile.ZIP_DEFLATED) for store in stores.iterator(): archive.write(store.abs_real_path.encode('utf-8'), store.abs_real_path[len(self.abs_real_path)+1:] .encode('utf-8')) archive.close() if path is not None: shutil.move(tempzipfile, path) else: return archivecontents.getvalue() finally: if tempzipfile is not None and os.path.exists(tempzipfile): os.remove(tempzipfile) try: archivecontents.close() except: pass ########################################################################### def make_indexer(self): """Get an indexing object for this project. Since we do not want to keep the indexing databases open for the lifetime of the TranslationProject (it is cached!), it may NOT be part of the Project object, but should be used via a short living local variable. """ logging.debug(u"Loading indexer for %s", self.pootle_path) indexdir = os.path.join(self.abs_real_path, self.index_directory) from translate.search import indexing indexer = indexing.get_indexer(indexdir) indexer.set_field_analyzers({ "pofilename": indexer.ANALYZER_EXACT, "pomtime": indexer.ANALYZER_EXACT, "dbid": indexer.ANALYZER_EXACT, }) return indexer def init_index(self, indexer): """Initializes the search index.""" #FIXME: stop relying on pomtime so virtual files can be searchable? try: indexer.begin_transaction() for store in self.stores.iterator(): try: self.update_index(indexer, store) except OSError: # Broken link or permission problem? logging.exception("Error indexing %s", store) indexer.commit_transaction() indexer.flush(optimize=True) except Exception: logging.exception(u"Error opening indexer for %s", self) try: indexer.cancel_transaction() except: pass def update_index(self, indexer, store, unitid=None): """Updates the index with the contents of store (limit to ``unitid`` if given). There are two reasons for calling this function: 1. Creating a new instance of :cls:`TranslationProject` (see :meth:`TranslationProject.init_index`) -> Check if the index is up-to-date / rebuild the index if necessary 2. Translating a unit via the web interface -> (re)index only the specified unit(s) The argument ``unitid`` should be None for 1. Known problems: 1. This function should get called, when the po file changes externally. WARNING: You have to stop the pootle server before manually changing po files, if you want to keep the index database in sync. """ #FIXME: leverage file updated signal to check if index needs updating if indexer is None: return False # Check if the pomtime in the index == the latest pomtime pomtime = str(hash(store.get_mtime()) ** 2) pofilenamequery = indexer.make_query([("pofilename", store.pootle_path)], True) pomtimequery = indexer.make_query([("pomtime", pomtime)], True) gooditemsquery = indexer.make_query([pofilenamequery, pomtimequery], True) gooditemsnum = indexer.get_query_result(gooditemsquery) \ .get_matches_count() # If there is at least one up-to-date indexing item, then the po file # was not changed externally -> no need to update the database units = None if (gooditemsnum > 0) and (not unitid): # Nothing to be done return elif unitid is not None: # Update only specific item - usually translation via the web # interface. All other items should still be up-to-date (even with # an older pomtime). # Delete the relevant item from the database units = store.units.filter(id=unitid) itemsquery = indexer.make_query([("dbid", str(unitid))], False) indexer.delete_doc([pofilenamequery, itemsquery]) else: # (item is None) # The po file is not indexed - or it was changed externally # delete all items of this file logging.debug(u"Updating %s indexer for file %s", self.pootle_path, store.pootle_path) indexer.delete_doc({"pofilename": store.pootle_path}) units = store.units addlist = [] for unit in units.iterator(): doc = { "pofilename": store.pootle_path, "pomtime": pomtime, "dbid": str(unit.id), } if unit.hasplural(): orig = "\n".join(unit.source.strings) trans = "\n".join(unit.target.strings) else: orig = unit.source trans = unit.target doc.update({ "source": orig, "target": trans, "notes": unit.getnotes(), "locations": unit.getlocations(), }) addlist.append(doc) if addlist: for add_item in addlist: indexer.index_document(add_item) ########################################################################### def gettermmatcher(self): """Returns the terminology matcher.""" terminology_stores = Store.objects.none() mtime = None if self.is_terminology_project: terminology_stores = self.stores.all() mtime = self.get_mtime() else: # Get global terminology first try: termproject = TranslationProject.objects.get( language=self.language_id, project__code='terminology', ) mtime = termproject.get_mtime() terminology_stores = termproject.stores.all() except TranslationProject.DoesNotExist: pass local_terminology = self.stores.filter( name__startswith='pootle-terminology') for store in local_terminology.iterator(): if mtime is None: mtime = store.get_mtime() else: mtime = max(mtime, store.get_mtime()) terminology_stores = terminology_stores | local_terminology if mtime is None: return if mtime != self.non_db_state.termmatchermtime: from translate.search import match self.non_db_state.termmatcher = match.terminologymatcher( terminology_stores.iterator(), ) self.non_db_state.termmatchermtime = mtime return self.non_db_state.termmatcher
class VirtualFolder(models.Model): # any changes to the `name` field may require updating the schema # see migration 0003_case_sensitive_schema.py name = models.CharField(_('Name'), blank=False, unique=True, max_length=70) title = models.CharField(_('Title'), blank=True, null=True, max_length=255) filter_rules = models.TextField( # Translators: This is a noun. _('Filter'), blank=False, help_text=_('Filtering rules that tell which stores this virtual ' 'folder comprises.'), ) priority = models.FloatField( _('Priority'), default=1, help_text=_('Number specifying importance. Greater priority means it ' 'is more important.'), ) is_public = models.BooleanField( _('Is public?'), default=True, help_text=_('Whether this virtual folder is public or not.'), ) description = MarkupField( _('Description'), blank=True, help_text=_( 'Use this to provide more information or instructions. ' 'Allowed markup: %s', get_markup_filter_display_name()), ) stores = models.ManyToManyField(Store, db_index=True, related_name='vfolders') all_projects = models.BooleanField(default=False) projects = models.ManyToManyField(Project, db_index=True, related_name='vfolders') all_languages = models.BooleanField(default=False) languages = models.ManyToManyField(Language, db_index=True, related_name='vfolders') @cached_property def path_matcher(self): return path_matcher.get(self.__class__)(self) def __unicode__(self): return self.name def save(self, *args, **kwargs): # Force validation of fields. self.clean_fields() self.name = self.name.lower() super(VirtualFolder, self).save(*args, **kwargs) def clean_fields(self): """Validate virtual folder fields.""" if self.priority <= 0: raise ValidationError(u'Priority must be greater than zero.') if not self.filter_rules: raise ValidationError(u'Some filtering rule must be specified.')
class TranslationProject(models.Model): description_help_text = _( 'A description of this translation project. ' 'This is useful to give more information or ' 'instructions. Allowed markup: %s', get_markup_filter_name()) description = MarkupField(blank=True, help_text=description_help_text) language = models.ForeignKey(Language, db_index=True) project = models.ForeignKey(Project, db_index=True) real_path = models.FilePathField(editable=False) directory = models.OneToOneField(Directory, db_index=True, editable=False) pootle_path = models.CharField(max_length=255, null=False, unique=True, db_index=True, editable=False) tags = TaggableManager(blank=True, verbose_name=_("Tags"), help_text=_("A comma-separated list of tags.")) _non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE, settings.PARSE_POOL_CULL_FREQUENCY) index_directory = ".translation_index" objects = TranslationProjectManager() class Meta: unique_together = ('language', 'project') db_table = 'pootle_app_translationproject' def __unicode__(self): return self.pootle_path def save(self, *args, **kwargs): created = self.id is None project_dir = self.project.get_real_path() from pootle_app.project_tree import get_translation_project_dir self.abs_real_path = get_translation_project_dir(self.language, project_dir, self.file_style, make_dirs=True) self.directory = self.language.directory \ .get_or_make_subdir(self.project.code) self.pootle_path = self.directory.pootle_path super(TranslationProject, self).save(*args, **kwargs) if created: self.scan_files() def delete(self, *args, **kwargs): directory = self.directory super(TranslationProject, self).delete(*args, **kwargs) directory.delete() deletefromcache(self, [ "getquickstats", "getcompletestats", "get_mtime", "get_suggestion_count" ]) def get_absolute_url(self): return l(self.pootle_path) def get_translate_url(self, **kwargs): lang, proj, dir, fn = split_pootle_path(self.pootle_path) return u''.join([ reverse('pootle-tp-translate', args=[lang, proj, dir, fn]), get_editor_filter(**kwargs), ]) def natural_key(self): return (self.pootle_path, ) natural_key.dependencies = [ 'pootle_app.Directory', 'pootle_language.Language', 'pootle_project.Project' ] ########################################################################### # Properties # ########################################################################### fullname = property(lambda self: "%s [%s]" % (self.project.fullname, self.language.name)) def _get_abs_real_path(self): return absolute_real_path(self.real_path) def _set_abs_real_path(self, value): self.real_path = relative_real_path(value) abs_real_path = property(_get_abs_real_path, _set_abs_real_path) def _get_treestyle(self): return self.project.get_treestyle() file_style = property(_get_treestyle) def _get_checker(self): from translate.filters import checks checkerclasses = [ checks.projectcheckers.get(self.project.checkstyle, checks.StandardChecker), checks.StandardUnitChecker ] excluded_filters = ['hassuggestion', 'spellcheck'] return checks.TeeChecker(checkerclasses=checkerclasses, excludefilters=excluded_filters, errorhandler=self.filtererrorhandler, languagecode=self.language.code) checker = property(_get_checker) def _get_non_db_state(self): if not hasattr(self, "_non_db_state"): try: self._non_db_state = self._non_db_state_cache[self.id] except KeyError: self._non_db_state = TranslationProjectNonDBState(self) self._non_db_state_cache[self.id] = \ TranslationProjectNonDBState(self) return self._non_db_state non_db_state = property(_get_non_db_state) def _get_units(self): self.require_units() # FIXME: we rely on implicit ordering defined in the model. We might # want to consider pootle_path as well return Unit.objects.filter(store__translation_project=self, state__gt=OBSOLETE).select_related('store') units = property(_get_units) @property def is_terminology_project(self): return self.pootle_path.endswith('/terminology/') @property def is_template_project(self): return self == self.project.get_template_translationproject() def _get_indexer(self): if (self.non_db_state.indexer is None and self.non_db_state._indexing_enabled): try: indexer = self.make_indexer() if not self.non_db_state._index_initialized: self.init_index(indexer) self.non_db_state._index_initialized = True self.non_db_state.indexer = indexer except Exception, e: logging.warning( u"Could not initialize indexer for %s in %s: " u"%s", self.project.code, self.language.code, str(e)) self.non_db_state._indexing_enabled = False return self.non_db_state.indexer
class TranslationProject(models.Model): description_help_text = _( 'A description of this translation project. ' 'This is useful to give more information or ' 'instructions. Allowed markup: %s', get_markup_filter_name()) description = MarkupField(blank=True, help_text=description_help_text) language = models.ForeignKey(Language, db_index=True) project = models.ForeignKey(Project, db_index=True) real_path = models.FilePathField(editable=False) directory = models.OneToOneField(Directory, db_index=True, editable=False) pootle_path = models.CharField(max_length=255, null=False, unique=True, db_index=True, editable=False) tags = TaggableManager(blank=True, verbose_name=_("Tags"), help_text=_("A comma-separated list of tags.")) _non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE, settings.PARSE_POOL_CULL_FREQUENCY) index_directory = ".translation_index" objects = TranslationProjectManager() class Meta: unique_together = ('language', 'project') db_table = 'pootle_app_translationproject' def __unicode__(self): return self.pootle_path def save(self, *args, **kwargs): created = self.id is None project_dir = self.project.get_real_path() from pootle_app.project_tree import get_translation_project_dir self.abs_real_path = get_translation_project_dir(self.language, project_dir, self.file_style, make_dirs=True) self.directory = self.language.directory \ .get_or_make_subdir(self.project.code) self.pootle_path = self.directory.pootle_path super(TranslationProject, self).save(*args, **kwargs) if created: self.scan_files() def delete(self, *args, **kwargs): directory = self.directory super(TranslationProject, self).delete(*args, **kwargs) directory.delete() deletefromcache(self, [ "getquickstats", "getcompletestats", "get_mtime", "get_suggestion_count" ]) def get_absolute_url(self): return l(self.pootle_path) def get_translate_url(self, **kwargs): lang, proj, dir, fn = split_pootle_path(self.pootle_path) return u''.join([ reverse('pootle-tp-translate', args=[lang, proj, dir, fn]), get_editor_filter(**kwargs), ]) def natural_key(self): return (self.pootle_path, ) natural_key.dependencies = [ 'pootle_app.Directory', 'pootle_language.Language', 'pootle_project.Project' ] ########################################################################### # Properties # ########################################################################### fullname = property(lambda self: "%s [%s]" % (self.project.fullname, self.language.name)) def _get_abs_real_path(self): return absolute_real_path(self.real_path) def _set_abs_real_path(self, value): self.real_path = relative_real_path(value) abs_real_path = property(_get_abs_real_path, _set_abs_real_path) def _get_treestyle(self): return self.project.get_treestyle() file_style = property(_get_treestyle) def _get_checker(self): from translate.filters import checks checkerclasses = [ checks.projectcheckers.get(self.project.checkstyle, checks.StandardChecker), checks.StandardUnitChecker ] excluded_filters = ['hassuggestion', 'spellcheck'] return checks.TeeChecker(checkerclasses=checkerclasses, excludefilters=excluded_filters, errorhandler=self.filtererrorhandler, languagecode=self.language.code) checker = property(_get_checker) def _get_non_db_state(self): if not hasattr(self, "_non_db_state"): try: self._non_db_state = self._non_db_state_cache[self.id] except KeyError: self._non_db_state = TranslationProjectNonDBState(self) self._non_db_state_cache[self.id] = \ TranslationProjectNonDBState(self) return self._non_db_state non_db_state = property(_get_non_db_state) def _get_units(self): self.require_units() # FIXME: we rely on implicit ordering defined in the model. We might # want to consider pootle_path as well return Unit.objects.filter(store__translation_project=self, state__gt=OBSOLETE).select_related('store') units = property(_get_units) @property def is_terminology_project(self): return self.pootle_path.endswith('/terminology/') @property def is_template_project(self): return self == self.project.get_template_translationproject() def _get_indexer(self): if (self.non_db_state.indexer is None and self.non_db_state._indexing_enabled): try: indexer = self.make_indexer() if not self.non_db_state._index_initialized: self.init_index(indexer) self.non_db_state._index_initialized = True self.non_db_state.indexer = indexer except Exception as e: logging.warning( u"Could not initialize indexer for %s in %s: " u"%s", self.project.code, self.language.code, str(e)) self.non_db_state._indexing_enabled = False return self.non_db_state.indexer indexer = property(_get_indexer) def _has_index(self): return (self.non_db_state._indexing_enabled and (self.non_db_state._index_initialized or self.indexer is not None)) has_index = property(_has_index) ########################################################################### def filtererrorhandler(self, functionname, str1, str2, e): logging.error(u"Error in filter %s: %r, %r, %s", functionname, str1, str2, e) return False def update(self): """Update all stores to reflect state on disk.""" stores = self.stores.exclude(file='').filter(state__gte=PARSED) for store in stores.iterator(): store.update(update_translation=True, update_structure=True) def sync(self, conservative=True, skip_missing=False, modified_since=0): """Sync unsaved work on all stores to disk.""" stores = self.stores.exclude(file='').filter(state__gte=PARSED) for store in stores.iterator(): store.sync(update_translation=True, update_structure=not conservative, conservative=conservative, create=False, skip_missing=skip_missing, modified_since=modified_since) def get_latest_submission(self): """Get the latest submission done in the Translation project.""" try: sub = Submission.objects.filter(translation_project=self).latest() except Submission.DoesNotExist: return '' return sub.get_submission_message() @getfromcache def get_mtime(self): tp_units = Unit.objects.filter(store__translation_project=self) return max_column(tp_units, 'mtime', None) def require_units(self): """Makes sure all stores are parsed""" errors = 0 for store in self.stores.filter(state__lt=PARSED).iterator(): try: store.require_units() except IntegrityError: logging.info(u"Duplicate IDs in %s", store.abs_real_path) errors += 1 except ParseError as e: logging.info(u"Failed to parse %s\n%s", store.abs_real_path, e) errors += 1 except (IOError, OSError) as e: logging.info(u"Can't access %s\n%s", store.abs_real_path, e) errors += 1 return errors @getfromcache def getquickstats(self): if self.is_template_project: return empty_quickstats errors = self.require_units() tp_not_obsolete_units = Unit.objects.filter( store__translation_project=self, state__gt=OBSOLETE, ) stats = calculate_stats(tp_not_obsolete_units) stats['errors'] = errors return stats @getfromcache def getcompletestats(self): if self.is_template_project: return empty_completestats for store in self.stores.filter(state__lt=CHECKED).iterator(): store.require_qualitychecks() query = QualityCheck.objects.filter( unit__store__translation_project=self, unit__state__gt=UNTRANSLATED, false_positive=False) return group_by_count_extra(query, 'name', 'category') @getfromcache def get_suggestion_count(self): """ Check if any unit in the stores for this translation project has suggestions. """ return Suggestion.objects.filter(unit__store__translation_project=self, unit__state__gt=OBSOLETE).count() def update_against_templates(self, pootle_path=None): """Update translation project from templates.""" if self.is_template_project: return template_translation_project = self.project \ .get_template_translationproject() if (template_translation_project is None or template_translation_project == self): return monolingual = self.project.is_monolingual() if not monolingual: self.sync() if pootle_path is None: oldstats = self.getquickstats() from pootle_app.project_tree import (convert_template, get_translated_name, get_translated_name_gnu) for store in template_translation_project.stores.iterator(): if self.file_style == 'gnu': new_pootle_path, new_path = get_translated_name_gnu( self, store) else: new_pootle_path, new_path = get_translated_name(self, store) if pootle_path is not None and new_pootle_path != pootle_path: continue relative_po_path = os.path.relpath(new_path, settings.PODIRECTORY) try: from pootle.scripts import hooks if not hooks.hook(self.project.code, "pretemplateupdate", relative_po_path): continue except: # Assume hook is not present. pass convert_template(self, store, new_pootle_path, new_path, monolingual) all_files, new_files = self.scan_files(vcs_sync=False) from pootle_misc import versioncontrol project_path = self.project.get_real_path() if new_files and versioncontrol.hasversioning(project_path): from pootle.scripts import hooks message = "New files added from %s based on templates" % \ (settings.TITLE) filestocommit = [] for new_file in new_files: try: filestocommit.extend( hooks.hook(self.project.code, "precommit", new_file.file.name, author=None, message=message)) except ImportError: # Failed to import the hook - we're going to assume there # just isn't a hook to import. That means we'll commit the # original file. filestocommit.append(new_file.file.name) success = True try: output = versioncontrol.add_files(project_path, filestocommit, message) except Exception as e: logging.error(u"Failed to add files: %s", e) success = False for new_file in new_files: try: hooks.hook(self.project.code, "postcommit", new_file.file.name, success=success) except: #FIXME: We should not hide the exception - makes # development impossible pass if pootle_path is None: newstats = self.getquickstats() from pootle_app.models.signals import post_template_update post_template_update.send(sender=self, oldstats=oldstats, newstats=newstats) def scan_files(self, vcs_sync=True): """Scans the file system and returns a list of translation files. :param vcs_sync: boolean on whether or not to synchronise the PO directory with the VCS checkout. """ projects = [p.strip() for p in self.project.ignoredfiles.split(',')] ignored_files = set(projects) ext = os.extsep + self.project.localfiletype # Scan for pots if template project if self.is_template_project: ext = os.extsep + self.project.get_template_filetype() from pootle_app.project_tree import (add_files, match_template_filename, direct_language_match_filename, sync_from_vcs) all_files = [] new_files = [] if self.file_style == 'gnu': if self.pootle_path.startswith('/templates/'): file_filter = lambda filename: match_template_filename( self.project, filename, ) else: file_filter = lambda filename: direct_language_match_filename( self.language.code, filename, ) else: file_filter = lambda filename: True if vcs_sync: sync_from_vcs(ignored_files, ext, self.real_path, file_filter) all_files, new_files = add_files( self, ignored_files, ext, self.real_path, self.directory, file_filter, ) return all_files, new_files def update_file_from_version_control(self, store): from pootle.scripts import hooks store.sync(update_translation=True) filetoupdate = store.file.name try: filetoupdate = hooks.hook(self.project.code, "preupdate", store.file.name) except: pass # Keep a copy of working files in memory before updating oldstats = store.getquickstats() working_copy = store.file.store try: logging.debug(u"Updating %s from version control", store.file.name) from pootle_misc import versioncontrol versioncontrol.update_file(filetoupdate) store.file._delete_store_cache() store.file._update_store_cache() except Exception as e: # Something wrong, file potentially modified, bail out # and replace with working copy logging.error( u"Near fatal catastrophe, exception %s while " u"updating %s from version control", e, store.file.name) working_copy.save() raise VersionControlError try: hooks.hook(self.project.code, "postupdate", store.file.name) except: pass try: logging.debug(u"Parsing version control copy of %s into db", store.file.name) store.update(update_structure=True, update_translation=True) remotestats = store.getquickstats() #FIXME: try to avoid merging if file was not updated logging.debug(u"Merging %s with version control update", store.file.name) store.mergefile(working_copy, None, allownewstrings=False, suggestions=True, notranslate=False, obsoletemissing=False) except Exception as e: logging.error( u"Near fatal catastrophe, exception %s while merging " u"%s with version control copy", e, store.file.name) working_copy.save() store.update(update_structure=True, update_translation=True) raise newstats = store.getquickstats() return oldstats, remotestats, newstats def update_dir(self, request=None, directory=None): """Updates translation project's files from version control, retaining uncommitted translations. """ old_stats = self.getquickstats() remote_stats = {} from pootle_misc import versioncontrol try: versioncontrol.update_dir(self.real_path) except IOError as e: logging.error(u"Error during update of %(path)s:\n%(error)s", { "path": self.real_path, "error": e, }) if request: msg = _("Failed to update from version control: %(error)s", {"error": e}) messages.error(request, msg) return all_files, new_files = self.scan_files() new_file_set = set(new_files) from pootle.scripts import hooks # Go through all stores except any pootle-terminology.* ones if directory.is_translationproject(): stores = self.stores.exclude(file="") else: stores = directory.stores.exclude(file="") for store in stores.iterator(): if store in new_file_set: # these won't have to be merged, since they are new remotestats = store.getquickstats() remote_stats = dictsum(remote_stats, remotestats) continue store.sync(update_translation=True) filetoupdate = store.file.name try: filetoupdate = hooks.hook(self.project.code, "preupdate", store.file.name) except: pass # keep a copy of working files in memory before updating working_copy = store.file.store versioncontrol.copy_to_podir(filetoupdate) store.file._delete_store_cache() store.file._update_store_cache() try: hooks.hook(self.project.code, "postupdate", store.file.name) except: pass try: logging.debug(u"Parsing version control copy of %s into db", store.file.name) store.update(update_structure=True, update_translation=True) remotestats = store.getquickstats() #FIXME: Try to avoid merging if file was not updated logging.debug(u"Merging %s with version control update", store.file.name) store.mergefile(working_copy, None, allownewstrings=False, suggestions=True, notranslate=False, obsoletemissing=False) except Exception as e: logging.error( u"Near fatal catastrophe, exception %s while " "merging %s with version control copy", e, store.file.name) working_copy.save() store.update(update_structure=True, update_translation=True) raise remote_stats = dictsum(remote_stats, remotestats) new_stats = self.getquickstats() if request: msg = [ _(u'Updated project <em>%(project)s</em> from version control', {'project': self.fullname}), stats_message(_(u"Working copy"), old_stats), stats_message(_(u"Remote copy"), remote_stats), stats_message(_(u"Merged copy"), new_stats) ] msg = u"<br/>".join([force_unicode(m) for m in msg]) messages.info(request, msg) from pootle_app.models.signals import post_vc_update post_vc_update.send(sender=self, oldstats=old_stats, remotestats=remote_stats, newstats=new_stats) def update_file(self, request, store): """Updates file from version control, retaining uncommitted translations""" try: old_stats, remote_stats, new_stats = \ self.update_file_from_version_control(store) # FIXME: This belongs to views msg = [ _(u'Updated file <em>%(filename)s</em> from version control', {'filename': store.file.name}), stats_message(_(u"Working copy"), old_stats), stats_message(_(u"Remote copy"), remote_stats), stats_message(_(u"Merged copy"), new_stats) ] msg = u"<br/>".join([force_unicode(m) for m in msg]) messages.info(request, msg) from pootle_app.models.signals import post_vc_update post_vc_update.send(sender=self, oldstats=old_stats, remotestats=remote_stats, newstats=new_stats) except VersionControlError as e: # FIXME: This belongs to views msg = _( u"Failed to update <em>%(filename)s</em> from " u"version control: %(error)s", { 'filename': store.file.name, 'error': e, }) messages.error(request, msg) self.scan_files() def commit_dir(self, user, directory, request=None): """Commits files under a directory to version control. This does not do permission checking. """ self.sync() stats = self.getquickstats() author = user.username message = stats_message_raw( "Commit from %s by user %s." % (settings.TITLE, author), stats) # Try to append email as well, since some VCS does not allow omitting # it (ie. Git). if user.is_authenticated() and len(user.email): author += " <%s>" % user.email if directory.is_translationproject(): stores = list(self.stores.exclude(file="")) else: stores = list(directory.stores.exclude(file="")) filestocommit = [] from pootle.scripts import hooks for store in stores: try: filestocommit.extend( hooks.hook(self.project.code, "precommit", store.file.name, author=author, message=message)) except ImportError: # Failed to import the hook - we're going to assume there just # isn't a hook to import. That means we'll commit the original # file. filestocommit.append(store.file.name) success = True try: from pootle_misc import versioncontrol project_path = self.project.get_real_path() versioncontrol.add_files(project_path, filestocommit, message, author) # FIXME: This belongs to views if request is not None: msg = _( "Committed all files under <em>%(path)s</em> to " "version control", {'path': directory.pootle_path}) messages.success(request, msg) except Exception as e: logging.error(u"Failed to commit: %s", e) # FIXME: This belongs to views if request is not None: msg = _("Failed to commit to version control: %(error)s", {'error': e}) messages.error(request, msg) success = False for store in stores: try: hooks.hook(self.project.code, "postcommit", store.file.name, success=success) except: #FIXME: We should not hide the exception - makes development # impossible pass from pootle_app.models.signals import post_vc_commit post_vc_commit.send(sender=self, path_obj=directory, stats=stats, user=user, success=success) return success def commit_file(self, user, store, request=None): """Commits an individual file to version control. This does not do permission checking. """ store.sync(update_structure=False, update_translation=True, conservative=True) stats = store.getquickstats() author = user.username message = stats_message_raw("Commit from %s by user %s." % \ (settings.TITLE, author), stats) # Try to append email as well, since some VCS does not allow omitting # it (ie. Git). if user.is_authenticated() and len(user.email): author += " <%s>" % user.email from pootle.scripts import hooks try: filestocommit = hooks.hook(self.project.code, "precommit", store.file.name, author=author, message=message) except ImportError: # Failed to import the hook - we're going to assume there just # isn't a hook to import. That means we'll commit the original # file. filestocommit = [store.file.name] success = True try: from pootle_misc import versioncontrol for file in filestocommit: versioncontrol.commit_file(file, message=message, author=author) # FIXME: This belongs to views if request is not None: msg = _( "Committed file <em>%(filename)s</em> to version " "control", {'filename': file}) messages.success(request, msg) except Exception as e: logging.error(u"Failed to commit file: %s", e) # FIXME: This belongs to views if request is not None: msg_params = { 'filename': filename, 'error': e, } msg = _( "Failed to commit <em>%(filename)s</em> to version " "control: %(error)s", msg_params) messages.error(request, msg) success = False try: hooks.hook(self.project.code, "postcommit", store.file.name, success=success) except: #FIXME: We should not hide the exception - makes development # impossible pass from pootle_app.models.signals import post_vc_commit post_vc_commit.send(sender=self, path_obj=store, stats=stats, user=user, success=success) return success def initialize(self): try: from pootle.scripts import hooks hooks.hook(self.project.code, "initialize", self.real_path, self.language.code) except Exception as e: logging.error(u"Failed to initialize (%s): %s", self.language.code, e) ########################################################################### def get_archive(self, stores, path=None): """Returns an archive of the given files.""" import shutil from pootle_misc import ptempfile as tempfile tempzipfile = None try: # Using zip command line is fast # The temporary file below is opened and immediately closed for # security reasons fd, tempzipfile = tempfile.mkstemp(prefix='pootle', suffix='.zip') os.close(fd) file_list = u" ".join( store.abs_real_path[len(self.abs_real_path)+1:] \ for store in stores.iterator() ) cmd = u"cd %(path)s ; zip -r - %(file_list)s > %(tmpfile)s" % { 'path': self.abs_real_path, 'file_list': file_list, 'tmpfile': tempzipfile, } result = os.system(cmd.encode('utf-8')) if result == 0: if path is not None: shutil.move(tempzipfile, path) return else: filedata = open(tempzipfile, "r").read() if filedata: return filedata finally: if tempzipfile is not None and os.path.exists(tempzipfile): os.remove(tempzipfile) # But if it doesn't work, we can do it from python archivecontents = None try: if path is not None: fd, tempzipfile = tempfile.mkstemp(prefix='pootle', suffix='.zip') os.close(fd) archivecontents = open(tempzipfile, "wb") else: import cStringIO archivecontents = cStringIO.StringIO() import zipfile archive = zipfile.ZipFile(archivecontents, 'w', zipfile.ZIP_DEFLATED) for store in stores.iterator(): archive.write( store.abs_real_path.encode('utf-8'), store.abs_real_path[len(self.abs_real_path) + 1:].encode('utf-8')) archive.close() if path is not None: shutil.move(tempzipfile, path) else: return archivecontents.getvalue() finally: if tempzipfile is not None and os.path.exists(tempzipfile): os.remove(tempzipfile) try: archivecontents.close() except: pass ########################################################################### def make_indexer(self): """Get an indexing object for this project. Since we do not want to keep the indexing databases open for the lifetime of the TranslationProject (it is cached!), it may NOT be part of the Project object, but should be used via a short living local variable. """ logging.debug(u"Loading indexer for %s", self.pootle_path) indexdir = os.path.join(self.abs_real_path, self.index_directory) from translate.search import indexing indexer = indexing.get_indexer(indexdir) indexer.set_field_analyzers({ "pofilename": indexer.ANALYZER_EXACT, "pomtime": indexer.ANALYZER_EXACT, "dbid": indexer.ANALYZER_EXACT, }) return indexer def init_index(self, indexer): """Initializes the search index.""" #FIXME: stop relying on pomtime so virtual files can be searchable? try: indexer.begin_transaction() for store in self.stores.iterator(): try: self.update_index(indexer, store) except OSError as e: # Broken link or permission problem? logging.error("Error indexing %s: %s", store, e) indexer.commit_transaction() indexer.flush(optimize=True) except Exception as e: logging.error(u"Error opening indexer for %s:\n%s", self, e) try: indexer.cancel_transaction() except: pass def update_index(self, indexer, store, unitid=None): """Updates the index with the contents of store (limit to ``unitid`` if given). There are two reasons for calling this function: 1. Creating a new instance of :cls:`TranslationProject` (see :meth:`TranslationProject.init_index`) -> Check if the index is up-to-date / rebuild the index if necessary 2. Translating a unit via the web interface -> (re)index only the specified unit(s) The argument ``unitid`` should be None for 1. Known problems: 1. This function should get called, when the po file changes externally. WARNING: You have to stop the pootle server before manually changing po files, if you want to keep the index database in sync. """ #FIXME: leverage file updated signal to check if index needs updating if indexer is None: return False # Check if the pomtime in the index == the latest pomtime pomtime = str(hash(store.get_mtime())**2) pofilenamequery = indexer.make_query( [("pofilename", store.pootle_path)], True) pomtimequery = indexer.make_query([("pomtime", pomtime)], True) gooditemsquery = indexer.make_query([pofilenamequery, pomtimequery], True) gooditemsnum = indexer.get_query_result(gooditemsquery) \ .get_matches_count() # If there is at least one up-to-date indexing item, then the po file # was not changed externally -> no need to update the database units = None if (gooditemsnum > 0) and (not unitid): # Nothing to be done return elif unitid is not None: # Update only specific item - usually translation via the web # interface. All other items should still be up-to-date (even with # an older pomtime). # Delete the relevant item from the database units = store.units.filter(id=unitid) itemsquery = indexer.make_query([("dbid", str(unitid))], False) indexer.delete_doc([pofilenamequery, itemsquery]) else: # (item is None) # The po file is not indexed - or it was changed externally # delete all items of this file logging.debug(u"Updating %s indexer for file %s", self.pootle_path, store.pootle_path) indexer.delete_doc({"pofilename": store.pootle_path}) units = store.units addlist = [] for unit in units.iterator(): doc = { "pofilename": store.pootle_path, "pomtime": pomtime, "dbid": str(unit.id), } if unit.hasplural(): orig = "\n".join(unit.source.strings) trans = "\n".join(unit.target.strings) else: orig = unit.source trans = unit.target doc.update({ "source": orig, "target": trans, "notes": unit.getnotes(), "locations": unit.getlocations(), }) addlist.append(doc) if addlist: for add_item in addlist: indexer.index_document(add_item) ########################################################################### def gettermmatcher(self): """Returns the terminology matcher.""" terminology_stores = Store.objects.none() mtime = None if self.is_terminology_project: terminology_stores = self.stores.all() mtime = self.get_mtime() else: # Get global terminology first try: termproject = TranslationProject.objects.get( language=self.language_id, project__code='terminology', ) mtime = termproject.get_mtime() terminology_stores = termproject.stores.all() except TranslationProject.DoesNotExist: pass local_terminology = self.stores.filter( name__startswith='pootle-terminology') for store in local_terminology.iterator(): if mtime is None: mtime = store.get_mtime() else: mtime = max(mtime, store.get_mtime()) terminology_stores = terminology_stores | local_terminology if mtime is None: return if mtime != self.non_db_state.termmatchermtime: from translate.search import match self.non_db_state.termmatcher = match.terminologymatcher( terminology_stores.iterator(), ) self.non_db_state.termmatchermtime = mtime return self.non_db_state.termmatcher ########################################################################### #FIXME: we should cache results to ease live translation def translate_message(self, singular, plural=None, n=1): for store in self.stores.iterator(): unit = store.findunit(singular) if unit is not None and unit.istranslated(): if unit.hasplural() and n != 1: pluralequation = self.language.pluralequation if pluralequation: pluralfn = gettext.c2py(pluralequation) target = unit.target.strings[pluralfn(n)] if target is not None: return target else: return unit.target # No translation found if n != 1 and plural is not None: return plural else: return singular
class Project(models.Model): objects = ProjectManager() class Meta: ordering = ['code'] db_table = 'pootle_app_project' code_help_text = _('A short code for the project. This should only contain ' 'ASCII characters, numbers, and the underscore (_) character.') code = models.CharField(max_length=255, null=False, unique=True, db_index=True, verbose_name=_('Code'), help_text=code_help_text) fullname = models.CharField(max_length=255, null=False, verbose_name=_("Full Name")) description_help_text = _('A description of this project. ' 'This is useful to give more information or instructions. ' 'Allowed markup: %s', get_markup_filter_name()) description = MarkupField(blank=True, help_text=description_help_text) checker_choices = [('standard', 'standard')] checkers = list(checks.projectcheckers.keys()) checkers.sort() checker_choices.extend([(checker, checker) for checker in checkers]) checkstyle = models.CharField(max_length=50, default='standard', null=False, choices=checker_choices, verbose_name=_('Quality Checks')) localfiletype = models.CharField(max_length=50, default="po", choices=filetype_choices, verbose_name=_('File Type')) treestyle_choices = ( # TODO: check that the None is stored and handled correctly ('auto', _('Automatic detection (slower)')), ('gnu', _('GNU style: files named by language code')), ('nongnu', _('Non-GNU: Each language in its own directory')), ) treestyle = models.CharField(max_length=20, default='auto', choices=treestyle_choices, verbose_name=_('Project Tree Style')) source_language = models.ForeignKey('pootle_language.Language', db_index=True, verbose_name=_('Source Language')) ignoredfiles = models.CharField(max_length=255, blank=True, null=False, default="", verbose_name=_('Ignore Files')) directory = models.OneToOneField('pootle_app.Directory', db_index=True, editable=False) report_target_help_text = _('A URL or an email address where issues ' 'with the source text can be reported.') report_target = models.CharField(max_length=512, blank=True, verbose_name=_("Report Target"), help_text=report_target_help_text) def natural_key(self): return (self.code,) natural_key.dependencies = ['pootle_app.Directory'] def __unicode__(self): return self.fullname def save(self, *args, **kwargs): # Create file system directory if needed project_path = self.get_real_path() if not os.path.exists(project_path): os.makedirs(project_path) from pootle_app.models.directory import Directory self.directory = Directory.objects.projects \ .get_or_make_subdir(self.code) super(Project, self).save(*args, **kwargs) # FIXME: far from ideal, should cache at the manager level instead cache.delete(CACHE_KEY) cache.set(CACHE_KEY, Project.objects.all(), 0) def delete(self, *args, **kwargs): directory = self.directory # Just doing a plain delete will collect all related objects in memory # before deleting: translation projects, stores, units, quality checks, # pootle_store suggestions, pootle_app suggestions and submissions. # This can easily take down a process. If we do a translation project # at a time and force garbage collection, things stay much more # managable. import gc gc.collect() for tp in self.translationproject_set.iterator(): tp.delete() gc.collect() # Here is a different version that first deletes all the related # objects, starting from the leaves. This will have to be maintained # doesn't seem to provide a real advantage in terms of performance. # Doing this finer grained garbage collection keeps memory usage even # lower but can take a bit longer. ''' from pootle_statistics.models import Submission from pootle_app.models import Suggestion as AppSuggestion from pootle_store.models import Suggestion as StoreSuggestion from pootle_store.models import QualityCheck Submission.objects.filter(from_suggestion__translation_project__project=self).delete() AppSuggestion.objects.filter(translation_project__project=self).delete() StoreSuggestion.objects.filter(unit__store__translation_project__project=self).delete() QualityCheck.objects.filter(unit__store__translation_project__project=self).delete() gc.collect() for tp in self.translationproject_set.iterator(): Unit.objects.filter(store__translation_project=tp).delete() gc.collect() ''' super(Project, self).delete(*args, **kwargs) directory.delete() # FIXME: far from ideal, should cache at the manager level instead cache.delete(CACHE_KEY) @getfromcache def get_mtime(self): project_units = Unit.objects.filter( store__translation_project__project=self ) return max_column(project_units, 'mtime', None) @getfromcache def getquickstats(self): return statssum(self.translationproject_set.iterator()) def translated_percentage(self): qs = self.getquickstats() max_words = max(qs['totalsourcewords'], 1) return int(100.0 * qs['translatedsourcewords'] / max_words) def _get_pootle_path(self): return "/projects/" + self.code + "/" pootle_path = property(_get_pootle_path) def get_real_path(self): return absolute_real_path(self.code) def get_absolute_url(self): return l(self.pootle_path) @cached_property def languages(self): """Returns a list of active :cls:`~pootle_languages.models.Language` objects for this :cls:`~pootle_project.models.Project`. """ from pootle_language.models import Language # FIXME: we should better have a way to automatically cache models with # built-in invalidation -- did I hear django-cache-machine? return Language.objects.filter(Q(translationproject__project=self), ~Q(code='templates')) def get_template_filetype(self): if self.localfiletype == 'po': return 'pot' else: return self.localfiletype def get_file_class(self): """Returns the TranslationStore subclass required for parsing project files.""" return factory_classes[self.localfiletype] def is_monolingual(self): """Returns ``True`` if this project is monolingual.""" return is_monolingual(self.get_file_class()) def _get_is_terminology(self): """Returns ``True`` if this project is a terminology project.""" return self.checkstyle == 'terminology' is_terminology = property(_get_is_terminology) def file_belongs_to_project(self, filename, match_templates=True): """Tests if ``filename`` matches project filetype (ie. extension). If ``match_templates`` is ``True``, this will also check if the file matches the template filetype. """ template_ext = os.path.extsep + self.get_template_filetype() return (filename.endswith(os.path.extsep + self.localfiletype) or match_templates and filename.endswith(template_ext)) def _detect_treestyle(self): try: dirlisting = os.walk(self.get_real_path()) dirpath, dirnames, filenames = dirlisting.next() if not dirnames: # No subdirectories if filter(self.file_belongs_to_project, filenames): # Translation files found, assume gnu return "gnu" else: # There are subdirectories if filter(lambda dirname: dirname == 'templates' or langcode_re.match(dirname), dirnames): # Found language dirs assume nongnu return "nongnu" else: # No language subdirs found, look for any translation file for dirpath, dirnames, filenames in os.walk(self.get_real_path()): if filter(self.file_belongs_to_project, filenames): return "gnu" except: pass # Unsure return None def get_treestyle(self): """Returns the real treestyle, if :attr:`Project.treestyle` is set to ``auto`` it checks the project directory and tries to guess if it is gnu style or nongnu style. We are biased towards nongnu because it makes managing projects from the web easier. """ if self.treestyle != "auto": return self.treestyle else: detected = self._detect_treestyle() if detected is not None: return detected # When unsure return nongnu return "nongnu" def get_template_translationproject(self): """Returns the translation project that will be used as a template for this project. First it tries to retrieve the translation project that has the special 'templates' language within this project, otherwise it falls back to the source language set for current project. """ try: return self.translationproject_set.get(language__code='templates') except ObjectDoesNotExist: try: return self.translationproject_set \ .get(language=self.source_language_id) except ObjectDoesNotExist: pass