class PootleSite(models.Model):
"""Model to store each specific Pootle site configuration.
The configuration includes some data for install/upgrade mechanisms.
"""
site = models.OneToOneField(Site, editable=False)
title = models.CharField(
max_length=50,
blank=False,
default="Pootle Demo",
verbose_name=_("Title"),
help_text=_("The name for this Pootle server"),
)
description = MarkupField(
blank=True,
default='',
verbose_name=_("Description"),
help_text=_(
"The description and instructions shown on the about "
"page. Allowed markup: %s", get_markup_filter_name()),
)
objects = PootleSiteManager()
class Meta:
app_label = "pootle_app"
def pootle_context(request):
"""Exposes settings to templates."""
# FIXME: maybe we should expose relevant settings only?
return {
'settings': {
'POOTLE_TITLE':
settings.POOTLE_TITLE,
'POOTLE_INSTANCE_ID':
settings.POOTLE_INSTANCE_ID,
'POOTLE_CONTACT_ENABLED': (settings.POOTLE_CONTACT_ENABLED
and settings.POOTLE_CONTACT_EMAIL),
'POOTLE_MARKUP_FILTER':
get_markup_filter_name(),
'POOTLE_SIGNUP_ENABLED':
settings.POOTLE_SIGNUP_ENABLED,
'SCRIPT_NAME':
settings.SCRIPT_NAME,
'POOTLE_CACHE_TIMEOUT':
settings.POOTLE_CACHE_TIMEOUT,
'DEBUG':
settings.DEBUG,
},
'custom': settings.POOTLE_CUSTOM_TEMPLATE_CONTEXT,
'ALL_LANGUAGES': Language.live.cached_dict(translation.get_language()),
'ALL_PROJECTS': Project.objects.cached_dict(request.user),
'SOCIAL_AUTH_PROVIDERS': _get_social_auth_providers(request),
'display_agreement': _agreement_context(request),
}
def pootle_context(request):
"""Exposes settings to templates."""
# FIXME: maybe we should expose relevant settings only?
return {
'settings': {
'POOTLE_TITLE': settings.POOTLE_TITLE,
'POOTLE_INSTANCE_ID': settings.POOTLE_INSTANCE_ID,
'POOTLE_CONTACT_ENABLED': (settings.POOTLE_CONTACT_ENABLED and
settings.POOTLE_CONTACT_EMAIL),
'POOTLE_MARKUP_FILTER': get_markup_filter_name(),
'POOTLE_SIGNUP_ENABLED': settings.POOTLE_SIGNUP_ENABLED,
'SCRIPT_NAME': settings.SCRIPT_NAME,
'POOTLE_CACHE_TIMEOUT': settings.POOTLE_CACHE_TIMEOUT,
'DEBUG': settings.DEBUG,
},
'custom': settings.POOTLE_CUSTOM_TEMPLATE_CONTEXT,
'ALL_LANGUAGES': Language.live.cached_dict(translation.get_language()),
'ALL_PROJECTS': Project.objects.cached_dict(request.user),
'SOCIAL_AUTH_PROVIDERS': jsonify(_get_social_auth_providers(request)),
'display_agreement': _agreement_context(request),
}
class Goal(TagBase):
"""Goal is a tag with a priority.
Also it might be used to set shared goals across a translation project, for
example a goal with all the files that must focus first their effor on all
the translators (independently of the language they are translating to).
It inherits from TagBase instead of Tag because that way it is possible to
reduce the number of DB queries.
"""
description = MarkupField(
verbose_name=_("Description"),
blank=True,
help_text=_('A description of this goal. This is useful to give more '
'information or instructions. Allowed markup: %s',
get_markup_filter_name()),
)
# Priority goes from 1 to 10, being 1 the greater and 10 the lower.
priority = models.IntegerField(
verbose_name=_("Priority"),
default=10,
help_text=_("The priority for this goal."),
)
# Tells if the goal is going to be shared across a project. This might be
# seen as a 'virtual goal' because it doesn't apply to any real TP, but to
# the templates one.
project_goal = models.BooleanField(
verbose_name=_("Project goal?"),
default=False,
help_text=_("Designates that this is a project goal (shared across "
"all languages in the project)."),
)
# Necessary for assigning and checking permissions.
directory = models.OneToOneField(
'pootle_app.Directory',
db_index=True,
editable=False,
)
CACHED_FUNCTIONS = ["get_raw_stats_for_path"]
class Meta:
ordering = ["priority"]
############################ Properties ###################################
@property
def pootle_path(self):
return "/goals/" + self.slug + "/"
@property
def goal_name(self):
"""Return the goal name, i.e. the name without the 'goal:' prefix.
If this is a project goal, then is appended a text indicating that.
"""
if self.project_goal:
return "%s %s" % (self.name[5:], _("(Project goal)"))
else:
return self.name[5:]
############################ Methods ######################################
@classmethod
def get_goals_for_path(cls, pootle_path):
"""Return the goals applied to the stores in this path.
If this is not the 'templates' translation project for the project then
also return the 'project goals' applied to the stores in the
'templates' translation project.
:param pootle_path: A string with a valid pootle path.
"""
# Putting the next imports at the top of the file causes circular
# import issues.
from pootle_app.models.directory import Directory
from pootle_store.models import Store
directory = Directory.objects.get(pootle_path=pootle_path)
stores_pks = directory.stores.values_list("pk", flat=True)
criteria = {
'items_with_goal__content_type': ContentType.objects \
.get_for_model(Store),
'items_with_goal__object_id__in': stores_pks,
}
try:
tp = directory.translation_project
except:
return []
if tp.is_template_project:
# Return the 'project goals' applied to stores in this path.
return cls.objects.filter(**criteria) \
.order_by('project_goal', 'priority').distinct()
else:
# Get the 'non-project goals' (aka regular goals) applied to stores
# in this path.
criteria['project_goal'] = False
regular_goals = cls.objects.filter(**criteria).distinct()
# Now get the 'project goals' applied to stores in the 'templates'
# TP for this TP's project.
template_tp = tp.project.get_template_translationproject()
if template_tp is None: # If this project has no 'templates' TP.
project_goals = cls.objects.none()
else:
tpl_dir_path = "/%s/%s" % (template_tp.language.code,
pootle_path.split("/", 2)[-1])
try:
tpl_dir = Directory.objects.get(pootle_path=tpl_dir_path)
except Directory.DoesNotExist:
project_goals = cls.objects.none()
else:
tpl_stores_pks = tpl_dir.stores.values_list('pk',
flat=True)
criteria.update({
'project_goal': True,
'items_with_goal__object_id__in': tpl_stores_pks,
})
project_goals = cls.objects.filter(**criteria).distinct()
return list(chain(regular_goals, project_goals))
@classmethod
def get_trail_for_path(cls, pootle_path):
"""Return a list with the trail for the given path.
If the pootle path does not exist, then an empty list is returned.
The trail is all the directories that correspond to the given pootle
path, plus the Translation project where the given pootle path is. For
example for the pootle path /ru/firefoxos/add-ons/dropbox/nvda.po the
following trail is returned:
* Translation project object for /ru/firefoxos/
* Directory object for /ru/firefoxos/add-ons/
* Directory object for /ru/firefoxos/add-ons/dropbox/
Note that no object for the store is included in the returned trail.
:param pootle_path: A string with a valid pootle path.
"""
# Putting the next import at the top of the file causes circular import
# issues.
from pootle_store.models import Store
try:
path_obj = Store.objects.get(pootle_path=pootle_path)
except Store.DoesNotExist:
# Putting the next import at the top of the file causes circular
# import issues.
from pootle_app.models.directory import Directory
try:
path_obj = Directory.objects.get(pootle_path=pootle_path)
except Directory.DoesNotExist:
# If it is not possible to retrieve any path_obj for the
# provided pootle_path, then abort.
return []
if isinstance(path_obj, Store):
path_dir = path_obj.parent
else: # Else it is a directory.
path_dir = path_obj
# Note: Not including path_obj (if it is a store) in path_objs since we
# still don't support including units in a goal.
path_objs = chain([path_obj.translation_project], path_dir.trail())
return path_objs
@classmethod
def get_most_important_incomplete_for_path(cls, path_obj):
"""Return the most important incomplete goal for this path or None.
If this is not the 'templates' translation project for the project then
also considers the 'project goals' applied to the stores in the
'templates' translation project.
The most important goal is the one with the lowest priority, or if more
than a goal have the lower priority then the alphabetical order is
taken in account.
:param path_obj: A pootle path object.
"""
most_important = None
for goal in cls.get_goals_for_path(path_obj.pootle_path):
if (most_important is None or
goal.priority < most_important.priority or
(goal.priority == most_important.priority and
goal.name < most_important.name)):
if goal.get_incomplete_words_in_path(path_obj):
most_important = goal
return most_important
@classmethod
def flush_all_caches_in_tp(cls, translation_project):
"""Remove the cache for all the goals in the given translation project.
:param translation_project: An instance of :class:`TranslationProject`.
"""
pootle_path = translation_project.pootle_path
keys = set()
for goal in cls.get_goals_for_path(pootle_path):
for store in goal.get_stores_for_path(pootle_path):
for path_obj in store.parent.trail():
for function_name in cls.CACHED_FUNCTIONS:
keys.add(iri_to_uri(goal.pootle_path + ":" +
path_obj.pootle_path + ":" +
function_name))
for function_name in cls.CACHED_FUNCTIONS:
keys.add(iri_to_uri(goal.pootle_path + ":" + pootle_path +
":" + function_name))
cache.delete_many(list(keys))
@classmethod
def flush_all_caches_for_path(cls, pootle_path):
"""Remove the cache for all the goals in the given path and upper
directories.
The cache is deleted for the given path, for the directories between
the given path and the translation project, and for the translation
project itself.
:param pootle_path: A string with a valid pootle path.
"""
# Get all the affected objects just once, to avoid querying the
# database all the time if there are too many objects involved.
affected_trail = cls.get_trail_for_path(pootle_path)
if not affected_trail:
return
affected_goals = cls.get_goals_for_path(pootle_path)
keys = []
for goal in affected_goals:
for path_obj in affected_trail:
for function_name in cls.CACHED_FUNCTIONS:
keys.append(iri_to_uri(goal.pootle_path + ":" +
path_obj.pootle_path + ":" +
function_name))
cache.delete_many(keys)
def save(self, *args, **kwargs):
# Putting the next import at the top of the file causes circular import
# issues.
from pootle_app.models.directory import Directory
self.directory = Directory.objects.goals.get_or_make_subdir(self.slug)
super(Goal, self).save(*args, **kwargs)
def delete(self, *args, **kwargs):
directory = self.directory
super(Goal, self).delete(*args, **kwargs)
directory.delete()
def get_translate_url_for_path(self, pootle_path, **kwargs):
"""Return this goal's translate URL for the given path.
:param pootle_path: A string with a valid pootle path.
"""
lang, proj, dir_path, fn = split_pootle_path(pootle_path)
return u''.join([
reverse('pootle-tp-translate', args=[lang, proj, dir_path, fn]),
get_editor_filter(goal=self.slug, **kwargs),
])
def get_critical_url_for_path(self, pootle_path, **kwargs):
"""Return this goal's translate URL for critical checks failures in the
given path.
:param pootle_path: A string with a valid pootle path.
"""
critical = ','.join(get_qualitychecks_by_category(Category.CRITICAL))
return self.get_translate_url_for_path(pootle_path, check=critical)
def get_drill_down_url_for_path(self, pootle_path):
"""Return this goal's drill down URL for the given path.
:param pootle_path: A string with a valid pootle path.
"""
lang, proj, dir_path, filename = split_pootle_path(pootle_path)
reverse_args = [lang, proj, self.slug, dir_path, filename]
return reverse('pootle-tp-goal-drill-down', args=reverse_args)
def get_stores_for_path(self, pootle_path):
"""Return the stores for this goal in the given pootle path.
If this is a project goal then the corresponding stores in the path to
that ones in the 'templates' TP for this goal are returned instead.
:param pootle_path: A string with a valid pootle path.
"""
# Putting the next imports at the top of the file causes circular
# import issues.
from pootle_store.models import Store
from pootle_translationproject.models import TranslationProject
lang, proj, dir_path, filename = split_pootle_path(pootle_path)
# Get the translation project for this pootle_path.
try:
tp = TranslationProject.objects.get(language__code=lang,
project__code=proj)
except TranslationProject.DoesNotExist:
return Store.objects.none()
if self.project_goal and not tp.is_template_project:
# Get the stores for this goal that are in the 'templates' TP.
templates_tp = tp.project.get_template_translationproject()
if templates_tp is None:
return Store.objects.none()
else:
path_in_templates = (templates_tp.pootle_path + dir_path +
filename)
lookups = {
'pootle_path__startswith': path_in_templates,
'goals__in': [self],
}
template_stores_in_goal = Store.objects.filter(**lookups)
# Putting the next imports at the top of the file causes circular
# import issues.
if tp.file_style == 'gnu':
from pootle_app.project_tree import (
get_translated_name_gnu as get_translated_name)
else:
from pootle_app.project_tree import get_translated_name
# Get the pootle path for the corresponding stores in the given
# TP for those stores in the 'templates' TP.
criteria = {
'pootle_path__in': [get_translated_name(tp, store)[0]
for store in template_stores_in_goal],
}
else:
# This is a regular goal or the given TP is the 'templates' TP, so
# just retrieve the goal stores on this TP.
criteria = {
'pootle_path__startswith': pootle_path,
'goals__in': [self],
}
# Return the stores.
return Store.objects.filter(**criteria)
def get_children_for_path(self, pootle_path):
"""Return this goal stores and subdirectories in the given directory.
The subdirectories returned are the ones that have any store for this
goal just below them, or in any of its subdirectories.
If this is a project goal then are returned instead:
* The stores in the given directory that correspond to the goal stores
in the corresponding directory in the 'templates' TP,
* The subdirectories in the given directory that have stores that
correspond to goal stores in the 'templates' TP.
:param pootle_path: The pootle path for a :class:`Directory` instance.
:return: Tuple with a stores list and a directories queryset.
"""
# Putting the next import at the top of the file causes circular import
# issues.
from pootle_app.models.directory import Directory
stores_in_dir = []
subdir_paths = set()
stores_for_path = self.get_stores_for_path(pootle_path)
# Put apart the stores that are just below the directory from those
# that are in subdirectories inside directory.
for store in stores_for_path:
trailing_path = store.pootle_path[len(pootle_path):]
if "/" in trailing_path:
# Store is in a subdirectory.
subdir_name = trailing_path.split("/")[0] + "/"
subdir_paths.add(pootle_path + subdir_name)
else:
# Store is in the directory.
stores_in_dir.append(store)
# Get the subdirectories that have stores for this goal.
subdirs_in_dir = Directory.objects.filter(pootle_path__in=subdir_paths)
# Return a tuple with stores and subdirectories in the given directory.
return (stores_in_dir, subdirs_in_dir)
def slugify(self, tag, i=None):
return slugify_tag_name(tag)
def get_incomplete_words_in_path(self, path_obj):
"""Return the number of incomplete words for this goal in the path.
:param path_obj: A pootle path object.
"""
total = path_obj.get_total_wordcount()
translated = path_obj.get_translated_wordcount()
return total - translated
class AbstractPage(DirtyFieldsMixin, models.Model):
active = models.BooleanField(
_('Active'),
help_text=_('Whether this page is active or not.'),
)
virtual_path = models.CharField(
_("Virtual Path"),
max_length=100,
default='',
unique=True,
help_text='/pages/',
)
# TODO: make title and body localizable fields
title = models.CharField(_("Title"), max_length=100)
body = MarkupField(
# Translators: Content that will be used to display this static page
_("Display Content"),
blank=True,
help_text=_('Allowed markup: %s', get_markup_filter_name()),
)
url = models.URLField(
_("Redirect to URL"),
blank=True,
help_text=_('If set, any references to this page will redirect to this'
' URL'),
)
# This will go away with bug 2830, but works fine for now.
modified_on = models.DateTimeField(
default=now,
editable=False,
auto_now_add=True,
)
objects = PageManager()
class Meta:
abstract = True
def __unicode__(self):
return self.virtual_path
def save(self):
# Update the `modified_on` timestamp only when specific fields change.
dirty_fields = self.get_dirty_fields()
if any(field in dirty_fields for field in ('title', 'body', 'url')):
self.modified_on = now()
super(AbstractPage, self).save()
def get_absolute_url(self):
if self.url:
return self.url
return reverse('pootle-staticpages-display', args=[self.virtual_path])
@staticmethod
def max_pk():
"""Returns the sum of all the highest PKs for each submodel."""
return reduce(
lambda x, y: x + y,
[
int(p.objects.aggregate(Max('pk')).values()[0] or 0)
for p in AbstractPage.__subclasses__()
],
)
def clean(self):
"""Fail validation if:
- URL and body are blank
- Current virtual path exists in other page models
"""
if not self.url and not self.body:
# Translators: 'URL' and 'content' refer to form fields.
raise ValidationError(_('URL or content must be provided.'))
pages = [
p.objects.filter(
Q(virtual_path=self.virtual_path),
~Q(pk=self.pk),
).exists() for p in AbstractPage.__subclasses__()
]
if True in pages:
raise ValidationError(_(u'Virtual path already in use.'))
class VirtualFolder(models.Model):
name = models.CharField(_('Name'), blank=False, max_length=70)
location = models.CharField(
_('Location'),
blank=False,
max_length=255,
help_text=_('Root path where this virtual folder is applied.'),
)
filter_rules = models.TextField(
# Translators: This is a noun.
_('Filter'),
blank=False,
help_text=_('Filtering rules that tell which stores this virtual '
'folder comprises.'),
)
priority = models.FloatField(
_('Priority'),
default=1,
help_text=_('Number specifying importance. Greater priority means it '
'is more important.'),
)
is_browsable = models.BooleanField(
_('Is browsable?'),
default=True,
help_text=_('Whether this virtual folder is active or not.'),
)
description = MarkupField(
_('Description'),
blank=True,
help_text=_(
'Use this to provide more information or instructions. '
'Allowed markup: %s', get_markup_filter_name()),
)
units = models.ManyToManyField(
Unit,
db_index=True,
related_name='vfolders',
)
class Meta:
unique_together = ('name', 'location')
ordering = ['-priority', 'name']
@classmethod
def get_matching_for(cls, pootle_path):
"""Return the matching virtual folders in the given pootle path.
Not all the applicable virtual folders have matching filtering rules.
This method further restricts the list of applicable virtual folders to
retrieve only those with filtering rules that actually match.
"""
return VirtualFolder.objects.filter(
units__store__pootle_path__startswith=pootle_path).distinct()
def __unicode__(self):
return ": ".join([self.name, self.location])
def save(self, *args, **kwargs):
# Force validation of fields.
self.clean_fields()
self.name = self.name.lower()
super(VirtualFolder, self).save(*args, **kwargs)
# Clean any existing relationship between units and this vfolder.
self.units.clear()
# Recreate relationships between this vfolder and units.
for location in self.get_all_pootle_paths():
for filename in self.filter_rules.split(","):
vf_file = "".join([location, filename])
qs = Store.objects.live().filter(pootle_path=vf_file)
if qs.exists():
self.units.add(*qs[0].units.all())
else:
if not vf_file.endswith("/"):
vf_file += "/"
if Directory.objects.filter(pootle_path=vf_file).exists():
qs = Unit.objects.filter(
store__pootle_path__startswith=vf_file)
self.units.add(*qs)
def clean_fields(self):
"""Validate virtual folder fields."""
if not self.priority > 0:
raise ValidationError(u'Priority must be greater than zero.')
elif self.location == "/":
raise ValidationError(u'The "/" location is not allowed. Use '
u'"/{LANG}/{PROJ}/" instead.')
def get_all_pootle_paths(self):
"""Return a list with all the locations this virtual folder applies.
If the virtual folder location has no {LANG} nor {PROJ} placeholders
then the list only contains its location. If any of the placeholders is
present, then they get expanded to match all the existing languages and
projects.
"""
# Locations like /project/<my_proj>/ are not handled correctly. So
# rewrite them.
if self.location.startswith("/projects/"):
self.location = self.location.replace("/projects/", "/{LANG}/")
if "{LANG}" in self.location and "{PROJ}" in self.location:
locations = []
for lang in Language.objects.all():
temp = self.location.replace("{LANG}", lang.code)
for proj in Project.objects.all():
locations.append(temp.replace("{PROJ}", proj.code))
return locations
elif "{LANG}" in self.location:
try:
project = Project.objects.get(code=self.location.split("/")[2])
languages = project.languages.iterator()
except:
languages = Language.objects.iterator()
return [
self.location.replace("{LANG}", lang.code)
for lang in languages
]
elif "{PROJ}" in self.location:
try:
projects = Project.objects.filter(
translationproject__language__code=self.location.split(
"/")[1]).iterator()
except:
projects = Project.objects.iterator()
return [
self.location.replace("{PROJ}", proj.code) for proj in projects
]
return [self.location]
class TranslationProject(models.Model):
_non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE,
settings.PARSE_POOL_CULL_FREQUENCY)
objects = TranslationProjectManager()
index_directory = ".translation_index"
class Meta:
unique_together = ('language', 'project')
db_table = 'pootle_app_translationproject'
description_help_text = _(
'A description of this translation project. '
'This is useful to give more information or instructions. '
'Allowed markup: %s', get_markup_filter_name())
description = MarkupField(blank=True, help_text=description_help_text)
language = models.ForeignKey(Language, db_index=True)
project = models.ForeignKey(Project, db_index=True)
real_path = models.FilePathField(editable=False)
directory = models.OneToOneField(Directory, db_index=True, editable=False)
pootle_path = models.CharField(max_length=255,
null=False,
unique=True,
db_index=True,
editable=False)
def natural_key(self):
return (self.pootle_path, )
natural_key.dependencies = [
'pootle_app.Directory', 'pootle_language.Language',
'pootle_project.Project'
]
def __unicode__(self):
return self.pootle_path
def save(self, *args, **kwargs):
created = self.id is None
project_dir = self.project.get_real_path()
from pootle_app.project_tree import get_translation_project_dir
self.abs_real_path = get_translation_project_dir(self.language,
project_dir,
self.file_style,
make_dirs=True)
self.directory = self.language.directory \
.get_or_make_subdir(self.project.code)
self.pootle_path = self.directory.pootle_path
super(TranslationProject, self).save(*args, **kwargs)
if created:
self.scan_files()
def delete(self, *args, **kwargs):
directory = self.directory
super(TranslationProject, self).delete(*args, **kwargs)
directory.delete()
deletefromcache(self, [
"getquickstats", "getcompletestats", "get_mtime",
"get_suggestion_count"
])
def get_absolute_url(self):
return l(self.pootle_path)
fullname = property(lambda self: "%s [%s]" %
(self.project.fullname, self.language.name))
def _get_abs_real_path(self):
return absolute_real_path(self.real_path)
def _set_abs_real_path(self, value):
self.real_path = relative_real_path(value)
abs_real_path = property(_get_abs_real_path, _set_abs_real_path)
def _get_treestyle(self):
return self.project.get_treestyle()
file_style = property(_get_treestyle)
def _get_checker(self):
from translate.filters import checks
checkerclasses = [
checks.projectcheckers.get(self.project.checkstyle,
checks.StandardChecker),
checks.StandardUnitChecker
]
return checks.TeeChecker(checkerclasses=checkerclasses,
excludefilters=['hassuggestion'],
errorhandler=self.filtererrorhandler,
languagecode=self.language.code)
checker = property(_get_checker)
def filtererrorhandler(self, functionname, str1, str2, e):
logging.error(u"Error in filter %s: %r, %r, %s", functionname, str1,
str2, e)
return False
def _get_non_db_state(self):
if not hasattr(self, "_non_db_state"):
try:
self._non_db_state = self._non_db_state_cache[self.id]
except KeyError:
self._non_db_state = TranslationProjectNonDBState(self)
self._non_db_state_cache[self.id] = \
TranslationProjectNonDBState(self)
return self._non_db_state
non_db_state = property(_get_non_db_state)
def update(self):
"""Update all stores to reflect state on disk"""
stores = self.stores.exclude(file='').filter(state__gte=PARSED)
for store in stores.iterator():
store.update(update_translation=True, update_structure=True)
def sync(self, conservative=True, skip_missing=False, modified_since=0):
"""Sync unsaved work on all stores to disk"""
stores = self.stores.exclude(file='').filter(state__gte=PARSED)
for store in stores.iterator():
store.sync(update_translation=True,
update_structure=not conservative,
conservative=conservative,
create=False,
skip_missing=skip_missing,
modified_since=modified_since)
@getfromcache
def get_mtime(self):
tp_units = Unit.objects.filter(store__translation_project=self)
return max_column(tp_units, 'mtime', None)
def require_units(self):
"""Makes sure all stores are parsed"""
errors = 0
for store in self.stores.filter(state__lt=PARSED).iterator():
try:
store.require_units()
except IntegrityError:
logging.info(u"Duplicate IDs in %s", store.abs_real_path)
errors += 1
except ParseError, e:
logging.info(u"Failed to parse %s\n%s", store.abs_real_path, e)
errors += 1
except (IOError, OSError), e:
logging.info(u"Can't access %s\n%s", store.abs_real_path, e)
errors += 1
class Language(models.Model, TreeItem):
code = models.CharField(
max_length=50,
null=False,
unique=True,
db_index=True,
verbose_name=_("Code"),
help_text=_('ISO 639 language code for the language, possibly '
'followed by an underscore (_) and an ISO 3166 country '
'code. <a href="http://www.w3.org/International/articles/'
'language-tags/">More information</a>'),
)
fullname = models.CharField(
max_length=255,
null=False,
verbose_name=_("Full Name"),
)
description = MarkupField(
blank=True,
help_text=_(
'A description of this language. This is useful to give '
'more information or instructions. Allowed markup: %s',
get_markup_filter_name()),
)
specialchars = models.CharField(
max_length=255,
blank=True,
verbose_name=_("Special Characters"),
help_text=_('Enter any special characters that users might find '
'difficult to type'),
)
nplurals = models.SmallIntegerField(
default=0,
choices=((0, _('Unknown')), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5),
(6, 6)),
verbose_name=_("Number of Plurals"),
help_text=_('For more information, visit <a href="'
'http://docs.translatehouse.org/projects/'
'localization-guide/en/latest/l10n/pluralforms.html">our '
'page</a> on plural forms.'),
)
pluralequation = models.CharField(
max_length=255,
blank=True,
verbose_name=_("Plural Equation"),
help_text=_('For more information, visit <a href="'
'http://docs.translatehouse.org/projects/'
'localization-guide/en/latest/l10n/pluralforms.html">our '
'page</a> on plural forms.'),
)
directory = models.OneToOneField(
'pootle_app.Directory',
db_index=True,
editable=False,
)
objects = LanguageManager()
live = LiveLanguageManager()
class Meta:
ordering = ['code']
db_table = 'pootle_app_language'
def natural_key(self):
return (self.code, )
natural_key.dependencies = ['pootle_app.Directory']
############################ Properties ###################################
@property
def pootle_path(self):
return '/%s/' % self.code
@property
def name(self):
"""Localized fullname for the language."""
return tr_lang(self.fullname)
@property
def direction(self):
"""Return the language direction."""
return language_dir(self.code)
############################ Methods ######################################
def __init__(self, *args, **kwargs):
super(Language, self).__init__(*args, **kwargs)
def __repr__(self):
return u'<%s: %s>' % (self.__class__.__name__, self.fullname)
def __unicode__(self):
return u"%s - %s" % (self.name, self.code)
def save(self, *args, **kwargs):
# create corresponding directory object.
from pootle_app.models.directory import Directory
self.directory = Directory.objects.root.get_or_make_subdir(self.code)
super(Language, self).save(*args, **kwargs)
# FIXME: far from ideal, should cache at the manager level instead.
cache.delete(CACHE_KEY)
def delete(self, *args, **kwargs):
directory = self.directory
super(Language, self).delete(*args, **kwargs)
directory.delete()
# FIXME: far from ideal, should cache at the manager level instead.
cache.delete(CACHE_KEY)
def get_absolute_url(self):
return reverse('pootle-language-overview', args=[self.code])
def get_translate_url(self, **kwargs):
return u''.join([
reverse('pootle-language-translate', args=[self.code]),
get_editor_filter(**kwargs),
])
### TreeItem
def get_children(self):
return self.translationproject_set.all()
def get_cachekey(self):
return self.directory.pootle_path
### /TreeItem
def translated_percentage(self):
total = max(self.get_total_wordcount(), 1)
translated = self.get_translated_wordcount()
return int(100.0 * translated / total)
class Project(models.Model, TreeItem):
code = models.CharField(
max_length=255,
null=False,
unique=True,
db_index=True,
verbose_name=_('Code'),
help_text=_('A short code for the project. This should only contain '
'ASCII characters, numbers, and the underscore (_) '
'character.'),
)
fullname = models.CharField(
max_length=255,
null=False,
verbose_name=_("Full Name"),
)
description = MarkupField(
blank=True,
help_text=_('A description of this project. This is useful to give '
'more information or instructions. Allowed markup: %s',
get_markup_filter_name()),
)
checker_choices = [('standard', 'standard')]
checkers = list(checks.projectcheckers.keys())
checkers.sort()
checker_choices.extend([(checker, checker) for checker in checkers])
checkstyle = models.CharField(
max_length=50,
default='standard',
null=False,
choices=checker_choices,
verbose_name=_('Quality Checks'),
)
localfiletype = models.CharField(
max_length=50,
default="po",
choices=filetype_choices,
verbose_name=_('File Type'),
)
treestyle = models.CharField(
max_length=20,
default='auto',
choices=(
# TODO: check that the None is stored and handled correctly
('auto', _('Automatic detection (slower)')),
('gnu', _('GNU style: files named by language code')),
('nongnu', _('Non-GNU: Each language in its own directory')),
),
verbose_name=_('Project Tree Style'),
)
source_language = models.ForeignKey(
'pootle_language.Language',
db_index=True,
verbose_name=_('Source Language'),
)
ignoredfiles = models.CharField(
max_length=255,
blank=True,
null=False,
default="",
verbose_name=_('Ignore Files'),
)
directory = models.OneToOneField(
'pootle_app.Directory',
db_index=True,
editable=False,
)
report_email = models.EmailField(
max_length=254,
blank=True,
verbose_name=_("Errors Report Email"),
help_text=_('An email address where issues with the source text can '
'be reported.'),
)
disabled = models.BooleanField(verbose_name=_('Disabled'), default=False)
objects = ProjectManager()
class Meta:
ordering = ['code']
db_table = 'pootle_app_project'
############################ Properties ###################################
@property
def name(self):
return self.fullname
@property
def pootle_path(self):
return "/projects/" + self.code + "/"
@property
def is_terminology(self):
"""Returns ``True`` if this project is a terminology project."""
return self.checkstyle == 'terminology'
@property
def is_monolingual(self):
"""Return ``True`` if this project is monolingual."""
return is_monolingual(self.get_file_class())
############################ Cached properties ############################
@cached_property
def languages(self):
"""Returns a list of active :cls:`~pootle_languages.models.Language`
objects for this :cls:`~pootle_project.models.Project`.
"""
from pootle_language.models import Language
# FIXME: we should better have a way to automatically cache models with
# built-in invalidation -- did I hear django-cache-machine?
return Language.objects.filter(Q(translationproject__project=self),
~Q(code='templates'))
@cached_property
def resources(self):
"""Returns a list of :cls:`~pootle_app.models.Directory` and
:cls:`~pootle_store.models.Store` objects available for this
:cls:`~pootle_project.models.Project` across all languages.
"""
from pootle_store.models import Store
resources_path = ''.join(['/%/', self.code, '/%'])
store_objs = Store.objects.extra(
where=[
'pootle_store_store.pootle_path LIKE %s',
'pootle_store_store.pootle_path NOT LIKE %s',
], params=[resources_path, '/templates/%']
).select_related('parent').distinct()
# Populate with stores and their parent directories, avoiding any
# duplicates
resources = []
for store in store_objs.iterator():
directory = store.parent
if (not directory.is_translationproject() and
all(directory.path != r.path for r in resources)):
resources.append(directory)
if all(store.path != r.path for r in resources):
resources.append(store)
resources.sort(key=get_path_sortkey)
return resources
############################ Methods ######################################
@classmethod
def for_username(self, username):
"""Returns a list of project codes available to `username`.
Checks for `view` permissions in project directories, and if no
explicit permissions are available, falls back to the root
directory for that user.
"""
key = iri_to_uri('projects:accessible:%s' % username)
user_projects = cache.get(key, None)
if user_projects is None:
logging.debug(u'Cache miss for %s', key)
lookup_args = {
'directory__permission_sets__positive_permissions__codename':
'view',
'directory__permission_sets__profile__user__username':
username,
}
user_projects = self.objects.cached().filter(**lookup_args) \
.values_list('code', flat=True)
# No explicit permissions for projects, let's examine the root
if not user_projects.count():
root_permissions = PermissionSet.objects.filter(
directory__pootle_path='/',
profile__user__username=username,
positive_permissions__codename='view',
)
if root_permissions.count():
user_projects = self.objects.cached() \
.values_list('code', flat=True)
cache.set(key, user_projects, settings.OBJECT_CACHE_TIMEOUT)
return user_projects
@classmethod
def accessible_by_user(self, user):
"""Returns a list of project codes accessible by `user`.
First checks for `user`, and if no explicit `view` permissions
have been found, falls back to `default` (if logged-in) and
`nobody` users.
"""
user_projects = []
check_usernames = ['nobody']
if user.is_authenticated():
check_usernames = [user.username, 'default', 'nobody']
for username in check_usernames:
user_projects = self.for_username(username)
if user_projects:
break
return user_projects
def __unicode__(self):
return self.fullname
def __init__(self, *args, **kwargs):
super(Project, self).__init__(*args, **kwargs)
def save(self, *args, **kwargs):
# Create file system directory if needed
project_path = self.get_real_path()
if not os.path.exists(project_path):
os.makedirs(project_path)
from pootle_app.models.directory import Directory
self.directory = Directory.objects.projects \
.get_or_make_subdir(self.code)
super(Project, self).save(*args, **kwargs)
# FIXME: far from ideal, should cache at the manager level instead
cache.delete(CACHE_KEY)
users_list = User.objects.values_list('username', flat=True)
cache.delete_many(map(lambda x: 'projects:accessible:%s' % x,
users_list))
def delete(self, *args, **kwargs):
directory = self.directory
# Just doing a plain delete will collect all related objects in memory
# before deleting: translation projects, stores, units, quality checks,
# pootle_store suggestions, pootle_app suggestions and submissions.
# This can easily take down a process. If we do a translation project
# at a time and force garbage collection, things stay much more
# managable.
import gc
gc.collect()
for tp in self.translationproject_set.iterator():
tp.delete()
gc.collect()
super(Project, self).delete(*args, **kwargs)
directory.delete()
# FIXME: far from ideal, should cache at the manager level instead
cache.delete(CACHE_KEY)
users_list = User.objects.values_list('username', flat=True)
cache.delete_many(map(lambda x: 'projects:accessible:%s' % x,
users_list))
def get_absolute_url(self):
return reverse('pootle-project-overview', args=[self.code])
def get_translate_url(self, **kwargs):
return u''.join([
reverse('pootle-project-translate', args=[self.code]),
get_editor_filter(**kwargs),
])
def clean(self):
if self.code in RESERVED_PROJECT_CODES:
raise ValidationError(
_('"%s" cannot be used as a project code' % (self.code,))
)
### TreeItem
def get_children(self):
return self.translationproject_set.all()
def get_cachekey(self):
return self.directory.pootle_path
def get_parents(self):
from pootle_app.models.directory import Directory
return [Directory.objects.projects]
### /TreeItem
def translated_percentage(self):
total = self.get_total_wordcount()
translated = self.get_translated_wordcount()
max_words = max(total, 1)
return int(100.0 * translated / max_words)
def get_real_path(self):
return absolute_real_path(self.code)
def is_accessible_by(self, user):
"""Returns `True` if the current project is accessible by
`user`.
"""
if user.is_superuser:
return True
return self.code in Project.accessible_by_user(user)
def get_template_filetype(self):
if self.localfiletype == 'po':
return 'pot'
else:
return self.localfiletype
def get_file_class(self):
"""Returns the TranslationStore subclass required for parsing
project files."""
return factory_classes[self.localfiletype]
def file_belongs_to_project(self, filename, match_templates=True):
"""Tests if ``filename`` matches project filetype (ie. extension).
If ``match_templates`` is ``True``, this will also check if the
file matches the template filetype.
"""
template_ext = os.path.extsep + self.get_template_filetype()
return (filename.endswith(os.path.extsep + self.localfiletype)
or match_templates and filename.endswith(template_ext))
def _detect_treestyle(self):
try:
dirlisting = os.walk(self.get_real_path())
dirpath, dirnames, filenames = dirlisting.next()
if not dirnames:
# No subdirectories
if filter(self.file_belongs_to_project, filenames):
# Translation files found, assume gnu
return "gnu"
else:
# There are subdirectories
if filter(lambda dirname: dirname == 'templates' or langcode_re.match(dirname), dirnames):
# Found language dirs assume nongnu
return "nongnu"
else:
# No language subdirs found, look for any translation file
for dirpath, dirnames, filenames in os.walk(self.get_real_path()):
if filter(self.file_belongs_to_project, filenames):
return "gnu"
except:
pass
# Unsure
return None
def get_treestyle(self):
"""Returns the real treestyle, if :attr:`Project.treestyle` is set
to ``auto`` it checks the project directory and tries to guess
if it is gnu style or nongnu style.
We are biased towards nongnu because it makes managing projects
from the web easier.
"""
if self.treestyle != "auto":
return self.treestyle
else:
detected = self._detect_treestyle()
if detected is not None:
return detected
# When unsure return nongnu
return "nongnu"
def get_template_translationproject(self):
"""Returns the translation project that will be used as a template
for this project.
First it tries to retrieve the translation project that has the
special 'templates' language within this project, otherwise it
falls back to the source language set for current project.
"""
try:
return self.translationproject_set.get(language__code='templates')
except ObjectDoesNotExist:
try:
return self.translationproject_set \
.get(language=self.source_language_id)
except ObjectDoesNotExist:
pass
class Language(models.Model):
objects = LanguageManager()
live = LiveLanguageManager()
class Meta:
ordering = ['code']
db_table = 'pootle_app_language'
code_help_text = _('ISO 639 language code for the language, possibly '
'followed by an underscore (_) and an ISO 3166 country code. '
'<a href="http://www.w3.org/International/articles/language-tags/">'
'More information</a>')
code = models.CharField(max_length=50, null=False, unique=True,
db_index=True, verbose_name=_("Code"), help_text=code_help_text)
fullname = models.CharField(max_length=255, null=False,
verbose_name=_("Full Name"))
description_help_text = _('A description of this language. '
'This is useful to give more information or instructions. '
'Allowed markup: %s', get_markup_filter_name())
description = MarkupField(blank=True, help_text=description_help_text)
specialchars_help_text = _('Enter any special characters that users '
'might find difficult to type')
specialchars = models.CharField(max_length=255, blank=True,
verbose_name=_("Special Characters"),
help_text=specialchars_help_text)
plurals_help_text = _('For more information, visit '
'<a href="http://translate.sourceforge.net/wiki/l10n/pluralforms">'
'our wiki page</a> on plural forms.')
nplural_choices = (
(0, _('Unknown')), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6)
)
nplurals = models.SmallIntegerField(default=0, choices=nplural_choices,
verbose_name=_("Number of Plurals"), help_text=plurals_help_text)
pluralequation = models.CharField(max_length=255, blank=True,
verbose_name=_("Plural Equation"), help_text=plurals_help_text)
directory = models.OneToOneField('pootle_app.Directory', db_index=True,
editable=False)
pootle_path = property(lambda self: '/%s/' % self.code)
def natural_key(self):
return (self.code,)
natural_key.dependencies = ['pootle_app.Directory']
def save(self, *args, **kwargs):
# create corresponding directory object
from pootle_app.models.directory import Directory
self.directory = Directory.objects.root.get_or_make_subdir(self.code)
super(Language, self).save(*args, **kwargs)
# FIXME: far from ideal, should cache at the manager level instead
cache.delete(CACHE_KEY)
cache.set(CACHE_KEY, Language.live.all(), 0)
def delete(self, *args, **kwargs):
directory = self.directory
super(Language, self).delete(*args, **kwargs)
directory.delete()
# FIXME: far from ideal, should cache at the manager level instead
cache.delete(CACHE_KEY)
def __repr__(self):
return u'<%s: %s>' % (self.__class__.__name__, self.fullname)
def __unicode__(self):
return u"%s - %s" % (self.name, self.code)
@getfromcache
def get_mtime(self):
return max_column(Unit.objects.filter(
store__translation_project__language=self), 'mtime', None)
@getfromcache
def getquickstats(self):
return statssum(self.translationproject_set.iterator())
def get_absolute_url(self):
return l(self.pootle_path)
def localname(self):
"""localized fullname"""
return tr_lang(self.fullname)
name = property(localname)
def get_direction(self):
"""returns language direction"""
return language_dir(self.code)
def translated_percentage(self):
qs = self.getquickstats()
word_count = max(qs['totalsourcewords'], 1)
return int(100.0 * qs['translatedsourcewords'] / word_count)
class TranslationProject(models.Model, TreeItem):
description = MarkupField(
blank=True,
help_text=_('A description of this translation project. This is '
'useful to give more information or instructions. Allowed '
'markup: %s', get_markup_filter_name()),
)
language = models.ForeignKey(Language, db_index=True)
project = models.ForeignKey(Project, db_index=True)
real_path = models.FilePathField(editable=False)
directory = models.OneToOneField(Directory, db_index=True, editable=False)
pootle_path = models.CharField(
max_length=255,
null=False,
unique=True,
db_index=True,
editable=False,
)
disabled = models.BooleanField(default=False)
tags = TaggableManager(
blank=True,
verbose_name=_("Tags"),
help_text=_("A comma-separated list of tags."),
)
goals = TaggableManager(
blank=True,
verbose_name=_("Goals"),
through=ItemWithGoal,
help_text=_("A comma-separated list of goals."),
)
# Cached Unit values
total_wordcount = models.PositiveIntegerField(
default=0,
null=True,
editable=False,
)
translated_wordcount = models.PositiveIntegerField(
default=0,
null=True,
editable=False,
)
fuzzy_wordcount = models.PositiveIntegerField(
default=0,
null=True,
editable=False,
)
suggestion_count = models.PositiveIntegerField(
default=0,
null=True,
editable=False,
)
failing_critical_count = models.PositiveIntegerField(
default=0,
null=True,
editable=False,
)
last_submission = models.OneToOneField(
Submission,
null=True,
editable=False,
)
last_unit = models.OneToOneField(Unit, null=True, editable=False)
_non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE,
settings.PARSE_POOL_CULL_FREQUENCY)
index_directory = ".translation_index"
objects = TranslationProjectManager()
class Meta:
unique_together = ('language', 'project')
db_table = 'pootle_app_translationproject'
############################ Properties ###################################
@property
def tag_like_objects(self):
"""Return the tag like objects applied to this translation project.
Tag like objects can be either tags or goals.
"""
return list(chain(self.tags.all().order_by("name"),
self.goals.all().order_by("name")))
@property
def name(self):
# TODO: See if `self.fullname` can be removed
return self.fullname
@property
def fullname(self):
return "%s [%s]" % (self.project.fullname, self.language.name)
@property
def abs_real_path(self):
return absolute_real_path(self.real_path)
@abs_real_path.setter
def abs_real_path(self, value):
self.real_path = relative_real_path(value)
@property
def file_style(self):
return self.project.get_treestyle()
@property
def checker(self):
from translate.filters import checks
checkerclasses = [checks.projectcheckers.get(self.project.checkstyle,
checks.StandardChecker),
checks.StandardUnitChecker]
return checks.TeeChecker(checkerclasses=checkerclasses,
excludefilters=excluded_filters,
errorhandler=self.filtererrorhandler,
languagecode=self.language.code)
@property
def non_db_state(self):
if not hasattr(self, "_non_db_state"):
try:
self._non_db_state = self._non_db_state_cache[self.id]
except KeyError:
self._non_db_state = TranslationProjectNonDBState(self)
self._non_db_state_cache[self.id] = \
TranslationProjectNonDBState(self)
return self._non_db_state
@property
def units(self):
self.require_units()
# FIXME: we rely on implicit ordering defined in the model. We might
# want to consider pootle_path as well
return Unit.objects.filter(store__translation_project=self,
state__gt=OBSOLETE).select_related('store')
@property
def is_terminology_project(self):
return self.pootle_path.endswith('/terminology/')
@property
def is_template_project(self):
return self == self.project.get_template_translationproject()
@property
def indexer(self):
if (self.non_db_state.indexer is None and
self.non_db_state._indexing_enabled):
try:
indexer = self.make_indexer()
if not self.non_db_state._index_initialized:
self.init_index(indexer)
self.non_db_state._index_initialized = True
self.non_db_state.indexer = indexer
except Exception as e:
logging.warning(u"Could not initialize indexer for %s in %s: "
u"%s", self.project.code, self.language.code,
str(e))
self.non_db_state._indexing_enabled = False
return self.non_db_state.indexer
@property
def has_index(self):
return (self.non_db_state._indexing_enabled and
(self.non_db_state._index_initialized or
self.indexer is not None))
############################ Cached properties ############################
@cached_property
def code(self):
return u'-'.join([self.language.code, self.project.code])
@cached_property
def all_goals(self):
# Putting the next import at the top of the file causes circular
# import issues.
from pootle_tagging.models import Goal
return Goal.get_goals_for_path(self.pootle_path)
############################ Methods ######################################
def __unicode__(self):
return self.pootle_path
def __init__(self, *args, **kwargs):
super(TranslationProject, self).__init__(*args, **kwargs)
def save(self, *args, **kwargs):
created = self.id is None
project_dir = self.project.get_real_path()
if not self.disabled:
from pootle_app.project_tree import get_translation_project_dir
self.abs_real_path = get_translation_project_dir(self.language,
project_dir, self.file_style, make_dirs=True)
self.directory = self.language.directory \
.get_or_make_subdir(self.project.code)
self.pootle_path = self.directory.pootle_path
super(TranslationProject, self).save(*args, **kwargs)
if created:
self.scan_files()
def delete(self, *args, **kwargs):
directory = self.directory
super(TranslationProject, self).delete(*args, **kwargs)
#TODO: avoid an access to directory while flushing the cache
directory.flush_cache()
directory.delete()
def get_absolute_url(self):
lang, proj, dir, fn = split_pootle_path(self.pootle_path)
return reverse('pootle-tp-overview', args=[lang, proj, dir, fn])
def get_translate_url(self, **kwargs):
lang, proj, dir, fn = split_pootle_path(self.pootle_path)
return u''.join([
reverse('pootle-tp-translate', args=[lang, proj, dir, fn]),
get_editor_filter(**kwargs),
])
def filtererrorhandler(self, functionname, str1, str2, e):
logging.error(u"Error in filter %s: %r, %r, %s", functionname, str1,
str2, e)
return False
def is_accessible_by(self, user):
"""Returns `True` if the current translation project is accessible
by `user`.
"""
if user.is_superuser:
return True
return self.project.code in Project.accessible_by_user(user)
def update(self):
"""Update all stores to reflect state on disk."""
stores = self.stores.exclude(file='').filter(state__gte=PARSED)
for store in stores.iterator():
store.update(update_translation=True, update_structure=True)
def sync(self, conservative=True, skip_missing=False, modified_since=0):
"""Sync unsaved work on all stores to disk."""
stores = self.stores.exclude(file='').filter(state__gte=PARSED)
for store in stores.iterator():
store.sync(update_translation=True,
update_structure=not conservative,
conservative=conservative, create=False,
skip_missing=skip_missing,
modified_since=modified_since)
def get_mtime(self):
return self.directory.get_mtime()
def require_units(self):
"""Makes sure all stores are parsed"""
errors = 0
for store in self.stores.filter(state__lt=PARSED).iterator():
try:
store.require_units()
except IntegrityError:
logging.info(u"Duplicate IDs in %s", store.abs_real_path)
errors += 1
except ParseError as e:
logging.info(u"Failed to parse %s\n%s", store.abs_real_path, e)
errors += 1
except (IOError, OSError) as e:
logging.info(u"Can't access %s\n%s", store.abs_real_path, e)
errors += 1
return errors
### TreeItem
def get_children_for_stats(self, goal=None):
if goal is None:
return super(TranslationProject, self).get_children_for_stats()
else:
from itertools import chain
stores, dirs = goal.get_children_for_path(self.pootle_path)
return list(chain(stores, dirs))
def get_progeny(self, goal=None):
if goal is None:
return super(TranslationProject, self).get_progeny()
else:
return goal.get_stores_for_path(self.pootle_path)
def get_self_stats(self, goal=None):
if goal is None:
return super(TranslationProject, self).get_self_stats()
else:
return {
'total': self.get_total_wordcount(goal),
'translated': self.get_translated_wordcount(goal),
'fuzzy': self.get_fuzzy_wordcount(goal),
'suggestions': self.get_suggestion_count(goal),
'critical': self.get_critical_error_unit_count(goal),
'lastupdated': self.get_last_updated(goal),
'lastaction': self.get_last_action(goal),
}
def get_children(self):
return self.directory.get_children()
def get_total_wordcount(self, goal=None):
if goal is None:
return self.total_wordcount
else:
return super(TranslationProject, self).get_total_wordcount(goal)
def get_translated_wordcount(self, goal=None):
if goal is None:
return self.translated_wordcount
else:
return super(TranslationProject, self).get_translated_wordcount(goal)
def get_fuzzy_wordcount(self, goal=None):
if goal is None:
return self.fuzzy_wordcount
else:
return super(TranslationProject, self).get_fuzzy_wordcount(goal)
def get_suggestion_count(self, goal=None):
if goal is None:
return self.suggestion_count
else:
return super(TranslationProject, self).get_suggestion_count(goal)
def get_critical_error_unit_count(self, goal=None):
if goal is None:
return self.failing_critical_count
else:
return super(TranslationProject, self).get_critical_error_unit_count(goal)
def get_next_goal_count(self):
# Putting the next import at the top of the file causes circular
# import issues.
from pootle_tagging.models import Goal
goal = Goal.get_most_important_incomplete_for_path(self.directory)
if goal is not None:
return goal.get_incomplete_words_in_path(self.directory)
return 0
def get_last_updated(self, goal=None):
if self.last_unit is None:
return {'id': 0, 'creation_time': 0, 'snippet': ''}
creation_time = dateformat.format(self.last_unit.creation_time, 'U')
return {
'id': self.last_unit.id,
'creation_time': int(creation_time),
'snippet': self.last_unit.get_last_updated_message()
}
def get_last_action(self, goal=None):
try:
if (self.last_submission is None or
(self.last_submission is not None and
self.last_submission.unit is None)):
return {'id': 0, 'mtime': 0, 'snippet': ''}
except Submission.DoesNotExist:
return {'id': 0, 'mtime': 0, 'snippet': ''}
mtime = dateformat.format(self.last_submission.creation_time, 'U')
return {
'id': self.last_submission.unit.id,
'mtime': int(mtime),
'snippet': self.last_submission.get_submission_message()
}
def get_next_goal_url(self):
# Putting the next import at the top of the file causes circular
# import issues.
from pootle_tagging.models import Goal
goal = Goal.get_most_important_incomplete_for_path(self.directory)
if goal is not None:
return goal.get_translate_url_for_path(self.directory.pootle_path,
state='incomplete')
return ''
def get_cachekey(self):
return self.directory.pootle_path
def get_parents(self):
return [self.language, self.project]
### /TreeItem
def update_against_templates(self, pootle_path=None):
"""Update translation project from templates."""
if self.is_template_project:
return
template_translation_project = self.project \
.get_template_translationproject()
if (template_translation_project is None or
template_translation_project == self):
return
monolingual = self.project.is_monolingual
if not monolingual:
self.sync()
from pootle_app.project_tree import (convert_template,
get_translated_name,
get_translated_name_gnu)
for store in template_translation_project.stores.iterator():
if self.file_style == 'gnu':
new_pootle_path, new_path = get_translated_name_gnu(self, store)
else:
new_pootle_path, new_path = get_translated_name(self, store)
if pootle_path is not None and new_pootle_path != pootle_path:
continue
convert_template(self, store, new_pootle_path, new_path,
monolingual)
all_files, new_files = self.scan_files(vcs_sync=False)
project_path = self.project.get_real_path()
if new_files and versioncontrol.hasversioning(project_path):
message = ("New files added from %s based on templates" %
get_site_title())
filestocommit = [f.file.name for f in new_files]
success = True
try:
output = versioncontrol.add_files(project_path, filestocommit,
message)
except Exception:
logging.exception(u"Failed to add files")
success = False
if pootle_path is None:
from pootle_app.signals import post_template_update
post_template_update.send(sender=self)
def scan_files(self, vcs_sync=True):
"""Scan the file system and return a list of translation files.
:param vcs_sync: boolean on whether or not to synchronise the PO
directory with the VCS checkout.
"""
proj_ignore = [p.strip() for p in self.project.ignoredfiles.split(',')]
ignored_files = set(proj_ignore)
ext = os.extsep + self.project.localfiletype
# Scan for pots if template project
if self.is_template_project:
ext = os.extsep + self.project.get_template_filetype()
from pootle_app.project_tree import (add_files,
match_template_filename,
direct_language_match_filename)
all_files = []
new_files = []
if self.file_style == 'gnu':
if self.pootle_path.startswith('/templates/'):
file_filter = lambda filename: match_template_filename(
self.project, filename,
)
else:
file_filter = lambda filename: direct_language_match_filename(
self.language.code, filename,
)
else:
file_filter = lambda filename: True
if vcs_sync:
from versioncontrol.utils import sync_from_vcs
sync_from_vcs(ignored_files, ext, self.real_path, file_filter)
all_files, new_files = add_files(
self,
ignored_files,
ext,
self.real_path,
self.directory,
file_filter,
)
return all_files, new_files
def update_file_from_version_control(self, store):
store.sync(update_translation=True)
filetoupdate = store.file.name
# Keep a copy of working files in memory before updating
working_copy = store.file.store
try:
logging.debug(u"Updating %s from version control", store.file.name)
versioncontrol.update_file(filetoupdate)
store.file._delete_store_cache()
store.file._update_store_cache()
except Exception:
# Something wrong, file potentially modified, bail out
# and replace with working copy
logging.exception(u"Near fatal catastrophe, while updating %s "
u"from version control", store.file.name)
working_copy.save()
raise versioncontrol.VersionControlError
try:
logging.debug(u"Parsing version control copy of %s into db",
store.file.name)
store.update(update_structure=True, update_translation=True)
#FIXME: try to avoid merging if file was not updated
logging.debug(u"Merging %s with version control update",
store.file.name)
store.mergefile(working_copy, None, allownewstrings=False,
suggestions=True, notranslate=False,
obsoletemissing=False)
except Exception:
logging.exception(u"Near fatal catastrophe, while merging %s with "
u"version control copy", store.file.name)
working_copy.save()
store.update(update_structure=True, update_translation=True)
raise
def update_dir(self, request=None, directory=None):
"""Updates translation project's files from version control, retaining
uncommitted translations.
"""
remote_stats = {}
try:
versioncontrol.update_dir(self.real_path)
except IOError as e:
logging.exception(u"Error during update of %s", self.real_path)
if request:
msg = _("Failed to update from version control: %(error)s",
{"error": e})
messages.error(request, msg)
return
all_files, new_files = self.scan_files()
new_file_set = set(new_files)
# Go through all stores except any pootle-terminology.* ones
if directory.is_translationproject():
stores = self.stores.exclude(file="")
else:
stores = directory.stores.exclude(file="")
for store in stores.iterator():
if store in new_file_set:
continue
store.sync(update_translation=True)
filetoupdate = store.file.name
# keep a copy of working files in memory before updating
working_copy = store.file.store
versioncontrol.copy_to_podir(filetoupdate)
store.file._delete_store_cache()
store.file._update_store_cache()
try:
logging.debug(u"Parsing version control copy of %s into db",
store.file.name)
store.update(update_structure=True, update_translation=True)
#FIXME: Try to avoid merging if file was not updated
logging.debug(u"Merging %s with version control update",
store.file.name)
store.mergefile(working_copy, None, allownewstrings=False,
suggestions=True, notranslate=False,
obsoletemissing=False)
except Exception:
logging.exception(u"Near fatal catastrophe, while merging %s "
"with version control copy", store.file.name)
working_copy.save()
store.update(update_structure=True, update_translation=True)
raise
if request:
msg = \
_(u'Updated project <em>%(project)s</em> from version control',
{'project': self.fullname})
messages.info(request, msg)
from pootle_app.signals import post_vc_update
post_vc_update.send(sender=self)
def update_file(self, request, store):
"""Updates file from version control, retaining uncommitted
translations"""
try:
self.update_file_from_version_control(store)
# FIXME: This belongs to views
msg = _(u'Updated file <em>%(filename)s</em> from version control',
{'filename': store.file.name})
messages.info(request, msg)
from pootle_app.signals import post_vc_update
post_vc_update.send(sender=self)
except versioncontrol.VersionControlError as e:
# FIXME: This belongs to views
msg = _(u"Failed to update <em>%(filename)s</em> from "
u"version control: %(error)s",
{
'filename': store.file.name,
'error': e,
}
)
messages.error(request, msg)
self.scan_files()
def commit_dir(self, user, directory, request=None):
"""Commits files under a directory to version control.
This does not do permission checking.
"""
self.sync()
total = directory.get_total_wordcount()
translated = directory.get_translated_wordcount()
fuzzy = directory.get_fuzzy_wordcount()
author = user.username
message = stats_message_raw("Commit from %s by user %s." %
(get_site_title(), author),
total, translated, fuzzy)
# Try to append email as well, since some VCS does not allow omitting
# it (ie. Git).
if user.is_authenticated() and len(user.email):
author += " <%s>" % user.email
if directory.is_translationproject():
stores = list(self.stores.exclude(file=""))
else:
stores = list(directory.stores.exclude(file=""))
filestocommit = [store.file.name for store in stores]
success = True
try:
project_path = self.project.get_real_path()
versioncontrol.add_files(project_path, filestocommit, message,
author)
# FIXME: This belongs to views
if request is not None:
msg = _("Committed all files under <em>%(path)s</em> to "
"version control", {'path': directory.pootle_path})
messages.success(request, msg)
except Exception as e:
logging.exception(u"Failed to commit directory")
# FIXME: This belongs to views
if request is not None:
msg = _("Failed to commit to version control: %(error)s",
{'error': e})
messages.error(request, msg)
success = False
from pootle_app.signals import post_vc_commit
post_vc_commit.send(sender=self, path_obj=directory,
user=user, success=success)
return success
def commit_file(self, user, store, request=None):
"""Commits an individual file to version control.
This does not do permission checking.
"""
from pootle_app.signals import post_vc_commit
store.sync(update_structure=False, update_translation=True,
conservative=True)
total = store.get_total_wordcount()
translated = store.get_translated_wordcount()
fuzzy = store.get_fuzzy_wordcount()
author = user.username
message = stats_message_raw("Commit from %s by user %s." % \
(get_site_title(), author), total, translated, fuzzy)
# Try to append email as well, since some VCS does not allow omitting
# it (ie. Git).
if user.is_authenticated() and len(user.email):
author += " <%s>" % user.email
filestocommit = [store.file.name]
success = True
for file in filestocommit:
try:
versioncontrol.commit_file(file, message=message,
author=author)
# FIXME: This belongs to views
if request is not None:
msg = _("Committed file <em>%(filename)s</em> to version "
"control", {'filename': file})
messages.success(request, msg)
except Exception as e:
logging.exception(u"Failed to commit file")
# FIXME: This belongs to views
if request is not None:
msg_params = {
"filename": file,
"error": e,
}
msg = _("Failed to commit <em>%(filename)s</em> to version "
"control: %(error)s", msg_params)
messages.error(request, msg)
success = False
post_vc_commit.send(sender=self, path_obj=store,
user=user, success=success)
return success
###########################################################################
def get_archive(self, stores, path=None):
"""Returns an archive of the given files."""
import shutil
import subprocess
from pootle_misc import ptempfile as tempfile
tempzipfile = None
archivecontents = None
try:
# Using zip command line is fast
# The temporary file below is opened and immediately closed for
# security reasons
fd, tempzipfile = tempfile.mkstemp(prefix='pootle', suffix='.zip')
os.close(fd)
archivecontents = open(tempzipfile, "wb")
file_list = u" ".join(
store.abs_real_path[len(self.abs_real_path)+1:] \
for store in stores.iterator()
)
process = subprocess.Popen(['zip', '-r', '-', file_list],
cwd=self.abs_real_path,
stdout=archivecontents)
result = process.wait()
if result == 0:
if path is not None:
shutil.move(tempzipfile, path)
return
else:
filedata = open(tempzipfile, "r").read()
if filedata:
return filedata
else:
raise Exception("failed to read temporary zip file")
else:
raise Exception("zip command returned error code: %d" % result)
except Exception as e:
# But if it doesn't work, we can do it from Python.
logging.debug(e)
logging.debug("falling back to zipfile module")
if path is not None:
if tempzipfile is None:
fd, tempzipfile = tempfile.mkstemp(prefix='pootle',
suffix='.zip')
os.close(fd)
archivecontents = open(tempzipfile, "wb")
else:
import cStringIO
archivecontents = cStringIO.StringIO()
import zipfile
archive = zipfile.ZipFile(archivecontents, 'w',
zipfile.ZIP_DEFLATED)
for store in stores.iterator():
archive.write(store.abs_real_path.encode('utf-8'),
store.abs_real_path[len(self.abs_real_path)+1:]
.encode('utf-8'))
archive.close()
if path is not None:
shutil.move(tempzipfile, path)
else:
return archivecontents.getvalue()
finally:
if tempzipfile is not None and os.path.exists(tempzipfile):
os.remove(tempzipfile)
try:
archivecontents.close()
except:
pass
###########################################################################
def make_indexer(self):
"""Get an indexing object for this project.
Since we do not want to keep the indexing databases open for the
lifetime of the TranslationProject (it is cached!), it may NOT be
part of the Project object, but should be used via a short living
local variable.
"""
logging.debug(u"Loading indexer for %s", self.pootle_path)
indexdir = os.path.join(self.abs_real_path, self.index_directory)
from translate.search import indexing
indexer = indexing.get_indexer(indexdir)
indexer.set_field_analyzers({
"pofilename": indexer.ANALYZER_EXACT,
"pomtime": indexer.ANALYZER_EXACT,
"dbid": indexer.ANALYZER_EXACT,
})
return indexer
def init_index(self, indexer):
"""Initializes the search index."""
#FIXME: stop relying on pomtime so virtual files can be searchable?
try:
indexer.begin_transaction()
for store in self.stores.iterator():
try:
self.update_index(indexer, store)
except OSError:
# Broken link or permission problem?
logging.exception("Error indexing %s", store)
indexer.commit_transaction()
indexer.flush(optimize=True)
except Exception:
logging.exception(u"Error opening indexer for %s", self)
try:
indexer.cancel_transaction()
except:
pass
def update_index(self, indexer, store, unitid=None):
"""Updates the index with the contents of store (limit to
``unitid`` if given).
There are two reasons for calling this function:
1. Creating a new instance of :cls:`TranslationProject`
(see :meth:`TranslationProject.init_index`)
-> Check if the index is up-to-date / rebuild the index if
necessary
2. Translating a unit via the web interface
-> (re)index only the specified unit(s)
The argument ``unitid`` should be None for 1.
Known problems:
1. This function should get called, when the po file changes
externally.
WARNING: You have to stop the pootle server before manually
changing po files, if you want to keep the index database in
sync.
"""
#FIXME: leverage file updated signal to check if index needs updating
if indexer is None:
return False
# Check if the pomtime in the index == the latest pomtime
pomtime = str(hash(store.get_mtime()) ** 2)
pofilenamequery = indexer.make_query([("pofilename",
store.pootle_path)], True)
pomtimequery = indexer.make_query([("pomtime", pomtime)], True)
gooditemsquery = indexer.make_query([pofilenamequery, pomtimequery],
True)
gooditemsnum = indexer.get_query_result(gooditemsquery) \
.get_matches_count()
# If there is at least one up-to-date indexing item, then the po file
# was not changed externally -> no need to update the database
units = None
if (gooditemsnum > 0) and (not unitid):
# Nothing to be done
return
elif unitid is not None:
# Update only specific item - usually translation via the web
# interface. All other items should still be up-to-date (even with
# an older pomtime).
# Delete the relevant item from the database
units = store.units.filter(id=unitid)
itemsquery = indexer.make_query([("dbid", str(unitid))], False)
indexer.delete_doc([pofilenamequery, itemsquery])
else:
# (item is None)
# The po file is not indexed - or it was changed externally
# delete all items of this file
logging.debug(u"Updating %s indexer for file %s", self.pootle_path,
store.pootle_path)
indexer.delete_doc({"pofilename": store.pootle_path})
units = store.units
addlist = []
for unit in units.iterator():
doc = {
"pofilename": store.pootle_path,
"pomtime": pomtime,
"dbid": str(unit.id),
}
if unit.hasplural():
orig = "\n".join(unit.source.strings)
trans = "\n".join(unit.target.strings)
else:
orig = unit.source
trans = unit.target
doc.update({
"source": orig,
"target": trans,
"notes": unit.getnotes(),
"locations": unit.getlocations(),
})
addlist.append(doc)
if addlist:
for add_item in addlist:
indexer.index_document(add_item)
###########################################################################
def gettermmatcher(self):
"""Returns the terminology matcher."""
terminology_stores = Store.objects.none()
mtime = None
if self.is_terminology_project:
terminology_stores = self.stores.all()
mtime = self.get_mtime()
else:
# Get global terminology first
try:
termproject = TranslationProject.objects.get(
language=self.language_id,
project__code='terminology',
)
mtime = termproject.get_mtime()
terminology_stores = termproject.stores.all()
except TranslationProject.DoesNotExist:
pass
local_terminology = self.stores.filter(
name__startswith='pootle-terminology')
for store in local_terminology.iterator():
if mtime is None:
mtime = store.get_mtime()
else:
mtime = max(mtime, store.get_mtime())
terminology_stores = terminology_stores | local_terminology
if mtime is None:
return
if mtime != self.non_db_state.termmatchermtime:
from translate.search import match
self.non_db_state.termmatcher = match.terminologymatcher(
terminology_stores.iterator(),
)
self.non_db_state.termmatchermtime = mtime
return self.non_db_state.termmatcher
class TranslationProject(models.Model):
description_help_text = _(
'A description of this translation project. '
'This is useful to give more information or '
'instructions. Allowed markup: %s', get_markup_filter_name())
description = MarkupField(blank=True, help_text=description_help_text)
language = models.ForeignKey(Language, db_index=True)
project = models.ForeignKey(Project, db_index=True)
real_path = models.FilePathField(editable=False)
directory = models.OneToOneField(Directory, db_index=True, editable=False)
pootle_path = models.CharField(max_length=255,
null=False,
unique=True,
db_index=True,
editable=False)
tags = TaggableManager(blank=True,
verbose_name=_("Tags"),
help_text=_("A comma-separated list of tags."))
_non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE,
settings.PARSE_POOL_CULL_FREQUENCY)
index_directory = ".translation_index"
objects = TranslationProjectManager()
class Meta:
unique_together = ('language', 'project')
db_table = 'pootle_app_translationproject'
def __unicode__(self):
return self.pootle_path
def save(self, *args, **kwargs):
created = self.id is None
project_dir = self.project.get_real_path()
from pootle_app.project_tree import get_translation_project_dir
self.abs_real_path = get_translation_project_dir(self.language,
project_dir,
self.file_style,
make_dirs=True)
self.directory = self.language.directory \
.get_or_make_subdir(self.project.code)
self.pootle_path = self.directory.pootle_path
super(TranslationProject, self).save(*args, **kwargs)
if created:
self.scan_files()
def delete(self, *args, **kwargs):
directory = self.directory
super(TranslationProject, self).delete(*args, **kwargs)
directory.delete()
deletefromcache(self, [
"getquickstats", "getcompletestats", "get_mtime",
"get_suggestion_count"
])
def get_absolute_url(self):
return l(self.pootle_path)
def get_translate_url(self, **kwargs):
lang, proj, dir, fn = split_pootle_path(self.pootle_path)
return u''.join([
reverse('pootle-tp-translate', args=[lang, proj, dir, fn]),
get_editor_filter(**kwargs),
])
def natural_key(self):
return (self.pootle_path, )
natural_key.dependencies = [
'pootle_app.Directory', 'pootle_language.Language',
'pootle_project.Project'
]
###########################################################################
# Properties #
###########################################################################
fullname = property(lambda self: "%s [%s]" %
(self.project.fullname, self.language.name))
def _get_abs_real_path(self):
return absolute_real_path(self.real_path)
def _set_abs_real_path(self, value):
self.real_path = relative_real_path(value)
abs_real_path = property(_get_abs_real_path, _set_abs_real_path)
def _get_treestyle(self):
return self.project.get_treestyle()
file_style = property(_get_treestyle)
def _get_checker(self):
from translate.filters import checks
checkerclasses = [
checks.projectcheckers.get(self.project.checkstyle,
checks.StandardChecker),
checks.StandardUnitChecker
]
excluded_filters = ['hassuggestion', 'spellcheck']
return checks.TeeChecker(checkerclasses=checkerclasses,
excludefilters=excluded_filters,
errorhandler=self.filtererrorhandler,
languagecode=self.language.code)
checker = property(_get_checker)
def _get_non_db_state(self):
if not hasattr(self, "_non_db_state"):
try:
self._non_db_state = self._non_db_state_cache[self.id]
except KeyError:
self._non_db_state = TranslationProjectNonDBState(self)
self._non_db_state_cache[self.id] = \
TranslationProjectNonDBState(self)
return self._non_db_state
non_db_state = property(_get_non_db_state)
def _get_units(self):
self.require_units()
# FIXME: we rely on implicit ordering defined in the model. We might
# want to consider pootle_path as well
return Unit.objects.filter(store__translation_project=self,
state__gt=OBSOLETE).select_related('store')
units = property(_get_units)
@property
def is_terminology_project(self):
return self.pootle_path.endswith('/terminology/')
@property
def is_template_project(self):
return self == self.project.get_template_translationproject()
def _get_indexer(self):
if (self.non_db_state.indexer is None
and self.non_db_state._indexing_enabled):
try:
indexer = self.make_indexer()
if not self.non_db_state._index_initialized:
self.init_index(indexer)
self.non_db_state._index_initialized = True
self.non_db_state.indexer = indexer
except Exception, e:
logging.warning(
u"Could not initialize indexer for %s in %s: "
u"%s", self.project.code, self.language.code, str(e))
self.non_db_state._indexing_enabled = False
return self.non_db_state.indexer
class TranslationProject(models.Model):
description_help_text = _(
'A description of this translation project. '
'This is useful to give more information or '
'instructions. Allowed markup: %s', get_markup_filter_name())
description = MarkupField(blank=True, help_text=description_help_text)
language = models.ForeignKey(Language, db_index=True)
project = models.ForeignKey(Project, db_index=True)
real_path = models.FilePathField(editable=False)
directory = models.OneToOneField(Directory, db_index=True, editable=False)
pootle_path = models.CharField(max_length=255,
null=False,
unique=True,
db_index=True,
editable=False)
tags = TaggableManager(blank=True,
verbose_name=_("Tags"),
help_text=_("A comma-separated list of tags."))
_non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE,
settings.PARSE_POOL_CULL_FREQUENCY)
index_directory = ".translation_index"
objects = TranslationProjectManager()
class Meta:
unique_together = ('language', 'project')
db_table = 'pootle_app_translationproject'
def __unicode__(self):
return self.pootle_path
def save(self, *args, **kwargs):
created = self.id is None
project_dir = self.project.get_real_path()
from pootle_app.project_tree import get_translation_project_dir
self.abs_real_path = get_translation_project_dir(self.language,
project_dir,
self.file_style,
make_dirs=True)
self.directory = self.language.directory \
.get_or_make_subdir(self.project.code)
self.pootle_path = self.directory.pootle_path
super(TranslationProject, self).save(*args, **kwargs)
if created:
self.scan_files()
def delete(self, *args, **kwargs):
directory = self.directory
super(TranslationProject, self).delete(*args, **kwargs)
directory.delete()
deletefromcache(self, [
"getquickstats", "getcompletestats", "get_mtime",
"get_suggestion_count"
])
def get_absolute_url(self):
return l(self.pootle_path)
def get_translate_url(self, **kwargs):
lang, proj, dir, fn = split_pootle_path(self.pootle_path)
return u''.join([
reverse('pootle-tp-translate', args=[lang, proj, dir, fn]),
get_editor_filter(**kwargs),
])
def natural_key(self):
return (self.pootle_path, )
natural_key.dependencies = [
'pootle_app.Directory', 'pootle_language.Language',
'pootle_project.Project'
]
###########################################################################
# Properties #
###########################################################################
fullname = property(lambda self: "%s [%s]" %
(self.project.fullname, self.language.name))
def _get_abs_real_path(self):
return absolute_real_path(self.real_path)
def _set_abs_real_path(self, value):
self.real_path = relative_real_path(value)
abs_real_path = property(_get_abs_real_path, _set_abs_real_path)
def _get_treestyle(self):
return self.project.get_treestyle()
file_style = property(_get_treestyle)
def _get_checker(self):
from translate.filters import checks
checkerclasses = [
checks.projectcheckers.get(self.project.checkstyle,
checks.StandardChecker),
checks.StandardUnitChecker
]
excluded_filters = ['hassuggestion', 'spellcheck']
return checks.TeeChecker(checkerclasses=checkerclasses,
excludefilters=excluded_filters,
errorhandler=self.filtererrorhandler,
languagecode=self.language.code)
checker = property(_get_checker)
def _get_non_db_state(self):
if not hasattr(self, "_non_db_state"):
try:
self._non_db_state = self._non_db_state_cache[self.id]
except KeyError:
self._non_db_state = TranslationProjectNonDBState(self)
self._non_db_state_cache[self.id] = \
TranslationProjectNonDBState(self)
return self._non_db_state
non_db_state = property(_get_non_db_state)
def _get_units(self):
self.require_units()
# FIXME: we rely on implicit ordering defined in the model. We might
# want to consider pootle_path as well
return Unit.objects.filter(store__translation_project=self,
state__gt=OBSOLETE).select_related('store')
units = property(_get_units)
@property
def is_terminology_project(self):
return self.pootle_path.endswith('/terminology/')
@property
def is_template_project(self):
return self == self.project.get_template_translationproject()
def _get_indexer(self):
if (self.non_db_state.indexer is None
and self.non_db_state._indexing_enabled):
try:
indexer = self.make_indexer()
if not self.non_db_state._index_initialized:
self.init_index(indexer)
self.non_db_state._index_initialized = True
self.non_db_state.indexer = indexer
except Exception as e:
logging.warning(
u"Could not initialize indexer for %s in %s: "
u"%s", self.project.code, self.language.code, str(e))
self.non_db_state._indexing_enabled = False
return self.non_db_state.indexer
indexer = property(_get_indexer)
def _has_index(self):
return (self.non_db_state._indexing_enabled
and (self.non_db_state._index_initialized
or self.indexer is not None))
has_index = property(_has_index)
###########################################################################
def filtererrorhandler(self, functionname, str1, str2, e):
logging.error(u"Error in filter %s: %r, %r, %s", functionname, str1,
str2, e)
return False
def update(self):
"""Update all stores to reflect state on disk."""
stores = self.stores.exclude(file='').filter(state__gte=PARSED)
for store in stores.iterator():
store.update(update_translation=True, update_structure=True)
def sync(self, conservative=True, skip_missing=False, modified_since=0):
"""Sync unsaved work on all stores to disk."""
stores = self.stores.exclude(file='').filter(state__gte=PARSED)
for store in stores.iterator():
store.sync(update_translation=True,
update_structure=not conservative,
conservative=conservative,
create=False,
skip_missing=skip_missing,
modified_since=modified_since)
def get_latest_submission(self):
"""Get the latest submission done in the Translation project."""
try:
sub = Submission.objects.filter(translation_project=self).latest()
except Submission.DoesNotExist:
return ''
return sub.get_submission_message()
@getfromcache
def get_mtime(self):
tp_units = Unit.objects.filter(store__translation_project=self)
return max_column(tp_units, 'mtime', None)
def require_units(self):
"""Makes sure all stores are parsed"""
errors = 0
for store in self.stores.filter(state__lt=PARSED).iterator():
try:
store.require_units()
except IntegrityError:
logging.info(u"Duplicate IDs in %s", store.abs_real_path)
errors += 1
except ParseError as e:
logging.info(u"Failed to parse %s\n%s", store.abs_real_path, e)
errors += 1
except (IOError, OSError) as e:
logging.info(u"Can't access %s\n%s", store.abs_real_path, e)
errors += 1
return errors
@getfromcache
def getquickstats(self):
if self.is_template_project:
return empty_quickstats
errors = self.require_units()
tp_not_obsolete_units = Unit.objects.filter(
store__translation_project=self,
state__gt=OBSOLETE,
)
stats = calculate_stats(tp_not_obsolete_units)
stats['errors'] = errors
return stats
@getfromcache
def getcompletestats(self):
if self.is_template_project:
return empty_completestats
for store in self.stores.filter(state__lt=CHECKED).iterator():
store.require_qualitychecks()
query = QualityCheck.objects.filter(
unit__store__translation_project=self,
unit__state__gt=UNTRANSLATED,
false_positive=False)
return group_by_count_extra(query, 'name', 'category')
@getfromcache
def get_suggestion_count(self):
"""
Check if any unit in the stores for this translation project has
suggestions.
"""
return Suggestion.objects.filter(unit__store__translation_project=self,
unit__state__gt=OBSOLETE).count()
def update_against_templates(self, pootle_path=None):
"""Update translation project from templates."""
if self.is_template_project:
return
template_translation_project = self.project \
.get_template_translationproject()
if (template_translation_project is None
or template_translation_project == self):
return
monolingual = self.project.is_monolingual()
if not monolingual:
self.sync()
if pootle_path is None:
oldstats = self.getquickstats()
from pootle_app.project_tree import (convert_template,
get_translated_name,
get_translated_name_gnu)
for store in template_translation_project.stores.iterator():
if self.file_style == 'gnu':
new_pootle_path, new_path = get_translated_name_gnu(
self, store)
else:
new_pootle_path, new_path = get_translated_name(self, store)
if pootle_path is not None and new_pootle_path != pootle_path:
continue
relative_po_path = os.path.relpath(new_path, settings.PODIRECTORY)
try:
from pootle.scripts import hooks
if not hooks.hook(self.project.code, "pretemplateupdate",
relative_po_path):
continue
except:
# Assume hook is not present.
pass
convert_template(self, store, new_pootle_path, new_path,
monolingual)
all_files, new_files = self.scan_files(vcs_sync=False)
from pootle_misc import versioncontrol
project_path = self.project.get_real_path()
if new_files and versioncontrol.hasversioning(project_path):
from pootle.scripts import hooks
message = "New files added from %s based on templates" % \
(settings.TITLE)
filestocommit = []
for new_file in new_files:
try:
filestocommit.extend(
hooks.hook(self.project.code,
"precommit",
new_file.file.name,
author=None,
message=message))
except ImportError:
# Failed to import the hook - we're going to assume there
# just isn't a hook to import. That means we'll commit the
# original file.
filestocommit.append(new_file.file.name)
success = True
try:
output = versioncontrol.add_files(project_path, filestocommit,
message)
except Exception as e:
logging.error(u"Failed to add files: %s", e)
success = False
for new_file in new_files:
try:
hooks.hook(self.project.code,
"postcommit",
new_file.file.name,
success=success)
except:
#FIXME: We should not hide the exception - makes
# development impossible
pass
if pootle_path is None:
newstats = self.getquickstats()
from pootle_app.models.signals import post_template_update
post_template_update.send(sender=self,
oldstats=oldstats,
newstats=newstats)
def scan_files(self, vcs_sync=True):
"""Scans the file system and returns a list of translation files.
:param vcs_sync: boolean on whether or not to synchronise the PO
directory with the VCS checkout.
"""
projects = [p.strip() for p in self.project.ignoredfiles.split(',')]
ignored_files = set(projects)
ext = os.extsep + self.project.localfiletype
# Scan for pots if template project
if self.is_template_project:
ext = os.extsep + self.project.get_template_filetype()
from pootle_app.project_tree import (add_files,
match_template_filename,
direct_language_match_filename,
sync_from_vcs)
all_files = []
new_files = []
if self.file_style == 'gnu':
if self.pootle_path.startswith('/templates/'):
file_filter = lambda filename: match_template_filename(
self.project,
filename,
)
else:
file_filter = lambda filename: direct_language_match_filename(
self.language.code,
filename,
)
else:
file_filter = lambda filename: True
if vcs_sync:
sync_from_vcs(ignored_files, ext, self.real_path, file_filter)
all_files, new_files = add_files(
self,
ignored_files,
ext,
self.real_path,
self.directory,
file_filter,
)
return all_files, new_files
def update_file_from_version_control(self, store):
from pootle.scripts import hooks
store.sync(update_translation=True)
filetoupdate = store.file.name
try:
filetoupdate = hooks.hook(self.project.code, "preupdate",
store.file.name)
except:
pass
# Keep a copy of working files in memory before updating
oldstats = store.getquickstats()
working_copy = store.file.store
try:
logging.debug(u"Updating %s from version control", store.file.name)
from pootle_misc import versioncontrol
versioncontrol.update_file(filetoupdate)
store.file._delete_store_cache()
store.file._update_store_cache()
except Exception as e:
# Something wrong, file potentially modified, bail out
# and replace with working copy
logging.error(
u"Near fatal catastrophe, exception %s while "
u"updating %s from version control", e, store.file.name)
working_copy.save()
raise VersionControlError
try:
hooks.hook(self.project.code, "postupdate", store.file.name)
except:
pass
try:
logging.debug(u"Parsing version control copy of %s into db",
store.file.name)
store.update(update_structure=True, update_translation=True)
remotestats = store.getquickstats()
#FIXME: try to avoid merging if file was not updated
logging.debug(u"Merging %s with version control update",
store.file.name)
store.mergefile(working_copy,
None,
allownewstrings=False,
suggestions=True,
notranslate=False,
obsoletemissing=False)
except Exception as e:
logging.error(
u"Near fatal catastrophe, exception %s while merging "
u"%s with version control copy", e, store.file.name)
working_copy.save()
store.update(update_structure=True, update_translation=True)
raise
newstats = store.getquickstats()
return oldstats, remotestats, newstats
def update_dir(self, request=None, directory=None):
"""Updates translation project's files from version control, retaining
uncommitted translations.
"""
old_stats = self.getquickstats()
remote_stats = {}
from pootle_misc import versioncontrol
try:
versioncontrol.update_dir(self.real_path)
except IOError as e:
logging.error(u"Error during update of %(path)s:\n%(error)s", {
"path": self.real_path,
"error": e,
})
if request:
msg = _("Failed to update from version control: %(error)s",
{"error": e})
messages.error(request, msg)
return
all_files, new_files = self.scan_files()
new_file_set = set(new_files)
from pootle.scripts import hooks
# Go through all stores except any pootle-terminology.* ones
if directory.is_translationproject():
stores = self.stores.exclude(file="")
else:
stores = directory.stores.exclude(file="")
for store in stores.iterator():
if store in new_file_set:
# these won't have to be merged, since they are new
remotestats = store.getquickstats()
remote_stats = dictsum(remote_stats, remotestats)
continue
store.sync(update_translation=True)
filetoupdate = store.file.name
try:
filetoupdate = hooks.hook(self.project.code, "preupdate",
store.file.name)
except:
pass
# keep a copy of working files in memory before updating
working_copy = store.file.store
versioncontrol.copy_to_podir(filetoupdate)
store.file._delete_store_cache()
store.file._update_store_cache()
try:
hooks.hook(self.project.code, "postupdate", store.file.name)
except:
pass
try:
logging.debug(u"Parsing version control copy of %s into db",
store.file.name)
store.update(update_structure=True, update_translation=True)
remotestats = store.getquickstats()
#FIXME: Try to avoid merging if file was not updated
logging.debug(u"Merging %s with version control update",
store.file.name)
store.mergefile(working_copy,
None,
allownewstrings=False,
suggestions=True,
notranslate=False,
obsoletemissing=False)
except Exception as e:
logging.error(
u"Near fatal catastrophe, exception %s while "
"merging %s with version control copy", e, store.file.name)
working_copy.save()
store.update(update_structure=True, update_translation=True)
raise
remote_stats = dictsum(remote_stats, remotestats)
new_stats = self.getquickstats()
if request:
msg = [
_(u'Updated project <em>%(project)s</em> from version control',
{'project': self.fullname}),
stats_message(_(u"Working copy"), old_stats),
stats_message(_(u"Remote copy"), remote_stats),
stats_message(_(u"Merged copy"), new_stats)
]
msg = u"<br/>".join([force_unicode(m) for m in msg])
messages.info(request, msg)
from pootle_app.models.signals import post_vc_update
post_vc_update.send(sender=self,
oldstats=old_stats,
remotestats=remote_stats,
newstats=new_stats)
def update_file(self, request, store):
"""Updates file from version control, retaining uncommitted
translations"""
try:
old_stats, remote_stats, new_stats = \
self.update_file_from_version_control(store)
# FIXME: This belongs to views
msg = [
_(u'Updated file <em>%(filename)s</em> from version control',
{'filename': store.file.name}),
stats_message(_(u"Working copy"), old_stats),
stats_message(_(u"Remote copy"), remote_stats),
stats_message(_(u"Merged copy"), new_stats)
]
msg = u"<br/>".join([force_unicode(m) for m in msg])
messages.info(request, msg)
from pootle_app.models.signals import post_vc_update
post_vc_update.send(sender=self,
oldstats=old_stats,
remotestats=remote_stats,
newstats=new_stats)
except VersionControlError as e:
# FIXME: This belongs to views
msg = _(
u"Failed to update <em>%(filename)s</em> from "
u"version control: %(error)s", {
'filename': store.file.name,
'error': e,
})
messages.error(request, msg)
self.scan_files()
def commit_dir(self, user, directory, request=None):
"""Commits files under a directory to version control.
This does not do permission checking.
"""
self.sync()
stats = self.getquickstats()
author = user.username
message = stats_message_raw(
"Commit from %s by user %s." % (settings.TITLE, author), stats)
# Try to append email as well, since some VCS does not allow omitting
# it (ie. Git).
if user.is_authenticated() and len(user.email):
author += " <%s>" % user.email
if directory.is_translationproject():
stores = list(self.stores.exclude(file=""))
else:
stores = list(directory.stores.exclude(file=""))
filestocommit = []
from pootle.scripts import hooks
for store in stores:
try:
filestocommit.extend(
hooks.hook(self.project.code,
"precommit",
store.file.name,
author=author,
message=message))
except ImportError:
# Failed to import the hook - we're going to assume there just
# isn't a hook to import. That means we'll commit the original
# file.
filestocommit.append(store.file.name)
success = True
try:
from pootle_misc import versioncontrol
project_path = self.project.get_real_path()
versioncontrol.add_files(project_path, filestocommit, message,
author)
# FIXME: This belongs to views
if request is not None:
msg = _(
"Committed all files under <em>%(path)s</em> to "
"version control", {'path': directory.pootle_path})
messages.success(request, msg)
except Exception as e:
logging.error(u"Failed to commit: %s", e)
# FIXME: This belongs to views
if request is not None:
msg = _("Failed to commit to version control: %(error)s",
{'error': e})
messages.error(request, msg)
success = False
for store in stores:
try:
hooks.hook(self.project.code,
"postcommit",
store.file.name,
success=success)
except:
#FIXME: We should not hide the exception - makes development
# impossible
pass
from pootle_app.models.signals import post_vc_commit
post_vc_commit.send(sender=self,
path_obj=directory,
stats=stats,
user=user,
success=success)
return success
def commit_file(self, user, store, request=None):
"""Commits an individual file to version control.
This does not do permission checking.
"""
store.sync(update_structure=False,
update_translation=True,
conservative=True)
stats = store.getquickstats()
author = user.username
message = stats_message_raw("Commit from %s by user %s." % \
(settings.TITLE, author), stats)
# Try to append email as well, since some VCS does not allow omitting
# it (ie. Git).
if user.is_authenticated() and len(user.email):
author += " <%s>" % user.email
from pootle.scripts import hooks
try:
filestocommit = hooks.hook(self.project.code,
"precommit",
store.file.name,
author=author,
message=message)
except ImportError:
# Failed to import the hook - we're going to assume there just
# isn't a hook to import. That means we'll commit the original
# file.
filestocommit = [store.file.name]
success = True
try:
from pootle_misc import versioncontrol
for file in filestocommit:
versioncontrol.commit_file(file,
message=message,
author=author)
# FIXME: This belongs to views
if request is not None:
msg = _(
"Committed file <em>%(filename)s</em> to version "
"control", {'filename': file})
messages.success(request, msg)
except Exception as e:
logging.error(u"Failed to commit file: %s", e)
# FIXME: This belongs to views
if request is not None:
msg_params = {
'filename': filename,
'error': e,
}
msg = _(
"Failed to commit <em>%(filename)s</em> to version "
"control: %(error)s", msg_params)
messages.error(request, msg)
success = False
try:
hooks.hook(self.project.code,
"postcommit",
store.file.name,
success=success)
except:
#FIXME: We should not hide the exception - makes development
# impossible
pass
from pootle_app.models.signals import post_vc_commit
post_vc_commit.send(sender=self,
path_obj=store,
stats=stats,
user=user,
success=success)
return success
def initialize(self):
try:
from pootle.scripts import hooks
hooks.hook(self.project.code, "initialize", self.real_path,
self.language.code)
except Exception as e:
logging.error(u"Failed to initialize (%s): %s", self.language.code,
e)
###########################################################################
def get_archive(self, stores, path=None):
"""Returns an archive of the given files."""
import shutil
from pootle_misc import ptempfile as tempfile
tempzipfile = None
try:
# Using zip command line is fast
# The temporary file below is opened and immediately closed for
# security reasons
fd, tempzipfile = tempfile.mkstemp(prefix='pootle', suffix='.zip')
os.close(fd)
file_list = u" ".join(
store.abs_real_path[len(self.abs_real_path)+1:] \
for store in stores.iterator()
)
cmd = u"cd %(path)s ; zip -r - %(file_list)s > %(tmpfile)s" % {
'path': self.abs_real_path,
'file_list': file_list,
'tmpfile': tempzipfile,
}
result = os.system(cmd.encode('utf-8'))
if result == 0:
if path is not None:
shutil.move(tempzipfile, path)
return
else:
filedata = open(tempzipfile, "r").read()
if filedata:
return filedata
finally:
if tempzipfile is not None and os.path.exists(tempzipfile):
os.remove(tempzipfile)
# But if it doesn't work, we can do it from python
archivecontents = None
try:
if path is not None:
fd, tempzipfile = tempfile.mkstemp(prefix='pootle',
suffix='.zip')
os.close(fd)
archivecontents = open(tempzipfile, "wb")
else:
import cStringIO
archivecontents = cStringIO.StringIO()
import zipfile
archive = zipfile.ZipFile(archivecontents, 'w',
zipfile.ZIP_DEFLATED)
for store in stores.iterator():
archive.write(
store.abs_real_path.encode('utf-8'),
store.abs_real_path[len(self.abs_real_path) +
1:].encode('utf-8'))
archive.close()
if path is not None:
shutil.move(tempzipfile, path)
else:
return archivecontents.getvalue()
finally:
if tempzipfile is not None and os.path.exists(tempzipfile):
os.remove(tempzipfile)
try:
archivecontents.close()
except:
pass
###########################################################################
def make_indexer(self):
"""Get an indexing object for this project.
Since we do not want to keep the indexing databases open for the
lifetime of the TranslationProject (it is cached!), it may NOT be
part of the Project object, but should be used via a short living
local variable.
"""
logging.debug(u"Loading indexer for %s", self.pootle_path)
indexdir = os.path.join(self.abs_real_path, self.index_directory)
from translate.search import indexing
indexer = indexing.get_indexer(indexdir)
indexer.set_field_analyzers({
"pofilename": indexer.ANALYZER_EXACT,
"pomtime": indexer.ANALYZER_EXACT,
"dbid": indexer.ANALYZER_EXACT,
})
return indexer
def init_index(self, indexer):
"""Initializes the search index."""
#FIXME: stop relying on pomtime so virtual files can be searchable?
try:
indexer.begin_transaction()
for store in self.stores.iterator():
try:
self.update_index(indexer, store)
except OSError as e:
# Broken link or permission problem?
logging.error("Error indexing %s: %s", store, e)
indexer.commit_transaction()
indexer.flush(optimize=True)
except Exception as e:
logging.error(u"Error opening indexer for %s:\n%s", self, e)
try:
indexer.cancel_transaction()
except:
pass
def update_index(self, indexer, store, unitid=None):
"""Updates the index with the contents of store (limit to
``unitid`` if given).
There are two reasons for calling this function:
1. Creating a new instance of :cls:`TranslationProject`
(see :meth:`TranslationProject.init_index`)
-> Check if the index is up-to-date / rebuild the index if
necessary
2. Translating a unit via the web interface
-> (re)index only the specified unit(s)
The argument ``unitid`` should be None for 1.
Known problems:
1. This function should get called, when the po file changes
externally.
WARNING: You have to stop the pootle server before manually
changing po files, if you want to keep the index database in
sync.
"""
#FIXME: leverage file updated signal to check if index needs updating
if indexer is None:
return False
# Check if the pomtime in the index == the latest pomtime
pomtime = str(hash(store.get_mtime())**2)
pofilenamequery = indexer.make_query(
[("pofilename", store.pootle_path)], True)
pomtimequery = indexer.make_query([("pomtime", pomtime)], True)
gooditemsquery = indexer.make_query([pofilenamequery, pomtimequery],
True)
gooditemsnum = indexer.get_query_result(gooditemsquery) \
.get_matches_count()
# If there is at least one up-to-date indexing item, then the po file
# was not changed externally -> no need to update the database
units = None
if (gooditemsnum > 0) and (not unitid):
# Nothing to be done
return
elif unitid is not None:
# Update only specific item - usually translation via the web
# interface. All other items should still be up-to-date (even with
# an older pomtime).
# Delete the relevant item from the database
units = store.units.filter(id=unitid)
itemsquery = indexer.make_query([("dbid", str(unitid))], False)
indexer.delete_doc([pofilenamequery, itemsquery])
else:
# (item is None)
# The po file is not indexed - or it was changed externally
# delete all items of this file
logging.debug(u"Updating %s indexer for file %s", self.pootle_path,
store.pootle_path)
indexer.delete_doc({"pofilename": store.pootle_path})
units = store.units
addlist = []
for unit in units.iterator():
doc = {
"pofilename": store.pootle_path,
"pomtime": pomtime,
"dbid": str(unit.id),
}
if unit.hasplural():
orig = "\n".join(unit.source.strings)
trans = "\n".join(unit.target.strings)
else:
orig = unit.source
trans = unit.target
doc.update({
"source": orig,
"target": trans,
"notes": unit.getnotes(),
"locations": unit.getlocations(),
})
addlist.append(doc)
if addlist:
for add_item in addlist:
indexer.index_document(add_item)
###########################################################################
def gettermmatcher(self):
"""Returns the terminology matcher."""
terminology_stores = Store.objects.none()
mtime = None
if self.is_terminology_project:
terminology_stores = self.stores.all()
mtime = self.get_mtime()
else:
# Get global terminology first
try:
termproject = TranslationProject.objects.get(
language=self.language_id,
project__code='terminology',
)
mtime = termproject.get_mtime()
terminology_stores = termproject.stores.all()
except TranslationProject.DoesNotExist:
pass
local_terminology = self.stores.filter(
name__startswith='pootle-terminology')
for store in local_terminology.iterator():
if mtime is None:
mtime = store.get_mtime()
else:
mtime = max(mtime, store.get_mtime())
terminology_stores = terminology_stores | local_terminology
if mtime is None:
return
if mtime != self.non_db_state.termmatchermtime:
from translate.search import match
self.non_db_state.termmatcher = match.terminologymatcher(
terminology_stores.iterator(), )
self.non_db_state.termmatchermtime = mtime
return self.non_db_state.termmatcher
###########################################################################
#FIXME: we should cache results to ease live translation
def translate_message(self, singular, plural=None, n=1):
for store in self.stores.iterator():
unit = store.findunit(singular)
if unit is not None and unit.istranslated():
if unit.hasplural() and n != 1:
pluralequation = self.language.pluralequation
if pluralequation:
pluralfn = gettext.c2py(pluralequation)
target = unit.target.strings[pluralfn(n)]
if target is not None:
return target
else:
return unit.target
# No translation found
if n != 1 and plural is not None:
return plural
else:
return singular
class Project(models.Model):
objects = ProjectManager()
class Meta:
ordering = ['code']
db_table = 'pootle_app_project'
code_help_text = _('A short code for the project. This should only contain '
'ASCII characters, numbers, and the underscore (_) character.')
code = models.CharField(max_length=255, null=False, unique=True,
db_index=True, verbose_name=_('Code'), help_text=code_help_text)
fullname = models.CharField(max_length=255, null=False,
verbose_name=_("Full Name"))
description_help_text = _('A description of this project. '
'This is useful to give more information or instructions. '
'Allowed markup: %s', get_markup_filter_name())
description = MarkupField(blank=True, help_text=description_help_text)
checker_choices = [('standard', 'standard')]
checkers = list(checks.projectcheckers.keys())
checkers.sort()
checker_choices.extend([(checker, checker) for checker in checkers])
checkstyle = models.CharField(max_length=50, default='standard',
null=False, choices=checker_choices,
verbose_name=_('Quality Checks'))
localfiletype = models.CharField(max_length=50, default="po",
choices=filetype_choices, verbose_name=_('File Type'))
treestyle_choices = (
# TODO: check that the None is stored and handled correctly
('auto', _('Automatic detection (slower)')),
('gnu', _('GNU style: files named by language code')),
('nongnu', _('Non-GNU: Each language in its own directory')),
)
treestyle = models.CharField(max_length=20, default='auto',
choices=treestyle_choices, verbose_name=_('Project Tree Style'))
source_language = models.ForeignKey('pootle_language.Language',
db_index=True, verbose_name=_('Source Language'))
ignoredfiles = models.CharField(max_length=255, blank=True, null=False,
default="", verbose_name=_('Ignore Files'))
directory = models.OneToOneField('pootle_app.Directory', db_index=True,
editable=False)
report_target_help_text = _('A URL or an email address where issues '
'with the source text can be reported.')
report_target = models.CharField(max_length=512, blank=True,
verbose_name=_("Report Target"), help_text=report_target_help_text)
def natural_key(self):
return (self.code,)
natural_key.dependencies = ['pootle_app.Directory']
def __unicode__(self):
return self.fullname
def save(self, *args, **kwargs):
# Create file system directory if needed
project_path = self.get_real_path()
if not os.path.exists(project_path):
os.makedirs(project_path)
from pootle_app.models.directory import Directory
self.directory = Directory.objects.projects \
.get_or_make_subdir(self.code)
super(Project, self).save(*args, **kwargs)
# FIXME: far from ideal, should cache at the manager level instead
cache.delete(CACHE_KEY)
cache.set(CACHE_KEY, Project.objects.all(), 0)
def delete(self, *args, **kwargs):
directory = self.directory
# Just doing a plain delete will collect all related objects in memory
# before deleting: translation projects, stores, units, quality checks,
# pootle_store suggestions, pootle_app suggestions and submissions.
# This can easily take down a process. If we do a translation project
# at a time and force garbage collection, things stay much more
# managable.
import gc
gc.collect()
for tp in self.translationproject_set.iterator():
tp.delete()
gc.collect()
# Here is a different version that first deletes all the related
# objects, starting from the leaves. This will have to be maintained
# doesn't seem to provide a real advantage in terms of performance.
# Doing this finer grained garbage collection keeps memory usage even
# lower but can take a bit longer.
'''
from pootle_statistics.models import Submission
from pootle_app.models import Suggestion as AppSuggestion
from pootle_store.models import Suggestion as StoreSuggestion
from pootle_store.models import QualityCheck
Submission.objects.filter(from_suggestion__translation_project__project=self).delete()
AppSuggestion.objects.filter(translation_project__project=self).delete()
StoreSuggestion.objects.filter(unit__store__translation_project__project=self).delete()
QualityCheck.objects.filter(unit__store__translation_project__project=self).delete()
gc.collect()
for tp in self.translationproject_set.iterator():
Unit.objects.filter(store__translation_project=tp).delete()
gc.collect()
'''
super(Project, self).delete(*args, **kwargs)
directory.delete()
# FIXME: far from ideal, should cache at the manager level instead
cache.delete(CACHE_KEY)
@getfromcache
def get_mtime(self):
project_units = Unit.objects.filter(
store__translation_project__project=self
)
return max_column(project_units, 'mtime', None)
@getfromcache
def getquickstats(self):
return statssum(self.translationproject_set.iterator())
def translated_percentage(self):
qs = self.getquickstats()
max_words = max(qs['totalsourcewords'], 1)
return int(100.0 * qs['translatedsourcewords'] / max_words)
def _get_pootle_path(self):
return "/projects/" + self.code + "/"
pootle_path = property(_get_pootle_path)
def get_real_path(self):
return absolute_real_path(self.code)
def get_absolute_url(self):
return l(self.pootle_path)
@cached_property
def languages(self):
"""Returns a list of active :cls:`~pootle_languages.models.Language`
objects for this :cls:`~pootle_project.models.Project`.
"""
from pootle_language.models import Language
# FIXME: we should better have a way to automatically cache models with
# built-in invalidation -- did I hear django-cache-machine?
return Language.objects.filter(Q(translationproject__project=self),
~Q(code='templates'))
def get_template_filetype(self):
if self.localfiletype == 'po':
return 'pot'
else:
return self.localfiletype
def get_file_class(self):
"""Returns the TranslationStore subclass required for parsing
project files."""
return factory_classes[self.localfiletype]
def is_monolingual(self):
"""Returns ``True`` if this project is monolingual."""
return is_monolingual(self.get_file_class())
def _get_is_terminology(self):
"""Returns ``True`` if this project is a terminology project."""
return self.checkstyle == 'terminology'
is_terminology = property(_get_is_terminology)
def file_belongs_to_project(self, filename, match_templates=True):
"""Tests if ``filename`` matches project filetype (ie. extension).
If ``match_templates`` is ``True``, this will also check if the
file matches the template filetype.
"""
template_ext = os.path.extsep + self.get_template_filetype()
return (filename.endswith(os.path.extsep + self.localfiletype)
or match_templates and filename.endswith(template_ext))
def _detect_treestyle(self):
try:
dirlisting = os.walk(self.get_real_path())
dirpath, dirnames, filenames = dirlisting.next()
if not dirnames:
# No subdirectories
if filter(self.file_belongs_to_project, filenames):
# Translation files found, assume gnu
return "gnu"
else:
# There are subdirectories
if filter(lambda dirname: dirname == 'templates' or langcode_re.match(dirname), dirnames):
# Found language dirs assume nongnu
return "nongnu"
else:
# No language subdirs found, look for any translation file
for dirpath, dirnames, filenames in os.walk(self.get_real_path()):
if filter(self.file_belongs_to_project, filenames):
return "gnu"
except:
pass
# Unsure
return None
def get_treestyle(self):
"""Returns the real treestyle, if :attr:`Project.treestyle` is set
to ``auto`` it checks the project directory and tries to guess
if it is gnu style or nongnu style.
We are biased towards nongnu because it makes managing projects
from the web easier.
"""
if self.treestyle != "auto":
return self.treestyle
else:
detected = self._detect_treestyle()
if detected is not None:
return detected
# When unsure return nongnu
return "nongnu"
def get_template_translationproject(self):
"""Returns the translation project that will be used as a template
for this project.
First it tries to retrieve the translation project that has the
special 'templates' language within this project, otherwise it
falls back to the source language set for current project.
"""
try:
return self.translationproject_set.get(language__code='templates')
except ObjectDoesNotExist:
try:
return self.translationproject_set \
.get(language=self.source_language_id)
except ObjectDoesNotExist:
pass
class VirtualFolder(models.Model):
name = models.CharField(_('Name'), blank=False, max_length=70)
location = models.CharField(
_('Location'),
blank=False,
max_length=255,
help_text=_('Root path where this virtual folder is applied.'),
)
filter_rules = models.TextField(
# Translators: This is a noun.
_('Filter'),
blank=False,
help_text=_('Filtering rules that tell which stores this virtual '
'folder comprises.'),
)
priority = models.FloatField(
_('Priority'),
default=1,
help_text=_('Number specifying importance. Greater priority means it '
'is more important.'),
)
is_public = models.BooleanField(
_('Is public?'),
default=True,
help_text=_('Whether this virtual folder is public or not.'),
)
description = MarkupField(
_('Description'),
blank=True,
help_text=_(
'Use this to provide more information or instructions. '
'Allowed markup: %s', get_markup_filter_name()),
)
units = models.ManyToManyField(
Unit,
db_index=True,
related_name='vfolders',
)
class Meta:
unique_together = ('name', 'location')
@property
def tp_relative_path(self):
"""Return the virtual folder path relative to any translation project.
This is the virtual folder location stripping out the language and
project parts and appending the virtual folder name as if it were a
folder.
For example a location /af/{PROJ}/browser/ for a virtual folder default
is returned as browser/default/
"""
return '/'.join(
self.location.strip('/').split('/')[2:] + [self.name, ''])
@property
def all_locations(self):
"""Return a list with all the locations this virtual folder applies.
If the virtual folder location has no {LANG} nor {PROJ} placeholders
then the list only contains its location. If any of the placeholders is
present, then they get expanded to match all the existing languages and
projects.
"""
if "{LANG}/{PROJ}" in self.location:
locations = []
for lang in Language.objects.all():
temp = self.location.replace("{LANG}", lang.code)
for proj in Project.objects.all():
locations.append(temp.replace("{PROJ}", proj.code))
return locations
elif "{LANG}" in self.location:
try:
project = Project.objects.get(code=self.location.split("/")[2])
languages = project.languages.iterator()
except Exception:
languages = Language.objects.iterator()
return [
self.location.replace("{LANG}", lang.code)
for lang in languages
]
elif "{PROJ}" in self.location:
try:
projects = Project.objects.filter(
translationproject__language__code=self.location.split(
"/")[1]).iterator()
except Exception:
projects = Project.objects.iterator()
return [
self.location.replace("{PROJ}", proj.code) for proj in projects
]
return [self.location]
def __unicode__(self):
return ": ".join([self.name, self.location])
def save(self, *args, **kwargs):
# Force validation of fields.
self.clean_fields()
self.name = self.name.lower()
if self.pk is None:
projects = set()
else:
# If this is an already existing vfolder, keep a list of the
# projects it was related to.
projects = set(
Project.objects.filter(
translationproject__stores__unit__vfolders=self).distinct(
).values_list('code', flat=True))
super(VirtualFolder, self).save(*args, **kwargs)
# Clean any existing relationship between units and this vfolder.
self.units.clear()
# Recreate relationships between this vfolder and units.
vfolder_stores_set = set()
for location in self.all_locations:
for filename in self.filter_rules.split(","):
vf_file = "".join([location, filename])
qs = Store.objects.live().filter(pootle_path=vf_file)
if qs.exists():
self.units.add(*qs[0].units.all())
vfolder_stores_set.add(qs[0])
else:
if not vf_file.endswith("/"):
vf_file += "/"
if Directory.objects.filter(pootle_path=vf_file).exists():
qs = Unit.objects.filter(
state__gt=OBSOLETE,
store__pootle_path__startswith=vf_file)
self.units.add(*qs)
vfolder_stores_set.update(
Store.objects.filter(
pootle_path__startswith=vf_file))
# For each store create all VirtualFolderTreeItem tree structure up to
# its adjusted vfolder location.
for store in vfolder_stores_set:
try:
VirtualFolderTreeItem.objects.get_or_create(
directory=store.parent,
vfolder=self,
)
except ValidationError:
# If there is some problem, e.g. a clash with a directory,
# delete the virtual folder and all its related items, and
# reraise the exception.
self.delete()
raise
# Get the set of projects whose resources cache must be invalidated.
# This includes the projects the projects it was previously related to
# for the already existing vfolders.
projects.update(
Project.objects.filter(
translationproject__stores__unit__vfolders=self).distinct().
values_list('code', flat=True))
# Send the signal. This is used to invalidate the cached resources for
# all the related projects.
vfolder_post_save.send(sender=self.__class__,
instance=self,
projects=list(projects))
def clean_fields(self):
"""Validate virtual folder fields."""
if not self.priority > 0:
raise ValidationError(u'Priority must be greater than zero.')
elif self.location == "/":
raise ValidationError(u'The "/" location is not allowed. Use '
u'"/{LANG}/{PROJ}/" instead.')
elif self.location.startswith("/projects/"):
raise ValidationError(u'Locations starting with "/projects/" are '
u'not allowed. Use "/{LANG}/" instead.')
if not self.filter_rules:
raise ValidationError(u'Some filtering rule must be specified.')
def get_adjusted_location(self, pootle_path):
"""Return the virtual folder location adjusted to the given path.
The virtual folder location might have placeholders, which affect the
actual filenames since those have to be concatenated to the virtual
folder location.
"""
count = self.location.count("/")
if pootle_path.count("/") < count:
raise ValueError("%s is not applicable in %s" %
(self, pootle_path))
pootle_path_parts = pootle_path.strip("/").split("/")
location_parts = self.location.strip("/").split("/")
try:
if (location_parts[0] != pootle_path_parts[0]
and location_parts[0] != "{LANG}"):
raise ValueError("%s is not applicable in %s" %
(self, pootle_path))
if (location_parts[1] != pootle_path_parts[1]
and location_parts[1] != "{PROJ}"):
raise ValueError("%s is not applicable in %s" %
(self, pootle_path))
except IndexError:
pass
return "/".join(pootle_path.split("/")[:count])
def get_adjusted_pootle_path(self, pootle_path):
"""Adjust the given pootle path to this virtual folder.
The provided pootle path is converted to a path that includes the
virtual folder name in the right place.
For example a virtual folder named vfolder8, with a location
/{LANG}/firefox/browser/ in a path
/af/firefox/browser/chrome/overrides/ gets converted to
/af/firefox/browser/vfolder8/chrome/overrides/
"""
count = self.location.count('/')
if pootle_path.count('/') < count:
# The provided pootle path is above the virtual folder location.
path_parts = pootle_path.rstrip('/').split('/')
pootle_path = '/'.join(path_parts +
self.location.split('/')[len(path_parts):])
if count < 3:
# If the virtual folder location is not long as a translation
# project pootle path then the returned adjusted location is too
# short, meaning that the returned translate URL will have the
# virtual folder name as the project or language code.
path_parts = pootle_path.split('/')
return '/'.join(path_parts[:3] + [self.name] + path_parts[3:])
# If the virtual folder location is as long as a TP pootle path and
# the provided pootle path isn't above the virtual folder location.
lead = self.get_adjusted_location(pootle_path)
trail = pootle_path.replace(lead, '').lstrip('/')
return '/'.join([lead, self.name, trail])
