class TranslationStoreFieldFile(FieldFile):
"""FieldFile is the file-like object of a FileField, that is found in a
TranslationStoreField.
"""
from translate.misc.lru import LRUCachingDict
_store_cache = LRUCachingDict(PARSE_POOL_SIZE, PARSE_POOL_CULL_FREQUENCY)
def getpomtime(self):
file_stat = os.stat(self.realpath)
return file_stat.st_mtime, file_stat.st_size
@property
def filename(self):
return os.path.basename(self.name)
def _get_realpath(self):
"""Return realpath resolving symlinks if necessary."""
if not hasattr(self, "_realpath"):
# Django's db.models.fields.files.FieldFile raises ValueError if
# if the file field has no name - and tests "if self" to check
if self:
self._realpath = os.path.realpath(self.path)
else:
self._realpath = ""
return self._realpath
@property
def realpath(self):
"""Get real path from cache before attempting to check for symlinks."""
if not hasattr(self, "_store_tuple"):
return self._get_realpath()
else:
return self._store_tuple.realpath
@property
def store(self):
"""Get translation store from dictionary cache, populate if store not
already cached.
"""
self._update_store_cache()
return self._store_tuple.store
def _update_store_cache(self):
"""Add translation store to dictionary cache, replace old cached
version if needed.
"""
if self.exists():
mod_info = self.getpomtime()
else:
mod_info = 0
if not hasattr(
self,
"_store_tuple") or self._store_tuple.mod_info != mod_info:
try:
self._store_tuple = self._store_cache[self.path]
if self._store_tuple.mod_info != mod_info:
# if file is modified act as if it doesn't exist in cache
raise KeyError
except KeyError:
logging.debug(u"Cache miss for %s", self.path)
from translate.storage import factory
syncer = self.instance.syncer
classes = {
syncer.extension: syncer.file_class,
}
store_obj = factory.getobject(self.path,
ignore=self.field.ignore,
classes=classes)
self._store_tuple = StoreTuple(store_obj, mod_info,
self.realpath)
self._store_cache[self.path] = self._store_tuple
def _touch_store_cache(self):
"""Update stored mod_info without reparsing file."""
if hasattr(self, "_store_tuple"):
mod_info = self.getpomtime()
if self._store_tuple.mod_info != mod_info:
self._store_tuple.mod_info = mod_info
else:
# FIXME: do we really need that?
self._update_store_cache()
def _delete_store_cache(self):
"""Remove translation store from cache."""
try:
del self._store_cache[self.path]
except KeyError:
pass
try:
del self._store_tuple
except AttributeError:
pass
def exists(self):
return os.path.exists(self.realpath)
def savestore(self):
"""Saves to temporary file then moves over original file. This way we
avoid the need for locking.
"""
import shutil
from pootle.core.utils import ptempfile as tempfile
tmpfile, tmpfilename = tempfile.mkstemp(suffix=self.filename)
os.close(tmpfile)
self.store.savefile(tmpfilename)
shutil.move(tmpfilename, self.realpath)
self._touch_store_cache()
def save(self, name, content, save=True):
# FIXME: implement save to tmp file then move instead of directly
# saving
super().save(name, content, save)
self._delete_store_cache()
def delete(self, save=True):
self._delete_store_cache()
if save:
super().delete(save)
class TranslationProject(models.Model):
_non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE,
settings.PARSE_POOL_CULL_FREQUENCY)
objects = TranslationProjectManager()
index_directory = ".translation_index"
class Meta:
unique_together = ('language', 'project')
db_table = 'pootle_app_translationproject'
language = models.ForeignKey(Language, db_index=True)
project = models.ForeignKey(Project, db_index=True)
real_path = models.FilePathField(editable=False)
directory = models.OneToOneField(Directory, db_index=True, editable=False)
pootle_path = models.CharField(max_length=255,
null=False,
unique=True,
db_index=True,
editable=False)
def natural_key(self):
return (self.pootle_path, )
natural_key.dependencies = [
'pootle_app.Directory', 'pootle_language.Language',
'pootle_project.Project'
]
def __unicode__(self):
return self.pootle_path
def save(self, *args, **kwargs):
created = self.id is None
project_dir = self.project.get_real_path()
self.abs_real_path = project_tree.get_translation_project_dir(
self.language, project_dir, self.file_style, make_dirs=True)
self.directory = self.language.directory.get_or_make_subdir(
self.project.code)
self.pootle_path = self.directory.pootle_path
super(TranslationProject, self).save(*args, **kwargs)
if created:
self.scan_files()
def delete(self, *args, **kwargs):
directory = self.directory
super(TranslationProject, self).delete(*args, **kwargs)
directory.delete()
deletefromcache(self, [
"getquickstats", "getcompletestats", "get_mtime", "has_suggestions"
])
def get_absolute_url(self):
return l(self.pootle_path)
fullname = property(lambda self: "%s [%s]" %
(self.project.fullname, self.language.name))
def _get_abs_real_path(self):
return absolute_real_path(self.real_path)
def _set_abs_real_path(self, value):
self.real_path = relative_real_path(value)
abs_real_path = property(_get_abs_real_path, _set_abs_real_path)
def _get_treestyle(self):
return self.project.get_treestyle()
file_style = property(_get_treestyle)
def _get_checker(self):
checkerclasses = [
checks.projectcheckers.get(self.project.checkstyle,
checks.StandardChecker),
checks.StandardUnitChecker
]
return checks.TeeChecker(checkerclasses=checkerclasses,
errorhandler=self.filtererrorhandler,
languagecode=self.language.code)
checker = property(_get_checker)
def filtererrorhandler(self, functionname, str1, str2, e):
logging.error(u"error in filter %s: %r, %r, %s", functionname, str1,
str2, e)
return False
def _get_non_db_state(self):
if not hasattr(self, "_non_db_state"):
try:
self._non_db_state = self._non_db_state_cache[self.id]
except KeyError:
self._non_db_state = TranslationProjectNonDBState(self)
self._non_db_state_cache[
self.id] = TranslationProjectNonDBState(self)
return self._non_db_state
non_db_state = property(_get_non_db_state)
def update(self, conservative=True):
"""update all stores to reflect state on disk"""
for store in self.stores.exclude(file='').filter(
state__gte=PARSED).iterator():
store.update(update_translation=True,
update_structure=not conservative,
conservative=conservative)
def sync(self, conservative=True):
"""sync unsaved work on all stores to disk"""
for store in self.stores.exclude(file='').filter(
state__gte=PARSED).iterator():
store.sync(update_translation=True,
update_structure=not conservative,
conservative=conservative,
create=False)
@getfromcache
def get_mtime(self):
return max_column(Unit.objects.filter(store__translation_project=self),
'mtime', None)
def require_units(self):
"""makes sure all stores are parsed"""
errors = 0
for store in self.stores.filter(state__lt=PARSED).iterator():
try:
store.require_units()
except IntegrityError:
logging.info(u"Duplicate IDs in %s", store.abs_real_path)
errors += 1
except ParseError, e:
logging.info(u"Failed to parse %s\n%s", store.abs_real_path, e)
errors += 1
except (IOError, OSError), e:
logging.info(u"Can't access %s\n%s", store.abs_real_path, e)
errors += 1
class TranslationProject(models.Model, CachedTreeItem):
language = models.ForeignKey(Language, db_index=True)
project = models.ForeignKey(Project, db_index=True)
real_path = models.FilePathField(editable=False)
directory = models.OneToOneField(Directory, db_index=True, editable=False)
pootle_path = models.CharField(max_length=255,
null=False,
unique=True,
db_index=True,
editable=False)
creation_time = models.DateTimeField(auto_now_add=True,
db_index=True,
editable=False,
null=True)
_non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE,
settings.PARSE_POOL_CULL_FREQUENCY)
objects = TranslationProjectManager()
class Meta:
unique_together = ('language', 'project')
db_table = 'pootle_app_translationproject'
@cached_property
def code(self):
return u'-'.join([self.language.code, self.project.code])
############################ Properties ###################################
@property
def name(self):
# TODO: See if `self.fullname` can be removed
return self.fullname
@property
def fullname(self):
return "%s [%s]" % (self.project.fullname, self.language.name)
@property
def abs_real_path(self):
return absolute_real_path(self.real_path)
@abs_real_path.setter
def abs_real_path(self, value):
self.real_path = relative_real_path(value)
@property
def file_style(self):
return self.project.get_treestyle()
@property
def checker(self):
from translate.filters import checks
# We do not use default Translate Toolkit checkers; instead use
# our own one
if settings.POOTLE_QUALITY_CHECKER:
from pootle_misc.util import import_func
checkerclasses = [import_func(settings.POOTLE_QUALITY_CHECKER)]
else:
checkerclasses = [
checks.projectcheckers.get(self.project.checkstyle,
checks.StandardUnitChecker)
]
return checks.TeeChecker(checkerclasses=checkerclasses,
excludefilters=excluded_filters,
errorhandler=self.filtererrorhandler,
languagecode=self.language.code)
@property
def non_db_state(self):
if not hasattr(self, "_non_db_state"):
try:
self._non_db_state = self._non_db_state_cache[self.id]
except KeyError:
self._non_db_state = TranslationProjectNonDBState(self)
self._non_db_state_cache[self.id] = \
TranslationProjectNonDBState(self)
return self._non_db_state
@property
def units(self):
# FIXME: we rely on implicit ordering defined in the model. We might
# want to consider pootle_path as well
return Unit.objects.filter(store__translation_project=self,
state__gt=OBSOLETE).select_related('store')
@property
def disabled(self):
return self.project.disabled
@property
def is_terminology_project(self):
return self.project.checkstyle == 'terminology'
@property
def is_template_project(self):
return self == self.project.get_template_translationproject()
############################ Methods ######################################
def __unicode__(self):
return self.pootle_path
def __init__(self, *args, **kwargs):
super(TranslationProject, self).__init__(*args, **kwargs)
def save(self, *args, **kwargs):
created = self.id is None
if created:
from pootle_app.project_tree import translation_project_dir_exists
template_tp = self.project.get_template_translationproject()
initialize_from_templates = (not self.is_template_project
and template_tp is not None and
not translation_project_dir_exists(
self.language, self.project))
self.directory = self.language.directory \
.get_or_make_subdir(self.project.code)
self.pootle_path = self.directory.pootle_path
project_dir = self.project.get_real_path()
from pootle_app.project_tree import get_translation_project_dir
self.abs_real_path = get_translation_project_dir(
self.language,
project_dir,
self.file_style,
make_dirs=not self.directory.obsolete)
super(TranslationProject, self).save(*args, **kwargs)
if created:
if initialize_from_templates:
# We are adding a new TP and there are no files to import from
# disk, so initialize the TP files using the templates TP ones.
from pootle_app.project_tree import init_store_from_template
for template_store in template_tp.stores.live().iterator():
init_store_from_template(self, template_store)
self.scan_files()
# Create units from disk store
for store in self.stores.live().iterator():
changed = store.update_from_disk()
# If there were changes stats will be refreshed anyway - otherwise...
# Trigger stats refresh for TP added from UI.
# FIXME: This won't be necessary once #3547 is fixed.
if not changed:
store.save(update_cache=True)
def delete(self, *args, **kwargs):
directory = self.directory
super(TranslationProject, self).delete(*args, **kwargs)
directory.delete()
def get_absolute_url(self):
lang, proj, dir, fn = split_pootle_path(self.pootle_path)
return reverse('pootle-tp-browse', args=[lang, proj, dir, fn])
def get_translate_url(self, **kwargs):
lang, proj, dir, fn = split_pootle_path(self.pootle_path)
return u''.join([
reverse('pootle-tp-translate', args=[lang, proj, dir, fn]),
get_editor_filter(**kwargs),
])
def get_announcement(self, user=None):
"""Return the related announcement, if any."""
return StaticPage.get_announcement_for(self.pootle_path, user)
def filtererrorhandler(self, functionname, str1, str2, e):
logging.error(u"Error in filter %s: %r, %r, %s", functionname, str1,
str2, e)
return False
def is_accessible_by(self, user):
"""Returns `True` if the current translation project is accessible
by `user`.
"""
if user.is_superuser:
return True
return self.project.code in Project.accessible_by_user(user)
def update(self):
"""Update all stores to reflect state on disk"""
stores = self.stores.live().exclude(file='').filter(state__gte=PARSED)
for store in stores.iterator():
store.update_from_disk()
def sync(self, conservative=True, skip_missing=False, only_newer=True):
"""Sync unsaved work on all stores to disk"""
stores = self.stores.live().exclude(file='').filter(state__gte=PARSED)
for store in stores.iterator():
store.sync(update_structure=not conservative,
conservative=conservative,
skip_missing=skip_missing,
only_newer=only_newer)
### TreeItem
def get_children(self):
return self.directory.children
def get_cachekey(self):
return self.directory.pootle_path
def get_parents(self):
return [self.project]
def clear_all_cache(self, children=True, parents=True):
super(TranslationProject, self).clear_all_cache(children=children,
parents=parents)
if 'virtualfolder' in settings.INSTALLED_APPS:
# VirtualFolderTreeItem can only have VirtualFolderTreeItem parents
# so it is necessary to flush their cache by calling them one by
# one.
from virtualfolder.models import VirtualFolderTreeItem
tp_vfolder_treeitems = VirtualFolderTreeItem.objects.filter(
pootle_path__startswith=self.pootle_path)
for vfolder_treeitem in tp_vfolder_treeitems.iterator():
vfolder_treeitem.clear_all_cache(children=False, parents=False)
### /TreeItem
def directory_exists(self):
"""Checks if the actual directory for the translation project
exists on-disk.
"""
return not does_not_exist(self.abs_real_path)
def scan_files(self):
"""Scans the file system and returns a list of translation files.
"""
projects = [p.strip() for p in self.project.ignoredfiles.split(',')]
ignored_files = set(projects)
ext = os.extsep + self.project.localfiletype
# Scan for pots if template project
if self.is_template_project:
ext = os.extsep + self.project.get_template_filetype()
from pootle_app.project_tree import (add_files,
match_template_filename,
direct_language_match_filename)
all_files = []
new_files = []
if self.file_style == 'gnu':
if self.pootle_path.startswith('/templates/'):
file_filter = lambda filename: match_template_filename(
self.project,
filename,
)
else:
file_filter = lambda filename: direct_language_match_filename(
self.language.code,
filename,
)
else:
file_filter = lambda filename: True
all_files, new_files, is_empty = add_files(
self,
ignored_files,
ext,
self.real_path,
self.directory,
file_filter,
)
return all_files, new_files
###########################################################################
def gettermmatcher(self):
"""Returns the terminology matcher."""
terminology_stores = Store.objects.none()
mtime = None
if not self.is_terminology_project:
# Get global terminology first
try:
termproject = TranslationProject.objects \
.get_terminology_project(self.language_id)
mtime = termproject.get_cached_value(CachedMethods.MTIME)
terminology_stores = termproject.stores.live()
except TranslationProject.DoesNotExist:
pass
local_terminology = self.stores.live().filter(
name__startswith='pootle-terminology')
for store in local_terminology.iterator():
if mtime is None:
mtime = store.get_cached_value(CachedMethods.MTIME)
else:
mtime = max(mtime,
store.get_cached_value(CachedMethods.MTIME))
terminology_stores = terminology_stores | local_terminology
if mtime is None:
return
if mtime != self.non_db_state.termmatchermtime:
from pootle_misc.match import Matcher
self.non_db_state.termmatcher = Matcher(
terminology_stores.iterator(), )
self.non_db_state.termmatchermtime = mtime
return self.non_db_state.termmatcher
class TranslationProject(models.Model, CachedTreeItem):
language = models.ForeignKey(Language, db_index=True)
project = models.ForeignKey(Project, db_index=True)
real_path = models.FilePathField(editable=False)
directory = models.OneToOneField(Directory, db_index=True, editable=False)
pootle_path = models.CharField(max_length=255,
null=False,
unique=True,
db_index=True,
editable=False)
creation_time = models.DateTimeField(auto_now_add=True,
db_index=True,
editable=False,
null=True)
_non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE,
settings.PARSE_POOL_CULL_FREQUENCY)
objects = TranslationProjectManager()
class Meta:
unique_together = ('language', 'project')
db_table = 'pootle_app_translationproject'
@cached_property
def code(self):
return u'-'.join([self.language.code, self.project.code])
############################ Properties ###################################
@property
def name(self):
# TODO: See if `self.fullname` can be removed
return self.fullname
@property
def fullname(self):
return "%s [%s]" % (self.project.fullname, self.language.name)
@property
def abs_real_path(self):
return absolute_real_path(self.real_path)
@abs_real_path.setter
def abs_real_path(self, value):
self.real_path = relative_real_path(value)
@property
def file_style(self):
return self.project.get_treestyle()
@property
def checker(self):
from translate.filters import checks
# We do not use default Translate Toolkit checkers; instead use
# our own one
checkerclasses = [ENChecker]
return checks.TeeChecker(checkerclasses=checkerclasses,
excludefilters=excluded_filters,
errorhandler=self.filtererrorhandler,
languagecode=self.language.code)
@property
def non_db_state(self):
if not hasattr(self, "_non_db_state"):
try:
self._non_db_state = self._non_db_state_cache[self.id]
except KeyError:
self._non_db_state = TranslationProjectNonDBState(self)
self._non_db_state_cache[self.id] = \
TranslationProjectNonDBState(self)
return self._non_db_state
@property
def units(self):
self.require_units()
# FIXME: we rely on implicit ordering defined in the model. We might
# want to consider pootle_path as well
return Unit.objects.filter(store__translation_project=self,
state__gt=OBSOLETE).select_related('store')
@property
def is_terminology_project(self):
return self.project.checkstyle == 'terminology'
@property
def is_template_project(self):
return self == self.project.get_template_translationproject()
############################ Methods ######################################
def __unicode__(self):
return self.pootle_path
def __init__(self, *args, **kwargs):
super(TranslationProject, self).__init__(*args, **kwargs)
def save(self, *args, **kwargs):
created = self.id is None
self.directory = self.language.directory \
.get_or_make_subdir(self.project.code)
self.pootle_path = self.directory.pootle_path
project_dir = self.project.get_real_path()
from pootle_app.project_tree import get_translation_project_dir
self.abs_real_path = get_translation_project_dir(
self.language,
project_dir,
self.file_style,
make_dirs=not self.directory.obsolete)
super(TranslationProject, self).save(*args, **kwargs)
if created:
self.scan_files()
def delete(self, *args, **kwargs):
directory = self.directory
super(TranslationProject, self).delete(*args, **kwargs)
directory.delete()
def get_absolute_url(self):
lang, proj, dir, fn = split_pootle_path(self.pootle_path)
return reverse('pootle-tp-overview', args=[lang, proj, dir, fn])
def get_translate_url(self, **kwargs):
lang, proj, dir, fn = split_pootle_path(self.pootle_path)
return u''.join([
reverse('pootle-tp-translate', args=[lang, proj, dir, fn]),
get_editor_filter(**kwargs),
])
def filtererrorhandler(self, functionname, str1, str2, e):
logging.error(u"Error in filter %s: %r, %r, %s", functionname, str1,
str2, e)
return False
def is_accessible_by(self, user):
"""Returns `True` if the current translation project is accessible
by `user`.
"""
if user.is_superuser:
return True
return self.project.code in Project.accessible_by_user(user)
def update(self, overwrite=True):
"""Update all stores to reflect state on disk"""
stores = self.stores.live().exclude(file='').filter(state__gte=PARSED)
for store in stores.iterator():
store.update(overwrite=overwrite)
def sync(self, conservative=True, skip_missing=False, only_newer=True):
"""Sync unsaved work on all stores to disk"""
stores = self.stores.live().exclude(file='').filter(state__gte=PARSED)
for store in stores.iterator():
store.sync(update_structure=not conservative,
conservative=conservative,
skip_missing=skip_missing,
only_newer=only_newer)
def require_units(self):
"""Makes sure all stores are parsed"""
for store in self.stores.live().filter(state__lt=PARSED).iterator():
try:
store.require_units()
except IntegrityError:
logging.info(u"Duplicate IDs in %s", store.abs_real_path)
except ParseError as e:
logging.info(u"Failed to parse %s\n%s", store.abs_real_path, e)
except (IOError, OSError) as e:
logging.info(u"Can't access %s\n%s", store.abs_real_path, e)
### TreeItem
def get_children(self):
return self.directory.children
def get_cachekey(self):
return self.directory.pootle_path
def get_parent(self):
return self.project
### /TreeItem
def directory_exists(self):
"""Checks if the actual directory for the translation project
exists on-disk.
"""
return not does_not_exist(self.abs_real_path)
def scan_files(self):
"""Scans the file system and returns a list of translation files.
"""
projects = [p.strip() for p in self.project.ignoredfiles.split(',')]
ignored_files = set(projects)
ext = os.extsep + self.project.localfiletype
# Scan for pots if template project
if self.is_template_project:
ext = os.extsep + self.project.get_template_filetype()
from pootle_app.project_tree import (add_files,
match_template_filename,
direct_language_match_filename)
all_files = []
new_files = []
if self.file_style == 'gnu':
if self.pootle_path.startswith('/templates/'):
file_filter = lambda filename: match_template_filename(
self.project,
filename,
)
else:
file_filter = lambda filename: direct_language_match_filename(
self.language.code,
filename,
)
else:
file_filter = lambda filename: True
all_files, new_files, is_empty = add_files(
self,
ignored_files,
ext,
self.real_path,
self.directory,
file_filter,
)
return all_files, new_files
###########################################################################
def gettermmatcher(self):
"""Returns the terminology matcher."""
terminology_stores = Store.objects.none()
mtime = None
if not self.is_terminology_project:
# Get global terminology first
try:
termproject = TranslationProject.objects \
.get_terminology_project(self.language_id)
mtime = termproject.get_cached_value(CachedMethods.MTIME)
terminology_stores = termproject.stores.live()
except TranslationProject.DoesNotExist:
pass
local_terminology = self.stores.live().filter(
name__startswith='pootle-terminology')
for store in local_terminology.iterator():
if mtime is None:
mtime = store.get_cached_value(CachedMethods.MTIME)
else:
mtime = max(mtime,
store.get_cached_value(CachedMethods.MTIME))
terminology_stores = terminology_stores | local_terminology
if mtime is None:
return
if mtime != self.non_db_state.termmatchermtime:
from pootle_misc.match import Matcher
self.non_db_state.termmatcher = Matcher(
terminology_stores.iterator(), )
self.non_db_state.termmatchermtime = mtime
return self.non_db_state.termmatcher
class TranslationProject(models.Model, CachedTreeItem):
language = models.ForeignKey(Language, db_index=True, on_delete=models.CASCADE)
project = models.ForeignKey(Project, db_index=True, on_delete=models.CASCADE)
real_path = models.FilePathField(editable=False, null=True, blank=True)
directory = models.OneToOneField(
Directory, db_index=True, editable=False, on_delete=models.CASCADE
)
pootle_path = models.CharField(
max_length=255, null=False, unique=True, db_index=True, editable=False
)
creation_time = models.DateTimeField(
auto_now_add=True, db_index=True, editable=False, null=True
)
_non_db_state_cache = LRUCachingDict(PARSE_POOL_SIZE, PARSE_POOL_CULL_FREQUENCY)
objects = TranslationProjectManager()
class Meta(object):
unique_together = ("language", "project")
db_table = "pootle_app_translationproject"
base_manager_name = "objects"
@cached_property
def code(self):
return u"-".join([self.language.code, self.project.code])
# # # # # # # # # # # # # # Properties # # # # # # # # # # # # # # # # # #
@property
def name(self):
# TODO: See if `self.fullname` can be removed
return self.fullname
@property
def fullname(self):
return "%s [%s]" % (self.project.fullname, self.language.name)
@property
def abs_real_path(self):
if self.real_path is not None:
return absolute_real_path(self.real_path)
@abs_real_path.setter
def abs_real_path(self, value):
if value is not None:
self.real_path = relative_real_path(value)
else:
self.real_path = None
@property
def checker(self):
from translate.filters import checks
# We do not use default Translate Toolkit checkers; instead use
# our own one
if settings.ZING_QUALITY_CHECKER:
from pootle_misc.util import import_func
checkerclasses = [import_func(settings.ZING_QUALITY_CHECKER)]
else:
checkerclasses = [
checks.projectcheckers.get(
self.project.checkstyle, checks.StandardChecker
)
]
return checks.TeeChecker(
checkerclasses=checkerclasses,
excludefilters=excluded_filters,
errorhandler=self.filtererrorhandler,
languagecode=self.language.code,
)
@property
def non_db_state(self):
if not hasattr(self, "_non_db_state"):
try:
self._non_db_state = self._non_db_state_cache[self.id]
except KeyError:
self._non_db_state = TranslationProjectNonDBState(self)
self._non_db_state_cache[self.id] = TranslationProjectNonDBState(self)
return self._non_db_state
@property
def disabled(self):
return self.project.disabled
@property
def is_terminology_project(self):
return self.project.checkstyle == "terminology"
# # # # # # # # # # # # # # Methods # # # # # # # # # # # # # # # # # # #
def __str__(self):
return self.pootle_path
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def save(self, *args, **kwargs):
self.directory = self.language.directory.get_or_make_subdir(self.project.code)
self.pootle_path = self.directory.pootle_path
self.abs_real_path = get_translation_project_dir(
self.language,
self.project.get_real_path(),
make_dirs=not self.directory.obsolete,
)
super().save(*args, **kwargs)
def delete(self, *args, **kwargs):
directory = self.directory
super().delete(*args, **kwargs)
directory.delete()
def get_absolute_url(self):
return reverse(
"pootle-tp-browse", args=split_pootle_path(self.pootle_path)[:-1]
)
def get_translate_url(self, **kwargs):
return u"".join(
[
reverse(
"pootle-tp-translate", args=split_pootle_path(self.pootle_path)[:-1]
),
get_editor_filter(**kwargs),
]
)
def filtererrorhandler(self, functionname, str1, str2, e):
logging.error(u"Error in filter %s: %r, %r, %s", functionname, str1, str2, e)
return False
def is_accessible_by(self, user):
"""Returns `True` if the current translation project is accessible
by `user`.
"""
if user.is_superuser:
return True
return self.project.code in Project.accessible_by_user(user)
def update_from_disk(self, force=False, overwrite=False):
"""Update all stores to reflect state on disk.
:return: `True` if any of the existing stores were updated.
FIXME note: `scan_files()` doesn't report whether something
changed or not, but it can obsolete dirs/stores. Hence if that
happened the return value will be `False`, which is misleading.
"""
changed = False
logging.info(u"Scanning for new files in %s", self)
# Create new, make obsolete in-DB stores to reflect state on disk
self.scan_files()
stores = self.stores.live().select_related("parent").exclude(file="")
# Update store content from disk store
for store in stores.iterator():
changed = (
store.updater.update_from_disk(force=force, overwrite=overwrite)
or changed
)
# If this TP has no stores, cache should be updated forcibly.
if not changed and stores.count() == 0:
self.update_all_cache()
return changed
def sync(self, conservative=True, skip_missing=False, only_newer=True):
"""Sync unsaved work on all stores to disk"""
stores = self.stores.live().exclude(file="").filter(state__gte=PARSED)
for store in stores.select_related("parent").iterator():
store.sync(
update_structure=not conservative,
conservative=conservative,
skip_missing=skip_missing,
only_newer=only_newer,
)
# # # TreeItem
def get_children(self):
return self.directory.children
def get_parent(self):
return self.project
# # # /TreeItem
def directory_exists_on_disk(self):
"""Checks if the actual directory for the translation project
exists on disk.
"""
return not does_not_exist(self.abs_real_path)
def scan_files(self):
"""Scans the file system and returns a list of translation files.
"""
from pootle_app.project_tree import add_files
all_files = []
new_files = []
all_files, new_files, __ = add_files(self, self.real_path, self.directory,)
return all_files, new_files
###########################################################################
def gettermmatcher(self):
"""Returns the terminology matcher."""
terminology_stores = Store.objects.none()
mtime = None
if not self.is_terminology_project:
# Get global terminology first
try:
termproject = TranslationProject.objects.get_terminology_project(
self.language_id
)
mtime = termproject.get_cached_value(CachedMethods.MTIME)
terminology_stores = termproject.stores.live()
except TranslationProject.DoesNotExist:
pass
if mtime is None:
return
if mtime != self.non_db_state.termmatchermtime:
from pootle_misc.match import Matcher
self.non_db_state.termmatcher = Matcher(terminology_stores.iterator())
self.non_db_state.termmatchermtime = mtime
return self.non_db_state.termmatcher
class TranslationStoreFieldFile(FieldFile):
"""FieldFile is the File-like object of a FileField, that is found in a
TranslationStoreField."""
from translate.misc.lru import LRUCachingDict
from django.conf import settings
_store_cache = LRUCachingDict(settings.PARSE_POOL_SIZE,
settings.PARSE_POOL_CULL_FREQUENCY)
def getpomtime(self):
file_stat = os.stat(self.realpath)
return file_stat.st_mtime, file_stat.st_size
def _get_filename(self):
return os.path.basename(self.name)
filename = property(_get_filename)
def _get_realpath(self):
"""return realpath resolving symlinks if neccessary"""
if not hasattr(self, "_realpath"):
self._realpath = os.path.realpath(self.path)
return self._realpath
def _get_cached_realpath(self):
"""get real path from cache before attempting to check for symlinks"""
if not hasattr(self, "_store_tuple"):
return self._get_realpath()
else:
return self._store_tuple.realpath
realpath = property(_get_cached_realpath)
def _get_store(self):
"""Get translation store from dictionary cache, populate if store not
already cached."""
self._update_store_cache()
return self._store_tuple.store
def _update_store_cache(self):
"""Add translation store to dictionary cache, replace old cached
version if needed."""
mod_info = self.getpomtime()
if not hasattr(
self,
"_store_tuple") or self._store_tuple.mod_info != mod_info:
try:
self._store_tuple = self._store_cache[self.path]
if self._store_tuple.mod_info != mod_info:
# if file is modified act as if it doesn't exist in cache
raise KeyError
except KeyError:
logging.debug(u"cache miss for %s", self.path)
from translate.storage import factory
from pootle_store.filetypes import factory_classes
self._store_tuple = StoreTuple(
factory.getobject(self.path,
ignore=self.field.ignore,
classes=factory_classes), mod_info,
self.realpath)
self._store_cache[self.path] = self._store_tuple
translation_file_updated.send(sender=self, path=self.path)
def _touch_store_cache(self):
"""Update stored mod_info without reparsing file."""
if hasattr(self, "_store_tuple"):
mod_info = self.getpomtime()
if self._store_tuple.mod_info != mod_info:
self._store_tuple.mod_info = mod_info
translation_file_updated.send(sender=self, path=self.path)
else:
#FIXME: do we really need that?
self._update_store_cache()
def _delete_store_cache(self):
"""Remove translation store from cache."""
try:
del self._store_cache[self.path]
except KeyError:
pass
try:
del self._store_tuple
except AttributeError:
pass
translation_file_updated.send(sender=self, path=self.path)
store = property(_get_store)
def savestore(self):
"""Saves to temporary file then moves over original file. This
way we avoid the need for locking."""
import shutil
import tempfile
tmpfile, tmpfilename = tempfile.mkstemp(suffix=self.filename)
os.close(tmpfile)
self.store.savefile(tmpfilename)
shutil.move(tmpfilename, self.realpath)
self._touch_store_cache()
def save(self, name, content, save=True):
#FIXME: implement save to tmp file then move instead of directly saving
super(TranslationStoreFieldFile, self).save(name, content, save)
self._delete_store_cache()
def delete(self, save=True):
self._delete_store_cache()
if save:
super(TranslationStoreFieldFile, self).delete(save)
class TranslationProject(models.Model, CachedTreeItem):
language = models.ForeignKey(Language, db_index=True)
project = models.ForeignKey(Project, db_index=True)
real_path = models.FilePathField(editable=False, null=True, blank=True)
directory = models.OneToOneField(Directory, db_index=True, editable=False)
pootle_path = models.CharField(max_length=255,
null=False,
unique=True,
db_index=True,
editable=False)
creation_time = models.DateTimeField(auto_now_add=True,
db_index=True,
editable=False,
null=True)
_non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE,
settings.PARSE_POOL_CULL_FREQUENCY)
objects = TranslationProjectManager()
class Meta(object):
unique_together = ('language', 'project')
db_table = 'pootle_app_translationproject'
@cached_property
def code(self):
return u'-'.join([self.language.code, self.project.code])
@cached_property
def data_tool(self):
return data_tool.get(self.__class__)(self)
# # # # # # # # # # # # # # Properties # # # # # # # # # # # # # # # # # #
@property
def name(self):
# TODO: See if `self.fullname` can be removed
return self.fullname
@property
def fullname(self):
return "%s [%s]" % (self.project.fullname, self.language.name)
@property
def abs_real_path(self):
if self.real_path is not None:
return absolute_real_path(self.real_path)
@abs_real_path.setter
def abs_real_path(self, value):
if value is not None:
self.real_path = relative_real_path(value)
else:
self.real_path = None
@property
def file_style(self):
return self.project.get_treestyle()
@property
def checker(self):
from translate.filters import checks
# We do not use default Translate Toolkit checkers; instead use
# our own one
if settings.POOTLE_QUALITY_CHECKER:
from pootle_misc.util import import_func
checkerclasses = [import_func(settings.POOTLE_QUALITY_CHECKER)]
else:
checkerclasses = [
checks.projectcheckers.get(self.project.checkstyle,
checks.StandardChecker)
]
return checks.TeeChecker(checkerclasses=checkerclasses,
excludefilters=excluded_filters,
errorhandler=self.filtererrorhandler,
languagecode=self.language.code)
@property
def non_db_state(self):
if not hasattr(self, "_non_db_state"):
try:
self._non_db_state = self._non_db_state_cache[self.id]
except KeyError:
self._non_db_state = TranslationProjectNonDBState(self)
self._non_db_state_cache[self.id] = \
TranslationProjectNonDBState(self)
return self._non_db_state
@property
def disabled(self):
return self.project.disabled
@property
def is_terminology_project(self):
return self.project.checkstyle == 'terminology'
@property
def is_template_project(self):
return self == self.project.get_template_translationproject()
# # # # # # # # # # # # # # Methods # # # # # # # # # # # # # # # # # # #
def __unicode__(self):
return self.pootle_path
def __init__(self, *args, **kwargs):
super(TranslationProject, self).__init__(*args, **kwargs)
def save(self, *args, **kwargs):
self.directory = self.language.directory \
.get_or_make_subdir(self.project.code)
self.pootle_path = self.directory.pootle_path
if self.project.treestyle != "none":
from pootle_app.project_tree import get_translation_project_dir
self.abs_real_path = get_translation_project_dir(
self.language,
self.project,
self.file_style,
make_dirs=not self.directory.obsolete)
else:
self.abs_real_path = None
super(TranslationProject, self).save(*args, **kwargs)
def delete(self, *args, **kwargs):
directory = self.directory
super(TranslationProject, self).delete(*args, **kwargs)
directory.delete()
def get_absolute_url(self):
return reverse('pootle-tp-browse',
args=split_pootle_path(self.pootle_path)[:-1])
def get_translate_url(self, **kwargs):
return u''.join([
reverse("pootle-tp-translate",
args=split_pootle_path(self.pootle_path)[:-1]),
get_editor_filter(**kwargs)
])
def get_announcement(self, user=None):
"""Return the related announcement, if any."""
return StaticPage.get_announcement_for(self.pootle_path, user)
def filtererrorhandler(self, functionname, str1, str2, e):
logging.error(u"Error in filter %s: %r, %r, %s", functionname, str1,
str2, e)
return False
def is_accessible_by(self, user):
"""Returns `True` if the current translation project is accessible
by `user`.
"""
if user.is_superuser:
return True
return self.project.code in Project.accessible_by_user(user)
def can_be_inited_from_templates(self):
"""Returns `True` if the current translation project hasn't been
saved yet and can be initialized from templates.
"""
# This method checks if the current translation project directory
# doesn't exist. So it won't work if the translation project is already
# saved the database because the translation project directory is
# auto-created in `save()` method.
template_tp = self.project.get_template_translationproject()
return (not self.is_template_project and template_tp is not None
and not translation_project_dir_exists(self.language,
self.project))
def init_from_templates(self):
"""Initializes the current translation project files using
the templates TP ones.
"""
template_tp = self.project.get_template_translationproject()
template_stores = template_tp.stores.live().exclude(file="")
for template_store in template_stores.iterator():
init_store_from_template(self, template_store)
self.update_from_disk()
def update_from_disk(self, force=False, overwrite=False):
"""Update all stores to reflect state on disk."""
changed = False
logging.info(u"Scanning for new files in %s", self)
# Create new, make obsolete in-DB stores to reflect state on disk
self.scan_files()
stores = self.stores.live().select_related('parent').exclude(file='')
# Update store content from disk store
for store in stores.iterator():
if not store.file:
continue
disk_mtime = store.get_file_mtime()
if not force and disk_mtime == store.file_mtime:
# The file on disk wasn't changed since the last sync
logging.debug(
u"File didn't change since last sync, "
u"skipping %s", store.pootle_path)
continue
changed = (store.updater.update_from_disk(overwrite=overwrite)
or changed)
# If this TP has no stores, cache should be updated forcibly.
if not changed and stores.count() == 0:
self.update_all_cache()
return changed
def sync(self, conservative=True, skip_missing=False, only_newer=True):
"""Sync unsaved work on all stores to disk"""
stores = self.stores.live().exclude(file='').filter(state__gte=PARSED)
for store in stores.select_related("parent").iterator():
store.sync(update_structure=not conservative,
conservative=conservative,
skip_missing=skip_missing,
only_newer=only_newer)
# # # TreeItem
def get_children(self):
return self.directory.children
def get_cachekey(self):
return self.pootle_path
def get_parents(self):
return [self.project]
def clear_all_cache(self, children=True, parents=True):
super(TranslationProject, self).clear_all_cache(children=children,
parents=parents)
if 'virtualfolder' in settings.INSTALLED_APPS:
# VirtualFolderTreeItem can only have VirtualFolderTreeItem parents
# so it is necessary to flush their cache by calling them one by
# one.
from virtualfolder.models import VirtualFolderTreeItem
tp_vfolder_treeitems = VirtualFolderTreeItem.objects.filter(
pootle_path__startswith=self.pootle_path)
for vfolder_treeitem in tp_vfolder_treeitems.iterator():
vfolder_treeitem.clear_all_cache(children=False, parents=False)
# # # /TreeItem
def directory_exists_on_disk(self):
"""Checks if the actual directory for the translation project
exists on disk.
"""
return not does_not_exist(self.abs_real_path)
def scan_files(self):
"""Scans the file system and returns a list of translation files.
"""
projects = [p.strip() for p in self.project.ignoredfiles.split(',')]
ignored_files = set(projects)
filetypes = self.project.filetype_tool
exts = filetypes.filetype_extensions
# Scan for pots if template project
if self.is_template_project:
exts = filetypes.template_extensions
from pootle_app.project_tree import (add_files,
match_template_filename,
direct_language_match_filename)
all_files = []
new_files = []
if self.file_style == 'gnu':
if self.pootle_path.startswith('/templates/'):
file_filter = lambda filename: match_template_filename(
self.project,
filename,
)
else:
file_filter = lambda filename: direct_language_match_filename(
self.language.code,
filename,
)
else:
file_filter = lambda filename: True
all_files, new_files, __ = add_files(
self,
ignored_files,
exts,
self.real_path,
self.directory,
file_filter,
)
return all_files, new_files
###########################################################################
def gettermmatcher(self):
"""Returns the terminology matcher."""
terminology_stores = Store.objects.none()
mtime = None
if not self.is_terminology_project:
# Get global terminology first
try:
termproject = TranslationProject.objects \
.get_terminology_project(self.language_id)
mtime = termproject.get_cached_value(CachedMethods.MTIME)
terminology_stores = termproject.stores.live()
except TranslationProject.DoesNotExist:
pass
local_terminology = self.stores.live().filter(
name__startswith='pootle-terminology')
for store in local_terminology.iterator():
if mtime is None:
mtime = store.get_cached_value(CachedMethods.MTIME)
else:
mtime = max(mtime,
store.get_cached_value(CachedMethods.MTIME))
terminology_stores = terminology_stores | local_terminology
if mtime is None:
return
if mtime != self.non_db_state.termmatchermtime:
from pootle_misc.match import Matcher
self.non_db_state.termmatcher = Matcher(
terminology_stores.iterator())
self.non_db_state.termmatchermtime = mtime
return self.non_db_state.termmatcher
class TranslationProject(models.Model, TreeItem):
description = MarkupField(
blank=True,
help_text=_('A description of this translation project. This is '
'useful to give more information or instructions. Allowed '
'markup: %s', get_markup_filter_name()),
)
language = models.ForeignKey(Language, db_index=True)
project = models.ForeignKey(Project, db_index=True)
real_path = models.FilePathField(editable=False)
directory = models.OneToOneField(Directory, db_index=True, editable=False)
pootle_path = models.CharField(
max_length=255,
null=False,
unique=True,
db_index=True,
editable=False,
)
disabled = models.BooleanField(default=False)
tags = TaggableManager(
blank=True,
verbose_name=_("Tags"),
help_text=_("A comma-separated list of tags."),
)
goals = TaggableManager(
blank=True,
verbose_name=_("Goals"),
through=ItemWithGoal,
help_text=_("A comma-separated list of goals."),
)
# Cached Unit values
total_wordcount = models.PositiveIntegerField(
default=0,
null=True,
editable=False,
)
translated_wordcount = models.PositiveIntegerField(
default=0,
null=True,
editable=False,
)
fuzzy_wordcount = models.PositiveIntegerField(
default=0,
null=True,
editable=False,
)
suggestion_count = models.PositiveIntegerField(
default=0,
null=True,
editable=False,
)
failing_critical_count = models.PositiveIntegerField(
default=0,
null=True,
editable=False,
)
last_submission = models.OneToOneField(
Submission,
null=True,
editable=False,
)
last_unit = models.OneToOneField(Unit, null=True, editable=False)
_non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE,
settings.PARSE_POOL_CULL_FREQUENCY)
index_directory = ".translation_index"
objects = TranslationProjectManager()
class Meta:
unique_together = ('language', 'project')
db_table = 'pootle_app_translationproject'
############################ Properties ###################################
@property
def tag_like_objects(self):
"""Return the tag like objects applied to this translation project.
Tag like objects can be either tags or goals.
"""
return list(chain(self.tags.all().order_by("name"),
self.goals.all().order_by("name")))
@property
def name(self):
# TODO: See if `self.fullname` can be removed
return self.fullname
@property
def fullname(self):
return "%s [%s]" % (self.project.fullname, self.language.name)
@property
def abs_real_path(self):
return absolute_real_path(self.real_path)
@abs_real_path.setter
def abs_real_path(self, value):
self.real_path = relative_real_path(value)
@property
def file_style(self):
return self.project.get_treestyle()
@property
def checker(self):
from translate.filters import checks
checkerclasses = [checks.projectcheckers.get(self.project.checkstyle,
checks.StandardChecker),
checks.StandardUnitChecker]
return checks.TeeChecker(checkerclasses=checkerclasses,
excludefilters=excluded_filters,
errorhandler=self.filtererrorhandler,
languagecode=self.language.code)
@property
def non_db_state(self):
if not hasattr(self, "_non_db_state"):
try:
self._non_db_state = self._non_db_state_cache[self.id]
except KeyError:
self._non_db_state = TranslationProjectNonDBState(self)
self._non_db_state_cache[self.id] = \
TranslationProjectNonDBState(self)
return self._non_db_state
@property
def units(self):
self.require_units()
# FIXME: we rely on implicit ordering defined in the model. We might
# want to consider pootle_path as well
return Unit.objects.filter(store__translation_project=self,
state__gt=OBSOLETE).select_related('store')
@property
def is_terminology_project(self):
return self.pootle_path.endswith('/terminology/')
@property
def is_template_project(self):
return self == self.project.get_template_translationproject()
@property
def indexer(self):
if (self.non_db_state.indexer is None and
self.non_db_state._indexing_enabled):
try:
indexer = self.make_indexer()
if not self.non_db_state._index_initialized:
self.init_index(indexer)
self.non_db_state._index_initialized = True
self.non_db_state.indexer = indexer
except Exception as e:
logging.warning(u"Could not initialize indexer for %s in %s: "
u"%s", self.project.code, self.language.code,
str(e))
self.non_db_state._indexing_enabled = False
return self.non_db_state.indexer
@property
def has_index(self):
return (self.non_db_state._indexing_enabled and
(self.non_db_state._index_initialized or
self.indexer is not None))
############################ Cached properties ############################
@cached_property
def code(self):
return u'-'.join([self.language.code, self.project.code])
@cached_property
def all_goals(self):
# Putting the next import at the top of the file causes circular
# import issues.
from pootle_tagging.models import Goal
return Goal.get_goals_for_path(self.pootle_path)
############################ Methods ######################################
def __unicode__(self):
return self.pootle_path
def __init__(self, *args, **kwargs):
super(TranslationProject, self).__init__(*args, **kwargs)
def save(self, *args, **kwargs):
created = self.id is None
project_dir = self.project.get_real_path()
if not self.disabled:
from pootle_app.project_tree import get_translation_project_dir
self.abs_real_path = get_translation_project_dir(self.language,
project_dir, self.file_style, make_dirs=True)
self.directory = self.language.directory \
.get_or_make_subdir(self.project.code)
self.pootle_path = self.directory.pootle_path
super(TranslationProject, self).save(*args, **kwargs)
if created:
self.scan_files()
def delete(self, *args, **kwargs):
directory = self.directory
super(TranslationProject, self).delete(*args, **kwargs)
#TODO: avoid an access to directory while flushing the cache
directory.flush_cache()
directory.delete()
def get_absolute_url(self):
lang, proj, dir, fn = split_pootle_path(self.pootle_path)
return reverse('pootle-tp-overview', args=[lang, proj, dir, fn])
def get_translate_url(self, **kwargs):
lang, proj, dir, fn = split_pootle_path(self.pootle_path)
return u''.join([
reverse('pootle-tp-translate', args=[lang, proj, dir, fn]),
get_editor_filter(**kwargs),
])
def filtererrorhandler(self, functionname, str1, str2, e):
logging.error(u"Error in filter %s: %r, %r, %s", functionname, str1,
str2, e)
return False
def is_accessible_by(self, user):
"""Returns `True` if the current translation project is accessible
by `user`.
"""
if user.is_superuser:
return True
return self.project.code in Project.accessible_by_user(user)
def update(self):
"""Update all stores to reflect state on disk."""
stores = self.stores.exclude(file='').filter(state__gte=PARSED)
for store in stores.iterator():
store.update(update_translation=True, update_structure=True)
def sync(self, conservative=True, skip_missing=False, modified_since=0):
"""Sync unsaved work on all stores to disk."""
stores = self.stores.exclude(file='').filter(state__gte=PARSED)
for store in stores.iterator():
store.sync(update_translation=True,
update_structure=not conservative,
conservative=conservative, create=False,
skip_missing=skip_missing,
modified_since=modified_since)
def get_mtime(self):
return self.directory.get_mtime()
def require_units(self):
"""Makes sure all stores are parsed"""
errors = 0
for store in self.stores.filter(state__lt=PARSED).iterator():
try:
store.require_units()
except IntegrityError:
logging.info(u"Duplicate IDs in %s", store.abs_real_path)
errors += 1
except ParseError as e:
logging.info(u"Failed to parse %s\n%s", store.abs_real_path, e)
errors += 1
except (IOError, OSError) as e:
logging.info(u"Can't access %s\n%s", store.abs_real_path, e)
errors += 1
return errors
### TreeItem
def get_children_for_stats(self, goal=None):
if goal is None:
return super(TranslationProject, self).get_children_for_stats()
else:
from itertools import chain
stores, dirs = goal.get_children_for_path(self.pootle_path)
return list(chain(stores, dirs))
def get_progeny(self, goal=None):
if goal is None:
return super(TranslationProject, self).get_progeny()
else:
return goal.get_stores_for_path(self.pootle_path)
def get_self_stats(self, goal=None):
if goal is None:
return super(TranslationProject, self).get_self_stats()
else:
return {
'total': self.get_total_wordcount(goal),
'translated': self.get_translated_wordcount(goal),
'fuzzy': self.get_fuzzy_wordcount(goal),
'suggestions': self.get_suggestion_count(goal),
'critical': self.get_critical_error_unit_count(goal),
'lastupdated': self.get_last_updated(goal),
'lastaction': self.get_last_action(goal),
}
def get_children(self):
return self.directory.get_children()
def get_total_wordcount(self, goal=None):
if goal is None:
return self.total_wordcount
else:
return super(TranslationProject, self).get_total_wordcount(goal)
def get_translated_wordcount(self, goal=None):
if goal is None:
return self.translated_wordcount
else:
return super(TranslationProject, self).get_translated_wordcount(goal)
def get_fuzzy_wordcount(self, goal=None):
if goal is None:
return self.fuzzy_wordcount
else:
return super(TranslationProject, self).get_fuzzy_wordcount(goal)
def get_suggestion_count(self, goal=None):
if goal is None:
return self.suggestion_count
else:
return super(TranslationProject, self).get_suggestion_count(goal)
def get_critical_error_unit_count(self, goal=None):
if goal is None:
return self.failing_critical_count
else:
return super(TranslationProject, self).get_critical_error_unit_count(goal)
def get_next_goal_count(self):
# Putting the next import at the top of the file causes circular
# import issues.
from pootle_tagging.models import Goal
goal = Goal.get_most_important_incomplete_for_path(self.directory)
if goal is not None:
return goal.get_incomplete_words_in_path(self.directory)
return 0
def get_last_updated(self, goal=None):
if self.last_unit is None:
return {'id': 0, 'creation_time': 0, 'snippet': ''}
creation_time = dateformat.format(self.last_unit.creation_time, 'U')
return {
'id': self.last_unit.id,
'creation_time': int(creation_time),
'snippet': self.last_unit.get_last_updated_message()
}
def get_last_action(self, goal=None):
try:
if (self.last_submission is None or
(self.last_submission is not None and
self.last_submission.unit is None)):
return {'id': 0, 'mtime': 0, 'snippet': ''}
except Submission.DoesNotExist:
return {'id': 0, 'mtime': 0, 'snippet': ''}
mtime = dateformat.format(self.last_submission.creation_time, 'U')
return {
'id': self.last_submission.unit.id,
'mtime': int(mtime),
'snippet': self.last_submission.get_submission_message()
}
def get_next_goal_url(self):
# Putting the next import at the top of the file causes circular
# import issues.
from pootle_tagging.models import Goal
goal = Goal.get_most_important_incomplete_for_path(self.directory)
if goal is not None:
return goal.get_translate_url_for_path(self.directory.pootle_path,
state='incomplete')
return ''
def get_cachekey(self):
return self.directory.pootle_path
def get_parents(self):
return [self.language, self.project]
### /TreeItem
def update_against_templates(self, pootle_path=None):
"""Update translation project from templates."""
if self.is_template_project:
return
template_translation_project = self.project \
.get_template_translationproject()
if (template_translation_project is None or
template_translation_project == self):
return
monolingual = self.project.is_monolingual
if not monolingual:
self.sync()
from pootle_app.project_tree import (convert_template,
get_translated_name,
get_translated_name_gnu)
for store in template_translation_project.stores.iterator():
if self.file_style == 'gnu':
new_pootle_path, new_path = get_translated_name_gnu(self, store)
else:
new_pootle_path, new_path = get_translated_name(self, store)
if pootle_path is not None and new_pootle_path != pootle_path:
continue
convert_template(self, store, new_pootle_path, new_path,
monolingual)
all_files, new_files = self.scan_files(vcs_sync=False)
project_path = self.project.get_real_path()
if new_files and versioncontrol.hasversioning(project_path):
message = ("New files added from %s based on templates" %
get_site_title())
filestocommit = [f.file.name for f in new_files]
success = True
try:
output = versioncontrol.add_files(project_path, filestocommit,
message)
except Exception:
logging.exception(u"Failed to add files")
success = False
if pootle_path is None:
from pootle_app.signals import post_template_update
post_template_update.send(sender=self)
def scan_files(self, vcs_sync=True):
"""Scan the file system and return a list of translation files.
:param vcs_sync: boolean on whether or not to synchronise the PO
directory with the VCS checkout.
"""
proj_ignore = [p.strip() for p in self.project.ignoredfiles.split(',')]
ignored_files = set(proj_ignore)
ext = os.extsep + self.project.localfiletype
# Scan for pots if template project
if self.is_template_project:
ext = os.extsep + self.project.get_template_filetype()
from pootle_app.project_tree import (add_files,
match_template_filename,
direct_language_match_filename)
all_files = []
new_files = []
if self.file_style == 'gnu':
if self.pootle_path.startswith('/templates/'):
file_filter = lambda filename: match_template_filename(
self.project, filename,
)
else:
file_filter = lambda filename: direct_language_match_filename(
self.language.code, filename,
)
else:
file_filter = lambda filename: True
if vcs_sync:
from versioncontrol.utils import sync_from_vcs
sync_from_vcs(ignored_files, ext, self.real_path, file_filter)
all_files, new_files = add_files(
self,
ignored_files,
ext,
self.real_path,
self.directory,
file_filter,
)
return all_files, new_files
def update_file_from_version_control(self, store):
store.sync(update_translation=True)
filetoupdate = store.file.name
# Keep a copy of working files in memory before updating
working_copy = store.file.store
try:
logging.debug(u"Updating %s from version control", store.file.name)
versioncontrol.update_file(filetoupdate)
store.file._delete_store_cache()
store.file._update_store_cache()
except Exception:
# Something wrong, file potentially modified, bail out
# and replace with working copy
logging.exception(u"Near fatal catastrophe, while updating %s "
u"from version control", store.file.name)
working_copy.save()
raise versioncontrol.VersionControlError
try:
logging.debug(u"Parsing version control copy of %s into db",
store.file.name)
store.update(update_structure=True, update_translation=True)
#FIXME: try to avoid merging if file was not updated
logging.debug(u"Merging %s with version control update",
store.file.name)
store.mergefile(working_copy, None, allownewstrings=False,
suggestions=True, notranslate=False,
obsoletemissing=False)
except Exception:
logging.exception(u"Near fatal catastrophe, while merging %s with "
u"version control copy", store.file.name)
working_copy.save()
store.update(update_structure=True, update_translation=True)
raise
def update_dir(self, request=None, directory=None):
"""Updates translation project's files from version control, retaining
uncommitted translations.
"""
remote_stats = {}
try:
versioncontrol.update_dir(self.real_path)
except IOError as e:
logging.exception(u"Error during update of %s", self.real_path)
if request:
msg = _("Failed to update from version control: %(error)s",
{"error": e})
messages.error(request, msg)
return
all_files, new_files = self.scan_files()
new_file_set = set(new_files)
# Go through all stores except any pootle-terminology.* ones
if directory.is_translationproject():
stores = self.stores.exclude(file="")
else:
stores = directory.stores.exclude(file="")
for store in stores.iterator():
if store in new_file_set:
continue
store.sync(update_translation=True)
filetoupdate = store.file.name
# keep a copy of working files in memory before updating
working_copy = store.file.store
versioncontrol.copy_to_podir(filetoupdate)
store.file._delete_store_cache()
store.file._update_store_cache()
try:
logging.debug(u"Parsing version control copy of %s into db",
store.file.name)
store.update(update_structure=True, update_translation=True)
#FIXME: Try to avoid merging if file was not updated
logging.debug(u"Merging %s with version control update",
store.file.name)
store.mergefile(working_copy, None, allownewstrings=False,
suggestions=True, notranslate=False,
obsoletemissing=False)
except Exception:
logging.exception(u"Near fatal catastrophe, while merging %s "
"with version control copy", store.file.name)
working_copy.save()
store.update(update_structure=True, update_translation=True)
raise
if request:
msg = \
_(u'Updated project <em>%(project)s</em> from version control',
{'project': self.fullname})
messages.info(request, msg)
from pootle_app.signals import post_vc_update
post_vc_update.send(sender=self)
def update_file(self, request, store):
"""Updates file from version control, retaining uncommitted
translations"""
try:
self.update_file_from_version_control(store)
# FIXME: This belongs to views
msg = _(u'Updated file <em>%(filename)s</em> from version control',
{'filename': store.file.name})
messages.info(request, msg)
from pootle_app.signals import post_vc_update
post_vc_update.send(sender=self)
except versioncontrol.VersionControlError as e:
# FIXME: This belongs to views
msg = _(u"Failed to update <em>%(filename)s</em> from "
u"version control: %(error)s",
{
'filename': store.file.name,
'error': e,
}
)
messages.error(request, msg)
self.scan_files()
def commit_dir(self, user, directory, request=None):
"""Commits files under a directory to version control.
This does not do permission checking.
"""
self.sync()
total = directory.get_total_wordcount()
translated = directory.get_translated_wordcount()
fuzzy = directory.get_fuzzy_wordcount()
author = user.username
message = stats_message_raw("Commit from %s by user %s." %
(get_site_title(), author),
total, translated, fuzzy)
# Try to append email as well, since some VCS does not allow omitting
# it (ie. Git).
if user.is_authenticated() and len(user.email):
author += " <%s>" % user.email
if directory.is_translationproject():
stores = list(self.stores.exclude(file=""))
else:
stores = list(directory.stores.exclude(file=""))
filestocommit = [store.file.name for store in stores]
success = True
try:
project_path = self.project.get_real_path()
versioncontrol.add_files(project_path, filestocommit, message,
author)
# FIXME: This belongs to views
if request is not None:
msg = _("Committed all files under <em>%(path)s</em> to "
"version control", {'path': directory.pootle_path})
messages.success(request, msg)
except Exception as e:
logging.exception(u"Failed to commit directory")
# FIXME: This belongs to views
if request is not None:
msg = _("Failed to commit to version control: %(error)s",
{'error': e})
messages.error(request, msg)
success = False
from pootle_app.signals import post_vc_commit
post_vc_commit.send(sender=self, path_obj=directory,
user=user, success=success)
return success
def commit_file(self, user, store, request=None):
"""Commits an individual file to version control.
This does not do permission checking.
"""
from pootle_app.signals import post_vc_commit
store.sync(update_structure=False, update_translation=True,
conservative=True)
total = store.get_total_wordcount()
translated = store.get_translated_wordcount()
fuzzy = store.get_fuzzy_wordcount()
author = user.username
message = stats_message_raw("Commit from %s by user %s." % \
(get_site_title(), author), total, translated, fuzzy)
# Try to append email as well, since some VCS does not allow omitting
# it (ie. Git).
if user.is_authenticated() and len(user.email):
author += " <%s>" % user.email
filestocommit = [store.file.name]
success = True
for file in filestocommit:
try:
versioncontrol.commit_file(file, message=message,
author=author)
# FIXME: This belongs to views
if request is not None:
msg = _("Committed file <em>%(filename)s</em> to version "
"control", {'filename': file})
messages.success(request, msg)
except Exception as e:
logging.exception(u"Failed to commit file")
# FIXME: This belongs to views
if request is not None:
msg_params = {
"filename": file,
"error": e,
}
msg = _("Failed to commit <em>%(filename)s</em> to version "
"control: %(error)s", msg_params)
messages.error(request, msg)
success = False
post_vc_commit.send(sender=self, path_obj=store,
user=user, success=success)
return success
###########################################################################
def get_archive(self, stores, path=None):
"""Returns an archive of the given files."""
import shutil
import subprocess
from pootle_misc import ptempfile as tempfile
tempzipfile = None
archivecontents = None
try:
# Using zip command line is fast
# The temporary file below is opened and immediately closed for
# security reasons
fd, tempzipfile = tempfile.mkstemp(prefix='pootle', suffix='.zip')
os.close(fd)
archivecontents = open(tempzipfile, "wb")
file_list = u" ".join(
store.abs_real_path[len(self.abs_real_path)+1:] \
for store in stores.iterator()
)
process = subprocess.Popen(['zip', '-r', '-', file_list],
cwd=self.abs_real_path,
stdout=archivecontents)
result = process.wait()
if result == 0:
if path is not None:
shutil.move(tempzipfile, path)
return
else:
filedata = open(tempzipfile, "r").read()
if filedata:
return filedata
else:
raise Exception("failed to read temporary zip file")
else:
raise Exception("zip command returned error code: %d" % result)
except Exception as e:
# But if it doesn't work, we can do it from Python.
logging.debug(e)
logging.debug("falling back to zipfile module")
if path is not None:
if tempzipfile is None:
fd, tempzipfile = tempfile.mkstemp(prefix='pootle',
suffix='.zip')
os.close(fd)
archivecontents = open(tempzipfile, "wb")
else:
import cStringIO
archivecontents = cStringIO.StringIO()
import zipfile
archive = zipfile.ZipFile(archivecontents, 'w',
zipfile.ZIP_DEFLATED)
for store in stores.iterator():
archive.write(store.abs_real_path.encode('utf-8'),
store.abs_real_path[len(self.abs_real_path)+1:]
.encode('utf-8'))
archive.close()
if path is not None:
shutil.move(tempzipfile, path)
else:
return archivecontents.getvalue()
finally:
if tempzipfile is not None and os.path.exists(tempzipfile):
os.remove(tempzipfile)
try:
archivecontents.close()
except:
pass
###########################################################################
def make_indexer(self):
"""Get an indexing object for this project.
Since we do not want to keep the indexing databases open for the
lifetime of the TranslationProject (it is cached!), it may NOT be
part of the Project object, but should be used via a short living
local variable.
"""
logging.debug(u"Loading indexer for %s", self.pootle_path)
indexdir = os.path.join(self.abs_real_path, self.index_directory)
from translate.search import indexing
indexer = indexing.get_indexer(indexdir)
indexer.set_field_analyzers({
"pofilename": indexer.ANALYZER_EXACT,
"pomtime": indexer.ANALYZER_EXACT,
"dbid": indexer.ANALYZER_EXACT,
})
return indexer
def init_index(self, indexer):
"""Initializes the search index."""
#FIXME: stop relying on pomtime so virtual files can be searchable?
try:
indexer.begin_transaction()
for store in self.stores.iterator():
try:
self.update_index(indexer, store)
except OSError:
# Broken link or permission problem?
logging.exception("Error indexing %s", store)
indexer.commit_transaction()
indexer.flush(optimize=True)
except Exception:
logging.exception(u"Error opening indexer for %s", self)
try:
indexer.cancel_transaction()
except:
pass
def update_index(self, indexer, store, unitid=None):
"""Updates the index with the contents of store (limit to
``unitid`` if given).
There are two reasons for calling this function:
1. Creating a new instance of :cls:`TranslationProject`
(see :meth:`TranslationProject.init_index`)
-> Check if the index is up-to-date / rebuild the index if
necessary
2. Translating a unit via the web interface
-> (re)index only the specified unit(s)
The argument ``unitid`` should be None for 1.
Known problems:
1. This function should get called, when the po file changes
externally.
WARNING: You have to stop the pootle server before manually
changing po files, if you want to keep the index database in
sync.
"""
#FIXME: leverage file updated signal to check if index needs updating
if indexer is None:
return False
# Check if the pomtime in the index == the latest pomtime
pomtime = str(hash(store.get_mtime()) ** 2)
pofilenamequery = indexer.make_query([("pofilename",
store.pootle_path)], True)
pomtimequery = indexer.make_query([("pomtime", pomtime)], True)
gooditemsquery = indexer.make_query([pofilenamequery, pomtimequery],
True)
gooditemsnum = indexer.get_query_result(gooditemsquery) \
.get_matches_count()
# If there is at least one up-to-date indexing item, then the po file
# was not changed externally -> no need to update the database
units = None
if (gooditemsnum > 0) and (not unitid):
# Nothing to be done
return
elif unitid is not None:
# Update only specific item - usually translation via the web
# interface. All other items should still be up-to-date (even with
# an older pomtime).
# Delete the relevant item from the database
units = store.units.filter(id=unitid)
itemsquery = indexer.make_query([("dbid", str(unitid))], False)
indexer.delete_doc([pofilenamequery, itemsquery])
else:
# (item is None)
# The po file is not indexed - or it was changed externally
# delete all items of this file
logging.debug(u"Updating %s indexer for file %s", self.pootle_path,
store.pootle_path)
indexer.delete_doc({"pofilename": store.pootle_path})
units = store.units
addlist = []
for unit in units.iterator():
doc = {
"pofilename": store.pootle_path,
"pomtime": pomtime,
"dbid": str(unit.id),
}
if unit.hasplural():
orig = "\n".join(unit.source.strings)
trans = "\n".join(unit.target.strings)
else:
orig = unit.source
trans = unit.target
doc.update({
"source": orig,
"target": trans,
"notes": unit.getnotes(),
"locations": unit.getlocations(),
})
addlist.append(doc)
if addlist:
for add_item in addlist:
indexer.index_document(add_item)
###########################################################################
def gettermmatcher(self):
"""Returns the terminology matcher."""
terminology_stores = Store.objects.none()
mtime = None
if self.is_terminology_project:
terminology_stores = self.stores.all()
mtime = self.get_mtime()
else:
# Get global terminology first
try:
termproject = TranslationProject.objects.get(
language=self.language_id,
project__code='terminology',
)
mtime = termproject.get_mtime()
terminology_stores = termproject.stores.all()
except TranslationProject.DoesNotExist:
pass
local_terminology = self.stores.filter(
name__startswith='pootle-terminology')
for store in local_terminology.iterator():
if mtime is None:
mtime = store.get_mtime()
else:
mtime = max(mtime, store.get_mtime())
terminology_stores = terminology_stores | local_terminology
if mtime is None:
return
if mtime != self.non_db_state.termmatchermtime:
from translate.search import match
self.non_db_state.termmatcher = match.terminologymatcher(
terminology_stores.iterator(),
)
self.non_db_state.termmatchermtime = mtime
return self.non_db_state.termmatcher
class TranslationProject(models.Model):
description_help_text = _(
'A description of this translation project. '
'This is useful to give more information or '
'instructions. Allowed markup: %s', get_markup_filter_name())
description = MarkupField(blank=True, help_text=description_help_text)
language = models.ForeignKey(Language, db_index=True)
project = models.ForeignKey(Project, db_index=True)
real_path = models.FilePathField(editable=False)
directory = models.OneToOneField(Directory, db_index=True, editable=False)
pootle_path = models.CharField(max_length=255,
null=False,
unique=True,
db_index=True,
editable=False)
tags = TaggableManager(blank=True,
verbose_name=_("Tags"),
help_text=_("A comma-separated list of tags."))
_non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE,
settings.PARSE_POOL_CULL_FREQUENCY)
index_directory = ".translation_index"
objects = TranslationProjectManager()
class Meta:
unique_together = ('language', 'project')
db_table = 'pootle_app_translationproject'
def __unicode__(self):
return self.pootle_path
def save(self, *args, **kwargs):
created = self.id is None
project_dir = self.project.get_real_path()
from pootle_app.project_tree import get_translation_project_dir
self.abs_real_path = get_translation_project_dir(self.language,
project_dir,
self.file_style,
make_dirs=True)
self.directory = self.language.directory \
.get_or_make_subdir(self.project.code)
self.pootle_path = self.directory.pootle_path
super(TranslationProject, self).save(*args, **kwargs)
if created:
self.scan_files()
def delete(self, *args, **kwargs):
directory = self.directory
super(TranslationProject, self).delete(*args, **kwargs)
directory.delete()
deletefromcache(self, [
"getquickstats", "getcompletestats", "get_mtime",
"get_suggestion_count"
])
def get_absolute_url(self):
return l(self.pootle_path)
def get_translate_url(self, **kwargs):
lang, proj, dir, fn = split_pootle_path(self.pootle_path)
return u''.join([
reverse('pootle-tp-translate', args=[lang, proj, dir, fn]),
get_editor_filter(**kwargs),
])
def natural_key(self):
return (self.pootle_path, )
natural_key.dependencies = [
'pootle_app.Directory', 'pootle_language.Language',
'pootle_project.Project'
]
###########################################################################
# Properties #
###########################################################################
fullname = property(lambda self: "%s [%s]" %
(self.project.fullname, self.language.name))
def _get_abs_real_path(self):
return absolute_real_path(self.real_path)
def _set_abs_real_path(self, value):
self.real_path = relative_real_path(value)
abs_real_path = property(_get_abs_real_path, _set_abs_real_path)
def _get_treestyle(self):
return self.project.get_treestyle()
file_style = property(_get_treestyle)
def _get_checker(self):
from translate.filters import checks
checkerclasses = [
checks.projectcheckers.get(self.project.checkstyle,
checks.StandardChecker),
checks.StandardUnitChecker
]
excluded_filters = ['hassuggestion', 'spellcheck']
return checks.TeeChecker(checkerclasses=checkerclasses,
excludefilters=excluded_filters,
errorhandler=self.filtererrorhandler,
languagecode=self.language.code)
checker = property(_get_checker)
def _get_non_db_state(self):
if not hasattr(self, "_non_db_state"):
try:
self._non_db_state = self._non_db_state_cache[self.id]
except KeyError:
self._non_db_state = TranslationProjectNonDBState(self)
self._non_db_state_cache[self.id] = \
TranslationProjectNonDBState(self)
return self._non_db_state
non_db_state = property(_get_non_db_state)
def _get_units(self):
self.require_units()
# FIXME: we rely on implicit ordering defined in the model. We might
# want to consider pootle_path as well
return Unit.objects.filter(store__translation_project=self,
state__gt=OBSOLETE).select_related('store')
units = property(_get_units)
@property
def is_terminology_project(self):
return self.pootle_path.endswith('/terminology/')
@property
def is_template_project(self):
return self == self.project.get_template_translationproject()
def _get_indexer(self):
if (self.non_db_state.indexer is None
and self.non_db_state._indexing_enabled):
try:
indexer = self.make_indexer()
if not self.non_db_state._index_initialized:
self.init_index(indexer)
self.non_db_state._index_initialized = True
self.non_db_state.indexer = indexer
except Exception, e:
logging.warning(
u"Could not initialize indexer for %s in %s: "
u"%s", self.project.code, self.language.code, str(e))
self.non_db_state._indexing_enabled = False
return self.non_db_state.indexer
class TranslationProject(models.Model):
description_help_text = _(
'A description of this translation project. '
'This is useful to give more information or '
'instructions. Allowed markup: %s', get_markup_filter_name())
description = MarkupField(blank=True, help_text=description_help_text)
language = models.ForeignKey(Language, db_index=True)
project = models.ForeignKey(Project, db_index=True)
real_path = models.FilePathField(editable=False)
directory = models.OneToOneField(Directory, db_index=True, editable=False)
pootle_path = models.CharField(max_length=255,
null=False,
unique=True,
db_index=True,
editable=False)
tags = TaggableManager(blank=True,
verbose_name=_("Tags"),
help_text=_("A comma-separated list of tags."))
_non_db_state_cache = LRUCachingDict(settings.PARSE_POOL_SIZE,
settings.PARSE_POOL_CULL_FREQUENCY)
index_directory = ".translation_index"
objects = TranslationProjectManager()
class Meta:
unique_together = ('language', 'project')
db_table = 'pootle_app_translationproject'
def __unicode__(self):
return self.pootle_path
def save(self, *args, **kwargs):
created = self.id is None
project_dir = self.project.get_real_path()
from pootle_app.project_tree import get_translation_project_dir
self.abs_real_path = get_translation_project_dir(self.language,
project_dir,
self.file_style,
make_dirs=True)
self.directory = self.language.directory \
.get_or_make_subdir(self.project.code)
self.pootle_path = self.directory.pootle_path
super(TranslationProject, self).save(*args, **kwargs)
if created:
self.scan_files()
def delete(self, *args, **kwargs):
directory = self.directory
super(TranslationProject, self).delete(*args, **kwargs)
directory.delete()
deletefromcache(self, [
"getquickstats", "getcompletestats", "get_mtime",
"get_suggestion_count"
])
def get_absolute_url(self):
return l(self.pootle_path)
def get_translate_url(self, **kwargs):
lang, proj, dir, fn = split_pootle_path(self.pootle_path)
return u''.join([
reverse('pootle-tp-translate', args=[lang, proj, dir, fn]),
get_editor_filter(**kwargs),
])
def natural_key(self):
return (self.pootle_path, )
natural_key.dependencies = [
'pootle_app.Directory', 'pootle_language.Language',
'pootle_project.Project'
]
###########################################################################
# Properties #
###########################################################################
fullname = property(lambda self: "%s [%s]" %
(self.project.fullname, self.language.name))
def _get_abs_real_path(self):
return absolute_real_path(self.real_path)
def _set_abs_real_path(self, value):
self.real_path = relative_real_path(value)
abs_real_path = property(_get_abs_real_path, _set_abs_real_path)
def _get_treestyle(self):
return self.project.get_treestyle()
file_style = property(_get_treestyle)
def _get_checker(self):
from translate.filters import checks
checkerclasses = [
checks.projectcheckers.get(self.project.checkstyle,
checks.StandardChecker),
checks.StandardUnitChecker
]
excluded_filters = ['hassuggestion', 'spellcheck']
return checks.TeeChecker(checkerclasses=checkerclasses,
excludefilters=excluded_filters,
errorhandler=self.filtererrorhandler,
languagecode=self.language.code)
checker = property(_get_checker)
def _get_non_db_state(self):
if not hasattr(self, "_non_db_state"):
try:
self._non_db_state = self._non_db_state_cache[self.id]
except KeyError:
self._non_db_state = TranslationProjectNonDBState(self)
self._non_db_state_cache[self.id] = \
TranslationProjectNonDBState(self)
return self._non_db_state
non_db_state = property(_get_non_db_state)
def _get_units(self):
self.require_units()
# FIXME: we rely on implicit ordering defined in the model. We might
# want to consider pootle_path as well
return Unit.objects.filter(store__translation_project=self,
state__gt=OBSOLETE).select_related('store')
units = property(_get_units)
@property
def is_terminology_project(self):
return self.pootle_path.endswith('/terminology/')
@property
def is_template_project(self):
return self == self.project.get_template_translationproject()
def _get_indexer(self):
if (self.non_db_state.indexer is None
and self.non_db_state._indexing_enabled):
try:
indexer = self.make_indexer()
if not self.non_db_state._index_initialized:
self.init_index(indexer)
self.non_db_state._index_initialized = True
self.non_db_state.indexer = indexer
except Exception as e:
logging.warning(
u"Could not initialize indexer for %s in %s: "
u"%s", self.project.code, self.language.code, str(e))
self.non_db_state._indexing_enabled = False
return self.non_db_state.indexer
indexer = property(_get_indexer)
def _has_index(self):
return (self.non_db_state._indexing_enabled
and (self.non_db_state._index_initialized
or self.indexer is not None))
has_index = property(_has_index)
###########################################################################
def filtererrorhandler(self, functionname, str1, str2, e):
logging.error(u"Error in filter %s: %r, %r, %s", functionname, str1,
str2, e)
return False
def update(self):
"""Update all stores to reflect state on disk."""
stores = self.stores.exclude(file='').filter(state__gte=PARSED)
for store in stores.iterator():
store.update(update_translation=True, update_structure=True)
def sync(self, conservative=True, skip_missing=False, modified_since=0):
"""Sync unsaved work on all stores to disk."""
stores = self.stores.exclude(file='').filter(state__gte=PARSED)
for store in stores.iterator():
store.sync(update_translation=True,
update_structure=not conservative,
conservative=conservative,
create=False,
skip_missing=skip_missing,
modified_since=modified_since)
def get_latest_submission(self):
"""Get the latest submission done in the Translation project."""
try:
sub = Submission.objects.filter(translation_project=self).latest()
except Submission.DoesNotExist:
return ''
return sub.get_submission_message()
@getfromcache
def get_mtime(self):
tp_units = Unit.objects.filter(store__translation_project=self)
return max_column(tp_units, 'mtime', None)
def require_units(self):
"""Makes sure all stores are parsed"""
errors = 0
for store in self.stores.filter(state__lt=PARSED).iterator():
try:
store.require_units()
except IntegrityError:
logging.info(u"Duplicate IDs in %s", store.abs_real_path)
errors += 1
except ParseError as e:
logging.info(u"Failed to parse %s\n%s", store.abs_real_path, e)
errors += 1
except (IOError, OSError) as e:
logging.info(u"Can't access %s\n%s", store.abs_real_path, e)
errors += 1
return errors
@getfromcache
def getquickstats(self):
if self.is_template_project:
return empty_quickstats
errors = self.require_units()
tp_not_obsolete_units = Unit.objects.filter(
store__translation_project=self,
state__gt=OBSOLETE,
)
stats = calculate_stats(tp_not_obsolete_units)
stats['errors'] = errors
return stats
@getfromcache
def getcompletestats(self):
if self.is_template_project:
return empty_completestats
for store in self.stores.filter(state__lt=CHECKED).iterator():
store.require_qualitychecks()
query = QualityCheck.objects.filter(
unit__store__translation_project=self,
unit__state__gt=UNTRANSLATED,
false_positive=False)
return group_by_count_extra(query, 'name', 'category')
@getfromcache
def get_suggestion_count(self):
"""
Check if any unit in the stores for this translation project has
suggestions.
"""
return Suggestion.objects.filter(unit__store__translation_project=self,
unit__state__gt=OBSOLETE).count()
def update_against_templates(self, pootle_path=None):
"""Update translation project from templates."""
if self.is_template_project:
return
template_translation_project = self.project \
.get_template_translationproject()
if (template_translation_project is None
or template_translation_project == self):
return
monolingual = self.project.is_monolingual()
if not monolingual:
self.sync()
if pootle_path is None:
oldstats = self.getquickstats()
from pootle_app.project_tree import (convert_template,
get_translated_name,
get_translated_name_gnu)
for store in template_translation_project.stores.iterator():
if self.file_style == 'gnu':
new_pootle_path, new_path = get_translated_name_gnu(
self, store)
else:
new_pootle_path, new_path = get_translated_name(self, store)
if pootle_path is not None and new_pootle_path != pootle_path:
continue
relative_po_path = os.path.relpath(new_path, settings.PODIRECTORY)
try:
from pootle.scripts import hooks
if not hooks.hook(self.project.code, "pretemplateupdate",
relative_po_path):
continue
except:
# Assume hook is not present.
pass
convert_template(self, store, new_pootle_path, new_path,
monolingual)
all_files, new_files = self.scan_files(vcs_sync=False)
from pootle_misc import versioncontrol
project_path = self.project.get_real_path()
if new_files and versioncontrol.hasversioning(project_path):
from pootle.scripts import hooks
message = "New files added from %s based on templates" % \
(settings.TITLE)
filestocommit = []
for new_file in new_files:
try:
filestocommit.extend(
hooks.hook(self.project.code,
"precommit",
new_file.file.name,
author=None,
message=message))
except ImportError:
# Failed to import the hook - we're going to assume there
# just isn't a hook to import. That means we'll commit the
# original file.
filestocommit.append(new_file.file.name)
success = True
try:
output = versioncontrol.add_files(project_path, filestocommit,
message)
except Exception as e:
logging.error(u"Failed to add files: %s", e)
success = False
for new_file in new_files:
try:
hooks.hook(self.project.code,
"postcommit",
new_file.file.name,
success=success)
except:
#FIXME: We should not hide the exception - makes
# development impossible
pass
if pootle_path is None:
newstats = self.getquickstats()
from pootle_app.models.signals import post_template_update
post_template_update.send(sender=self,
oldstats=oldstats,
newstats=newstats)
def scan_files(self, vcs_sync=True):
"""Scans the file system and returns a list of translation files.
:param vcs_sync: boolean on whether or not to synchronise the PO
directory with the VCS checkout.
"""
projects = [p.strip() for p in self.project.ignoredfiles.split(',')]
ignored_files = set(projects)
ext = os.extsep + self.project.localfiletype
# Scan for pots if template project
if self.is_template_project:
ext = os.extsep + self.project.get_template_filetype()
from pootle_app.project_tree import (add_files,
match_template_filename,
direct_language_match_filename,
sync_from_vcs)
all_files = []
new_files = []
if self.file_style == 'gnu':
if self.pootle_path.startswith('/templates/'):
file_filter = lambda filename: match_template_filename(
self.project,
filename,
)
else:
file_filter = lambda filename: direct_language_match_filename(
self.language.code,
filename,
)
else:
file_filter = lambda filename: True
if vcs_sync:
sync_from_vcs(ignored_files, ext, self.real_path, file_filter)
all_files, new_files = add_files(
self,
ignored_files,
ext,
self.real_path,
self.directory,
file_filter,
)
return all_files, new_files
def update_file_from_version_control(self, store):
from pootle.scripts import hooks
store.sync(update_translation=True)
filetoupdate = store.file.name
try:
filetoupdate = hooks.hook(self.project.code, "preupdate",
store.file.name)
except:
pass
# Keep a copy of working files in memory before updating
oldstats = store.getquickstats()
working_copy = store.file.store
try:
logging.debug(u"Updating %s from version control", store.file.name)
from pootle_misc import versioncontrol
versioncontrol.update_file(filetoupdate)
store.file._delete_store_cache()
store.file._update_store_cache()
except Exception as e:
# Something wrong, file potentially modified, bail out
# and replace with working copy
logging.error(
u"Near fatal catastrophe, exception %s while "
u"updating %s from version control", e, store.file.name)
working_copy.save()
raise VersionControlError
try:
hooks.hook(self.project.code, "postupdate", store.file.name)
except:
pass
try:
logging.debug(u"Parsing version control copy of %s into db",
store.file.name)
store.update(update_structure=True, update_translation=True)
remotestats = store.getquickstats()
#FIXME: try to avoid merging if file was not updated
logging.debug(u"Merging %s with version control update",
store.file.name)
store.mergefile(working_copy,
None,
allownewstrings=False,
suggestions=True,
notranslate=False,
obsoletemissing=False)
except Exception as e:
logging.error(
u"Near fatal catastrophe, exception %s while merging "
u"%s with version control copy", e, store.file.name)
working_copy.save()
store.update(update_structure=True, update_translation=True)
raise
newstats = store.getquickstats()
return oldstats, remotestats, newstats
def update_dir(self, request=None, directory=None):
"""Updates translation project's files from version control, retaining
uncommitted translations.
"""
old_stats = self.getquickstats()
remote_stats = {}
from pootle_misc import versioncontrol
try:
versioncontrol.update_dir(self.real_path)
except IOError as e:
logging.error(u"Error during update of %(path)s:\n%(error)s", {
"path": self.real_path,
"error": e,
})
if request:
msg = _("Failed to update from version control: %(error)s",
{"error": e})
messages.error(request, msg)
return
all_files, new_files = self.scan_files()
new_file_set = set(new_files)
from pootle.scripts import hooks
# Go through all stores except any pootle-terminology.* ones
if directory.is_translationproject():
stores = self.stores.exclude(file="")
else:
stores = directory.stores.exclude(file="")
for store in stores.iterator():
if store in new_file_set:
# these won't have to be merged, since they are new
remotestats = store.getquickstats()
remote_stats = dictsum(remote_stats, remotestats)
continue
store.sync(update_translation=True)
filetoupdate = store.file.name
try:
filetoupdate = hooks.hook(self.project.code, "preupdate",
store.file.name)
except:
pass
# keep a copy of working files in memory before updating
working_copy = store.file.store
versioncontrol.copy_to_podir(filetoupdate)
store.file._delete_store_cache()
store.file._update_store_cache()
try:
hooks.hook(self.project.code, "postupdate", store.file.name)
except:
pass
try:
logging.debug(u"Parsing version control copy of %s into db",
store.file.name)
store.update(update_structure=True, update_translation=True)
remotestats = store.getquickstats()
#FIXME: Try to avoid merging if file was not updated
logging.debug(u"Merging %s with version control update",
store.file.name)
store.mergefile(working_copy,
None,
allownewstrings=False,
suggestions=True,
notranslate=False,
obsoletemissing=False)
except Exception as e:
logging.error(
u"Near fatal catastrophe, exception %s while "
"merging %s with version control copy", e, store.file.name)
working_copy.save()
store.update(update_structure=True, update_translation=True)
raise
remote_stats = dictsum(remote_stats, remotestats)
new_stats = self.getquickstats()
if request:
msg = [
_(u'Updated project <em>%(project)s</em> from version control',
{'project': self.fullname}),
stats_message(_(u"Working copy"), old_stats),
stats_message(_(u"Remote copy"), remote_stats),
stats_message(_(u"Merged copy"), new_stats)
]
msg = u"<br/>".join([force_unicode(m) for m in msg])
messages.info(request, msg)
from pootle_app.models.signals import post_vc_update
post_vc_update.send(sender=self,
oldstats=old_stats,
remotestats=remote_stats,
newstats=new_stats)
def update_file(self, request, store):
"""Updates file from version control, retaining uncommitted
translations"""
try:
old_stats, remote_stats, new_stats = \
self.update_file_from_version_control(store)
# FIXME: This belongs to views
msg = [
_(u'Updated file <em>%(filename)s</em> from version control',
{'filename': store.file.name}),
stats_message(_(u"Working copy"), old_stats),
stats_message(_(u"Remote copy"), remote_stats),
stats_message(_(u"Merged copy"), new_stats)
]
msg = u"<br/>".join([force_unicode(m) for m in msg])
messages.info(request, msg)
from pootle_app.models.signals import post_vc_update
post_vc_update.send(sender=self,
oldstats=old_stats,
remotestats=remote_stats,
newstats=new_stats)
except VersionControlError as e:
# FIXME: This belongs to views
msg = _(
u"Failed to update <em>%(filename)s</em> from "
u"version control: %(error)s", {
'filename': store.file.name,
'error': e,
})
messages.error(request, msg)
self.scan_files()
def commit_dir(self, user, directory, request=None):
"""Commits files under a directory to version control.
This does not do permission checking.
"""
self.sync()
stats = self.getquickstats()
author = user.username
message = stats_message_raw(
"Commit from %s by user %s." % (settings.TITLE, author), stats)
# Try to append email as well, since some VCS does not allow omitting
# it (ie. Git).
if user.is_authenticated() and len(user.email):
author += " <%s>" % user.email
if directory.is_translationproject():
stores = list(self.stores.exclude(file=""))
else:
stores = list(directory.stores.exclude(file=""))
filestocommit = []
from pootle.scripts import hooks
for store in stores:
try:
filestocommit.extend(
hooks.hook(self.project.code,
"precommit",
store.file.name,
author=author,
message=message))
except ImportError:
# Failed to import the hook - we're going to assume there just
# isn't a hook to import. That means we'll commit the original
# file.
filestocommit.append(store.file.name)
success = True
try:
from pootle_misc import versioncontrol
project_path = self.project.get_real_path()
versioncontrol.add_files(project_path, filestocommit, message,
author)
# FIXME: This belongs to views
if request is not None:
msg = _(
"Committed all files under <em>%(path)s</em> to "
"version control", {'path': directory.pootle_path})
messages.success(request, msg)
except Exception as e:
logging.error(u"Failed to commit: %s", e)
# FIXME: This belongs to views
if request is not None:
msg = _("Failed to commit to version control: %(error)s",
{'error': e})
messages.error(request, msg)
success = False
for store in stores:
try:
hooks.hook(self.project.code,
"postcommit",
store.file.name,
success=success)
except:
#FIXME: We should not hide the exception - makes development
# impossible
pass
from pootle_app.models.signals import post_vc_commit
post_vc_commit.send(sender=self,
path_obj=directory,
stats=stats,
user=user,
success=success)
return success
def commit_file(self, user, store, request=None):
"""Commits an individual file to version control.
This does not do permission checking.
"""
store.sync(update_structure=False,
update_translation=True,
conservative=True)
stats = store.getquickstats()
author = user.username
message = stats_message_raw("Commit from %s by user %s." % \
(settings.TITLE, author), stats)
# Try to append email as well, since some VCS does not allow omitting
# it (ie. Git).
if user.is_authenticated() and len(user.email):
author += " <%s>" % user.email
from pootle.scripts import hooks
try:
filestocommit = hooks.hook(self.project.code,
"precommit",
store.file.name,
author=author,
message=message)
except ImportError:
# Failed to import the hook - we're going to assume there just
# isn't a hook to import. That means we'll commit the original
# file.
filestocommit = [store.file.name]
success = True
try:
from pootle_misc import versioncontrol
for file in filestocommit:
versioncontrol.commit_file(file,
message=message,
author=author)
# FIXME: This belongs to views
if request is not None:
msg = _(
"Committed file <em>%(filename)s</em> to version "
"control", {'filename': file})
messages.success(request, msg)
except Exception as e:
logging.error(u"Failed to commit file: %s", e)
# FIXME: This belongs to views
if request is not None:
msg_params = {
'filename': filename,
'error': e,
}
msg = _(
"Failed to commit <em>%(filename)s</em> to version "
"control: %(error)s", msg_params)
messages.error(request, msg)
success = False
try:
hooks.hook(self.project.code,
"postcommit",
store.file.name,
success=success)
except:
#FIXME: We should not hide the exception - makes development
# impossible
pass
from pootle_app.models.signals import post_vc_commit
post_vc_commit.send(sender=self,
path_obj=store,
stats=stats,
user=user,
success=success)
return success
def initialize(self):
try:
from pootle.scripts import hooks
hooks.hook(self.project.code, "initialize", self.real_path,
self.language.code)
except Exception as e:
logging.error(u"Failed to initialize (%s): %s", self.language.code,
e)
###########################################################################
def get_archive(self, stores, path=None):
"""Returns an archive of the given files."""
import shutil
from pootle_misc import ptempfile as tempfile
tempzipfile = None
try:
# Using zip command line is fast
# The temporary file below is opened and immediately closed for
# security reasons
fd, tempzipfile = tempfile.mkstemp(prefix='pootle', suffix='.zip')
os.close(fd)
file_list = u" ".join(
store.abs_real_path[len(self.abs_real_path)+1:] \
for store in stores.iterator()
)
cmd = u"cd %(path)s ; zip -r - %(file_list)s > %(tmpfile)s" % {
'path': self.abs_real_path,
'file_list': file_list,
'tmpfile': tempzipfile,
}
result = os.system(cmd.encode('utf-8'))
if result == 0:
if path is not None:
shutil.move(tempzipfile, path)
return
else:
filedata = open(tempzipfile, "r").read()
if filedata:
return filedata
finally:
if tempzipfile is not None and os.path.exists(tempzipfile):
os.remove(tempzipfile)
# But if it doesn't work, we can do it from python
archivecontents = None
try:
if path is not None:
fd, tempzipfile = tempfile.mkstemp(prefix='pootle',
suffix='.zip')
os.close(fd)
archivecontents = open(tempzipfile, "wb")
else:
import cStringIO
archivecontents = cStringIO.StringIO()
import zipfile
archive = zipfile.ZipFile(archivecontents, 'w',
zipfile.ZIP_DEFLATED)
for store in stores.iterator():
archive.write(
store.abs_real_path.encode('utf-8'),
store.abs_real_path[len(self.abs_real_path) +
1:].encode('utf-8'))
archive.close()
if path is not None:
shutil.move(tempzipfile, path)
else:
return archivecontents.getvalue()
finally:
if tempzipfile is not None and os.path.exists(tempzipfile):
os.remove(tempzipfile)
try:
archivecontents.close()
except:
pass
###########################################################################
def make_indexer(self):
"""Get an indexing object for this project.
Since we do not want to keep the indexing databases open for the
lifetime of the TranslationProject (it is cached!), it may NOT be
part of the Project object, but should be used via a short living
local variable.
"""
logging.debug(u"Loading indexer for %s", self.pootle_path)
indexdir = os.path.join(self.abs_real_path, self.index_directory)
from translate.search import indexing
indexer = indexing.get_indexer(indexdir)
indexer.set_field_analyzers({
"pofilename": indexer.ANALYZER_EXACT,
"pomtime": indexer.ANALYZER_EXACT,
"dbid": indexer.ANALYZER_EXACT,
})
return indexer
def init_index(self, indexer):
"""Initializes the search index."""
#FIXME: stop relying on pomtime so virtual files can be searchable?
try:
indexer.begin_transaction()
for store in self.stores.iterator():
try:
self.update_index(indexer, store)
except OSError as e:
# Broken link or permission problem?
logging.error("Error indexing %s: %s", store, e)
indexer.commit_transaction()
indexer.flush(optimize=True)
except Exception as e:
logging.error(u"Error opening indexer for %s:\n%s", self, e)
try:
indexer.cancel_transaction()
except:
pass
def update_index(self, indexer, store, unitid=None):
"""Updates the index with the contents of store (limit to
``unitid`` if given).
There are two reasons for calling this function:
1. Creating a new instance of :cls:`TranslationProject`
(see :meth:`TranslationProject.init_index`)
-> Check if the index is up-to-date / rebuild the index if
necessary
2. Translating a unit via the web interface
-> (re)index only the specified unit(s)
The argument ``unitid`` should be None for 1.
Known problems:
1. This function should get called, when the po file changes
externally.
WARNING: You have to stop the pootle server before manually
changing po files, if you want to keep the index database in
sync.
"""
#FIXME: leverage file updated signal to check if index needs updating
if indexer is None:
return False
# Check if the pomtime in the index == the latest pomtime
pomtime = str(hash(store.get_mtime())**2)
pofilenamequery = indexer.make_query(
[("pofilename", store.pootle_path)], True)
pomtimequery = indexer.make_query([("pomtime", pomtime)], True)
gooditemsquery = indexer.make_query([pofilenamequery, pomtimequery],
True)
gooditemsnum = indexer.get_query_result(gooditemsquery) \
.get_matches_count()
# If there is at least one up-to-date indexing item, then the po file
# was not changed externally -> no need to update the database
units = None
if (gooditemsnum > 0) and (not unitid):
# Nothing to be done
return
elif unitid is not None:
# Update only specific item - usually translation via the web
# interface. All other items should still be up-to-date (even with
# an older pomtime).
# Delete the relevant item from the database
units = store.units.filter(id=unitid)
itemsquery = indexer.make_query([("dbid", str(unitid))], False)
indexer.delete_doc([pofilenamequery, itemsquery])
else:
# (item is None)
# The po file is not indexed - or it was changed externally
# delete all items of this file
logging.debug(u"Updating %s indexer for file %s", self.pootle_path,
store.pootle_path)
indexer.delete_doc({"pofilename": store.pootle_path})
units = store.units
addlist = []
for unit in units.iterator():
doc = {
"pofilename": store.pootle_path,
"pomtime": pomtime,
"dbid": str(unit.id),
}
if unit.hasplural():
orig = "\n".join(unit.source.strings)
trans = "\n".join(unit.target.strings)
else:
orig = unit.source
trans = unit.target
doc.update({
"source": orig,
"target": trans,
"notes": unit.getnotes(),
"locations": unit.getlocations(),
})
addlist.append(doc)
if addlist:
for add_item in addlist:
indexer.index_document(add_item)
###########################################################################
def gettermmatcher(self):
"""Returns the terminology matcher."""
terminology_stores = Store.objects.none()
mtime = None
if self.is_terminology_project:
terminology_stores = self.stores.all()
mtime = self.get_mtime()
else:
# Get global terminology first
try:
termproject = TranslationProject.objects.get(
language=self.language_id,
project__code='terminology',
)
mtime = termproject.get_mtime()
terminology_stores = termproject.stores.all()
except TranslationProject.DoesNotExist:
pass
local_terminology = self.stores.filter(
name__startswith='pootle-terminology')
for store in local_terminology.iterator():
if mtime is None:
mtime = store.get_mtime()
else:
mtime = max(mtime, store.get_mtime())
terminology_stores = terminology_stores | local_terminology
if mtime is None:
return
if mtime != self.non_db_state.termmatchermtime:
from translate.search import match
self.non_db_state.termmatcher = match.terminologymatcher(
terminology_stores.iterator(), )
self.non_db_state.termmatchermtime = mtime
return self.non_db_state.termmatcher
###########################################################################
#FIXME: we should cache results to ease live translation
def translate_message(self, singular, plural=None, n=1):
for store in self.stores.iterator():
unit = store.findunit(singular)
if unit is not None and unit.istranslated():
if unit.hasplural() and n != 1:
pluralequation = self.language.pluralequation
if pluralequation:
pluralfn = gettext.c2py(pluralequation)
target = unit.target.strings[pluralfn(n)]
if target is not None:
return target
else:
return unit.target
# No translation found
if n != 1 and plural is not None:
return plural
else:
return singular
class TranslationStoreFieldFile(FieldFile):
"""FieldFile is the file-like object of a FileField, that is found in a
TranslationStoreField."""
from translate.misc.lru import LRUCachingDict
from django.conf import settings
_store_cache = LRUCachingDict(settings.PARSE_POOL_SIZE,
settings.PARSE_POOL_CULL_FREQUENCY)
def getpomtime(self):
file_stat = os.stat(self.realpath)
return file_stat.st_mtime, file_stat.st_size
@property
def filename(self):
return os.path.basename(self.name)
def _get_realpath(self):
"""Return realpath resolving symlinks if necessary."""
if not hasattr(self, "_realpath"):
self._realpath = os.path.realpath(self.path)
return self._realpath
@property
def realpath(self):
"""Get real path from cache before attempting to check for symlinks."""
if not hasattr(self, "_store_tuple"):
return self._get_realpath()
else:
return self._store_tuple.realpath
@property
def store(self):
"""Get translation store from dictionary cache, populate if store not
already cached."""
self._update_store_cache()
return self._store_tuple.store
def _update_store_cache(self):
"""Add translation store to dictionary cache, replace old cached
version if needed."""
mod_info = self.getpomtime()
if (not hasattr(self, "_store_tuple")
or self._store_tuple.mod_info != mod_info):
try:
self._store_tuple = self._store_cache[self.path]
if self._store_tuple.mod_info != mod_info:
# if file is modified act as if it doesn't exist in cache
raise KeyError
except KeyError:
logging.debug(u"Cache miss for %s", self.path)
from translate.storage import factory
from pootle_store.filetypes import factory_classes
store_obj = factory.getobject(self.path,
ignore=self.field.ignore,
classes=factory_classes)
self._store_tuple = StoreTuple(store_obj, mod_info,
self.realpath)
self._store_cache[self.path] = self._store_tuple
translation_file_updated.send(sender=self, path=self.path)
def _touch_store_cache(self):
"""Update stored mod_info without reparsing file."""
if hasattr(self, "_store_tuple"):
mod_info = self.getpomtime()
if self._store_tuple.mod_info != mod_info:
self._store_tuple.mod_info = mod_info
translation_file_updated.send(sender=self, path=self.path)
else:
#FIXME: do we really need that?
self._update_store_cache()
def _delete_store_cache(self):
"""Remove translation store from cache."""
try:
del self._store_cache[self.path]
except KeyError:
pass
try:
del self._store_tuple
except AttributeError:
pass
translation_file_updated.send(sender=self, path=self.path)
def exists(self):
return os.path.exists(self.realpath)
def savestore(self):
"""Saves to temporary file then moves over original file. This
way we avoid the need for locking."""
import shutil
from pootle_misc import ptempfile as tempfile
tmpfile, tmpfilename = tempfile.mkstemp(suffix=self.filename)
os.close(tmpfile)
self.store.savefile(tmpfilename)
#### HACK WHICH GLOBS MSGSTR ONTO SINGLE LINE
with open(tmpfilename, 'r') as f:
text = f.read()
new_text = re.sub(r'msgid ".+"\nmsgstr ".*"\n(?:".*"\n)*',
lambda m: m.group(0).replace('"\n"', ''), text)
if new_text != text:
with open(tmpfilename, 'w') as f:
f.write(new_text)
####
shutil.move(tmpfilename, self.realpath)
self._touch_store_cache()
def save(self, name, content, save=True):
#FIXME: implement save to tmp file then move instead of directly saving
super(TranslationStoreFieldFile, self).save(name, content, save)
self._delete_store_cache()
def delete(self, save=True):
self._delete_store_cache()
if save:
super(TranslationStoreFieldFile, self).delete(save)