def locales(self):
"""
Yield an iterable of Locales whose strings are stored within
this repo.
"""
from pontoon.sync.utils import locale_directory_path
locales = [] # Use list since we're caching the result.
for locale in self.project.locales.all():
try:
locale_directory_path(self.checkout_path, locale.code)
locales.append(locale)
except IOError:
pass # Directory missing, not in this repo.
return locales
def test_obsolete_vcs_resources_paths(self):
"""Tests if we remove obsolete resources"""
obsolete_vcs_resources = [
'obsolete.properties',
'second_obsolete.properties',
]
obsolete_paths = [
os.path.join(
locale_directory_path(self.vcs_project.checkout_path,
self.translated_locale.code), path)
for path in obsolete_vcs_resources
]
# We test if we remove only resources available on disk
self.changeset.changes['obsolete_vcs_resources'] = obsolete_paths + [
'second_obsolete.properties'
]
with patch('os.path.exists', side_effect=lambda p: p in obsolete_paths[:1]) as exists_mock,\
patch('os.remove') as remove_mock:
self.changeset.execute_obsolete_vcs_resources()
exists_mock.assert_has_calls(
[call(obsolete_paths[0]),
call(obsolete_paths[1])])
remove_mock.assert_called_once_with(obsolete_paths[0])
assert_equal(self.changeset.locales_to_commit,
{self.translated_locale})
def locales(self):
"""
Yield an iterable of Locales whose strings are stored within
this repo.
"""
from pontoon.sync.utils import locale_directory_path
locales = [] # Use list since we're caching the result.
for locale in self.project.locales.all():
try:
locale_directory_path(self.checkout_path, locale.code)
locales.append(locale)
except IOError:
pass # Directory missing, not in this repo.
return locales
def locale_directory_paths(self):
"""
A map of locale codes and their absolute directory paths.
Create locale directory, if not in repository yet.
"""
locale_directory_paths = {}
parent_directories = set()
for locale in self.locales:
try:
if self.configuration:
locale_directory_paths[locale.code] = self.configuration.l10n_base
else:
locale_directory_paths[locale.code] = locale_directory_path(
self.checkout_path, locale.code, parent_directories,
)
parent_directory = get_parent_directory(
locale_directory_paths[locale.code]
)
except IOError:
if not self.db_project.has_multi_locale_repositories:
source_directory = self.source_directory_path
parent_directory = get_parent_directory(source_directory)
locale_code = locale.code
if uses_undercore_as_separator(parent_directory):
locale_code = locale_code.replace("-", "_")
locale_directory = os.path.join(parent_directory, locale_code)
# For asymmetric formats, create empty folder
if is_asymmetric_resource(next(self.relative_resource_paths())):
os.makedirs(locale_directory)
# For other formats, copy resources from source directory
else:
shutil.copytree(source_directory, locale_directory)
for root, dirnames, filenames in scandir.walk(locale_directory):
for filename in filenames:
path = os.path.join(root, filename)
if is_resource(filename):
os.rename(path, source_to_locale_path(path))
else:
os.remove(path)
locale_directory_paths[locale.code] = locale_directory
else:
raise MissingLocaleDirectoryError(
"Directory for locale `{0}` not found".format(locale.code)
)
parent_directories.add(parent_directory)
return locale_directory_paths
def locale_directory_paths(self):
"""
A map of locale codes and their absolute directory paths.
Create locale directory, if not in repository yet.
"""
locale_directory_paths = {}
parent_directories = set()
for locale in self.locales:
try:
if self.configuration:
locale_directory_paths[locale.code] = self.configuration.l10n_base
else:
locale_directory_paths[locale.code] = locale_directory_path(
self.checkout_path,
locale.code,
parent_directories,
)
parent_directory = get_parent_directory(locale_directory_paths[locale.code])
except IOError:
if not self.db_project.has_multi_locale_repositories:
source_directory = self.source_directory_path
parent_directory = get_parent_directory(source_directory)
locale_code = locale.code
if uses_undercore_as_separator(parent_directory):
locale_code = locale_code.replace('-', '_')
locale_directory = os.path.join(parent_directory, locale_code)
# For asymmetric formats, create empty folder
if is_asymmetric_resource(next(self.relative_resource_paths())):
os.makedirs(locale_directory)
# For other formats, copy resources from source directory
else:
shutil.copytree(source_directory, locale_directory)
for root, dirnames, filenames in scandir.walk(locale_directory):
for filename in filenames:
path = os.path.join(root, filename)
if is_resource(filename):
os.rename(path, source_to_locale_path(path))
else:
os.remove(path)
locale_directory_paths[locale.code] = locale_directory
else:
raise MissingLocaleDirectoryError(
'Directory for locale `{0}` not found'.format(locale.code)
)
parent_directories.add(parent_directory)
return locale_directory_paths
def __init__(self, vcs_project, path, locales=None):
"""
Load the resource file for each enabled locale and store its
translations in VCSEntity instances.
"""
from pontoon.base.models import Locale
from pontoon.sync import formats # Avoid circular import.
self.vcs_project = vcs_project
self.path = path
self.locales = locales or []
self.files = {}
self.entities = {}
# Create entities using resources from the source directory,
source_resource_path = os.path.join(vcs_project.source_directory_path(), self.path)
source_resource_path = locale_to_source_path(source_resource_path)
source_resource_file = formats.parse(source_resource_path, locale=Locale.objects.get(code='en-US'))
for index, translation in enumerate(source_resource_file.translations):
vcs_entity = VCSEntity(
resource=self,
key=translation.key,
string=translation.source_string,
string_plural=translation.source_string_plural,
comments=translation.comments,
source=translation.source,
order=translation.order or index
)
self.entities[vcs_entity.key] = vcs_entity
# Fill in translations from the locale resources.
for locale in locales:
resource_path = os.path.join(
locale_directory_path(vcs_project.checkout_path, locale.code),
self.path
)
log.debug('Parsing resource file: %s', resource_path)
try:
resource_file = formats.parse(resource_path, source_resource_path, locale)
except (IOError, ParseError):
continue # File doesn't exist or is invalid, let's move on
self.files[locale] = resource_file
log.debug('Discovered %s translations.', len(resource_file.translations))
for translation in resource_file.translations:
try:
self.entities[translation.key].translations[locale.code] = translation
except KeyError:
# If the source is missing an entity, we consider it
# deleted and don't add it.
pass
def execute_obsolete_vcs_resources(self):
for path in self.changes['obsolete_vcs_resources']:
for locale in self.db_project.locales.all():
file_path = os.path.join(
locale_directory_path(self.vcs_project.checkout_path, locale.code),
path
)
if os.path.exists(file_path):
log.info('Removing obsolete file {} for {}.'.format(path, locale.code))
os.remove(file_path)
self.locales_to_commit.add(locale)
def locale_directories(self, repo):
"""
A map of paths to their respective locales.
"""
locales_paths = {}
for locale in self.db_project.locales.all():
path = locale_directory_path(repo.checkout_path, locale.code)[len(repo.checkout_path):].lstrip(os.sep)
locales_paths[path] = locale
return locales_paths
def execute_obsolete_vcs_resources(self):
for path in self.changes['obsolete_vcs_resources']:
for locale in self.db_project.locales.all():
file_path = os.path.join(
locale_directory_path(self.vcs_project.checkout_path,
locale.code), path)
if os.path.exists(file_path):
log.info('Removing obsolete file {} for {}.'.format(
path, locale.code))
os.remove(file_path)
self.locales_to_commit.add(locale)
def commit_changes(db_project, vcs_project, changeset, locale):
"""Commit the changes we've made back to the VCS."""
authors = changeset.commit_authors_per_locale.get(locale.code, [])
# Use the top translator for this batch as commit author, or
# the fake Pontoon user if there are no authors.
if len(authors) > 0:
commit_author = Counter(authors).most_common(1)[0][0]
else:
commit_author = User(first_name="Mozilla Pontoon", email="*****@*****.**")
commit_message = render_to_string('sync/commit_message.jinja', {
'locale': locale,
'project': db_project,
'authors': set(authors)
})
locale_path = locale_directory_path(vcs_project.checkout_path, locale.code)
repo = db_project.repository_for_path(locale_path)
repo.commit(commit_message, commit_author, locale_path)
def commit_changes(db_project, vcs_project, changeset, locale):
"""Commit the changes we've made back to the VCS."""
authors = changeset.commit_authors_per_locale.get(locale.code, [])
# Use the top translator for this batch as commit author, or
# the fake Pontoon user if there are no authors.
if len(authors) > 0:
commit_author = Counter(authors).most_common(1)[0][0]
else:
commit_author = User(first_name="Mozilla Pontoon", email="*****@*****.**")
commit_message = render_to_string('sync/commit_message.jinja', {
'locale': locale,
'project': db_project,
'authors': set(authors)
})
locale_path = locale_directory_path(vcs_project.checkout_path, locale.code)
repo = db_project.repository_for_path(locale_path)
repo.commit(commit_message, commit_author, locale_path)
def test_obsolete_vcs_resources_paths(self):
"""Tests if we remove obsolete resources"""
obsolete_vcs_resources = [
'obsolete.properties',
'second_obsolete.properties',
]
obsolete_paths = [
os.path.join(locale_directory_path(self.vcs_project.checkout_path, self.translated_locale.code), path)
for path in obsolete_vcs_resources
]
# We test if we remove only resources available on disk
self.changeset.changes['obsolete_vcs_resources'] = obsolete_paths + ['second_obsolete.properties']
with patch('os.path.exists', side_effect=lambda p: p in obsolete_paths[:1]) as exists_mock,\
patch('os.remove') as remove_mock:
self.changeset.execute_obsolete_vcs_resources()
exists_mock.assert_has_calls([
call(obsolete_paths[0]),
call(obsolete_paths[1])
])
remove_mock.assert_called_once_with(obsolete_paths[0])
assert_equal(self.changeset.locales_to_commit, {self.translated_locale})
