def extract_lang(project, locale, path, entities=False):
"""Extract .lang file with path and save or update in DB."""
lang = parse_lang(path)
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path, format='lang')
if entities:
for order, (key, value) in enumerate(lang):
save_entity(
resource=resource, string=key, comment=value[1], order=order)
update_entity_count(resource)
else:
for key, value in lang:
if key != value[2] or '{ok}' in value[3]:
try:
e = Entity.objects.get(resource=resource, string=key)
save_translation(
entity=e, locale=locale, string=value[2])
except Entity.DoesNotExist:
continue
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
def extract_ini(project, path):
"""Extract .ini file with path and save or update in DB."""
config = configparser.ConfigParser()
with codecs.open(path, 'r', 'utf-8') as f:
try:
config.read_file(f)
except Exception as e:
log.debug("INI configparser: " + str(e))
sections = config.sections()
source_locale = None
for s in ('templates', 'en-US', 'en-GB', 'en'):
if s in sections:
source_locale = s
break
if source_locale is None:
log.error("Unable to detect source locale")
raise Exception("error")
# Move source locale on top, so we save entities first, then translations
sections.insert(0, sections.pop(sections.index(source_locale)))
resource, created = Resource.objects.get_or_create(project=project,
path=path,
format='ini')
for section in sections:
try:
locale = Locale.objects.get(code=section)
except Locale.DoesNotExist:
log.debug("Locale not supported: " + section)
break
order = 0
for item in config.items(section):
if section == source_locale:
save_entity(resource=resource,
string=item[1],
key=item[0],
order=order)
order += 1
else:
try:
e = Entity.objects.get(resource=resource, key=item[0])
save_translation(entity=e, locale=locale, string=item[1])
except Entity.DoesNotExist:
log.debug("[" + section + "]: line ID " + item[0] +
" is obsolete.")
continue
if section == source_locale:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + section + "]: saved to DB.")
def extract_ini(project, path):
"""Extract .ini file with path and save or update in DB."""
config = configparser.ConfigParser()
with codecs.open(path, 'r', 'utf-8') as f:
try:
config.read_file(f)
except Exception as e:
log.debug("INI configparser: " + str(e))
sections = config.sections()
source_locale = None
for s in ('templates', 'en-US', 'en-GB', 'en'):
if s in sections:
source_locale = s
break
if source_locale is None:
log.error("Unable to detect source locale")
raise Exception("error")
# Move source locale on top, so we save entities first, then translations
sections.insert(0, sections.pop(sections.index(source_locale)))
resource, created = Resource.objects.get_or_create(
project=project, path=path, format='ini')
for section in sections:
try:
locale = Locale.objects.get(code=section)
except Locale.DoesNotExist:
log.debug("Locale not supported: " + section)
break
order = 0
for item in config.items(section):
if section == source_locale:
save_entity(resource=resource, string=item[1],
key=item[0], order=order)
order += 1
else:
try:
e = Entity.objects.get(
resource=resource, key=item[0])
save_translation(
entity=e, locale=locale, string=item[1])
except Entity.DoesNotExist:
log.debug("[" + section + "]: line ID " +
item[0] + " is obsolete.")
continue
if section == source_locale:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + section + "]: saved to DB.")
def update_project_stats(db_project, vcs_project, changeset, locale):
"""Update the Stats entries in the database."""
for resource in db_project.resources.all():
# We only want to create/update the stats object if the resource
# exists in the current locale, UNLESS the file is asymmetric.
vcs_resource = vcs_project.resources[resource.path]
resource_exists = vcs_resource.files.get(locale) is not None
if resource_exists or resource.is_asymmetric:
update_stats(resource, locale)
def extract_inc(project, locale, path, entities=False):
"""Extract .inc file with path and save or update in DB."""
with codecs.open(path, 'r', 'utf-8', errors='replace') as inc_file:
comment = []
order = 0
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path, format='inc')
for line in inc_file:
# Uncomment MOZ_LANGPACK_CONTRIBUTORS (commented out)
# http://hg.mozilla.org/releases/mozilla-aurora/file/572c8f4c8fed/browser/locales/en-US/defines.inc#l10
if entities and line.startswith('# #define'):
line = line.lstrip('#').strip()
# Comments
if entities and line.startswith('# '):
comment.append(line.lstrip('# ').strip())
# Strings
elif line.startswith('#define'):
parts = line.lstrip('#define').strip().split(None, 1)
if not parts:
continue
if len(parts) == 1:
key, string = parts[0], ""
else:
key, string = parts
if entities:
save_entity(resource=resource, string=string, key=key,
comment=" ".join(comment), order=order)
comment = []
order += 1
else:
try:
e = Entity.objects.get(resource=resource, key=key)
save_translation(
entity=e, locale=locale, string=string)
except Entity.DoesNotExist:
continue
elif entities:
comment = []
if entities:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
def update_project_stats(db_project, vcs_project, changeset, locale):
"""Update the Stats entries in the database."""
for resource in db_project.resources.all():
# We only want to create/update the stats object if the resource
# exists in the current locale, UNLESS the file is asymmetric.
vcs_resource = vcs_project.resources.get(resource.path, None)
if vcs_resource is not None:
resource_exists = vcs_resource.files.get(locale) is not None
if resource_exists or resource.is_asymmetric:
update_stats(resource, locale)
def extract_silme(parser, project, locale, path, entities=False):
"""Extract file with path using silme and save or update in DB."""
try:
f = open(path)
structure = parser.get_structure(f.read())
format = str(parser).split('.')[-1].split('Format')[0].lower()
comment = ""
order = 0
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(project=project,
path=relative_path,
format=format)
for obj in structure:
if isinstance(obj, silme.core.entity.Entity):
if entities:
save_entity(resource=resource,
string=obj.value,
key=obj.id,
comment=comment,
order=order)
comment = ""
order += 1
else:
try:
e = Entity.objects.get(resource=resource, key=obj.id)
save_translation(entity=e,
locale=locale,
string=obj.value)
except Entity.DoesNotExist:
continue
elif isinstance(obj, silme.core.structure.Comment):
if entities:
comment = str(obj)
if entities:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
f.close()
except IOError:
log.debug("[" + locale.code + "]: " + path +
" doesn't exist. Skipping.")
def extract_properties(project, locale, paths, entities=False):
"""Extract .properties files from paths and save or update in DB."""
parser = silme.format.properties.PropertiesFormatParser
for path in paths:
try:
f = open(path)
structure = parser.get_structure(f.read())
comment = ""
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path)
for obj in structure:
if isinstance(obj, silme.core.entity.Entity):
if entities:
save_entity(resource=resource, string=obj.value,
key=obj.id, comment=comment)
comment = ""
else:
try:
e = Entity.objects.get(
resource=resource,
key=obj.id)
save_translation(
entity=e,
locale=locale,
string=obj.value)
except Entity.DoesNotExist:
continue
elif isinstance(obj, silme.core.structure.Comment):
if entities:
comment = str(obj)
if entities:
update_entity_count(resource, project)
else:
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
f.close()
except IOError:
log.debug("[" + locale.code + "]: " +
path + " doesn't exist. Skipping.")
def extract_silme(parser, project, locale, path, entities=False):
"""Extract file with path using silme and save or update in DB."""
try:
f = open(path)
structure = parser.get_structure(f.read())
format = str(parser).split('.')[-1].split('Format')[0].lower()
comment = ""
order = 0
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path, format=format)
for obj in structure:
if isinstance(obj, silme.core.entity.Entity):
if entities:
save_entity(resource=resource, string=obj.value,
key=obj.id, comment=comment, order=order)
comment = ""
order += 1
else:
try:
e = Entity.objects.get(resource=resource, key=obj.id)
save_translation(
entity=e,
locale=locale,
string=obj.value)
except Entity.DoesNotExist:
continue
elif isinstance(obj, silme.core.structure.Comment):
if entities:
comment = str(obj)
if entities:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
f.close()
except IOError:
log.debug("[" + locale.code + "]: " +
path + " doesn't exist. Skipping.")
def extract_silme(parser, project, locale, path, entities=False):
"""Extract file with path using silme and save or update in DB."""
with codecs.open(path, 'r', 'utf-8') as f:
structure = parser.get_structure(f.read())
format = str(parser).split('.')[2]
comment = ""
order = 0
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(project=project,
path=relative_path,
format=format)
for obj in structure:
if isinstance(obj, silme.core.entity.Entity):
if entities:
save_entity(resource=resource,
string=obj.value,
key=obj.id,
comment=comment,
order=order)
comment = ""
order += 1
else:
try:
e = Entity.objects.get(resource=resource, key=obj.id)
save_translation(entity=e,
locale=locale,
string=obj.value)
except Entity.DoesNotExist:
continue
elif isinstance(obj, silme.core.structure.Comment):
if entities:
comment = ''.join(unicode(i) for i in obj)
if entities:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
def extract_silme(parser, project, locale, path, entities=False):
"""Extract file with path using silme and save or update in DB."""
with codecs.open(path, 'r', 'utf-8') as f:
structure = parser.get_structure(f.read())
format = str(parser).split('.')[2]
comment = ""
order = 0
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path, format=format)
for obj in structure:
if isinstance(obj, silme.core.entity.Entity):
if entities:
save_entity(resource=resource, string=obj.value,
key=obj.id, comment=comment, order=order)
comment = ""
order += 1
else:
try:
e = Entity.objects.get(resource=resource, key=obj.id)
save_translation(
entity=e,
locale=locale,
string=obj.value)
except Entity.DoesNotExist:
continue
elif isinstance(obj, silme.core.structure.Comment):
if entities:
comment = ''.join(unicode(i) for i in obj)
if entities:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
def extract_xliff(project, locale, path, entities=False):
"""Extract .xliff file with path and save or update in DB."""
with open(path) as f:
xf = xliff.xlifffile(f)
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(project=project,
path=relative_path,
format='xliff')
if entities:
for order, unit in enumerate(xf.units):
save_entity(resource=resource,
string=unicode(unit.get_rich_source()[0]),
key=unit.getid(),
comment=unit.getnotes(),
order=order)
update_entity_count(resource)
else:
for unit in xf.units:
translation = unicode(unit.get_rich_target()[0])
if translation:
try:
e = Entity.objects.get(resource=resource,
key=unit.getid())
save_translation(entity=e,
locale=locale,
string=translation)
except Entity.DoesNotExist:
continue
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
def extract_xliff(project, locale, path, entities=False):
"""Extract .xliff file with path and save or update in DB."""
with open(path) as f:
xf = xliff.xlifffile(f)
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path, format='xliff')
if entities:
for order, unit in enumerate(xf.units):
save_entity(
resource=resource,
string=unicode(unit.get_rich_source()[0]),
key=unit.getid(),
comment=unit.getnotes(),
order=order)
update_entity_count(resource)
else:
for unit in xf.units:
translation = unicode(unit.get_rich_target()[0])
if translation:
try:
e = Entity.objects.get(
resource=resource, key=unit.getid())
save_translation(
entity=e, locale=locale, string=translation)
except Entity.DoesNotExist:
continue
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
def extract_l20n(project, locale, path, entities=False):
"""Extract .l20n file with path and save or update in DB."""
parser = L20nParser.L20nParser()
with codecs.open(path, 'r', 'utf-8') as f:
structure = parser.parse(f.read())
comment = ""
order = 0
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path, format='l20n')
for obj in structure.body:
# Entities
if obj.type == "Entity":
# Simple
if obj.value.type == "String":
key = obj.id.name
string = obj.value.source
store_l20n(
entities, resource, key, string, comment, locale,
order)
comment = ""
order += 1
# Plurals
elif obj.value.type == "Hash":
key = obj.id.name
string_plural = ""
plural_form = None
# Get strings
for item in obj.value.items:
if entities:
if item.id.name == "one":
string = item.value.source
elif item.id.name == "many":
string_plural = item.value.source
else:
string = item.value.source
idx = Locale.cldr_plural_to_id(item.id.name)
plural_form = locale.cldr_plurals_list().index(idx)
store_l20n(
entities, resource, key, string, comment, locale,
order, string_plural, plural_form)
comment = ""
order += 1
# Attributes
for attr in obj.attrs:
key = ".".join([obj.id.name, attr.id.name])
string = attr.value.source
store_l20n(
entities, resource, key, string, comment, locale,
order)
comment = ""
order += 1
# Comments
elif obj.type == "Comment":
comment = obj.body
if entities:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
def handle_upload_content(slug, code, part, f, user):
"""
Update translations in the database from uploaded file.
:param str slug: Project slug.
:param str code: Locale code.
:param str part: Resource path or Subpage name.
:param UploadedFile f: UploadedFile instance.
:param User user: User uploading the file.
"""
# Avoid circular import; someday we should refactor to avoid.
from pontoon.sync import formats
from pontoon.sync.changeset import ChangeSet
from pontoon.sync.vcs_models import VCSProject
from pontoon.base.models import (
ChangedEntityLocale,
Entity,
Locale,
Project,
Resource,
Translation,
update_stats,
)
relative_path = _get_relative_path_from_part(slug, part)
project = get_object_or_404(Project, slug=slug)
locale = get_object_or_404(Locale, code__iexact=code)
resource = get_object_or_404(Resource,
project__slug=slug,
path=relative_path)
# Store uploaded file to a temporary file and parse it
extension = os.path.splitext(f.name)[1]
with tempfile.NamedTemporaryFile(suffix=extension) as temp:
for chunk in f.chunks():
temp.write(chunk)
temp.flush()
resource_file = formats.parse(temp.name)
# Update database objects from file
changeset = ChangeSet(project, VCSProject(project, locales=[locale]),
timezone.now())
entities_qs = Entity.objects.filter(
resource__project=project,
resource__path=relative_path,
obsolete=False).prefetch_related(
Prefetch('translation_set',
queryset=Translation.objects.filter(locale=locale),
to_attr='db_translations')).prefetch_related(
Prefetch('translation_set',
queryset=Translation.objects.filter(
locale=locale,
approved_date__lte=timezone.now()),
to_attr='old_translations'))
entities_dict = {entity.key: entity for entity in entities_qs}
for vcs_translation in resource_file.translations:
key = vcs_translation.key
if key in entities_dict:
entity = entities_dict[key]
changeset.update_entity_translations_from_vcs(
entity, locale.code, vcs_translation, user,
entity.db_translations, entity.old_translations)
changeset.bulk_create_translations()
changeset.bulk_update_translations()
update_stats(resource, locale)
# Mark translations as changed
changed_entities = {}
existing = ChangedEntityLocale.objects.values_list('entity',
'locale').distinct()
for t in changeset.translations_to_create + changeset.translations_to_update:
key = (t.entity.pk, t.locale.pk)
# Remove duplicate changes to prevent unique constraint violation
if not key in existing:
changed_entities[key] = ChangedEntityLocale(entity=t.entity,
locale=t.locale)
ChangedEntityLocale.objects.bulk_create(changed_entities.values())
def extract_po(project, locale, path, entities=False):
"""Extract .po (gettext) file with path and save or update in DB."""
try:
po = polib.pofile(path)
relative_path = get_relative_path(path, locale)
if relative_path[-1] == 't':
relative_path = relative_path[:-1]
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path, format='po')
if entities:
for order, entry in enumerate(po):
if not entry.obsolete:
save_entity(resource=resource,
string=entry.msgid,
string_plural=entry.msgid_plural,
comment=entry.comment,
order=order,
source=entry.occurrences)
update_entity_count(resource)
else:
for entry in (po.translated_entries() + po.fuzzy_entries()):
if not entry.obsolete:
# Entities without plurals
if len(entry.msgstr) > 0:
try:
e = Entity.objects.get(
resource=resource,
string=entry.msgid)
save_translation(
entity=e,
locale=locale,
string=entry.msgstr,
fuzzy='fuzzy' in entry.flags)
except Entity.DoesNotExist:
continue
# Pluralized entities
elif len(entry.msgstr_plural) > 0:
try:
e = Entity.objects.get(
resource=resource,
string=entry.msgid)
for k in entry.msgstr_plural:
save_translation(
entity=e,
locale=locale,
string=entry.msgstr_plural[k],
plural_form=k,
fuzzy='fuzzy' in entry.flags)
except Entity.DoesNotExist:
continue
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
except Exception as e:
log.critical('PoExtractError for %s: %s' % (path, e))
def handle_upload_content(slug, code, part, f, user):
"""
Update translations in the database from uploaded file.
:param str slug: Project slug.
:param str code: Locale code.
:param str part: Resource path or Subpage name.
:param UploadedFile f: UploadedFile instance.
:param User user: User uploading the file.
"""
# Avoid circular import; someday we should refactor to avoid.
from pontoon.sync import formats
from pontoon.sync.changeset import ChangeSet
from pontoon.sync.vcs_models import VCSProject
from pontoon.base.models import (
ChangedEntityLocale,
Entity,
Locale,
Project,
Resource,
Translation,
update_stats,
)
relative_path = _get_relative_path_from_part(slug, part)
project = get_object_or_404(Project, slug=slug)
locale = get_object_or_404(Locale, code__iexact=code)
resource = get_object_or_404(Resource, project__slug=slug, path=relative_path)
# Store uploaded file to a temporary file and parse it
extension = os.path.splitext(f.name)[1]
with tempfile.NamedTemporaryFile(suffix=extension) as temp:
for chunk in f.chunks():
temp.write(chunk)
temp.flush()
resource_file = formats.parse(temp.name)
# Update database objects from file
changeset = ChangeSet(
project,
VCSProject(project, locales=[locale]),
timezone.now()
)
entities_qs = Entity.objects.filter(
resource__project=project,
resource__path=relative_path,
obsolete=False
).prefetch_related(
Prefetch(
'translation_set',
queryset=Translation.objects.filter(locale=locale),
to_attr='db_translations'
)
).prefetch_related(
Prefetch(
'translation_set',
queryset=Translation.objects.filter(locale=locale, approved_date__lte=timezone.now()),
to_attr='old_translations'
)
)
entities_dict = {entity.key: entity for entity in entities_qs}
for vcs_translation in resource_file.translations:
key = vcs_translation.key
if key in entities_dict:
entity = entities_dict[key]
changeset.update_entity_translations_from_vcs(
entity, locale.code, vcs_translation, user,
entity.db_translations, entity.old_translations
)
changeset.bulk_create_translations()
changeset.bulk_update_translations()
update_stats(resource, locale)
# Mark translations as changed
changed_entities = {}
existing = ChangedEntityLocale.objects.values_list('entity', 'locale').distinct()
for t in changeset.translations_to_create + changeset.translations_to_update:
key = (t.entity.pk, t.locale.pk)
# Remove duplicate changes to prevent unique constraint violation
if not key in existing:
changed_entities[key] = ChangedEntityLocale(entity=t.entity, locale=t.locale)
ChangedEntityLocale.objects.bulk_create(changed_entities.values())