def extract_lang(project, locale, path, entities=False):
"""Extract .lang file with path and save or update in DB."""
lang = parse_lang(path)
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path, format='lang')
if entities:
for order, (key, value) in enumerate(lang):
save_entity(
resource=resource, string=key, comment=value[1], order=order)
update_entity_count(resource)
else:
for key, value in lang:
if key != value[2] or '{ok}' in value[3]:
try:
e = Entity.objects.get(resource=resource, string=key)
save_translation(
entity=e, locale=locale, string=value[2])
except Entity.DoesNotExist:
continue
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
def extract_ini(project, path):
"""Extract .ini file with path and save or update in DB."""
config = configparser.ConfigParser()
with codecs.open(path, 'r', 'utf-8') as f:
try:
config.read_file(f)
except Exception as e:
log.debug("INI configparser: " + str(e))
sections = config.sections()
source_locale = None
for s in ('templates', 'en-US', 'en-GB', 'en'):
if s in sections:
source_locale = s
break
if source_locale is None:
log.error("Unable to detect source locale")
raise Exception("error")
# Move source locale on top, so we save entities first, then translations
sections.insert(0, sections.pop(sections.index(source_locale)))
resource, created = Resource.objects.get_or_create(project=project,
path=path,
format='ini')
for section in sections:
try:
locale = Locale.objects.get(code=section)
except Locale.DoesNotExist:
log.debug("Locale not supported: " + section)
break
order = 0
for item in config.items(section):
if section == source_locale:
save_entity(resource=resource,
string=item[1],
key=item[0],
order=order)
order += 1
else:
try:
e = Entity.objects.get(resource=resource, key=item[0])
save_translation(entity=e, locale=locale, string=item[1])
except Entity.DoesNotExist:
log.debug("[" + section + "]: line ID " + item[0] +
" is obsolete.")
continue
if section == source_locale:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + section + "]: saved to DB.")
def extract_ini(project, path):
"""Extract .ini file with path and save or update in DB."""
config = configparser.ConfigParser()
with codecs.open(path, 'r', 'utf-8') as f:
try:
config.read_file(f)
except Exception as e:
log.debug("INI configparser: " + str(e))
sections = config.sections()
source_locale = None
for s in ('templates', 'en-US', 'en-GB', 'en'):
if s in sections:
source_locale = s
break
if source_locale is None:
log.error("Unable to detect source locale")
raise Exception("error")
# Move source locale on top, so we save entities first, then translations
sections.insert(0, sections.pop(sections.index(source_locale)))
resource, created = Resource.objects.get_or_create(
project=project, path=path, format='ini')
for section in sections:
try:
locale = Locale.objects.get(code=section)
except Locale.DoesNotExist:
log.debug("Locale not supported: " + section)
break
order = 0
for item in config.items(section):
if section == source_locale:
save_entity(resource=resource, string=item[1],
key=item[0], order=order)
order += 1
else:
try:
e = Entity.objects.get(
resource=resource, key=item[0])
save_translation(
entity=e, locale=locale, string=item[1])
except Entity.DoesNotExist:
log.debug("[" + section + "]: line ID " +
item[0] + " is obsolete.")
continue
if section == source_locale:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + section + "]: saved to DB.")
def extract_inc(project, locale, path, entities=False):
"""Extract .inc file with path and save or update in DB."""
with codecs.open(path, 'r', 'utf-8', errors='replace') as inc_file:
comment = []
order = 0
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path, format='inc')
for line in inc_file:
# Uncomment MOZ_LANGPACK_CONTRIBUTORS (commented out)
# http://hg.mozilla.org/releases/mozilla-aurora/file/572c8f4c8fed/browser/locales/en-US/defines.inc#l10
if entities and line.startswith('# #define'):
line = line.lstrip('#').strip()
# Comments
if entities and line.startswith('# '):
comment.append(line.lstrip('# ').strip())
# Strings
elif line.startswith('#define'):
parts = line.lstrip('#define').strip().split(None, 1)
if not parts:
continue
if len(parts) == 1:
key, string = parts[0], ""
else:
key, string = parts
if entities:
save_entity(resource=resource, string=string, key=key,
comment=" ".join(comment), order=order)
comment = []
order += 1
else:
try:
e = Entity.objects.get(resource=resource, key=key)
save_translation(
entity=e, locale=locale, string=string)
except Entity.DoesNotExist:
continue
elif entities:
comment = []
if entities:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
def extract_silme(parser, project, locale, path, entities=False):
"""Extract file with path using silme and save or update in DB."""
try:
f = open(path)
structure = parser.get_structure(f.read())
format = str(parser).split('.')[-1].split('Format')[0].lower()
comment = ""
order = 0
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(project=project,
path=relative_path,
format=format)
for obj in structure:
if isinstance(obj, silme.core.entity.Entity):
if entities:
save_entity(resource=resource,
string=obj.value,
key=obj.id,
comment=comment,
order=order)
comment = ""
order += 1
else:
try:
e = Entity.objects.get(resource=resource, key=obj.id)
save_translation(entity=e,
locale=locale,
string=obj.value)
except Entity.DoesNotExist:
continue
elif isinstance(obj, silme.core.structure.Comment):
if entities:
comment = str(obj)
if entities:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
f.close()
except IOError:
log.debug("[" + locale.code + "]: " + path +
" doesn't exist. Skipping.")
def extract_silme(parser, project, locale, path, entities=False):
"""Extract file with path using silme and save or update in DB."""
try:
f = open(path)
structure = parser.get_structure(f.read())
format = str(parser).split('.')[-1].split('Format')[0].lower()
comment = ""
order = 0
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path, format=format)
for obj in structure:
if isinstance(obj, silme.core.entity.Entity):
if entities:
save_entity(resource=resource, string=obj.value,
key=obj.id, comment=comment, order=order)
comment = ""
order += 1
else:
try:
e = Entity.objects.get(resource=resource, key=obj.id)
save_translation(
entity=e,
locale=locale,
string=obj.value)
except Entity.DoesNotExist:
continue
elif isinstance(obj, silme.core.structure.Comment):
if entities:
comment = str(obj)
if entities:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
f.close()
except IOError:
log.debug("[" + locale.code + "]: " +
path + " doesn't exist. Skipping.")
def extract_silme(parser, project, locale, path, entities=False):
"""Extract file with path using silme and save or update in DB."""
with codecs.open(path, 'r', 'utf-8') as f:
structure = parser.get_structure(f.read())
format = str(parser).split('.')[2]
comment = ""
order = 0
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(project=project,
path=relative_path,
format=format)
for obj in structure:
if isinstance(obj, silme.core.entity.Entity):
if entities:
save_entity(resource=resource,
string=obj.value,
key=obj.id,
comment=comment,
order=order)
comment = ""
order += 1
else:
try:
e = Entity.objects.get(resource=resource, key=obj.id)
save_translation(entity=e,
locale=locale,
string=obj.value)
except Entity.DoesNotExist:
continue
elif isinstance(obj, silme.core.structure.Comment):
if entities:
comment = ''.join(unicode(i) for i in obj)
if entities:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
def extract_silme(parser, project, locale, path, entities=False):
"""Extract file with path using silme and save or update in DB."""
with codecs.open(path, 'r', 'utf-8') as f:
structure = parser.get_structure(f.read())
format = str(parser).split('.')[2]
comment = ""
order = 0
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path, format=format)
for obj in structure:
if isinstance(obj, silme.core.entity.Entity):
if entities:
save_entity(resource=resource, string=obj.value,
key=obj.id, comment=comment, order=order)
comment = ""
order += 1
else:
try:
e = Entity.objects.get(resource=resource, key=obj.id)
save_translation(
entity=e,
locale=locale,
string=obj.value)
except Entity.DoesNotExist:
continue
elif isinstance(obj, silme.core.structure.Comment):
if entities:
comment = ''.join(unicode(i) for i in obj)
if entities:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
def extract_xliff(project, locale, path, entities=False):
"""Extract .xliff file with path and save or update in DB."""
with open(path) as f:
xf = xliff.xlifffile(f)
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(project=project,
path=relative_path,
format='xliff')
if entities:
for order, unit in enumerate(xf.units):
save_entity(resource=resource,
string=unicode(unit.get_rich_source()[0]),
key=unit.getid(),
comment=unit.getnotes(),
order=order)
update_entity_count(resource)
else:
for unit in xf.units:
translation = unicode(unit.get_rich_target()[0])
if translation:
try:
e = Entity.objects.get(resource=resource,
key=unit.getid())
save_translation(entity=e,
locale=locale,
string=translation)
except Entity.DoesNotExist:
continue
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
def extract_xliff(project, locale, path, entities=False):
"""Extract .xliff file with path and save or update in DB."""
with open(path) as f:
xf = xliff.xlifffile(f)
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path, format='xliff')
if entities:
for order, unit in enumerate(xf.units):
save_entity(
resource=resource,
string=unicode(unit.get_rich_source()[0]),
key=unit.getid(),
comment=unit.getnotes(),
order=order)
update_entity_count(resource)
else:
for unit in xf.units:
translation = unicode(unit.get_rich_target()[0])
if translation:
try:
e = Entity.objects.get(
resource=resource, key=unit.getid())
save_translation(
entity=e, locale=locale, string=translation)
except Entity.DoesNotExist:
continue
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
def extract_l20n(project, locale, path, entities=False):
"""Extract .l20n file with path and save or update in DB."""
parser = L20nParser.L20nParser()
with codecs.open(path, 'r', 'utf-8') as f:
structure = parser.parse(f.read())
comment = ""
order = 0
relative_path = get_relative_path(path, locale)
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path, format='l20n')
for obj in structure.body:
# Entities
if obj.type == "Entity":
# Simple
if obj.value.type == "String":
key = obj.id.name
string = obj.value.source
store_l20n(
entities, resource, key, string, comment, locale,
order)
comment = ""
order += 1
# Plurals
elif obj.value.type == "Hash":
key = obj.id.name
string_plural = ""
plural_form = None
# Get strings
for item in obj.value.items:
if entities:
if item.id.name == "one":
string = item.value.source
elif item.id.name == "many":
string_plural = item.value.source
else:
string = item.value.source
idx = Locale.cldr_plural_to_id(item.id.name)
plural_form = locale.cldr_plurals_list().index(idx)
store_l20n(
entities, resource, key, string, comment, locale,
order, string_plural, plural_form)
comment = ""
order += 1
# Attributes
for attr in obj.attrs:
key = ".".join([obj.id.name, attr.id.name])
string = attr.value.source
store_l20n(
entities, resource, key, string, comment, locale,
order)
comment = ""
order += 1
# Comments
elif obj.type == "Comment":
comment = obj.body
if entities:
update_entity_count(resource)
else:
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
def extract_po(project, locale, path, entities=False):
"""Extract .po (gettext) file with path and save or update in DB."""
try:
po = polib.pofile(path)
relative_path = get_relative_path(path, locale)
if relative_path[-1] == 't':
relative_path = relative_path[:-1]
resource, created = Resource.objects.get_or_create(
project=project, path=relative_path, format='po')
if entities:
for order, entry in enumerate(po):
if not entry.obsolete:
save_entity(resource=resource,
string=entry.msgid,
string_plural=entry.msgid_plural,
comment=entry.comment,
order=order,
source=entry.occurrences)
update_entity_count(resource)
else:
for entry in (po.translated_entries() + po.fuzzy_entries()):
if not entry.obsolete:
# Entities without plurals
if len(entry.msgstr) > 0:
try:
e = Entity.objects.get(
resource=resource,
string=entry.msgid)
save_translation(
entity=e,
locale=locale,
string=entry.msgstr,
fuzzy='fuzzy' in entry.flags)
except Entity.DoesNotExist:
continue
# Pluralized entities
elif len(entry.msgstr_plural) > 0:
try:
e = Entity.objects.get(
resource=resource,
string=entry.msgid)
for k in entry.msgstr_plural:
save_translation(
entity=e,
locale=locale,
string=entry.msgstr_plural[k],
plural_form=k,
fuzzy='fuzzy' in entry.flags)
except Entity.DoesNotExist:
continue
update_stats(resource, locale)
log.debug("[" + locale.code + "]: " + path + " saved to DB.")
except Exception as e:
log.critical('PoExtractError for %s: %s' % (path, e))