def findEmptyFiles(repository_path):
"""
Find empty files
"""
empty_files_list = []
file_list = extractFileList(repository_path)
for file_path in file_list:
file_parser = parser.getParser(os.path.splitext(file_path)[1])
file_parser.readFile(os.path.join(repository_path, file_path))
try:
empty_file = True
entities = file_parser.parse()
for entity in entities:
# Ignore Junk
if isinstance(entity, parser.Junk):
continue
empty_file = False
if empty_file:
empty_files_list.append(file_path)
except Exception as e:
print("Error parsing file: {}".format(file_path))
print(e)
return empty_files_list
def extractStrings(self):
"""Extract strings in files"""
# Create a list of files to analyze
self.extractFileList()
for file_path in self.file_list:
file_extension = os.path.splitext(file_path)[1]
file_name = self.getRelativePath(file_path)
if file_name.endswith("region.properties"):
continue
file_parser = parser.getParser(file_extension)
file_parser.readFile(file_path)
try:
entities = file_parser.parse()
for entity in entities:
# Ignore Junk
if isinstance(entity, parser.Junk):
continue
string_id = f"{file_name}:{entity}"
if file_extension == ".ftl":
if entity.raw_val != "":
self.strings[string_id] = entity.raw_val
# Store attributes
for attribute in entity.attributes:
attr_string_id = f"{file_name}:{entity}.{attribute}"
self.strings[attr_string_id] = attribute.raw_val
else:
self.strings[string_id] = entity.raw_val
except Exception as e:
print(f"Error parsing file: {file_path}")
print(e)
def test_license_header(self):
p = parser.getParser('foo.dtd')
p.readContents(self.resource('triple-license.dtd'))
entities = list(p.walk())
self.assert_(isinstance(entities[0], parser.Comment))
self.assertIn('MPL', entities[0].all)
e = entities[1]
self.assert_(isinstance(e, parser.Entity))
self.assertEqual(e.key, 'foo')
self.assertEqual(e.val, 'value')
self.assertEqual(len(entities), 2)
p.readContents('''\
<!-- This Source Code Form is subject to the terms of the Mozilla Public
- License, v. 2.0. If a copy of the MPL was not distributed with this file,
- You can obtain one at http://mozilla.org/MPL/2.0/. -->
<!ENTITY foo "value">
''')
entities = list(p.walk())
self.assert_(isinstance(entities[0], parser.Comment))
self.assertIn('MPL', entities[0].all)
e = entities[1]
self.assert_(isinstance(e, parser.Entity))
self.assertEqual(e.key, 'foo')
self.assertEqual(e.val, 'value')
self.assertEqual(len(entities), 2)
def testMissing(self):
self.assertTrue(os.path.isdir(self.tmp))
self.reference("""foo = fooVal
bar = barVal
eff = effVal""")
self.localized("""bar = lBar
""")
cc = ContentComparer([Observer()])
cc.compare(File(self.ref, "en-reference.properties", ""),
File(self.l10n, "l10n.properties", ""),
mozpath.join(self.tmp, "merge", "l10n.properties"))
self.assertDictEqual(
cc.observers[0].toJSON(), {
'summary': {
None: {
'changed': 1,
'changed_w': 1,
'missing': 2,
'missing_w': 2
}
},
'details': {
'l10n.properties': [{
'missingEntity': u'foo'
}, {
'missingEntity': u'eff'
}]
}
})
mergefile = mozpath.join(self.tmp, "merge", "l10n.properties")
self.assertTrue(os.path.isfile(mergefile))
p = getParser(mergefile)
p.readFile(mergefile)
[m, n] = p.parse()
self.assertEqual(map(lambda e: e.key, m), ["bar", "foo", "eff"])
def test_license_header(self):
p = parser.getParser('foo.dtd')
p.readContents(self.resource('triple-license.dtd'))
entities = list(p.walk())
self.assert_(isinstance(entities[0], parser.Comment))
self.assertIn('MPL', entities[0].all)
e = entities[1]
self.assert_(isinstance(e, parser.Entity))
self.assertEqual(e.key, 'foo')
self.assertEqual(e.val, 'value')
self.assertEqual(len(entities), 2)
p.readContents('''\
<!-- This Source Code Form is subject to the terms of the Mozilla Public
- License, v. 2.0. If a copy of the MPL was not distributed with this file,
- You can obtain one at http://mozilla.org/MPL/2.0/. -->
<!ENTITY foo "value">
''')
entities = list(p.walk())
self.assert_(isinstance(entities[0], parser.Comment))
self.assertIn('MPL', entities[0].all)
e = entities[1]
self.assert_(isinstance(e, parser.Entity))
self.assertEqual(e.key, 'foo')
self.assertEqual(e.val, 'value')
self.assertEqual(len(entities), 2)
def testMissing(self):
self.assertTrue(os.path.isdir(self.tmp))
self.reference("""foo = fooVal
bar = barVal
eff = effVal""")
self.localized("""bar = lBar
""")
cc = ContentComparer([Observer()])
cc.compare(File(self.ref, "en-reference.properties", ""),
File(self.l10n, "l10n.properties", ""),
mozpath.join(self.tmp, "merge", "l10n.properties"))
self.assertDictEqual(
cc.observers[0].toJSON(),
{'summary':
{None: {
'changed': 1,
'changed_w': 1,
'missing': 2,
'missing_w': 2
}},
'details': {
'l10n.properties': [
{'missingEntity': u'foo'},
{'missingEntity': u'eff'}]
}
})
mergefile = mozpath.join(self.tmp, "merge", "l10n.properties")
self.assertTrue(os.path.isfile(mergefile))
p = getParser(mergefile)
p.readFile(mergefile)
[m, n] = p.parse()
self.assertEqual(map(lambda e: e.key, m), ["bar", "foo", "eff"])
def handleFile(self, file_blame):
path = mozpath.normsep(file_blame['path'])
try:
parser = getParser(path)
except UserWarning:
return
self.blame[path] = {}
self.readFile(parser, path)
entities = parser.parse()
for e in entities:
if isinstance(e, Junk):
continue
entity_lines = file_blame['lines'][(e.value_position()[0] -
1):e.value_position(-1)[0]]
# ignore timezone
entity_lines.sort(key=lambda blame: -blame['date'][0])
line_blame = entity_lines[0]
user = line_blame['user']
timestamp = line_blame['date'][0] # ignore timezone
if user not in self.users:
self.users.append(user)
userid = self.users.index(user)
self.blame[path][e.key] = [userid, timestamp]
def extractStrings(file_list, repository_path):
"""Extract strings from all files."""
translations = {}
for file_name in file_list:
file_path = os.path.join(repository_path, file_name)
file_extension = os.path.splitext(file_path)[1]
file_parser = parser.getParser(file_extension)
file_parser.readFile(file_path)
try:
entities = file_parser.parse()
for entity in entities:
# Ignore Junk
if isinstance(entity, parser.Junk):
continue
string_id = "{}:{}".format(file_name, str(entity))
if file_extension == ".ftl":
if entity.raw_val is not None:
translations[string_id] = entity.raw_val
# Store attributes
for attribute in entity.attributes:
attr_string_id = "{}:{}.{}".format(
file_name, str(entity), str(attribute))
translations[attr_string_id] = attribute.raw_val
else:
translations[string_id] = entity.raw_val
except Exception as e:
print("Error parsing file: {}".format(file_name))
print(e)
return translations
def testMissing(self):
self.assertTrue(os.path.isdir(self.tmp))
self.reference("""foo = fooVal
bar = barVal
eff = effVal""")
self.localized("""bar = lBar
""")
cc = ContentComparer()
cc.set_merge_stage(os.path.join(self.tmp, "merge"))
cc.compare(File(self.ref, "en-reference.properties", ""),
File(self.l10n, "l10n.properties", ""))
self.assertDictEqual(
cc.observer.toJSON(),
{'summary':
{None: {
'changed': 1, 'missing': 2
}},
'details': {
'children': [
('l10n.properties',
{'value': {'missingEntity': [u'eff', u'foo']}}
)
]}
}
)
mergefile = os.path.join(self.tmp, "merge", "l10n.properties")
self.assertTrue(os.path.isfile(mergefile))
p = getParser(mergefile)
p.readFile(mergefile)
[m, n] = p.parse()
self.assertEqual(map(lambda e: e.key, m), ["bar", "eff", "foo"])
def testMissing(self):
self.assertTrue(os.path.isdir(self.tmp))
self.reference("""foo = fooVal
bar = barVal
eff = effVal""")
self.localized("""bar = lBar
""")
cc = ContentComparer()
cc.set_merge_stage(os.path.join(self.tmp, "merge"))
cc.compare(File(self.ref, "en-reference.properties", ""),
File(self.l10n, "l10n.properties", ""))
self.assertDictEqual(
cc.observer.toJSON(),
{'summary':
{None: {
'changed': 1, 'missing': 2
}},
'details': {
'children': [
('l10n.properties',
{'value': {'missingEntity': [u'eff', u'foo']}}
)
]}
}
)
mergefile = os.path.join(self.tmp, "merge", "l10n.properties")
self.assertTrue(os.path.isfile(mergefile))
p = getParser(mergefile)
p.readFile(mergefile)
[m, n] = p.parse()
self.assertEqual(map(lambda e: e.key, m), ["bar", "eff", "foo"])
def handleFile(self, file_blame):
path = mozpath.normsep(file_blame['path'])
try:
parser = getParser(path)
except UserWarning:
return
self.blame[path] = {}
self.readFile(parser, path)
entities = parser.parse()
for e in entities:
if isinstance(e, Junk):
continue
if e.val_span:
key_vals = [(e.key, e.val_span)]
else:
key_vals = []
if isinstance(e, FluentEntity):
key_vals += [('{}.{}'.format(e.key, attr.key), attr.val_span)
for attr in e.attributes]
for key, (val_start, val_end) in key_vals:
entity_lines = file_blame['lines'][(
e.ctx.linecol(val_start)[0] - 1):e.ctx.linecol(val_end)[0]]
# ignore timezone
entity_lines.sort(key=lambda blame: -blame['date'][0])
line_blame = entity_lines[0]
user = line_blame['user']
timestamp = line_blame['date'][0] # ignore timezone
if user not in self.users:
self.users.append(user)
userid = self.users.index(user)
self.blame[path][key] = [userid, timestamp]
def handleFile(self, file_blame):
abspath = file_blame['abspath']
try:
parser = getParser(abspath)
except UserWarning:
return
self.blame[abspath] = {}
parser.readFile(file_blame['path'])
entities, emap = parser.parse()
for e in entities:
if isinstance(e, Junk):
continue
entity_lines = file_blame['lines'][
(e.value_position()[0] - 1):e.value_position(-1)[0]
]
# ignore timezone
entity_lines.sort(key=lambda blame: -blame['date'][0])
line_blame = entity_lines[0]
user = line_blame['user']
timestamp = line_blame['date'][0] # ignore timezone
if user not in self.users:
self.users.append(user)
userid = self.users.index(user)
self.blame[abspath][e.key] = [userid, timestamp]
def testMismatchingAttributes(self):
self.reference("""
foo = Foo
bar = Bar
.tender = Attribute value
eff = Eff
""")
self.localized("""\
foo = lFoo
.obsolete = attr
bar = lBar
eff = lEff
""")
cc = ContentComparer()
cc.observers.append(Observer())
cc.compare(File(self.ref, "en-reference.ftl", ""),
File(self.l10n, "l10n.ftl", ""),
mozpath.join(self.tmp, "merge", "l10n.ftl"))
self.assertDictEqual(
cc.observers.toJSON(), {
'details': {
'l10n.ftl': [
{
'error':
u'Obsolete attribute: '
'obsolete at line 2, column 3 for foo'
},
{
'error':
u'Missing attribute: tender at line 3,'
' column 1 for bar',
},
],
},
'summary': {
None: {
'changed': 3,
'changed_w': 5,
'errors': 2
}
}
})
# validate merge results
mergepath = mozpath.join(self.tmp, "merge", "l10n.ftl")
self.assertTrue(os.path.exists(mergepath))
p = getParser(mergepath)
p.readFile(mergepath)
merged_entities = p.parse()
self.assertEqual(list(merged_entities.keys()), ["eff"])
merged_eff = merged_entities['eff']
# eff should be l10n
p.readFile(self.l10n)
l10n_entities = p.parse()
l10n_eff = l10n_entities['eff']
self.assertTrue(merged_eff.equals(l10n_eff))
def get_dtd_entities(dtd_path):
if dtd_path in dtd_entities_cache:
return dtd_entities_cache[dtd_path]
dtd_parser = parser.getParser('.dtd')
dtd_parser.readFile(dtd_path)
dtd_entities_cache[dtd_path] = dtd_parser.parse()
return dtd_entities_cache[dtd_path]
def get_dtd_entities(dtd_path):
if dtd_path in dtd_entities_cache:
return dtd_entities_cache[dtd_path]
dtd_parser = parser.getParser('.dtd')
dtd_parser.readFile(dtd_path)
dtd_entities_cache[dtd_path] = dtd_parser.parse()
return dtd_entities_cache[dtd_path]
def testMismatchingValues(self):
self.reference("""
foo = Foo
.foottr = something
bar =
.tender = Attribute value
""")
self.localized("""\
foo =
.foottr = attr
bar = lBar
.tender = localized
""")
cc = ContentComparer()
cc.observers.append(Observer())
cc.compare(File(self.ref, "en-reference.ftl", ""),
File(self.l10n, "l10n.ftl", ""),
mozpath.join(self.tmp, "merge", "l10n.ftl"))
self.assertDictEqual(
cc.observers.toJSON(), {
'details': {
'l10n.ftl': [
{
'error':
u'Missing value at line 1, column 1 for foo'
},
{
'error':
u'Obsolete value at line 3, column 7 for bar',
},
]
},
'summary': {
None: {
'errors': 2,
'warnings': 0,
'missing': 0,
'missing_w': 0,
'report': 0,
'obsolete': 0,
'changed': 2,
'changed_w': 4,
'unchanged': 0,
'unchanged_w': 0,
'keys': 0,
}
}
})
# validate merge results
mergepath = mozpath.join(self.tmp, "merge", "l10n.ftl")
self.assertTrue(os.path.exists(mergepath))
p = getParser(mergepath)
p.readFile(mergepath)
merged_entities = p.parse()
self.assertEqual(merged_entities, tuple())
def merge_channels(name, resources):
try:
parser = cl.getParser(name)
except UserWarning:
raise MergeNotSupportedError(
'Unsupported file format ({}).'.format(name))
entities = merge_resources(parser, resources)
return encode(serialize_legacy_resource(entities), parser.encoding)
def extractStrings(self):
''' Extract strings from all files '''
# If storage_mode is append, read existing translations (if available)
# before overriding them
if self.storage_mode == 'append':
file_name = '{}.json'.format(self.storage_file)
if os.path.isfile(file_name):
with open(file_name) as f:
self.translations = json.load(f)
f.close()
# Create a list of files to analyze
self.extractFileList()
for file_name in self.file_list:
file_extension = os.path.splitext(file_name)[1]
file_parser = parser.getParser(file_extension)
file_parser.readFile(file_name)
try:
entities, map = file_parser.parse()
for entity in entities:
# Ignore Junk
if isinstance(entity, parser.Junk):
continue
string_id = u'{0}:{1}'.format(
self.getRelativePath(file_name), six.text_type(entity))
if file_extension == '.ftl':
if entity.raw_val is not None:
self.translations[string_id] = entity.raw_val
# Store attributes
for attribute in entity.attributes:
attr_string_id = u'{0}:{1}.{2}'.format(
self.getRelativePath(file_name),
six.text_type(entity),
six.text_type(attribute))
self.translations[
attr_string_id] = attribute.raw_val
else:
self.translations[string_id] = entity.raw_val
except Exception as e:
print('Error parsing file: {0}'.format(file_name))
print(e)
# Remove extra strings from locale
if self.reference_locale != self.locale:
# Read the JSON cache for reference locale if available
file_name = '{}.json'.format(self.reference_storage_file)
if os.path.isfile(file_name):
with open(file_name) as f:
reference_strings = json.load(f)
f.close()
for string_id in list(self.translations.keys()):
if string_id not in reference_strings:
del (self.translations[string_id])
def read_legacy_resource(self, path):
"""Read a legacy resource and parse it into a dict."""
parser = getParser(path)
parser.readFile(path)
# Transform the parsed result which is an iterator into a dict.
return {
entity.key: entity.val for entity in parser
if entity.localized or self.enforce_translated
}
def merge_channels(name, resources):
try:
parser = cl.getParser(name)
except UserWarning:
raise MergeNotSupportedError(
'Unsupported file format ({}).'.format(name))
entities = merge_resources(parser, resources)
return encode(serialize_legacy_resource(entities), parser.encoding)
def extractStrings(self):
'''Extract strings from all files.'''
# If storage_mode is append, read existing translations (if available)
# before overriding them
if self.storage_mode == 'append':
file_name = '{}.json'.format(self.storage_file)
if os.path.isfile(file_name):
with open(file_name) as f:
self.translations = json.load(f)
f.close()
# Create a list of files to analyze
self.extractFileList()
for file_name in self.file_list:
file_extension = os.path.splitext(file_name)[1]
file_parser = parser.getParser(file_extension)
file_parser.readFile(file_name)
try:
entities = file_parser.parse()
for entity in entities:
# Ignore Junk
if isinstance(entity, parser.Junk):
continue
string_id = u'{0}:{1}'.format(
self.getRelativePath(file_name), six.text_type(entity))
if file_extension == '.ftl':
if entity.raw_val is not None:
self.translations[string_id] = entity.raw_val
# Store attributes
for attribute in entity.attributes:
attr_string_id = u'{0}:{1}.{2}'.format(
self.getRelativePath(file_name),
six.text_type(entity),
six.text_type(attribute))
self.translations[attr_string_id] = \
attribute.raw_val
else:
self.translations[string_id] = entity.raw_val
except Exception as e:
print('Error parsing file: {0}'.format(file_name))
print(e)
# Remove extra strings from locale
if self.reference_locale != self.locale:
# Read the JSON cache for reference locale if available
file_name = '{}.json'.format(self.reference_storage_file)
if os.path.isfile(file_name):
with open(file_name) as f:
reference_strings = json.load(f)
f.close()
for string_id in list(self.translations.keys()):
if string_id not in reference_strings:
del(self.translations[string_id])
def testMismatchingAttributes(self):
self.reference("""
foo = Foo
bar = Bar
.tender = Attribute value
eff = Eff
""")
self.localized("""\
foo = lFoo
.obsolete = attr
bar = lBar
eff = lEff
""")
cc = ContentComparer([Observer()])
cc.compare(File(self.ref, "en-reference.ftl", ""),
File(self.l10n, "l10n.ftl", ""),
mozpath.join(self.tmp, "merge", "l10n.ftl"))
self.assertDictEqual(
cc.observers[0].toJSON(),
{
'details': {
'l10n.ftl': [
{
'error':
u'Obsolete attribute: '
'obsolete at line 2, column 3 for foo'
},
{
'error':
u'Missing attribute: tender at line 3,'
' column 1 for bar',
},
],
},
'summary': {
None: {'changed': 3, 'changed_w': 5, 'errors': 2}
}
}
)
# validate merge results
mergepath = mozpath.join(self.tmp, "merge", "l10n.ftl")
self.assert_(os.path.exists(mergepath))
p = getParser(mergepath)
p.readFile(mergepath)
merged_entities, merged_map = p.parse()
self.assertEqual([e.key for e in merged_entities], ["eff"])
merged_eff = merged_entities[merged_map['eff']]
# eff should be l10n
p.readFile(self.l10n)
l10n_entities, l10n_map = p.parse()
l10n_eff = l10n_entities[l10n_map['eff']]
self.assertTrue(merged_eff.equals(l10n_eff))
def _test(self, content, refWarnOrErrors):
p = getParser(self.file.file)
p.readContents(content)
l10n = [e for e in p]
assert len(l10n) == 1
l10n = l10n[0]
checker = getChecker(self.file)
ref = self.refList[self.refMap[l10n.key]]
found = tuple(checker.check(ref, l10n))
self.assertEqual(found, refWarnOrErrors)
def compare(self, ref_file, l10n, merge_file, extra_tests=None):
try:
p = parser.getParser(ref_file.file)
except UserWarning:
# no comparison, XXX report?
return
try:
p.readContents(ref_file.getContents())
except Exception, e:
self.notify('error', ref_file, str(e))
return
def compare(self, ref_file, l10n, merge_file, extra_tests=None):
try:
p = parser.getParser(ref_file.file)
except UserWarning:
# no comparison, XXX report?
return
try:
p.readContents(ref_file.getContents())
except Exception, e:
self.notify('error', ref_file, str(e))
return
def _test(self, content, refWarnOrErrors):
p = parser.getParser(self.file.file)
p.readContents(content)
l10n = [e for e in p]
assert len(l10n) == 1
l10n = l10n[0]
checker = getChecker(self.file)
if checker.needs_reference:
checker.set_reference(self.refList)
ref = self.refList[l10n.key]
found = tuple(checker.check(ref, l10n))
self.assertEqual(found, refWarnOrErrors)
def get_ids_for_file(self, lang_ids, lang_files):
from compare_locales.parser import getParser
parser = getParser('foo.lang')
ids_for_file = defaultdict(list)
for lf in lang_files.keys():
f = settings.LOCALES_PATH / 'en-US' / lf
parser.readFile(str(f))
mapping = parser.parse()
for string_id in lang_ids.keys():
if string_id in mapping:
ids_for_file[lf].append(string_id)
return ids_for_file
def extractStrings(self):
"""Extract strings from all files."""
# Read strings for all locales, including the reference locale
print("Parsing locales")
for locale in self.locales:
# Create a list of files to analyze
file_list = self.extractFileList(locale)
self.translations[locale] = {}
for file_name in file_list:
file_extension = os.path.splitext(file_name)[1]
file_parser = parser.getParser(file_extension)
file_parser.readFile(file_name)
try:
entities = file_parser.parse()
for entity in entities:
# Ignore Junk
if isinstance(entity, parser.Junk):
continue
string_id = "{}:{}".format(
self.getRelativePath(file_name, locale), entity)
if file_extension == ".ftl":
if entity.raw_val is not None:
self.translations[locale][
string_id] = entity.raw_val
# Store attributes
for attribute in entity.attributes:
attr_string_id = "{}:{}.{}".format(
self.getRelativePath(file_name, locale),
entity,
attribute,
)
self.translations[locale][
attr_string_id] = attribute.raw_val
else:
self.translations[locale][
string_id] = entity.raw_val
except Exception as e:
print("Error parsing file: {}".format(file_name))
print(e)
# Remove extra strings from locale
reference_ids = list(self.translations[self.reference_locale].keys())
for locale in self.locales:
if locale == self.reference_locale:
continue
for string_id in list(self.translations[locale].keys()):
if string_id not in reference_ids:
del self.translations[locale][string_id]
def parse_file(file_path, storage):
file_extension = os.path.splitext(file_path)[1]
file_parser = parser.getParser(file_extension)
file_parser.readFile(file_path)
try:
entities, map = file_parser.parse()
for entity in entities:
# Ignore Junk
if isinstance(entity, parser.Junk):
continue
storage[unicode(entity)] = entity.val
except Exception as e:
print('Error parsing file: {}'.format(file_path))
print(e)
def compare(self, ref_file, l10n):
try:
p = parser.getParser(ref_file.file)
except UserWarning:
# no comparison, XXX report?
return
if ref_file not in self.reference:
# we didn't parse this before
try:
p.readContents(ref_file.getContents())
except Exception, e:
self.notify('error', ref_file, str(e))
return
self.reference[ref_file] = p.parse()
def _test(self, content, refWarnOrErrors, with_ref_file=False):
p = getParser(self.file.file)
p.readContents(content)
l10n = [e for e in p]
assert len(l10n) == 1
l10n = l10n[0]
if with_ref_file:
kwargs = {'reference': self.refList}
else:
kwargs = {}
checker = getChecker(self.file, **kwargs)
ref = self.refList[self.refMap[l10n.key]]
found = tuple(checker.check(ref, l10n))
self.assertEqual(found, refWarnOrErrors)
def testJunk(self):
self.assertTrue(os.path.isdir(self.tmp))
self.reference("""<!ENTITY foo 'fooVal'>
<!ENTITY bar 'barVal'>
<!ENTITY eff 'effVal'>""")
self.localized("""<!ENTITY foo 'fooVal'>
<!ENTY bar 'gimmick'>
<!ENTITY eff 'effVal'>
""")
cc = ContentComparer()
cc.observers.append(Observer())
cc.compare(File(self.ref, "en-reference.dtd", ""),
File(self.l10n, "l10n.dtd", ""),
mozpath.join(self.tmp, "merge", "l10n.dtd"))
self.assertDictEqual(
cc.observers.toJSON(), {
'summary': {
None: {
'errors': 1,
'warnings': 0,
'missing': 1,
'missing_w': 1,
'report': 0,
'obsolete': 0,
'changed': 0,
'changed_w': 0,
'unchanged': 2,
'unchanged_w': 2,
'keys': 0,
}
},
'details': {
'l10n.dtd': [{
'error':
u'Unparsed content "<!ENTY bar '
u'\'gimmick\'>\n" '
u'from line 2 column 1 to '
u'line 3 column 1'
}, {
'missingEntity': u'bar'
}]
}
})
mergefile = mozpath.join(self.tmp, "merge", "l10n.dtd")
self.assertTrue(os.path.isfile(mergefile))
p = getParser(mergefile)
p.readFile(mergefile)
entities = p.parse()
self.assertEqual(list(entities.keys()), ["foo", "eff", "bar"])
def testMismatchingValues(self):
self.reference("""
foo = Foo
.foottr = something
bar
.tender = Attribute value
""")
self.localized("""\
foo
.foottr = attr
bar = lBar
.tender = localized
""")
cc = ContentComparer([Observer()])
cc.compare(File(self.ref, "en-reference.ftl", ""),
File(self.l10n, "l10n.ftl", ""),
mozpath.join(self.tmp, "merge", "l10n.ftl"))
self.assertDictEqual(
cc.observers[0].toJSON(), {
'details': {
'l10n.ftl': [
{
'error':
u'Missing value at line 1, column 1 for foo'
},
{
'error':
u'Obsolete value at line 3, column 7 for bar',
},
]
},
'summary': {
None: {
'changed': 2,
'changed_w': 4,
'errors': 2
}
}
})
# validate merge results
mergepath = mozpath.join(self.tmp, "merge", "l10n.ftl")
self.assert_(os.path.exists(mergepath))
p = getParser(mergepath)
p.readFile(mergepath)
merged_entities, _ = p.parse()
self.assertEqual([e.key for e in merged_entities], [])
def add(self, orig, missing):
if self.notify('missingFile', missing, None) == "ignore":
# filter said that we don't need this file, don't count it
return
f = orig
try:
p = parser.getParser(f.file)
except UserWarning:
return
try:
p.readContents(f.getContents())
entities, map = p.parse()
except Exception, e:
self.notify('error', f, str(e))
return
def testError(self):
self.assertTrue(os.path.isdir(self.tmp))
self.reference("""foo = fooVal
bar = %d barVal
eff = effVal""")
self.localized("""\
bar = %S lBar
eff = leffVal
""")
cc = ContentComparer()
cc.observers.append(Observer())
cc.compare(File(self.ref, "en-reference.properties", ""),
File(self.l10n, "l10n.properties", ""),
mozpath.join(self.tmp, "merge", "l10n.properties"))
self.assertDictEqual(
cc.observers.toJSON(), {
'summary': {
None: {
'errors': 1,
'warnings': 0,
'missing': 1,
'missing_w': 1,
'report': 0,
'obsolete': 0,
'changed': 2,
'changed_w': 3,
'unchanged': 0,
'unchanged_w': 0,
'keys': 0,
}
},
'details': {
'l10n.properties': [{
'missingEntity': u'foo'
}, {
'error':
u'argument 1 `S` should be `d` '
u'at line 1, column 7 for bar'
}]
}
})
mergefile = mozpath.join(self.tmp, "merge", "l10n.properties")
self.assertTrue(os.path.isfile(mergefile))
p = getParser(mergefile)
p.readFile(mergefile)
entities = p.parse()
self.assertEqual(list(entities.keys()), ["eff", "foo", "bar"])
self.assertEqual(entities['bar'].val, '%d barVal')
def testMissing(self):
self.assertTrue(os.path.isdir(self.tmp))
l10n = os.path.join(self.tmp, "l10n.dtd")
open(l10n, "w").write("""<!ENTITY bar 'lBar'>
""")
cc = ContentComparer()
cc.set_merge_stage(os.path.join(self.tmp, "merge"))
cc.compare(File(self.ref, "en-reference.dtd", ""),
File(l10n, "l10n.dtd", ""))
print cc.observer.serialize()
mergefile = os.path.join(self.tmp, "merge", "l10n.dtd")
self.assertTrue(os.path.isfile(mergefile))
p = getParser(mergefile)
p.readFile(mergefile)
[m, n] = p.parse()
self.assertEqual(map(lambda e: e.key, m), ["bar", "eff", "foo"])
def testMissing(self):
self.assertTrue(os.path.isdir(self.tmp))
l10n = os.path.join(self.tmp, "l10n.dtd")
open(l10n, "w").write("""<!ENTITY bar 'lBar'>
""")
cc = ContentComparer()
cc.set_merge_stage(os.path.join(self.tmp, "merge"))
cc.compare(File(self.ref, "en-reference.dtd", ""),
File(l10n, "l10n.dtd", ""))
print cc.observer.serialize()
mergefile = os.path.join(self.tmp, "merge", "l10n.dtd")
self.assertTrue(os.path.isfile(mergefile))
p = getParser(mergefile)
p.readFile(mergefile)
[m, n] = p.parse()
self.assertEqual(map(lambda e: e.key, m), ["bar", "eff", "foo"])
def _test(self, content, refWarnOrErrors, with_ref_file=False):
p = getParser(self.file.file)
p.readContents(content)
l10n = [e for e in p]
assert len(l10n) == 1
l10n = l10n[0]
if with_ref_file:
kwargs = {
'reference': self.refList
}
else:
kwargs = {}
checker = getChecker(self.file, **kwargs)
ref = self.refList[self.refMap[l10n.key]]
found = tuple(checker.check(ref, l10n))
self.assertEqual(found, refWarnOrErrors)
def compare_content(f, past, now):
try:
p = getParser(f)
except UserWarning:
return
p.readContents(past)
past_entities, past_map = p.parse()
p.readContents(now)
now_entities, now_map = p.parse()
data = {'strings': 0, 'words': 0}
for k in now_map.keys():
if k in past_map or not isinstance(now_entities[now_map[k]], Entity):
continue
data['strings'] += 1
data['words'] += now_entities[now_map[k]].count_words()
return data
def __init__(self, path, source_resource=None):
self.path = path
self.entities = OrderedDict() # Preserve entity order.
self.source_resource = source_resource
try:
self.parser = parser.getParser(self.path)
except UserWarning as err:
raise ParseError(err)
self.parsed_objects = []
# A monolingual l10n file might not contain all entities, but the code
# expects ParsedResource to contain representations of all of them. So
# when parsing the l10n resource, we first create empty entity for each
# source resource entity.
if source_resource:
for key, entity in source_resource.entities.items():
self.entities[key] = CompareLocalesEntity(
entity.key,
None,
None,
0,
)
try:
self.parser.readFile(self.path)
except IOError as err:
# If the file doesn't exist, but we have a source resource,
# we can keep going, we'll just not have any translations.
if source_resource:
return
else:
raise ParseError(err)
self.parsed_objects = list(self.parser.walk())
order = 0
for entity in self.parsed_objects:
if isinstance(entity, parser.Entity):
self.entities[entity.key] = CompareLocalesEntity(
entity.key,
entity.unwrap(),
entity.pre_comment,
order,
)
order += 1
def testMismatchingValues(self):
self.reference("""
foo = Foo
.foottr = something
bar
.tender = Attribute value
""")
self.localized("""\
foo
.foottr = attr
bar = lBar
.tender = localized
""")
cc = ContentComparer([Observer()])
cc.compare(File(self.ref, "en-reference.ftl", ""),
File(self.l10n, "l10n.ftl", ""),
mozpath.join(self.tmp, "merge", "l10n.ftl"))
self.assertDictEqual(
cc.observers[0].toJSON(),
{
'details': {
'l10n.ftl': [
{
'error':
u'Missing value at line 1, column 1 for foo'
},
{
'error':
u'Obsolete value at line 3, column 7 for bar',
},
]
},
'summary': {
None: {'changed': 2, 'changed_w': 4, 'errors': 2}
}
}
)
# validate merge results
mergepath = mozpath.join(self.tmp, "merge", "l10n.ftl")
self.assert_(os.path.exists(mergepath))
p = getParser(mergepath)
p.readFile(mergepath)
merged_entities, _ = p.parse()
self.assertEqual([e.key for e in merged_entities], [])
def test_license_header(self):
p = getParser('foo.dtd')
p.readContents(self.resource('triple-license.dtd'))
for e in p:
self.assertEqual(e.key, 'foo')
self.assertEqual(e.val, 'value')
self.assert_('MPL' in p.header)
p.readContents('''\
<!-- This Source Code Form is subject to the terms of the Mozilla Public
- License, v. 2.0. If a copy of the MPL was not distributed with this file,
- You can obtain one at http://mozilla.org/MPL/2.0/. -->
<!ENTITY foo "value">
''')
for e in p:
self.assertEqual(e.key, 'foo')
self.assertEqual(e.val, 'value')
self.assert_('MPL' in p.header)
def extractStrings(self):
'''Extract strings from all locales.'''
basedir = os.path.dirname(self.toml_path)
project_config = paths.TOMLParser().parse(
self.toml_path, env={'l10n_base': ''})
basedir = os.path.join(basedir, project_config.root)
reference_cache = {}
self.translations[self.reference_locale] = {}
for locale in project_config.all_locales:
files = paths.ProjectFiles(locale, [project_config])
self.translations[locale] = {}
for l10n_file, reference_file, _, _ in files:
if not os.path.exists(l10n_file):
# File not available in localization
continue
if not os.path.exists(reference_file):
# File not available in reference
continue
key_path = os.path.relpath(reference_file, basedir)
try:
p = getParser(reference_file)
except UserWarning:
continue
if key_path not in reference_cache:
p.readFile(reference_file)
reference_cache[key_path] = set(p.parse().keys())
self.translations[self.reference_locale].update(
('{}/{}:{}'.format(
self.repository_name, key_path, entity.key),
entity.raw_val)
for entity in p.parse()
)
p.readFile(l10n_file)
self.translations[locale].update(
('{}/{}:{}'.format(
self.repository_name, key_path, entity.key),
entity.raw_val)
for entity in p.parse()
)
def serialize(filename, reference, old_l10n, new_data):
'''Returns a byte string of the serialized content to use.
Input are a filename to create the right parser, a reference and
an existing localization, both as the result of parser.walk().
Finally, new_data is a dictionary of key to raw values to serialize.
Raises a SerializationNotSupportedError if we don't support the file
format.
'''
try:
parser = getParser(filename)
except UserWarning:
raise SerializationNotSupportedError(
'Unsupported file format ({}).'.format(filename))
# create template, whitespace and all
placeholders = [
placeholder(entry)
for entry in reference
if not isinstance(entry, Junk)
]
ref_mapping = {
entry.key: entry
for entry in reference
if isinstance(entry, Entity)
}
# strip obsolete strings
old_l10n = sanitize_old(ref_mapping.keys(), old_l10n, new_data)
# create new Entities
# .val can just be "", merge_channels doesn't need that
new_l10n = []
for key, new_raw_val in six.iteritems(new_data):
if new_raw_val is None or key not in ref_mapping:
continue
ref_ent = ref_mapping[key]
new_l10n.append(ref_ent.wrap(new_raw_val))
merged = merge_resources(
parser,
[placeholders, old_l10n, new_l10n],
keep_newest=False
)
pruned = prune_placeholders(merged)
return encode(serialize_legacy_resource(pruned), parser.encoding)
def merge_channels(name, *resources):
try:
parser = cl.getParser(name)
except UserWarning:
raise MergeNotSupportedError(
'Unsupported file format ({}).'.format(name))
# A map of comments to the keys of entities they belong to.
comments = {}
def parse_resource(resource):
# The counter dict keeps track of number of identical comments.
counter = defaultdict(int)
parser.readContents(resource)
pairs = [get_key_value(entity, counter) for entity in parser.walk()]
return OrderedDict(pairs)
def get_key_value(entity, counter):
if isinstance(entity, cl.Comment):
counter[entity.all] += 1
# Use the (value, index) tuple as the key. AddRemove will
# de-deplicate identical comments at the same index.
return ((entity.all, counter[entity.all]), entity)
if isinstance(entity, cl.Whitespace):
# Use the Whitespace instance as the key so that it's always
# unique. Adjecent whitespace will be folded into the longer one in
# prune.
return (entity, entity)
# When comments change, AddRemove gives us one 'add' and one 'delete'
# (because a comment's key is its content). In merge_two we'll try to
# de-duplicate comments by looking at the entity they belong to. Set
# up the back-reference from the comment to its entity here.
if isinstance(entity, cl.Entity) and entity.pre_comment:
comments[entity.pre_comment] = entity.key
return (entity.key, entity)
entities = reduce(
lambda x, y: merge_two(comments, x, y),
map(parse_resource, resources))
return encode(serialize_legacy_resource(entities), parser.encoding)
def extractStrings(self):
''' Extract strings from all files '''
# If storage_mode is append, read existing translations (if available)
# before overriding them
if self.storage_mode == 'append':
file_name = self.storage_file + '.json'
if os.path.isfile(file_name):
with open(file_name) as f:
self.translations = json.load(f)
f.close()
# Create a list of files to analyze
self.extractFileList()
for file_name in self.file_list:
file_extension = os.path.splitext(file_name)[1]
file_parser = parser.getParser(file_extension)
file_parser.readFile(file_name)
try:
entities, map = file_parser.parse()
for entity in entities:
string_id = u'{0}:{1}'.format(
self.getRelativePath(file_name), unicode(entity))
if not isinstance(entity, parser.Junk):
self.translations[string_id] = entity.raw_val
except Exception as e:
print 'Error parsing file: {0}'.format(file_name)
print e
# Remove extra strings from locale
if self.reference_locale != self.locale:
# Read the JSON cache for reference locale if available
file_name = self.reference_storage_file + '.json'
if os.path.isfile(file_name):
with open(file_name) as f:
reference_strings = json.load(f)
f.close()
for string_id in self.translations.keys():
if string_id not in reference_strings:
del(self.translations[string_id])
def __init__(self, path, source_resource=None):
self.path = path
self.entities = OrderedDict() # Preserve entity order.
self.source_resource = source_resource
self.parser = parser.getParser(self.path)
self.parsed_objects = []
# A monolingual l10n file might not contain all entities, but the code
# expects ParsedResource to contain representations of all of them. So
# when parsing the l10n resource, we first create empty entity for each
# source resource entity.
if source_resource:
for key, entity in source_resource.entities.items():
self.entities[key] = CompareLocalesEntity(
entity.key,
None,
None,
None,
)
try:
self.parser.readFile(self.path)
except IOError as err:
# If the file doesn't exist, but we have a source resource,
# we can keep going, we'll just not have any translations.
if source_resource:
return
else:
raise ParseError(err)
self.parsed_objects = list(self.parser.walk())
order = 0
for entity in self.parsed_objects:
if isinstance(entity, parser.Entity):
self.entities[entity.key] = CompareLocalesEntity(
entity.key,
entity.unwrap(),
entity.pre_comment,
order,
)
order += 1
def lint_file(self, path, ref, extra_tests):
file_parser = parser.getParser(path)
if ref is not None and os.path.isfile(ref):
file_parser.readFile(ref)
reference = file_parser.parse()
else:
reference = {}
file_parser.readFile(path)
current = file_parser.parse()
checker = checks.getChecker(
File(path, path, locale=REFERENCE_LOCALE),
extra_tests=extra_tests
)
if checker and checker.needs_reference:
checker.set_reference(current)
linter = EntityLinter(current, checker, reference)
for current_entity in current:
for result in linter.lint_entity(current_entity):
result['path'] = path
yield result
def extractStrings(self):
'''Extract strings in files'''
# Create a list of files to analyze
self.extractFileList()
for file_path in self.file_list:
file_extension = os.path.splitext(file_path)[1]
file_name = self.getRelativePath(file_path)
# Ignore folders unrelated to Firefox Desktop or Fennec
if file_name.startswith(self.excluded_folders):
continue
if file_name.endswith('region.properties'):
continue
file_parser = parser.getParser(file_extension)
file_parser.readFile(file_path)
try:
entities, map = file_parser.parse()
for entity in entities:
# Ignore Junk
if isinstance(entity, parser.Junk):
continue
string_id = u'{}:{}'.format(
file_name, entity)
if file_extension == '.ftl':
if entity.raw_val != '':
self.strings[string_id] = entity.raw_val
# Store attributes
for attribute in entity.attributes:
attr_string_id = u'{0}:{1}.{2}'.format(
file_name, entity, attribute)
self.strings[attr_string_id] = attribute.raw_val
else:
self.strings[string_id] = entity.raw_val
except Exception as e:
print('Error parsing file: {}'.format(file_path))
print(e)
def add(self, orig, missing, merge_file):
''' Add missing localized file.'''
f = orig
try:
p = parser.getParser(f.file)
except UserWarning:
p = None
# if we don't support this file, assume CAN_COPY to mimick
# l10n dir as closely as possible
caps = p.capabilities if p else parser.CAN_COPY
if (caps & (parser.CAN_COPY | parser.CAN_MERGE)):
# even if we can merge, pretend we can only copy
self.merge(
KeyedTuple([]), orig, missing, merge_file,
['trigger copy'], [], None, parser.CAN_COPY, None
)
if self.observers.notify('missingFile', missing, None) == "ignore":
# filter said that we don't need this file, don't count it
return
if p is None:
# We don't have a parser, cannot count missing strings
return
try:
p.readFile(f)
entities = p.parse()
except Exception as ex:
self.observers.notify('error', f, str(ex))
return
# strip parse errors
entities = [e for e in entities if not isinstance(e, parser.Junk)]
self.observers.updateStats(missing, {'missingInFiles': len(entities)})
missing_w = 0
for e in entities:
missing_w += e.count_words()
self.observers.updateStats(missing, {'missing_w': missing_w})
def testJunk(self):
self.assertTrue(os.path.isdir(self.tmp))
self.reference("""<!ENTITY foo 'fooVal'>
<!ENTITY bar 'barVal'>
<!ENTITY eff 'effVal'>""")
self.localized("""<!ENTITY foo 'fooVal'>
<!ENTY bar 'gimmick'>
<!ENTITY eff 'effVal'>
""")
cc = ContentComparer()
cc.set_merge_stage(os.path.join(self.tmp, "merge"))
cc.compare(File(self.ref, "en-reference.dtd", ""),
File(self.l10n, "l10n.dtd", ""))
self.assertDictEqual(
cc.observer.toJSON(),
{'summary':
{None: {
'errors': 1, 'missing': 1, 'unchanged': 2
}},
'details': {
'children': [
('l10n.dtd',
{'value': {
'error': [u'Unparsed content "<!ENTY bar '
u'\'gimmick\'>" '
u'from line 2 colum 1 to '
u'line 2 column 22'],
'missingEntity': [u'bar']}}
)
]}
}
)
mergefile = os.path.join(self.tmp, "merge", "l10n.dtd")
self.assertTrue(os.path.isfile(mergefile))
p = getParser(mergefile)
p.readFile(mergefile)
[m, n] = p.parse()
self.assertEqual(map(lambda e: e.key, m), ["foo", "eff", "bar"])
def handleFile(self, leaf):
try:
parser = getParser(leaf)
except UserWarning:
return
args = cmdbuilder(b('annotate'), d=True, u=True, T='json',
*['path:' + leaf])
blame_json = ''.join(self.client.rawcommand(args))
blames = json.loads(blame_json)
fname = os.path.join(self.client.root(), leaf)
parser.readFile(fname)
entities, emap = parser.parse()
self.blame[leaf] = {}
for e in entities:
blines = blames[(e.value_position()[0] - 1):e.value_position(-1)[0]]
blines.sort(key=lambda blame: -blame['date'][0]) # ignore timezone
blame = blines[0]
user = blame['user']
timestamp = blame['date'][0] # ignore timezone
if user not in self.users:
self.users.append(user)
userid = self.users.index(user)
self.blame[leaf][e.key] = [userid, timestamp]
def test_entities_across_dtd(self):
f = File("browser/strings.dtd", "strings.dtd", "browser")
p = getParser(f.file)
p.readContents('<!ENTITY other "some &good.ref;">')
ref = p.parse()
checker = getChecker(f, reference=ref[0])
# good string
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
# dtd warning
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string &ref;")
self.assertEqual(tuple(checker.check(ref, l10n)),
(('warning', (0, 0),
'Referencing unknown entity `ref` (good.ref known)',
'xmlparse'),))
# no report on stray ampersand
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string with &good.ref;")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
def testError(self):
self.assertTrue(os.path.isdir(self.tmp))
self.reference("""foo = fooVal
bar = %d barVal
eff = effVal""")
self.localized("""\
bar = %S lBar
eff = leffVal
""")
cc = ContentComparer()
cc.set_merge_stage(os.path.join(self.tmp, "merge"))
cc.compare(File(self.ref, "en-reference.properties", ""),
File(self.l10n, "l10n.properties", ""))
self.assertDictEqual(
cc.observer.toJSON(),
{'summary':
{None: {
'changed': 2, 'errors': 1, 'missing': 1
}},
'details': {
'children': [
('l10n.properties',
{'value': {
'error': [u'argument 1 `S` should be `d` '
u'at line 1, column 7 for bar'],
'missingEntity': [u'foo']}}
)
]}
}
)
mergefile = os.path.join(self.tmp, "merge", "l10n.properties")
self.assertTrue(os.path.isfile(mergefile))
p = getParser(mergefile)
p.readFile(mergefile)
[m, n] = p.parse()
self.assertEqual([e.key for e in m], ["eff", "foo", "bar"])
self.assertEqual(m[n['bar']].val, '%d barVal')
def read_legacy_resource(self, path):
"""Read a legacy resource and parse it into a dict."""
parser = getParser(path)
parser.readFile(path)
# Transform the parsed result which is an iterator into a dict.
return {entity.key: entity.val for entity in parser}
def setUp(self):
p = getParser(self.file.file)
p.readContents(self.refContent)
self.refList, self.refMap = p.parse()
