def test_android_prop(self):
f = File("embedding/android/strings.properties", "strings.properties",
"embedding/android")
checker = getChecker(f)
# good plain string
ref = self.getEntity("plain string")
l10n = self.getEntity("plain localized string")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
# no dtd warning
ref = self.getEntity("plain string")
l10n = self.getEntity("plain localized string &ref;")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
# no report on stray ampersand
ref = self.getEntity("plain string")
l10n = self.getEntity("plain localized string with apos: '")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
# report on bad printf
ref = self.getEntity("string with %s")
l10n = self.getEntity("string with %S")
self.assertEqual(tuple(checker.check(ref, l10n)),
(('error', 0, 'argument 1 `S` should be `s`',
'printf'),))
def run_checks(entity, locale_code, string):
"""
Run all compare-locales checks on provided translation and entity.
:arg pontoon.base.models.Entity entity: Source entity instance
:arg basestring locale_code: Locale of a translation
:arg basestring string: translation string
:return: Dictionary with the following structure:
{
'clErrors': [
'Error1',
],
'clWarnings': [
'Warning1',
]
}
Both keys are optional.
"""
resource_ext = ".{}".format(entity.resource.format)
extra_tests = None
if "mobile/android/base" in entity.resource.path:
extra_tests = {"android-dtd"}
entity.string = escape_quotes(entity.string)
string = escape_quotes(string)
source_ent, translation_ent = cast_to_compare_locales(
resource_ext,
entity,
string,
)
checker = getChecker(
File(entity.resource.path, entity.resource.path, locale=locale_code),
extra_tests,
)
if checker is None:
# compare-locales has no checks for this format, it's OK.
return {}
# Currently, references are required only by DTD files but that may change in the future.
if checker.needs_reference:
references = KeyedTuple(
CompareDTDEntity(
e.key,
e.string,
e.comment,
) for e in entity.resource.entities.all())
checker.set_reference(references)
errors = {}
for severity, _, message, _ in checker.check(source_ent, translation_ent):
messages = errors.setdefault("cl%ss" % severity.capitalize(), [])
# Old-school duplicate prevention - set() is not JSON serializable
if message not in messages:
messages.append(message)
return errors
def _test(self, content, refWarnOrErrors):
p = getParser(self.file.file)
p.readContents(content)
l10n = [e for e in p]
assert len(l10n) == 1
l10n = l10n[0]
checker = getChecker(self.file)
ref = self.refList[self.refMap[l10n.key]]
found = tuple(checker.check(ref, l10n))
self.assertEqual(found, refWarnOrErrors)
def _test(self, content, refWarnOrErrors):
p = parser.getParser(self.file.file)
p.readContents(content)
l10n = [e for e in p]
assert len(l10n) == 1
l10n = l10n[0]
checker = getChecker(self.file)
if checker.needs_reference:
checker.set_reference(self.refList)
ref = self.refList[l10n.key]
found = tuple(checker.check(ref, l10n))
self.assertEqual(found, refWarnOrErrors)
def _test(self, content, refWarnOrErrors, with_ref_file=False):
p = getParser(self.file.file)
p.readContents(content)
l10n = [e for e in p]
assert len(l10n) == 1
l10n = l10n[0]
if with_ref_file:
kwargs = {'reference': self.refList}
else:
kwargs = {}
checker = getChecker(self.file, **kwargs)
ref = self.refList[self.refMap[l10n.key]]
found = tuple(checker.check(ref, l10n))
self.assertEqual(found, refWarnOrErrors)
def _test(self, content, refWarnOrErrors, with_ref_file=False):
p = getParser(self.file.file)
p.readContents(content)
l10n = [e for e in p]
assert len(l10n) == 1
l10n = l10n[0]
if with_ref_file:
kwargs = {
'reference': self.refList
}
else:
kwargs = {}
checker = getChecker(self.file, **kwargs)
ref = self.refList[self.refMap[l10n.key]]
found = tuple(checker.check(ref, l10n))
self.assertEqual(found, refWarnOrErrors)
def lint_file(self, path, ref, extra_tests):
file_parser = parser.getParser(path)
if ref is not None and os.path.isfile(ref):
file_parser.readFile(ref)
reference = file_parser.parse()
else:
reference = {}
file_parser.readFile(path)
current = file_parser.parse()
checker = checks.getChecker(File(path, path, locale=REFERENCE_LOCALE),
extra_tests=extra_tests)
if checker and checker.needs_reference:
checker.set_reference(current)
linter = EntityLinter(current, checker, reference)
for current_entity in current:
for result in linter.lint_entity(current_entity):
result['path'] = path
yield result
def test_non_android_dtd(self):
f = File("browser/strings.dtd", "strings.dtd", "browser")
checker = getChecker(f)
# good string
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string")
self.assertEqual(tuple(checker.check(ref, l10n)), ())
# dtd warning
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string &ref;")
self.assertEqual(
tuple(checker.check(ref, l10n)),
(('warning',
(0, 0), 'Referencing unknown entity `ref`', 'xmlparse'), ))
# no report on stray ampersand
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string with apos: '")
self.assertEqual(tuple(checker.check(ref, l10n)), ())
def test_non_android_dtd(self):
f = File("browser/strings.dtd", "strings.dtd", "browser")
checker = getChecker(f)
# good string
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
# dtd warning
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string &ref;")
self.assertEqual(tuple(checker.check(ref, l10n)),
(('warning', (0, 0),
'Referencing unknown entity `ref`', 'xmlparse'),))
# no report on stray ampersand
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string with apos: '")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
def lint_file(self, path, ref, extra_tests):
file_parser = parser.getParser(path)
if ref is not None and os.path.isfile(ref):
file_parser.readFile(ref)
reference = file_parser.parse()
else:
reference = {}
file_parser.readFile(path)
current = file_parser.parse()
checker = checks.getChecker(
File(path, path, locale=REFERENCE_LOCALE),
extra_tests=extra_tests
)
if checker and checker.needs_reference:
checker.set_reference(current)
linter = EntityLinter(current, checker, reference)
for current_entity in current:
for result in linter.lint_entity(current_entity):
result['path'] = path
yield result
def test_entities_across_dtd(self):
f = File("browser/strings.dtd", "strings.dtd", "browser")
p = getParser(f.file)
p.readContents('<!ENTITY other "some &good.ref;">')
ref = p.parse()
checker = getChecker(f, reference=ref[0])
# good string
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string")
self.assertEqual(tuple(checker.check(ref, l10n)), ())
# dtd warning
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string &ref;")
self.assertEqual(
tuple(checker.check(ref, l10n)),
(('warning',
(0, 0), 'Referencing unknown entity `ref` (good.ref known)',
'xmlparse'), ))
# no report on stray ampersand
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string with &good.ref;")
self.assertEqual(tuple(checker.check(ref, l10n)), ())
def test_entities_across_dtd(self):
f = File("browser/strings.dtd", "strings.dtd", "browser")
p = getParser(f.file)
p.readContents('<!ENTITY other "some &good.ref;">')
ref = p.parse()
checker = getChecker(f, reference=ref[0])
# good string
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
# dtd warning
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string &ref;")
self.assertEqual(tuple(checker.check(ref, l10n)),
(('warning', (0, 0),
'Referencing unknown entity `ref` (good.ref known)',
'xmlparse'),))
# no report on stray ampersand
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string with &good.ref;")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
try:
p.readContents(l10n.getContents())
l10n_entities, l10n_map = p.parse()
l10n_ctx = p.ctx
except Exception, e:
self.notify('error', l10n, str(e))
return
ar = AddRemove()
ar.set_left(e.key for e in ref_entities)
ar.set_right(e.key for e in l10n_entities)
report = missing = obsolete = changed = unchanged = keys = 0
missing_w = changed_w = unchanged_w = 0 # word stats
missings = []
skips = []
checker = getChecker(l10n, extra_tests=extra_tests)
if checker and checker.needs_reference:
checker.set_reference(ref_entities)
for msg in p.findDuplicates(ref_entities):
self.notify('warning', l10n, msg)
for msg in p.findDuplicates(l10n_entities):
self.notify('error', l10n, msg)
for action, entity_id in ar:
if action == 'delete':
# missing entity
if isinstance(ref_entities[ref_map[entity_id]], parser.Junk):
self.notify('warning', l10n, 'Parser error in en-US')
continue
_rv = self.notify('missingEntity', l10n, entity_id)
if _rv == "ignore":
continue
def run_checks(entity, locale_code, string):
"""
Run all compare-locales checks on provided translation and entity.
:arg pontoon.base.models.Entity entity: Source entity instance
:arg basestring locale_code: Locale of a translation
:arg basestring string: translation string
:return: Dictionary with the following structure:
{
'clErrors': [
'Error1',
],
'clWarnings': [
'Warning1',
]
}
Both keys are optional.
"""
resource_ext = '.{}'.format(entity.resource.format)
extra_tests = None
if 'mobile/android/base' in entity.resource.path:
extra_tests = {'android-dtd'}
entity.string = escape_quotes(entity.string)
string = escape_quotes(string)
source_ent, translation_ent = cast_to_compare_locales(
resource_ext,
entity,
string,
)
checker = getChecker(
File(entity.resource.path, entity.resource.path, locale=locale_code),
extra_tests
)
if checker is None:
# compare-locales has no checks for this format, it's OK.
return {}
# Currently, references are required only by DTD files but that may change in the future.
if checker.needs_reference:
references = KeyedTuple(
CompareDTDEntity(
e.key,
e.string,
e.comment,
)
for e in entity.resource.entities.all()
)
checker.set_reference(references)
errors = {}
for severity, _, message, _ in checker.check(source_ent, translation_ent):
messages = errors.setdefault('cl%ss' % severity.capitalize(), [])
# Old-school duplicate prevention - set() is not JSON serializable
if message not in messages:
messages.append(message)
return errors
def test_android_dtd(self):
"""Testing the actual android checks. The logic is involved,
so this is a lot of nitty gritty detail tests.
"""
f = File("embedding/android/strings.dtd", "strings.dtd",
"embedding/android")
checker = getChecker(f)
# good string
ref = self.getDTDEntity("plain string")
l10n = self.getDTDEntity("plain localized string")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
# dtd warning
l10n = self.getDTDEntity("plain localized string &ref;")
self.assertEqual(tuple(checker.check(ref, l10n)),
(('warning', (0, 0),
'Referencing unknown entity `ref`', 'xmlparse'),))
# no report on stray ampersand or quote, if not completely quoted
for i in xrange(3):
# make sure we're catching unescaped apostrophes,
# try 0..5 backticks
l10n = self.getDTDEntity("\\"*(2*i) + "'")
self.assertEqual(tuple(checker.check(ref, l10n)),
(('error', 2*i, self.apos_msg, 'android'),))
l10n = self.getDTDEntity("\\"*(2*i + 1) + "'")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
# make sure we don't report if apos string is quoted
l10n = self.getDTDEntity('"' + "\\"*(2*i) + "'\"")
tpl = tuple(checker.check(ref, l10n))
self.assertEqual(tpl, (),
"`%s` shouldn't fail but got %s"
% (l10n.val, str(tpl)))
l10n = self.getDTDEntity('"' + "\\"*(2*i+1) + "'\"")
tpl = tuple(checker.check(ref, l10n))
self.assertEqual(tpl, (),
"`%s` shouldn't fail but got %s"
% (l10n.val, str(tpl)))
# make sure we're catching unescaped quotes, try 0..5 backticks
l10n = self.getDTDEntity("\\"*(2*i) + "\"")
self.assertEqual(tuple(checker.check(ref, l10n)),
(('error', 2*i, self.quot_msg, 'android'),))
l10n = self.getDTDEntity("\\"*(2*i + 1) + "'")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
# make sure we don't report if quote string is single quoted
l10n = self.getDTDEntity("'" + "\\"*(2*i) + "\"'")
tpl = tuple(checker.check(ref, l10n))
self.assertEqual(tpl, (),
"`%s` shouldn't fail but got %s" %
(l10n.val, str(tpl)))
l10n = self.getDTDEntity('"' + "\\"*(2*i+1) + "'\"")
tpl = tuple(checker.check(ref, l10n))
self.assertEqual(tpl, (),
"`%s` shouldn't fail but got %s" %
(l10n.val, str(tpl)))
# check for mixed quotes and ampersands
l10n = self.getDTDEntity("'\"")
self.assertEqual(tuple(checker.check(ref, l10n)),
(('error', 0, self.apos_msg, 'android'),
('error', 1, self.quot_msg, 'android')))
l10n = self.getDTDEntity("''\"'")
self.assertEqual(tuple(checker.check(ref, l10n)),
(('error', 1, self.apos_msg, 'android'),))
l10n = self.getDTDEntity('"\'""')
self.assertEqual(tuple(checker.check(ref, l10n)),
(('error', 2, self.quot_msg, 'android'),))
# broken unicode escape
l10n = self.getDTDEntity("Some broken \u098 unicode")
self.assertEqual(tuple(checker.check(ref, l10n)),
(('error', 12, 'truncated \\uXXXX escape',
'android'),))
# broken unicode escape, try to set the error off
l10n = self.getDTDEntity(u"\u9690"*14+"\u006"+" "+"\u0064")
self.assertEqual(tuple(checker.check(ref, l10n)),
(('error', 14, 'truncated \\uXXXX escape',
'android'),))
def compare(self, ref_file, l10n, merge_file, extra_tests=None):
try:
p = parser.getParser(ref_file.file)
except UserWarning:
# no comparison, XXX report?
# At least, merge
self.merge([], {}, ref_file, l10n, merge_file, [], [], None,
parser.CAN_COPY, None)
return
try:
p.readContents(ref_file.getContents())
except Exception as e:
self.notify('error', ref_file, str(e))
return
ref_entities, ref_map = p.parse()
try:
p.readContents(l10n.getContents())
l10n_entities, l10n_map = p.parse()
l10n_ctx = p.ctx
except Exception as e:
self.notify('error', l10n, str(e))
return
ar = AddRemove()
ar.set_left(e.key for e in ref_entities)
ar.set_right(e.key for e in l10n_entities)
report = missing = obsolete = changed = unchanged = keys = 0
missing_w = changed_w = unchanged_w = 0 # word stats
missings = []
skips = []
checker = getChecker(l10n, extra_tests=extra_tests)
if checker and checker.needs_reference:
checker.set_reference(ref_entities)
for msg in p.findDuplicates(ref_entities):
self.notify('warning', l10n, msg)
for msg in p.findDuplicates(l10n_entities):
self.notify('error', l10n, msg)
for action, entity_id in ar:
if action == 'delete':
# missing entity
if isinstance(ref_entities[ref_map[entity_id]], parser.Junk):
self.notify('warning', l10n, 'Parser error in en-US')
continue
_rv = self.notify('missingEntity', l10n, entity_id)
if _rv == "ignore":
continue
if _rv == "error":
# only add to missing entities for l10n-merge on error,
# not report
missings.append(entity_id)
missing += 1
refent = ref_entities[ref_map[entity_id]]
missing_w += refent.count_words()
else:
# just report
report += 1
elif action == 'add':
# obsolete entity or junk
if isinstance(l10n_entities[l10n_map[entity_id]], parser.Junk):
junk = l10n_entities[l10n_map[entity_id]]
params = (junk.val, ) + junk.position() + junk.position(-1)
self.notify(
'error', l10n,
'Unparsed content "%s" from line %d column %d'
' to line %d column %d' % params)
if merge_file is not None:
skips.append(junk)
elif self.notify('obsoleteEntity', l10n,
entity_id) != 'ignore':
obsolete += 1
else:
# entity found in both ref and l10n, check for changed
refent = ref_entities[ref_map[entity_id]]
l10nent = l10n_entities[l10n_map[entity_id]]
if self.keyRE.search(entity_id):
keys += 1
else:
if refent.equals(l10nent):
self.doUnchanged(l10nent)
unchanged += 1
unchanged_w += refent.count_words()
else:
self.doChanged(ref_file, refent, l10nent)
changed += 1
changed_w += refent.count_words()
# run checks:
if checker:
for tp, pos, msg, cat in checker.check(refent, l10nent):
line, col = l10nent.value_position(pos)
# skip error entities when merging
if tp == 'error' and merge_file is not None:
skips.append(l10nent)
self.notify(
tp, l10n, u"%s at line %d, column %d for %s" %
(msg, line, col, refent.key))
pass
if merge_file is not None:
self.merge(ref_entities, ref_map, ref_file, l10n, merge_file,
missings, skips, l10n_ctx, p.capabilities, p.encoding)
stats = {}
for cat, value in (('missing', missing), ('missing_w', missing_w),
('report', report), ('obsolete', obsolete),
('changed', changed), ('changed_w', changed_w),
('unchanged', unchanged),
('unchanged_w', unchanged_w), ('keys', keys)):
if value:
stats[cat] = value
self.updateStats(l10n, stats)
pass
def compare(self, ref_file, l10n, merge_file, extra_tests=None):
try:
p = parser.getParser(ref_file.file)
except UserWarning:
# no comparison, XXX report?
# At least, merge
self.merge(
KeyedTuple([]), ref_file, l10n, merge_file, [], [], None,
parser.CAN_COPY, None)
return
try:
p.readFile(ref_file)
except Exception as e:
self.observers.notify('error', ref_file, str(e))
return
ref_entities = p.parse()
try:
p.readFile(l10n)
l10n_entities = p.parse()
l10n_ctx = p.ctx
except Exception as e:
self.observers.notify('error', l10n, str(e))
return
ar = AddRemove()
ar.set_left(ref_entities.keys())
ar.set_right(l10n_entities.keys())
report = missing = obsolete = changed = unchanged = keys = 0
missing_w = changed_w = unchanged_w = 0 # word stats
missings = []
skips = []
checker = getChecker(l10n, extra_tests=extra_tests)
if checker and checker.needs_reference:
checker.set_reference(ref_entities)
for msg in p.findDuplicates(ref_entities):
self.observers.notify('warning', l10n, msg)
for msg in p.findDuplicates(l10n_entities):
self.observers.notify('error', l10n, msg)
for action, entity_id in ar:
if action == 'delete':
# missing entity
if isinstance(ref_entities[entity_id], parser.Junk):
self.observers.notify(
'warning', l10n, 'Parser error in en-US'
)
continue
_rv = self.observers.notify('missingEntity', l10n, entity_id)
if _rv == "ignore":
continue
if _rv == "error":
# only add to missing entities for l10n-merge on error,
# not report
missings.append(entity_id)
missing += 1
refent = ref_entities[entity_id]
missing_w += refent.count_words()
else:
# just report
report += 1
elif action == 'add':
# obsolete entity or junk
if isinstance(l10n_entities[entity_id],
parser.Junk):
junk = l10n_entities[entity_id]
self.observers.notify(
'error', l10n,
junk.error_message()
)
if merge_file is not None:
skips.append(junk)
elif (
self.observers.notify('obsoleteEntity', l10n, entity_id)
!= 'ignore'
):
obsolete += 1
else:
# entity found in both ref and l10n, check for changed
refent = ref_entities[entity_id]
l10nent = l10n_entities[entity_id]
if self.keyRE.search(entity_id):
keys += 1
else:
if refent.equals(l10nent):
self.doUnchanged(l10nent)
unchanged += 1
unchanged_w += refent.count_words()
else:
self.doChanged(ref_file, refent, l10nent)
changed += 1
changed_w += refent.count_words()
# run checks:
if checker:
for tp, pos, msg, cat in checker.check(refent, l10nent):
line, col = l10nent.value_position(pos)
# skip error entities when merging
if tp == 'error' and merge_file is not None:
skips.append(l10nent)
self.observers.notify(
tp, l10n,
u"%s at line %d, column %d for %s" %
(msg, line, col, refent.key)
)
pass
if merge_file is not None:
self.merge(
ref_entities, ref_file,
l10n, merge_file, missings, skips, l10n_ctx,
p.capabilities, p.encoding)
stats = {}
for cat, value in (
('missing', missing),
('missing_w', missing_w),
('report', report),
('obsolete', obsolete),
('changed', changed),
('changed_w', changed_w),
('unchanged', unchanged),
('unchanged_w', unchanged_w),
('keys', keys)):
if value:
stats[cat] = value
self.observers.updateStats(l10n, stats)
pass
try:
p.readContents(l10n.getContents())
l10n_entities, l10n_map = p.parse()
l10n_ctx = p.ctx
except Exception, e:
self.notify('error', l10n, str(e))
return
ar = AddRemove()
ar.set_left(e.key for e in ref_entities)
ar.set_right(e.key for e in l10n_entities)
report = missing = obsolete = changed = unchanged = keys = 0
missing_w = changed_w = unchanged_w = 0 # word stats
missings = []
skips = []
checker = getChecker(l10n, extra_tests=extra_tests)
if checker and checker.needs_reference:
checker.set_reference(ref_entities)
for msg in p.findDuplicates(ref_entities):
self.notify('warning', l10n, msg)
for msg in p.findDuplicates(l10n_entities):
self.notify('error', l10n, msg)
for action, entity_id in ar:
if action == 'delete':
# missing entity
if isinstance(ref_entities[ref_map[entity_id]], parser.Junk):
self.notify('warning', l10n, 'Parser error in en-US')
continue
_rv = self.notify('missingEntity', l10n, entity_id)
if _rv == "ignore":
continue
def compare(self, ref_file, l10n, merge_file, extra_tests=None):
try:
p = parser.getParser(ref_file.file)
except UserWarning:
# no comparison, XXX report?
# At least, merge
self.merge(KeyedTuple([]), ref_file, l10n, merge_file, [], [],
None, parser.CAN_COPY, None)
return
try:
p.readFile(ref_file)
except Exception as e:
self.observers.notify('error', ref_file, str(e))
return
ref_entities = p.parse()
try:
p.readFile(l10n)
l10n_entities = p.parse()
l10n_ctx = p.ctx
except Exception as e:
self.observers.notify('error', l10n, str(e))
return
ar = AddRemove()
ar.set_left(ref_entities.keys())
ar.set_right(l10n_entities.keys())
report = missing = obsolete = changed = unchanged = keys = 0
missing_w = changed_w = unchanged_w = 0 # word stats
missings = []
skips = []
checker = getChecker(l10n, extra_tests=extra_tests)
if checker and checker.needs_reference:
checker.set_reference(ref_entities)
for msg in p.findDuplicates(ref_entities):
self.observers.notify('warning', l10n, msg)
for msg in p.findDuplicates(l10n_entities):
self.observers.notify('error', l10n, msg)
for action, entity_id in ar:
if action == 'delete':
# missing entity
if isinstance(ref_entities[entity_id], parser.Junk):
self.observers.notify('warning', l10n,
'Parser error in en-US')
continue
_rv = self.observers.notify('missingEntity', l10n, entity_id)
if _rv == "ignore":
continue
if _rv == "error":
# only add to missing entities for l10n-merge on error,
# not report
missings.append(entity_id)
missing += 1
refent = ref_entities[entity_id]
missing_w += refent.count_words()
else:
# just report
report += 1
elif action == 'add':
# obsolete entity or junk
if isinstance(l10n_entities[entity_id], parser.Junk):
junk = l10n_entities[entity_id]
self.observers.notify('error', l10n, junk.error_message())
if merge_file is not None:
skips.append(junk)
elif (self.observers.notify('obsoleteEntity', l10n, entity_id)
!= 'ignore'):
obsolete += 1
else:
# entity found in both ref and l10n, check for changed
refent = ref_entities[entity_id]
l10nent = l10n_entities[entity_id]
if self.keyRE.search(entity_id):
keys += 1
else:
if refent.equals(l10nent):
self.doUnchanged(l10nent)
unchanged += 1
unchanged_w += refent.count_words()
else:
self.doChanged(ref_file, refent, l10nent)
changed += 1
changed_w += refent.count_words()
# run checks:
if checker:
for tp, pos, msg, cat in checker.check(refent, l10nent):
if isinstance(pos, EntityPos):
line, col = l10nent.position(pos)
else:
line, col = l10nent.value_position(pos)
# skip error entities when merging
if tp == 'error' and merge_file is not None:
skips.append(l10nent)
self.observers.notify(
tp, l10n, u"%s at line %d, column %d for %s" %
(msg, line, col, refent.key))
pass
if merge_file is not None:
self.merge(ref_entities, ref_file, l10n, merge_file, missings,
skips, l10n_ctx, p.capabilities, p.encoding)
stats = {
'missing': missing,
'missing_w': missing_w,
'report': report,
'obsolete': obsolete,
'changed': changed,
'changed_w': changed_w,
'unchanged': unchanged,
'unchanged_w': unchanged_w,
'keys': keys,
}
self.observers.updateStats(l10n, stats)
pass
for m in self.nl.finditer(p.contents):
lines.append(m.end())
for i in xrange(len(lines), 0, -1):
if offset >= lines[i - 1]:
return (i, offset - lines[i - 1])
return (1, offset)
l10n_list = l10n_map.keys()
l10n_list.sort()
ar = AddRemove()
ar.set_left(ref_list)
ar.set_right(l10n_list)
report = missing = obsolete = changed = unchanged = keys = 0
missings = []
skips = []
checker = getChecker(l10n, reference=ref[0])
for action, item_or_pair in ar:
if action == 'delete':
# missing entity
_rv = self.notify('missingEntity', l10n, item_or_pair)
if _rv == "ignore":
continue
if _rv == "error":
# only add to missing entities for l10n-merge on error,
# not report
missings.append(item_or_pair)
missing += 1
else:
# just report
report += 1
elif action == 'add':