def convertmixedunit(self, labeldtd, accesskeydtd):
labelpo = self.convertunit(labeldtd)
accesskeypo = self.convertunit(accesskeydtd)
if labelpo is None:
return accesskeypo
if accesskeypo is None:
return labelpo
thepo = po.pounit(encoding="UTF-8")
thepo.addlocations(labelpo.getlocations())
thepo.addlocations(accesskeypo.getlocations())
thepo.msgidcomment = thepo._extract_msgidcomments() + labelpo._extract_msgidcomments()
thepo.msgidcomment = thepo._extract_msgidcomments() + accesskeypo._extract_msgidcomments()
thepo.addnote(labelpo.getnotes("developer"), "developer")
thepo.addnote(accesskeypo.getnotes("developer"), "developer")
thepo.addnote(labelpo.getnotes("translator"), "translator")
thepo.addnote(accesskeypo.getnotes("translator"), "translator")
# redo the strings from original dtd...
label = dtd.unquotefromdtd(labeldtd.definition).decode('UTF-8')
accesskey = dtd.unquotefromdtd(accesskeydtd.definition).decode('UTF-8')
label = accesskeyfn.combine(label, accesskey)
if label is None:
return None
thepo.source = label
thepo.target = ""
return thepo
def convertmixedunit(self, labeldtd, accesskeydtd):
labelpo = self.convertunit(labeldtd)
accesskeypo = self.convertunit(accesskeydtd)
if labelpo is None:
return accesskeypo
if accesskeypo is None:
return labelpo
thepo = po.pounit(encoding="UTF-8")
thepo.addlocations(labelpo.getlocations())
thepo.addlocations(accesskeypo.getlocations())
thepo.msgidcomment = thepo._extract_msgidcomments() + labelpo._extract_msgidcomments()
thepo.msgidcomment = thepo._extract_msgidcomments() + accesskeypo._extract_msgidcomments()
thepo.addnote(labelpo.getnotes("developer"), "developer")
thepo.addnote(accesskeypo.getnotes("developer"), "developer")
thepo.addnote(labelpo.getnotes("translator"), "translator")
thepo.addnote(accesskeypo.getnotes("translator"), "translator")
# redo the strings from original dtd...
label = dtd.unquotefromdtd(labeldtd.definition).decode('UTF-8')
accesskey = dtd.unquotefromdtd(accesskeydtd.definition).decode('UTF-8')
label = accesskeyfn.combine(label, accesskey)
if label is None:
return None
thepo.source = label
thepo.target = ""
return thepo
def test_accesskeycase(self):
"""tests that access keys come out with the same case as the original, regardless"""
simplepo_template = '''#: simple.label\n#: simple.accesskey\nmsgid "%s"\nmsgstr "%s"\n'''
simpledtd_template = '''<!ENTITY simple.label "Simple %s">\n<!ENTITY simple.accesskey "%s">'''
possibilities = [
#(en label, en akey, en po, af po, af label, expected af akey)
("Sis", "S", "&Sis", "&Sies", "Sies", "S"),
("Sis", "s", "Si&s", "&Sies", "Sies", "S"),
("Sis", "S", "&Sis", "Sie&s", "Sies", "s"),
("Sis", "s", "Si&s", "Sie&s", "Sies", "s"),
# untranslated strings should have the casing of the source
("Sis", "S", "&Sis", "", "Sis", "S"),
("Sis", "s", "Si&s", "", "Sis", "s"),
("Suck", "S", "&Suck", "", "Suck", "S"),
("Suck", "s", "&Suck", "", "Suck", "s"),
]
for (en_label, en_akey, po_source, po_target, target_label,
target_akey) in possibilities:
simplepo = simplepo_template % (po_source, po_target)
simpledtd = simpledtd_template % (en_label, en_akey)
dtdfile = self.merge2dtd(simpledtd, simplepo)
dtdfile.makeindex()
accel = dtd.unquotefromdtd(
dtdfile.id_index["simple.accesskey"].definition)
assert accel == target_akey
def applytranslation(entity, dtdunit, inputunit, mixedentities):
"""applies the translation for entity in the po unit to the dtd unit"""
# this converts the po-style string to a dtd-style string
unquotedstr = inputunit.target
# check there aren't missing entities...
if len(unquotedstr.strip()) == 0:
return
# handle mixed entities
for labelsuffix in dtd.labelsuffixes:
if entity.endswith(labelsuffix):
if entity in mixedentities:
unquotedstr, akey = accesskey.extract(unquotedstr)
break
else:
for akeytype in dtd.accesskeysuffixes:
if entity.endswith(akeytype):
if entity in mixedentities:
label, unquotedstr = accesskey.extract(unquotedstr)
if not unquotedstr:
warnings.warn("Could not find accesskey for %s" %
entity)
else:
original = dtd.unquotefromdtd(dtdunit.definition)
# For the sake of diffs we keep the case of the
# accesskey the same if we know the translation didn't
# change. Casing matters in XUL.
if unquotedstr == dtdunit.source and original.lower(
) == unquotedstr.lower():
if original.isupper():
unquotedstr = unquotedstr.upper()
elif original.islower():
unquotedstr = unquotedstr.lower()
if len(unquotedstr) > 0:
dtdunit.definition = dtd.quotefordtd(
dtd.removeinvalidamps(entity, unquotedstr))
def applytranslation(entity, dtdunit, inputunit, mixedentities):
"""applies the translation for entity in the po unit to the dtd unit"""
# this converts the po-style string to a dtd-style string
unquotedstr = inputunit.target
# check there aren't missing entities...
if len(unquotedstr.strip()) == 0:
return
# handle mixed entities
for labelsuffix in dtd.labelsuffixes:
if entity.endswith(labelsuffix):
if entity in mixedentities:
unquotedstr, akey = accesskey.extract(unquotedstr)
break
else:
for akeytype in dtd.accesskeysuffixes:
if entity.endswith(akeytype):
if entity in mixedentities:
label, unquotedstr = accesskey.extract(unquotedstr)
if not unquotedstr:
warnings.warn("Could not find accesskey for %s" % entity)
else:
original = dtd.unquotefromdtd(dtdunit.definition)
# For the sake of diffs we keep the case of the
# accesskey the same if we know the translation didn't
# change. Casing matters in XUL.
if unquotedstr == dtdunit.source and original.lower() == unquotedstr.lower():
if original.isupper():
unquotedstr = unquotedstr.upper()
elif original.islower():
unquotedstr = unquotedstr.lower()
if len(unquotedstr) > 0:
dtdunit.definition = dtd.quotefordtd(dtd.removeinvalidamps(entity, unquotedstr))
def openDTD(file): f = open(file, "r") dtdobj = dtd.dtdfile(f) f.close() dtdentries = dtdobj.units d = OrderedDict() for i in dtdentries: if d.has_key(i.entity): print "Duplicate entry: %s" %i.entity else: d[i.entity] = dtd.unquotefromdtd(i.definition)
def test_invalid_quoting(self):
"""checks that invalid quoting doesn't work - quotes can't be reopened"""
# TODO: we should rather raise an error
dtdsource = '<!ENTITY test.me "bananas for sale""room">\n'
assert dtd.unquotefromdtd(dtdsource[dtdsource.find('"'):]) == 'bananas for sale'
dtdfile = self.dtdparse(dtdsource)
assert len(dtdfile.units) == 1
dtdunit = dtdfile.units[0]
assert dtdunit.definition == '"bananas for sale"'
assert dtdfile.serialize() == '<!ENTITY test.me "bananas for sale">\n'
def test_invalid_quoting(self):
"""checks that invalid quoting doesn't work - quotes can't be reopened"""
# TODO: we should rather raise an error
dtdsource = '<!ENTITY test.me "bananas for sale""room">\n'
assert dtd.unquotefromdtd(dtdsource[dtdsource.find('"'):]) == 'bananas for sale'
dtdfile = self.dtdparse(dtdsource)
assert len(dtdfile.units) == 1
dtdunit = dtdfile.units[0]
assert dtdunit.definition == '"bananas for sale"'
assert dtdfile.serialize() == '<!ENTITY test.me "bananas for sale">\n'
def test_accesskey_types(self):
"""tests that we can detect the various styles of accesskey"""
simplepo_template = '''#: simple.%s\n#: simple.%s\nmsgid "&File"\nmsgstr "F&aele"\n'''
simpledtd_template = '''<!ENTITY simple.%s "File">\n<!ENTITY simple.%s "a">'''
for label in ("label", "title"):
for accesskey in ("accesskey", "accessKey", "akey"):
simplepo = simplepo_template % (label, accesskey)
simpledtd = simpledtd_template % (label, accesskey)
dtdfile = self.merge2dtd(simpledtd, simplepo)
dtdfile.makeindex()
assert dtd.unquotefromdtd(dtdfile.index["simple.%s" % accesskey].definition) == "a"
def test_accesskey_types(self):
"""tests that we can detect the various styles of accesskey"""
simplepo_template = '''#: simple.%s\n#: simple.%s\nmsgid "&File"\nmsgstr "F&aele"\n'''
simpledtd_template = '''<!ENTITY simple.%s "File">\n<!ENTITY simple.%s "a">'''
for label in ("label", "title"):
for accesskey in ("accesskey", "accessKey", "akey"):
simplepo = simplepo_template % (label, accesskey)
simpledtd = simpledtd_template % (label, accesskey)
dtdfile = self.merge2dtd(simpledtd, simplepo)
dtdfile.makeindex()
assert dtd.unquotefromdtd(dtdfile.index["simple.%s" % accesskey].definition) == "a"
def test_roundtrip_quoting():
specials = ['Fish & chips', 'five < six', 'six > five',
'Use ', 'Use &nbsp;'
'A "solution"', "skop 'n bal", '"""', "'''",
'\n', '\t', '\r',
'Escape at end \\',
'\\n', '\\t', '\\r', '\\"', '\r\n', '\\r\\n', '\\']
for special in specials:
quoted_special = dtd.quotefordtd(special)
unquoted_special = dtd.unquotefromdtd(quoted_special)
print "special: %r\nquoted: %r\nunquoted: %r\n" % (special, quoted_special, unquoted_special)
assert special == unquoted_special
def test_roundtrip_quoting():
specials = [
'Fish & chips', 'five < six', 'six > five', 'Use ',
'Use &nbsp;'
'A "solution"', "skop 'n bal", '"""', "'''", '\n', '\t', '\r',
'Escape at end \\', '\\n', '\\t', '\\r', '\\"', '\r\n', '\\r\\n', '\\'
]
for special in specials:
quoted_special = dtd.quotefordtd(special)
unquoted_special = dtd.unquotefromdtd(quoted_special)
print "special: %r\nquoted: %r\nunquoted: %r\n" % (
special, quoted_special, unquoted_special)
assert special == unquoted_special
def convertstrings(self, thedtd, thepo):
# extract the string, get rid of quoting
unquoted = dtd.unquotefromdtd(thedtd.definition).replace("\r", "")
# escape backslashes... but not if they're for a newline
# unquoted = unquoted.replace("\\", "\\\\").replace("\\\\n", "\\n")
# now split the string into lines and quote them
lines = unquoted.split('\n')
while lines and not lines[0].strip():
del lines[0]
while lines and not lines[-1].strip():
del lines[-1]
# quotes have been escaped already by escapeforpo, so just add the start and end quotes
if len(lines) > 1:
thepo.source = "\n".join([lines[0].rstrip() + ' '] + \
[line.strip() + ' ' for line in lines[1:-1]] + \
[lines[-1].lstrip()])
elif lines:
thepo.source = lines[0]
else:
thepo.source = ""
thepo.target = ""
def convertstrings(self, thedtd, thepo):
# extract the string, get rid of quoting
unquoted = dtd.unquotefromdtd(thedtd.definition).replace("\r", "")
# escape backslashes... but not if they're for a newline
# unquoted = unquoted.replace("\\", "\\\\").replace("\\\\n", "\\n")
# now split the string into lines and quote them
lines = unquoted.split('\n')
while lines and not lines[0].strip():
del lines[0]
while lines and not lines[-1].strip():
del lines[-1]
# quotes have been escaped already by escapeforpo, so just add the start and end quotes
if len(lines) > 1:
thepo.source = "\n".join([lines[0].rstrip() + ' '] + \
[line.strip() + ' ' for line in lines[1:-1]] + \
[lines[-1].lstrip()])
elif lines:
thepo.source = lines[0]
else:
thepo.source = ""
thepo.target = ""
def test_roundtrip_quoting():
specials = [
"Fish & chips",
"five < six",
"six > five",
"Use ",
'Use &nbsp;A "solution"',
"skop 'n bal",
'"""',
"'''",
"\n",
"\t",
"\r",
"Escape at end \\",
"",
"\\n",
"\\t",
"\\r",
'\\"',
"\r\n",
"\\r\\n",
"\\",
"Completed %S",
"&blockAttackSites;",
" ",
"&intro-point2-a;",
"&basePBMenu.label;",
# "Don't buy",
# "Don't \"buy\"",
'A "thing"',
'<a href="http',
]
for special in specials:
quoted_special = dtd.quotefordtd(special)
unquoted_special = dtd.unquotefromdtd(quoted_special)
print(
"special: %r\nquoted: %r\nunquoted: %r\n"
% (special, quoted_special, unquoted_special)
)
assert special == unquoted_special
def test_roundtrip_quoting():
specials = [
'Fish & chips',
'five < six',
'six > five',
'Use ',
'Use &nbsp;A "solution"',
"skop 'n bal",
'"""',
"'''",
'\n',
'\t',
'\r',
'Escape at end \\',
'',
'\\n',
'\\t',
'\\r',
'\\"',
'\r\n',
'\\r\\n',
'\\',
"Completed %S",
"&blockAttackSites;",
" ",
"&intro-point2-a;",
"&basePBMenu.label;",
#"Don't buy",
#"Don't \"buy\"",
"A \"thing\"",
"<a href=\"http"
]
for special in specials:
quoted_special = dtd.quotefordtd(special)
unquoted_special = dtd.unquotefromdtd(quoted_special)
print("special: %r\nquoted: %r\nunquoted: %r\n" % (special,
quoted_special,
unquoted_special))
assert special == unquoted_special
def test_roundtrip_quoting():
specials = [
'Fish & chips',
'five < six',
'six > five',
'Use ',
'Use &nbsp;A "solution"',
"skop 'n bal",
'"""',
"'''",
'\n',
'\t',
'\r',
'Escape at end \\',
'',
'\\n',
'\\t',
'\\r',
'\\"',
'\r\n',
'\\r\\n',
'\\',
"Completed %S",
"&blockAttackSites;",
" ",
"&intro-point2-a;",
"&basePBMenu.label;",
#"Don't buy",
#"Don't \"buy\"",
"A \"thing\"",
"<a href=\"http"
]
for special in specials:
quoted_special = dtd.quotefordtd(special)
unquoted_special = dtd.unquotefromdtd(quoted_special)
print("special: %r\nquoted: %r\nunquoted: %r\n" % (special,
quoted_special,
unquoted_special))
assert special == unquoted_special
def test_accesskeycase(self):
"""tests that access keys come out with the same case as the original, regardless"""
simplepo_template = '''#: simple.label\n#: simple.accesskey\nmsgid "%s"\nmsgstr "%s"\n'''
simpledtd_template = '''<!ENTITY simple.label "Simple %s">\n<!ENTITY simple.accesskey "%s">'''
possibilities = [
#(en label, en akey, en po, af po, af label, expected af akey)
("Sis", "S", "&Sis", "&Sies", "Sies", "S"),
("Sis", "s", "Si&s", "&Sies", "Sies", "S"),
("Sis", "S", "&Sis", "Sie&s", "Sies", "s"),
("Sis", "s", "Si&s", "Sie&s", "Sies", "s"),
# untranslated strings should have the casing of the source
("Sis", "S", "&Sis", "", "Sis", "S"),
("Sis", "s", "Si&s", "", "Sis", "s"),
("Suck", "S", "&Suck", "", "Suck", "S"),
("Suck", "s", "&Suck", "", "Suck", "s"),
]
for (en_label, en_akey, po_source, po_target, target_label, target_akey) in possibilities:
simplepo = simplepo_template % (po_source, po_target)
simpledtd = simpledtd_template % (en_label, en_akey)
dtdfile = self.merge2dtd(simpledtd, simplepo)
dtdfile.makeindex()
accel = dtd.unquotefromdtd(dtdfile.index["simple.accesskey"].definition)
assert accel == target_akey
def tester(raw_original, dtd_ready_result):
#print dtd.quotefordtd(raw_original)
assert dtd.quotefordtd(raw_original) == dtd_ready_result
#print dtd.unquotefromdtd(dtd_ready_result)
assert dtd.unquotefromdtd(dtd_ready_result) == raw_original
def test_unquotefromdtd_unimplemented_cases():
"""Test unimplemented unquoting DTD cases."""
assert dtd.unquotefromdtd('"<p> and </p>"') == "<p> and </p>"
def tester(raw_original, dtd_ready_result):
#print dtd.quotefordtd(raw_original)
assert dtd.quotefordtd(raw_original) == dtd_ready_result
#print dtd.unquotefromdtd(dtd_ready_result)
assert dtd.unquotefromdtd(dtd_ready_result) == raw_original
def test_unquotefromdtd():
"""Test unquoting DTD definitions"""
# %
assert dtd.unquotefromdtd('"Completed %S"') == "Completed %S"
assert dtd.unquotefromdtd('"Completed %S"') == "Completed %S"
assert dtd.unquotefromdtd('"Completed %S"') == "Completed %S"
# &entity;
assert dtd.unquotefromdtd('"Color&light █"') == "Color&light █"
assert dtd.unquotefromdtd('"Color & Light; Red"') == "Color & Light; Red"
assert dtd.unquotefromdtd('"&blockAttackSites;"') == "&blockAttackSites;"
assert dtd.unquotefromdtd('"&intro-point2-a;"') == "&intro-point2-a;"
assert dtd.unquotefromdtd('"&basePBMenu.label"') == "&basePBMenu.label"
# &
assert dtd.unquotefromdtd('"Color & Light"') == "Color & Light"
assert dtd.unquotefromdtd('"Color & █"') == "Color & █"
# nbsp
assert dtd.unquotefromdtd('" "') == " "
# '
assert dtd.unquotefromdtd("'Don't buy'") == "Don't buy"
# "
assert dtd.unquotefromdtd("'Don't "buy"'") == 'Don\'t "buy"'
assert dtd.unquotefromdtd('"A "thing""') == "A \"thing\""
assert dtd.unquotefromdtd('"A "thing""') == "A \"thing\""
assert dtd.unquotefromdtd("'<a href=\"http'") == "<a href=\"http"
# other chars
assert dtd.unquotefromdtd('"»"') == u"»"
def test_unquotefromdtd_unimplemented_cases():
"""Test unimplemented unquoting DTD cases."""
assert dtd.unquotefromdtd('"Color & Light"') == "Color & Light"
assert dtd.unquotefromdtd('"Color & █"') == "Color & █"
assert dtd.unquotefromdtd('"<p> and </p>"') == "<p> and </p>"
def test_unquotefromdtd_unimplemented_cases():
"""Test unimplemented unquoting DTD cases."""
assert dtd.unquotefromdtd('"<p> and </p>"') == "<p> and </p>"
def test_unquotefromdtd():
"""Test unquoting DTD definitions"""
# %
assert dtd.unquotefromdtd('"Completed %S"') == "Completed %S"
assert dtd.unquotefromdtd('"Completed %S"') == "Completed %S"
assert dtd.unquotefromdtd('"Completed %S"') == "Completed %S"
# &entity;
assert dtd.unquotefromdtd('"Color&light █"') == "Color&light █"
assert dtd.unquotefromdtd('"Color & Light; Red"') == "Color & Light; Red"
assert dtd.unquotefromdtd('"&blockAttackSites;"') == "&blockAttackSites;"
assert dtd.unquotefromdtd('"&intro-point2-a;"') == "&intro-point2-a;"
assert dtd.unquotefromdtd('"&basePBMenu.label"') == "&basePBMenu.label"
# &
assert dtd.unquotefromdtd('"Color & Light"') == "Color & Light"
assert dtd.unquotefromdtd('"Color & █"') == "Color & █"
# nbsp
assert dtd.unquotefromdtd('" "') == " "
# '
assert dtd.unquotefromdtd("'Don't buy'") == "Don't buy"
# "
assert dtd.unquotefromdtd("'Don't "buy"'") == 'Don\'t "buy"'
assert dtd.unquotefromdtd('"A "thing""') == 'A "thing"'
assert dtd.unquotefromdtd('"A "thing""') == 'A "thing"'
assert dtd.unquotefromdtd("'<a href=\"http'") == '<a href="http'
# other chars
assert dtd.unquotefromdtd('"»"') == "»"
def test_unquotefromdtd_unimplemented_cases():
"""Test unimplemented unquoting DTD cases."""
assert dtd.unquotefromdtd('"Color & Light"') == "Color & Light"
assert dtd.unquotefromdtd('"Color & █"') == "Color & █"
assert dtd.unquotefromdtd('"<p> and </p>"') == "<p> and </p>"
def test_unquotefromdtd():
"""Test unquoting DTD definitions"""
assert dtd.unquotefromdtd('"Completed %S"') == "Completed %S"
assert dtd.unquotefromdtd('"Completed %S"') == "Completed %S"
assert dtd.unquotefromdtd('"Completed %S"') == "Completed %S"
assert dtd.unquotefromdtd('"Color&light █"') == "Color&light █"
assert dtd.unquotefromdtd('"Color & Light; Red"') == "Color & Light; Red"
assert dtd.unquotefromdtd('"&blockAttackSites;"') == "&blockAttackSites;"
assert dtd.unquotefromdtd('" "') == " "
assert dtd.unquotefromdtd('"&intro-point2-a;"') == "&intro-point2-a;"
assert dtd.unquotefromdtd('"&basePBMenu.label"') == "&basePBMenu.label"
assert dtd.unquotefromdtd("'Don't buy'") == "Don't buy"
assert dtd.unquotefromdtd("'Don't "buy"'") == 'Don\'t "buy"'
assert dtd.unquotefromdtd('"A "thing""') == "A \"thing\""
assert dtd.unquotefromdtd("'<a href=\"http'") == "<a href=\"http"
