def test_python_tokenizing():
vartok = VariableTokenizer(['python'])
for string, expected in [
('Hello %s', ['Hello ', '%s', '']),
('Hello %(username)s', ['Hello ', '%(username)s', '']),
('Hello %(user)s%(name)s', ['Hello ', '%(user)s', '', '%(name)s', '']),
('Hello {username}', ['Hello ', '{username}', '']),
('Hello {user}{name}', ['Hello ', '{user}', '', '{name}', '']),
('Products and Services', ['Products and Services']),
]:
eq_(vartok.tokenize(string), expected)
def test_python_tokenizing():
vartok = VariableTokenizer(['pysprintf', 'pyformat'])
data = [
('Hello %s', ['Hello ', '%s', '']),
('Hello %(username)s', ['Hello ', '%(username)s', '']),
('Hello %(user)s%(name)s', ['Hello ', '%(user)s', '', '%(name)s', '']),
('Hello {username}', ['Hello ', '{username}', '']),
('Hello {user}{name}', ['Hello ', '{user}', '', '{name}', '']),
('Products and Services', ['Products and Services']),
]
for text, expected in data:
eq_(vartok.tokenize(text), expected)
def test_python_tokenizing():
vartok = VariableTokenizer(['python-format', 'python-brace-format'])
data = [
('Hello %s', ['Hello ', '%s', '']),
('Hello %(username)s', ['Hello ', '%(username)s', '']),
('Hello %(user)s%(name)s', ['Hello ', '%(user)s', '', '%(name)s', '']),
('Hello {username}', ['Hello ', '{username}', '']),
('Hello {user}{name}', ['Hello ', '{user}', '', '{name}', '']),
('Products and Services', ['Products and Services']),
]
for text, expected in data:
eq_(vartok.tokenize(text), expected)
class Linter(object):
def __init__(self, var_types):
# FIXME - this is a horrible name
self.vartok = VariableTokenizer(var_types)
def verify_file(self, filename_or_string):
"""Verifies strings in file.
:arg filename_or_string: filename to verify or the contents of
a pofile as a string
:returns: for each string in the pofile, this returns a None
if there were no issues or a LintError if there were
issues
:raises IOError: if the file is not a valid .po file or
doesn't exist
"""
po = polib.pofile(filename_or_string)
results = []
for entry in po:
if not entry.msgid_plural:
if not entry.msgid and entry.msgstr:
continue
id_tokens = self.vartok.extract_tokens(entry.msgid)
str_tokens = self.vartok.extract_tokens(entry.msgstr)
results.append(
LintItem(entry, entry.msgid, entry.msgid,
id_tokens, entry.msgstr,
str_tokens, None))
else:
for key in sorted(entry.msgstr_plural.keys()):
if key == '0':
# This is the 1 case.
text = entry.msgid
else:
text = entry.msgid_plural
id_tokens = self.vartok.extract_tokens(text)
str_tokens = self.vartok.extract_tokens(
entry.msgstr_plural[key])
results.append(
LintItem(entry, entry.msgid, text, id_tokens,
entry.msgstr_plural[key],
str_tokens, key))
return results
def test_varformat_empty(self):
vartok = VariableTokenizer([])
linted_entry = build_linted_entry('#: foo/foo.py:5\n'
'msgid "%s foo"\n'
'msgstr "%a FOO"\n')
msgs = self.lintrule.lint(vartok, linted_entry)
assert msgs == []
def test_varformat_empty(self):
vartok = VariableTokenizer([])
linted_entry = build_linted_entry("#: foo/foo.py:5\n"
'msgid "Foo {bar}"\n'
'msgstr "Oof: {foo}"\n')
msgs = self.lintrule.lint(vartok, linted_entry)
assert msgs == []
def test_empty_tokenizer():
vartok = VariableTokenizer([])
assert vartok.contains('python-format') is False
assert vartok.tokenize('a b c d e') == ['a b c d e']
assert vartok.extract_tokens('a b c d e') == set()
assert vartok.is_token('{0}') is False
assert vartok.extract_variable_name('{0}') is None
def test_empty_tokenizer():
vartok = VariableTokenizer([])
assert vartok.contains("python-format") is False
assert vartok.tokenize("a b c d e") == ["a b c d e"]
assert vartok.extract_tokens("a b c d e") == set()
assert vartok.is_token("{0}") is False
assert vartok.extract_variable_name("{0}") is None
def test_varformat_empty(self):
vartok = VariableTokenizer([])
linted_entry = build_linted_entry(
'#: kitsune/kbforums/feeds.py:23\n'
'msgid "Recently updated threads about %s"\n'
'msgstr "RECENTLY UPDATED THREADS"\n')
msgs = self.lintrule.lint(vartok, linted_entry)
assert msgs == []
def test_varformat_empty(self):
vartok = VariableTokenizer([])
linted_entry = build_linted_entry(
'msgid "Value for key \\"{0}\\" exceeds the length of {1}"\n'
'msgstr "Valor para la clave \\"{0}\\" excede el tamano de {1]"\n')
msgs = self.lintrule.lint(vartok, linted_entry)
assert msgs == []
class HTMLExtractorTest(TestCase):
vartok = VariableTokenizer(['python-format', 'python-brace-format'])
def test_basic(self):
trans = HTMLExtractorTransform()
output = trans.transform(self.vartok, [Token('')])
eq_(output, [Token(u'', 'text', True)])
output = trans.transform(self.vartok, [Token('<b>hi</b>')])
eq_(output, [
Token(u'<b>', 'html', False),
Token(u'hi', 'text', True),
Token(u'</b>', 'html', False),
])
def test_alt_title_placeholder(self):
trans = HTMLExtractorTransform()
output = trans.transform(self.vartok, [Token('<img alt="foo">')])
eq_(output, [
Token(u'<img alt="', 'html', False),
Token(u'foo', 'text', True),
Token(u'">', 'html', False),
])
output = trans.transform(self.vartok, [Token('<img title="foo">')])
eq_(output, [
Token(u'<img title="', 'html', False),
Token(u'foo', 'text', True),
Token(u'">', 'html', False),
])
output = trans.transform(self.vartok,
[Token('<input placeholder="foo">')])
eq_(output, [
Token(u'<input placeholder="', 'html', False),
Token(u'foo', 'text', True),
Token(u'">', 'html', False),
])
def test_script_style(self):
trans = HTMLExtractorTransform()
output = trans.transform(
self.vartok, [Token('<style>TR {white-space: nowrap;}</style>')])
eq_(output, [
Token(u'<style>', 'html', False),
Token(u'TR {white-space: nowrap;}', 'style', False),
Token(u'</style>', 'html', False)
])
output = trans.transform(
self.vartok, [Token('<script>console.log("foo");</script>')])
eq_(output, [
Token(u'<script>', 'html', False),
Token(u'console.log("foo");', 'script', False),
Token(u'</script>', 'html', False)
])
def test_varformat_empty(self):
vartok = VariableTokenizer([])
linted_entry = build_linted_entry(
'#: kitsune/questions/templates/questions/question_details.html:14\n'
'msgid "{q} | {product} Support Forum"\n'
'msgstr "{q} | {product}} foo bar"\n')
msgs = self.lintrule.lint(vartok, linted_entry)
assert msgs == []
def test_varformat_empty(self):
vartok = VariableTokenizer([])
linted_entry = build_linted_entry(
'#: kitsune/questions/templates/questions/answers.html:56\n'
'msgid "%(count)s view"\n'
'msgid_plural "%(count)s views"\n'
'msgstr[0] "%(count) zoo"\n')
msgs = self.lintrule.lint(vartok, linted_entry)
assert msgs == []
def test_empty_tokenizer():
vartok = VariableTokenizer([])
assert vartok.contains('python-format') is False
assert vartok.tokenize('a b c d e') == ['a b c d e']
assert vartok.extract_tokens('a b c d e') == set()
assert vartok.is_token('{0}') is False
assert vartok.extract_variable_name('{0}') is None
class LintRuleTestCase:
vartok = VariableTokenizer(["python-format", "python-brace-format"])
class TransformTestCase:
vartok = VariableTokenizer(["python-format", "python-brace-format"])
class TestHTMLExtractor:
vartok = VariableTokenizer(["python-format", "python-brace-format"])
def test_basic(self):
trans = HTMLExtractorTransform()
output = trans.transform(self.vartok, [Token("")])
assert output == [Token("", "text", True)]
output = trans.transform(self.vartok, [Token("<b>hi</b>")])
assert output == [
Token("<b>", "html", False),
Token("hi", "text", True),
Token("</b>", "html", False),
]
def test_alt_title_placeholder(self):
trans = HTMLExtractorTransform()
output = trans.transform(self.vartok, [Token('<img alt="foo">')])
assert output == [
Token('<img alt="', "html", False),
Token("foo", "text", True),
Token('">', "html", False),
]
output = trans.transform(self.vartok, [Token('<img title="foo">')])
assert output == [
Token('<img title="', "html", False),
Token("foo", "text", True),
Token('">', "html", False),
]
output = trans.transform(self.vartok, [Token('<input placeholder="foo">')])
assert output == [
Token('<input placeholder="', "html", False),
Token("foo", "text", True),
Token('">', "html", False),
]
def test_script_style(self):
trans = HTMLExtractorTransform()
output = trans.transform(
self.vartok, [Token("<style>TR {white-space: nowrap;}</style>")]
)
assert output == [
Token("<style>", "html", False),
Token("TR {white-space: nowrap;}", "style", False),
Token("</style>", "html", False),
]
output = trans.transform(
self.vartok, [Token('<script>console.log("foo");</script>')]
)
assert output == [
Token("<script>", "html", False),
Token('console.log("foo");', "script", False),
Token("</script>", "html", False),
]
def test_whitespace_collapse(self):
def _trans(text):
return HTMLExtractorTransform().transform(self.vartok, [Token(text)])
assert _trans("<html><body>hello!</body></html>") == _trans(
"<html><body> hello! </body></html>"
)
def test_parse(self, text, expected):
vartok = VariableTokenizer(['python-brace-format'])
assert vartok.tokenize(text) == expected
class LintRuleTestCase:
vartok = VariableTokenizer(['python-format', 'python-brace-format'])
def test_parse(self, text, expected):
vartok = VariableTokenizer(["python-brace-format"])
assert vartok.tokenize(text) == expected
def test_python_tokenizing(text, expected):
vartok = VariableTokenizer(["python-format", "python-brace-format"])
assert vartok.tokenize(text) == expected
class TLRTestCase(TestCase):
vartok = VariableTokenizer(['python-format', 'python-brace-format'])
def __init__(self, vars_, rules_spec):
self.vartok = VariableTokenizer(vars_)
self.rules_spec = rules_spec
self.rules = convert_rules(self.rules_spec)
def __init__(self, var_types):
# FIXME - this is a horrible name
self.vartok = VariableTokenizer(var_types)
def test_python_tokenizing(text, expected):
vartok = VariableTokenizer(['python-format', 'python-brace-format'])
assert vartok.tokenize(text) == expected
class TransformTestCase:
vartok = VariableTokenizer(['python-format', 'python-brace-format'])
def __init__(self, variable_formats, pipeline_spec):
self.vartok = VariableTokenizer(variable_formats)
self.pipeline_spec = pipeline_spec
self._pipeline = convert_pipeline(self.pipeline_spec)
class TestHTMLExtractor:
vartok = VariableTokenizer(['python-format', 'python-brace-format'])
def test_basic(self):
trans = HTMLExtractorTransform()
output = trans.transform(self.vartok, [Token('')])
assert output == [Token(u'', 'text', True)]
output = trans.transform(self.vartok, [Token('<b>hi</b>')])
assert (output == [
Token(u'<b>', 'html', False),
Token(u'hi', 'text', True),
Token(u'</b>', 'html', False),
])
def test_alt_title_placeholder(self):
trans = HTMLExtractorTransform()
output = trans.transform(self.vartok, [Token('<img alt="foo">')])
assert (output == [
Token(u'<img alt="', 'html', False),
Token(u'foo', 'text', True),
Token(u'">', 'html', False),
])
output = trans.transform(self.vartok, [Token('<img title="foo">')])
assert (output == [
Token(u'<img title="', 'html', False),
Token(u'foo', 'text', True),
Token(u'">', 'html', False),
])
output = trans.transform(self.vartok,
[Token('<input placeholder="foo">')])
assert (output == [
Token(u'<input placeholder="', 'html', False),
Token(u'foo', 'text', True),
Token(u'">', 'html', False),
])
def test_script_style(self):
trans = HTMLExtractorTransform()
output = trans.transform(
self.vartok, [Token('<style>TR {white-space: nowrap;}</style>')])
assert (output == [
Token(u'<style>', 'html', False),
Token(u'TR {white-space: nowrap;}', 'style', False),
Token(u'</style>', 'html', False)
])
output = trans.transform(
self.vartok, [Token('<script>console.log("foo");</script>')])
assert (output == [
Token(u'<script>', 'html', False),
Token(u'console.log("foo");', 'script', False),
Token(u'</script>', 'html', False)
])
def test_whitespace_collapse(self):
def _trans(text):
return HTMLExtractorTransform().transform(self.vartok,
[Token(text)])
assert (_trans('<html><body>hello!</body></html>') == _trans(
'<html><body> hello! </body></html>'))
