def test_utf8_bom():
tree = marso.parse(unicode_bom + 'a = 1')
expr_stmt = tree.children[0]
assert expr_stmt.start_pos == (1, 0)
tree = marso.parse(unicode_bom + '\n')
endmarker = tree.children[0]
parts = list(endmarker._split_prefix())
assert [p.type for p in parts] == ['bom', 'newline', 'spacing']
assert [p.start_pos for p in parts] == [(1, 0), (1, 0), (2, 0)]
assert [p.end_pos for p in parts] == [(1, 0), (2, 0), (2, 0)]
def test_utf8_bom():
unicode_bom = BOM_UTF8.decode('utf-8')
module = marso.parse(unicode_bom)
endmarker = module.children[0]
assert endmarker.type == 'endmarker'
assert unicode_bom == endmarker.prefix
module = marso.parse(unicode_bom + 'foo = 1')
expr_stmt = module.children[0]
assert expr_stmt.type == 'expr_stmt'
assert unicode_bom == expr_stmt.get_first_leaf().prefix
def test_one_line_function(each_version):
module = parse('def x(): f.', version=each_version)
assert module.children[0].type == 'funcdef'
def_, name, parameters, colon, f = module.children[0].children
assert f.type == 'error_node'
module = parse('def x(a:', version=each_version)
func = module.children[0]
assert func.type == 'error_node'
if each_version.startswith('2'):
assert func.children[-1].value == 'a'
else:
assert func.children[-1] == ':'
def test_python2_octal(each_version):
module = parse('0660', version=each_version)
first = module.children[0]
if each_version.startswith('2'):
assert first.type == 'number'
else:
assert first.type == 'error_node'
def node(self, request):
parsed = parse(dedent(request.param[0]), version='3.6')
request.keywords['expected'] = request.param[1]
child = parsed.children[0]
if child.type == 'simple_stmt':
child = child.children[0]
return child
def test_get_code():
"""Use the same code that the parser also generates, to compare"""
s = '''"""a docstring"""
class SomeClass(object, mixin):
def __init__(self):
self.xy = 3.0
"""statement docstr"""
def some_method(self):
return 1
def yield_method(self):
while hasattr(self, 'xy'):
yield True
for x in [1, 2]:
yield x
def empty(self):
pass
class Empty:
pass
class WithDocstring:
"""class docstr"""
pass
def method_with_docstring():
"""class docstr"""
pass
'''
assert parse(s).get_code() == s
def test_ellipsis_py2(each_py2_version):
module = parse('[0][...]', version=each_py2_version, error_recovery=False)
expr = module.children[0]
trailer = expr.children[-1]
subscript = trailer.children[1]
assert subscript.type == 'subscript'
assert [leaf.value for leaf in subscript.children] == ['.', '.', '.']
def test_round_trip():
code = dedent('''
def x():
"""hahaha"""
func''')
assert parse(code).get_code() == code
def test_dont_break_imports_without_namespaces():
"""
The code checking for ``from __future__ import absolute_import`` shouldn't
assume that all imports have non-``None`` namespaces.
"""
src = "from __future__ import absolute_import\nimport xyzzy"
assert parse(src)._has_explicit_absolute_import()
def _split_comment_param_declaration(decl_text):
"""
Split decl_text on commas, but group generic expressions
together.
For example, given "foo, Bar[baz, biz]" we return
['foo', 'Bar[baz, biz]'].
"""
try:
node = parse(decl_text, error_recovery=False).children[0]
except ParserSyntaxError:
debug.warning('Comment annotation is not valid Python: %s' % decl_text)
return []
if node.type in ['name', 'atom_expr', 'power']:
return [node.get_code().strip()]
params = []
try:
children = node.children
except AttributeError:
return []
else:
for child in children:
if child.type in ['name', 'atom_expr', 'power']:
params.append(child.get_code().strip())
return params
def test_incomplete_list_comprehension(each_version):
""" Shouldn't raise an error, same bug as #418. """
# With the old parser this actually returned a statement. With the new
# parser only valid statements generate one.
children = parse('(1 for def', version=each_version).children
assert [c.type for c in children] == \
['error_node', 'error_node', 'endmarker']
def _expand_typestr(type_str):
"""
Attempts to interpret the possible types in `type_str`
"""
# Check if alternative types are specified with 'or'
if re.search(r'\bor\b', type_str):
for t in type_str.split('or'):
yield t.split('of')[0].strip()
# Check if like "list of `type`" and set type to list
elif re.search(r'\bof\b', type_str):
yield type_str.split('of')[0]
# Check if type has is a set of valid literal values eg: {'C', 'F', 'A'}
elif type_str.startswith('{'):
node = parse(type_str, version='3.7').children[0]
if node.type == 'atom':
for leaf in node.children[1].children:
if leaf.type == 'number':
if '.' in leaf.value:
yield 'float'
else:
yield 'int'
elif leaf.type == 'string':
if 'b' in leaf.string_prefix.lower():
yield 'bytes'
else:
yield 'str'
# Ignore everything else.
# Otherwise just work with what we have.
else:
yield type_str
def _invalid_syntax(self, code):
with pytest.raises(marso.ParserSyntaxError):
module = marso.parse(code,
version=self.version,
error_recovery=False)
# For debugging
print(module.children)
def parse(self, code):
if self._is_passing:
return marso.parse(code,
version=self.version,
error_recovery=False)
else:
self._invalid_syntax(code)
def test_basic_parsing():
"""Validate the parsing features"""
m = parse(code_basic_features)
diff_code_assert(
code_basic_features,
m.get_code()
)
def test_invalid_token():
module = parse('a + ? + b')
error_node, q, plus_b, endmarker = module.children
assert error_node.get_code() == 'a +'
assert q.value == '?'
assert q.type == 'error_leaf'
assert plus_b.type == 'factor'
assert plus_b.get_code() == ' + b'
def test_end_pos_line(each_version):
# jedi issue #150
s = "x()\nx( )\nx( )\nx ( )\n"
module = parse(s, version=each_version)
for i, simple_stmt in enumerate(module.children[:-1]):
expr_stmt = simple_stmt.children[0]
assert expr_stmt.end_pos == (i + 1, i + 3)
def test_with_stmt():
module = parse('with x: f.\na')
assert module.children[0].type == 'with_stmt'
w, with_item, colon, f = module.children[0].children
assert f.type == 'error_node'
assert f.get_code(include_prefix=False) == 'f.'
assert module.children[2].type == 'name'
def test_dedent_at_end(each_version):
code = dedent('''
for foobar in [1]:
foobar''')
module = parse(code, version=each_version)
assert module.get_code() == code
suite = module.children[0].children[-1]
foobar = suite.children[-1]
assert foobar.type == 'name'
def test_carriage_return_statements(each_version):
source = dedent('''
foo = 'ns1!'
# this is a namespace package
''')
source = source.replace('\n', '\r\n')
stmt = parse(source, version=each_version).children[0]
assert '#' not in stmt.get_code()
def test_carriage_return_at_end(code, types):
"""
By adding an artificial newline this created weird side effects for
\r at the end of files.
"""
tree = parse(code)
assert tree.get_code() == code
assert [c.type for c in tree.children] == types
assert tree.end_pos == (len(code) + 1, 0)
def test_open_string_literal(each_version, code):
"""
Testing mostly if removing the last newline works.
"""
lines = split_lines(code, keepends=True)
end_pos = (len(lines), len(lines[-1]))
module = parse(code, version=each_version)
assert module.get_code() == code
assert module.end_pos == end_pos == module.children[1].end_pos
def test_cache_limit():
def cache_size():
return sum(len(v) for v in parser_cache.values())
try:
parser_cache.clear()
future_node_cache_item = _NodeCacheItem('bla', [], change_time=time.time() + 10e6)
old_node_cache_item = _NodeCacheItem('bla', [], change_time=time.time() - 10e4)
parser_cache['some_hash_old'] = {
'/path/%s' % i: old_node_cache_item for i in range(300)
}
parser_cache['some_hash_new'] = {
'/path/%s' % i: future_node_cache_item for i in range(300)
}
assert cache_size() == 600
parse('somecode', cache=True, path='/path/somepath')
assert cache_size() == 301
finally:
parser_cache.clear()
def test_end_pos(each_version):
s = dedent('''
x = ['a', 'b', 'c']
def func():
y = None
''')
parser = parse(s, version=each_version)
scope = next(parser.iter_funcdefs())
assert scope.start_pos == (3, 0)
assert scope.end_pos == (5, 0)
def test_if_else():
module = parse('if x:\n f.\nelse:\n g(')
if_stmt = module.children[0]
if_, test, colon, suite1, else_, colon, suite2 = if_stmt.children
f = suite1.children[1]
assert f.type == 'error_node'
assert f.children[0].value == 'f'
assert f.children[1].value == '.'
g = suite2.children[1]
assert g.children[0].value == 'g'
assert g.children[1].value == '('
def check(src, result):
# Python 2 tuple params should be ignored for now.
m = parse(src, version=each_version)
if each_version.startswith('2'):
# We don't want b and c to be a part of the param enumeration. Just
# ignore them, because it's not what we want to support in the
# future.
func = next(m.iter_funcdefs())
assert [param.name.value for param in func.get_params()] == result
else:
assert not list(m.iter_funcdefs())
def check_p(src,
number_parsers_used,
number_of_splits=None,
number_of_misses=0):
if number_of_splits is None:
number_of_splits = number_parsers_used
module_node = parse(src)
assert src == module_node.get_code()
return module_node
def test_is_definition(code, name_index, is_definition, include_setitem):
module = parse(code, version='3.8')
name = module.get_first_leaf()
while True:
if name.type == 'name':
if name_index == 0:
break
name_index -= 1
name = name.get_next_leaf()
assert name.is_definition(include_setitem=include_setitem) == is_definition
def initialize(self, code):
logging.debug('differ: initialize')
try:
del cache.parser_cache[self.grammar._hashed][None]
except KeyError:
pass
self.lines = split_lines(code, keepends=True)
self.module = parse(code, diff_cache=True, cache=True)
assert code == self.module.get_code()
_assert_valid_graph(self.module)
return self.module
def assert_params(param_string, version=None, **wanted_dct):
source = dedent('''
def x(%s):
pass
''') % param_string
module = parse(source, version=version)
funcdef = next(module.iter_funcdefs())
dct = dict((p.name.value, p.default and p.default.get_code())
for p in funcdef.get_params())
assert dct == wanted_dct
assert module.get_code() == source
