def test_utf8_bom(): tree = marso.parse(unicode_bom + 'a = 1') expr_stmt = tree.children[0] assert expr_stmt.start_pos == (1, 0) tree = marso.parse(unicode_bom + '\n') endmarker = tree.children[0] parts = list(endmarker._split_prefix()) assert [p.type for p in parts] == ['bom', 'newline', 'spacing'] assert [p.start_pos for p in parts] == [(1, 0), (1, 0), (2, 0)] assert [p.end_pos for p in parts] == [(1, 0), (2, 0), (2, 0)]
def test_utf8_bom(): unicode_bom = BOM_UTF8.decode('utf-8') module = marso.parse(unicode_bom) endmarker = module.children[0] assert endmarker.type == 'endmarker' assert unicode_bom == endmarker.prefix module = marso.parse(unicode_bom + 'foo = 1') expr_stmt = module.children[0] assert expr_stmt.type == 'expr_stmt' assert unicode_bom == expr_stmt.get_first_leaf().prefix
def test_one_line_function(each_version): module = parse('def x(): f.', version=each_version) assert module.children[0].type == 'funcdef' def_, name, parameters, colon, f = module.children[0].children assert f.type == 'error_node' module = parse('def x(a:', version=each_version) func = module.children[0] assert func.type == 'error_node' if each_version.startswith('2'): assert func.children[-1].value == 'a' else: assert func.children[-1] == ':'
def test_python2_octal(each_version): module = parse('0660', version=each_version) first = module.children[0] if each_version.startswith('2'): assert first.type == 'number' else: assert first.type == 'error_node'
def node(self, request): parsed = parse(dedent(request.param[0]), version='3.6') request.keywords['expected'] = request.param[1] child = parsed.children[0] if child.type == 'simple_stmt': child = child.children[0] return child
def test_get_code(): """Use the same code that the parser also generates, to compare""" s = '''"""a docstring""" class SomeClass(object, mixin): def __init__(self): self.xy = 3.0 """statement docstr""" def some_method(self): return 1 def yield_method(self): while hasattr(self, 'xy'): yield True for x in [1, 2]: yield x def empty(self): pass class Empty: pass class WithDocstring: """class docstr""" pass def method_with_docstring(): """class docstr""" pass ''' assert parse(s).get_code() == s
def test_ellipsis_py2(each_py2_version): module = parse('[0][...]', version=each_py2_version, error_recovery=False) expr = module.children[0] trailer = expr.children[-1] subscript = trailer.children[1] assert subscript.type == 'subscript' assert [leaf.value for leaf in subscript.children] == ['.', '.', '.']
def test_round_trip(): code = dedent(''' def x(): """hahaha""" func''') assert parse(code).get_code() == code
def test_dont_break_imports_without_namespaces(): """ The code checking for ``from __future__ import absolute_import`` shouldn't assume that all imports have non-``None`` namespaces. """ src = "from __future__ import absolute_import\nimport xyzzy" assert parse(src)._has_explicit_absolute_import()
def _split_comment_param_declaration(decl_text): """ Split decl_text on commas, but group generic expressions together. For example, given "foo, Bar[baz, biz]" we return ['foo', 'Bar[baz, biz]']. """ try: node = parse(decl_text, error_recovery=False).children[0] except ParserSyntaxError: debug.warning('Comment annotation is not valid Python: %s' % decl_text) return [] if node.type in ['name', 'atom_expr', 'power']: return [node.get_code().strip()] params = [] try: children = node.children except AttributeError: return [] else: for child in children: if child.type in ['name', 'atom_expr', 'power']: params.append(child.get_code().strip()) return params
def test_incomplete_list_comprehension(each_version): """ Shouldn't raise an error, same bug as #418. """ # With the old parser this actually returned a statement. With the new # parser only valid statements generate one. children = parse('(1 for def', version=each_version).children assert [c.type for c in children] == \ ['error_node', 'error_node', 'endmarker']
def _expand_typestr(type_str): """ Attempts to interpret the possible types in `type_str` """ # Check if alternative types are specified with 'or' if re.search(r'\bor\b', type_str): for t in type_str.split('or'): yield t.split('of')[0].strip() # Check if like "list of `type`" and set type to list elif re.search(r'\bof\b', type_str): yield type_str.split('of')[0] # Check if type has is a set of valid literal values eg: {'C', 'F', 'A'} elif type_str.startswith('{'): node = parse(type_str, version='3.7').children[0] if node.type == 'atom': for leaf in node.children[1].children: if leaf.type == 'number': if '.' in leaf.value: yield 'float' else: yield 'int' elif leaf.type == 'string': if 'b' in leaf.string_prefix.lower(): yield 'bytes' else: yield 'str' # Ignore everything else. # Otherwise just work with what we have. else: yield type_str
def _invalid_syntax(self, code): with pytest.raises(marso.ParserSyntaxError): module = marso.parse(code, version=self.version, error_recovery=False) # For debugging print(module.children)
def parse(self, code): if self._is_passing: return marso.parse(code, version=self.version, error_recovery=False) else: self._invalid_syntax(code)
def test_basic_parsing(): """Validate the parsing features""" m = parse(code_basic_features) diff_code_assert( code_basic_features, m.get_code() )
def test_invalid_token(): module = parse('a + ? + b') error_node, q, plus_b, endmarker = module.children assert error_node.get_code() == 'a +' assert q.value == '?' assert q.type == 'error_leaf' assert plus_b.type == 'factor' assert plus_b.get_code() == ' + b'
def test_end_pos_line(each_version): # jedi issue #150 s = "x()\nx( )\nx( )\nx ( )\n" module = parse(s, version=each_version) for i, simple_stmt in enumerate(module.children[:-1]): expr_stmt = simple_stmt.children[0] assert expr_stmt.end_pos == (i + 1, i + 3)
def test_with_stmt(): module = parse('with x: f.\na') assert module.children[0].type == 'with_stmt' w, with_item, colon, f = module.children[0].children assert f.type == 'error_node' assert f.get_code(include_prefix=False) == 'f.' assert module.children[2].type == 'name'
def test_dedent_at_end(each_version): code = dedent(''' for foobar in [1]: foobar''') module = parse(code, version=each_version) assert module.get_code() == code suite = module.children[0].children[-1] foobar = suite.children[-1] assert foobar.type == 'name'
def test_carriage_return_statements(each_version): source = dedent(''' foo = 'ns1!' # this is a namespace package ''') source = source.replace('\n', '\r\n') stmt = parse(source, version=each_version).children[0] assert '#' not in stmt.get_code()
def test_carriage_return_at_end(code, types): """ By adding an artificial newline this created weird side effects for \r at the end of files. """ tree = parse(code) assert tree.get_code() == code assert [c.type for c in tree.children] == types assert tree.end_pos == (len(code) + 1, 0)
def test_open_string_literal(each_version, code): """ Testing mostly if removing the last newline works. """ lines = split_lines(code, keepends=True) end_pos = (len(lines), len(lines[-1])) module = parse(code, version=each_version) assert module.get_code() == code assert module.end_pos == end_pos == module.children[1].end_pos
def test_cache_limit(): def cache_size(): return sum(len(v) for v in parser_cache.values()) try: parser_cache.clear() future_node_cache_item = _NodeCacheItem('bla', [], change_time=time.time() + 10e6) old_node_cache_item = _NodeCacheItem('bla', [], change_time=time.time() - 10e4) parser_cache['some_hash_old'] = { '/path/%s' % i: old_node_cache_item for i in range(300) } parser_cache['some_hash_new'] = { '/path/%s' % i: future_node_cache_item for i in range(300) } assert cache_size() == 600 parse('somecode', cache=True, path='/path/somepath') assert cache_size() == 301 finally: parser_cache.clear()
def test_end_pos(each_version): s = dedent(''' x = ['a', 'b', 'c'] def func(): y = None ''') parser = parse(s, version=each_version) scope = next(parser.iter_funcdefs()) assert scope.start_pos == (3, 0) assert scope.end_pos == (5, 0)
def test_if_else(): module = parse('if x:\n f.\nelse:\n g(') if_stmt = module.children[0] if_, test, colon, suite1, else_, colon, suite2 = if_stmt.children f = suite1.children[1] assert f.type == 'error_node' assert f.children[0].value == 'f' assert f.children[1].value == '.' g = suite2.children[1] assert g.children[0].value == 'g' assert g.children[1].value == '('
def check(src, result): # Python 2 tuple params should be ignored for now. m = parse(src, version=each_version) if each_version.startswith('2'): # We don't want b and c to be a part of the param enumeration. Just # ignore them, because it's not what we want to support in the # future. func = next(m.iter_funcdefs()) assert [param.name.value for param in func.get_params()] == result else: assert not list(m.iter_funcdefs())
def check_p(src, number_parsers_used, number_of_splits=None, number_of_misses=0): if number_of_splits is None: number_of_splits = number_parsers_used module_node = parse(src) assert src == module_node.get_code() return module_node
def test_is_definition(code, name_index, is_definition, include_setitem): module = parse(code, version='3.8') name = module.get_first_leaf() while True: if name.type == 'name': if name_index == 0: break name_index -= 1 name = name.get_next_leaf() assert name.is_definition(include_setitem=include_setitem) == is_definition
def initialize(self, code): logging.debug('differ: initialize') try: del cache.parser_cache[self.grammar._hashed][None] except KeyError: pass self.lines = split_lines(code, keepends=True) self.module = parse(code, diff_cache=True, cache=True) assert code == self.module.get_code() _assert_valid_graph(self.module) return self.module
def assert_params(param_string, version=None, **wanted_dct): source = dedent(''' def x(%s): pass ''') % param_string module = parse(source, version=version) funcdef = next(module.iter_funcdefs()) dct = dict((p.name.value, p.default and p.default.get_code()) for p in funcdef.get_params()) assert dct == wanted_dct assert module.get_code() == source