def test_simple():
"""
The diff parser reuses modules. So check for that.
"""
grammar = load_grammar()
module_a = grammar.parse('a', diff_cache=True)
assert grammar.parse('b', diff_cache=True) == module_a
def test_modulepickling_simulate_deleted_cache(tmpdir):
"""
Tests loading from a cache file after it is deleted.
According to macOS `dev docs`__,
Note that the system may delete the Caches/ directory to free up disk
space, so your app must be able to re-create or download these files as
needed.
It is possible that other supported platforms treat cache files the same
way.
__ https://developer.apple.com/library/content/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html
"""
grammar = load_grammar()
module = 'fake parser'
# Create the file
path = tmpdir.dirname + '/some_path'
with open(path, 'w'):
pass
io = file_io.FileIO(path)
save_module(grammar._hashed, io, module, lines=[])
assert load_module(grammar._hashed, io) == module
unlink(_get_hashed_path(grammar._hashed, path))
parser_cache.clear()
cached2 = load_module(grammar._hashed, io)
assert cached2 is None
def test_load_grammar():
grammar = load_grammar(language="demo")._pgen_grammar
assert grammar.start_nonterminal == 'file_input'
nonterminals = {'file_input', 'rule', 'rhs', 'items', 'item', 'atom'}
assert nonterminals == {x for x in grammar.nonterminal_to_dfas.keys()}
reserved = {':', '|', '[', '*', '+', ']', '(', ')', 'ademo'}
assert reserved == {x for x in grammar.reserved_syntax_strings.keys()}
def test_parse_demo_lang_error():
code = "abc : def + 'ss'\na"
grammar = load_grammar(language="demo")
root = grammar.parse(code)
assert root.children[-2].get_code() == 'a'
assert root.children[-1].type == 'endmarker'
with pytest.raises(marso.parser.ParserSyntaxError):
grammar.parse(code, error_recovery=False)
def test_on_itself(each_version):
"""
There are obviously no syntax erros in the Python code of marso. However
marso should output the same for all versions.
"""
grammar = marso.load_grammar(version=each_version)
path = os.path.dirname(os.path.dirname(__file__)) + '/marso'
for file in get_python_files(path):
tree = grammar.parse(path=file)
errors = list(grammar.iter_errors(tree))
assert not errors
def test_py_LL1():
grammar = load_grammar(version="3.8")
st1 = grammar.parse("print('OK') if (a := 1) > 0 else None",
error_recovery=False)
# https://www.python.org/dev/peps/pep-0617/#some-rules-are-not-actually-ll-1
# namedexpr_test: test [':=' test]
# should be:
# namedexpr_test: NAME [':=' test]
# so as to reject following code:
st2 = grammar.parse(
"print('OK') if ([x for x in y] := [1,2,3]) > 0 else None",
error_recovery=False)
def test_invalid_token_in_fstr():
module = load_grammar(version='3.6').parse('f"{a + ? + b}"')
error_node, q, plus_b, error1, error2, endmarker = module.children
assert error_node.get_code() == 'f"{a +'
assert q.value == '?'
assert q.type == 'error_leaf'
assert plus_b.type == 'error_node'
assert plus_b.get_code() == ' + b'
assert error1.value == '}'
assert error1.type == 'error_leaf'
assert error2.value == '"'
assert error2.type == 'error_leaf'
def test_dedent_issues1():
code = dedent('''\
class C:
@property
f
g
end
''')
module = load_grammar(version='3.8').parse(code)
klass, endmarker = module.children
suite = klass.children[-1]
assert suite.children[2].type == 'error_leaf'
assert suite.children[3].get_code(include_prefix=False) == 'f\n'
assert suite.children[5].get_code(include_prefix=False) == 'g\n'
assert suite.type == 'suite'
def test_dedent_issues3():
code = dedent('''\
class C:
f
g
''')
module = load_grammar(version='3.8').parse(code)
klass, endmarker = module.children
suite = klass.children[-1]
assert len(suite.children) == 4
assert suite.children[1].get_code() == ' f\n'
assert suite.children[1].type == 'simple_stmt'
assert suite.children[2].get_code() == ''
assert suite.children[2].type == 'error_leaf'
assert suite.children[2].token_type == 'ERROR_DEDENT'
assert suite.children[3].get_code() == ' g\n'
assert suite.children[3].type == 'simple_stmt'
class Differ(object):
grammar = load_grammar()
def initialize(self, code):
logging.debug('differ: initialize')
try:
del cache.parser_cache[self.grammar._hashed][None]
except KeyError:
pass
self.lines = split_lines(code, keepends=True)
self.module = parse(code, diff_cache=True, cache=True)
assert code == self.module.get_code()
_assert_valid_graph(self.module)
return self.module
def parse(self, code, copies=0, parsers=0, expect_error_leaves=False):
logging.debug('differ: parse copies=%s parsers=%s', copies, parsers)
lines = split_lines(code, keepends=True)
diff_parser = DiffParser(
self.grammar._pgen_grammar,
self.grammar._tokenizer,
self.module,
)
new_module = diff_parser.update(self.lines, lines)
self.lines = lines
assert code == new_module.get_code()
_assert_valid_graph(new_module)
without_diff_parser_module = parse(code)
_assert_nodes_are_equal(new_module, without_diff_parser_module)
error_node = _check_error_leaves_nodes(new_module)
assert expect_error_leaves == (error_node is not None), error_node
if parsers is not ANY:
assert diff_parser._parser_count == parsers
if copies is not ANY:
assert diff_parser._copy_count == copies
return new_module
def test_parse_demo_language():
code = "abc : def + 'ss'\n"
grammar = load_grammar(language="demo")
root = grammar.parse(code)
assert root.start_pos == (1, 0)
assert root.end_pos == (2, 0)
assert root.parent is None
assert root.type == "file_input"
assert root.children.__len__() == 2
assert root.children[0].type == "rule"
assert root.children[1].type == "endmarker"
rule = root.children[0]
assert rule.parent == root
assert rule.children.__len__() == 4
assert rule.children[0].type == "name"
assert rule.children[0].value == "abc"
assert rule.children[1].type == "operator"
assert rule.children[1].value == ":"
assert rule.children[1].start_pos == (1, 4)
assert rule.children[1].prefix == " "
assert rule.children[2].type == "items"
assert rule.children[3].type == "newline"
assert rule.children[3].value == "\n"
items = rule.children[2]
assert items.parent == rule
assert items.children.__len__() == 2
assert items.children[0].type == "item"
assert items.children[1].type == "string"
assert items.children[1].value == "'ss'"
item = items.children[0]
assert item.children[0].type == "name"
assert item.children[0].value == "def"
assert item.children[1].type == "operator"
assert item.children[1].value == "+"
def test_modulepickling_change_cache_dir(tmpdir):
"""
ParserPickling should not save old cache when cache_directory is changed.
See: `#168 <https://github.com/davidhalter/jedi/pull/168>`_
"""
dir_1 = str(tmpdir.mkdir('first'))
dir_2 = str(tmpdir.mkdir('second'))
item_1 = _NodeCacheItem('bla', [])
item_2 = _NodeCacheItem('bla', [])
path_1 = 'fake path 1'
path_2 = 'fake path 2'
hashed_grammar = load_grammar()._hashed
_save_to_file_system(hashed_grammar, path_1, item_1, cache_path=dir_1)
parser_cache.clear()
cached = load_stored_item(hashed_grammar, path_1, item_1, cache_path=dir_1)
assert cached == item_1.node
_save_to_file_system(hashed_grammar, path_2, item_2, cache_path=dir_2)
cached = load_stored_item(hashed_grammar, path_1, item_1, cache_path=dir_2)
assert cached is None
def test_dedent_issues2():
code = dedent('''\
class C:
@property
if 1:
g
else:
h
end
''')
module = load_grammar(version='3.8').parse(code)
klass, endmarker = module.children
suite = klass.children[-1]
assert suite.children[2].type == 'error_leaf'
if_ = suite.children[3]
assert if_.children[0] == 'if'
assert if_.children[3].type == 'suite'
assert if_.children[3].get_code() == '\n g\n'
assert if_.children[4] == 'else'
assert if_.children[6].type == 'suite'
assert if_.children[6].get_code() == '\n h\n'
assert suite.children[4].get_code(include_prefix=False) == 'end\n'
assert suite.type == 'suite'
def __init__(self, project, environment=None, script_path=None):
if environment is None:
environment = project.get_environment()
self.environment = environment
self.script_path = script_path
self.compiled_subprocess = environment.get_inference_state_subprocess(self)
self.grammar = environment.get_grammar() # TTODO: this grammar is incorrect.
self.latest_grammar = marso.load_grammar(version='3.7')
self.memoize_cache = {} # for memoize decorators
self.module_cache = imports.ModuleCache() # does the job of `sys.modules`.
self.stub_module_cache = {} # Dict[Tuple[str, ...], Optional[ModuleValue]]
self.compiled_cache = {} # see `inference.compiled.create()`
self.inferred_element_counts = {}
self.mixed_cache = {} # see `inference.compiled.mixed._create()`
self.analysis = []
self.dynamic_params_depth = 0
self.is_analysis = False
self.project = project
self.access_cache = {}
self.allow_descriptor_getattr = False
self.flow_analysis_enabled = True
self.reset_recursion_limitations()
def get_grammar(self):
version_string = '%s.%s' % (self.version_info.major,
self.version_info.minor)
return marso.load_grammar(version=version_string)
def _get_error_list(code, version=None):
grammar = marso.load_grammar(version=version)
tree = grammar.parse(code)
return list(grammar.iter_errors(tree))
def test_parse_py_error():
code = "abc : def + 'ss'\na"
grammar = load_grammar(language="python")
root = grammar.parse(code)
def __init__(self, code=None, line=None, column=None, path=None,
encoding=None, sys_path=None, environment=None,
project=None, source=None, language="python"):
self._orig_path = path
# An empty path (also empty string) should always result in no path.
self.path = os.path.abspath(path) if path else None
self.language = language
self.grammar = marso.load_grammar(language=self.language)
if encoding is None:
encoding = 'utf-8'
else:
warnings.warn(
"Deprecated since version 0.17.0. You should cast to valid "
"unicode yourself, especially if you are not using utf-8.",
DeprecationWarning,
stacklevel=2
)
if line is not None:
warnings.warn(
"Providing the line is now done in the functions themselves "
"like `Script(...).complete(line, column)`",
DeprecationWarning,
stacklevel=2
)
if column is not None:
warnings.warn(
"Providing the column is now done in the functions themselves "
"like `Script(...).complete(line, column)`",
DeprecationWarning,
stacklevel=2
)
if source is not None:
code = source
warnings.warn(
"Use the code keyword argument instead.",
DeprecationWarning,
stacklevel=2
)
if code is None:
# TODO add a better warning than the traceback!
with open(path, 'rb') as f:
code = f.read()
if sys_path is not None and not is_py3:
sys_path = list(map(force_unicode, sys_path))
if project is None:
# Load the Python grammar of the current interpreter.
project = get_default_project(
self.language,
os.path.dirname(self.path) if path else None
)
# TODO deprecate and remove sys_path from the Script API.
if sys_path is not None:
project._sys_path = sys_path
warnings.warn(
"Deprecated since version 0.17.0. Use the project API instead, "
"which means Script(project=Project(dir, sys_path=sys_path)) instead.",
DeprecationWarning,
stacklevel=2
)
self._inference_state = InferenceState(
project, environment=environment, script_path=self.path
)
debug.speed('init')
self._module_node, code = self._inference_state.parse_and_get_code(
code=code,
path=self.path,
encoding=encoding,
use_latest_grammar=path and path.endswith('.pyi'),
cache=False, # No disk cache, because the current script often changes.
diff_cache=settings.fast_parser,
cache_path=settings.cache_directory,
)
debug.speed('parsed')
self._code_lines = marso.split_lines(code, keepends=True)
self._code = code
self._pos = line, column
cache.clear_time_caches()
debug.reset_time()
from medi._compatibility import unicode, is_py3
from medi.api.classes import Name
from medi.api.completion import get_user_context
from medi import parser_utils
from medi.api.environment import get_default_environment, get_system_environment
from medi.inference.gradual.conversion import convert_values
from medi.inference.analysis import Warning
TEST_COMPLETIONS = 0
TEST_INFERENCE = 1
TEST_GOTO = 2
TEST_REFERENCES = 3
grammar37 = marso.load_grammar(version='3.7')
class BaseTestCase(object):
def __init__(self, skip_version_info=None):
self._skip_version_info = skip_version_info
self._skip = None
def set_skip(self, reason):
self._skip = reason
def get_skip_reason(self, environment):
if self._skip is not None:
return self._skip
if self._skip_version_info is None:
def issues(code):
grammar = marso.load_grammar()
module = marso.parse(code)
return grammar._get_normalizer_issues(module)
def __init__(self, version, is_passing):
self.version = version
self._is_passing = is_passing
self.grammar = marso.load_grammar(version=self.version)
def _parse(code, version=None):
code = dedent(code) + "\n\n"
grammar = load_grammar(version=version)
return grammar.parse(code, error_recovery=False)
def grammar():
return load_grammar(version='3.8')
