def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._grammar_2 = parso.load_grammar(version="2.7")
self._grammar_3 = parso.load_grammar(version="3.5")
# A cache map of `pathlib.Path` -> {external IDs field names} to not compute
# the field names repeatedly for each row in a file.
self._csv_external_id_fields = collections.defaultdict(set)
def _load_grammar(cls):
import parso
if cls._fstring_grammar is None:
cls._fstring_grammar = parso.load_grammar(
language='python-f-string')
return cls._fstring_grammar
def test_simple():
"""
The diff parser reuses modules. So check for that.
"""
grammar = load_grammar()
module_a = grammar.parse('a', diff_cache=True)
assert grammar.parse('b', diff_cache=True) == module_a
async def syntax_error(buff):
start = buff.get_start_iter()
end = buff.get_end_iter()
text = buff.get_text(start, end, True)
if buff.get_tag_table().lookup("err_t") is None:
buff.create_tag("err_t", underline="error")
buff.remove_tag_by_name("err_t", start, end)
errors = Script(text).get_syntax_errors()
grammer = par.load_grammar()
error_code = par.parse(text)
error_found = grammer.iter_errors(error_code)
for e in errors:
lin = e.line - 1
col = e.column - 1
lin2 = e.until_line - 1
col2 = e.until_column - 1
if lin < 1:
lin = 0
if col < 1:
col = 0
if lin2 < 1:
lin2 = 0
if col2 < 1:
col2 = 0
start.set_line(lin)
end.set_line(lin2)
start.set_line_offset(col)
end.set_line_offset(col2)
buff.apply_tag_by_name("err_t", start, end)
def __init__(self, project, environment=None, script_path=None):
if environment is None:
environment = project.get_environment()
self.environment = environment
self.script_path = script_path
self.compiled_subprocess = environment.get_inference_state_subprocess(
self)
self.grammar = environment.get_grammar()
self.latest_grammar = parso.load_grammar(version='3.7')
self.memoize_cache = {} # for memoize decorators
self.module_cache = imports.ModuleCache(
) # does the job of `sys.modules`.
self.stub_module_cache = {
} # Dict[Tuple[str, ...], Optional[ModuleValue]]
self.compiled_cache = {} # see `inference.compiled.create()`
self.inferred_element_counts = {}
self.mixed_cache = {} # see `inference.compiled.mixed._create()`
self.analysis = []
self.dynamic_params_depth = 0
self.is_analysis = False
self.project = project
self.access_cache = {}
self.allow_descriptor_getattr = False
self.flow_analysis_enabled = True
self.reset_recursion_limitations()
def __init__(self, filename):
self.code = ""
self.message = ""
error_message = ""
error_start_pos = ""
if filename == "ssg":
file_path = Path.cwd() / "ssg.py"
else:
file_path = Path.cwd() / "ssg" / "{}.py".format(filename)
grammar = parso.load_grammar()
module = grammar.parse(path=file_path.resolve())
self.success = len(grammar.iter_errors(module)) == 0
if self.success:
with open(file_path.resolve(), "r") as source_code:
self.code = RedBaron(source_code.read())
else:
error_message = grammar.iter_errors(module)[0].message
error_start_pos = grammar.iter_errors(module)[0].start_pos[0]
self.message = "{} on or around line {} in `{}`.".format(
error_message, error_start_pos, file_path.name
)
def test_simple():
"""
The diff parser reuses modules. So check for that.
"""
grammar = load_grammar()
module_a = grammar.parse('a', diff_cache=True)
assert grammar.parse('b', diff_cache=True) == module_a
def __init__(self, grammar, sys_path=None):
self.grammar = grammar
self.latest_grammar = parso.load_grammar(version='3.6')
self.memoize_cache = {} # for memoize decorators
# To memorize modules -> equals `sys.modules`.
self.modules = {} # like `sys.modules`.
self.compiled_cache = {} # see `evaluate.compiled.create()`
self.inferred_element_counts = {}
self.mixed_cache = {} # see `evaluate.compiled.mixed._create()`
self.analysis = []
self.dynamic_params_depth = 0
self.is_analysis = False
self.python_version = sys.version_info[:2]
if sys_path is None:
sys_path = sys.path
self.sys_path = copy.copy(sys_path)
try:
self.sys_path.remove('')
except ValueError:
pass
self.reset_recursion_limitations()
# Constants
self.BUILTINS = compiled.get_special_object(self, 'BUILTINS')
def __init__(self, project, environment=None, script_path=None):
if environment is None:
environment = project.get_environment()
self.environment = environment
self.script_path = script_path
self.compiled_subprocess = environment.get_evaluator_subprocess(self)
self.grammar = environment.get_grammar()
self.latest_grammar = parso.load_grammar(version='3.6')
self.memoize_cache = {} # for memoize decorators
self.module_cache = imports.ModuleCache() # does the job of `sys.modules`.
self.compiled_cache = {} # see `evaluate.compiled.create()`
self.inferred_element_counts = {}
self.mixed_cache = {} # see `evaluate.compiled.mixed._create()`
self.analysis = []
self.dynamic_params_depth = 0
self.is_analysis = False
self.project = project
self.access_cache = {}
# This setting is only temporary to limit the work we have to do with
# tensorflow and others.
self.infer_enabled = True
self.reset_recursion_limitations()
self.allow_different_encoding = True
def __init__(self, project, environment=None, script_path=None):
if environment is None:
environment = project.get_environment()
self.environment = environment
self.script_path = script_path
self.compiled_subprocess = environment.get_evaluator_subprocess(self)
self.grammar = environment.get_grammar()
self.latest_grammar = parso.load_grammar(version='3.7')
self.memoize_cache = {} # for memoize decorators
self.module_cache = imports.ModuleCache(
) # does the job of `sys.modules`.
self.stub_module_cache = {
} # Dict[Tuple[str, ...], Optional[ModuleContext]]
self.compiled_cache = {} # see `evaluate.compiled.create()`
self.inferred_element_counts = {}
self.mixed_cache = {} # see `evaluate.compiled.mixed._create()`
self.analysis = []
self.dynamic_params_depth = 0
self.is_analysis = False
self.project = project
self.access_cache = {}
self.reset_recursion_limitations()
self.allow_different_encoding = True
# Plugin API
from jedi.plugins import plugin_manager
plugin_callbacks = plugin_manager.get_callbacks(self)
self.execute = plugin_callbacks.decorate('execute', callback=_execute)
self._import_module = partial(
plugin_callbacks.decorate('import_module',
callback=imports.import_module),
self,
)
def __init__(self, source=None, line=None, column=None, path=None,
encoding='utf-8', sys_path=None, environment=None):
self._orig_path = path
# An empty path (also empty string) should always result in no path.
self.path = os.path.abspath(path) if path else None
if source is None:
# TODO add a better warning than the traceback!
with open(path, 'rb') as f:
source = f.read()
# Load the Python grammar of the current interpreter.
self._grammar = parso.load_grammar()
if sys_path is not None and not is_py3:
sys_path = list(map(force_unicode, sys_path))
# Load the Python grammar of the current interpreter.
project = get_default_project(
os.path.dirname(self.path)if path else os.getcwd()
)
# TODO deprecate and remove sys_path from the Script API.
if sys_path is not None:
project._sys_path = sys_path
self._evaluator = Evaluator(
project, environment=environment, script_path=self.path
)
debug.speed('init')
self._module_node, source = self._evaluator.parse_and_get_code(
code=source,
path=self.path,
encoding=encoding,
cache=False, # No disk cache, because the current script often changes.
diff_cache=settings.fast_parser,
cache_path=settings.cache_directory,
)
debug.speed('parsed')
self._code_lines = parso.split_lines(source, keepends=True)
self._code = source
line = max(len(self._code_lines), 1) if line is None else line
if not (0 < line <= len(self._code_lines)):
raise ValueError('`line` parameter is not in a valid range.')
line_string = self._code_lines[line - 1]
line_len = len(line_string)
if line_string.endswith('\r\n'):
line_len -= 1
if line_string.endswith('\n'):
line_len -= 1
column = line_len if column is None else column
if not (0 <= column <= line_len):
raise ValueError('`column` parameter (%d) is not in a valid range '
'(0-%d) for line %d (%r).' % (
column, line_len, line, line_string))
self._pos = line, column
self._path = path
cache.clear_time_caches()
debug.reset_time()
class Differ(object):
grammar = load_grammar()
def initialize(self, code):
logging.debug('differ: initialize')
try:
del cache.parser_cache[self.grammar._hashed][None]
except KeyError:
pass
self.lines = split_lines(code, keepends=True)
self.module = parse(code, diff_cache=True, cache=True)
return self.module
def parse(self, code, copies=0, parsers=0, expect_error_leaves=False):
logging.debug('differ: parse copies=%s parsers=%s', copies, parsers)
lines = split_lines(code, keepends=True)
diff_parser = DiffParser(
self.grammar._pgen_grammar,
self.grammar._tokenizer,
self.module,
)
new_module = diff_parser.update(self.lines, lines)
self.lines = lines
assert code == new_module.get_code()
assert diff_parser._copy_count == copies
#assert diff_parser._parser_count == parsers
assert expect_error_leaves == _check_error_leaves_nodes(new_module)
_assert_valid_graph(new_module)
return new_module
def __init__(self, project, environment=None, script_path=None):
if environment is None:
environment = project.get_environment()
self.environment = environment
self.script_path = script_path
self.compiled_subprocess = environment.get_evaluator_subprocess(self)
self.grammar = environment.get_grammar()
self.latest_grammar = parso.load_grammar(version='3.6')
self.memoize_cache = {} # for memoize decorators
self.module_cache = imports.ModuleCache(
) # does the job of `sys.modules`.
self.compiled_cache = {} # see `evaluate.compiled.create()`
self.inferred_element_counts = {}
self.mixed_cache = {} # see `evaluate.compiled.mixed._create()`
self.analysis = []
self.dynamic_params_depth = 0
self.is_analysis = False
self.project = project
self.access_cache = {}
# This setting is only temporary to limit the work we have to do with
# tensorflow and others.
self.infer_enabled = True
self.reset_recursion_limitations()
self.allow_different_encoding = True
def test_modulepickling_simulate_deleted_cache(tmpdir):
"""
Tests loading from a cache file after it is deleted.
According to macOS `dev docs`__,
Note that the system may delete the Caches/ directory to free up disk
space, so your app must be able to re-create or download these files as
needed.
It is possible that other supported platforms treat cache files the same
way.
__ https://developer.apple.com/library/content/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html
"""
grammar = load_grammar()
module = 'fake parser'
# Create the file
path = tmpdir.dirname + '/some_path'
with open(path, 'w'):
pass
io = file_io.FileIO(path)
try_to_save_module(grammar._hashed, io, module, lines=[])
assert load_module(grammar._hashed, io) == module
os.unlink(_get_hashed_path(grammar._hashed, path))
parser_cache.clear()
cached2 = load_module(grammar._hashed, io)
assert cached2 is None
def get_new_source(target, kind, filepath=None):
"""Get the new source code of the target if given kind ('class' or 'def').
This works by using RedBaron to fetch the source code of the first object
that shares its name and kind with the target, inside the Python file from
which the target has been loaded.
"""
assert kind in ('class', 'def')
if kind == 'class':
kind_class = parso.python.tree.Class
else:
kind_class = parso.python.tree.Function
grammar = parso.load_grammar()
filepath = filepath or inspect.getsourcefile(target)
with open(filepath) as f:
parse = grammar.parse(f.read())
errors = grammar.iter_errors(parse)
if errors:
raise ParsingError('\n'.join(f'{err.start_pos}: {err.message}'
for err in errors))
for child in parse.children:
if child.type == 'decorated':
obj = child.children[1]
if isinstance(obj,
kind_class) and obj.name.value == target.__name__:
return obj.get_code()
raise ValueError("function or class not found")
def __init__(self, source=None, line=None, column=None, path=None,
encoding='utf-8', sys_path=None):
self._orig_path = path
# An empty path (also empty string) should always result in no path.
self.path = os.path.abspath(path) if path else None
if source is None:
# TODO add a better warning than the traceback!
with open(path, 'rb') as f:
source = f.read()
# TODO do we really want that?
self._source = python_bytes_to_unicode(source, encoding, errors='replace')
self._code_lines = split_lines(self._source)
line = max(len(self._code_lines), 1) if line is None else line
if not (0 < line <= len(self._code_lines)):
raise ValueError('`line` parameter is not in a valid range.')
line_len = len(self._code_lines[line - 1])
column = line_len if column is None else column
if not (0 <= column <= line_len):
raise ValueError('`column` parameter is not in a valid range.')
self._pos = line, column
self._path = path
cache.clear_time_caches()
debug.reset_time()
# Load the Python grammar of the current interpreter.
self._grammar = parso.load_grammar()
project = Project(sys_path=sys_path)
self._evaluator = Evaluator(self._grammar, project)
project.add_script_path(self.path)
debug.speed('init')
def test_modulepickling_simulate_deleted_cache(tmpdir):
"""
Tests loading from a cache file after it is deleted.
According to macOS `dev docs`__,
Note that the system may delete the Caches/ directory to free up disk
space, so your app must be able to re-create or download these files as
needed.
It is possible that other supported platforms treat cache files the same
way.
__ https://developer.apple.com/library/content/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html
"""
grammar = load_grammar()
module = 'fake parser'
# Create the file
path = tmpdir.dirname + '/some_path'
with open(path, 'w'):
pass
save_module(grammar._hashed, path, module, [])
assert load_module(grammar._hashed, path) == module
unlink(_get_hashed_path(grammar._hashed, path))
parser_cache.clear()
cached2 = load_module(grammar._hashed, path)
assert cached2 is None
def __init__(self,
source=None,
line=None,
column=None,
path=None,
encoding='utf-8',
sys_path=None,
environment=None):
self._orig_path = path
# An empty path (also empty string) should always result in no path.
self.path = os.path.abspath(path) if path else None
if source is None:
# TODO add a better warning than the traceback!
with open(path, 'rb') as f:
source = f.read()
# Load the Python grammar of the current interpreter.
self._grammar = parso.load_grammar()
if sys_path is not None and not is_py3:
sys_path = list(map(force_unicode, sys_path))
# Load the Python grammar of the current interpreter.
project = get_default_project(self.path or os.getcwd())
# TODO deprecate and remove sys_path from the Script API.
if sys_path is not None:
project._sys_path = sys_path
self._evaluator = Evaluator(project,
environment=environment,
script_path=self.path)
self._project = project
debug.speed('init')
self._module_node, source = self._evaluator.parse_and_get_code(
code=source,
path=self.path,
cache=
False, # No disk cache, because the current script often changes.
diff_cache=True,
cache_path=settings.cache_directory)
debug.speed('parsed')
self._code_lines = parso.split_lines(source)
line = max(len(self._code_lines), 1) if line is None else line
if not (0 < line <= len(self._code_lines)):
raise ValueError('`line` parameter is not in a valid range.')
line_len = len(self._code_lines[line - 1])
column = line_len if column is None else column
if not (0 <= column <= line_len):
raise ValueError('`column` parameter is not in a valid range.')
self._pos = line, column
self._path = path
cache.clear_time_caches()
debug.reset_time()
def test_on_itself(each_version):
"""
There are obviously no syntax erros in the Python code of parso. However
parso should output the same for all versions.
"""
grammar = parso.load_grammar(version=each_version)
path = os.path.dirname(os.path.dirname(__file__)) + '/parso'
for file in get_python_files(path):
tree = grammar.parse(path=file)
errors = list(grammar.iter_errors(tree))
assert not errors
def _get_errors(code):
grammar = parso.load_grammar()
module = grammar.parse(code)
errors = list(grammar.iter_errors(module))
for error in copy.copy(errors):
if error.code in _ALLOWED_ERROR_CODES:
errors.remove(error)
return errors
def test_on_itself(each_version):
"""
There are obviously no syntax erros in the Python code of parso. However
parso should output the same for all versions.
"""
grammar = parso.load_grammar(version=each_version)
path = os.path.dirname(os.path.dirname(__file__)) + '/parso'
for file in get_python_files(path):
tree = grammar.parse(path=file)
errors = list(grammar.iter_errors(tree))
assert not errors
def test_invalid_token_in_fstr():
module = load_grammar(version='3.6').parse('f"{a + ? + b}"')
error_node, q, plus_b, error1, error2, endmarker = module.children
assert error_node.get_code() == 'f"{a +'
assert q.value == '?'
assert q.type == 'error_leaf'
assert plus_b.type == 'error_node'
assert plus_b.get_code() == ' + b'
assert error1.value == '}'
assert error1.type == 'error_leaf'
assert error2.value == '"'
assert error2.type == 'error_leaf'
def test_sys_path_with_modifications():
code = dedent("""
import os
""")
path = os.path.abspath(os.path.join(os.curdir, 'module_name.py'))
grammar = parso.load_grammar()
module_node = parso.parse(code, path=path)
module_context = ModuleContext(Evaluator(grammar), module_node, path=path)
paths = sys_path_with_modifications(module_context.evaluator,
module_context)
assert '/tmp/.buildout/eggs/important_package.egg' in paths
def __init__(self,
source=None,
line=None,
column=None,
path=None,
encoding='utf-8',
sys_path=None,
environment=None,
_project=None):
self._orig_path = path
# An empty path (also empty string) should always result in no path.
self.path = os.path.abspath(path) if path else None
if source is None:
# TODO add a better warning than the traceback!
with open(path, 'rb') as f:
source = f.read()
# Load the Python grammar of the current interpreter.
self._grammar = parso.load_grammar()
if sys_path is not None and not is_py3:
sys_path = list(map(force_unicode, sys_path))
project = _project
if project is None:
# Load the Python grammar of the current interpreter.
project = get_default_project(
os.path.dirname(self.path) if path else os.getcwd())
# TODO deprecate and remove sys_path from the Script API.
if sys_path is not None:
project._sys_path = sys_path
self._inference_state = InferenceState(project,
environment=environment,
script_path=self.path)
debug.speed('init')
self._module_node, source = self._inference_state.parse_and_get_code(
code=source,
path=self.path,
encoding=encoding,
use_latest_grammar=path and path.endswith('.pyi'),
cache=
False, # No disk cache, because the current script often changes.
diff_cache=settings.fast_parser,
cache_path=settings.cache_directory,
)
debug.speed('parsed')
self._code_lines = parso.split_lines(source, keepends=True)
self._code = source
self._pos = line, column
cache.clear_time_caches()
debug.reset_time()
def tokenize_without_endmarker(code):
safeword = 'ZZZ_USER_WANTS_TO_COMPLETE_HERE'
grammar = load_grammar()
tokens = grammar._tokenize(code + safeword)
for token_ in tokens:
if token_.string == safeword:
return
elif token_.string.endswith(safeword):
yield PythonToken(token_.type, token_.string[:-len(safeword)],
token_.start_pos, token_.prefix)
return
else:
yield token_
def test_dedent_issues1():
code = dedent('''\
class C:
@property
f
g
end
''')
module = load_grammar(version='3.8').parse(code)
klass, endmarker = module.children
suite = klass.children[-1]
assert suite.children[2].type == 'error_leaf'
assert suite.children[3].get_code(include_prefix=False) == 'f\n'
assert suite.children[5].get_code(include_prefix=False) == 'g\n'
assert suite.type == 'suite'
def parse(source: str, filename: str) -> ast.Module:
with importlib.resources.path(__package__, "Grammar") as path:
grammar = parso.load_grammar(path=path)
module = grammar.parse(source)
errs = grammar.iter_errors(module)
if errs:
msg = errs[0].message.replace("SyntaxError: ", "")
raise SyntaxError(msg, (filename, *errs[0].start_pos, None))
return reduce(
lambda tree, trans: ast.fix_missing_locations(
trans(source, filename).visit(_setparents(tree))
),
TRANSFORMERS,
ast.parse(source, filename),
)
def validate_syntax(self):
try:
self.code_runner = CodeRunner(
self.code, flags=PyCF_ALLOW_TOP_LEVEL_AWAIT).compile()
except SyntaxError:
import parso
r = parso.parse(self.code)
errors = []
for error in parso.load_grammar().iter_errors(r):
error_dict = dict(start_pos=error.start_pos,
end_pos=error.end_pos,
msg=error.message)
errors.append(error_dict)
self.dispose()
return to_js(dict(valid=False, errors=errors))
return to_js(dict(valid=True))
def test_dedent_issues3():
code = dedent('''\
class C:
f
g
''')
module = load_grammar(version='3.8').parse(code)
klass, endmarker = module.children
suite = klass.children[-1]
assert len(suite.children) == 4
assert suite.children[1].get_code() == ' f\n'
assert suite.children[1].type == 'simple_stmt'
assert suite.children[2].get_code() == ''
assert suite.children[2].type == 'error_leaf'
assert suite.children[2].token_type == 'ERROR_DEDENT'
assert suite.children[3].get_code() == ' g\n'
assert suite.children[3].type == 'simple_stmt'
def main(arguments):
debugger = 'pdb' if arguments['--pdb'] else \
'ipdb' if arguments['--ipdb'] else None
redo_file = os.path.join(os.path.dirname(__file__), 'fuzz-redo.pickle')
if arguments['--logging']:
root = logging.getLogger()
root.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
root.addHandler(ch)
grammar = parso.load_grammar()
parso.python.diff.DEBUG_DIFF_PARSER = True
if arguments['redo']:
with open(redo_file, 'rb') as f:
file_tests_obj = pickle.load(f)
only_last = arguments['--only-last'] and int(arguments['--only-last'])
file_tests_obj.redo(
grammar,
debugger,
only_last=only_last,
print_code=arguments['--print-code']
)
elif arguments['random']:
# A random file is used to do diff parser checks if no file is given.
# This helps us to find errors in a lot of different files.
file_paths = list(find_python_files_in_tree(arguments['<path>'] or '.'))
max_tries = int(arguments['--maxtries'])
tries = 0
try:
while tries < max_tries:
path = random.choice(file_paths)
print("Checking %s: %s tries" % (path, tries))
now_tries = min(1000, max_tries - tries)
file_tests_obj = FileTests(path, now_tries, int(arguments['--changes']))
file_tests_obj.run(grammar, debugger)
tries += now_tries
except Exception:
with open(redo_file, 'wb') as f:
pickle.dump(file_tests_obj, f)
raise
else:
raise NotImplementedError('Command is not implemented')
def main(arguments):
debugger = 'pdb' if arguments['--pdb'] else \
'ipdb' if arguments['--ipdb'] else None
redo_file = os.path.join(os.path.dirname(__file__), 'fuzz-redo.pickle')
if arguments['--logging']:
root = logging.getLogger()
root.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
root.addHandler(ch)
grammar = parso.load_grammar()
parso.python.diff.DEBUG_DIFF_PARSER = True
if arguments['redo']:
with open(redo_file, 'rb') as f:
file_tests_obj = pickle.load(f)
only_last = arguments['--only-last'] and int(arguments['--only-last'])
file_tests_obj.redo(
grammar,
debugger,
only_last=only_last,
print_code=arguments['--print-code']
)
elif arguments['random']:
# A random file is used to do diff parser checks if no file is given.
# This helps us to find errors in a lot of different files.
file_paths = list(find_python_files_in_tree(arguments['<path>'] or '.'))
max_tries = int(arguments['--maxtries'])
tries = 0
try:
while tries < max_tries:
path = random.choice(file_paths)
print("Checking %s: %s tries" % (path, tries))
now_tries = min(1000, max_tries - tries)
file_tests_obj = FileTests(path, now_tries, int(arguments['--changes']))
file_tests_obj.run(grammar, debugger)
tries += now_tries
except Exception:
with open(redo_file, 'wb') as f:
pickle.dump(file_tests_obj, f)
raise
else:
raise NotImplementedError('Command is not implemented')
def __init__(self,
source=None,
line=None,
column=None,
path=None,
encoding='utf-8',
sys_path=None):
self._orig_path = path
# An empty path (also empty string) should always result in no path.
self.path = os.path.abspath(path) if path else None
if source is None:
# TODO add a better warning than the traceback!
with open(path, 'rb') as f:
source = f.read()
# Load the Python grammar of the current interpreter.
self._grammar = parso.load_grammar()
project = Project(sys_path=sys_path)
self._evaluator = Evaluator(self._grammar, project)
self._module_node, source = self._evaluator.parse_and_get_code(
code=source,
path=self.path,
cache=
False, # No disk cache, because the current script often changes.
diff_cache=True,
cache_path=settings.cache_directory)
self._code_lines = split_lines(source)
line = max(len(self._code_lines), 1) if line is None else line
if not (0 < line <= len(self._code_lines)):
raise ValueError('`line` parameter is not in a valid range.')
line_len = len(self._code_lines[line - 1])
column = line_len if column is None else column
if not (0 <= column <= line_len):
raise ValueError('`column` parameter is not in a valid range.')
self._pos = line, column
self._path = path
cache.clear_time_caches()
debug.reset_time()
project.add_script_path(self.path)
debug.speed('init')
def __init__(self,
source=None,
line=None,
column=None,
path=None,
encoding='utf-8',
sys_path=None):
self._orig_path = path
# An empty path (also empty string) should always result in no path.
self.path = os.path.abspath(path) if path else None
if source is None:
# TODO add a better warning than the traceback!
with open(path, 'rb') as f:
source = f.read()
# TODO do we really want that?
self._source = python_bytes_to_unicode(source,
encoding,
errors='replace')
self._code_lines = split_lines(self._source)
line = max(len(self._code_lines), 1) if line is None else line
if not (0 < line <= len(self._code_lines)):
raise ValueError('`line` parameter is not in a valid range.')
line_len = len(self._code_lines[line - 1])
column = line_len if column is None else column
if not (0 <= column <= line_len):
raise ValueError('`column` parameter is not in a valid range.')
self._pos = line, column
self._path = path
cache.clear_time_caches()
debug.reset_time()
# Load the Python grammar of the current interpreter.
self._grammar = parso.load_grammar()
if sys_path is None:
venv = os.getenv('VIRTUAL_ENV')
if venv:
sys_path = list(get_venv_path(venv))
self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
debug.speed('init')
def __init__(self, grammar, project):
self.grammar = grammar
self.latest_grammar = parso.load_grammar(version='3.6')
self.memoize_cache = {} # for memoize decorators
# To memorize modules -> equals `sys.modules`.
self.modules = {} # like `sys.modules`.
self.compiled_cache = {} # see `evaluate.compiled.create()`
self.inferred_element_counts = {}
self.mixed_cache = {} # see `evaluate.compiled.mixed._create()`
self.analysis = []
self.dynamic_params_depth = 0
self.is_analysis = False
self.python_version = sys.version_info[:2]
self.project = project
project.add_evaluator(self)
self.reset_recursion_limitations()
# Constants
self.BUILTINS = compiled.get_special_object(self, 'BUILTINS')
def test_normalizer_issue(normalizer_issue_case):
def sort(issues):
issues = sorted(issues, key=lambda i: (i.start_pos, i.code))
return ["(%s, %s): %s" % (i.start_pos[0], i.start_pos[1], i.code)
for i in issues]
with open(normalizer_issue_case.path, 'rb') as f:
code = python_bytes_to_unicode(f.read())
desired = sort(collect_errors(code))
grammar = parso.load_grammar(version=normalizer_issue_case.python_version)
module = grammar.parse(code)
issues = grammar._get_normalizer_issues(module)
actual = sort(issues)
diff = '\n'.join(difflib.ndiff(desired, actual))
# To make the pytest -v diff a bit prettier, stop pytest to rewrite assert
# statements by executing the comparison earlier.
_bool = desired == actual
assert _bool, '\n' + diff
def __init__(self, project, environment=None, script_path=None):
if environment is None:
environment = project.get_environment()
self.environment = environment
self.script_path = script_path
self.compiled_subprocess = environment.get_evaluator_subprocess(self)
self.grammar = environment.get_grammar()
self.latest_grammar = parso.load_grammar(version='3.6')
self.memoize_cache = {} # for memoize decorators
# To memorize modules -> equals `sys.modules`.
self.modules = {} # like `sys.modules`.
self.compiled_cache = {} # see `evaluate.compiled.create()`
self.inferred_element_counts = {}
self.mixed_cache = {} # see `evaluate.compiled.mixed._create()`
self.analysis = []
self.dynamic_params_depth = 0
self.is_analysis = False
self.project = project
self.access_cache = {}
self.reset_recursion_limitations()
self.allow_different_encoding = True
def test_modulepickling_change_cache_dir(tmpdir):
"""
ParserPickling should not save old cache when cache_directory is changed.
See: `#168 <https://github.com/davidhalter/jedi/pull/168>`_
"""
dir_1 = str(tmpdir.mkdir('first'))
dir_2 = str(tmpdir.mkdir('second'))
item_1 = _NodeCacheItem('bla', [])
item_2 = _NodeCacheItem('bla', [])
path_1 = 'fake path 1'
path_2 = 'fake path 2'
hashed_grammar = load_grammar()._hashed
_save_to_file_system(hashed_grammar, path_1, item_1, cache_path=dir_1)
parser_cache.clear()
cached = load_stored_item(hashed_grammar, path_1, item_1, cache_path=dir_1)
assert cached == item_1.node
_save_to_file_system(hashed_grammar, path_2, item_2, cache_path=dir_2)
cached = load_stored_item(hashed_grammar, path_1, item_1, cache_path=dir_2)
assert cached is None
def test_modulepickling_change_cache_dir(tmpdir):
"""
ParserPickling should not save old cache when cache_directory is changed.
See: `#168 <https://github.com/davidhalter/jedi/pull/168>`_
"""
dir_1 = str(tmpdir.mkdir('first'))
dir_2 = str(tmpdir.mkdir('second'))
item_1 = _NodeCacheItem('bla', [])
item_2 = _NodeCacheItem('bla', [])
path_1 = 'fake path 1'
path_2 = 'fake path 2'
hashed_grammar = load_grammar()._hashed
_save_to_file_system(hashed_grammar, path_1, item_1, cache_path=dir_1)
parser_cache.clear()
cached = load_stored_item(hashed_grammar, path_1, item_1, cache_path=dir_1)
assert cached == item_1.node
_save_to_file_system(hashed_grammar, path_2, item_2, cache_path=dir_2)
cached = load_stored_item(hashed_grammar, path_1, item_1, cache_path=dir_2)
assert cached is None
def _evaluator():
return Evaluator(parso.load_grammar())
def issues(code):
grammar = parso.load_grammar()
module = parso.parse(code)
return grammar._get_normalizer_issues(module)
import jedi
from jedi import debug
from jedi._compatibility import unicode, is_py3
from jedi.api.classes import Definition
from jedi.api.completion import get_user_scope
from jedi import parser_utils
from jedi.api.environment import get_default_environment, get_system_environment
TEST_COMPLETIONS = 0
TEST_DEFINITIONS = 1
TEST_ASSIGNMENTS = 2
TEST_USAGES = 3
grammar36 = parso.load_grammar(version='3.6')
class IntegrationTestCase(object):
def __init__(self, test_type, correct, line_nr, column, start, line,
path=None, skip_version_info=None):
self.test_type = test_type
self.correct = correct
self.line_nr = line_nr
self.column = column
self.start = start
self.line = line
self.path = path
self._skip_version_info = skip_version_info
self._skip = None
def _parse(code, version=None):
code = dedent(code) + "\n\n"
grammar = load_grammar(version=version)
return grammar.parse(code, error_recovery=False)
--pdb Launch pdb when error is raised
--ipdb Launch ipdb when error is raised
"""
from __future__ import print_function
import logging
import sys
import os
import random
import pickle
import parso
from parso.utils import split_lines
from test.test_diff_parser import _check_error_leaves_nodes
_latest_grammar = parso.load_grammar(version='3.8')
_python_reserved_strings = tuple(
# Keywords are ususally only interesting in combination with spaces after
# them. We don't put a space before keywords, to avoid indentation errors.
s + (' ' if s.isalpha() else '')
for s in _latest_grammar._pgen_grammar.reserved_syntax_strings.keys()
)
_random_python_fragments = _python_reserved_strings + (
' ', '\t', '\n', '\r', '\f', 'f"', 'F"""', "fr'", "RF'''", '"', '"""', "'",
"'''", ';', ' some_random_word ', '\\', '#',
)
def find_python_files_in_tree(file_path):
if not os.path.isdir(file_path):
yield file_path
def grammar():
return load_grammar(version='3.6')
def grammar():
return load_grammar(language="python-f-string")
def __init__(self, version, is_passing):
self.version = version
self._is_passing = is_passing
self.grammar = parso.load_grammar(version=self.version)
def _get_error_list(code, version=None):
grammar = parso.load_grammar(version=version)
tree = grammar.parse(code)
return list(grammar.iter_errors(tree))
def get_grammar(self):
version_string = '%s.%s' % (self.version_info.major, self.version_info.minor)
return parso.load_grammar(version=version_string)