def parse_and_get_code(self, code=None, path=None, encoding='utf-8',
use_latest_grammar=False, file_io=None, **kwargs):
if self.allow_different_encoding:
if code is None:
if file_io is None:
file_io = FileIO(path)
code = file_io.read()
code = python_bytes_to_unicode(code, encoding=encoding, errors='replace')
grammar = self.latest_grammar if use_latest_grammar else self.grammar
return grammar.parse(code=code, path=path, file_io=file_io, **kwargs), code
def _load_from_typeshed(inference_state, python_value_set, parent_module_value,
import_names):
import_name = import_names[-1]
map_ = None
if len(import_names) == 1:
map_ = _cache_stub_file_map(inference_state.grammar.version_info)
import_name = _IMPORT_MAP.get(import_name, import_name)
elif isinstance(parent_module_value, ModuleValue):
if not parent_module_value.is_package():
# Only if it's a package (= a folder) something can be
# imported.
return None
paths = parent_module_value.py__path__()
# Once the initial package has been loaded, the sub packages will
# always be loaded, regardless if they are there or not. This makes
# sense, IMO, because stubs take preference, even if the original
# library doesn't provide a module (it could be dynamic). ~dave
map_ = _merge_create_stub_map(
[PathInfo(p, is_third_party=False) for p in paths])
if map_ is not None:
path_info = map_.get(import_name)
if path_info is not None and (not path_info.is_third_party
or python_value_set):
return _try_to_load_stub_from_file(
inference_state,
python_value_set,
file_io=FileIO(path_info.path),
import_names=import_names,
)
def parse_and_get_code(self, code=None, path=None,
use_latest_grammar=False, file_io=None, **kwargs):
if path is not None:
path = str(path)
if code is None:
if file_io is None:
file_io = FileIO(path)
code = file_io.read()
# We cannot just use parso, because it doesn't use errors='replace'.
code = parso.python_bytes_to_unicode(code, encoding='utf-8', errors='replace')
if len(code) > settings._cropped_file_size:
code = code[:settings._cropped_file_size]
grammar = self.latest_grammar if use_latest_grammar else self.grammar
return grammar.parse(code=code, path=path, file_io=file_io, **kwargs), code
def get_file_ios_to_check():
for folder_io, base_names in folders_with_names_to_be_checked:
for file_io in check_directory(folder_io):
yield file_io, base_names
for p in settings.additional_dynamic_modules:
p = os.path.abspath(p)
if p not in used_mod_paths:
yield FileIO(p), None
def test_get_modules_containing_name(evaluator, path, goal, is_package):
module = imports._load_python_module(
evaluator,
FileIO(path),
import_names=('ok', 'lala', 'x'),
is_package=is_package,
)
assert module
input_module, found_module = imports.get_modules_containing_name(
evaluator, [module], 'string_that_only_exists_here')
assert input_module is module
assert found_module.string_names == goal
def test_get_modules_containing_name(inference_state, path, goal, is_package):
module = imports._load_python_module(
inference_state,
FileIO(path),
import_names=('ok', 'lala', 'x'),
is_package=is_package,
)
assert module
module_context = module.as_context()
input_module, found_module = imports.get_module_contexts_containing_name(
inference_state, [module_context], 'string_that_only_exists_here')
assert input_module is module_context
assert found_module.string_names == goal
def _get_paths_from_buildout_script(inference_state, buildout_script_path):
file_io = FileIO(str(buildout_script_path))
try:
module_node = inference_state.parse(
file_io=file_io, cache=True, cache_path=settings.cache_directory)
except IOError:
debug.warning('Error trying to read buildout_script: %s',
buildout_script_path)
return
from jedi.inference.value import ModuleValue
module_context = ModuleValue(
inference_state,
module_node,
file_io=file_io,
string_names=None,
code_lines=get_cached_code_lines(inference_state.grammar,
str(buildout_script_path)),
).as_context()
yield from check_sys_path_modifications(module_context)
def _get_paths_from_buildout_script(evaluator, buildout_script_path):
file_io = FileIO(buildout_script_path)
try:
module_node = evaluator.parse(file_io=file_io,
cache=True,
cache_path=settings.cache_directory)
except IOError:
debug.warning('Error trying to read buildout_script: %s',
buildout_script_path)
return
from jedi.evaluate.context import ModuleContext
module = ModuleContext(
evaluator,
module_node,
file_io,
string_names=None,
code_lines=get_cached_code_lines(evaluator.grammar,
buildout_script_path),
)
for path in check_sys_path_modifications(module):
yield path
def _load_from_typeshed(inference_state, python_value_set, parent_module_value, import_names):
import_name = import_names[-1]
map_ = None
if len(import_names) == 1:
map_ = _cache_stub_file_map(inference_state.grammar.version_info)
import_name = _IMPORT_MAP.get(import_name, import_name)
elif isinstance(parent_module_value, ModuleValue):
if not parent_module_value.is_package():
# Only if it's a package (= a folder) something can be
# imported.
return None
path = parent_module_value.py__path__()
map_ = _merge_create_stub_map(path)
if map_ is not None:
path = map_.get(import_name)
if path is not None:
return _try_to_load_stub_from_file(
inference_state,
python_value_set,
file_io=FileIO(path),
import_names=import_names,
)
def _load_from_typeshed(evaluator, actual_context_set, parent_module_context,
import_names):
import_name = import_names[-1]
map_ = None
if len(import_names) == 1:
map_ = _cache_stub_file_map(evaluator.grammar.version_info)
elif isinstance(parent_module_context, StubModuleContext):
if not parent_module_context.is_package:
# Only if it's a package (= a folder) something can be
# imported.
return None
path = parent_module_context.py__path__()
map_ = _merge_create_stub_map(path)
if map_ is not None:
path = map_.get(import_name)
if path is not None:
return _try_to_load_stub_from_file(
evaluator,
actual_context_set,
file_io=FileIO(path),
import_names=import_names,
)
def _try_to_load_stub(inference_state, import_names, python_value_set,
parent_module_value, sys_path):
"""
Trying to load a stub for a set of import_names.
This is modelled to work like "PEP 561 -- Distributing and Packaging Type
Information", see https://www.python.org/dev/peps/pep-0561.
"""
if parent_module_value is None and len(import_names) > 1:
try:
parent_module_value = try_to_load_stub_cached(
inference_state,
import_names[:-1],
NO_VALUES,
parent_module_value=None,
sys_path=sys_path)
except KeyError:
pass
# 1. Try to load foo-stubs folders on path for import name foo.
if len(import_names) == 1:
# foo-stubs
for p in sys_path:
p = cast_path(p)
init = os.path.join(
p, *import_names) + '-stubs' + os.path.sep + '__init__.pyi'
m = _try_to_load_stub_from_file(
inference_state,
python_value_set,
file_io=FileIO(init),
import_names=import_names,
)
if m is not None:
return m
if import_names[0] == 'django' and python_value_set:
return _try_to_load_stub_from_file(
inference_state,
python_value_set,
file_io=FileIO(DJANGO_INIT_PATH),
import_names=import_names,
)
# 2. Try to load pyi files next to py files.
for c in python_value_set:
try:
method = c.py__file__
except AttributeError:
pass
else:
file_path = method()
file_paths = []
if c.is_namespace():
file_paths = [
os.path.join(p, '__init__.pyi') for p in c.py__path__()
]
elif file_path is not None and file_path.endswith('.py'):
file_paths = [file_path + 'i']
for file_path in file_paths:
m = _try_to_load_stub_from_file(
inference_state,
python_value_set,
# The file path should end with .pyi
file_io=FileIO(file_path),
import_names=import_names,
)
if m is not None:
return m
# 3. Try to load typeshed
m = _load_from_typeshed(inference_state, python_value_set,
parent_module_value, import_names)
if m is not None:
return m
# 4. Try to load pyi file somewhere if python_value_set was not defined.
if not python_value_set:
if parent_module_value is not None:
check_path = parent_module_value.py__path__() or []
# In case import_names
names_for_path = (import_names[-1], )
else:
check_path = sys_path
names_for_path = import_names
for p in check_path:
m = _try_to_load_stub_from_file(
inference_state,
python_value_set,
file_io=FileIO(os.path.join(p, *names_for_path) + '.pyi'),
import_names=import_names,
)
if m is not None:
return m
# If no stub is found, that's fine, the calling function has to deal with
# it.
return None
def _find_syntax_node_name(inference_state, python_object):
original_object = python_object
try:
python_object = _get_object_to_check(python_object)
path = inspect.getsourcefile(python_object)
except TypeError:
# The type might not be known (e.g. class_with_dict.__weakref__)
return None
if path is None or not os.path.exists(path):
# The path might not exist or be e.g. <stdin>.
return None
file_io = FileIO(path)
module_node = _load_module(inference_state, path)
if inspect.ismodule(python_object):
# We don't need to check names for modules, because there's not really
# a way to write a module in a module in Python (and also __name__ can
# be something like ``email.utils``).
code_lines = get_cached_code_lines(inference_state.grammar, path)
return module_node, module_node, file_io, code_lines
try:
name_str = python_object.__name__
except AttributeError:
# Stuff like python_function.__code__.
return None
if name_str == '<lambda>':
return None # It's too hard to find lambdas.
# Doesn't always work (e.g. os.stat_result)
names = module_node.get_used_names().get(name_str, [])
# Only functions and classes are relevant. If a name e.g. points to an
# import, it's probably a builtin (like collections.deque) and needs to be
# ignored.
names = [
n for n in names
if n.parent.type in ('funcdef', 'classdef') and n.parent.name == n
]
if not names:
return None
try:
code = python_object.__code__
# By using the line number of a code object we make the lookup in a
# file pretty easy. There's still a possibility of people defining
# stuff like ``a = 3; foo(a); a = 4`` on the same line, but if people
# do so we just don't care.
line_nr = code.co_firstlineno
except AttributeError:
pass
else:
line_names = [name for name in names if name.start_pos[0] == line_nr]
# There's a chance that the object is not available anymore, because
# the code has changed in the background.
if line_names:
names = line_names
code_lines = get_cached_code_lines(inference_state.grammar, path)
# It's really hard to actually get the right definition, here as a last
# resort we just return the last one. This chance might lead to odd
# completions at some points but will lead to mostly correct type
# inference, because people tend to define a public name in a module only
# once.
tree_node = names[-1].parent
if tree_node.type == 'funcdef' and get_api_type(
original_object) == 'instance':
# If an instance is given and we're landing on a function (e.g.
# partial in 3.5), something is completely wrong and we should not
# return that.
return None
return module_node, tree_node, file_io, code_lines
def _find_syntax_node_name(evaluator, access_handle):
# TODO accessing this is bad, but it probably doesn't matter that much,
# because we're working with interpreteters only here.
python_object = access_handle.access._obj
try:
python_object = _get_object_to_check(python_object)
path = inspect.getsourcefile(python_object)
except TypeError:
# The type might not be known (e.g. class_with_dict.__weakref__)
return None
if path is None or not os.path.exists(path):
# The path might not exist or be e.g. <stdin>.
return None
file_io = FileIO(path)
module_node = _load_module(evaluator, path)
if inspect.ismodule(python_object):
# We don't need to check names for modules, because there's not really
# a way to write a module in a module in Python (and also __name__ can
# be something like ``email.utils``).
code_lines = get_cached_code_lines(evaluator.grammar, path)
return module_node, module_node, file_io, code_lines
try:
name_str = python_object.__name__
except AttributeError:
# Stuff like python_function.__code__.
return None
if name_str == '<lambda>':
return None # It's too hard to find lambdas.
# Doesn't always work (e.g. os.stat_result)
names = module_node.get_used_names().get(name_str, [])
names = [n for n in names if n.is_definition()]
if not names:
return None
try:
code = python_object.__code__
# By using the line number of a code object we make the lookup in a
# file pretty easy. There's still a possibility of people defining
# stuff like ``a = 3; foo(a); a = 4`` on the same line, but if people
# do so we just don't care.
line_nr = code.co_firstlineno
except AttributeError:
pass
else:
line_names = [name for name in names if name.start_pos[0] == line_nr]
# There's a chance that the object is not available anymore, because
# the code has changed in the background.
if line_names:
names = line_names
code_lines = get_cached_code_lines(evaluator.grammar, path)
# It's really hard to actually get the right definition, here as a last
# resort we just return the last one. This chance might lead to odd
# completions at some points but will lead to mostly correct type
# inference, because people tend to define a public name in a module only
# once.
return module_node, names[-1].parent, file_io, code_lines
