async def infer_python_dependencies(
request: InferPythonDependencies, python_inference: PythonInference
) -> InferredDependencies:
if not python_inference.imports:
return InferredDependencies()
stripped_sources = await Get(StrippedSourceFiles, SourceFilesRequest([request.sources_field]))
modules = tuple(
PythonModule.create_from_stripped_path(PurePath(fp))
for fp in stripped_sources.snapshot.files
)
digest_contents = await Get(DigestContents, Digest, stripped_sources.snapshot.digest)
imports_per_file = tuple(
find_python_imports(file_content.content.decode(), module_name=module.module)
for file_content, module in zip(digest_contents, modules)
)
owner_per_import = await MultiGet(
Get(PythonModuleOwner, PythonModule(imported_module))
for file_imports in imports_per_file
for imported_module in file_imports.explicit_imports
if imported_module not in combined_stdlib
)
return InferredDependencies(
owner.address
for owner in owner_per_import
if (
owner.address
and owner.address.maybe_convert_to_base_target() != request.sources_field.address
)
)
async def infer_python_dependencies(
request: InferPythonDependencies,
python_inference: PythonInference) -> InferredDependencies:
if not python_inference.imports:
return InferredDependencies([], sibling_dependencies_inferrable=False)
stripped_sources = await Get(StrippedSourceFiles,
SourceFilesRequest([request.sources_field]))
modules = tuple(
PythonModule.create_from_stripped_path(PurePath(fp))
for fp in stripped_sources.snapshot.files)
digest_contents = await Get(DigestContents, Digest,
stripped_sources.snapshot.digest)
owner_requests: List[Get[PythonModuleOwner, PythonModule]] = []
for file_content, module in zip(digest_contents, modules):
file_imports_obj = find_python_imports(file_content.content.decode(),
module_name=module.module)
detected_imports = (file_imports_obj.all_imports
if python_inference.string_imports else
file_imports_obj.explicit_imports)
owner_requests.extend(
Get(PythonModuleOwner, PythonModule(imported_module))
for imported_module in detected_imports
if imported_module not in combined_stdlib)
owner_per_import = await MultiGet(owner_requests)
result = (
owner.address for owner in owner_per_import
if owner.address and owner.address != request.sources_field.address)
return InferredDependencies(result, sibling_dependencies_inferrable=True)
def test_relative_imports() -> None:
imports = find_python_imports(
filename="foo.py",
content=dedent("""\
from . import sibling
from .subdir.child import Child
from ..parent import Parent
"""),
module_name="project.util.test_utils",
)
assert set(imports.explicit_imports) == {
"project.util.sibling",
"project.util.subdir.child.Child",
"project.parent.Parent",
}
assert not imports.inferred_imports
def test_works_with_python2() -> None:
imports = find_python_imports(
filename="foo.py",
content=dedent("""\
print "Python 2 lives on."
import demo
from project.demo import Demo
importlib.import_module(b"dep.from.bytes")
importlib.import_module(u"dep.from.str")
"""),
module_name="project.app",
)
assert set(imports.explicit_imports) == {"demo", "project.demo.Demo"}
assert set(imports.inferred_imports) == {"dep.from.bytes", "dep.from.str"}
def test_works_with_python38() -> None:
imports = find_python_imports(
filename="foo.py",
content=dedent("""\
is_py38 = True
if walrus := is_py38:
print(walrus)
import demo
from project.demo import Demo
importlib.import_module("dep.from.str")
"""),
module_name="project.app",
)
assert set(imports.explicit_imports) == {"demo", "project.demo.Demo"}
assert set(imports.inferred_imports) == {"dep.from.str"}
def test_imports_from_strings() -> None:
imports = find_python_imports(
dedent(
"""\
modules = [
# Valid strings
'a.b.d',
'a.b2.d',
'a.b.c.Foo',
'a.b.c.d.Foo',
'a.b.c.d.FooBar',
'a.b.c.d.e.f.g.Baz',
'a.b_c.d._bar',
'a.b2.c.D',
# Invalid strings
'..a.b.c.d',
'a.b',
'a.B.d',
'a.2b.d',
'a..b..c',
'a.b.c.d.2Bar',
'a.b_c.D.bar',
'a.b_c.D.Bar',
'a.2b.c.D',
]
for module in modules:
importlib.import_module(module)
"""
),
module_name="project.app",
)
assert not imports.explicit_imports
assert set(imports.inferred_imports) == {
"a.b.d",
"a.b2.d",
"a.b.c.Foo",
"a.b.c.d.Foo",
"a.b.c.d.FooBar",
"a.b.c.d.e.f.g.Baz",
"a.b_c.d._bar",
"a.b2.c.D",
}
async def infer_python_dependencies(
request: InferPythonDependencies) -> InferredDependencies:
stripped_sources = await Get[SourceRootStrippedSources](
StripSourcesFieldRequest(request.sources_field))
modules = tuple(
PythonModule.create_from_stripped_path(PurePath(fp))
for fp in stripped_sources.snapshot.files)
files_content = await Get[FilesContent](Digest,
stripped_sources.snapshot.digest)
imports_per_file = tuple(
find_python_imports(fc.content.decode(), module_name=module.module)
for fc, module in zip(files_content, modules))
owner_per_import = await MultiGet(
Get[PythonModuleOwner](PythonModule(imported_module))
for file_imports in imports_per_file
for imported_module in file_imports.all_imports
if imported_module not in combined_stdlib)
return InferredDependencies(
owner.address for owner in owner_per_import
if owner.address and owner.address != request.sources_field.address)
def test_normal_imports() -> None:
imports = find_python_imports(
dedent(
"""\
from __future__ import print_function
import os
import os.path
from typing import TYPE_CHECKING
import requests
import demo
from project.demo import Demo
from project.demo import OriginalName as Renamed
if TYPE_CHECKING:
from project.circular_dep import CircularDep
try:
import subprocess
except ImportError:
import subprocess23 as subprocess
"""
),
module_name="project.app",
)
assert set(imports.explicit_imports) == {
"__future__.print_function",
"os",
"os.path",
"typing.TYPE_CHECKING",
"requests",
"demo",
"project.demo.Demo",
"project.demo.OriginalName",
"project.circular_dep.CircularDep",
"subprocess",
"subprocess23",
}
assert not imports.inferred_imports
async def infer_python_dependencies(
request: InferPythonDependencies) -> InferredDependencies:
stripped_sources = await Get[SourceRootStrippedSources](
StripSourcesFieldRequest(request.sources_field))
modules = tuple(
PythonModule.create_from_stripped_path(PurePath(fp))
for fp in stripped_sources.snapshot.files)
files_content = await Get[FilesContent](Digest,
stripped_sources.snapshot.digest)
imports_per_file = tuple(
find_python_imports(fc.content.decode(), module_name=module.module)
for fc, module in zip(files_content, modules))
owners_per_import = await MultiGet(
Get[PythonModuleOwners](PythonModule(imported_module))
for file_imports in imports_per_file
for imported_module in file_imports.all_imports
if imported_module not in combined_stdlib)
# We conservatively only use dep inference if there is exactly one owner for an import.
return InferredDependencies(
itertools.chain.from_iterable(
owners for owners in owners_per_import if len(owners) == 1
and tuple(owners)[0] != request.sources_field.address))
async def infer_python_dependencies(
request: InferPythonDependencies,
python_inference: PythonInference) -> InferredDependencies:
if not python_inference.imports:
return InferredDependencies([], sibling_dependencies_inferrable=False)
stripped_sources = await Get(StrippedSourceFiles,
SourceFilesRequest([request.sources_field]))
modules = tuple(
PythonModule.create_from_stripped_path(PurePath(fp))
for fp in stripped_sources.snapshot.files)
digest_contents = await Get(DigestContents, Digest,
stripped_sources.snapshot.digest)
owners_requests: List[Get[PythonModuleOwners, PythonModule]] = []
for file_content, module in zip(digest_contents, modules):
file_imports_obj = find_python_imports(
filename=file_content.path,
content=file_content.content.decode(),
module_name=module.module,
)
detected_imports = (file_imports_obj.all_imports
if python_inference.string_imports else
file_imports_obj.explicit_imports)
owners_requests.extend(
Get(PythonModuleOwners, PythonModule(imported_module))
for imported_module in detected_imports
if imported_module not in combined_stdlib)
owners_per_import = await MultiGet(owners_requests)
# We remove the request's address so that we don't infer dependencies on self.
merged_result = sorted(
set(itertools.chain.from_iterable(owners_per_import)) -
{request.sources_field.address})
return InferredDependencies(merged_result,
sibling_dependencies_inferrable=True)
def test_gracefully_handle_syntax_errors() -> None:
imports = find_python_imports(filename="foo.py",
content="x =",
module_name="project.app")
assert not imports.explicit_imports
assert not imports.inferred_imports