def test_transitivity(self):
with temppathlib.TemporaryDirectory() as tmp:
script_pth = tmp.path / "some_script.py"
script_pth.write_text("#!/usr/bin/env python\n\n"
"import something_local\n")
module_pth = tmp.path / "something_local.py"
module_pth.write_text("#!/usr/bin/env python\n"
"import unittest\n"
"import PIL.Image\n"
"import something_else_local")
another_module_pth = tmp.path / "something_else_local.py"
another_module_pth.write_text("#!/usr/bin/env python\n")
requirements = packagery.parse_requirements(text="pillow==5.2.0\n")
module_to_requirement = packagery.parse_module_to_requirement(
text="PIL.Image\tpillow")
pkg = packagery.collect_dependency_graph(
root_dir=tmp.path,
rel_paths=[pathlib.Path("some_script.py")],
requirements=requirements,
module_to_requirement=module_to_requirement)
self.assertListEqual([], pkg.unresolved_modules)
self.assertListEqual(["pillow"], list(pkg.requirements.keys()))
self.assertSetEqual(
{
pathlib.Path("some_script.py"),
pathlib.Path("something_local.py"),
pathlib.Path("something_else_local.py")
}, pkg.rel_paths)
def test_builtin_local_pip_dependencies(self): # pylint: disable=invalid-name
with temppathlib.TemporaryDirectory() as tmp:
script_pth = tmp.path / "some_script.py"
script_pth.write_text("#!/usr/bin/env python\n\n"
"import sys\n\n"
"import PIL.Image\n\n"
"import something_local\n")
module_pth = tmp.path / "something_local.py"
module_pth.write_text("#!/usr/bin/env python\n")
requirements = packagery.parse_requirements(text="pillow==5.2.0\n")
module_to_requirement = packagery.parse_module_to_requirement(
text="PIL.Image\tpillow")
pkg = packagery.collect_dependency_graph(
root_dir=tmp.path,
rel_paths=[pathlib.Path("some_script.py")],
requirements=requirements,
module_to_requirement=module_to_requirement)
self.assertListEqual([], pkg.unresolved_modules)
self.assertListEqual(["pillow"], list(pkg.requirements.keys()))
self.assertSetEqual(
{
pathlib.Path("some_script.py"),
pathlib.Path("something_local.py")
}, pkg.rel_paths)
def test_local_dependency_with_init(self):
with temppathlib.TemporaryDirectory(dont_delete=True) as tmp:
script_pth = tmp.path / "some_script.py"
script_pth.write_text(
"#!/usr/bin/env python\nimport mapried.config\n")
module_pth = tmp.path / "mapried/config/__init__.py"
module_pth.parent.mkdir(parents=True)
module_pth.write_text("#!/usr/bin/env python\n")
(tmp.path /
"mapried/__init__.py").write_text("#!/usr/bin/env python\n")
pkg = packagery.collect_dependency_graph(
root_dir=tmp.path,
rel_paths=[pathlib.Path("some_script.py")],
requirements={},
module_to_requirement={})
self.assertListEqual([], pkg.unresolved_modules)
self.assertSetEqual(
{
pathlib.Path("some_script.py"),
pathlib.Path("mapried/__init__.py"),
pathlib.Path("mapried/config/__init__.py")
}, pkg.rel_paths)
def test_local_dependency_without_init(self): # pylint: disable=invalid-name
with temppathlib.TemporaryDirectory() as tmp:
script_pth = tmp.path / "some_script.py"
script_pth.write_text(
"#!/usr/bin/env python\nimport some_module.something\n")
module_pth = tmp.path / "some_module/something.py"
module_pth.parent.mkdir()
(tmp.path / "some_module/__init__.py").write_text("'''hello'''\n")
module_pth.write_text("#!/usr/bin/env python\n'''hello'''\n")
pkg = packagery.collect_dependency_graph(
root_dir=tmp.path,
rel_paths=[pathlib.Path("some_script.py")],
requirements={},
module_to_requirement={})
self.assertListEqual([], pkg.unresolved_modules)
self.assertSetEqual(
{
pathlib.Path("some_script.py"),
pathlib.Path("some_module/something.py"),
pathlib.Path("some_module/__init__.py")
}, pkg.rel_paths)
def test_builtin_without_file(self):
with temppathlib.TemporaryDirectory() as tmp:
script_pth = tmp.path / "some_script.py"
script_pth.write_text("#!/usr/bin/env python\n\n" "import sys\n")
pkg = packagery.collect_dependency_graph(
root_dir=tmp.path,
rel_paths=[pathlib.Path("some_script.py")],
requirements=dict(),
module_to_requirement=dict())
self.assertListEqual([], pkg.unresolved_modules)
self.assertListEqual([], list(pkg.requirements.keys()))
self.assertSetEqual({pathlib.Path("some_script.py")},
pkg.rel_paths)
def test_missing_module(self):
with temppathlib.TemporaryDirectory() as tmp:
script_pth = tmp.path / "some_script.py"
script_pth.write_text("#!/usr/bin/env python\n\n"
"import missing\n")
pkg = packagery.collect_dependency_graph(
root_dir=tmp.path,
rel_paths=[pathlib.Path("some_script.py")],
requirements={},
module_to_requirement={})
self.assertListEqual([
packagery.UnresolvedModule(
name="missing",
importer_rel_path=pathlib.Path("some_script.py")).__dict__
], [
unresolved_module.__dict__
for unresolved_module in pkg.unresolved_modules
])
def generate_package_with_local_pip_and_missing_deps() -> packagery.Package: # pylint: disable=invalid-name
with temppathlib.TemporaryDirectory() as tmp:
script_pth = tmp.path / "some_script.py"
script_pth.write_text("#!/usr/bin/env python\n\n"
"import PIL.Image\n\n"
"import something_local\n"
"import something_missing\n")
module_pth = tmp.path / "something_local.py"
module_pth.write_text("#!/usr/bin/env python\n")
requirements = packagery.parse_requirements(text="pillow==5.2.0\n")
module_to_requirement = packagery.parse_module_to_requirement(
text="PIL.Image\tpillow")
pkg = packagery.collect_dependency_graph(
root_dir=tmp.path,
rel_paths=[pathlib.Path("some_script.py")],
requirements=requirements,
module_to_requirement=module_to_requirement)
return pkg
def main() -> int:
"""Execute the main routine."""
# pylint: disable=too-many-locals
parser = argparse.ArgumentParser(description=pypackagery_meta.__description__)
parser.add_argument("--root_dir", help="Root directory of the Python files in the monorepo", required=True)
parser.add_argument(
"--initial_set",
help="Paths to the files for which we want to compute the dependencies.\n\n"
"If you specify a directory, all *.py files beneath (including subdirectories) are considered as part of "
"the initial set.",
nargs='+',
required=True)
parser.add_argument(
"--format",
help="The format of the output depedendency graph; default is the verbose, human-readable format",
choices=packagery.FORMATS,
default='verbose')
parser.add_argument(
"--dont_panic",
help="If set, does not return an error code if some of the dependencies could not be resolved",
action="store_true")
parser.add_argument("--output_path", help="If set, outputs the result to a file instead of STDOUT")
args = parser.parse_args()
root_dir = pathlib.Path(args.root_dir).absolute()
fmt = str(args.format)
dont_panic = bool(args.dont_panic)
output_path = None if not args.output_path else pathlib.Path(args.output_path)
assert isinstance(args.initial_set, list)
assert all(isinstance(item, str) for item in args.initial_set)
initial_set = args.initial_set # type: List[str]
initial_pths = [pathlib.Path(pth).absolute() for pth in initial_set]
for pth in initial_pths:
if root_dir != pth and root_dir not in pth.parents:
raise ValueError(("Expected all the files of the initial set to reside beneath root directory {}, "
"but at least one does not: {}").format(root_dir, pth))
initial_files = packagery.resolve_initial_paths(initial_paths=initial_pths)
# yapf: disable
rel_pths = [pth.relative_to(root_dir) for pth in initial_files]
# yapf: enable
requirements_txt = root_dir / "requirements.txt"
if not requirements_txt.exists():
raise FileNotFoundError(("requirements.txt expected beneath the root directory {}, "
"but could not be found: {})".format(root_dir, requirements_txt)))
module_to_requirement_tsv = root_dir / "module_to_requirement.tsv"
if not module_to_requirement_tsv.exists():
raise FileNotFoundError(("module_to_requirement.tsv expected beneath the root directory {}, "
"but could not be found: {})".format(root_dir, module_to_requirement_tsv)))
requirements = packagery.parse_requirements(text=requirements_txt.read_text(), filename=str(requirements_txt))
module_to_requirement = packagery.parse_module_to_requirement(
text=module_to_requirement_tsv.read_text(), filename=str(module_to_requirement_tsv))
missing_reqs = packagery.missing_requirements(
module_to_requirement=module_to_requirement, requirements=requirements)
if len(missing_reqs) > 0:
raise RuntimeError(
("The requirements listed in moudle_to_requirement mapping are missing in requirements file:\n"
"module-to-requirement file: {}\n"
"requirements file: {}\n"
"Missing requirements:\n{}").format(module_to_requirement_tsv, requirements_txt, "\n".join(missing_reqs)))
pkg = packagery.collect_dependency_graph(
root_dir=root_dir, rel_paths=rel_pths, requirements=requirements, module_to_requirement=module_to_requirement)
if output_path is None:
packagery.output(package=pkg, a_format=fmt)
else:
with output_path.open('w') as fid:
packagery.output(package=pkg, out=fid, a_format=fmt)
if not dont_panic and len(pkg.unresolved_modules) > 0:
return 1
return 0
