def test_invalid(self): text = "//wrong" value_error = None # type: Optional[ValueError] try: packagery.parse_requirements( text=text, filename="/some/path/requirements.txt") except ValueError as err: value_error = err self.assertIsNotNone(value_error) self.assertEqual( "Invalid requirements in file /some/path/requirements.txt", str(value_error))
def test_transitivity(self): with temppathlib.TemporaryDirectory() as tmp: script_pth = tmp.path / "some_script.py" script_pth.write_text("#!/usr/bin/env python\n\n" "import something_local\n") module_pth = tmp.path / "something_local.py" module_pth.write_text("#!/usr/bin/env python\n" "import unittest\n" "import PIL.Image\n" "import something_else_local") another_module_pth = tmp.path / "something_else_local.py" another_module_pth.write_text("#!/usr/bin/env python\n") requirements = packagery.parse_requirements(text="pillow==5.2.0\n") module_to_requirement = packagery.parse_module_to_requirement( text="PIL.Image\tpillow") pkg = packagery.collect_dependency_graph( root_dir=tmp.path, rel_paths=[pathlib.Path("some_script.py")], requirements=requirements, module_to_requirement=module_to_requirement) self.assertListEqual([], pkg.unresolved_modules) self.assertListEqual(["pillow"], list(pkg.requirements.keys())) self.assertSetEqual( { pathlib.Path("some_script.py"), pathlib.Path("something_local.py"), pathlib.Path("something_else_local.py") }, pkg.rel_paths)
def test_builtin_local_pip_dependencies(self): # pylint: disable=invalid-name with temppathlib.TemporaryDirectory() as tmp: script_pth = tmp.path / "some_script.py" script_pth.write_text("#!/usr/bin/env python\n\n" "import sys\n\n" "import PIL.Image\n\n" "import something_local\n") module_pth = tmp.path / "something_local.py" module_pth.write_text("#!/usr/bin/env python\n") requirements = packagery.parse_requirements(text="pillow==5.2.0\n") module_to_requirement = packagery.parse_module_to_requirement( text="PIL.Image\tpillow") pkg = packagery.collect_dependency_graph( root_dir=tmp.path, rel_paths=[pathlib.Path("some_script.py")], requirements=requirements, module_to_requirement=module_to_requirement) self.assertListEqual([], pkg.unresolved_modules) self.assertListEqual(["pillow"], list(pkg.requirements.keys())) self.assertSetEqual( { pathlib.Path("some_script.py"), pathlib.Path("something_local.py") }, pkg.rel_paths)
def test_without_name(self): url = "http://download.pytorch.org/whl/cpu/torch-0.3.1-cp35-cp35m-linux_x86_64.whl" # no egg fragment text = "{}\n".format(url) value_error = None # type: Optional[ValueError] try: packagery.parse_requirements(text=text, filename="/some/requirements.txt") except ValueError as err: value_error = err self.assertIsNotNone(value_error) self.assertEqual( "The name is missing in the requirement from the file /some/requirements.txt: " "http://download.pytorch.org/whl/cpu/torch-0.3.1-cp35-cp35m-linux_x86_64.whl " "(did you specify the egg fragment?)", str(value_error))
def test_without_version(self): text = "some_pack-age\n" reqs = packagery.parse_requirements(text=text) self.assertEqual(1, len(reqs)) self.assertIn('some_pack-age', reqs) requirement = reqs["some_pack-age"] self.assertEqual("some_pack-age", requirement.name) self.assertEqual("some_pack-age\n", requirement.line)
def test_with_whitespace(self): text = " some_package " reqs = packagery.parse_requirements(text=text) self.assertEqual(1, len(reqs)) self.assertIn('some_package', reqs) requirement = reqs['some_package'] self.assertEqual('some_package', requirement.name) self.assertEqual("some_package\n", requirement.line)
def test_with_comment_on_the_same_line(self): # pylint: disable=invalid-name text = "some_package # some comment" reqs = packagery.parse_requirements(text=text) self.assertEqual(1, len(reqs)) self.assertIn('some_package', reqs) requirement = reqs['some_package'] self.assertEqual('some_package', requirement.name) self.assertEqual("some_package # some comment\n", requirement.line)
def test_with_pinned_version(self): text = "some_package12 == 1.2.3" reqs = packagery.parse_requirements(text=text) self.assertEqual(1, len(reqs)) self.assertIn('some_package12', reqs) requirement = reqs['some_package12'] self.assertEqual('some_package12', requirement.name) self.assertEqual("some_package12 == 1.2.3\n", requirement.line)
def test_that_it_works(self): # yapf: disable module_to_requirement = collections.OrderedDict([("some_module", "some_missing_package"), ("another_module", "some_package")]) # yapf: enable requirements = packagery.parse_requirements( "some_package\nsome_unused_package") missing = packagery.missing_requirements( module_to_requirement=module_to_requirement, requirements=requirements) self.assertListEqual(['some_missing_package'], missing)
def test_url(self): url = "http://download.pytorch.org/whl/cpu/torch-0.3.1-cp35-cp35m-linux_x86_64.whl#egg=torch" text = "{}\n".format(url) reqs = packagery.parse_requirements(text=text) self.assertEqual(1, len(reqs)) self.assertIn("torch", reqs) requirement = reqs["torch"] self.assertEqual("torch", requirement.name) self.assertEqual( "http://download.pytorch.org/whl/cpu/torch-0.3.1-cp35-cp35m-linux_x86_64.whl#egg=torch\n", requirement.line)
def test_multiple(self): text = "some_package\nanother_package==1.2" reqs = packagery.parse_requirements(text=text) self.assertEqual(2, len(reqs)) self.assertIn('some_package', reqs) self.assertIn('another_package', reqs) requirement1 = reqs['some_package'] self.assertEqual('some_package', requirement1.name) self.assertEqual("some_package\n", requirement1.line) requirement2 = reqs['another_package'] self.assertEqual('another_package', requirement2.name) self.assertEqual("another_package==1.2\n", requirement2.line)
def generate_package_with_local_pip_and_missing_deps() -> packagery.Package: # pylint: disable=invalid-name with temppathlib.TemporaryDirectory() as tmp: script_pth = tmp.path / "some_script.py" script_pth.write_text("#!/usr/bin/env python\n\n" "import PIL.Image\n\n" "import something_local\n" "import something_missing\n") module_pth = tmp.path / "something_local.py" module_pth.write_text("#!/usr/bin/env python\n") requirements = packagery.parse_requirements(text="pillow==5.2.0\n") module_to_requirement = packagery.parse_module_to_requirement( text="PIL.Image\tpillow") pkg = packagery.collect_dependency_graph( root_dir=tmp.path, rel_paths=[pathlib.Path("some_script.py")], requirements=requirements, module_to_requirement=module_to_requirement) return pkg
def main() -> int: """Execute the main routine.""" # pylint: disable=too-many-locals parser = argparse.ArgumentParser(description=pypackagery_meta.__description__) parser.add_argument("--root_dir", help="Root directory of the Python files in the monorepo", required=True) parser.add_argument( "--initial_set", help="Paths to the files for which we want to compute the dependencies.\n\n" "If you specify a directory, all *.py files beneath (including subdirectories) are considered as part of " "the initial set.", nargs='+', required=True) parser.add_argument( "--format", help="The format of the output depedendency graph; default is the verbose, human-readable format", choices=packagery.FORMATS, default='verbose') parser.add_argument( "--dont_panic", help="If set, does not return an error code if some of the dependencies could not be resolved", action="store_true") parser.add_argument("--output_path", help="If set, outputs the result to a file instead of STDOUT") args = parser.parse_args() root_dir = pathlib.Path(args.root_dir).absolute() fmt = str(args.format) dont_panic = bool(args.dont_panic) output_path = None if not args.output_path else pathlib.Path(args.output_path) assert isinstance(args.initial_set, list) assert all(isinstance(item, str) for item in args.initial_set) initial_set = args.initial_set # type: List[str] initial_pths = [pathlib.Path(pth).absolute() for pth in initial_set] for pth in initial_pths: if root_dir != pth and root_dir not in pth.parents: raise ValueError(("Expected all the files of the initial set to reside beneath root directory {}, " "but at least one does not: {}").format(root_dir, pth)) initial_files = packagery.resolve_initial_paths(initial_paths=initial_pths) # yapf: disable rel_pths = [pth.relative_to(root_dir) for pth in initial_files] # yapf: enable requirements_txt = root_dir / "requirements.txt" if not requirements_txt.exists(): raise FileNotFoundError(("requirements.txt expected beneath the root directory {}, " "but could not be found: {})".format(root_dir, requirements_txt))) module_to_requirement_tsv = root_dir / "module_to_requirement.tsv" if not module_to_requirement_tsv.exists(): raise FileNotFoundError(("module_to_requirement.tsv expected beneath the root directory {}, " "but could not be found: {})".format(root_dir, module_to_requirement_tsv))) requirements = packagery.parse_requirements(text=requirements_txt.read_text(), filename=str(requirements_txt)) module_to_requirement = packagery.parse_module_to_requirement( text=module_to_requirement_tsv.read_text(), filename=str(module_to_requirement_tsv)) missing_reqs = packagery.missing_requirements( module_to_requirement=module_to_requirement, requirements=requirements) if len(missing_reqs) > 0: raise RuntimeError( ("The requirements listed in moudle_to_requirement mapping are missing in requirements file:\n" "module-to-requirement file: {}\n" "requirements file: {}\n" "Missing requirements:\n{}").format(module_to_requirement_tsv, requirements_txt, "\n".join(missing_reqs))) pkg = packagery.collect_dependency_graph( root_dir=root_dir, rel_paths=rel_pths, requirements=requirements, module_to_requirement=module_to_requirement) if output_path is None: packagery.output(package=pkg, a_format=fmt) else: with output_path.open('w') as fid: packagery.output(package=pkg, out=fid, a_format=fmt) if not dont_panic and len(pkg.unresolved_modules) > 0: return 1 return 0
def test_with_comment(self): text = "# some comment\n # some comment\n" reqs = packagery.parse_requirements(text=text) self.assertEqual(0, len(reqs))
def test_empty_requirements(self): text = "" reqs = packagery.parse_requirements(text=text) self.assertEqual(0, len(reqs))