def test_invalid(self):
text = "//wrong"
value_error = None # type: Optional[ValueError]
try:
packagery.parse_requirements(
text=text, filename="/some/path/requirements.txt")
except ValueError as err:
value_error = err
self.assertIsNotNone(value_error)
self.assertEqual(
"Invalid requirements in file /some/path/requirements.txt",
str(value_error))
def test_transitivity(self):
with temppathlib.TemporaryDirectory() as tmp:
script_pth = tmp.path / "some_script.py"
script_pth.write_text("#!/usr/bin/env python\n\n"
"import something_local\n")
module_pth = tmp.path / "something_local.py"
module_pth.write_text("#!/usr/bin/env python\n"
"import unittest\n"
"import PIL.Image\n"
"import something_else_local")
another_module_pth = tmp.path / "something_else_local.py"
another_module_pth.write_text("#!/usr/bin/env python\n")
requirements = packagery.parse_requirements(text="pillow==5.2.0\n")
module_to_requirement = packagery.parse_module_to_requirement(
text="PIL.Image\tpillow")
pkg = packagery.collect_dependency_graph(
root_dir=tmp.path,
rel_paths=[pathlib.Path("some_script.py")],
requirements=requirements,
module_to_requirement=module_to_requirement)
self.assertListEqual([], pkg.unresolved_modules)
self.assertListEqual(["pillow"], list(pkg.requirements.keys()))
self.assertSetEqual(
{
pathlib.Path("some_script.py"),
pathlib.Path("something_local.py"),
pathlib.Path("something_else_local.py")
}, pkg.rel_paths)
def test_builtin_local_pip_dependencies(self): # pylint: disable=invalid-name
with temppathlib.TemporaryDirectory() as tmp:
script_pth = tmp.path / "some_script.py"
script_pth.write_text("#!/usr/bin/env python\n\n"
"import sys\n\n"
"import PIL.Image\n\n"
"import something_local\n")
module_pth = tmp.path / "something_local.py"
module_pth.write_text("#!/usr/bin/env python\n")
requirements = packagery.parse_requirements(text="pillow==5.2.0\n")
module_to_requirement = packagery.parse_module_to_requirement(
text="PIL.Image\tpillow")
pkg = packagery.collect_dependency_graph(
root_dir=tmp.path,
rel_paths=[pathlib.Path("some_script.py")],
requirements=requirements,
module_to_requirement=module_to_requirement)
self.assertListEqual([], pkg.unresolved_modules)
self.assertListEqual(["pillow"], list(pkg.requirements.keys()))
self.assertSetEqual(
{
pathlib.Path("some_script.py"),
pathlib.Path("something_local.py")
}, pkg.rel_paths)
def test_without_name(self):
url = "http://download.pytorch.org/whl/cpu/torch-0.3.1-cp35-cp35m-linux_x86_64.whl" # no egg fragment
text = "{}\n".format(url)
value_error = None # type: Optional[ValueError]
try:
packagery.parse_requirements(text=text,
filename="/some/requirements.txt")
except ValueError as err:
value_error = err
self.assertIsNotNone(value_error)
self.assertEqual(
"The name is missing in the requirement from the file /some/requirements.txt: "
"http://download.pytorch.org/whl/cpu/torch-0.3.1-cp35-cp35m-linux_x86_64.whl "
"(did you specify the egg fragment?)", str(value_error))
def test_without_version(self):
text = "some_pack-age\n"
reqs = packagery.parse_requirements(text=text)
self.assertEqual(1, len(reqs))
self.assertIn('some_pack-age', reqs)
requirement = reqs["some_pack-age"]
self.assertEqual("some_pack-age", requirement.name)
self.assertEqual("some_pack-age\n", requirement.line)
def test_with_whitespace(self):
text = " some_package "
reqs = packagery.parse_requirements(text=text)
self.assertEqual(1, len(reqs))
self.assertIn('some_package', reqs)
requirement = reqs['some_package']
self.assertEqual('some_package', requirement.name)
self.assertEqual("some_package\n", requirement.line)
def test_with_comment_on_the_same_line(self): # pylint: disable=invalid-name
text = "some_package # some comment"
reqs = packagery.parse_requirements(text=text)
self.assertEqual(1, len(reqs))
self.assertIn('some_package', reqs)
requirement = reqs['some_package']
self.assertEqual('some_package', requirement.name)
self.assertEqual("some_package # some comment\n", requirement.line)
def test_with_pinned_version(self):
text = "some_package12 == 1.2.3"
reqs = packagery.parse_requirements(text=text)
self.assertEqual(1, len(reqs))
self.assertIn('some_package12', reqs)
requirement = reqs['some_package12']
self.assertEqual('some_package12', requirement.name)
self.assertEqual("some_package12 == 1.2.3\n", requirement.line)
def test_that_it_works(self):
# yapf: disable
module_to_requirement = collections.OrderedDict([("some_module", "some_missing_package"),
("another_module", "some_package")])
# yapf: enable
requirements = packagery.parse_requirements(
"some_package\nsome_unused_package")
missing = packagery.missing_requirements(
module_to_requirement=module_to_requirement,
requirements=requirements)
self.assertListEqual(['some_missing_package'], missing)
def test_url(self):
url = "http://download.pytorch.org/whl/cpu/torch-0.3.1-cp35-cp35m-linux_x86_64.whl#egg=torch"
text = "{}\n".format(url)
reqs = packagery.parse_requirements(text=text)
self.assertEqual(1, len(reqs))
self.assertIn("torch", reqs)
requirement = reqs["torch"]
self.assertEqual("torch", requirement.name)
self.assertEqual(
"http://download.pytorch.org/whl/cpu/torch-0.3.1-cp35-cp35m-linux_x86_64.whl#egg=torch\n",
requirement.line)
def test_multiple(self):
text = "some_package\nanother_package==1.2"
reqs = packagery.parse_requirements(text=text)
self.assertEqual(2, len(reqs))
self.assertIn('some_package', reqs)
self.assertIn('another_package', reqs)
requirement1 = reqs['some_package']
self.assertEqual('some_package', requirement1.name)
self.assertEqual("some_package\n", requirement1.line)
requirement2 = reqs['another_package']
self.assertEqual('another_package', requirement2.name)
self.assertEqual("another_package==1.2\n", requirement2.line)
def generate_package_with_local_pip_and_missing_deps() -> packagery.Package: # pylint: disable=invalid-name
with temppathlib.TemporaryDirectory() as tmp:
script_pth = tmp.path / "some_script.py"
script_pth.write_text("#!/usr/bin/env python\n\n"
"import PIL.Image\n\n"
"import something_local\n"
"import something_missing\n")
module_pth = tmp.path / "something_local.py"
module_pth.write_text("#!/usr/bin/env python\n")
requirements = packagery.parse_requirements(text="pillow==5.2.0\n")
module_to_requirement = packagery.parse_module_to_requirement(
text="PIL.Image\tpillow")
pkg = packagery.collect_dependency_graph(
root_dir=tmp.path,
rel_paths=[pathlib.Path("some_script.py")],
requirements=requirements,
module_to_requirement=module_to_requirement)
return pkg
def main() -> int:
"""Execute the main routine."""
# pylint: disable=too-many-locals
parser = argparse.ArgumentParser(description=pypackagery_meta.__description__)
parser.add_argument("--root_dir", help="Root directory of the Python files in the monorepo", required=True)
parser.add_argument(
"--initial_set",
help="Paths to the files for which we want to compute the dependencies.\n\n"
"If you specify a directory, all *.py files beneath (including subdirectories) are considered as part of "
"the initial set.",
nargs='+',
required=True)
parser.add_argument(
"--format",
help="The format of the output depedendency graph; default is the verbose, human-readable format",
choices=packagery.FORMATS,
default='verbose')
parser.add_argument(
"--dont_panic",
help="If set, does not return an error code if some of the dependencies could not be resolved",
action="store_true")
parser.add_argument("--output_path", help="If set, outputs the result to a file instead of STDOUT")
args = parser.parse_args()
root_dir = pathlib.Path(args.root_dir).absolute()
fmt = str(args.format)
dont_panic = bool(args.dont_panic)
output_path = None if not args.output_path else pathlib.Path(args.output_path)
assert isinstance(args.initial_set, list)
assert all(isinstance(item, str) for item in args.initial_set)
initial_set = args.initial_set # type: List[str]
initial_pths = [pathlib.Path(pth).absolute() for pth in initial_set]
for pth in initial_pths:
if root_dir != pth and root_dir not in pth.parents:
raise ValueError(("Expected all the files of the initial set to reside beneath root directory {}, "
"but at least one does not: {}").format(root_dir, pth))
initial_files = packagery.resolve_initial_paths(initial_paths=initial_pths)
# yapf: disable
rel_pths = [pth.relative_to(root_dir) for pth in initial_files]
# yapf: enable
requirements_txt = root_dir / "requirements.txt"
if not requirements_txt.exists():
raise FileNotFoundError(("requirements.txt expected beneath the root directory {}, "
"but could not be found: {})".format(root_dir, requirements_txt)))
module_to_requirement_tsv = root_dir / "module_to_requirement.tsv"
if not module_to_requirement_tsv.exists():
raise FileNotFoundError(("module_to_requirement.tsv expected beneath the root directory {}, "
"but could not be found: {})".format(root_dir, module_to_requirement_tsv)))
requirements = packagery.parse_requirements(text=requirements_txt.read_text(), filename=str(requirements_txt))
module_to_requirement = packagery.parse_module_to_requirement(
text=module_to_requirement_tsv.read_text(), filename=str(module_to_requirement_tsv))
missing_reqs = packagery.missing_requirements(
module_to_requirement=module_to_requirement, requirements=requirements)
if len(missing_reqs) > 0:
raise RuntimeError(
("The requirements listed in moudle_to_requirement mapping are missing in requirements file:\n"
"module-to-requirement file: {}\n"
"requirements file: {}\n"
"Missing requirements:\n{}").format(module_to_requirement_tsv, requirements_txt, "\n".join(missing_reqs)))
pkg = packagery.collect_dependency_graph(
root_dir=root_dir, rel_paths=rel_pths, requirements=requirements, module_to_requirement=module_to_requirement)
if output_path is None:
packagery.output(package=pkg, a_format=fmt)
else:
with output_path.open('w') as fid:
packagery.output(package=pkg, out=fid, a_format=fmt)
if not dont_panic and len(pkg.unresolved_modules) > 0:
return 1
return 0
def test_with_comment(self):
text = "# some comment\n # some comment\n"
reqs = packagery.parse_requirements(text=text)
self.assertEqual(0, len(reqs))
def test_empty_requirements(self):
text = ""
reqs = packagery.parse_requirements(text=text)
self.assertEqual(0, len(reqs))
