def test_repo_with_extra():
dists = DistributionCollection()
root = DistInfo("root",
"1.0",
pkg_resources.parse_requirements(["a[test]"]),
meta=True)
metadata_a = DistInfo(
"a", "1.0.0",
pkg_resources.parse_requirements(['b ; extra=="test"', "c"]))
metadata_b = DistInfo("b", "2.0.0", [])
metadata_c = DistInfo("c", "2.0.0", [])
root = next(iter(dists.add_dist(root, None, None)))
root_a = next(
iter(
dists.add_dist(metadata_a, None,
pkg_resources.Requirement.parse("a[test]"))))
dists.add_dist(metadata_b, root_a,
pkg_resources.Requirement.parse('b ; extra=="test"'))
dists.add_dist(metadata_c, root_a, pkg_resources.Requirement.parse("a"))
lines = dists.generate_lines({root})
assert sorted(lines) == [
(("a[test]", "1.0.0"), "root"),
(("b", "2.0.0"), "a[test]"),
(("c", "2.0.0"), "a"),
]
def _compile(scenario, reqs, constraint_reqs=None, limit_reqs=None):
mock_pypi.load_scenario(scenario, limit_reqs=limit_reqs)
if constraint_reqs is not None:
constraint_reqs = [
DistInfo(
"test_constraints",
None,
[
pkg_resources.Requirement.parse(req)
for req in constraint_reqs
],
)
]
if isinstance(reqs, list):
reqs = {"test_reqs": reqs}
input_reqs = [
DistInfo(
key,
None,
[pkg_resources.Requirement.parse(req) for req in value],
meta=True,
) for key, value in reqs.items()
]
return _real_outputs(
req_compile.compile.perform_compile(
input_reqs, mock_pypi, constraint_reqs=constraint_reqs))
def test_metadata_transitive_violated():
dists = DistributionCollection()
metadata_a = DistInfo("a", "1.0.0", [])
metadata_b = DistInfo("b",
"1.0.0",
reqs=pkg_resources.parse_requirements(["a>1.0"]))
dists.add_dist(metadata_a, None, None)
dists.add_dist(metadata_b, None, None)
assert dists.nodes["a"].metadata is None
def setup_build_constraints():
global dists
dists = DistributionCollection()
info1 = DistInfo('a', pkg_resources.parse_version('1.0.0'),
pkg_resources.parse_requirements(['b<1', 'c==2.0.0']))
dists.add_dist(info1, None, None)
info2 = DistInfo('x', pkg_resources.parse_version('1.0.0'),
pkg_resources.parse_requirements(['b>0.5']))
dists.add_dist(info2, None, None)
info3 = DistInfo('y', pkg_resources.parse_version('1.0.0'),
pkg_resources.parse_requirements(['b<2']))
dists.add_dist(info3, None, None)
def _create_input_reqs(input_arg, extra_sources):
input_arg = input_arg.strip()
if input_arg == "-":
stdin_contents = sys.stdin.readlines()
def _create_stdin_input_req(line):
try:
result = _create_req_from_path(line)
extra_sources.append(line)
return utils.parse_requirement(
"{}=={}".format(*result.to_definition(None))
)
except ValueError:
return utils.parse_requirement(line)
reqs = (
_create_stdin_input_req(line.strip())
for line in stdin_contents
if line.strip()
)
reqs = (req for req in reqs if req is not None)
return DistInfo("-", None, reqs, meta=True)
if os.path.isfile(input_arg):
return RequirementsFile.from_file(input_arg)
return _create_req_from_path(input_arg)
def test_two_sources_same():
dists = DistributionCollection()
dists.add_dist(
DistInfo("aaa", "1.2.0",
pkg_resources.parse_requirements(["bbb<1.0"])),
None,
Requirement.parse("aaa"),
)
dists.add_dist(
DistInfo("ccc", "1.0.0",
pkg_resources.parse_requirements(["bbb<1.0"])),
None,
Requirement.parse("ccc"),
)
assert dists["bbb"].build_constraints() == pkg_resources.Requirement.parse(
"bbb<1.0")
def test_regular_and_extra_constraints():
dists = DistributionCollection()
root = DistInfo("root",
"1.0",
pkg_resources.parse_requirements(["a[test]"]),
meta=True)
metadata_a = DistInfo(
"a", "1.0.0",
pkg_resources.parse_requirements(['b>3 ; extra=="test"', "b>2"]))
dists.add_dist(root, None, None)
dists.add_dist(metadata_a, None,
pkg_resources.Requirement.parse("a[test]"))
assert dists["b"].build_constraints() == pkg_resources.Requirement.parse(
"b>2,>3")
def test_unnormalized_dist_with_extra():
dists = DistributionCollection()
metadata = DistInfo("A", "1.0.0", [])
dists.add_dist(metadata, None, Requirement.parse("A[x]"))
assert dists["A"].metadata.version == "1.0.0"
assert dists["A[x]"].metadata.version == "1.0.0"
def test_source_dirs_dont_hit_pypi(mocker, basic_compile_mock):
mocker.patch("os.path.exists")
metadata_mock = mocker.patch("req_compile.metadata.extract_metadata")
metadata_mock.return_value = DistInfo("myproj", "1.0", ["unknown_req"])
compile_main(["source/myproj"])
perform_compile_args = basic_compile_mock.mock_calls[0][1]
assert perform_compile_args[0][0].name == "myproj"
def test_metadata_violated():
dists = DistributionCollection()
metadata_a = DistInfo("a", "1.0.0", [])
dists.add_dist(metadata_a, None, None)
dists.add_dist(metadata_a, None, Requirement.parse("a>1.0"))
assert dists.nodes["a"].metadata is None
def test_compile_source_user1(local_tree):
results = req_compile.compile.perform_compile(
[
DistInfo("test",
None, [pkg_resources.Requirement.parse("user1")],
meta=True)
],
local_tree,
)
assert _real_outputs(results) == {"framework==1.0.1", "user1==2.0.0"}
def test_add_remove_dist():
dists = DistributionCollection()
nodes = dists.add_dist(
DistInfo("aaa", "1.2.0",
pkg_resources.parse_requirements(["bbb<1.0"])),
None,
Requirement.parse("aaa"),
)
assert len(nodes) == 1
dists.remove_dists(nodes)
assert "bbb" not in dists
def test_metadata_violated_removes_transitive():
dists = DistributionCollection()
metadata_a = DistInfo("a",
"1.0.0",
reqs=pkg_resources.parse_requirements(["b"]))
dists.add_dist(metadata_a, None, None)
dists.add_dist(metadata_a, None, Requirement.parse("a>1.0"))
assert dists["a"].metadata is None
assert "b" not in dists
def test_dist_with_unselected_extra():
dists = DistributionCollection()
dists.add_dist(
DistInfo(
"aaa",
"1.2.0",
reqs=pkg_resources.parse_requirements(['bbb<1.0 ; extra=="x1"']),
),
None,
None,
)
assert str(dists.nodes["aaa"].metadata) == "aaa==1.2.0"
def _parse_flat_metadata(contents):
name = None
version = None
raw_reqs = []
for line in contents.split("\n"):
lower_line = line.lower()
if name is None and lower_line.startswith("name:"):
name = line.split(":")[1].strip()
elif version is None and lower_line.startswith("version:"):
version = utils.parse_version(line.split(":")[1].strip())
elif lower_line.startswith("requires-dist:"):
raw_reqs.append(line.partition(":")[2].strip())
return DistInfo(name, version, list(utils.parse_requirements(raw_reqs)))
def test_compile_source_user2_recursive_root():
base_dir = os.path.join(os.path.dirname(__file__), "local-tree")
repo = SourceRepository(base_dir)
results = req_compile.compile.perform_compile(
[
DistInfo("test",
None, [pkg_resources.Requirement.parse("user-2")],
meta=True)
],
repo,
)
assert _real_outputs(results) == {
"framework==1.0.1",
"user-2==1.1.0",
"util==8.0.0",
}
def test_round_trip(scenario, roots, mock_metadata, mock_pypi):
mock_pypi.load_scenario("normal")
results, nodes = req_compile.compile.perform_compile(
[
DistInfo("test",
None,
pkg_resources.parse_requirements(roots),
meta=True)
],
mock_pypi,
)
fd, name = tempfile.mkstemp()
for line in results.generate_lines(nodes):
os.write(
fd, "{}=={} # {}\n".format(line[0][0], line[0][1],
line[1]).encode("utf-8"))
os.close(fd)
solution_result = load_from_file(name)
for node in results:
if isinstance(node.metadata, DistInfo) and node.key != "test":
assert node.key in solution_result
def resolve_candidate(self, candidate):
return DistInfo(candidate.name, candidate.version, []), False
def run_add_source():
dists = DistributionCollection()
info = DistInfo('a', pkg_resources.parse_version('1.0.0'),
pkg_resources.parse_requirements(['b<1', 'c==2.0.0']))
dists.add_dist(info, None, None)
def compile_main(args=None):
parser = argparse.ArgumentParser(
description="Req-Compile: Python requirements compiler"
)
group = parser.add_argument_group("requirement compilation")
group.add_argument(
"requirement_files",
nargs="*",
metavar="requirements_file",
help="Input requirements file or project directory to compile. Pass - to compile"
"from stdin",
)
group.add_argument(
"-c",
"--constraints",
action="append",
metavar="constraints_file",
help="Constraints file or project directory to use as constraints. ",
)
group.add_argument(
"-e",
"--extra",
action="append",
dest="extras",
default=[],
metavar="extra",
help="Extras to apply automatically to source packages",
)
group.add_argument(
"-P",
"--upgrade-package",
action="append",
dest="upgrade_packages",
metavar="package_name",
help="Package to omit from solutions. Use this to upgrade packages",
)
group.add_argument(
"--remove-source",
default=False,
action="store_true",
help="Remove distributions satisfied via --source from the output",
)
group.add_argument(
"--remove-non-source",
default=False,
action="store_true",
help="Remove distributions not satisfied via --source from the output",
)
group.add_argument(
"-p",
"--pre",
dest="allow_prerelease",
default=False,
action="store_true",
help="Allow prereleases from all sources",
)
group.add_argument(
"--annotate",
default=False,
action="store_true",
help="Annotate the output file with the sources of each requirement",
)
group.add_argument(
"--no-comments",
default=False,
action="store_true",
help="Disable comments in the output",
)
group.add_argument(
"--no-pins",
default=False,
action="store_true",
help="Disable version pins, just list distributions",
)
add_logging_args(parser)
add_repo_args(parser)
args = parser.parse_args(args=args)
logger = logging.getLogger("req_compile")
if args.verbose:
logging.basicConfig(level=logging.DEBUG, stream=sys.stderr)
logger.setLevel(logging.DEBUG)
logger.getChild("compile").addFilter(IndentFilter())
else:
logging.basicConfig(level=logging.CRITICAL, stream=sys.stderr)
wheeldir = args.wheel_dir
if wheeldir:
try:
if not os.path.exists(wheeldir):
os.mkdir(wheeldir)
except OSError:
pass
delete_wheeldir = False
else:
wheeldir = tempfile.mkdtemp()
delete_wheeldir = True
input_args = args.requirement_files
if not input_args:
# Check to see whether stdin is hooked up to piped data or the console
if not sys.stdin.isatty():
input_args = ("-",)
else:
input_args = (".",)
extra_sources = []
input_reqs = [
_create_input_reqs(input_arg, extra_sources) for input_arg in input_args
]
constraint_reqs = []
if args.constraints is not None:
constraint_reqs = [
_create_input_reqs(input_arg, extra_sources)
for input_arg in args.constraints
]
if args.extras:
for req in input_reqs:
try:
extra_req = pkg_resources.Requirement.parse(
req.name + "[{}]".format(",".join(args.extras))
)
except pkg_resources.RequirementParseError:
continue
extra_constraint = DistInfo(
"{}-extra".format(req.name), None, [extra_req], meta=True
)
constraint_reqs.append(extra_constraint)
try:
repo = build_repo(
args.solutions,
args.upgrade_packages,
extra_sources + args.sources,
args.excluded_sources,
args.find_links,
args.index_urls,
args.no_index,
wheeldir,
allow_prerelease=args.allow_prerelease,
)
results, roots = perform_compile(
input_reqs, repo, extras=args.extras, constraint_reqs=constraint_reqs
)
except RepositoryInitializationError as ex:
logger.exception("Error initialization repository")
print("Error initializing {}: {}".format(ex.type.__name__, ex), file=sys.stderr)
sys.exit(1)
except (
req_compile.errors.NoCandidateException,
req_compile.errors.MetadataError,
) as ex:
_generate_no_candidate_display(ex.req, repo, ex.results, ex)
sys.exit(1)
finally:
if delete_wheeldir:
shutil.rmtree(wheeldir)
write_requirements_file(
results,
roots,
annotate_source=args.annotate,
input_reqs=input_reqs,
repo=repo,
remove_non_source=args.remove_non_source,
remove_source=args.remove_source,
no_pins=args.no_pins,
no_comments=args.no_comments,
)