def test_multiple_versions(rule_runner: RuleRunner) -> None:
"""This tests that we correctly create a new python_requirement for each unique dependency name
in a requirements.txt file, grouping duplicated dependency names to handle multiple requirement
strings per PEP 508."""
file_addr = Address("", target_name="reqs", relative_file_path="requirements.txt")
assert_python_requirements(
rule_runner,
"python_requirements(name='reqs')",
dedent(
"""\
Django>=3.2
Django==3.2.7
confusedmonkey==86
repletewateringcan>=7
"""
),
expected_targets={
PythonRequirementTarget(
{
"requirements": ["Django>=3.2", "Django==3.2.7"],
"dependencies": [file_addr.spec],
},
Address("", target_name="reqs", generated_name="Django"),
),
PythonRequirementTarget(
{"requirements": ["confusedmonkey==86"], "dependencies": [file_addr.spec]},
Address("", target_name="reqs", generated_name="confusedmonkey"),
),
PythonRequirementTarget(
{"requirements": ["repletewateringcan>=7"], "dependencies": [file_addr.spec]},
Address("", target_name="reqs", generated_name="repletewateringcan"),
),
TargetGeneratorSourcesHelperTarget({"sources": ["requirements.txt"]}, file_addr),
},
)
def test_pipfile_lock(rule_runner: RuleRunner) -> None:
"""This tests that we correctly create a new `python_requirement` target for each entry in a
Pipfile.lock file.
Edge cases:
* Develop and Default requirements are used
* If a module_mapping is given, and the project is in the map, we set `modules`. It
works regardless of capitalization.
"""
assert_pipenv_requirements(
rule_runner,
"pipenv_requirements(module_mapping={'ANSIcolors': ['colors']})",
{
"default": {
"ansicolors": {
"version": ">=1.18.0"
}
},
"develop": {
"cachetools": {
"markers": "python_version ~= '3.5'",
"version": "==4.1.1"
}
},
},
expected_file_dep=PythonRequirementsFile(
{"sources": ["Pipfile.lock"]},
Address("", target_name="Pipfile.lock")),
expected_targets=[
PythonRequirementTarget(
{
"requirements":
[PipRequirement.parse("ansicolors>=1.18.0")],
"dependencies": [":Pipfile.lock"],
"modules": ["colors"],
},
Address("", target_name="ansicolors"),
),
PythonRequirementTarget(
{
"requirements": [
PipRequirement.parse(
"cachetools==4.1.1;python_version ~= '3.5'")
],
"dependencies": [":Pipfile.lock"],
},
Address("", target_name="cachetools"),
),
],
)
def test_pipfile_lock(rule_runner: RuleRunner) -> None:
"""This tests that we correctly create a new python_requirement_library for each entry in a
Pipfile.lock file.
Edge cases:
* Develop and Default requirements are used
* module_mapping works.
"""
file_addr = Address("",
target_name="reqs",
relative_file_path="Pipfile.lock")
assert_pipenv_requirements(
rule_runner,
"pipenv_requirements(name='reqs', module_mapping={'ansicolors': ['colors']})",
{
"default": {
"ansicolors": {
"version": ">=1.18.0"
}
},
"develop": {
"cachetools": {
"markers": "python_version ~= '3.5'",
"version": "==4.1.1",
"extras": ["ring", "mongo"],
}
},
},
expected_targets={
PythonRequirementTarget(
{
"requirements": ["ansicolors>=1.18.0"],
"modules": ["colors"],
"dependencies": [file_addr.spec],
},
Address("", target_name="reqs", generated_name="ansicolors"),
),
PythonRequirementTarget(
{
"requirements":
["cachetools[ring, mongo]==4.1.1;python_version ~= '3.5'"],
"dependencies": [file_addr.spec],
},
Address("", target_name="reqs", generated_name="cachetools"),
),
TargetGeneratorSourcesHelperTarget({"source": "Pipfile.lock"},
file_addr),
},
)
def test_supply_python_requirements_file(rule_runner: RuleRunner) -> None:
"""This tests that we can supply our own `_python_requirements_file`."""
assert_pipenv_requirements(
rule_runner,
dedent(
"""
pipenv_requirements(
source='custom/pipfile/Pipfile.lock',
pipfile_target='//:custom_pipfile_target'
)
_generator_sources_helper(
name='custom_pipfile_target',
sources=['custom/pipfile/Pipfile.lock']
)
"""
),
{"default": {"ansicolors": {"version": ">=1.18.0"}}},
expected_file_dep=TargetGeneratorSourcesHelperTarget(
{"sources": ["custom/pipfile/Pipfile.lock"]},
Address("", target_name="custom_pipfile_target"),
),
expected_targets=[
PythonRequirementTarget(
{
"requirements": [PipRequirement.parse("ansicolors>=1.18.0")],
"dependencies": ["//:custom_pipfile_target"],
},
Address("", target_name="ansicolors"),
),
],
pipfile_lock_relpath="custom/pipfile/Pipfile.lock",
)
def generate_tgt(raw_req: str, info: dict) -> PythonRequirementTarget:
if info.get("extras"):
raw_req += f"[{','.join(info['extras'])}]"
raw_req += info.get("version", "")
if info.get("markers"):
raw_req += f";{info['markers']}"
parsed_req = PipRequirement.parse(raw_req)
normalized_proj_name = canonicalize_project_name(
parsed_req.project_name)
tgt_overrides = overrides.pop(normalized_proj_name, {})
if Dependencies.alias in tgt_overrides:
tgt_overrides[Dependencies.alias] = list(
tgt_overrides[Dependencies.alias]) + [file_tgt.address.spec]
return PythonRequirementTarget(
{
**inherited_fields,
PythonRequirementsField.alias: [parsed_req],
PythonRequirementModulesField.alias:
module_mapping.get(normalized_proj_name),
PythonRequirementTypeStubModulesField.alias:
stubs_mapping.get(normalized_proj_name),
# This may get overridden by `tgt_overrides`, which will have already added in
# the file tgt.
Dependencies.alias: [file_tgt.address.spec],
**tgt_overrides,
},
generator.address.create_generated(parsed_req.project_name),
)
def test_no_compatible_resolve_error() -> None:
python_setup = create_subsystem(PythonSetup, resolves={"a": "", "b": ""}, enable_resolves=True)
targets = [
PythonRequirementTarget(
{PythonRequirementsField.alias: [], PythonResolveField.alias: "a"},
Address("", target_name="t1"),
),
PythonSourceTarget(
{PythonSourceField.alias: "f.py", PythonResolveField.alias: "a"},
Address("", target_name="t2"),
),
PythonSourceTarget(
{PythonSourceField.alias: "f.py", PythonResolveField.alias: "b"},
Address("", target_name="t3"),
),
]
assert str(NoCompatibleResolveException(python_setup, "Prefix", targets)).startswith(
dedent(
"""\
Prefix:
a:
* //:t1
* //:t2
b:
* //:t3
"""
)
)
def generate_tgt(
project_name: str, parsed_reqs: Iterable[PipRequirement]
) -> PythonRequirementTarget:
normalized_proj_name = canonicalize_project_name(project_name)
tgt_overrides = overrides.pop(normalized_proj_name, {})
if Dependencies.alias in tgt_overrides:
tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [
file_tgt.address.spec
]
return PythonRequirementTarget(
{
**inherited_fields,
PythonRequirementsField.alias: list(parsed_reqs),
PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name),
PythonRequirementTypeStubModulesField.alias: stubs_mapping.get(
normalized_proj_name
),
# This may get overridden by `tgt_overrides`, which will have already added in
# the file tgt.
Dependencies.alias: [file_tgt.address.spec],
**tgt_overrides,
},
generator.address.create_generated(project_name),
)
def test_source_override(rule_runner: RuleRunner) -> None:
assert_poetry_requirements(
rule_runner,
"poetry_requirements(source='subdir/pyproject.toml')",
dedent("""\
[tool.poetry.dependencies]
ansicolors = ">=1.18.0"
[tool.poetry.dev-dependencies]
"""),
pyproject_toml_relpath="subdir/pyproject.toml",
expected_file_dep=TargetGeneratorSourcesHelperTarget(
{"sources": ["subdir/pyproject.toml"]},
address=Address("", target_name="subdir_pyproject.toml"),
),
expected_targets=[
PythonRequirementTarget(
{
"dependencies": [":subdir_pyproject.toml"],
"requirements":
[PipRequirement.parse("ansicolors>=1.18.0")],
},
address=Address("", target_name="ansicolors"),
),
],
)
def test_source_override(rule_runner: RuleRunner) -> None:
file_addr = Address("",
target_name="reqs",
relative_file_path="subdir/pyproject.toml")
assert_poetry_requirements(
rule_runner,
"poetry_requirements(name='reqs', source='subdir/pyproject.toml')",
dedent("""\
[tool.poetry.dependencies]
ansicolors = ">=1.18.0"
[tool.poetry.dev-dependencies]
"""),
pyproject_toml_relpath="subdir/pyproject.toml",
expected_targets={
PythonRequirementTarget(
{
"dependencies": [file_addr.spec],
"requirements": ["ansicolors>=1.18.0"]
},
address=Address("",
target_name="reqs",
generated_name="ansicolors"),
),
TargetGeneratorSourcesHelperTarget(
{"sources": ["subdir/pyproject.toml"]}, file_addr),
},
)
def test_multiple_versions(rule_runner: RuleRunner) -> None:
"""This tests that we correctly create a new python_requirement for each unique dependency name
in a requirements.txt file, grouping duplicated dependency names to handle multiple requirement
strings per PEP 508."""
assert_python_requirements(
rule_runner,
"python_requirements()",
dedent("""\
Django>=3.2
Django==3.2.7
confusedmonkey==86
repletewateringcan>=7
"""),
expected_file_dep=TargetGeneratorSourcesHelperTarget(
{"sources": ["requirements.txt"]},
Address("", target_name="requirements.txt"),
),
expected_targets=[
PythonRequirementTarget(
{
"dependencies": [":requirements.txt"],
"requirements": [
PipRequirement.parse("Django>=3.2"),
PipRequirement.parse("Django==3.2.7"),
],
},
Address("", target_name="Django"),
),
PythonRequirementTarget(
{
"dependencies": [":requirements.txt"],
"requirements":
[PipRequirement.parse("confusedmonkey==86")],
},
Address("", target_name="confusedmonkey"),
),
PythonRequirementTarget(
{
"dependencies": [":requirements.txt"],
"requirements":
[PipRequirement.parse("repletewateringcan>=7")],
},
Address("", target_name="repletewateringcan"),
),
],
)
def test_properly_creates_extras_requirements(rule_runner: RuleRunner) -> None:
"""This tests the proper parsing of requirements installed with specified extras."""
assert_pipenv_requirements(
rule_runner,
"pipenv_requirements()",
{
"default": {
"ansicolors": {
"version": ">=1.18.0",
"extras": ["neon"]
}
},
"develop": {
"cachetools": {
"markers": "python_version ~= '3.5'",
"version": "==4.1.1",
"extras": ["ring", "mongo"],
}
},
},
expected_file_dep=PythonRequirementsFile(
{"sources": ["Pipfile.lock"]},
Address("", target_name="Pipfile.lock")),
expected_targets=[
PythonRequirementTarget(
{
"requirements":
[PipRequirement.parse("ansicolors[neon]>=1.18.0")],
"dependencies": [":Pipfile.lock"],
},
Address("", target_name="ansicolors"),
),
PythonRequirementTarget(
{
"requirements": [
PipRequirement.parse(
"cachetools[ring,mongo]==4.1.1;python_version ~= '3.5'"
)
],
"dependencies": [":Pipfile.lock"],
},
Address("", target_name="cachetools"),
),
],
)
def create_tgt(dist: str, module: str) -> PythonRequirementTarget:
return PythonRequirementTarget(
{
PythonRequirementsField.alias: (f"{dist}{version}", ),
PythonRequirementModulesField.alias: (module, ),
**request.template,
},
request.template_address.create_generated(dist),
union_membership,
)
def create_tgt(dist: str, module: str) -> PythonRequirementTarget:
return PythonRequirementTarget(
{
PythonRequirementsField.alias: (f"{dist}{version}", ),
PythonRequirementModulesField.alias: (module, ),
PythonRequirementResolveField.alias:
generator[PythonRequirementResolveField].value,
},
generator.address.create_generated(dist),
)
def test_source_override(rule_runner: RuleRunner) -> None:
file_addr = Address("", target_name="reqs", relative_file_path="subdir/requirements.txt")
assert_python_requirements(
rule_runner,
"python_requirements(name='reqs', source='subdir/requirements.txt')",
"ansicolors>=1.18.0",
requirements_txt_relpath="subdir/requirements.txt",
expected_targets={
PythonRequirementTarget(
{"requirements": ["ansicolors>=1.18.0"], "dependencies": [file_addr.spec]},
Address("", target_name="reqs", generated_name="ansicolors"),
),
TargetGeneratorSourcesHelperTarget({"sources": ["subdir/requirements.txt"]}, file_addr),
},
)
def test_source_override(rule_runner: RuleRunner) -> None:
assert_python_requirements(
rule_runner,
"python_requirements(source='subdir/requirements.txt')",
"ansicolors>=1.18.0",
requirements_txt_relpath="subdir/requirements.txt",
expected_file_dep=TargetGeneratorSourcesHelperTarget(
{"sources": ["subdir/requirements.txt"]},
Address("", target_name="subdir_requirements.txt"),
),
expected_targets=[
PythonRequirementTarget(
{
"dependencies": [":subdir_requirements.txt"],
"requirements":
[PipRequirement.parse("ansicolors>=1.18.0")],
},
Address("", target_name="ansicolors"),
),
],
)
def generate_tgt(parsed_req: PipRequirement) -> PythonRequirementTarget:
normalized_proj_name = canonicalize_project_name(parsed_req.project_name)
tgt_overrides = overrides.pop(normalized_proj_name, {})
if Dependencies.alias in tgt_overrides:
tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [
file_tgt.address.spec
]
return PythonRequirementTarget(
{
**request.template,
PythonRequirementsField.alias: [parsed_req],
PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name),
PythonRequirementTypeStubModulesField.alias: stubs_mapping.get(
normalized_proj_name
),
# This may get overridden by `tgt_overrides`, which will have already added in
# the file tgt.
Dependencies.alias: [file_tgt.address.spec],
**tgt_overrides,
},
request.template_address.create_generated(parsed_req.project_name),
union_membership,
)
def test_pyproject_toml(rule_runner: RuleRunner) -> None:
"""This tests that we correctly create a new python_requirement for each entry in a
pyproject.toml file.
Note that this just ensures proper targets are created; see prior tests for specific parsing
edge cases.
"""
assert_poetry_requirements(
rule_runner,
dedent("""\
poetry_requirements(
# module_mapping should work regardless of capitalization.
module_mapping={'ansiCOLORS': ['colors']},
type_stubs_module_mapping={'Django-types': ['django']},
)
"""),
dedent("""\
[tool.poetry.dependencies]
Django = {version = "3.2", python = "3"}
Django-types = "2"
Un-Normalized-PROJECT = "1.0.0"
[tool.poetry.dev-dependencies]
ansicolors = ">=1.18.0"
"""),
expected_file_dep=TargetGeneratorSourcesHelperTarget(
{"sources": ["pyproject.toml"]},
address=Address("", target_name="pyproject.toml"),
),
expected_targets=[
PythonRequirementTarget(
{
"dependencies": [":pyproject.toml"],
"requirements":
[PipRequirement.parse("ansicolors>=1.18.0")],
"modules": ["colors"],
},
address=Address("", target_name="ansicolors"),
),
PythonRequirementTarget(
{
"dependencies": [":pyproject.toml"],
"requirements": [
PipRequirement.parse(
"Django==3.2 ; python_version == '3'")
],
},
address=Address("", target_name="Django"),
),
PythonRequirementTarget(
{
"dependencies": [":pyproject.toml"],
"requirements": [PipRequirement.parse("Django-types==2")],
"type_stub_modules": ["django"],
},
address=Address("", target_name="Django-types"),
),
PythonRequirementTarget(
{
"dependencies": [":pyproject.toml"],
"requirements":
[PipRequirement.parse("Un_Normalized_PROJECT == 1.0.0")],
},
address=Address("", target_name="Un-Normalized-PROJECT"),
),
],
)
def test_requirements_txt(rule_runner: RuleRunner) -> None:
"""This tests that we correctly create a new python_requirement for each entry in a
requirements.txt file, where each dependency is unique.
Some edge cases:
* We ignore comments and options (values that start with `--`).
* module_mapping works regardless of capitalization.
* Projects get normalized thanks to Requirement.parse().
* Overrides works, including for dependencies.
"""
file_addr = Address("",
target_name="reqs",
relative_file_path="requirements.txt")
assert_python_requirements(
rule_runner,
dedent("""\
python_requirements(
name='reqs',
module_mapping={'ansiCOLORS': ['colors']},
type_stubs_module_mapping={'Django-types': ['django']},
overrides={
"ansicolors": {"tags": ["overridden"]},
"Django": {"dependencies": ["#Django-types"]},
},
)
"""),
dedent("""\
# Comment.
--find-links=https://duckduckgo.com
ansicolors>=1.18.0
Django==3.2 ; python_version>'3'
Django-types
Un-Normalized-PROJECT # Inline comment.
pip@ git+https://github.com/pypa/pip.git
"""),
expected_targets={
PythonRequirementTarget(
{
"requirements": ["ansicolors>=1.18.0"],
"modules": ["colors"],
"dependencies": [file_addr.spec],
"tags": ["overridden"],
},
Address("", target_name="reqs", generated_name="ansicolors"),
),
PythonRequirementTarget(
{
"requirements": ["Django==3.2 ; python_version>'3'"],
"dependencies": ["#Django-types", file_addr.spec],
},
Address("", target_name="reqs", generated_name="Django"),
),
PythonRequirementTarget(
{
"requirements": ["Django-types"],
"type_stub_modules": ["django"],
"dependencies": [file_addr.spec],
},
Address("", target_name="reqs", generated_name="Django-types"),
),
PythonRequirementTarget(
{
"requirements": ["Un_Normalized_PROJECT"],
"dependencies": [file_addr.spec]
},
Address("",
target_name="reqs",
generated_name="Un-Normalized-PROJECT"),
),
PythonRequirementTarget(
{
"requirements":
["pip@ git+https://github.com/pypa/pip.git"],
"dependencies": [file_addr.spec],
},
Address("", target_name="reqs", generated_name="pip"),
),
TargetGeneratorSourcesHelperTarget({"source": "requirements.txt"},
file_addr),
},
)
def test_no_compatible_resolve_error() -> None:
python_setup = create_subsystem(PythonSetup,
resolves={
"a": "",
"b": ""
},
enable_resolves=True)
t1 = PythonRequirementTarget(
{
PythonRequirementsField.alias: [],
PythonResolveField.alias: "a",
Dependencies.alias: ["//:t3"],
},
Address("", target_name="t1"),
)
t2 = PythonSourceTarget(
{
PythonSourceField.alias: "f.py",
PythonResolveField.alias: "a",
Dependencies.alias: ["//:t3"],
},
Address("", target_name="t2"),
)
t3 = PythonSourceTarget(
{
PythonSourceField.alias: "f.py",
PythonResolveField.alias: "b"
},
Address("", target_name="t3"),
)
def maybe_get_resolve(t: Target) -> str | None:
if not t.has_field(PythonResolveField):
return None
return t[PythonResolveField].normalized_value(python_setup)
bad_roots_err = str(
NoCompatibleResolveException.bad_input_roots(
[t2, t3],
maybe_get_resolve=maybe_get_resolve,
doc_url_slug="",
workaround=None))
assert bad_roots_err.startswith(
softwrap("""
The input targets did not have a resolve in common.
a:
* //:t2
b:
* //:t3
Targets used together must use the same resolve, set by the `resolve` field.
"""))
bad_single_dep_err = str(
NoCompatibleResolveException.bad_dependencies(
maybe_get_resolve=maybe_get_resolve,
doc_url_slug="",
root_targets=[t1],
root_resolve="a",
dependencies=[t3],
))
assert bad_single_dep_err.startswith(
softwrap("""
The target //:t1 uses the `resolve` `a`, but some of its
dependencies are not compatible with that resolve:
* //:t3 (b)
All dependencies must work with the same `resolve`. To fix this, either change
the `resolve=` field on those dependencies to `a`, or change
the `resolve=` of the target //:t1.
"""))
bad_multiple_deps_err = str(
NoCompatibleResolveException.bad_dependencies(
maybe_get_resolve=maybe_get_resolve,
doc_url_slug="",
root_targets=[t1, t2],
root_resolve="a",
dependencies=[t3],
))
assert bad_multiple_deps_err.startswith(
softwrap("""
The input targets use the `resolve` `a`, but some of their
dependencies are not compatible with that resolve.
Input targets:
* //:t1
* //:t2
Bad dependencies:
* //:t3 (b)
All dependencies must work with the same `resolve`. To fix this, either change
the `resolve=` field on those dependencies to `a`, or change
the `resolve=` of the input targets.
"""))
def test_requirements_txt(rule_runner: RuleRunner) -> None:
"""This tests that we correctly create a new python_requirement for each entry in a
requirements.txt file, where each dependency is unique.
Some edge cases:
* We ignore comments and options (values that start with `--`).
* If a module_mapping is given, and the project is in the map, we copy over a subset of the
mapping to the created target. It works regardless of capitalization.
* Projects get normalized thanks to Requirement.parse().
"""
assert_python_requirements(
rule_runner,
dedent("""\
python_requirements(
module_mapping={'ansiCOLORS': ['colors']},
type_stubs_module_mapping={'Django-types': ['django']},
)
"""),
dedent("""\
# Comment.
--find-links=https://duckduckgo.com
ansicolors>=1.18.0
Django==3.2 ; python_version>'3'
Django-types
Un-Normalized-PROJECT # Inline comment.
pip@ git+https://github.com/pypa/pip.git
"""),
expected_file_dep=TargetGeneratorSourcesHelperTarget(
{"sources": ["requirements.txt"]},
Address("", target_name="requirements.txt"),
),
expected_targets=[
PythonRequirementTarget(
{
"dependencies": [":requirements.txt"],
"requirements":
[PipRequirement.parse("ansicolors>=1.18.0")],
"modules": ["colors"],
},
Address("", target_name="ansicolors"),
),
PythonRequirementTarget(
{
"dependencies": [":requirements.txt"],
"requirements":
[PipRequirement.parse("Django==3.2 ; python_version>'3'")],
},
Address("", target_name="Django"),
),
PythonRequirementTarget(
{
"dependencies": [":requirements.txt"],
"requirements": [PipRequirement.parse("Django-types")],
"type_stub_modules": ["django"],
},
Address("", target_name="Django-types"),
),
PythonRequirementTarget(
{
"dependencies": [":requirements.txt"],
"requirements":
[PipRequirement.parse("Un_Normalized_PROJECT")],
},
Address("", target_name="Un-Normalized-PROJECT"),
),
PythonRequirementTarget(
{
"dependencies": [":requirements.txt"],
"requirements": [
PipRequirement.parse(
"pip@ git+https://github.com/pypa/pip.git")
],
},
Address("", target_name="pip"),
),
],
)