def test_generated_targets_address_validation() -> None:
"""Ensure that all addresses are well formed."""
class MockTarget(Target):
alias = "tgt"
core_fields = ()
generator = MockTarget({}, Address("dir", target_name="generator"))
with pytest.raises(InvalidGeneratedTargetException):
GeneratedTargets(
generator,
[
MockTarget({},
Address("a_different_dir",
target_name="generator",
generated_name="gen"))
],
)
with pytest.raises(InvalidGeneratedTargetException):
GeneratedTargets(
generator,
[
MockTarget({},
Address("dir",
target_name="a_different_generator",
generated_name="gen"))
],
)
with pytest.raises(InvalidGeneratedTargetException):
GeneratedTargets(
generator,
[
MockTarget(
{},
Address(
"dir",
target_name="a_different_generator",
generated_name=None,
relative_file_path=None,
),
)
],
)
# These are fine.
GeneratedTargets(
generator,
[
MockTarget(
{},
Address("dir", target_name="generator", generated_name="gen")),
MockTarget({},
Address("dir",
target_name="generator",
relative_file_path="gen")),
],
)
async def generate_terraform_module_targets(
request: GenerateTerraformModuleTargetsRequest,
) -> GeneratedTargets:
generator = request.generator
sources_paths = await Get(
SourcesPaths, SourcesPathsRequest(generator.get(TerraformModulesSources))
)
dir_to_filenames = group_by_dir(sources_paths.files)
dirs_with_terraform_files = []
for dir, filenames in dir_to_filenames.items():
if any(filename.endswith(".tf") for filename in filenames):
dirs_with_terraform_files.append(dir)
def gen_target(dir: str) -> Target:
generated_target_fields = {}
for field in generator.field_values.values():
value: ImmutableValue | None
if isinstance(field, Sources):
value = tuple(sorted(dir_to_filenames[dir]))
else:
value = field.value
generated_target_fields[field.alias] = value
return TerraformModule(generated_target_fields, generator.address.create_generated(dir))
return GeneratedTargets(
request.generator, [gen_target(dir) for dir in dirs_with_terraform_files]
)
def test_target_generation_at_subdir(rule_runner: RuleRunner) -> None:
rule_runner.write_files({
"src/tf/BUILD": "terraform_modules()\n",
"src/tf/versions.tf": "",
"src/tf/foo/versions.tf": "",
})
generator_addr = Address("src/tf")
generator = rule_runner.get_target(generator_addr)
targets = rule_runner.request(
GeneratedTargets, [GenerateTerraformModuleTargetsRequest(generator)])
assert targets == GeneratedTargets(
generator,
[
TerraformModuleTarget(
{TerraformModuleSourcesField.alias: ("foo/versions.tf", )},
generator_addr.create_generated("foo"),
residence_dir="src/tf/foo",
),
TerraformModuleTarget(
{TerraformModuleSourcesField.alias: ("versions.tf", )},
generator_addr.create_generated("."),
residence_dir="src/tf",
),
],
)
async def generate_mock_generated_target(
request: MockGenerateTargetsRequest,
union_membership: UnionMembership,
) -> GeneratedTargets:
paths = await Get(
SourcesPaths,
SourcesPathsRequest(request.generator[MultipleSourcesField]))
# Generate using both "file address" and "generated target" syntax.
return GeneratedTargets(
request.generator,
[
*_generate_file_level_targets(
MockGeneratedTarget,
request.generator,
paths.files,
request.template_address,
request.template,
request.overrides,
union_membership,
add_dependencies_on_all_siblings=True,
use_generated_address_syntax=False,
).values(),
*_generate_file_level_targets(
MockGeneratedTarget,
request.generator,
paths.files,
request.template_address,
request.template,
request.overrides,
union_membership,
add_dependencies_on_all_siblings=True,
use_generated_address_syntax=True,
).values(),
],
)
async def generate_targets_from_go_mod(
request: GenerateTargetsFromGoModRequest,
union_membership: UnionMembership,
) -> GeneratedTargets:
generator_addr = request.generator.address
go_mod_info = await Get(
GoModInfo, GoModInfoRequest(request.generator[GoModSourcesField]))
all_packages = await Get(
AllThirdPartyPackages,
AllThirdPartyPackagesRequest(go_mod_info.digest, go_mod_info.mod_path),
)
def create_tgt(
pkg_info: ThirdPartyPkgAnalysis) -> GoThirdPartyPackageTarget:
return GoThirdPartyPackageTarget(
{GoImportPathField.alias: pkg_info.import_path},
# E.g. `src/go:mod#github.com/google/uuid`.
generator_addr.create_generated(pkg_info.import_path),
union_membership,
residence_dir=generator_addr.spec_path,
)
return GeneratedTargets(
request.generator,
(create_tgt(pkg_info)
for pkg_info in all_packages.import_paths_to_pkg_info.values()),
)
async def generate_mock_generated_target(
request: MockGenerateTargetsRequest) -> GeneratedTargets:
paths = await Get(SourcesPaths,
SourcesPathsRequest(request.generator[Sources]))
# Generate using both "file address" and "generated target" syntax.
return GeneratedTargets(
request.generator,
[
*generate_file_level_targets(
MockGeneratedTarget,
request.generator,
paths.files,
None,
add_dependencies_on_all_siblings=True,
use_generated_address_syntax=False,
).values(),
*generate_file_level_targets(
MockGeneratedTarget,
request.generator,
paths.files,
None,
add_dependencies_on_all_siblings=True,
use_generated_address_syntax=True,
).values(),
],
)
async def generate_go_external_package_targets(
request: GenerateGoExternalPackageTargetsRequest, ) -> GeneratedTargets:
generator_addr = request.generator.address
resolved_module = await Get(ResolvedGoModule,
ResolveGoModuleRequest(generator_addr))
all_resolved_packages = await MultiGet(
Get(
ResolveExternalGoModuleToPackagesResult,
ResolveExternalGoModuleToPackagesRequest(
path=module_descriptor.path,
version=module_descriptor.version,
go_sum_digest=resolved_module.digest,
),
) for module_descriptor in resolved_module.modules)
def create_tgt(pkg: ResolvedGoPackage) -> GoExternalPackageTarget:
return GoExternalPackageTarget(
{
GoExternalModulePathField.alias: pkg.module_path,
GoExternalModuleVersionField.alias: pkg.module_version,
GoExternalPackageImportPathField.alias: pkg.import_path,
},
# E.g. `src/go:mod#github.com/google/uuid`.
Address(
generator_addr.spec_path,
target_name=generator_addr.target_name,
generated_name=pkg.import_path,
),
)
return GeneratedTargets(
request.generator,
(create_tgt(pkg) for resolved_pkgs in all_resolved_packages
for pkg in resolved_pkgs.packages),
)
async def generate_mock_generated_target(
request: MockGenerateTargetsRequest) -> GeneratedTargets:
return GeneratedTargets(
request.generator,
[
MockGeneratedNonfileTarget(
request.template,
request.generator.address.create_generated("gen"))
],
)
async def generate_docker_image_rule(
request: GenerateDockerImageTargetFromOrigin, union_membership: UnionMembership
) -> GeneratedTargets:
return GeneratedTargets(
request.generator,
[
DockerImageTarget(
{
"instructions": DOCKERFILE.strip().split("\n"),
},
request.generator.address.create_generated("generated-image"),
union_membership,
)
],
)
def test_generate_source_targets() -> None:
rule_runner = RuleRunner(
rules=[
*target_types.rules(),
QueryRule(GeneratedTargets, [GenerateTargetsFromProtobufSources]),
],
target_types=[ProtobufSourcesGeneratorTarget],
)
rule_runner.write_files({
"src/proto/BUILD":
dedent("""\
protobuf_sources(
name='lib',
sources=['**/*.proto'],
overrides={'f1.proto': {'tags': ['overridden']}},
)
"""),
"src/proto/f1.proto":
"",
"src/proto/f2.proto":
"",
"src/proto/subdir/f.proto":
"",
})
generator = rule_runner.get_target(Address("src/proto", target_name="lib"))
def gen_tgt(rel_fp: str,
tags: list[str] | None = None) -> ProtobufSourceTarget:
return ProtobufSourceTarget(
{
SingleSourceField.alias: rel_fp,
Tags.alias: tags
},
Address("src/proto", target_name="lib", relative_file_path=rel_fp),
residence_dir=os.path.dirname(os.path.join("src/proto", rel_fp)),
)
generated = rule_runner.request(
GeneratedTargets, [GenerateTargetsFromProtobufSources(generator)])
assert generated == GeneratedTargets(
generator,
{
gen_tgt("f1.proto", tags=["overridden"]),
gen_tgt("f2.proto"),
gen_tgt("subdir/f.proto"),
},
)
async def generate_targets_from_pex_binaries(
request: GenerateTargetsFromPexBinaries,
union_membership: UnionMembership,
) -> GeneratedTargets:
generator_addr = request.generator.address
entry_points_field = request.generator[PexEntryPointsField].value or []
overrides = request.require_unparametrized_overrides()
inherited_fields = {
field.alias: field.value
for field in request.generator.field_values.values()
if not isinstance(field, (PexEntryPointsField, OverridesField))
}
# Note that we don't check for overlap because it seems unlikely to be a problem.
# If it does, we should add this check. (E.g. `path.to.app` and `path/to/app.py`)
def create_pex_binary(entry_point_spec: str) -> PexBinary:
return PexBinary(
{
PexEntryPointField.alias: entry_point_spec,
**inherited_fields,
# Note that overrides comes last to make sure that it indeed overrides.
**overrides.pop(entry_point_spec, {}),
},
# ":" is a forbidden character in target names
generator_addr.create_generated(entry_point_spec.replace(":",
"-")),
union_membership,
residence_dir=generator_addr.spec_path,
)
pex_binaries = [
create_pex_binary(entry_point) for entry_point in entry_points_field
]
if overrides:
raise InvalidFieldException(
f"Unused key in the `overrides` field for {request.generator.address}: "
f"{sorted(overrides)}"
f"Tip: if you'd like to override a field's value for every `{PexBinary.alias}` target "
"generated by this target, change the field directly on this target rather than using "
"the `overrides` field.")
return GeneratedTargets(request.generator, pex_binaries)
async def generate_targets_from_go_mod(
request: GenerateTargetsFromGoModRequest,
union_membership: UnionMembership,
) -> GeneratedTargets:
generator_addr = request.generator.address
go_mod_sources = request.generator[GoModSourcesField]
go_mod_info = await Get(GoModInfo, GoModInfoRequest(go_mod_sources))
go_mod_snapshot = await Get(Snapshot, Digest, go_mod_info.digest)
all_packages = await Get(
AllThirdPartyPackages,
AllThirdPartyPackagesRequest(go_mod_info.digest, go_mod_info.mod_path),
)
def gen_file_tgt(fp: str) -> TargetGeneratorSourcesHelperTarget:
return TargetGeneratorSourcesHelperTarget(
{TargetGeneratorSourcesHelperSourcesField.alias: fp},
generator_addr.create_file(fp),
union_membership,
)
file_tgts = [gen_file_tgt("go.mod")]
if go_mod_sources.go_sum_path in go_mod_snapshot.files:
file_tgts.append(gen_file_tgt("go.sum"))
def create_tgt(pkg_info: ThirdPartyPkgAnalysis) -> GoThirdPartyPackageTarget:
return GoThirdPartyPackageTarget(
{
**request.template,
GoImportPathField.alias: pkg_info.import_path,
Dependencies.alias: [t.address.spec for t in file_tgts],
},
# E.g. `src/go:mod#github.com/google/uuid`.
generator_addr.create_generated(pkg_info.import_path),
union_membership,
residence_dir=generator_addr.spec_path,
)
result = tuple(
create_tgt(pkg_info) for pkg_info in all_packages.import_paths_to_pkg_info.values()
) + tuple(file_tgts)
return GeneratedTargets(request.generator, result)
def generate_from_pants_requirements(
request: GenerateFromPantsRequirementsRequest, ) -> GeneratedTargets:
generator = request.generator
version = determine_version()
def create_tgt(dist: str, module: str) -> PythonRequirementTarget:
return PythonRequirementTarget(
{
PythonRequirementsField.alias: (f"{dist}{version}", ),
PythonRequirementModulesField.alias: (module, ),
PythonRequirementResolveField.alias:
generator[PythonRequirementResolveField].value,
},
generator.address.create_generated(dist),
)
result = [create_tgt("pantsbuild.pants", "pants")]
if generator[PantsRequirementsTestutilField].value:
result.append(create_tgt("pantsbuild.pants.testutil",
"pants.testutil"))
return GeneratedTargets(generator, result)
def test_target_generation_at_build_root(rule_runner: RuleRunner) -> None:
rule_runner.write_files({
"BUILD":
"terraform_modules(name='tf_mods')\n",
"src/tf/versions.tf":
"",
"src/tf/outputs.tf":
"",
"src/tf/foo/versions.tf":
"",
"src/tf/not-terraform/README.md":
"This should not trigger target generation.",
})
generator_addr = Address("", target_name="tf_mods")
generator = rule_runner.get_target(generator_addr)
targets = rule_runner.request(
GeneratedTargets, [GenerateTerraformModuleTargetsRequest(generator)])
assert targets == GeneratedTargets(
generator,
[
TerraformModuleTarget(
{
TerraformModuleSourcesField.alias:
("src/tf/foo/versions.tf", )
},
generator_addr.create_generated("src/tf/foo"),
residence_dir="src/tf/foo",
),
TerraformModuleTarget(
{
TerraformModuleSourcesField.alias:
("src/tf/outputs.tf", "src/tf/versions.tf")
},
generator_addr.create_generated("src/tf"),
residence_dir="src/tf",
),
],
)
def test_generate_source_and_test_targets() -> None:
rule_runner = RuleRunner(
rules=[
*target_types.rules(),
QueryRule(GeneratedTargets, [GenerateTargetsFromShunit2Tests]),
QueryRule(GeneratedTargets, [GenerateTargetsFromShellSources]),
],
target_types=[
Shunit2TestsGeneratorTarget, ShellSourcesGeneratorTarget
],
)
rule_runner.write_files({
"src/sh/BUILD":
dedent("""\
shell_sources(
name='lib',
sources=['**/*.sh', '!**/*_test.sh'],
overrides={'f1.sh': {'tags': ['overridden']}},
)
shunit2_tests(
name='tests',
sources=['**/*_test.sh'],
overrides={'f1_test.sh': {'tags': ['overridden']}},
)
"""),
"src/sh/f1.sh":
"",
"src/sh/f1_test.sh":
"",
"src/sh/f2.sh":
"",
"src/sh/f2_test.sh":
"",
"src/sh/subdir/f.sh":
"",
"src/sh/subdir/f_test.sh":
"",
})
sources_generator = rule_runner.get_target(
Address("src/sh", target_name="lib"))
tests_generator = rule_runner.get_target(
Address("src/sh", target_name="tests"))
def gen_source_tgt(rel_fp: str,
tags: list[str] | None = None) -> ShellSourceTarget:
return ShellSourceTarget(
{
SingleSourceField.alias: rel_fp,
Tags.alias: tags
},
Address("src/sh", target_name="lib", relative_file_path=rel_fp),
residence_dir=os.path.dirname(os.path.join("src/sh", rel_fp)),
)
def gen_test_tgt(rel_fp: str,
tags: list[str] | None = None) -> Shunit2TestTarget:
return Shunit2TestTarget(
{
SingleSourceField.alias: rel_fp,
Tags.alias: tags
},
Address("src/sh", target_name="tests", relative_file_path=rel_fp),
residence_dir=os.path.dirname(os.path.join("src/sh", rel_fp)),
)
sources_generated = rule_runner.request(
GeneratedTargets, [GenerateTargetsFromShellSources(sources_generator)])
tests_generated = rule_runner.request(
GeneratedTargets, [GenerateTargetsFromShunit2Tests(tests_generator)])
assert sources_generated == GeneratedTargets(
sources_generator,
{
gen_source_tgt("f1.sh", tags=["overridden"]),
gen_source_tgt("f2.sh"),
gen_source_tgt("subdir/f.sh"),
},
)
assert tests_generated == GeneratedTargets(
tests_generator,
{
gen_test_tgt("f1_test.sh", tags=["overridden"]),
gen_test_tgt("f2_test.sh"),
gen_test_tgt("subdir/f_test.sh"),
},
)
async def generate_targets_from_go_mod(
request: GenerateTargetsFromGoModRequest,
files_not_found_behavior: FilesNotFoundBehavior,
union_membership: UnionMembership,
) -> GeneratedTargets:
generator_addr = request.generator.address
go_mod_info, go_paths = await MultiGet(
Get(GoModInfo, GoModInfoRequest(generator_addr)),
Get(
Paths,
PathGlobs,
request.generator[GoModPackageSourcesField].path_globs(
files_not_found_behavior),
),
)
all_third_party_packages = await Get(
AllThirdPartyPackages,
AllThirdPartyPackagesRequest(go_mod_info.stripped_digest),
)
dir_to_filenames = group_by_dir(go_paths.files)
matched_dirs = [
dir for dir, filenames in dir_to_filenames.items() if filenames
]
def create_first_party_package_tgt(dir: str) -> GoFirstPartyPackageTarget:
subpath = fast_relpath(dir, generator_addr.spec_path)
import_path = f"{go_mod_info.import_path}/{subpath}" if subpath else go_mod_info.import_path
return GoFirstPartyPackageTarget(
{
GoImportPathField.alias:
import_path,
GoFirstPartyPackageSubpathField.alias:
subpath,
GoFirstPartyPackageSourcesField.alias:
tuple(
sorted(
os.path.join(subpath, f)
for f in dir_to_filenames[dir])),
},
# E.g. `src/go:mod#./subdir`.
generator_addr.create_generated(f"./{subpath}"),
union_membership,
residence_dir=dir,
)
first_party_pkgs = (create_first_party_package_tgt(dir)
for dir in matched_dirs)
def create_third_party_package_tgt(
pkg_info: ThirdPartyPkgInfo) -> GoThirdPartyPackageTarget:
return GoThirdPartyPackageTarget(
{GoImportPathField.alias: pkg_info.import_path},
# E.g. `src/go:mod#github.com/google/uuid`.
generator_addr.create_generated(pkg_info.import_path),
union_membership,
residence_dir=generator_addr.spec_path,
)
third_party_pkgs = (create_third_party_package_tgt(pkg_info)
for pkg_info in all_third_party_packages.
import_paths_to_pkg_info.values())
return GeneratedTargets(request.generator,
(*first_party_pkgs, *third_party_pkgs))
def test_generate_package_targets(rule_runner: RuleRunner) -> None:
rule_runner.write_files({
"src/go/BUILD":
"go_mod()\n",
"src/go/go.mod":
dedent("""\
module example.com/src/go
go 1.17
require (
github.com/google/go-cmp v0.4.0
github.com/google/uuid v1.2.0
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
)
"""),
"src/go/go.sum":
dedent("""\
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
"""),
"src/go/hello.go":
"",
"src/go/subdir/f.go":
"",
"src/go/subdir/f2.go":
"",
"src/go/another_dir/subdir/f.go":
"",
})
generator = rule_runner.get_target(Address("src/go"))
generated = rule_runner.request(
GeneratedTargets, [GenerateTargetsFromGoModRequest(generator)])
def gen_first_party_tgt(rel_dir: str,
sources: list[str]) -> GoFirstPartyPackageTarget:
return GoFirstPartyPackageTarget(
{
GoImportPathField.alias:
(os.path.join("example.com/src/go", rel_dir)
if rel_dir else "example.com/src/go"),
GoFirstPartyPackageSubpathField.alias:
rel_dir,
GoFirstPartyPackageSourcesField.alias:
tuple(sources),
},
Address("src/go", generated_name=f"./{rel_dir}"),
residence_dir=os.path.join("src/go", rel_dir).rstrip("/"),
)
def gen_third_party_tgt(import_path: str) -> GoThirdPartyPackageTarget:
return GoThirdPartyPackageTarget(
{GoImportPathField.alias: import_path},
Address("src/go", generated_name=import_path),
)
expected = GeneratedTargets(
generator,
{
gen_first_party_tgt("", ["hello.go"]),
gen_first_party_tgt("subdir", ["subdir/f.go", "subdir/f2.go"]),
gen_first_party_tgt("another_dir/subdir",
["another_dir/subdir/f.go"]),
*(gen_third_party_tgt(pkg) for pkg in (
"github.com/google/uuid",
"github.com/google/go-cmp/cmp",
"github.com/google/go-cmp/cmp/cmpopts",
"github.com/google/go-cmp/cmp/internal/diff",
"github.com/google/go-cmp/cmp/internal/flags",
"github.com/google/go-cmp/cmp/internal/function",
"github.com/google/go-cmp/cmp/internal/testprotos",
"github.com/google/go-cmp/cmp/internal/teststructs",
"github.com/google/go-cmp/cmp/internal/value",
"golang.org/x/xerrors",
"golang.org/x/xerrors/internal",
)),
},
)
assert list(generated.keys()) == list(expected.keys())
for addr, tgt in generated.items():
assert tgt == expected[addr]
async def generate_from_python_requirement(
request: GenerateFromPoetryRequirementsRequest,
build_root: BuildRoot,
union_membership: UnionMembership,
) -> GeneratedTargets:
generator = request.generator
pyproject_rel_path = generator[PoetryRequirementsSourceField].value
pyproject_full_path = generator[PoetryRequirementsSourceField].file_path
overrides = {
canonicalize_project_name(k): v
for k, v in request.require_unparametrized_overrides().items()
}
file_tgt = TargetGeneratorSourcesHelperTarget(
{TargetGeneratorSourcesHelperSourcesField.alias: pyproject_rel_path},
Address(
request.template_address.spec_path,
target_name=request.template_address.target_name,
relative_file_path=pyproject_rel_path,
),
union_membership,
)
digest_contents = await Get(
DigestContents,
PathGlobs(
[pyproject_full_path],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=f"{generator}'s field `{PoetryRequirementsSourceField.alias}`",
),
)
requirements = parse_pyproject_toml(
PyProjectToml(
build_root=PurePath(build_root.path),
toml_relpath=PurePath(pyproject_full_path),
toml_contents=digest_contents[0].content.decode(),
)
)
module_mapping = generator[ModuleMappingField].value
stubs_mapping = generator[TypeStubsModuleMappingField].value
def generate_tgt(parsed_req: PipRequirement) -> PythonRequirementTarget:
normalized_proj_name = canonicalize_project_name(parsed_req.project_name)
tgt_overrides = overrides.pop(normalized_proj_name, {})
if Dependencies.alias in tgt_overrides:
tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [
file_tgt.address.spec
]
return PythonRequirementTarget(
{
**request.template,
PythonRequirementsField.alias: [parsed_req],
PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name),
PythonRequirementTypeStubModulesField.alias: stubs_mapping.get(
normalized_proj_name
),
# This may get overridden by `tgt_overrides`, which will have already added in
# the file tgt.
Dependencies.alias: [file_tgt.address.spec],
**tgt_overrides,
},
request.template_address.create_generated(parsed_req.project_name),
union_membership,
)
result = tuple(generate_tgt(requirement) for requirement in requirements) + (file_tgt,)
if overrides:
raise InvalidFieldException(
softwrap(
f"""
Unused key in the `overrides` field for {request.template_address}:
{sorted(overrides)}
"""
)
)
return GeneratedTargets(generator, result)
async def generate_from_pipenv_requirement(
request: GenerateFromPipenvRequirementsRequest,
python_setup: PythonSetup) -> GeneratedTargets:
generator = request.generator
lock_rel_path = generator[PipenvSourceField].value
lock_full_path = generator[PipenvSourceField].file_path
overrides = {
canonicalize_project_name(k): v
for k, v in request.require_unparametrized_overrides().items()
}
file_tgt = TargetGeneratorSourcesHelperTarget(
{TargetGeneratorSourcesHelperSourcesField.alias: [lock_rel_path]},
Address(
generator.address.spec_path,
target_name=generator.address.target_name,
relative_file_path=lock_rel_path,
),
)
digest_contents = await Get(
DigestContents,
PathGlobs(
[lock_full_path],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=
f"{generator}'s field `{PipenvSourceField.alias}`",
),
)
lock_info = json.loads(digest_contents[0].content)
# Validate the resolve is legal.
generator[PythonRequirementResolveField].normalized_value(python_setup)
module_mapping = generator[ModuleMappingField].value
stubs_mapping = generator[TypeStubsModuleMappingField].value
inherited_fields = {
field.alias: field.value
for field in request.generator.field_values.values()
if isinstance(field, (*COMMON_TARGET_FIELDS,
PythonRequirementResolveField))
}
def generate_tgt(raw_req: str, info: dict) -> PythonRequirementTarget:
if info.get("extras"):
raw_req += f"[{','.join(info['extras'])}]"
raw_req += info.get("version", "")
if info.get("markers"):
raw_req += f";{info['markers']}"
parsed_req = PipRequirement.parse(raw_req)
normalized_proj_name = canonicalize_project_name(
parsed_req.project_name)
tgt_overrides = overrides.pop(normalized_proj_name, {})
if Dependencies.alias in tgt_overrides:
tgt_overrides[Dependencies.alias] = list(
tgt_overrides[Dependencies.alias]) + [file_tgt.address.spec]
return PythonRequirementTarget(
{
**inherited_fields,
PythonRequirementsField.alias: [parsed_req],
PythonRequirementModulesField.alias:
module_mapping.get(normalized_proj_name),
PythonRequirementTypeStubModulesField.alias:
stubs_mapping.get(normalized_proj_name),
# This may get overridden by `tgt_overrides`, which will have already added in
# the file tgt.
Dependencies.alias: [file_tgt.address.spec],
**tgt_overrides,
},
generator.address.create_generated(parsed_req.project_name),
)
result = tuple(
generate_tgt(req, info) for req, info in {
**lock_info.get("default", {}),
**lock_info.get("develop", {})
}.items()) + (file_tgt, )
if overrides:
raise InvalidFieldException(
f"Unused key in the `overrides` field for {request.generator.address}: "
f"{sorted(overrides)}")
return GeneratedTargets(generator, result)
def test_generate_file_and_resource_targets() -> None:
rule_runner = RuleRunner(
rules=[
core_target_types.generate_targets_from_files,
core_target_types.generate_targets_from_resources,
QueryRule(GeneratedTargets, [GenerateTargetsFromFiles]),
QueryRule(GeneratedTargets, [GenerateTargetsFromResources]),
],
target_types=[FilesGeneratorTarget, ResourcesGeneratorTarget],
)
rule_runner.write_files({
"assets/BUILD":
dedent("""\
files(
name='files',
sources=['**/*.ext'],
overrides={'f1.ext': {'tags': ['overridden']}},
)
resources(
name='resources',
sources=['**/*.ext'],
overrides={'f1.ext': {'tags': ['overridden']}},
)
"""),
"assets/f1.ext":
"",
"assets/f2.ext":
"",
"assets/subdir/f.ext":
"",
})
files_generator = rule_runner.get_target(
Address("assets", target_name="files"))
resources_generator = rule_runner.get_target(
Address("assets", target_name="resources"))
def gen_file_tgt(rel_fp: str, tags: list[str] | None = None) -> FileTarget:
return FileTarget(
{
SingleSourceField.alias: rel_fp,
Tags.alias: tags
},
Address("assets", target_name="files", relative_file_path=rel_fp),
residence_dir=os.path.dirname(os.path.join("assets", rel_fp)),
)
def gen_resource_tgt(rel_fp: str,
tags: list[str] | None = None) -> ResourceTarget:
return ResourceTarget(
{
SingleSourceField.alias: rel_fp,
Tags.alias: tags
},
Address("assets",
target_name="resources",
relative_file_path=rel_fp),
residence_dir=os.path.dirname(os.path.join("assets", rel_fp)),
)
generated_files = rule_runner.request(
GeneratedTargets, [GenerateTargetsFromFiles(files_generator)])
generated_resources = rule_runner.request(
GeneratedTargets, [GenerateTargetsFromResources(resources_generator)])
assert generated_files == GeneratedTargets(
files_generator,
{
gen_file_tgt("f1.ext", tags=["overridden"]),
gen_file_tgt("f2.ext"),
gen_file_tgt("subdir/f.ext"),
},
)
assert generated_resources == GeneratedTargets(
resources_generator,
{
gen_resource_tgt("f1.ext", tags=["overridden"]),
gen_resource_tgt("f2.ext"),
gen_resource_tgt("subdir/f.ext"),
},
)
async def generate_from_python_requirement(
request: GenerateFromPythonRequirementsRequest, python_setup: PythonSetup
) -> GeneratedTargets:
generator = request.generator
requirements_rel_path = generator[PythonRequirementsSourceField].value
requirements_full_path = generator[PythonRequirementsSourceField].file_path
overrides = {
canonicalize_project_name(k): v
for k, v in request.require_unparametrized_overrides().items()
}
file_tgt = TargetGeneratorSourcesHelperTarget(
{TargetGeneratorSourcesHelperSourcesField.alias: [requirements_rel_path]},
Address(
generator.address.spec_path,
target_name=generator.address.target_name,
relative_file_path=requirements_rel_path,
),
)
digest_contents = await Get(
DigestContents,
PathGlobs(
[requirements_full_path],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=f"{generator}'s field `{PythonRequirementsSourceField.alias}`",
),
)
requirements = parse_requirements_file(
digest_contents[0].content.decode(), rel_path=requirements_full_path
)
grouped_requirements = itertools.groupby(
requirements, lambda parsed_req: parsed_req.project_name
)
# Validate the resolve is legal.
generator[PythonRequirementResolveField].normalized_value(python_setup)
module_mapping = generator[ModuleMappingField].value
stubs_mapping = generator[TypeStubsModuleMappingField].value
inherited_fields = {
field.alias: field.value
for field in request.generator.field_values.values()
if isinstance(field, (*COMMON_TARGET_FIELDS, PythonRequirementResolveField))
}
def generate_tgt(
project_name: str, parsed_reqs: Iterable[PipRequirement]
) -> PythonRequirementTarget:
normalized_proj_name = canonicalize_project_name(project_name)
tgt_overrides = overrides.pop(normalized_proj_name, {})
if Dependencies.alias in tgt_overrides:
tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [
file_tgt.address.spec
]
return PythonRequirementTarget(
{
**inherited_fields,
PythonRequirementsField.alias: list(parsed_reqs),
PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name),
PythonRequirementTypeStubModulesField.alias: stubs_mapping.get(
normalized_proj_name
),
# This may get overridden by `tgt_overrides`, which will have already added in
# the file tgt.
Dependencies.alias: [file_tgt.address.spec],
**tgt_overrides,
},
generator.address.create_generated(project_name),
)
result = tuple(
generate_tgt(project_name, parsed_reqs_)
for project_name, parsed_reqs_ in grouped_requirements
) + (file_tgt,)
if overrides:
raise InvalidFieldException(
f"Unused key in the `overrides` field for {request.generator.address}: "
f"{sorted(overrides)}"
)
return GeneratedTargets(generator, result)
def test_generate_source_and_test_targets() -> None:
rule_runner = RuleRunner(
rules=[
*target_types_rules.rules(),
*import_rules(),
*python_sources.rules(),
QueryRule(GeneratedTargets, [GenerateTargetsFromPythonTests]),
QueryRule(GeneratedTargets, [GenerateTargetsFromPythonSources]),
QueryRule(GeneratedTargets, [GenerateTargetsFromPythonTestUtils]),
],
target_types=[
PythonTestsGeneratorTarget,
PythonSourcesGeneratorTarget,
PythonTestUtilsGeneratorTarget,
],
)
rule_runner.write_files({
"src/py/BUILD":
dedent("""\
python_sources(
name='lib',
sources=['**/*.py', '!**/*_test.py', '!**/conftest.py'],
overrides={'f1.py': {'tags': ['overridden']}},
)
python_tests(
name='tests',
sources=['**/*_test.py'],
overrides={'f1_test.py': {'tags': ['overridden']}},
)
python_test_utils(
name='test_utils',
sources=['**/conftest.py'],
overrides={'conftest.py': {'tags': ['overridden']}},
)
"""),
"src/py/f1.py":
"",
"src/py/f1_test.py":
"",
"src/py/conftest.py":
"",
"src/py/f2.py":
"",
"src/py/f2_test.py":
"",
"src/py/subdir/f.py":
"",
"src/py/subdir/f_test.py":
"",
"src/py/subdir/conftest.py":
"",
})
sources_generator = rule_runner.get_target(
Address("src/py", target_name="lib"))
tests_generator = rule_runner.get_target(
Address("src/py", target_name="tests"))
test_utils_generator = rule_runner.get_target(
Address("src/py", target_name="test_utils"))
def gen_source_tgt(rel_fp: str,
tags: list[str] | None = None,
*,
tgt_name: str) -> PythonSourceTarget:
return PythonSourceTarget(
{
SingleSourceField.alias: rel_fp,
Tags.alias: tags
},
Address("src/py", target_name=tgt_name, relative_file_path=rel_fp),
residence_dir=os.path.dirname(os.path.join("src/py", rel_fp)),
)
def gen_test_tgt(rel_fp: str,
tags: list[str] | None = None) -> PythonTestTarget:
return PythonTestTarget(
{
SingleSourceField.alias: rel_fp,
Tags.alias: tags
},
Address("src/py", target_name="tests", relative_file_path=rel_fp),
residence_dir=os.path.dirname(os.path.join("src/py", rel_fp)),
)
sources_generated = rule_runner.request(
GeneratedTargets,
[GenerateTargetsFromPythonSources(sources_generator)])
tests_generated = rule_runner.request(
GeneratedTargets, [GenerateTargetsFromPythonTests(tests_generator)])
test_utils_generated = rule_runner.request(
GeneratedTargets,
[GenerateTargetsFromPythonTestUtils(test_utils_generator)])
assert sources_generated == GeneratedTargets(
sources_generator,
{
gen_source_tgt("f1.py", tags=["overridden"], tgt_name="lib"),
gen_source_tgt("f2.py", tgt_name="lib"),
gen_source_tgt("subdir/f.py", tgt_name="lib"),
},
)
assert tests_generated == GeneratedTargets(
tests_generator,
{
gen_test_tgt("f1_test.py", tags=["overridden"]),
gen_test_tgt("f2_test.py"),
gen_test_tgt("subdir/f_test.py"),
},
)
assert test_utils_generated == GeneratedTargets(
test_utils_generator,
{
gen_source_tgt(
"conftest.py", tags=["overridden"], tgt_name="test_utils"),
gen_source_tgt("subdir/conftest.py", tgt_name="test_utils"),
},
)