def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
self.graph, self.scheduler = setup_json_scheduler(build_root)
self.guava = self.graph.resolve(Address.parse('3rdparty/jvm:guava'))
self.thrift = self.graph.resolve(Address.parse('src/thrift/codegen/simple'))
self.java = self.graph.resolve(Address.parse('src/java/codegen/simple'))
def test_derivation(self):
a = self.make_target('a')
a_addr = a.address
b_addr = Address.parse('b')
self.build_graph.inject_synthetic_target(b_addr, Target, derived_from=a)
b = self.build_graph.get_target(b_addr)
c_addr = Address.parse('c')
self.build_graph.inject_synthetic_target(c_addr, Target, derived_from=a)
c = self.build_graph.get_target(c_addr)
d_addr = Address.parse('d')
self.build_graph.inject_synthetic_target(d_addr, Target, derived_from=b)
d = self.build_graph.get_target(d_addr)
self.assertEqual(a, self.build_graph.get_derived_from(a_addr))
self.assertEqual(a, self.build_graph.get_derived_from(b_addr))
self.assertEqual(a, self.build_graph.get_derived_from(c_addr))
self.assertEqual(b, self.build_graph.get_derived_from(d_addr))
self.assertEqual(a, self.build_graph.get_concrete_derived_from(a_addr))
self.assertEqual(a, self.build_graph.get_concrete_derived_from(b_addr))
self.assertEqual(a, self.build_graph.get_concrete_derived_from(c_addr))
self.assertEqual(a, self.build_graph.get_concrete_derived_from(d_addr))
self.assertEqual([b, c], self.build_graph.get_direct_derivatives(a_addr))
self.assertEqual([d], self.build_graph.get_direct_derivatives(b_addr))
self.assertEqual([], self.build_graph.get_direct_derivatives(c_addr))
self.assertEqual([], self.build_graph.get_direct_derivatives(d_addr))
self.assertEqual([b, c, d], self.build_graph.get_all_derivatives(a_addr))
self.assertEqual([d], self.build_graph.get_all_derivatives(b_addr))
self.assertEqual([], self.build_graph.get_all_derivatives(c_addr))
self.assertEqual([], self.build_graph.get_all_derivatives(d_addr))
def make_target(self, spec="", target_type=Target, dependencies=None, derived_from=None, **kwargs):
"""Creates a target and injects it into the test's build graph.
:param string spec: The target address spec that locates this target.
:param type target_type: The concrete target subclass to create this new target from.
:param list dependencies: A list of target instances this new target depends on.
:param derived_from: The target this new target was derived from.
:type derived_from: :class:`pants.build_graph.target.Target`
"""
address = Address.parse(spec)
target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs)
dependencies = dependencies or []
self.build_graph.inject_target(
target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from
)
# TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
# Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
for traversable_dependency_spec in target.traversable_dependency_specs:
traversable_dependency_address = Address.parse(traversable_dependency_spec, relative_to=address.spec_path)
traversable_dependency_target = self.build_graph.get_target(traversable_dependency_address)
if not traversable_dependency_target:
raise ValueError(
"Tests must make targets for traversable dependency specs ahead of them "
"being traversed, {} tried to traverse {} which does not exist.".format(
target, traversable_dependency_address
)
)
if traversable_dependency_target not in target.dependencies:
self.build_graph.inject_dependency(dependent=target.address, dependency=traversable_dependency_address)
target.mark_transitive_invalidation_hash_dirty()
return target
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
derived_from=None,
synthetic=False,
make_missing_sources=True,
**kwargs):
"""Creates a target and injects it into the test's build graph.
:API: public
:param string spec: The target address spec that locates this target.
:param type target_type: The concrete target subclass to create this new target from.
:param list dependencies: A list of target instances this new target depends on.
:param derived_from: The target this new target was derived from.
:type derived_from: :class:`pants.build_graph.target.Target`
"""
self._init_target_subsystem()
address = Address.parse(spec)
if make_missing_sources and 'sources' in kwargs:
for source in kwargs['sources']:
if '*' not in source:
self.create_file(os.path.join(address.spec_path, source), mode='a', contents='')
kwargs['sources'] = self.sources_for(kwargs['sources'], address.spec_path)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
self.build_graph.apply_injectables([target])
self.build_graph.inject_target(target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from,
synthetic=synthetic)
# TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
# Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
traversables = [target.compute_dependency_specs(payload=target.payload)]
for dependency_spec in itertools.chain(*traversables):
dependency_address = Address.parse(dependency_spec, relative_to=address.spec_path)
dependency_target = self.build_graph.get_target(dependency_address)
if not dependency_target:
raise ValueError('Tests must make targets for dependency specs ahead of them '
'being traversed, {} tried to traverse {} which does not exist.'
.format(target, dependency_address))
if dependency_target not in target.dependencies:
self.build_graph.inject_dependency(dependent=target.address,
dependency=dependency_address)
target.mark_transitive_invalidation_hash_dirty()
return target
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), "examples", "scheduler_inputs")
self.graph, self.scheduler = setup_json_scheduler(build_root)
self.guava = self.graph.resolve(Address.parse("3rdparty/jvm:guava"))
self.thrift = self.graph.resolve(Address.parse("src/thrift/codegen/simple"))
self.java = self.graph.resolve(Address.parse("src/java/codegen/simple"))
self.java_multi = self.graph.resolve(Address.parse("src/java/multiple_classpath_entries"))
self.unconfigured_thrift = self.graph.resolve(Address.parse("src/thrift/codegen/unconfigured"))
def test_target_invalid(self):
self.add_to_build_file('a/BUILD', 'target(name="a")')
with self.assertRaises(AddressLookupError):
self.build_graph.inject_address_closure(Address.parse('a:nope'))
self.add_to_build_file('b/BUILD', 'target(name="a")')
with self.assertRaises(AddressLookupError):
self.build_graph.inject_address_closure(Address.parse('b'))
with self.assertRaises(AddressLookupError):
self.build_graph.inject_address_closure(Address.parse('b:b'))
def test_create_single(self):
address_family = AddressFamily.create('',
[AddressMap('0', {
'one': Thing(name='one', age=42),
'two': Thing(name='two', age=37)
})])
self.assertEqual('', address_family.namespace)
self.assertEqual({Address.parse('//:one'): Thing(name='one', age=42),
Address.parse('//:two'): Thing(name='two', age=37)},
address_family.addressables)
def test_create_multiple(self):
address_family = AddressFamily.create('name/space',
[AddressMap('name/space/0',
{'one': Thing(name='one', age=42)}),
AddressMap('name/space/1',
{'two': Thing(name='two', age=37)})])
self.assertEqual('name/space', address_family.namespace)
self.assertEqual({Address.parse('name/space:one'): Thing(name='one', age=42),
Address.parse('name/space:two'): Thing(name='two', age=37)},
address_family.addressables)
def test_invalidation_relative(self):
resolved = self.address_mapper.resolve(Address.parse('a/b'))
self.assertEqual(self.a_b_target, resolved)
build_file = os.path.join(self.build_root, 'a/b/b.BUILD.json')
os.unlink(build_file)
self.assertIs(resolved, self.address_mapper.resolve(Address.parse('a/b')))
self.address_mapper.invalidate_build_file('a/b/b.BUILD.json')
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/b'))
def inject_address_closure(self, address):
if self.contains_address(address):
# The address was either mapped in or synthetically injected already.
return
if address in self._addresses_already_closed:
# We've visited this address already in the course of the active recursive injection.
return
mapper = self._address_mapper
target_address, target_addressable = mapper.resolve(address)
self._addresses_already_closed.add(target_address)
try:
dep_addresses = list(mapper.specs_to_addresses(target_addressable.dependency_specs,
relative_to=target_address.spec_path))
deps_seen = set()
for dep_address in dep_addresses:
if dep_address in deps_seen:
raise self.DuplicateAddressError(
"Addresses in dependencies must be unique. '{spec}' is referenced more than once."
.format(spec=dep_address.spec))
deps_seen.add(dep_address)
self.inject_address_closure(dep_address)
if not self.contains_address(target_address):
target = self._target_addressable_to_target(target_address, target_addressable)
self.apply_injectables([target])
self.inject_target(target, dependencies=dep_addresses)
else:
for dep_address in dep_addresses:
if dep_address not in self.dependencies_of(target_address):
self.inject_dependency(target_address, dep_address)
target = self.get_target(target_address)
for traversable_spec in target.compute_dependency_specs(payload=target.payload):
traversable_address = Address.parse(traversable_spec, relative_to=target_address.spec_path)
self.maybe_inject_address_closure(traversable_address)
if not any(traversable_address == t.address for t in target.dependencies):
self.inject_dependency(dependent=target.address, dependency=traversable_address)
target.mark_transitive_invalidation_hash_dirty()
for traversable_spec in target.compute_injectable_specs(payload=target.payload):
traversable_address = Address.parse(traversable_spec, relative_to=target_address.spec_path)
self.maybe_inject_address_closure(traversable_address)
target.mark_transitive_invalidation_hash_dirty()
except AddressLookupError as e:
raise self.TransitiveLookupError("{message}\n referenced from {spec}"
.format(message=e, spec=target_address.spec))
def test_resolve_cache(self):
graph = self.create_json_graph()
nonstrict_address = Address.parse('examples/graph_test:nonstrict')
nonstrict = graph.resolve(nonstrict_address)
self.assertIs(nonstrict, graph.resolve(nonstrict_address))
# The already resolved `nonstrict` interior node should be re-used by `java1`.
java1_address = Address.parse('examples/graph_test:java1')
java1 = graph.resolve(java1_address)
self.assertIs(nonstrict, java1.configurations[1])
self.assertIs(java1, graph.resolve(java1_address))
def test_invalidation_un_normalized(self):
resolved = self.address_mapper.resolve(Address.parse('a/b'))
self.assertEqual(self.a_b_target, resolved)
os.unlink(os.path.join(self.build_root, 'a/b/b.BUILD.json'))
self.assertIs(resolved, self.address_mapper.resolve(Address.parse('a/b')))
un_normalized_build_root = os.path.join(self.work_dir, 'build_root_linked')
os.symlink(self.build_root, un_normalized_build_root)
un_normalized_build_file = os.path.join(un_normalized_build_root, 'a/b/b.BUILD.json')
self.address_mapper.invalidate_build_file(un_normalized_build_file)
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/b'))
def test_resolve_cache(self):
scheduler = self.create_json()
nonstrict_address = Address.parse('graph_test:nonstrict')
nonstrict = self.resolve(scheduler, nonstrict_address)
self.assertEquals(nonstrict, self.resolve(scheduler, nonstrict_address))
# The already resolved `nonstrict` interior node should be re-used by `java1`.
java1_address = Address.parse('graph_test:java1')
java1 = self.resolve(scheduler, java1_address)
self.assertEquals(nonstrict, java1.configurations[1])
self.assertEquals(java1, self.resolve(scheduler, java1_address))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
derived_from=None,
synthetic=False,
**kwargs):
"""Creates a target and injects it into the test's build graph.
:API: public
:param string spec: The target address spec that locates this target.
:param type target_type: The concrete target subclass to create this new target from.
:param list dependencies: A list of target instances this new target depends on.
:param derived_from: The target this new target was derived from.
:type derived_from: :class:`pants.build_graph.target.Target`
"""
address = Address.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
self.build_graph.apply_injectables([target])
self.build_graph.inject_target(target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from,
synthetic=synthetic)
# TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
# Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
traversables = [target.compute_dependency_specs(payload=target.payload)]
# Only poke `traversable_dependency_specs` if a concrete implementation is defined
# in order to avoid spurious deprecation warnings.
if type(target).traversable_dependency_specs is not Target.traversable_dependency_specs:
traversables.append(target.traversable_dependency_specs)
for dependency_spec in itertools.chain(*traversables):
dependency_address = Address.parse(dependency_spec, relative_to=address.spec_path)
dependency_target = self.build_graph.get_target(dependency_address)
if not dependency_target:
raise ValueError('Tests must make targets for dependency specs ahead of them '
'being traversed, {} tried to traverse {} which does not exist.'
.format(target, dependency_address))
if dependency_target not in target.dependencies:
self.build_graph.inject_dependency(dependent=target.address,
dependency=dependency_address)
target.mark_transitive_invalidation_hash_dirty()
return target
def test_no_address_no_family(self):
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/c'))
# Errors are not cached.
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/c'))
build_file = os.path.join(self.build_root, 'a/c/c.BUILD.json')
with safe_open(build_file, 'w') as fp:
fp.write('{"type_alias": "struct", "name": "c"}')
resolved = self.address_mapper.resolve(Address.parse('a/c'))
self.assertEqual(Struct(name='c'), resolved)
def populate_target_dict(self, target_map):
"""Return a dict containing targets with files generated according to `target_map`.
The keys of `target_map` are target address strings, while the values of `target_map` should be
a dict which contains keyword arguments fed into `self.make_target()`, along with a few special
keys. Special keys are:
- 'key': used to access the target in the returned dict. Defaults to the target address spec.
- 'filemap': creates files at the specified relative paths to the target.
An `OrderedDict` of 2-tuples must be used with the targets topologically ordered, if
they have dependencies on each other. Note that dependency cycles are not currently supported
with this method.
:param target_map: Dict mapping each target address to generate -> kwargs for
`self.make_target()`, along with a 'key' and optionally a 'filemap' argument.
:return: Dict mapping the required 'key' argument -> target instance for each element of
`target_map`.
:rtype: dict
"""
target_dict = {}
# Create a target from each specification and insert it into `target_dict`.
for target_spec, target_kwargs in target_map.items():
unprocessed_kwargs = target_kwargs.copy()
target_base = Address.parse(target_spec).spec_path
# Populate the target's owned files from the specification.
filemap = unprocessed_kwargs.pop('filemap', {})
for rel_path, content in filemap.items():
buildroot_path = os.path.join(target_base, rel_path)
self.create_file(buildroot_path, content)
# Ensure any dependencies exist in the target dict (`target_map` must then be an
# OrderedDict).
# The 'key' is used to access the target in `target_dict`, and defaults to `target_spec`.
target_address = Address.parse(target_spec)
key = unprocessed_kwargs.pop('key', target_address.target_name)
dep_targets = []
for dep_spec in unprocessed_kwargs.pop('dependencies', []):
existing_tgt_key = target_map[dep_spec]['key']
dep_targets.append(target_dict[existing_tgt_key])
# Register the generated target.
generated_target = self.make_target(
spec=target_spec, dependencies=dep_targets, **unprocessed_kwargs)
target_dict[key] = generated_target
return target_dict
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), "examples", "scheduler_inputs")
self.graph, self.scheduler = setup_json_scheduler(build_root)
self.engine = LocalSerialEngine(self.scheduler)
self.guava = self.graph.resolve(Address.parse("3rdparty/jvm:guava"))
self.thrift = self.graph.resolve(Address.parse("src/thrift/codegen/simple"))
self.java = self.graph.resolve(Address.parse("src/java/codegen/simple"))
self.java_multi = self.graph.resolve(Address.parse("src/java/multiple_classpath_entries"))
self.unconfigured_thrift = self.graph.resolve(Address.parse("src/thrift/codegen/unconfigured"))
self.resources = self.graph.resolve(Address.parse("src/resources/simple"))
self.consumes_resources = self.graph.resolve(Address.parse("src/java/consumes_resources"))
self.consumes_managed_thirdparty = self.graph.resolve(Address.parse("src/java/managed_thirdparty"))
self.managed_guava = self.graph.resolve(Address.parse("3rdparty/jvm/managed:guava"))
self.managed_hadoop = self.graph.resolve(Address.parse("3rdparty/jvm/managed:hadoop-common"))
def test_max_recursion(self):
target_a = self.make_target('a', Target)
target_b = self.make_target('b', Target, dependencies=[target_a])
self.make_target('c', Target, dependencies=[target_b])
target_a.inject_dependency(Address.parse('c'))
with self.assertRaises(Target.RecursiveDepthError):
target_a.transitive_invalidation_hash()
def create_sources_field(self, sources, sources_rel_path, address=None, key_arg=None):
"""Factory method to create a SourcesField appropriate for the type of the sources object.
Note that this method is called before the call to Target.__init__ so don't expect fields to
be populated!
:return: a payload field object representing the sources parameter
:rtype: SourcesField
"""
if isinstance(sources, Addresses):
# Currently, this is only created by the result of from_target() which takes a single argument
if len(sources.addresses) != 1:
raise self.WrongNumberOfAddresses(
"Expected a single address to from_target() as argument to {spec}"
.format(spec=address.spec))
referenced_address = Address.parse(sources.addresses[0], relative_to=sources.rel_path)
return DeferredSourcesField(ref_address=referenced_address)
elif isinstance(sources, FilesetWithSpec):
filespec = sources.filespec
else:
sources = sources or []
assert_list(sources, key_arg=key_arg)
filespec = {'globs': [os.path.join(sources_rel_path, src) for src in (sources or [])]}
return SourcesField(sources=sources, sources_rel_path=sources_rel_path, filespec=filespec)
def java_sources(self):
for spec in self._java_sources_specs:
address = Address.parse(spec, relative_to=self.address.spec_path)
target = self._build_graph.get_target(address)
if target is None:
raise TargetDefinitionException(self, 'No such java target: {}'.format(spec))
yield target
def _synthetic_resources_target(self):
if not self.payload.resources.source_paths:
return None
# Create an address for the synthetic target.
spec = self.address.spec + '_synthetic_resources'
resource_address = Address.parse(spec=spec)
# For safety, ensure an address that's not used already, even though that's highly unlikely.
while self._build_graph.contains_address(resource_address):
spec += '_'
resource_address = Address.parse(spec=spec)
self._build_graph.inject_synthetic_target(resource_address, Resources,
sources=self.payload.resources.source_paths,
derived_from=self)
return self._build_graph.get_target(resource_address)
def traversable_specs(self):
for spec in super(PythonTarget, self).traversable_specs:
yield spec
if self._provides:
for spec in self._provides._binaries.values():
address = Address.parse(spec, relative_to=self.address.spec_path)
yield address.spec
def execute(self):
dist_targets = self.context.targets(is_local_python_dist)
build_graph = self.context.build_graph
if dist_targets:
with self.invalidated(dist_targets,
fingerprint_strategy=DefaultFingerprintStrategy(),
invalidate_dependents=True) as invalidation_check:
for vt in invalidation_check.invalid_vts:
if vt.target.dependencies:
raise TargetDefinitionException(
vt.target, 'The `dependencies` field is disallowed on `python_dist` targets. '
'List any 3rd party requirements in the install_requirements argument '
'of your setup function.'
)
self._create_dist(vt.target, vt.results_dir)
for vt in invalidation_check.all_vts:
dist = self._get_whl_from_dir(os.path.join(vt.results_dir, 'dist'))
req_lib_addr = Address.parse('{}__req_lib'.format(vt.target.address.spec))
self._inject_synthetic_dist_requirements(dist, req_lib_addr)
# Make any target that depends on the dist depend on the synthetic req_lib,
# for downstream consumption.
for dependent in build_graph.dependents_of(vt.target.address):
build_graph.inject_dependency(dependent, req_lib_addr)
def to_jar_dependencies(relative_to, jar_library_specs, build_graph):
"""Convenience method to resolve a list of specs to JarLibraries and return its jars attributes.
Expects that the jar_libraries are declared relative to this target.
:API: public
:param Address relative_to: address target that references jar_library_specs, for
error messages
:param list jar_library_specs: string specs to JavaLibrary targets. Note, this list should be returned
by the caller's traversable_specs() implementation to make sure that the jar_dependency jars
have been added to the build graph.
:param BuildGraph build_graph: build graph instance used to search for specs
:return: list of JarDependency instances represented by the library_specs
"""
jar_deps = OrderedSet()
for spec in jar_library_specs:
if not isinstance(spec, string_types):
raise JarLibrary.ExpectedAddressError(
"{address}: expected imports to contain string addresses, got {found_class}."
.format(address=relative_to.spec,
found_class=type(spec).__name__))
lookup = Address.parse(spec, relative_to=relative_to.spec_path)
target = build_graph.get_target(lookup)
if not isinstance(target, JarLibrary):
raise JarLibrary.WrongTargetTypeError(
"{address}: expected {spec} to be jar_library target type, got {found_class}"
.format(address=relative_to.spec,
spec=spec,
found_class=type(target).__name__))
jar_deps.update(target.jar_dependencies)
return list(jar_deps)
def __init__(self,
name,
build_file,
build_file_source_lines,
target_source_lines,
target_interval,
dependencies,
dependencies_interval):
"""See BuildFileManipulator.load() for how to construct one as a user."""
self.name = name
self.build_file = build_file
self.target_address = BuildFileAddress(build_file, name)
self._build_file_source_lines = build_file_source_lines
self._target_source_lines = target_source_lines
self._target_interval = target_interval
self._dependencies_interval = dependencies_interval
self._dependencies_by_address = {}
for dep in dependencies:
dep_address = Address.parse(dep.spec, relative_to=build_file.spec_path)
if dep_address in self._dependencies_by_address:
raise BuildTargetParseError('The address {dep_address} occurred multiple times in the '
'dependency specs for target {name} in {build_file}. '
.format(dep_address=dep_address.spec,
name=name,
build_file=build_file))
self._dependencies_by_address[dep_address] = dep
def _create_intermediate_target(self, address, suffix):
"""
:param string address: A target address.
:param string suffix: A string used as a suffix of the intermediate target name.
:returns: The address of a synthetic intermediary target.
"""
if not isinstance(address, six.string_types):
raise self.ExpectedAddressError("Expected string address argument, got type {type}"
.format(type=type(address)))
address = Address.parse(address, self._parse_context.rel_path)
# NB(gmalmquist): Ideally there should be a way to indicate that these targets are synthetic
# and shouldn't show up in `./pants list` etc, because we really don't want people to write
# handwritten dependencies on them. For now just give them names containing "-unstable-" as a
# hint.
hash_str = hash_target(str(address), suffix)
name = '{name}-unstable-{suffix}-{index}'.format(
name=address.target_name,
suffix=suffix.replace(' ', '.'),
index=hash_str,
)
self._parse_context.create_object_if_not_exists(
'target',
name=name,
dependencies=[address.spec],
**self.extra_target_arguments
)
return ':{}'.format(name)
def test_sources_ordering(self):
spec = 'testprojects/src/resources/org/pantsbuild/testproject/ordering'
with self.open_scheduler([spec]) as (graph, _, _):
target = graph.get_target(Address.parse(spec))
sources = [os.path.basename(s) for s in target.sources_relative_to_buildroot()]
self.assertEquals(['p', 'a', 'n', 't', 's', 'b', 'u', 'i', 'l', 'd'],
sources)
def test_codegen_simple(self):
build_request = BuildRequest(goals=["compile"], addressable_roots=[self.java.address])
execution_graph = self.scheduler.execution_graph(build_request)
plans = list(execution_graph.walk())
self.assertEqual(4, len(plans))
thrift_jars = [
Jar(org="org.apache.thrift", name="libthrift", rev="0.9.2"),
Jar(org="commons-lang", name="commons-lang", rev="2.5"),
self.graph.resolve(Address.parse("src/thrift:slf4j-api")),
]
jars = [self.guava] + thrift_jars
# Independent leaves 1st
self.assertEqual(
{
(
Sources.of(".java"),
Plan(
func_or_task_type=gen_apache_thrift,
subjects=[self.thrift],
strict=True,
rev="0.9.2",
gen="java",
sources=["src/thrift/codegen/simple/simple.thrift"],
),
),
(Classpath, Plan(func_or_task_type=IvyResolve, subjects=jars, jars=jars)),
},
set(self.extract_product_type_and_plan(p) for p in plans[0:2]),
)
# The rest is linked.
self.assertEqual(
(
Classpath,
Plan(
func_or_task_type=Javac,
subjects=[self.thrift],
sources=Promise(Sources.of(".java"), self.thrift),
classpath=[Promise(Classpath, jar) for jar in thrift_jars],
),
),
self.extract_product_type_and_plan(plans[2]),
)
self.assertEqual(
(
Classpath,
Plan(
func_or_task_type=Javac,
subjects=[self.java],
sources=["src/java/codegen/simple/Simple.java"],
classpath=[Promise(Classpath, self.guava), Promise(Classpath, self.thrift)],
),
),
self.extract_product_type_and_plan(plans[3]),
)
def injectables(self, build_graph):
junit_addr = Address.parse(self.injectables_spec_for_key('library'))
if not build_graph.contains_address(junit_addr):
build_graph.inject_synthetic_target(junit_addr,
JarLibrary,
jars=[JUnit.LIBRARY_JAR],
scope='forced')
def test_create_bad_targets(self):
with self.assertRaises(TypeError):
BuildFileAliases(targets={'fred': object()})
target = Target('fred', Address.parse('a:b'), MutableBuildGraph(address_mapper=None))
with self.assertRaises(TypeError):
BuildFileAliases(targets={'fred': target})
def __init__(self, *args, **kwargs):
super(MetaRename, self).__init__(*args, **kwargs)
self._from_address = Address.parse(self.get_options()['from'])
self._to_address = Address.parse(self.get_options().to)
def binary_iter():
if self.payload.provides:
for key, binary_spec in self.payload.provides.binaries.items():
address = Address.parse(binary_spec, relative_to=self.address.spec_path)
yield (key, self._build_graph.get_target(address))
def test_address_name_conflict(self):
with self.assertRaises(ValidationError):
Configuration(name='a', address=Address.parse('a:b'))
def test_generate_java_from_wsdl_with_embedded_xsd(rule_runner: RuleRunner) -> None:
rule_runner.write_files(
{
"src/wsdl/dir1/BUILD": "wsdl_sources()",
"src/wsdl/dir1/HelloService.wsdl": dedent(
"""\
<?xml version="1.0" encoding="UTF-8"?>
<wsdl:definitions name="HelloService"
targetNamespace="http://www.example.com/wsdl/HelloService/"
xmlns:wsdl="http://schemas.xmlsoap.org/wsdl/"
xmlns:soap="http://schemas.xmlsoap.org/wsdl/soap/"
xmlns:tns="http://www.example.com/wsdl/HelloService/"
xmlns:xsd="http://www.w3.org/2001/XMLSchema">
<wsdl:types>
<xsd:schema targetNamespace="http://www.example.com/wsdl/HelloService/">
<xsd:element name="Greeter" type="tns:Greeter" />
<xsd:complexType name="Greeter">
<xsd:sequence>
<xsd:element name="name" type="xsd:string" />
</xsd:sequence>
</xsd:complexType>
<xsd:element name="Greeting" type="tns:Greeting" />
<xsd:complexType name="Greeting">
<xsd:sequence>
<xsd:element name="message" type="xsd:string" />
</xsd:sequence>
</xsd:complexType>
</xsd:schema>
</wsdl:types>
<wsdl:message name="SayHelloRequest">
<wsdl:part name="greeter" type="tns:Greeter" />
</wsdl:message>
<wsdl:message name="SayHelloResponse">
<wsdl:part name="greeting" type="tns:Greeting" />
</wsdl:message>
<wsdl:portType name="HelloPortType">
<wsdl:operation name="sayHello">
<wsdl:input message="tns:SayHelloRequest" />
<wsdl:output message="tns:SayHelloResponse" />
</wsdl:operation>
</wsdl:portType>
<wsdl:binding name="HelloBinding" type="tns:HelloPortType">
<soap:binding transport="http://schemas.xmlsoap.org/soap/http" style="rpc"/>
<wsdl:operation name="sayHello">
<soap:operation soapAction="http://www.example.com/wsdl/HelloService/sayHello" />
<wsdl:input>
<soap:body use="literal" namespace="http://www.example.com/wsdl/HelloService/" />
</wsdl:input>
<wsdl:output>
<soap:body use="literal" namespace="http://www.example.com/wsdl/HelloService/" />
</wsdl:output>
</wsdl:operation>
</wsdl:binding>
<wsdl:service name="HelloService">
<wsdl:documentation>WSDL File for HelloService</wsdl:documentation>
<wsdl:port name="HelloPort" binding="tns:HelloBinding">
<soap:address location="http://www.example.com/HelloService" />
</wsdl:port>
</wsdl:service>
</wsdl:definitions>
"""
),
}
)
def assert_gen(addr: Address, expected: Iterable[str]) -> None:
assert_files_generated(
rule_runner, addr, source_roots=["src/wsdl"], expected_files=list(expected)
)
assert_gen(
Address("src/wsdl/dir1", relative_file_path="HelloService.wsdl"),
(
"src/wsdl/com/example/wsdl/helloservice/Greeter.java",
"src/wsdl/com/example/wsdl/helloservice/Greeting.java",
"src/wsdl/com/example/wsdl/helloservice/HelloPortType.java",
"src/wsdl/com/example/wsdl/helloservice/HelloService.java",
"src/wsdl/com/example/wsdl/helloservice/ObjectFactory.java",
"src/wsdl/com/example/wsdl/helloservice/package-info.java",
),
)
def injectables(self, build_graph):
junit_addr = Address.parse(self.injectables_address_spec_for_key("library"))
if not build_graph.contains_address(junit_addr):
build_graph.inject_synthetic_target(
junit_addr, JarLibrary, jars=[JUnit.LIBRARY_JAR], scope="forced"
)
def test_equivalence(self):
self.assertEqual(Address('a/b', 'c'), Address('a/b', 'c'))
self.assertEqual(Address('a/b', 'c'), Address.parse('a/b:c'))
self.assertEqual(Address.parse('a/b:c'), Address.parse('a/b:c'))
def test_generates_java(rule_runner: RuleRunner) -> None:
# This tests a few things:
# * We generate the correct file names.
# * Thrift files can import other thrift files, and those can import others
# (transitive dependencies). We'll only generate the requested target, though.
# * We can handle multiple source roots, which need to be preserved in the final output.
rule_runner.write_files({
"src/thrift/dir1/f.thrift":
dedent("""\
namespace java org.pantsbuild.example
struct Person {
1: string name
2: i32 id
3: string email
}
"""),
"src/thrift/dir1/f2.thrift":
dedent("""\
namespace java org.pantsbuild.example
include "dir1/f.thrift"
struct ManagedPerson {
1: f.Person employee
2: f.Person manager
}
"""),
"src/thrift/dir1/BUILD":
"thrift_sources()",
"src/thrift/dir2/g.thrift":
dedent("""\
namespace java org.pantsbuild.example
include "dir1/f2.thrift"
struct ManagedPersonWrapper {
1: f2.ManagedPerson managed_person
}
"""),
"src/thrift/dir2/BUILD":
"thrift_sources(dependencies=['src/thrift/dir1'])",
# Test another source root.
"tests/thrift/test_thrifts/f.thrift":
dedent("""\
namespace java org.pantsbuild.example
include "dir2/g.thrift"
struct Executive {
1: g.ManagedPersonWrapper managed_person_wrapper
}
"""),
"tests/thrift/test_thrifts/BUILD":
"thrift_sources(dependencies=['src/thrift/dir2'])",
})
def assert_gen(addr: Address, expected: list[str]) -> None:
assert_files_generated(
rule_runner,
addr,
source_roots=["src/python", "/src/thrift", "/tests/thrift"],
expected_files=expected,
)
assert_gen(
Address("src/thrift/dir1", relative_file_path="f.thrift"),
[
"src/thrift/org/pantsbuild/example/Person.java",
],
)
assert_gen(
Address("src/thrift/dir1", relative_file_path="f2.thrift"),
[
"src/thrift/org/pantsbuild/example/ManagedPerson.java",
],
)
assert_gen(
Address("src/thrift/dir2", relative_file_path="g.thrift"),
[
"src/thrift/org/pantsbuild/example/ManagedPersonWrapper.java",
],
)
assert_gen(
Address("tests/thrift/test_thrifts", relative_file_path="f.thrift"),
[
"tests/thrift/org/pantsbuild/example/Executive.java",
],
)
def test_not_found_and_family_does_not_exist(self):
scheduler = self.create_json()
dne = Address.parse('this/dir/does/not/exist')
self.assert_resolve_failure_type(ResolveError, dne, scheduler)
self.do_test_trace_message(scheduler, dne)
def test_not_found_but_family_exists(self):
scheduler = self.create_json()
dne = Address.parse('graph_test:this_addressable_does_not_exist')
self.assert_resolve_failure_type(ResolveError, dne, scheduler)
self.do_test_trace_message(scheduler, dne)
def test_type_mismatch_error(self):
scheduler = self.create_json()
mismatch = Address.parse('graph_test:type_mismatch')
self.assert_resolve_failure_type(ResolvedTypeMismatchError, mismatch, scheduler)
self.do_test_trace_message(scheduler, mismatch)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._from_address = Address.parse(self.get_options()["from"])
self._to_address = Address.parse(self.get_options().to)
def address(name): return Address(spec_path='graph_test', target_name=name)
def make_target(
self,
spec="",
target_type=TargetV1,
dependencies=None,
derived_from=None,
synthetic=False,
make_missing_sources=True,
**kwargs,
):
"""Creates a target and injects it into the test's build graph.
:API: public
:param string spec: The target address spec that locates this target.
:param type target_type: The concrete target subclass to create this new target from.
:param list dependencies: A list of target instances this new target depends on.
:param derived_from: The target this new target was derived from.
:type derived_from: :class:`pants.build_graph.target.Target`
"""
self._init_target_subsystem()
address = Address.parse(spec)
if make_missing_sources and "sources" in kwargs:
for source in kwargs["sources"]:
if "*" not in source:
self.create_file(os.path.join(address.spec_path, source),
mode="a",
contents="")
kwargs["sources"] = self.sources_for(kwargs["sources"],
address.spec_path)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
self.build_graph.apply_injectables([target])
self.build_graph.inject_target(
target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from,
synthetic=synthetic,
)
# TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
# Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
traversables = [
target.compute_dependency_address_specs(payload=target.payload)
]
for dependency_spec in itertools.chain(*traversables):
dependency_address = Address.parse(dependency_spec,
relative_to=address.spec_path)
dependency_target = self.build_graph.get_target(dependency_address)
if not dependency_target:
raise ValueError(
"Tests must make targets for dependency specs ahead of them "
"being traversed, {} tried to traverse {} which does not exist."
.format(target, dependency_address))
if dependency_target not in target.dependencies:
self.build_graph.inject_dependency(
dependent=target.address, dependency=dependency_address)
target.mark_transitive_invalidation_hash_dirty()
return target
def library_specs(self):
"""Lists of specs to resolve to jar_libraries containing more jars."""
return [
Address.parse(spec, relative_to=self.address.spec_path).spec
for spec in self.payload.library_specs
]
def checker_target(self):
self.context.resolve(self._CHECKER_ADDRESS_SPEC)
return self.context.build_graph.get_target(
Address.parse(self._CHECKER_ADDRESS_SPEC))
def get_alternate_target_roots(cls, options, address_mapper, build_graph):
processed = set()
for jvm_tool in JvmToolMixin.get_registered_tools():
dep_spec = jvm_tool.dep_spec(options)
dep_address = Address.parse(dep_spec)
# Some JVM tools are requested multiple times, we only need to handle them once.
if dep_address not in processed:
processed.add(dep_address)
try:
if build_graph.resolve_address(dep_address):
# The user has defined a tool classpath override - we let that stand.
continue
except AddressLookupError as e:
if jvm_tool.classpath is None:
raise cls._tool_resolve_error(e, dep_spec, jvm_tool)
else:
if not jvm_tool.is_default(options):
# The user specified a target spec for this jvm tool that doesn't actually exist.
# We want to error out here instead of just silently using the default option while
# appearing to respect their config.
raise cls.ToolResolveError(
dedent("""
Failed to resolve target for tool: {tool}. This target was obtained from
option {option} in scope {scope}.
Make sure you didn't make a typo in the tool's address. You specified that the
tool should use the target found at "{tool}".
This target has a default classpath configured, so you can simply remove:
[{scope}]
{option}: {tool}
from pants.ini (or any other config file) to use the default tool.
The default classpath is: {default_classpath}
Note that tool target addresses in pants.ini should be specified *without* quotes.
""").strip().format(tool=dep_spec,
option=jvm_tool.key,
scope=jvm_tool.scope,
default_classpath=':'.join(
map(str, jvm_tool.classpath or ()))))
if jvm_tool.classpath:
tool_classpath_target = JarLibrary(
name=dep_address.target_name,
address=dep_address,
build_graph=build_graph,
jars=jvm_tool.classpath)
else:
# The tool classpath is empty by default, so we just inject a dummy target that
# ivy resolves as the empty list classpath. JarLibrary won't do since it requires
# one or more jars, so we just pick a target type ivy has no resolve work to do for.
tool_classpath_target = Target(
name=dep_address.target_name,
address=dep_address,
build_graph=build_graph)
build_graph.inject_target(tool_classpath_target,
synthetic=True)
# We use the trick of not returning alternate roots, but instead just filling the dep_spec
# holes with a JarLibrary built from a tool's default classpath JarDependency list if there is
# no over-riding targets present. This means we do modify the build_graph, but we at least do
# it at a time in the engine lifecycle cut out for handling that.
return None
def _transitive_download_remote_libs(self, go_remote_libs, all_known_addresses=None):
"""Recursively attempt to resolve / download all remote transitive deps of go_remote_libs.
Returns a dict<GoRemoteLibrary, set<tuple<str, Address>>>, which maps a go remote library to a
set of unresolved remote dependencies, each dependency expressed as a tuple containing the
the import path of the dependency and the expected target address. If all transitive
dependencies were successfully resolved, returns an empty dict.
Downloads as many invalidated transitive dependencies as possible, and returns as many
undeclared dependencies as possible. However, because the dependencies of a remote library
can only be determined _after_ it has been downloaded, a transitive dependency of an undeclared
remote library will never be detected.
Because go_remote_libraries do not declare dependencies (rather, they are inferred), injects
all successfully resolved transitive dependencies into the build graph.
"""
if not go_remote_libs:
return {}
all_known_addresses = all_known_addresses or set()
all_known_addresses.update(lib.address for lib in go_remote_libs)
resolved_remote_libs = set()
undeclared_deps = defaultdict(set)
go_remote_lib_src = self.context.products.get_data('go_remote_lib_src')
with self.invalidated(go_remote_libs) as invalidation_check:
for vt in invalidation_check.all_vts:
go_remote_lib = vt.target
gopath = vt.results_dir
fetcher = self._get_fetcher(go_remote_lib.import_path)
if not vt.valid:
root = fetcher.root(go_remote_lib.import_path)
fetch_dir = os.path.join(self.workdir, 'fetches')
root_dir = os.path.join(fetch_dir, root)
# Only fetch each remote root once.
if not os.path.exists(root_dir):
with temporary_dir() as tmp_fetch_root:
fetcher.fetch(go_remote_lib.import_path, dest=tmp_fetch_root, rev=go_remote_lib.rev)
safe_mkdir(root_dir)
for path in os.listdir(tmp_fetch_root):
shutil.move(os.path.join(tmp_fetch_root, path), os.path.join(root_dir, path))
# TODO(John Sirois): Circle back and get get rid of this symlink tree.
# GoWorkspaceTask will further symlink a single package from the tree below into a
# target's workspace when it could just be linking from the fetch_dir. The only thing
# standing in the way is a determination of what we want to artifact cache. If we don't
# want to cache fetched zips, linking straight from the fetch_dir works simply. Otherwise
# thought needs to be applied to using the artifact cache directly or synthesizing a
# canonical owner target for the fetched files that 'child' targets (subpackages) can
# depend on and share the fetch from.
dest_dir = os.path.join(gopath, 'src', root)
# We may have been `invalidate`d and not `clean-all`ed so we need a new empty symlink
# chroot to avoid collision; thus `clean=True`.
safe_mkdir(dest_dir, clean=True)
for path in os.listdir(root_dir):
os.symlink(os.path.join(root_dir, path), os.path.join(dest_dir, path))
# Map the fetched remote sources.
pkg = go_remote_lib.import_path
go_remote_lib_src[go_remote_lib] = os.path.join(gopath, 'src', pkg)
for remote_import_path in self._get_remote_import_paths(pkg, gopath=gopath):
fetcher = self._get_fetcher(remote_import_path)
remote_root = fetcher.root(remote_import_path)
spec_path = os.path.join(go_remote_lib.target_base, remote_root)
package_path = GoRemoteLibrary.remote_package_path(remote_root, remote_import_path)
target_name = package_path or os.path.basename(remote_root)
address = Address(spec_path, target_name)
if address not in all_known_addresses:
try:
# If we've already resolved a package from this remote root, its ok to define an
# implicit synthetic remote target for all other packages in the same remote root.
implicit_ok = any(spec_path == a.spec_path for a in all_known_addresses)
remote_lib = self._resolve(go_remote_lib, address, package_path, implicit_ok)
resolved_remote_libs.add(remote_lib)
all_known_addresses.add(address)
except self.UndeclaredRemoteLibError as e:
undeclared_deps[go_remote_lib].add((remote_import_path, e.address))
self.context.build_graph.inject_dependency(go_remote_lib.address, address)
# Recurse after the invalidated block, so the libraries we downloaded are now "valid"
# and thus we don't try to download a library twice.
trans_undeclared_deps = self._transitive_download_remote_libs(resolved_remote_libs,
all_known_addresses)
undeclared_deps.update(trans_undeclared_deps)
return undeclared_deps
def _parse_config(address):
target_name, _, config_specifier = address.target_name.partition('@')
config_specifier = config_specifier or None
normalized_address = Address(spec_path=address.spec_path,
target_name=target_name)
return normalized_address, config_specifier
def resolve(self, spec):
"""Returns an iterator over the target(s) the given address points to."""
address = Address.parse(spec)
# NB: This is an idempotent, short-circuiting call.
self.inject_address_closure(address)
return self.transitive_subgraph_of_addresses([address])
def execute(self):
targets = self.codegen_targets()
build_graph = self.context.build_graph
with self.invalidated(targets, invalidate_dependents=True) as invalidation_check:
invalid_targets = list(chain.from_iterable(
vts.targets for vts in invalidation_check.invalid_vts
))
if invalid_targets:
self._execute_codegen(invalid_targets)
invalid_vts_by_target = dict([(vt.target, vt) for vt in invalidation_check.invalid_vts])
vts_artifactfiles_pairs = defaultdict(list)
for target in targets:
synthetic_name = '{0}-{1}'.format(target.id, 'scala')
spec_path = os.path.relpath(self.namespace_out, get_buildroot())
synthetic_address = Address(spec_path, synthetic_name)
generated_scala_sources = [
'{0}.{1}'.format(source, 'scala')
for source in self.sources_generated_by_target(target)
]
generated_java_sources = [
os.path.join(os.path.dirname(source), 'java_{0}.java'.format(os.path.basename(source)))
for source in self.sources_generated_by_target(target)
]
relative_generated_sources = [
os.path.relpath(src, self.namespace_out)
for src in generated_scala_sources + generated_java_sources
]
synthetic_target = self.context.add_new_target(
address=synthetic_address,
target_type=ScalaLibrary,
dependencies=self.synthetic_target_extra_dependencies,
sources=relative_generated_sources,
derived_from=target,
)
# NOTE: This bypasses the convenience function (Target.inject_dependency) in order
# to improve performance. Note that we can walk the transitive dependee subgraph once
# for transitive invalidation rather than walking a smaller subgraph for every single
# dependency injected. This walk also covers the invalidation for the java synthetic
# target above.
for dependent_address in build_graph.dependents_of(target.address):
build_graph.inject_dependency(dependent=dependent_address,
dependency=synthetic_target.address)
# NOTE: See the above comment. The same note applies.
for concrete_dependency_address in build_graph.dependencies_of(target.address):
build_graph.inject_dependency(
dependent=synthetic_target.address,
dependency=concrete_dependency_address,
)
build_graph.walk_transitive_dependee_graph(
[target.address],
work=lambda t: t.mark_transitive_invalidation_hash_dirty(),
)
if target in self.context.target_roots:
self.context.target_roots.append(synthetic_target)
if target in invalid_vts_by_target:
vts_artifactfiles_pairs[invalid_vts_by_target[target]].extend(
generated_scala_sources + generated_java_sources
)
if self.artifact_cache_writes_enabled():
self.update_artifact_cache(vts_artifactfiles_pairs.items())
def _ordering_test(self, spec, expected_sources=None):
expected_sources = expected_sources or ['p', 'a', 'n', 't', 's', 'b', 'u', 'i', 'l', 'd']
with self.open_scheduler([spec]) as (graph, _, _):
target = graph.get_target(Address.parse(spec))
sources = [os.path.basename(s) for s in target.sources_relative_to_buildroot()]
self.assertEquals(expected_sources, sources)
def get_target_from_spec(self, spec, relative_to=''):
"""Converts `spec` into an address and returns the result of `get_target`
:API: public
"""
return self.get_target(Address.parse(spec, relative_to=relative_to))
def test_contains_address(self):
a = Address.parse('a')
self.assertFalse(self.build_graph.contains_address(a))
target = Target(name='a', address=a, build_graph=self.build_graph)
self.build_graph.inject_target(target)
self.assertTrue(self.build_graph.contains_address(a))
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples',
'scheduler_inputs')
self.scheduler = setup_json_scheduler(build_root, self._native)
self.java = Address.parse('src/java/simple')
def test_no_targets(self):
self.add_to_build_file('empty/BUILD', 'pass')
with self.assertRaises(AddressLookupError):
self.build_graph.inject_address_closure(Address.parse('empty'))
with self.assertRaises(AddressLookupError):
self.build_graph.inject_address_closure(Address.parse('empty:foo'))
def inject_address_closure(self, spec):
self.build_graph.inject_address_closure(Address.parse(spec))
def inject(target, dep_spec, is_dependency):
address = Address.parse(dep_spec, relative_to=target.address.spec_path)
if not any(address == t.address for t in target.dependencies):
addresses_to_inject.add(address)
if is_dependency:
deps_to_inject.add((target.address, address))
def get_target(spec):
address = Address.parse(spec, relative_to=self.address.spec_path)
tgt = self._build_graph.get_target(address)
if tgt is None:
raise TargetDefinitionException(self, 'No such resource target: {}'.format(address))
return tgt
def do_test_cycle(self, address_str): scheduler = self.create_json() parsed_address = Address.parse(address_str) self.do_test_trace_message(scheduler, parsed_address, 'Dep graph contained a cycle.')
def test_address_no_name(self):
config = Configuration(address=Address.parse('a:b'))
self.assertEqual('b', config.name)
