def test_detect_cycle_direct(self):
a = self.make_target(':a')
# no cycles yet
sort_targets([a])
self.build_graph.inject_dependency(a.address, a.address)
with pytest.raises(CycleException):
sort_targets([a])
def test_detect_cycle_direct(self):
a = self.make_target(':a')
# no cycles yet
sort_targets([a])
self.build_graph.inject_dependency(a.address, a.address)
with pytest.raises(CycleException):
sort_targets([a])
def test_sort(self):
a = self.make_target(':a')
b = self.make_target(':b', dependencies=[a])
c = self.make_target(':c', dependencies=[b])
d = self.make_target(':d', dependencies=[c, a])
e = self.make_target(':e', dependencies=[d])
self.assertEquals(sort_targets([a, b, c, d, e]), [e, d, c, b, a])
self.assertEquals(sort_targets([b, d, a, e, c]), [e, d, c, b, a])
self.assertEquals(sort_targets([e, d, c, b, a]), [e, d, c, b, a])
def test_sort(self):
a = self.make_target(':a')
b = self.make_target(':b', dependencies=[a])
c = self.make_target(':c', dependencies=[b])
d = self.make_target(':d', dependencies=[c, a])
e = self.make_target(':e', dependencies=[d])
self.assertEquals(sort_targets([a, b, c, d, e]), [e, d, c, b, a])
self.assertEquals(sort_targets([b, d, a, e, c]), [e, d, c, b, a])
self.assertEquals(sort_targets([e, d, c, b, a]), [e, d, c, b, a])
def test_detect_cycle_indirect(self):
c = self.make_target(':c')
b = self.make_target(':b', dependencies=[c])
a = self.make_target(':a', dependencies=[c, b])
# no cycles yet
sort_targets([a])
self.build_graph.inject_dependency(c.address, a.address)
with pytest.raises(CycleException):
sort_targets([a])
def test_detect_cycle_indirect(self):
c = self.make_target(':c')
b = self.make_target(':b', dependencies=[c])
a = self.make_target(':a', dependencies=[c, b])
# no cycles yet
sort_targets([a])
self.build_graph.inject_dependency(c.address, a.address)
with pytest.raises(CycleException):
sort_targets([a])
def test_detect_cycle_direct(self):
a = self.make_target(':a')
# no cycles yet
sort_targets([a])
self.build_graph.inject_dependency(a.address, a.address)
try:
sort_targets([a])
self.fail("Expected a cycle to be detected")
except CycleException:
# expected
pass
def test_detect_cycle_direct(self):
a = self.make_target(':a')
# no cycles yet
sort_targets([a])
self.build_graph.inject_dependency(a.address, a.address)
try:
sort_targets([a])
self.fail("Expected a cycle to be detected")
except CycleException:
# expected
pass
def validate_platform_dependencies(self):
"""Check all jvm targets in the context, throwing an error or warning if there are bad targets.
If there are errors, this method fails slow rather than fails fast -- that is, it continues
checking the rest of the targets before spitting error messages. This is useful, because it's
nice to have a comprehensive list of all errors rather than just the first one we happened to
hit.
"""
conflicts = []
def is_conflicting(target, dependency):
return self.jvm_version(dependency) > self.jvm_version(target)
try:
sort_targets(self.jvm_targets)
except CycleException:
self.context.log.warn(
'Cannot validate dependencies when cycles exist in the build graph.'
)
return
try:
with self.invalidated(
self.jvm_targets,
fingerprint_strategy=self.PlatformFingerprintStrategy(),
invalidate_dependents=True) as vts:
dependency_map = self.jvm_dependency_map
for vts_target in vts.invalid_vts:
for target in vts_target.targets:
if target in dependency_map:
deps = dependency_map[target]
invalid_dependencies = [
dep for dep in deps
if is_conflicting(target, dep)
]
if invalid_dependencies:
conflicts.append(
(target, invalid_dependencies))
if conflicts:
# NB(gmalmquist): It's important to unconditionally raise an exception, then decide later
# whether to continue raising it or just print a warning, to make sure the targets aren't
# marked as valid if there are invalid platform dependencies.
error_message = self._create_full_error_message(conflicts)
raise self.IllegalJavaTargetLevelDependency(error_message)
except self.IllegalJavaTargetLevelDependency as e:
if self.check == 'fatal':
raise e
else:
assert self.check == 'warn'
self.context.log.warn(error_message)
return error_message
def test_detect_cycle_indirect(self):
c = self.make_target(':c')
b = self.make_target(':b', dependencies=[c])
a = self.make_target(':a', dependencies=[c, b])
# no cycles yet
sort_targets([a])
self.build_graph.inject_dependency(c.address, a.address)
try:
sort_targets([a])
self.fail("Expected a cycle to be detected")
except CycleException:
# expected
pass
def test_detect_cycle_indirect(self):
c = self.make_target(':c')
b = self.make_target(':b', dependencies=[c])
a = self.make_target(':a', dependencies=[c, b])
# no cycles yet
sort_targets([a])
self.build_graph.inject_dependency(c.address, a.address)
try:
sort_targets([a])
self.fail("Expected a cycle to be detected")
except CycleException:
# expected
pass
def exported_targets(self):
candidates = set()
if self.transitive:
candidates.update(self.context.targets())
else:
candidates.update(self.context.target_roots)
def get_synthetic(lang, target):
mappings = self.context.products.get(lang).get(target)
if mappings:
for key, generated in mappings.items():
for synthetic in generated:
yield synthetic
# Handle the case where a code gen target is in the listed roots and thus the publishable
# target is a synthetic twin generated by a code gen task upstream.
for candidate in self.context.target_roots:
candidates.update(get_synthetic('java', candidate))
candidates.update(get_synthetic('scala', candidate))
def exportable(tgt):
return tgt in candidates and tgt.is_exported
return OrderedSet(filter(exportable,
reversed(sort_targets(filter(exportable, candidates)))))
def exported_targets(self):
candidates = set()
if self.transitive:
candidates.update(self.context.targets())
else:
candidates.update(self.context.target_roots)
def get_synthetic(lang, target):
mappings = self.context.products.get(lang).get(target)
if mappings:
for key, generated in mappings.items():
for synthetic in generated:
yield synthetic
# Handle the case where a code gen target is in the listed roots and the thus the publishable
# target is a synthetic twin generated by a code gen task upstream.
for candidate in self.context.target_roots:
candidates.update(get_synthetic('java', candidate))
candidates.update(get_synthetic('scala', candidate))
def exportable(tgt):
return tgt in candidates and tgt.is_exported
return OrderedSet(
filter(exportable,
reversed(sort_targets(filter(exportable, candidates)))))
def execute_codegen(self, targets):
with self._task.context.new_workunit(name='execute', labels=[WorkUnit.MULTITOOL]):
ordered = [target for target in reversed(sort_targets(targets)) if target in targets]
for target in ordered:
with self._task.context.new_workunit(name=target.address.spec):
# TODO(gm): add a test-case to ensure this is correctly eliminating stale generated code.
safe_rmtree(self._task.codegen_workdir(target))
self._do_execute_codegen([target])
def vt_iter():
if topological_order:
sorted_targets = [t for t in reversed(sort_targets(targets)) if t in targets]
else:
sorted_targets = sorted(targets)
for target in sorted_targets:
target_key = self._key_for(target)
if target_key is not None:
yield VersionedTarget(self, target, target_key)
def validate_platform_dependencies(self):
"""Check all jvm targets in the context, throwing an error or warning if there are bad targets.
If there are errors, this method fails slow rather than fails fast -- that is, it continues
checking the rest of the targets before spitting error messages. This is useful, because it's
nice to have a comprehensive list of all errors rather than just the first one we happened to
hit.
"""
conflicts = []
def is_conflicting(target, dependency):
return self.jvm_version(dependency) > self.jvm_version(target)
try:
sort_targets(self.jvm_targets)
except CycleException:
self.context.log.warn('Cannot validate dependencies when cycles exist in the build graph.')
return
try:
with self.invalidated(self.jvm_targets,
fingerprint_strategy=self.PlatformFingerprintStrategy(),
invalidate_dependents=True) as vts:
dependency_map = self.jvm_dependency_map
for vts_target in vts.invalid_vts:
for target in vts_target.targets:
if target in dependency_map:
deps = dependency_map[target]
invalid_dependencies = [dep for dep in deps if is_conflicting(target, dep)]
if invalid_dependencies:
conflicts.append((target, invalid_dependencies))
if conflicts:
# NB(gmalmquist): It's important to unconditionally raise an exception, then decide later
# whether to continue raising it or just print a warning, to make sure the targets aren't
# marked as valid if there are invalid platform dependencies.
error_message = self._create_full_error_message(conflicts)
raise self.IllegalJavaTargetLevelDependency(error_message)
except self.IllegalJavaTargetLevelDependency as e:
if self.check == 'fatal':
raise e
else:
assert self.check == 'warn'
self.context.log.warn(error_message)
return error_message
def coalesce_targets(targets, discriminator):
"""Returns a list of Targets that `targets` depend on sorted from most dependent to least.
The targets are grouped where possible by target type as categorized by the given discriminator.
This algorithm was historically known as the "bang" algorithm from a time when it was
optionally enabled by appending a '!' (bang) to the command line target.
"""
sorted_targets = filter(discriminator, sort_targets(targets))
# can do no better for any of these:
# []
# [a]
# [a,b]
if len(sorted_targets) <= 2:
return sorted_targets
# For these, we'd like to coalesce if possible, like:
# [a,b,a,c,a,c] -> [a,a,a,b,c,c]
# adopt a quadratic worst case solution, when we find a type change edge, scan forward for
# the opposite edge and then try to swap dependency pairs to move the type back left to its
# grouping. If the leftwards migration fails due to a dependency constraint, we just stop
# and move on leaving "type islands".
current_type = None
# main scan left to right no backtracking
for i in range(len(sorted_targets) - 1):
current_target = sorted_targets[i]
if current_type != discriminator(current_target):
scanned_back = False
# scan ahead for next type match
for j in range(i + 1, len(sorted_targets)):
look_ahead_target = sorted_targets[j]
if current_type == discriminator(look_ahead_target):
scanned_back = True
# swap this guy as far back as we can
for k in range(j, i, -1):
previous_target = sorted_targets[k - 1]
mismatching_types = current_type != discriminator(previous_target)
not_a_dependency = look_ahead_target not in previous_target.closure()
if mismatching_types and not_a_dependency:
sorted_targets[k] = sorted_targets[k - 1]
sorted_targets[k - 1] = look_ahead_target
else:
break # out of k
break # out of j
if not scanned_back: # done with coalescing the current type, move on to next
current_type = discriminator(current_target)
return sorted_targets
def coalesce_targets(targets, discriminator):
"""Returns a list of Targets that `targets` depend on sorted from most dependent to least.
The targets are grouped where possible by target type as categorized by the given discriminator.
This algorithm was historically known as the "bang" algorithm from a time when it was
optionally enabled by appending a '!' (bang) to the command line target.
"""
sorted_targets = filter(discriminator, sort_targets(targets))
# can do no better for any of these:
# []
# [a]
# [a,b]
if len(sorted_targets) <= 2:
return sorted_targets
# For these, we'd like to coalesce if possible, like:
# [a,b,a,c,a,c] -> [a,a,a,b,c,c]
# adopt a quadratic worst case solution, when we find a type change edge, scan forward for
# the opposite edge and then try to swap dependency pairs to move the type back left to its
# grouping. If the leftwards migration fails due to a dependency constraint, we just stop
# and move on leaving "type islands".
current_type = None
# main scan left to right no backtracking
for i in range(len(sorted_targets) - 1):
current_target = sorted_targets[i]
if current_type != discriminator(current_target):
scanned_back = False
# scan ahead for next type match
for j in range(i + 1, len(sorted_targets)):
look_ahead_target = sorted_targets[j]
if current_type == discriminator(look_ahead_target):
scanned_back = True
# swap this guy as far back as we can
for k in range(j, i, -1):
previous_target = sorted_targets[k - 1]
mismatching_types = current_type != discriminator(previous_target)
not_a_dependency = look_ahead_target not in previous_target.closure()
if mismatching_types and not_a_dependency:
sorted_targets[k] = sorted_targets[k - 1]
sorted_targets[k - 1] = look_ahead_target
else:
break # out of k
break # out of j
if not scanned_back: # done with coalescing the current type, move on to next
current_type = discriminator(current_target)
return sorted_targets
def vt_iter():
if topological_order:
sorted_targets = [
t for t in reversed(sort_targets(targets)) if t in targets
]
else:
sorted_targets = sorted(targets)
for target in sorted_targets:
target_key = self._key_for(target)
if target_key is not None:
yield VersionedTarget(self, target, target_key)
def execute(self):
targets = [
target for target in self.context.target_roots
if self.has_provides(target)
]
if not targets:
raise TaskError('setup-py target(s) must provide an artifact.')
dist_dir = self.get_options().pants_distdir
# NB: We have to create and then run in 2 steps so that we can discover all exported targets
# in-play in the creation phase which then allows a tsort of these exported targets in the run
# phase to ensure an exported target is, for example (--run="sdist upload"), uploaded before any
# exported target that depends on it is uploaded.
created = {}
def create(target):
if target not in created:
self.context.log.info(
'Creating setup.py project for {}'.format(target))
setup_dir, dependencies = self.create_setup_py(
target, dist_dir)
created[target] = setup_dir
if self._recursive:
for dep in dependencies:
if self.has_provides(dep):
create(dep)
for target in targets:
create(target)
executed = [] # Collected and returned for tests.
for target in reversed(sort_targets(created.keys())):
setup_dir = created.get(target)
if setup_dir:
if not self._run:
self.context.log.info(
'Running packager against {}'.format(setup_dir))
setup_runner = Packager(setup_dir)
tgz_name = os.path.basename(setup_runner.sdist())
self.context.log.info('Writing {}'.format(
os.path.join(dist_dir, tgz_name)))
shutil.move(setup_runner.sdist(),
os.path.join(dist_dir, tgz_name))
safe_rmtree(setup_dir)
else:
self.context.log.info('Running {} against {}'.format(
self._run, setup_dir))
setup_runner = SetupPyRunner(setup_dir, self._run)
setup_runner.run()
executed.append(target)
return executed
def execute_codegen(self, targets):
with self._task.context.new_workunit(
name='execute', labels=[WorkUnitLabel.MULTITOOL]):
ordered = [
target for target in reversed(sort_targets(targets))
if target in targets
]
for target in ordered:
with self._task.context.new_workunit(
name=target.address.spec):
# TODO(gm): add a test-case to ensure this is correctly eliminating stale generated code.
safe_rmtree(self._task.codegen_workdir(target))
self._do_execute_codegen([target])
def _compute_transitive_deps_by_target(self):
"""Map from target to all the targets it depends on, transitively."""
# Sort from least to most dependent.
sorted_targets = reversed(sort_targets(self._context.targets()))
transitive_deps_by_target = defaultdict(set)
# Iterate in dep order, to accumulate the transitive deps for each target.
for target in sorted_targets:
transitive_deps = set()
for dep in target.dependencies:
transitive_deps.update(transitive_deps_by_target.get(dep, []))
transitive_deps.add(dep)
transitive_deps_by_target[target] = transitive_deps
return transitive_deps_by_target
def _compute_transitive_deps_by_target(self):
"""Map from target to all the targets it depends on, transitively."""
# Sort from least to most dependent.
sorted_targets = reversed(sort_targets(self._context.targets()))
transitive_deps_by_target = defaultdict(set)
# Iterate in dep order, to accumulate the transitive deps for each target.
for target in sorted_targets:
transitive_deps = set()
for dep in target.dependencies:
transitive_deps.update(transitive_deps_by_target.get(dep, []))
transitive_deps.add(dep)
transitive_deps_by_target[target] = transitive_deps
return transitive_deps_by_target
def execute(self):
targets = [target for target in self.context.target_roots if self.has_provides(target)]
if not targets:
raise TaskError('setup-py target(s) must provide an artifact.')
dist_dir = self.get_options().pants_distdir
# NB: We have to create and then run in 2 steps so that we can discover all exported targets
# in-play in the creation phase which then allows a tsort of these exported targets in the run
# phase to ensure an exported target is, for example (--run="sdist upload"), uploaded before any
# exported target that depends on it is uploaded.
created = {}
def create(target):
if target not in created:
self.context.log.info('Creating setup.py project for {}'.format(target))
setup_dir, dependencies = self.create_setup_py(target, dist_dir)
created[target] = setup_dir
if self._recursive:
for dep in dependencies:
if self.has_provides(dep):
create(dep)
for target in targets:
create(target)
executed = {} # Collected and returned for tests, processed target -> sdist|setup_dir.
for target in reversed(sort_targets(created.keys())):
setup_dir = created.get(target)
if setup_dir:
if not self._run:
self.context.log.info('Running packager against {}'.format(setup_dir))
setup_runner = Packager(setup_dir)
tgz_name = os.path.basename(setup_runner.sdist())
sdist_path = os.path.join(dist_dir, tgz_name)
self.context.log.info('Writing {}'.format(sdist_path))
shutil.move(setup_runner.sdist(), sdist_path)
safe_rmtree(setup_dir)
executed[target] = sdist_path
else:
self.context.log.info('Running {} against {}'.format(self._run, setup_dir))
setup_runner = SetupPyRunner(setup_dir, self._run)
setup_runner.run()
executed[target] = setup_dir
return executed
def _compute_transitive_deps_by_target(self):
"""Map from target to all the targets it depends on, transitively."""
# Sort from least to most dependent.
sorted_targets = reversed(sort_targets(self._context.targets()))
transitive_deps_by_target = defaultdict(set)
# Iterate in dep order, to accumulate the transitive deps for each target.
for target in sorted_targets:
transitive_deps = set()
for dep in target.dependencies:
transitive_deps.update(transitive_deps_by_target.get(dep, []))
transitive_deps.add(dep)
# Need to handle the case where a java_sources target has dependencies.
# In particular if it depends back on the original target.
if hasattr(target, 'java_sources'):
for java_source_target in target.java_sources:
for transitive_dep in java_source_target.dependencies:
transitive_deps_by_target[java_source_target].add(transitive_dep)
transitive_deps_by_target[target] = transitive_deps
return transitive_deps_by_target
def _compute_transitive_deps_by_target(self):
"""Map from target to all the targets it depends on, transitively."""
# Sort from least to most dependent.
sorted_targets = reversed(sort_targets(self.context.targets()))
transitive_deps_by_target = defaultdict(set)
# Iterate in dep order, to accumulate the transitive deps for each target.
for target in sorted_targets:
transitive_deps = set()
for dep in target.dependencies:
transitive_deps.update(transitive_deps_by_target.get(dep, []))
transitive_deps.add(dep)
# Need to handle the case where a java_sources target has dependencies.
# In particular if it depends back on the original target.
if hasattr(target, "java_sources"):
for java_source_target in target.java_sources:
for transitive_dep in java_source_target.dependencies:
transitive_deps_by_target[java_source_target].add(transitive_dep)
transitive_deps_by_target[target] = transitive_deps
return transitive_deps_by_target
def _order_target_list(self, targets): """Orders the targets topologically, from least to most dependent.""" return filter(targets.__contains__, reversed(sort_targets(targets)))
def execute_codegen(self, targets):
ordered = [target for target in reversed(sort_targets(targets)) if target in targets]
for target in ordered:
# TODO(gm): add a test-case to ensure this is correctly eliminating stale generated code.
safe_rmtree(self._task.codegen_workdir(target))
self._task.execute_codegen([target])
def _order_target_list(self, targets): """Orders the targets topologically, from least to most dependent.""" return filter(targets.__contains__, reversed(sort_targets(targets)))
