def _state_validation(self, result):
# State validation.
unknown_state_types = tuple(
type(state) for _, state in result.root_products
if type(state) not in (Throw, Return))
if unknown_state_types:
State.raise_unrecognized(unknown_state_types)
def product_request(self, product, subjects):
"""Executes a request for a singular product type from the scheduler for one or more subjects
and yields the products.
:param class product: A product type for the request.
:param list subjects: A list of subjects for the request.
:yields: The requested products.
"""
request = self._scheduler.execution_request([product], subjects)
result = self.execute(request)
if result.error:
raise result.error
result_items = self._scheduler.root_entries(request).items()
# State validation.
unknown_state_types = tuple(
type(state) for _, state in result_items if type(state) not in (Throw, Return)
)
if unknown_state_types:
State.raise_unrecognized(unknown_state_types)
# Throw handling.
# TODO: See https://github.com/pantsbuild/pants/issues/3912
throw_roots = tuple(root for root, state in result_items if type(state) is Throw)
if throw_roots:
cumulative_trace = '\n'.join(self._scheduler.trace())
raise ExecutionError('Received unexpected Throw state(s):\n{}'.format(cumulative_trace))
# Return handling.
returns = tuple(state.value for _, state in result_items if type(state) is Return)
for return_value in returns:
for computed_product in maybe_list(return_value, expected_type=product):
yield computed_product
def _index(self, roots):
"""Index from the given roots into the storage provided by the base class.
This is an additive operation: any existing connections involving these nodes are preserved.
"""
all_addresses = set()
new_targets = list()
# Index the ProductGraph.
for node, state in roots.items():
if type(state) is Throw:
trace = 'TODO: restore trace!\n {}'.format(
state) #'\n'.join(self._graph.trace(node))
raise AddressLookupError(
'Build graph construction failed for {}:\n{}'.format(
node, trace))
elif type(state) is not Return:
State.raise_unrecognized(state)
if type(state.value) is not HydratedTargets:
raise TypeError('Expected roots to hold {}; got: {}'.format(
HydratedTargets, type(state.value)))
# We have a successful HydratedTargets value (for a particular input Spec).
for hydrated_target in state.value.dependencies:
target_adaptor = hydrated_target.adaptor
address = target_adaptor.address
all_addresses.add(address)
if address not in self._target_by_address:
new_targets.append(self._index_target(target_adaptor))
# Once the declared dependencies of all targets are indexed, inject their
# additional "traversable_(dependency_)?specs".
deps_to_inject = OrderedSet()
addresses_to_inject = set()
def inject(target, dep_spec, is_dependency):
address = Address.parse(dep_spec,
relative_to=target.address.spec_path)
if not any(address == t.address for t in target.dependencies):
addresses_to_inject.add(address)
if is_dependency:
deps_to_inject.add((target.address, address))
for target in new_targets:
for spec in target.traversable_dependency_specs:
inject(target, spec, is_dependency=True)
for spec in target.traversable_specs:
inject(target, spec, is_dependency=False)
# Inject all addresses, then declare injected dependencies.
self.inject_addresses_closure(addresses_to_inject)
for target_address, dep_address in deps_to_inject:
self.inject_dependency(dependent=target_address,
dependency=dep_address)
return all_addresses
def _index(self, roots):
"""Index from the given roots into the storage provided by the base class.
This is an additive operation: any existing connections involving these nodes are preserved.
"""
all_addresses = set()
new_targets = list()
# Index the ProductGraph.
for node, state in self._graph.walk(roots=roots):
# Locate nodes that contain LegacyTarget values.
if type(state) is Throw:
trace = '\n'.join(self._graph.trace(node))
raise AddressLookupError(
'Build graph construction failed for {}:\n{}'.format(
node.subject, trace))
elif type(state) is not Return:
State.raise_unrecognized(state)
if node.product is not LegacyTarget:
continue
if type(node) is not TaskNode:
continue
# We have a successfully parsed LegacyTarget, which includes its declared dependencies.
address = state.value.adaptor.address
all_addresses.add(address)
if address not in self._target_by_address:
new_targets.append(self._index_target(state.value))
# Once the declared dependencies of all targets are indexed, inject their
# additional "traversable_(dependency_)?specs".
deps_to_inject = OrderedSet()
addresses_to_inject = set()
def inject(target, dep_spec, is_dependency):
address = Address.parse(dep_spec,
relative_to=target.address.spec_path)
if not any(address == t.address for t in target.dependencies):
addresses_to_inject.add(address)
if is_dependency:
deps_to_inject.add((target.address, address))
for target in new_targets:
for spec in target.traversable_dependency_specs:
inject(target, spec, is_dependency=True)
for spec in target.traversable_specs:
inject(target, spec, is_dependency=False)
# Inject all addresses, then declare injected dependencies.
self.inject_addresses_closure(addresses_to_inject)
for target_address, dep_address in deps_to_inject:
self.inject_dependency(dependent=target_address,
dependency=dep_address)
return all_addresses
def step(self, step_context):
select_state = step_context.select_for(Select(Files), self.subject, self.variants)
if type(select_state) in {Waiting, Noop, Throw}:
return select_state
elif type(select_state) is not Return:
State.raise_unrecognized(select_state)
file_list = select_state.value
snapshot = _create_snapshot_archive(file_list, step_context)
return Return(snapshot)
def _index(self, roots):
"""Index from the given roots into the storage provided by the base class.
This is an additive operation: any existing connections involving these nodes are preserved.
"""
all_addresses = set()
new_targets = list()
# Index the ProductGraph.
for node, state in self._graph.walk(roots=roots):
# Locate nodes that contain LegacyTarget values.
if type(state) is Throw:
trace = '\n'.join(self._graph.trace(node))
raise AddressLookupError(
'Build graph construction failed for {}:\n{}'.format(node.subject, trace))
elif type(state) is not Return:
State.raise_unrecognized(state)
if node.product is not LegacyTarget:
continue
if type(node) is not TaskNode:
continue
# We have a successfully parsed LegacyTarget, which includes its declared dependencies.
address = state.value.adaptor.address
all_addresses.add(address)
if address not in self._target_by_address:
new_targets.append(self._index_target(state.value))
# Once the declared dependencies of all targets are indexed, inject their
# additional "traversable_(dependency_)?specs".
deps_to_inject = set()
addresses_to_inject = set()
def inject(target, dep_spec, is_dependency):
address = Address.parse(dep_spec, relative_to=target.address.spec_path)
if not any(address == t.address for t in target.dependencies):
addresses_to_inject.add(address)
if is_dependency:
deps_to_inject.add((target.address, address))
for target in new_targets:
for spec in target.traversable_dependency_specs:
inject(target, spec, is_dependency=True)
for spec in target.traversable_specs:
inject(target, spec, is_dependency=False)
# Inject all addresses, then declare injected dependencies.
self.inject_addresses_closure(addresses_to_inject)
for target_address, dep_address in deps_to_inject:
self.inject_dependency(dependent=target_address, dependency=dep_address)
return all_addresses
def _index(self, roots):
"""Index from the given roots into the storage provided by the base class.
This is an additive operation: any existing connections involving these nodes are preserved.
"""
all_addresses = set()
new_targets = list()
# Index the ProductGraph.
for node, state in roots.items():
if type(state) is Throw:
trace = 'TODO: restore trace!\n {}'.format(state) #'\n'.join(self._graph.trace(node))
raise AddressLookupError(
'Build graph construction failed for {}:\n{}'.format(node, trace))
elif type(state) is not Return:
State.raise_unrecognized(state)
if type(state.value) is not HydratedTargets:
raise TypeError('Expected roots to hold {}; got: {}'.format(
HydratedTargets, type(state.value)))
# We have a successful HydratedTargets value (for a particular input Spec).
for hydrated_target in state.value.dependencies:
target_adaptor = hydrated_target.adaptor
address = target_adaptor.address
all_addresses.add(address)
if address not in self._target_by_address:
new_targets.append(self._index_target(target_adaptor))
# Once the declared dependencies of all targets are indexed, inject their
# additional "traversable_(dependency_)?specs".
deps_to_inject = OrderedSet()
addresses_to_inject = set()
def inject(target, dep_spec, is_dependency):
address = Address.parse(dep_spec, relative_to=target.address.spec_path)
if not any(address == t.address for t in target.dependencies):
addresses_to_inject.add(address)
if is_dependency:
deps_to_inject.add((target.address, address))
for target in new_targets:
for spec in target.traversable_dependency_specs:
inject(target, spec, is_dependency=True)
for spec in target.traversable_specs:
inject(target, spec, is_dependency=False)
# Inject all addresses, then declare injected dependencies.
self.inject_addresses_closure(addresses_to_inject)
for target_address, dep_address in deps_to_inject:
self.inject_dependency(dependent=target_address, dependency=dep_address)
return all_addresses
def products_request(self, products, subjects):
"""Executes a request for multiple products for some subjects, and returns the products.
:param list products: A list of product type for the request.
:param list subjects: A list of subjects for the request.
:returns: A dict from product type to lists of products each with length matching len(subjects).
"""
request = self.execution_request(products, subjects)
result = self.execute(request)
if result.error:
raise result.error
# State validation.
unknown_state_types = tuple(
type(state) for _, state in result.root_products
if type(state) not in (Throw, Return))
if unknown_state_types:
State.raise_unrecognized(unknown_state_types)
# Throw handling.
# TODO: See https://github.com/pantsbuild/pants/issues/3912
throw_root_states = tuple(state for root, state in result.root_products
if type(state) is Throw)
if throw_root_states:
unique_exceptions = tuple(set(t.exc for t in throw_root_states))
if self._scheduler.include_trace_on_error:
cumulative_trace = '\n'.join(self.trace(request))
raise ExecutionError(
'Received unexpected Throw state(s):\n{}'.format(
cumulative_trace),
unique_exceptions,
)
if len(unique_exceptions) == 1:
raise throw_root_states[0].exc
else:
raise ExecutionError(
'Multiple exceptions encountered:\n {}'.format(
'\n '.join('{}: {}'.format(type(t).__name__, str(t))
for t in unique_exceptions)),
unique_exceptions)
# Everything is a Return: we rely on the fact that roots are ordered to preserve subject
# order in output lists.
product_results = defaultdict(list)
for (_, product), state in result.root_products:
product_results[product].append(state.value)
return product_results
def product_request(self, product, subjects):
"""Executes a request for a singular product type from the scheduler for one or more subjects
and yields the products.
:param class product: A product type for the request.
:param list subjects: A list of subjects for the request.
:yields: The requested products.
"""
request = self._scheduler.execution_request([product], subjects)
result = self.execute(request)
if result.error:
raise result.error
result_items = self._scheduler.root_entries(request).items()
# State validation.
unknown_state_types = tuple(
type(state) for _, state in result_items
if type(state) not in (Throw, Return, Noop))
if unknown_state_types:
State.raise_unrecognized(unknown_state_types)
# Throw handling.
throw_roots = tuple(root for root, state in result_items
if type(state) is Throw)
if throw_roots:
cumulative_trace = '\n'.join(
'\n'.join(self._scheduler.product_graph.trace(root))
for root in throw_roots)
stringified_throw_roots = ', '.join(str(x) for x in throw_roots)
raise ExecutionError(
'received unexpected Throw state(s) for root(s): {}\n{}'.
format(stringified_throw_roots, cumulative_trace))
# Noop handling.
noop_roots = tuple(root for root, state in result_items
if type(state) is Noop)
if noop_roots:
raise ExecutionError(
'received unexpected Noop state(s) for the following root(s): {}'
.format(noop_roots))
# Return handling.
returns = tuple(state.value for _, state in result_items
if type(state) is Return)
for return_value in returns:
for computed_product in maybe_list(return_value,
expected_type=product):
yield computed_product
def products_request(self, products, subjects):
"""Executes a request for multiple products for some subjects, and returns the products.
:param list products: A list of product type for the request.
:param list subjects: A list of subjects for the request.
:returns: A dict from product type to lists of products each with length matching len(subjects).
"""
request = self.execution_request(products, subjects)
result = self.execute(request)
if result.error:
raise result.error
# State validation.
unknown_state_types = tuple(
type(state) for _, state in result.root_products if type(state) not in (Throw, Return)
)
if unknown_state_types:
State.raise_unrecognized(unknown_state_types)
# Throw handling.
# TODO: See https://github.com/pantsbuild/pants/issues/3912
throw_root_states = tuple(state for root, state in result.root_products if type(state) is Throw)
if throw_root_states:
unique_exceptions = tuple(set(t.exc for t in throw_root_states))
if self._scheduler.include_trace_on_error:
cumulative_trace = '\n'.join(self.trace(request))
raise ExecutionError(
'Received unexpected Throw state(s):\n{}'.format(cumulative_trace),
unique_exceptions,
)
if len(unique_exceptions) == 1:
raise throw_root_states[0].exc
else:
raise ExecutionError(
'Multiple exceptions encountered:\n {}'.format(
'\n '.join('{}: {}'.format(type(t).__name__, str(t)) for t in unique_exceptions)),
unique_exceptions
)
# Everything is a Return: we rely on the fact that roots are ordered to preserve subject
# order in output lists.
product_results = defaultdict(list)
for (_, product), state in result.root_products:
product_results[product].append(state.value)
return product_results
def product_request(self, product, subjects):
"""Executes a request for a singular product type from the scheduler for one or more subjects
and yields the products.
:param class product: A product type for the request.
:param list subjects: A list of subjects for the request.
:yields: The requested products.
"""
request = self._scheduler.execution_request([product], subjects)
result = self.execute(request)
if result.error:
raise result.error
result_items = self._scheduler.root_entries(request).items()
# State validation.
unknown_state_types = tuple(
type(state) for _, state in result_items if type(state) not in (Throw, Return, Noop)
)
if unknown_state_types:
State.raise_unrecognized(unknown_state_types)
# Throw handling.
throw_roots = tuple(root for root, state in result_items if type(state) is Throw)
if throw_roots:
cumulative_trace = '\n'.join(
'\n'.join(self._scheduler.product_graph.trace(root)) for root in throw_roots
)
stringified_throw_roots = ', '.join(str(x) for x in throw_roots)
raise ExecutionError('received unexpected Throw state(s) for root(s): {}\n{}'
.format(stringified_throw_roots, cumulative_trace))
# Noop handling.
noop_roots = tuple(root for root, state in result_items if type(state) is Noop)
if noop_roots:
raise ExecutionError('received unexpected Noop state(s) for the following root(s): {}'
.format(noop_roots))
# Return handling.
returns = tuple(state.value for _, state in result_items if type(state) is Return)
for return_value in returns:
for computed_product in maybe_list(return_value, expected_type=product):
yield computed_product
def update_state(self, node, state):
"""Updates the Node with the given State, creating any Nodes which do not already exist."""
entry = self._ensure_entry(node)
if entry.state is not None:
# It's important not to allow state changes on completed Nodes, because that invariant
# is used in cycle detection to avoid walking into completed Nodes.
raise CompletedNodeException('Node {} is already completed:\n {}\n {}'
.format(node, entry.state, state))
if type(state) in [Return, Throw, Noop]:
# Validate that a completed Node depends only on other completed Nodes.
for dep in entry.dependencies:
if dep.state is None:
raise IncompleteDependencyException(
'Cannot complete {} with {} while it has an incomplete dep:\n {}'
.format(node, state, dep.node))
entry.state = state
elif type(state) is Waiting:
self._add_dependencies(entry, state.dependencies)
else:
raise State.raise_unrecognized(state)
def step(self, step_context):
waiting_nodes = []
# Get the binary.
binary_select_node = step_context.select_node(Select(self.snapshotted_process.binary_type),
subject=self.subject,
variants=self.variants)
binary_state = step_context.get(binary_select_node)
if type(binary_state) is Throw:
return binary_state
elif type(binary_state) is Waiting:
waiting_nodes.append(binary_select_node)
elif type(binary_state) is Noop:
return Noop("Couldn't find binary: {}".format(binary_state))
elif type(binary_state) is not Return:
State.raise_unrecognized(binary_state)
# Create the request from the request callback after resolving its input clauses.
input_select_nodes = [step_context.select_node(s, self.subject, self.variants)
for s in self.snapshotted_process.input_selectors]
input_values = []
for input_selector, input_select_node in zip(self.snapshotted_process.input_selectors, input_select_nodes):
sn_state = step_context.get(input_select_node)
if type(sn_state) is Waiting:
waiting_nodes.extend(sn_state.dependencies)
elif type(sn_state) is Return:
input_values.append(sn_state.value)
elif type(sn_state) is Noop:
if input_selector.optional:
input_values.append(None)
else:
return Noop('Was missing value for (at least) input {}'.format(input_select_node))
elif type(sn_state) is Throw:
return sn_state
else:
State.raise_unrecognized(sn_state)
if waiting_nodes:
return Waiting(waiting_nodes)
# Now that we've returned on waiting, we can assume that the binary has a value, and that we're ready to do input
# conversion.
binary_value = binary_state.value
try:
process_request = self.snapshotted_process.input_conversion(*input_values)
except Exception as e:
return Throw(e)
# Request snapshots for the snapshot_subjects from the process request.
if process_request.snapshot_subjects:
snapshot_subjects_node = step_context.select_node(SelectDependencies(Snapshot,
SnapshottedProcessRequest,
'snapshot_subjects'),
process_request,
self.variants)
snapshot_subjects_state = step_context.get(snapshot_subjects_node)
if type(snapshot_subjects_state) is not Return:
return snapshot_subjects_state
# TODO resolve what to do with output files, then make these tmp dirs cleaned up.
with temporary_dir(cleanup=False) as sandbox_dir:
if process_request.snapshot_subjects:
snapshots_and_subjects = zip(snapshot_subjects_state.value, process_request.snapshot_subjects)
for snapshot, subject in snapshots_and_subjects:
_extract_snapshot(step_context, snapshot, sandbox_dir, subject)
# All of the snapshots have been checked out now.
if process_request.directories_to_create:
for d in process_request.directories_to_create:
safe_mkdir(os.path.join(sandbox_dir, d))
popen = self._run_command(binary_value, sandbox_dir, process_request)
process_result = SnapshottedProcessResult(popen.stdout.read(), popen.stderr.read(), popen.returncode)
if process_result.exit_code != 0:
return Throw(Exception('Running {} failed with non-zero exit code: {}'.format(binary_value,
process_result.exit_code)))
try:
converted_output = self.snapshotted_process.output_conversion(process_result, sandbox_dir)
except Exception as e:
return Throw(e)
return Return(converted_output)
def step(self, step_context):
waiting_nodes = []
# Get the binary.
binary_state = step_context.select_for(Select(self.snapshotted_process.binary_type),
subject=self.subject,
variants=self.variants)
if type(binary_state) is Throw:
return binary_state
elif type(binary_state) is Waiting:
waiting_nodes.extend(binary_state.dependencies)
elif type(binary_state) is Noop:
return Noop("Couldn't find binary: {}".format(binary_state))
elif type(binary_state) is not Return:
State.raise_unrecognized(binary_state)
# Create the request from the request callback after resolving its input clauses.
input_values = []
for input_selector in self.snapshotted_process.input_selectors:
sn_state = step_context.select_for(input_selector, self.subject, self.variants)
if type(sn_state) is Waiting:
waiting_nodes.extend(sn_state.dependencies)
elif type(sn_state) is Return:
input_values.append(sn_state.value)
elif type(sn_state) is Noop:
if input_selector.optional:
input_values.append(None)
else:
return Noop('Was missing value for (at least) input {}'.format(input_selector))
elif type(sn_state) is Throw:
return sn_state
else:
State.raise_unrecognized(sn_state)
if waiting_nodes:
return Waiting(waiting_nodes)
# Now that we've returned on waiting, we can assume that relevant inputs have values.
try:
process_request = self.snapshotted_process.input_conversion(*input_values)
except Exception as e:
return Throw(e)
# Request snapshots for the snapshot_subjects from the process request.
snapshot_subjects_value = []
if process_request.snapshot_subjects:
snapshot_subjects_state = step_context.select_for(SelectDependencies(Snapshot,
SnapshottedProcessRequest,
'snapshot_subjects',
field_types=(Files,)),
process_request,
self.variants)
if type(snapshot_subjects_state) is not Return:
return snapshot_subjects_state
snapshot_subjects_value = snapshot_subjects_state.value
# Ready to run.
execution = _Process(step_context.snapshot_archive_root,
process_request,
binary_state.value,
snapshot_subjects_value,
self.snapshotted_process.output_conversion)
return Runnable(_execute, (execution,))
def get_state(self, state_key):
"""The inverse of put_state: get a State given its Key."""
return State.from_components(
tuple(
self.get(Key.create_from_digest(d))
for d in self.get(state_key)))
def get_state(self, state_key): """The inverse of put_state: get a State given its Key.""" return State.from_components(tuple(self.get(Key.create_from_digest(d)) for d in self.get(state_key)))