def _create_step(self, node):
"""Creates a Step and Promise with the currently available dependencies of the given Node.
If the dependencies of a Node are not available, returns None.
TODO: Content addressing node and its dependencies should only happen if node is cacheable
or in a multi-process environment.
"""
Node.validate_node(node)
# See whether all of the dependencies for the node are available.
deps = dict()
for dep in self._product_graph.dependencies_of(node):
state = self._product_graph.state(dep)
if state is None:
return None
deps[dep] = state
# Additionally, include Noops for any dependencies that were cyclic.
for dep in self._product_graph.cyclic_dependencies_of(node):
noop_state = Noop('Dep from {} to {} would cause a cycle.'.format(node, dep))
deps[dep] = noop_state
# Ready.
self._step_id += 1
return (StepRequest(self._step_id, node, deps, self._project_tree), Promise())
def _create_step(self, node_entry):
"""Creates a Step and Promise with the currently available dependencies of the given Node.
If the dependencies of a Node are not available, returns None.
TODO: Content addressing node and its dependencies should only happen if node is cacheable
or in a multi-process environment.
"""
Node.validate_node(node_entry.node)
# See whether all of the dependencies for the node are available.
deps = dict()
for dep_entry in node_entry.dependencies:
if not dep_entry.is_complete:
return None
deps[dep_entry.node] = dep_entry.state
# Additionally, include Noops for any dependencies that were cyclic.
for dep in node_entry.cyclic_dependencies:
deps[dep] = Noop.cycle(node_entry.node, dep)
# Ready.
self._step_id += 1
step_request = StepRequest(self._step_id,
node_entry.node,
deps,
self._inline_nodes,
self._project_tree)
return (step_request, Promise())
def _create_step(self, node_entry):
"""Creates a Step and Promise with the currently available dependencies of the given Node.
If the dependencies of a Node are not available, returns None.
TODO: Content addressing node and its dependencies should only happen if node is cacheable
or in a multi-process environment.
"""
Node.validate_node(node_entry.node)
# See whether all of the dependencies for the node are available.
deps = dict()
for dep_entry in node_entry.dependencies:
if not dep_entry.is_complete:
return None
deps[dep_entry.node] = dep_entry.state
# Additionally, include Noops for any dependencies that were cyclic.
for dep in node_entry.cyclic_dependencies:
deps[dep] = Noop.cycle(node_entry.node, dep)
# Ready.
self._step_id += 1
step_request = StepRequest(self._step_id,
node_entry.node,
deps,
self._inline_nodes,
self._project_tree)
return (step_request, Promise())
def _trace(entry, level):
if type(entry.state) in (Noop, Return):
return
yield '{}{}'.format(' ' * level, entry.state)
for dep in entry.cyclic_dependencies:
yield '{}{}'.format(' ' * level, Noop.cycle(entry.node, dep))
for dep_entry in entry.dependencies:
for l in _trace(dep_entry, level+1):
yield l
def _trace(entry, level):
if type(entry.state) in (Noop, Return):
return
yield '{}{}'.format(' ' * level, entry.state)
for dep in entry.cyclic_dependencies:
yield '{}{}'.format(' ' * level, Noop.cycle(entry.node, dep))
for dep_entry in entry.dependencies:
for l in _trace(dep_entry, level + 1):
yield l
def _trace(entry, level):
if is_bottom(entry):
return
traced.add(entry)
yield _format(level, entry, entry.state)
for dep in entry.cyclic_dependencies:
yield _format(level, entry, Noop.cycle(entry.node, dep))
for dep_entry in entry.dependencies:
for l in _trace(dep_entry, level+1):
yield l
def _trace(entry, level):
if is_bottom(entry):
return
traced.add(entry)
yield _format(level, entry, entry.state)
for dep in entry.cyclic_dependencies:
yield _format(level, entry, Noop.cycle(entry.node, dep))
for dep_entry in entry.dependencies:
for l in _trace(dep_entry, level + 1):
yield l
def _trace(entry, level):
if type(entry.state) in (Noop, Return) or entry in traced:
return
traced.add(entry)
yield _format(level, entry.node, entry.state)
for dep in entry.cyclic_dependencies:
yield _format(level, entry.node, Noop.cycle(entry.node, dep))
for dep_entry in entry.dependencies:
for l in _trace(dep_entry, level+1):
yield l
def _trace(entry, level):
if type(entry.state) in (Noop, Return) or entry in traced:
return
traced.add(entry)
yield _format(level, entry.node, entry.state)
for dep in entry.cyclic_dependencies:
yield _format(level, entry.node, Noop.cycle(entry.node, dep))
for dep_entry in entry.dependencies:
for l in _trace(dep_entry, level + 1):
yield l
def test_state_roundtrips(self):
states = [
Return('a'),
Throw(PickleableException()),
Waiting([TaskNode(None, None, None)]),
Runnable(_runnable, ('an arg', )),
Noop('nada {}', ('op', ))
]
with closing(self.storage) as storage:
for state in states:
key = storage.put_state(state)
actual = storage.get_state(key)
self.assertEquals(state, actual)
self.assertEquals(key, storage.put_state(actual))
def _attempt_run_step(self, node_entry):
"""Attempt to run a Step with the currently available dependencies of the given Node.
If the currently declared dependencies of a Node are not yet available, returns None. If
they are available, runs a Step and returns the resulting State.
"""
# See whether all of the dependencies for the node are available.
if any(not dep_entry.is_complete for dep_entry in node_entry.dependencies):
return None
# Collect the deps.
deps = dict()
for dep_entry in node_entry.dependencies:
deps[dep_entry.node] = dep_entry.state
# Additionally, include Noops for any dependencies that were cyclic.
for dep in node_entry.cyclic_dependencies:
deps[dep] = Noop.cycle(node_entry.node, dep)
# Run.
step_context = StepContext(self.node_builder, self._project_tree, deps, self._inline_nodes)
return node_entry.node.step(step_context)
def _attempt_run_step(self, node_entry):
"""Attempt to run a Step with the currently available dependencies of the given Node.
If the currently declared dependencies of a Node are not yet available, returns None. If
they are available, runs a Step and returns the resulting State.
"""
# See whether all of the dependencies for the node are available.
if any(not dep_entry.is_complete for dep_entry in node_entry.dependencies):
return None
# Collect the deps.
deps = dict()
for dep_entry in node_entry.dependencies:
deps[dep_entry.node] = dep_entry.state
# Additionally, include Noops for any dependencies that were cyclic.
for dep in node_entry.cyclic_dependencies:
deps[dep] = Noop.cycle(node_entry.node, dep)
# Run.
step_context = StepContext(self.node_builder, self._project_tree, deps, self._inline_nodes)
return node_entry.node.step(step_context)
def step(self, step_context):
waiting_nodes = []
# Get the binary.
binary_state = step_context.select_for(Select(self.snapshotted_process.binary_type),
subject=self.subject,
variants=self.variants)
if type(binary_state) is Throw:
return binary_state
elif type(binary_state) is Waiting:
waiting_nodes.extend(binary_state.dependencies)
elif type(binary_state) is Noop:
return Noop("Couldn't find binary: {}".format(binary_state))
elif type(binary_state) is not Return:
State.raise_unrecognized(binary_state)
# Create the request from the request callback after resolving its input clauses.
input_values = []
for input_selector in self.snapshotted_process.input_selectors:
sn_state = step_context.select_for(input_selector, self.subject, self.variants)
if type(sn_state) is Waiting:
waiting_nodes.extend(sn_state.dependencies)
elif type(sn_state) is Return:
input_values.append(sn_state.value)
elif type(sn_state) is Noop:
if input_selector.optional:
input_values.append(None)
else:
return Noop('Was missing value for (at least) input {}'.format(input_selector))
elif type(sn_state) is Throw:
return sn_state
else:
State.raise_unrecognized(sn_state)
if waiting_nodes:
return Waiting(waiting_nodes)
# Now that we've returned on waiting, we can assume that relevant inputs have values.
try:
process_request = self.snapshotted_process.input_conversion(*input_values)
except Exception as e:
return Throw(e)
# Request snapshots for the snapshot_subjects from the process request.
snapshot_subjects_value = []
if process_request.snapshot_subjects:
snapshot_subjects_state = step_context.select_for(SelectDependencies(Snapshot,
SnapshottedProcessRequest,
'snapshot_subjects',
field_types=(Files,)),
process_request,
self.variants)
if type(snapshot_subjects_state) is not Return:
return snapshot_subjects_state
snapshot_subjects_value = snapshot_subjects_state.value
# Ready to run.
execution = _Process(step_context.snapshot_archive_root,
process_request,
binary_state.value,
snapshot_subjects_value,
self.snapshotted_process.output_conversion)
return Runnable(_execute, (execution,))