def assert_root(self, walk, node, return_value):
"""Asserts that the first Node in a walk was a DependenciesNode with the single given result."""
root, root_state = walk[0]
self.assertEquals(type(root), DependenciesNode)
self.assertEquals(Return([return_value]), root_state)
self.assertIn(
(node, Return(return_value)),
[(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)])
def _run_and_return_roots(self, session, execution_request):
raw_roots = self._native.lib.scheduler_execute(self._scheduler, session, execution_request)
remaining_runtime_exceptions_to_capture = list(self._native.consume_cffi_extern_method_runtime_exceptions())
try:
roots = []
for raw_root in self._native.unpack(raw_roots.nodes_ptr, raw_roots.nodes_len):
# Check if there were any uncaught exceptions within rules that were executed.
remaining_runtime_exceptions_to_capture.extend(self._native.consume_cffi_extern_method_runtime_exceptions())
if raw_root.is_throw:
state = Throw(self._from_value(raw_root.handle))
elif raw_root.handle == self._native.ffi.NULL:
# NB: We expect all NULL handles to correspond to uncaught exceptions which are collected
# in `self._native._peek_cffi_extern_method_runtime_exceptions()`!
if not remaining_runtime_exceptions_to_capture:
raise ExecutionError('Internal logic error in scheduler: expected more elements in '
'`self._native._peek_cffi_extern_method_runtime_exceptions()`.')
matching_runtime_exception = remaining_runtime_exceptions_to_capture.pop(0)
state = Throw(matching_runtime_exception)
else:
state = Return(self._from_value(raw_root.handle))
roots.append(state)
finally:
self._native.lib.nodes_destroy(raw_roots)
if remaining_runtime_exceptions_to_capture:
raise ExecutionError('Internal logic error in scheduler: expected elements in '
'`self._native._peek_cffi_extern_method_runtime_exceptions()`.')
return roots
def test_serial_execution_simple(self):
request = self.request(['compile'], self.java)
result = self.scheduler.execute(request)
self.scheduler.visualize_graph_to_file(request, 'blah/run.0.dot')
self.assertEqual(Return(Classpath(creator='javac')),
result.root_products[0][1])
self.assertIsNone(result.error)
def root_entries(self, execution_request):
"""Returns the roots for the given ExecutionRequest as a dict of tuples to State."""
with self._product_graph_lock:
if self._execution_request is not execution_request:
raise AssertionError(
"Multiple concurrent executions are not supported! {} vs {}"
.format(self._execution_request, execution_request))
raw_roots = self._native.gc(
self._native.lib.execution_roots(self._scheduler),
self._native.lib.nodes_destroy)
roots = {}
for root, raw_root in zip(
execution_request.roots,
self._native.unpack(raw_roots.nodes_ptr,
raw_roots.nodes_len)):
if raw_root.union_tag is 0:
state = None
elif raw_root.union_tag is 1:
state = Return(self._from_value(raw_root.union_return))
elif raw_root.union_tag is 2:
state = Throw("Failed")
elif raw_root.union_tag is 3:
state = Throw("Nooped")
else:
raise ValueError(
'Unrecognized State type `{}` on: {}'.format(
raw_root.union_tag, raw_root))
roots[root] = state
return roots
def test_serial_execution_simple(self):
request = self.request([Classpath], self.java)
result = self.scheduler.execute(request)
with temporary_dir() as tempdir:
self.scheduler.visualize_graph_to_file(os.path.join(tempdir, 'run.0.dot'))
self.assertEqual(Return(Classpath(creator='javac')), result.root_products[0][1])
self.assertIsNone(result.error)
def test_disallow_completing_with_incomplete_deps(self):
self.pg.add_dependencies('A', ['B'])
self.pg.add_dependencies('B', ['C'])
with self.assertRaises(
'TODO: IncompleteDependencyException: These tests should be ported to native tests.'
):
self.pg.complete_node('A', Return('done!'))
def assert_engine(self, engine):
result = engine.execute(self.request(['compile'], self.java))
self.assertEqual(
{
SelectNode(self.java, Classpath, None, None):
Return(Classpath(creator='javac'))
}, result.root_products)
self.assertIsNone(result.error)
def execute():
try:
runnable = storage.get_state(runnable_key)
result = Return(runnable.func(*runnable.args))
if debug:
_try_pickle(result)
result_key = storage.put_state(result)
if is_cacheable:
cache.put(runnable_key, result)
except Exception as e:
result_key = storage.put_state(Throw(e))
return result_key
def step(self, step_context):
select_state = step_context.select_for(Select(Files), self.subject, self.variants)
if type(select_state) in {Waiting, Noop, Throw}:
return select_state
elif type(select_state) is not Return:
State.raise_unrecognized(select_state)
file_list = select_state.value
snapshot = _create_snapshot_archive(file_list, step_context)
return Return(snapshot)
def _run_and_return_roots(self, session, execution_request):
raw_roots = self._native.lib.scheduler_execute(self._scheduler, session, execution_request)
try:
roots = []
for raw_root in self._native.unpack(raw_roots.nodes_ptr, raw_roots.nodes_len):
if raw_root.is_throw:
state = Throw(self._from_value(raw_root.handle))
else:
state = Return(self._from_value(raw_root.handle))
roots.append(state)
finally:
self._native.lib.nodes_destroy(raw_roots)
return roots
def _execute_step(self, step_entry, runnable):
"""A function to help support local step execution.
:param step_entry: Entry that the step is for.
:param runnable: Runnable to execute.
"""
key, result = self._maybe_cache_get(step_entry, runnable)
if result is None:
try:
result = Return(runnable.func(*runnable.args))
self._maybe_cache_put(key, result)
except Exception as e:
result = Throw(e)
return step_entry, result
def reduce(self, execution_request):
generator = self._scheduler.schedule(execution_request)
for runnable_batch in generator:
completed = []
for entry, runnable in runnable_batch:
key, result = self._maybe_cache_get(entry, runnable)
if result is None:
try:
result = Return(runnable.func(*runnable.args))
self._maybe_cache_put(key, result)
except Exception as e:
result = Throw(e)
completed.append((entry, result))
generator.send(completed)
def test_state_roundtrips(self):
states = [
Return('a'),
Throw(PickleableException()),
Waiting([TaskNode(None, None, None)]),
Runnable(_runnable, ('an arg', )),
Noop('nada {}', ('op', ))
]
with closing(self.storage) as storage:
for state in states:
key = storage.put_state(state)
actual = storage.get_state(key)
self.assertEquals(state, actual)
self.assertEquals(key, storage.put_state(actual))
def run_and_return_roots(self, execution_request):
raw_roots = self._native.lib.execution_execute(self._scheduler, execution_request)
try:
roots = []
for raw_root in self._native.unpack(raw_roots.nodes_ptr, raw_roots.nodes_len):
if raw_root.state_tag is 1:
state = Return(self._from_value(raw_root.state_value))
elif raw_root.state_tag in (2, 3, 4):
state = Throw(self._from_value(raw_root.state_value))
else:
raise ValueError(
'Unrecognized State type `{}` on: {}'.format(raw_root.state_tag, raw_root))
roots.append(state)
finally:
self._native.lib.nodes_destroy(raw_roots)
return roots
def root_entries(self, execution_request):
raw_roots = self._native.lib.execution_roots(self._scheduler)
try:
roots = {}
for root, raw_root in zip(execution_request.roots,
self._native.unpack(raw_roots.nodes_ptr,
raw_roots.nodes_len)):
if raw_root.state_tag is 0:
state = None
elif raw_root.state_tag is 1:
state = Return(self._from_value(raw_root.state_value))
elif raw_root.state_tag is 2:
state = Throw(self._from_value(raw_root.state_value))
elif raw_root.state_tag is 3:
state = Throw(self._from_value(raw_root.state_value))
else:
raise ValueError(
'Unrecognized State type `{}` on: {}'.format(raw_root.state_tag, raw_root))
roots[root] = state
finally:
self._native.lib.nodes_destroy(raw_roots)
return roots
def test_disallow_completed_state_change(self):
self.pg.complete_node('A', Return('done!'))
with self.assertRaises(CompletedNodeException):
self.pg.add_dependencies('A', ['B'])
def test_disallow_completing_with_incomplete_deps(self):
self.pg.update_state('A', Waiting(['B']))
self.pg.update_state('B', Waiting(['C']))
with self.assertRaises(IncompleteDependencyException):
self.pg.update_state('A', Return('done!'))
def test_disallow_completed_state_change(self):
self.pg.complete_node('A', Return('done!'))
with self.assertRaises(
'TODO: CompletedNodeException: These tests should be ported to native tests.'
):
self.pg.add_dependencies('A', ['B'])
def assert_engine(self, engine):
result = engine.execute(self.request(['compile'], self.java))
self.scheduler.visualize_graph_to_file('blah/run.0.dot')
self.assertEqual([Return(Classpath(creator='javac'))],
result.root_products.values())
self.assertIsNone(result.error)
def assert_engine(self, engine):
result = engine.execute(self.request(['compile'], self.java))
self.assertEqual([Return(Classpath(creator='javac'))],
result.root_products.values())
self.assertIsNone(result.error)
def test_disallow_completed_state_change(self):
self.pg.update_state('A', Return('done!'))
with self.assertRaises(CompletedNodeException):
self.pg.update_state('A', Waiting(['B']))
def assert_root(self, root, subject, return_value):
"""Asserts that the given root has the given result."""
self.assertEquals(subject, root[0][0])
self.assertEquals(Return(return_value), root[1])
def test_disallow_completing_with_incomplete_deps(self):
self.pg.add_dependencies('A', ['B'])
self.pg.add_dependencies('B', ['C'])
with self.assertRaises(IncompleteDependencyException):
self.pg.complete_node('A', Return('done!'))