def test_virtual_placement(placer):
machine = virtual_machine(width=8, height=8)
graph = MachineGraph("Test")
virtual_vertex = MachineSpiNNakerLinkVertex(spinnaker_link_id=0)
graph.add_vertex(virtual_vertex)
extended_machine = MallocBasedChipIdAllocator()(machine, graph)
n_keys_map = DictBasedMachinePartitionNKeysMap()
inputs = {
"MemoryExtendedMachine": machine,
"MemoryMachine": machine,
"MemoryMachineGraph": graph,
"PlanNTimeSteps": 1000,
"MemoryMachinePartitionNKeysMap": n_keys_map
}
algorithms = [placer]
xml_paths = []
executor = PACMANAlgorithmExecutor(algorithms, [], inputs, [], [], [],
xml_paths)
executor.execute_mapping()
placements = executor.get_item("MemoryPlacements")
placement = placements.get_placement_of_vertex(virtual_vertex)
chip = extended_machine.get_chip_at(placement.x, placement.y)
assert chip.virtual
def test_failing_function_workflow(self):
inputs = {}
executor = PACMANAlgorithmExecutor(
algorithms=["exception_when_called"],
optional_algorithms=[], inputs=inputs, required_outputs=[],
tokens=[], required_output_tokens=[])
with self.assertRaises(SpecificException):
executor.execute_mapping()
def test_failing_function_workflow(self):
inputs = {}
executor = PACMANAlgorithmExecutor(
algorithms=["exception_when_called"],
optional_algorithms=[],
inputs=inputs,
required_outputs=[],
tokens=[],
required_output_tokens=[])
with self.assertRaises(SpecificException):
executor.execute_mapping()
def _do_test(self, placer):
machine = virtual_machine(width=8, height=8)
graph = MachineGraph("Test")
vertices = [
SimpleMachineVertex(ResourceContainer(), label="v{}".format(i))
for i in range(100)
]
for vertex in vertices:
graph.add_vertex(vertex)
same_vertices = [
SimpleMachineVertex(ResourceContainer(), label="same{}".format(i))
for i in range(10)
]
random.seed(12345)
for vertex in same_vertices:
graph.add_vertex(vertex)
for _i in range(0, random.randint(1, 5)):
vertex.add_constraint(
SameChipAsConstraint(
vertices[random.randint(0, 99)]))
n_keys_map = DictBasedMachinePartitionNKeysMap()
inputs = {
"MemoryExtendedMachine": machine,
"MemoryMachine": machine,
"MemoryMachineGraph": graph,
"PlanNTimeSteps": None,
"MemoryMachinePartitionNKeysMap": n_keys_map
}
algorithms = [placer]
xml_paths = []
executor = PACMANAlgorithmExecutor(
algorithms, [], inputs, [], [], [], xml_paths)
executor.execute_mapping()
placements = executor.get_item("MemoryPlacements")
for same in same_vertices:
print("{0.vertex.label}, {0.x}, {0.y}, {0.p}: {1}".format(
placements.get_placement_of_vertex(same),
["{0.vertex.label}, {0.x}, {0.y}, {0.p}".format(
placements.get_placement_of_vertex(constraint.vertex))
for constraint in same.constraints]))
placement = placements.get_placement_of_vertex(same)
for constraint in same.constraints:
if isinstance(constraint, SameChipAsConstraint):
other_placement = placements.get_placement_of_vertex(
constraint.vertex)
self.assertTrue(
other_placement.x == placement.x and
other_placement.y == placement.y,
"Vertex was not placed on the same chip as requested")
def test_external_algorithm(self):
if not os.access("/bin/sh", os.X_OK):
raise self.skipTest("need Bourne shell to run this test")
fd, name = tempfile.mkstemp()
inputs = {"ExampleFilePath": name}
xmlfile = os.path.join(os.path.dirname(__file__), "test_algos.xml")
executor = PACMANAlgorithmExecutor(
algorithms=["SimpleExternal"], xml_paths=[xmlfile],
optional_algorithms=[], inputs=inputs, required_outputs=[],
tokens=[], required_output_tokens=[])
executor.execute_mapping()
self.assertEqual(executor.get_item("Foo"), name)
with os.fdopen(fd) as f:
self.assertEqual(f.read(), "foo\n")
def test_optional_algorithm_token_workflow(self):
""" Tests that a workflow with tokens works with optional algorithms
"""
TestPartTokenOutput1.called = False
TestPartTokenOutput2.called = False
TestWholeTokenRequired.called = False
executor = PACMANAlgorithmExecutor(
algorithms=[],
optional_algorithms=[
"TestPartTokenOutput2", "TestPartTokenOutput1"],
inputs={}, required_outputs=[],
tokens=[], required_output_tokens=["Test"])
executor.execute_mapping()
self.assertTrue(TestPartTokenOutput1.called)
self.assertTrue(TestPartTokenOutput2.called)
def test_basic_workflow(self):
""" Test the basic operation of the executor
"""
TestAlgorithm.called = False
TestNoChangesAlgorithm.called = False
TestAlgorithm3.called = False
inputs = {"TestType1": "TestType1"}
executor = PACMANAlgorithmExecutor(
algorithms=[
"TestAlgorithm3", "TestAlgorithm", "TestNoChangesAlgorithm"],
optional_algorithms=[], inputs=inputs, required_outputs=[],
tokens=[], required_output_tokens=[])
executor.execute_mapping()
self.assertTrue(TestAlgorithm.called)
self.assertTrue(TestNoChangesAlgorithm.called)
self.assertTrue(TestAlgorithm3.called)
def test_recursive_optional_workflow(self):
""" Test the basic operation of the executor
"""
TestRecursiveOptionalAlgorithm.called = False
TestAlgorithm3.called = False
inputs = {"TestType1": "TestType1"}
executor = PACMANAlgorithmExecutor(
algorithms=[
"TestRecursiveOptionalAlgorithm", "TestAlgorithm3"],
optional_algorithms=[], inputs=inputs, required_outputs=[],
tokens=[], required_output_tokens=[])
executor.execute_mapping()
self.assertTrue(TestRecursiveOptionalAlgorithm.called)
self.assertTrue(TestAlgorithm3.called)
self.assertEqual(
[algorithm.algorithm_id for algorithm in executor._algorithms],
["TestRecursiveOptionalAlgorithm", "TestAlgorithm3"])
def test_external_algorithm(self):
if not os.access("/bin/sh", os.X_OK):
raise self.skipTest("need Bourne shell to run this test")
fd, name = tempfile.mkstemp()
inputs = {"ExampleFilePath": name}
xmlfile = os.path.join(os.path.dirname(__file__), "test_algos.xml")
executor = PACMANAlgorithmExecutor(algorithms=["SimpleExternal"],
xml_paths=[xmlfile],
optional_algorithms=[],
inputs=inputs,
required_outputs=[],
tokens=[],
required_output_tokens=[])
executor.execute_mapping()
self.assertEqual(executor.get_item("Foo"), name)
with os.fdopen(fd) as f:
self.assertEqual(f.read(), "foo\n")
def test_failing_incomplete_workflow(self):
inputs = {}
with self.assertRaises(PacmanConfigurationException):
PACMANAlgorithmExecutor(algorithms=["NotThereAtAll"],
optional_algorithms=[],
inputs=inputs,
required_outputs=[],
tokens=[],
required_output_tokens=[])
def test_optional_algorithm_token_workflow(self):
""" Tests that a workflow with tokens works with optional algorithms
"""
TestPartTokenOutput1.called = False
TestPartTokenOutput2.called = False
TestWholeTokenRequired.called = False
executor = PACMANAlgorithmExecutor(algorithms=[],
optional_algorithms=[
"TestPartTokenOutput2",
"TestPartTokenOutput1"
],
inputs={},
required_outputs=[],
tokens=[],
required_output_tokens=["Test"])
executor.execute_mapping()
self.assertTrue(TestPartTokenOutput1.called)
self.assertTrue(TestPartTokenOutput2.called)
def __call__(
self, system_pre_allocated_resources_inputs, max_machine_outputs,
max_machine_available, steps, partitioning_algorithm,
system_pre_alloc_res_algorithms, print_timings, do_timings,
xml_paths, pacman_executor_provenance_path, nengo_operator_graph,
machine_time_step, first_machine_time_step,
machine_time_step_in_seconds):
inputs = dict()
algorithms = list()
# update inputs with the system pre allocated inputs
inputs.update(system_pre_allocated_resources_inputs)
inputs.update(max_machine_outputs)
# add partitioning inputs
inputs["MemoryVirtualMachine"] = max_machine_available
inputs["TotalMachineTimeSteps"] = steps
inputs["NengoOperatorGraph"] = nengo_operator_graph
inputs["MachineTimeStep"] = machine_time_step
inputs["FirstMachineTimeStep"] = first_machine_time_step
inputs["MachineTimeStepInSeconds"] = machine_time_step_in_seconds
# update algorithms with system pre allocated algor's
algorithms.extend(system_pre_alloc_res_algorithms)
algorithms.append(partitioning_algorithm)
outputs = ["MemoryMachineGraph", "NengoGraphMapper"]
# Execute the algorithms
executor = PACMANAlgorithmExecutor(
algorithms=algorithms, optional_algorithms=[],
inputs=inputs, tokens=[],
required_output_tokens=[],
required_outputs=outputs, print_timings=print_timings,
do_timings=do_timings, xml_paths=xml_paths,
provenance_name="nengo_application_graph_to_machine_graph",
provenance_path=pacman_executor_provenance_path)
executor.execute_mapping()
# update spinnaker with app graph
return executor.get_items()
def test_recursive_optional_workflow(self):
""" Test the basic operation of the executor
"""
TestRecursiveOptionalAlgorithm.called = False
TestAlgorithm3.called = False
inputs = {"TestType1": "TestType1"}
executor = PACMANAlgorithmExecutor(
algorithms=["TestRecursiveOptionalAlgorithm", "TestAlgorithm3"],
optional_algorithms=[],
inputs=inputs,
required_outputs=[],
tokens=[],
required_output_tokens=[])
executor.execute_mapping()
self.assertTrue(TestRecursiveOptionalAlgorithm.called)
self.assertTrue(TestAlgorithm3.called)
self.assertEqual(
[algorithm.algorithm_id for algorithm in executor._algorithms],
["TestRecursiveOptionalAlgorithm", "TestAlgorithm3"])
def test_basic_workflow(self):
""" Test the basic operation of the executor
"""
TestAlgorithm.called = False
TestNoChangesAlgorithm.called = False
TestAlgorithm3.called = False
inputs = {"TestType1": "TestType1"}
executor = PACMANAlgorithmExecutor(algorithms=[
"TestAlgorithm3", "TestAlgorithm", "TestNoChangesAlgorithm"
],
optional_algorithms=[],
inputs=inputs,
required_outputs=[],
tokens=[],
required_output_tokens=[])
executor.execute_mapping()
self.assertTrue(TestAlgorithm.called)
self.assertTrue(TestNoChangesAlgorithm.called)
self.assertTrue(TestAlgorithm3.called)
def test_optional_workflow(self):
""" Tests that an optional algorithm that doesn't do anything doesn't
get called
"""
TestAlgorithm.called = False
TestNoChangesAlgorithm.called = False
TestAlgorithm3.called = False
inputs = {"TestType1": "TestType1"}
executor = PACMANAlgorithmExecutor(
algorithms=["TestAlgorithm"],
optional_algorithms=["TestNoChangesAlgorithm", "TestAlgorithm3"],
inputs=inputs,
required_outputs=["TestType3"],
tokens=[], required_output_tokens=[])
executor.execute_mapping()
self.assertTrue(TestAlgorithm.called)
self.assertFalse(TestNoChangesAlgorithm.called)
self.assertTrue(TestAlgorithm3.called)
self.assertEqual(executor.get_item("TestType3"), "TestType3")
def test_failing_external_algorithm(self):
if not os.access("/bin/sh", os.X_OK):
raise self.skipTest("need Bourne shell to run this test")
fd, name = tempfile.mkstemp()
inputs = {"ExampleFilePath": name}
xmlfile = os.path.join(os.path.dirname(__file__), "test_algos.xml")
executor = PACMANAlgorithmExecutor(
algorithms=["FailingExternal"], xml_paths=[xmlfile],
optional_algorithms=[], inputs=inputs, required_outputs=[],
tokens=[], required_output_tokens=[])
with self.assertRaises(
PacmanExternalAlgorithmFailedToCompleteException) as e:
executor.execute_mapping()
self.assertIn(
"Algorithm FailingExternal returned a non-zero error code 1",
str(e.exception))
self.assertEqual(executor.get_item("Foo"), None)
with os.fdopen(fd) as f:
self.assertEqual(f.read(), "foo\n")
def test_optional_workflow(self):
""" Tests that an optional algorithm that doesn't do anything doesn't
get called
"""
TestAlgorithm.called = False
TestNoChangesAlgorithm.called = False
TestAlgorithm3.called = False
inputs = {"TestType1": "TestType1"}
executor = PACMANAlgorithmExecutor(
algorithms=["TestAlgorithm"],
optional_algorithms=["TestNoChangesAlgorithm", "TestAlgorithm3"],
inputs=inputs,
required_outputs=["TestType3"],
tokens=[],
required_output_tokens=[])
executor.execute_mapping()
self.assertTrue(TestAlgorithm.called)
self.assertFalse(TestNoChangesAlgorithm.called)
self.assertTrue(TestAlgorithm3.called)
self.assertEqual(executor.get_item("TestType3"), "TestType3")
def test_optional_token_workflow_not_needed(self):
""" Tests that a workflow with tokens works with optional algorithms\
that are not needed
"""
TestPartTokenOutput1.called = False
TestPartTokenOutput2.called = False
TestWholeTokenRequired.called = False
executor = PACMANAlgorithmExecutor(
algorithms=["TestWholeTokenRequired"],
optional_algorithms=[
"TestPartTokenOutput2", "TestPartTokenOutput1"
],
inputs={},
required_outputs=[],
tokens=[Token("Test", "Part1"),
Token("Test", "Part2")],
required_output_tokens=[])
executor.execute_mapping()
self.assertFalse(TestPartTokenOutput1.called)
self.assertFalse(TestPartTokenOutput2.called)
self.assertTrue(TestWholeTokenRequired.called)
def test_failing_external_algorithm(self):
if not os.access("/bin/sh", os.X_OK):
raise self.skipTest("need Bourne shell to run this test")
fd, name = tempfile.mkstemp()
inputs = {"ExampleFilePath": name}
xmlfile = os.path.join(os.path.dirname(__file__), "test_algos.xml")
executor = PACMANAlgorithmExecutor(algorithms=["FailingExternal"],
xml_paths=[xmlfile],
optional_algorithms=[],
inputs=inputs,
required_outputs=[],
tokens=[],
required_output_tokens=[])
with self.assertRaises(
PacmanExternalAlgorithmFailedToCompleteException) as e:
executor.execute_mapping()
self.assertIn(
"Algorithm FailingExternal returned a non-zero error code 1",
str(e.exception))
self.assertEqual(executor.get_item("Foo"), None)
with os.fdopen(fd) as f:
self.assertEqual(f.read(), "foo\n")
def test_memory_io():
vertex = MyVertex()
graph = MachineGraph("Test")
graph.add_vertex(vertex)
placements = Placements()
placements.add_placement(Placement(vertex, 0, 0, 1))
transceiver = _MockTransceiver()
temp = tempfile.mkdtemp()
print("ApplicationDataFolder = {}".format(temp))
inputs = {
"MemoryTransceiver": transceiver,
"MemoryMachineGraph": graph,
"MemoryPlacements": placements,
"IPAddress": "testing",
"ApplicationDataFolder": temp,
"APPID": 30
}
algorithms = ["WriteMemoryIOData"]
executor = PACMANAlgorithmExecutor(
algorithms, [], inputs, [], [], [],
xml_paths=get_front_end_common_pacman_xml_paths())
executor.execute_mapping()
assert(vertex._test_tag == vertex._tag)
def test_memory_io():
vertex = MyVertex()
graph = MachineGraph("Test")
graph.add_vertex(vertex)
placements = Placements()
placements.add_placement(Placement(vertex, 0, 0, 1))
transceiver = _MockTransceiver()
temp = tempfile.mkdtemp()
print("ApplicationDataFolder = {}".format(temp))
inputs = {
"MemoryTransceiver": transceiver,
"MemoryMachineGraph": graph,
"MemoryPlacements": placements,
"IPAddress": "testing",
"ReportFolder": temp,
"APPID": 30
}
algorithms = ["WriteMemoryIOData"]
executor = PACMANAlgorithmExecutor(
algorithms, [],
inputs, [], [], [],
xml_paths=get_front_end_common_pacman_xml_paths())
executor.execute_mapping()
assert (vertex._test_tag == vertex._tag)
def __call__(self, nengo_network, machine_time_step,
nengo_random_number_generator_seed, decoder_cache,
utilise_extra_core_for_output_types_probe,
nengo_nodes_as_function_of_time,
function_of_time_nodes_time_period, insert_interposers,
print_timings, do_timings, xml_paths,
pacman_executor_provenance_path, nengo_ensemble_profile,
nengo_ensemble_profile_num_samples, receive_buffer_port,
receive_buffer_host, minimum_buffer_sdram,
maximum_sdram_for_sink_vertex_buffing,
using_auto_pause_and_resume, time_between_requests,
buffer_size_before_receive, spike_buffer_max_size,
variable_buffer_max_size, machine_time_step_in_seconds):
# create data holders
inputs = dict()
algorithms = list()
outputs = list()
tokens = list()
required_tokens = list()
optional_algorithms = list()
# add nengo_spinnaker_gfe algorithms
algorithms.append("NengoApplicationGraphBuilder")
if insert_interposers:
algorithms.append("NengoUtiliseInterposers")
# add nengo_spinnaker_gfe inputs
inputs["NengoModel"] = nengo_network
inputs["MachineTimeStep"] = machine_time_step
inputs["NengoRandomNumberGeneratorSeed"] = (
nengo_random_number_generator_seed)
inputs["NengoDecoderCache"] = decoder_cache
inputs["NengoUtiliseExtraCoreForProbes"] = (
utilise_extra_core_for_output_types_probe)
inputs["NengoNodesAsFunctionOfTime"] = nengo_nodes_as_function_of_time
inputs["NengoNodesAsFunctionOfTimeTimePeriod"] = (
function_of_time_nodes_time_period)
inputs["NengoEnsembleProfile"] = nengo_ensemble_profile
inputs["NengoEnsembleProfileNumSamples"] = (
nengo_ensemble_profile_num_samples)
inputs["ReceiveBufferPort"] = receive_buffer_port
inputs["ReceiveBufferHost"] = receive_buffer_host
inputs["MinBufferSize"] = minimum_buffer_sdram
inputs["MaxSinkBuffingSize"] = maximum_sdram_for_sink_vertex_buffing
inputs["UsingAutoPauseAndResume"] = using_auto_pause_and_resume
inputs["TimeBetweenRequests"] = time_between_requests
inputs["BufferSizeBeforeReceive"] = buffer_size_before_receive
inputs["SpikeBufferMaxSize"] = spike_buffer_max_size
inputs["VariableBufferMaxSize"] = variable_buffer_max_size
inputs["MachineTimeStepInSeconds"] = machine_time_step_in_seconds
# Execute the algorithms
executor = PACMANAlgorithmExecutor(
algorithms=algorithms,
optional_algorithms=optional_algorithms,
inputs=inputs,
tokens=tokens,
required_output_tokens=required_tokens,
xml_paths=xml_paths,
required_outputs=outputs,
do_timings=do_timings,
print_timings=print_timings,
provenance_name="nengo_graph_to_application_graph",
provenance_path=pacman_executor_provenance_path)
executor.execute_mapping()
return (executor.get_item("NengoOperatorGraph"),
executor.get_item("NengoHostGraph"),
executor.get_item("NengoGraphToAppGraphMap"),
executor.get_item("AppGraphToNengoOperatorMap"),
executor.get_item("NengoRandomNumberGenerator"))