def setUp(self): self.registry = ProbeFactoryRegistry() self.plugin = self.registry.plugin factory = self.registry.data_source_factories[0] self.layer = ExecutionLayer( data_sources=[factory.create_model(), factory.create_model()])
def sample_workflow(self): wf = Workflow() wf.mco_model = self.mco_factory.create_model() wf.mco_model.parameters = [self.mco_parameter_factory.create_model()] wf.mco_model.kpis = [KPISpecification()] wf.execution_layers = [ ExecutionLayer(data_sources=[ self.data_source_factory.create_model(), self.data_source_factory.create_model(), ]), ExecutionLayer( data_sources=[self.data_source_factory.create_model()]), ] return wf
def test_from_json(self): json_path = fixtures.get("test_probe.json") with open(json_path) as f: data = json.load(f) layer_data = {"data_sources": data["workflow"]["execution_layers"][0]} layer = ExecutionLayer.from_json(self.registry, layer_data) self.assertDictEqual( layer_data["data_sources"][0], { "id": "force.bdss.enthought.plugin.test.v0." "factory.probe_data_source", "model_data": { "input_slot_info": [{ "source": "Environment", "name": "foo" }], "output_slot_info": [{ "name": "bar" }], }, }, ) layer_data["data_sources"][0]["model_data"].update({ "input_slots_type": "PRESSURE", "output_slots_type": "PRESSURE", "input_slots_size": 1, "output_slots_size": 1, }) self.assertDictEqual(layer.__getstate__(), layer_data)
def test_data_sources(self): wf = self.workflow mco_factory = self.plugin.mco_factories[0] wf.mco_model = mco_factory.create_model() parameter_factory = mco_factory.parameter_factories[0] wf.mco_model.parameters.append(parameter_factory.create_model()) wf.mco_model.parameters[0].name = "name" wf.mco_model.parameters[0].type = "type" wf.mco_model.kpis.append(KPISpecification(name='name')) layer = ExecutionLayer() wf.execution_layers.append(layer) ds_factory = self.plugin.data_source_factories[0] ds_model = ds_factory.create_model() layer.data_sources.append(ds_model) errors = verify_workflow(wf) self.assertEqual(errors[0].subject, ds_model) self.assertIn( "The number of input slots (1 values) returned by " "'Dummy data source' does not match the number " "of user-defined names specified (0 values). This " "is either a plugin error or a file error.", errors[0].local_error) ds_model.input_slot_info.append(InputSlotInfo(name="name")) errors = verify_workflow(wf) self.assertEqual(errors[0].subject, ds_model) self.assertIn( "The number of output slots (1 values) returned by " "'Dummy data source' does not match the number " "of user-defined names specified (0 values). This " "is either a plugin error or a file error.", errors[0].local_error) ds_model.output_slot_info.append(OutputSlotInfo(name="name")) errors = verify_workflow(wf) self.assertEqual(len(errors), 0) ds_model.input_slot_info[0].name = '' errors = verify_workflow(wf) self.assertEqual(len(errors), 1) self.assertIn("Input slot is not named", errors[0].local_error) ds_model.output_slot_info[0].name = '' errors = verify_workflow(wf) self.assertEqual(len(errors), 3) self.assertIn("All output variables have undefined names", errors[1].local_error) self.assertIn("An output variable has an undefined name", errors[2].local_error)
def test_empty_execution_layer(self): wf = self.workflow mco_factory = self.plugin.mco_factories[0] wf.mco_model = mco_factory.create_model() parameter_factory = mco_factory.parameter_factories[0] wf.mco_model.parameters.append(parameter_factory.create_model()) wf.mco_model.parameters[0].name = "name" wf.mco_model.parameters[0].type = "type" wf.mco_model.kpis.append(KPISpecification(name='name')) layer = ExecutionLayer() wf.execution_layers.append(layer) errors = verify_workflow(wf) self.assertEqual(len(errors), 1) self.assertEqual(errors[0].subject, wf.execution_layers[0])
def _extract_execution_layers(factory_registry, workflow_data): """ Generates the List(ExecutionLayer) from the `workflow_data` dictionary. Parameters ---------- factory_registry: IFactoryRegistry Generating factory registry for the data sources inside the execution layers workflow_data: dict Dictionary with the content of the `ExecutionLayer`s in serialized format Returns ------- execution_layers: List(ExecutionLayer) list of ExecutionLayer instances. """ execution_layers = [] for layer_data in workflow_data["execution_layers"]: layer = ExecutionLayer.from_json(factory_registry, layer_data) execution_layers.append(layer) return execution_layers
def test_empty_layer_from_json(self): layer = ExecutionLayer.from_json(self.registry, {}) self.assertEqual(0, len(layer.data_sources))
class TestExecutionLayer(TestCase, UnittestTools): def setUp(self): self.registry = ProbeFactoryRegistry() self.plugin = self.registry.plugin factory = self.registry.data_source_factories[0] self.layer = ExecutionLayer( data_sources=[factory.create_model(), factory.create_model()]) def test__init__(self): self.assertEqual(2, len(self.layer.data_sources)) def test_verify(self): errors = self.layer.verify() messages = [error.local_error for error in errors] self.assertEqual(4, len(messages)) self.layer.data_sources = [] errors = self.layer.verify() messages = [error.local_error for error in errors] self.assertEqual(1, len(messages)) self.assertIn("Layer has no data sources", messages) def test_create_data_source_error(self): factory = self.registry.data_source_factories[0] factory.raises_on_create_data_source = True with testfixtures.LogCapture() as capture: with self.assertRaises(Exception): self.layer.execute_layer([]) capture.check(( "force_bdss.core.execution_layer", "ERROR", "Unable to create data source from factory " "'force.bdss.enthought.plugin.test.v0.factory." "probe_data_source' in plugin " "'force.bdss.enthought.plugin.test.v0'." " This may indicate a programming " "error in the plugin", )) def test_data_source_run_error(self): data_values = [DataValue(name="foo")] self.layer.data_sources[0].input_slot_info = [ InputSlotInfo(name="foo") ] factory = self.registry.data_source_factories[0] factory.raises_on_data_source_run = True with testfixtures.LogCapture() as capture: with self.assertRaises(Exception): self.layer.execute_layer(data_values) capture.check( ( "force_bdss.core.execution_layer", "INFO", "Evaluating for Data Source test_data_source", ), ("force_bdss.core.execution_layer", "INFO", "Passed values:"), ( "force_bdss.core.execution_layer", "INFO", "0: foo = None (AVERAGE)", ), ( "force_bdss.core.execution_layer", "ERROR", "Evaluation could not be performed. " "Run method raised exception.", ), ) def test_error_for_incorrect_return_type(self): data_values = [DataValue(name="foo")] self.layer.data_sources[0].input_slot_info = [ InputSlotInfo(name="foo") ] def probe_run(self, *args, **kwargs): return "hello" factory = self.registry.data_source_factories[0] factory.run_function = probe_run with testfixtures.LogCapture(): with self.assertRaisesRegex( RuntimeError, "The run method of data source " "test_data_source must return a list. It " r"returned instead <.* 'str'>. Fix the run\(\)" " method to return the appropriate entity.", ): self.layer.execute_layer(data_values) def test_error_for_output_slots(self): data_values = [DataValue(name="foo")] self.layer.data_sources[0].input_slot_info = [ InputSlotInfo(name="foo") ] def probe_run(self, *args, **kwargs): return ["too", "many", "data", "values"] factory = self.registry.data_source_factories[0] factory.run_function = probe_run with testfixtures.LogCapture(): with self.assertRaisesRegex( RuntimeError, r"The number of data values \(4 values\) returned" " by 'test_data_source' does not match the number" r" of output slots it specifies \(1 values\)." " This is likely a plugin error.", ): self.layer.execute_layer(data_values) def test_error_for_non_data_source(self): data_values = [DataValue(name="foo")] self.layer.data_sources[0].input_slot_info = [ InputSlotInfo(name="foo") ] self.layer.data_sources[0].output_slot_info = [ OutputSlotInfo(name="one") ] def probe_run(self, *args, **kwargs): return ["hello"] factory = self.registry.data_source_factories[0] factory.run_function = probe_run with testfixtures.LogCapture(): with self.assertRaisesRegex( RuntimeError, "The result list returned by DataSource " "test_data_source" " contains an entry that is not a DataValue." " An entry of type <.* 'str'> was instead found" " in position 0." r" Fix the DataSource.run\(\) method to" " return the appropriate entity.", ): self.layer.execute_layer(data_values) def test_execute_layer_results(self): data_values = [ DataValue(name="foo"), DataValue(name="bar"), DataValue(name="baz"), DataValue(name="quux"), ] def run(self, *args, **kwargs): return [DataValue(value=1), DataValue(value=2), DataValue(value=3)] ds_factory = self.registry.data_source_factories[0] ds_factory.input_slots_size = 2 ds_factory.output_slots_size = 3 ds_factory.run_function = run evaluator_model = ds_factory.create_model() evaluator_model.input_slot_info = [ InputSlotInfo(name="foo"), InputSlotInfo(name="quux"), ] evaluator_model.output_slot_info = [ OutputSlotInfo(name="one"), OutputSlotInfo(name=""), OutputSlotInfo(name="three"), ] self.layer.data_sources = [evaluator_model] res = self.layer.execute_layer(data_values) self.assertEqual(len(res), 2) self.assertEqual(res[0].name, "one") self.assertEqual(res[0].value, 1) self.assertEqual(res[1].name, "three") self.assertEqual(res[1].value, 3) with mock.patch( "force_bdss.data_sources.base_data_source.BaseDataSource._run", return_value=run(None)) as mock_run: # mock_run.side_effect = [1, 2] self.layer.execute_layer(data_values) mock_run.assert_called_once() def test_from_json(self): json_path = fixtures.get("test_probe.json") with open(json_path) as f: data = json.load(f) layer_data = {"data_sources": data["workflow"]["execution_layers"][0]} layer = ExecutionLayer.from_json(self.registry, layer_data) self.assertDictEqual( layer_data["data_sources"][0], { "id": "force.bdss.enthought.plugin.test.v0." "factory.probe_data_source", "model_data": { "input_slot_info": [{ "source": "Environment", "name": "foo" }], "output_slot_info": [{ "name": "bar" }], }, }, ) layer_data["data_sources"][0]["model_data"].update({ "input_slots_type": "PRESSURE", "output_slots_type": "PRESSURE", "input_slots_size": 1, "output_slots_size": 1, }) self.assertDictEqual(layer.__getstate__(), layer_data) def test_empty_layer_from_json(self): layer = ExecutionLayer.from_json(self.registry, {}) self.assertEqual(0, len(layer.data_sources)) def test_notify_driver_event(self): with self.assertTraitChanges(self.layer, "event", count=1): self.layer.data_sources[0].notify(BaseDriverEvent())
def test_multilayer_execution(self): # The multilayer peforms the following execution # layer 0: in1 + in2 | in3 + in4 # res1 res2 # layer 1: res1 + res2 # res3 # layer 2: res3 * res1 # res4 # layer 3: res4 * res2 # out1 # Final result should be # out1 = ((in1 + in2 + in3 + in4) * (in1 + in2) * (in3 + in4) data_values = [ DataValue(value=10, name="in1"), DataValue(value=15, name="in2"), DataValue(value=3, name="in3"), DataValue(value=7, name="in4"), ] def adder(model, parameters): first = parameters[0].value second = parameters[1].value return [DataValue(value=(first + second))] adder_factory = ProbeDataSourceFactory( self.plugin, input_slots_size=2, output_slots_size=1, run_function=adder, ) def multiplier(model, parameters): first = parameters[0].value second = parameters[1].value return [DataValue(value=(first * second))] multiplier_factory = ProbeDataSourceFactory( self.plugin, input_slots_size=2, output_slots_size=1, run_function=multiplier, ) mco_factory = ProbeMCOFactory(self.plugin) mco_model = mco_factory.create_model() parameter_factory = mco_factory.parameter_factories[0] mco_model.parameters = [ parameter_factory.create_model({"name": "in1"}), parameter_factory.create_model({"name": "in2"}), parameter_factory.create_model({"name": "in3"}), parameter_factory.create_model({"name": "in4"}), ] mco_model.kpis = [KPISpecification(name="out1")] wf = Workflow( mco_model=mco_model, execution_layers=[ ExecutionLayer(), ExecutionLayer(), ExecutionLayer(), ExecutionLayer(), ], ) # Layer 0 model = adder_factory.create_model() model.input_slot_info = [ InputSlotInfo(name="in1"), InputSlotInfo(name="in2"), ] model.output_slot_info = [OutputSlotInfo(name="res1")] wf.execution_layers[0].data_sources.append(model) model = adder_factory.create_model() model.input_slot_info = [ InputSlotInfo(name="in3"), InputSlotInfo(name="in4"), ] model.output_slot_info = [OutputSlotInfo(name="res2")] wf.execution_layers[0].data_sources.append(model) # layer 1 model = adder_factory.create_model() model.input_slot_info = [ InputSlotInfo(name="res1"), InputSlotInfo(name="res2"), ] model.output_slot_info = [OutputSlotInfo(name="res3")] wf.execution_layers[1].data_sources.append(model) # layer 2 model = multiplier_factory.create_model() model.input_slot_info = [ InputSlotInfo(name="res3"), InputSlotInfo(name="res1"), ] model.output_slot_info = [OutputSlotInfo(name="res4")] wf.execution_layers[2].data_sources.append(model) # layer 3 model = multiplier_factory.create_model() model.input_slot_info = [ InputSlotInfo(name="res4"), InputSlotInfo(name="res2"), ] model.output_slot_info = [OutputSlotInfo(name="out1")] wf.execution_layers[3].data_sources.append(model) kpi_results = wf.execute(data_values) self.assertEqual(1, len(kpi_results)) self.assertEqual(8750, kpi_results[0].value)
def test_kpi_specification_adherence(self): # Often the user may only wish to treat a subset of DataSource # output slots as KPIs. This test makes sure they get what they # ask for! # keep input DataValues constant data_values = [ DataValue(value=99, name="in1"), DataValue(value=1, name="in2"), ] # dummy addition DataSource(a, b) that also returns its inputs # [a, b, a+b] def adder(model, parameters): first = parameters[0].value second = parameters[1].value return [ DataValue(value=first), DataValue(value=second), DataValue(value=(first + second)), ] adder_factory = ProbeDataSourceFactory( self.plugin, input_slots_size=2, output_slots_size=3, run_function=adder, ) mco_factory = ProbeMCOFactory(self.plugin) parameter_factory = mco_factory.parameter_factories[0] mco_model = mco_factory.create_model() # DataSourceModel stats constant throughout model = adder_factory.create_model() model.input_slot_info = [ InputSlotInfo(name="in1"), InputSlotInfo(name="in2"), ] model.output_slot_info = [ OutputSlotInfo(name="out1"), OutputSlotInfo(name="out2"), OutputSlotInfo(name="out3"), ] # test Parameter and KPI spec that follows DataSource slots # exactly mco_model.parameters = [ parameter_factory.create_model({"name": "in1"}), parameter_factory.create_model({"name": "in2"}), ] mco_model.kpis = [ KPISpecification(name="out1"), KPISpecification(name="out2"), KPISpecification(name="out3"), ] # need to make a new workflow for each KPISpecification wf = Workflow(mco_model=mco_model, execution_layers=[ExecutionLayer()]) wf.execution_layers[0].data_sources.append(model) kpi_results = wf.execute(data_values) self.assertEqual(len(kpi_results), 3) self.assertEqual(kpi_results[0].value, 99) self.assertEqual(kpi_results[1].value, 1) self.assertEqual(kpi_results[2].value, 100) self.assertEqual(kpi_results[0].name, "out1") self.assertEqual(kpi_results[1].name, "out2") self.assertEqual(kpi_results[2].name, "out3") # now test all possible combinations of KPISpecification, including # those with KPIs repeated, and empty KPI specification import itertools out_options = [("out1", 99), ("out2", 1), ("out3", 100)] for num_outputs in range(len(out_options) + 2, 0, -1): for spec in itertools.permutations(out_options, r=num_outputs): mco_model.kpis = [ KPISpecification(name=opt[0]) for opt in spec ] wf = Workflow( mco_model=mco_model, execution_layers=[ExecutionLayer()] ) wf.execution_layers[0].data_sources.append(model) kpi_results = wf.execute(data_values) self.assertEqual(len(kpi_results), num_outputs) for i in range(num_outputs): self.assertEqual(kpi_results[i].name, spec[i][0]) self.assertEqual(kpi_results[i].value, spec[i][1])