def setUp(self): super(TestDataPipeBuilder, self).setUp() test_utils.register_new_class("reader1", nc7.data.DataReader) test_utils.register_new_class("reader2", nc7.data.DataReader) test_utils.register_new_class("processor1", nc7.data.DataProcessor) test_utils.register_new_class("processor2", nc7.data.DataProcessor) test_utils.register_new_class("nucleotide1", nc7.core.Nucleotide)
def test_get_sample_config_for_nucleotide(self): class_name = 'DummyPluginWithDummyParameter' register_new_class(class_name, DummyPluginWithDummyParameter) inbound_nodes = ['TODO_SET_CORRECT_INBOUND_NODES'] incoming_keys_mapping = { 'TODO_SET_INPUT_PLUGIN_NAME_0': { 'TODO_SET_INPUT_PLUGIN_OUTPUT_KEY_NAME_0': 'inputs_mlp' } } correct_dict = { 'inbound_nodes': inbound_nodes, 'incoming_keys_mapping': incoming_keys_mapping, 'dummy_parameter': 'TODO_SET_CORRECT_VALUE', 'name': class_name, 'activation': 'elu', 'initializer': None, 'dropout': 'normal', 'trainable': True, 'stop_gradient_from_inputs': False, 'load_fname': None, 'load_var_scope': None, 'exclude_from_restore': False, 'optimization_configs': None, 'data_format': None, 'allow_mixed_precision': True, 'class_name': class_name } out_dict = nucleotide_utils.get_sample_config_for_nucleotide( class_name) keys_out = list(out_dict) keys_correct = list(correct_dict) self.assertSetEqual(set(keys_correct), set(keys_out)) self.assertDictEqual(correct_dict, out_dict)
def test_build_registered_object(self): register_new_class('dummy_plugin', nc7.model.ModelPlugin) name = "plugin_name" inbound_nodes = ['node1', 'node2'] plugin = builder_lib.build_registered_object( base_cls=nc7.model.ModelPlugin, class_name="dummy_plugin", name=name, inbound_nodes=inbound_nodes) self.assertIsInstance(plugin, nc7.model.ModelPlugin) self.assertTrue(plugin.build) self.assertEqual(plugin.name, name) self.assertListEqual(plugin.inbound_nodes, inbound_nodes) with self.assertRaises(ValueError): plugin2 = builder_lib.build_registered_object( base_cls=nc7.model.ModelMetric, class_name="dummy_plugin", name=name, inbound_nodes=inbound_nodes) plugin3 = builder_lib.build_registered_object( base_cls=nc7.model.ModelMetric, default_cls=nc7.model.ModelPlugin, inbound_nodes=inbound_nodes) self.assertIsInstance(plugin3, nc7.model.ModelPlugin) self.assertEqual(plugin3.name, plugin3.__class__.__name__)
def test_build_single_file_list(self): register_new_class('file_list1', FileListDummy) config_file_list = { 'class_name': 'file_list1', 'file_names': self.file_names1 } file_list = file_list_builder.build(config_file_list) self.assertIsInstance(file_list, nc7.data.FileList) self.assertEqual(self.number_of_files1, len(file_list)) self.assertDictEqual(self.file_names1, file_list.get())
def setUp(self): reset_register_and_logger() register_new_class("plugin1", _KPIPlugin) register_new_class('plugin2', _KPIPlugin) register_new_class("accumulator1", _KPIAccumulator) register_new_class("accumulator2", _KPIAccumulator) self.plugins_and_accumulators_configs = [ { "class_name": "plugin1", "parameter1": 10 }, { "class_name": "plugin2", "parameter1": 20, "cachers": [{ "class_name": "kpi_builder_test._KPICacher" }] }, { "class_name": "accumulator1", "parameter2": 100, "cachers": [{ "class_name": "kpi_builder_test._KPICacher", "p": 10 }], "savers": [{ "class_name": "kpi_builder_test._KPISaver", "p3": 5 }] }, { "class_name": "accumulator2", "parameter2": 200, "savers": [{ "class_name": "kpi_builder_test._KPISaver2", "p4": 50 }] }, ] self.project_global_config = { "KPIPlugin": { "cachers": [{ "class_name": "kpi_builder_test._KPICacher2", "p2": 5 }], "savers": [{ "class_name": "kpi_builder_test._KPISaver2", "p4": 100 }] } } project_global_config.clear_project_global_config() super().setUp()
def test_build_file_list_with_filter(self, single_data_filter): register_new_class('file_list1', FileListDummy) class _DummyDataFilter1(DataFilter): pass class _DummyDataFilter2(DataFilter): pass if single_data_filter: data_filter_config = { "data_filter": { "class_name": "_DummyDataFilter1" } } else: data_filter_config = { "data_filter": [{ "class_name": "_DummyDataFilter1" }, { "class_name": "_DummyDataFilter2" }, { "class_name": "_DummyDataFilter2" }] } config_file_list = { "class_name": "file_list1", "file_names": self.file_names1 } config_file_list.update(data_filter_config) file_list = file_list_builder.build(config_file_list) number_of_data_filters_must = single_data_filter and 1 or 3 self.assertEqual(number_of_data_filters_must, len(file_list.data_filters)) data_filter_types_must = ( single_data_filter and [_DummyDataFilter1] or [_DummyDataFilter1, _DummyDataFilter2, _DummyDataFilter2]) for each_data_filter, each_data_filter_type_must in zip( file_list.data_filters, data_filter_types_must): self.assertIsInstance(each_data_filter, each_data_filter_type_must)
def test_build_list_of_file_list(self): register_new_class('file_list2', FileListDummy) register_new_class('file_list3', FileListDummy) config_file_list = [{ 'class_name': 'file_list2', 'file_names': self.file_names1 }, { 'class_name': 'file_list2', 'file_names': self.file_names2 }, { 'class_name': 'file_list3', 'file_names': self.file_names3 }] file_list = file_list_builder.build(config_file_list) self.assertIsInstance(file_list, nc7.data.FileList) self.assertEqual( self.number_of_files1 + self.number_of_files2 + self.number_of_files3, len(file_list)) file_names_1_2_3 = {} for each_key in self.file_names1.keys(): file_names_1_2_3[each_key] = (self.file_names1[each_key] + self.file_names2[each_key]) self.assertDictEqual(self.file_names_1_2_3, file_list.get())
def test_get_callbacks(self): register_new_class('dummy_callback_1', nc7.coordinator.CoordinatorCallback) register_new_class('dummy_callback_2', nc7.coordinator.CoordinatorCallback) config_callbacks = [ { 'class_name': 'dummy_callback_1', 'name': 'callback1' }, { 'class_name': 'dummy_callback_2', 'name': 'callback2' }, ] callbacks = callback_builder.build_callbacks_chain(config_callbacks) for each_callback in callbacks: self.assertTrue(each_callback.built) self.assertEqual(2, len(callbacks)) for callback in callbacks: self.assertIsInstance(callback, nc7.coordinator.CoordinatorCallback) self.assertTrue(callback.built)
def setUp(self): reset_register_and_logger() register_new_class("dataset_with_file_list", _DatasetWithFileList) register_new_class("dataset_without_file_list", _DatasetWithoutFileList) register_new_class("FileListDummy", FileListDummy) self.number_of_files1 = 20 self.number_of_files2 = 10 self.file_names1 = {'data': ['input_1_{:03d}.ext'.format(i) for i in range(self.number_of_files1)]} self.file_names2 = {'data': ['input2_1_{:03d}.ext'.format(i) for i in range(self.number_of_files2)]}
def setUp(self): reset_register_and_logger() register_new_class('dummy_dataset_train', nc7.data.Dataset) register_new_class('dummy_dataset_eval', nc7.data.Dataset) register_new_class('dummy_callback_train', nc7.coordinator.CoordinatorCallback) register_new_class('dummy_plugin_train', nc7.model.ModelPlugin) register_new_class('dummy_loss', nc7.model.ModelLoss) register_new_class('dummy_pp', nc7.model.ModelPostProcessor) register_new_class('dummy_metrics', nc7.model.ModelMetric) register_new_class('dummy_summaries', nc7.model.ModelSummary) register_new_class('dummy_dataset_fl_train', _DatasetFileListDummy) register_new_class('dummy_dataset_fl_eval', _DatasetFileListDummy) register_new_class('dummy_file_list', FileListDummy) register_new_class('dummy_data_feeder', nc7.data.DataFeeder) register_new_class('dummy_callback_infer', nc7.coordinator.CoordinatorCallback) register_new_class('dummy_data_feeder_fl', _DatafeederFileListDummy) register_new_class('dummy_file_list_infer', FileListDummy) register_new_class("dummy_kpi_plugin1", nc7.kpi.KPIPlugin) register_new_class("dummy_kpi_plugin2", nc7.kpi.KPIPlugin) register_new_class("dummy_kpi_accumulator", nc7.kpi.KPIAccumulator)
def setUp(self) -> None: reset_register_and_logger() register_new_class("data_filter1", _DummyDataFilter1) register_new_class("data_filter2", _DummyDataFilter2) self.config1 = {"class_name": "data_filter1", "p1": 20, "p2": 500} self.config2 = {"class_name": "data_filter2", "p1": 100}
def test_build_data_object_from_config(self, with_file_list, with_data_filter, with_data_pipe=False, with_file_list_mapping=False): config_object = {"random_seed": 65477} if with_data_pipe: built_fn = lambda x: Dataset.from_data_pipe(**x).build() dataset_base_cls = Dataset else: if with_file_list: register_new_class("dummy_dataset", _DummyDatasetFileList) dataset_base_cls = _DummyDatasetFileList built_fn = None else: register_new_class("dummy_dataset", _DummyDataset) dataset_base_cls = _DummyDataset built_fn = None config_object.update({"class_name": "dummy_dataset", "p1": 100}) file_list_keys_mapping = {"key1": "key1_r"} if with_file_list: config_object["file_list"] = { "class_name": "file_list1", "file_names": { "key1": ["value1"] }, "name": "file_list_name" } _DummyDatasetFileList.file_list_keys = ["key1"] if with_file_list_mapping: config_object["file_list_keys_mapping"] = {"key1": "key1_r"} _DummyDatasetFileList.file_list_keys = ["key1_r"] if with_data_filter: config_object["data_filter"] = { "class_name": "data_filter1", "dp1": 1 } if with_data_pipe: reader_config1 = {"class_name": "reader1", "name": "reader1_name"} reader_config2 = {"class_name": "reader2", "name": "reader2_name"} processor_config1 = { "class_name": "processor1", "name": "processor1_name" } processor_config2 = { "class_name": "processor2", "name": "processor2_name" } if with_file_list: config_object["data_pipe"] = [ processor_config1, reader_config1, reader_config2, processor_config2 ] else: config_object["data_pipe"] = [ processor_config1, processor_config2 ] if with_data_pipe and with_file_list_mapping: with self.assertRaises(ValueError): data_builder_lib.build_data_object_from_config( config_object, base_cls=dataset_base_cls, built_fn=built_fn) return built_object = data_builder_lib.build_data_object_from_config( config_object, base_cls=dataset_base_cls, built_fn=built_fn) self.assertTrue(built_object.built) self.assertEqual(65477, built_object.random_seed) self.assertIsInstance(built_object, dataset_base_cls) if with_data_filter: self.assertEqual(1, len(built_object.data_filters)) data_filter = built_object.data_filters[0] self.assertTrue(data_filter.built) self.assertIsInstance(data_filter, _DummyDataFilter) self.assertEqual(1, data_filter.dp1) else: self.assertIsNone(built_object.data_filters) if with_file_list: file_list = built_object.file_list self.assertIsInstance(file_list, FileListDummy) self.assertTrue(file_list.built) self.assertEqual("file_list_name", file_list.name) file_names_must = ({ "key1_r": ["value1"] } if with_file_list_mapping else { "key1": ["value1"] }) self.assertDictEqual(file_names_must, file_list.get()) if with_file_list_mapping: self.assertDictEqual(file_list_keys_mapping, built_object.file_list_keys_mapping) else: self.assertIsNone(built_object.file_list_keys_mapping) else: self.assertFalse(hasattr(built_object, "file_list")) self.assertFalse(hasattr(built_object, "file_list_keys_mapping")) if with_data_pipe: data_pipe = built_object.data_pipe self.assertIsInstance(data_pipe, DataPipe) self.assertTrue(data_pipe.built) if with_file_list: self.assertSetEqual( {"reader1_name", "reader2_name"}, {r.name for r in data_pipe.readers.values()}) else: self.assertDictEqual({}, data_pipe.readers) self.assertSetEqual( {"processor1_name", "processor2_name"}, {r.name for r in data_pipe.processors.values()})
def setUp(self): reset_register_and_logger() register_new_class("data_filter1", _DummyDataFilter) register_new_class("file_list1", FileListDummy) register_new_class("reader1", _DataReaderTF) register_new_class("reader2", _DataReaderTF) register_new_class("processor1", _DataProcessorTF) register_new_class("processor2", _DataProcessorTF)