class TestZeroCenter(TestCase): def setUp(self): self.orga = Organizer("./") self.orga.cfg.zero_center_folder = "nope/" def test_get_xs_mean_existing(self): target = "asd" self.orga.xs_mean = target value = self.orga.get_xs_mean(logging=False) self.assertEqual(target, value) def test_get_xs_mean_no_folder(self): self.orga.cfg.zero_center_folder = None with self.assertRaises(ValueError): self.orga.get_xs_mean(logging=False) @patch('orcanet.core.load_zero_center_data') def test_get_xs_mean_new(self, load_mock): target = "blub" def mock_xs_mean(): return target load_mock.return_value = mock_xs_mean() xs_mean = self.orga.get_xs_mean(logging=False) self.assertEqual(target, xs_mean) self.assertEqual(target, self.orga.xs_mean)
def inference(directory, list_file=None, config_file=None, epoch=None, fileno=None): from orcanet.core import Organizer orga = Organizer(directory, list_file, config_file, tf_log_level=1) return orga.inference(epoch=epoch, fileno=fileno)
def make_orga(): orga = Organizer(self.output_folder, config_file=config_file) orga.cfg._files_dict["train"] = self.train_pathes orga.cfg._files_dict["val"] = self.val_pathes orga.cfg._list_file = "test.toml" orga.cfg.zero_center_folder = self.temp_dir orga.cfg.label_modifier = label_modifier orga.cfg.custom_objects = get_custom_objects() return orga
def get_orga(dims): dimdict = { 2: get_input_shapes_2d, 3: get_input_shapes_3d, "graph": get_input_shapes_graph } orga = Organizer(".") orga.io.get_input_shapes = dimdict[dims] return orga
def setUp(self): self.data_folder = os.path.join(os.path.dirname(__file__), "data") self.output_folder = self.data_folder + "/dummy_model" list_file = self.data_folder + "/in_out_test_list.toml" config_file = None self.orga = Organizer(self.output_folder, list_file, config_file) self.batchsize = 2 self.orga.cfg.batchsize = self.batchsize self.orga.cfg.label_modifier = label_modifier
def orca_pred(output_folder, list_file, config_file, model_file, epoch=None, fileno=None): """ Run orga.predict with predefined ModelBuilder networks using a parser. Per default, the most recent saved model will be loaded. Parameters ---------- output_folder : str Path to the folder where everything gets saved to, e.g. the summary log file, the plots, the trained models, etc. list_file : str Path to a list file which contains pathes to all the h5 files that should be used for training and validation. config_file : str Path to a .toml file which overwrites some of the default settings for training and validating a model. model_file : str Path to a file with parameters to build a model of a predefined architecture with OrcaNet. epoch : int, optional The epoch of the saved model to predict with. fileno : int, optional The filenumber of the saved model to predict with. """ # Set up the Organizer with the input data orga = Organizer(output_folder, list_file, config_file, tf_log_level=1) # When predicting with a orga model, the right modifiers and custom # objects need to be given update_objects(orga, model_file) # Per default, a prediction will be done for the model with the # highest epoch and filenumber. orga.predict(epoch=epoch, fileno=fileno)
def train(directory, list_file=None, config_file=None, model_file=None, to_epoch=None): from orcanet.core import Organizer from orcanet.model_builder import ModelBuilder from orcanet.misc import find_file orga = Organizer(directory, list_file, config_file, tf_log_level=1) if orga.io.get_latest_epoch() is None: # Start of training print("Building new model") if model_file is None: model_file = find_file(directory, "model.toml") model = ModelBuilder(model_file).build(orga, verbose=False) else: model = None return orga.train_and_validate(model=model, to_epoch=to_epoch)
def setUp(self): self.orga = Organizer(self.temp_dir) self.orga.cfg.batchsize = 9 self.orga.io.get_local_files = MagicMock(return_value=self.filepaths) self.orga.io.get_file_sizes = MagicMock(return_value=self.file_sizes) self.orga.cfg.get_list_file = MagicMock(return_value='/path/to/a/listfilename.toml') self.orga.io.get_next_pred_path = MagicMock(return_value=self.pred_filepath) self.model = build_dummy_model(self.input_shapes, self.output_shapes) self.model.compile(loss="mse", optimizer="sgd") self.orga._auto_label_modifier = lib.label_modifiers.ColumnLabels(self.model)
def setUp(self): # Size of train files. No actual files get generated (they are mocked). file_sizes = [100, 66] self.temp_dir = os.path.join(os.path.dirname(__file__), ".temp") self.summary_file = os.path.join(self.temp_dir, "summary.txt") self.orga = Organizer(self.temp_dir) self.orga.io.get_file_sizes = MagicMock(return_value=file_sizes) model = build_test_model() self.smry = SummaryLogger(self.orga, model) # tf 2.2: train the model to set metric_names model.train_on_batch(np.ones((2, 1)), np.ones((2, 1))) self.metrics = model.metrics_names
def setUp(self): self.temp_dir = os.path.join(os.path.dirname(__file__), ".temp", "batch_logger") self.model = build_test_model() samples_file1 = 40 samples_file2 = 60 self.data_file1 = get_test_data(samples_file1) self.data_file2 = get_test_data(samples_file2) self.orga = Organizer(self.temp_dir) self.orga.io.get_file_sizes = MagicMock( return_value=[samples_file1, samples_file2]) self.orga.io.get_subfolder("train_log", create=True) self.orga.cfg.batchsize = 10 self.orga.cfg.train_logger_display = 4
def setUp(self): # Pathes # Temporary output folder self.temp_dir = os.path.join(os.path.dirname(__file__), ".temp", "nn_utilities") self.output_folder = os.path.join(self.temp_dir, "model") self.data_folder = os.path.join(os.path.dirname(__file__), "data") config_file = os.path.join(self.data_folder, "config_test.toml") # Make sure temp dir does either not exist or is empty if os.path.exists(self.temp_dir): assert len(os.listdir(self.temp_dir)) == 0 else: os.makedirs(self.temp_dir) # Make dummy data self.train_inp_A = (self.temp_dir + "/train1_A.h5", self.temp_dir + "/train2_A.h5") self.train_inp_B = ( self.temp_dir + "/train1_B.h5", self.temp_dir + "/train2_B.h5", ) self.train_pathes = { "testing_input_A": self.train_inp_A, "testing_input_B": self.train_inp_B, } self.shape = (3, 3, 3) make_dummy_data(self.train_inp_A[0], self.train_inp_A[1], self.shape, mode=1) make_dummy_data(self.train_inp_B[0], self.train_inp_B[1], self.shape, mode=2) orga = Organizer(self.output_folder, config_file=config_file) orga.cfg._files_dict["train"] = self.train_pathes orga.cfg._list_file = "test.toml" orga.cfg.zero_center_folder = self.temp_dir self.orga = orga
def setUp(self): """ Make a .temp directory in the current working directory, generate dummy data in it and set up the cfg object. """ self.data_folder = os.path.join(os.path.dirname(__file__), "data") self.model_file = os.path.join(self.data_folder, "model_builder_test_CNN_cat.toml") self.input_shapes = { "input_A": (3, 3, 3, 3), } orga = Organizer(".") orga.io.get_input_shapes = MagicMock(return_value=self.input_shapes) orga.cfg.custom_objects = get_custom_objects() self.orga = orga
def use_orcanet(): temp_folder = "output/" os.mkdir(temp_folder) make_dummy_data(temp_folder) list_file = "example_list.toml" organizer = Organizer(temp_folder + "sum_model", list_file) organizer.cfg.train_logger_display = 10 model = make_dummy_model() organizer.train_and_validate(model, epochs=3) organizer.predict()
def orca_train(output_folder, list_file, config_file, model_file, recompile_model=False): """ Run orga.train with predefined ModelBuilder networks using a parser. Parameters ---------- output_folder : str Path to the folder where everything gets saved to, e.g. the summary log file, the plots, the trained models, etc. list_file : str Path to a list file which contains pathes to all the h5 files that should be used for training and validation. config_file : str Path to a .toml file which overwrites some of the default settings for training and validating a model. model_file : str Path to a file with parameters to build a model of a predefined architecture with OrcaNet. recompile_model : bool If the model should be recompiled or not. Necessary, if e.g. the loss_weights are changed during the training. """ # Set up the Organizer with the input data orga = Organizer(output_folder, list_file, config_file, tf_log_level=1) # Load in the orga sample-, label-, and dataset-modifiers, as well as # the custom objects update_objects(orga, model_file) # If this is the start of the training, a compiled model needs to be # handed to the orga.train function if orga.io.get_latest_epoch() is None: # The ModelBuilder class allows to construct models from a toml file, # adapted to the datasets in the orga instance. Its modifiers will # be taken into account for this builder = ModelBuilder(model_file) model = builder.build(orga, log_comp_opts=True) elif recompile_model is True: builder = ModelBuilder(model_file) path_of_model = orga.io.get_model_path(-1, -1) model = ks.models.load_model( path_of_model, custom_objects=orga.cfg.get_custom_objects()) print("Recompiling the saved model") model = builder.compile_model( model, custom_objects=orga.cfg.get_custom_objects()) builder.log_model_properties(orga) else: model = None try: # Use a custom LR schedule user_lr = orga.cfg.learning_rate lr = orca_learning_rates(user_lr, orga.io.get_no_of_files("train")) orga.cfg.learning_rate = lr except NameError: pass # start the training orga.train_and_validate(model=model)
class TestOrganizer(tf.test.TestCase): """ Test the organizer by creating a dummy directory .temp/core. """ def setUp(self): self.temp_dir = os.path.join(os.path.dirname(__file__), ".temp", "core") os.mkdir(self.temp_dir) self.orga = Organizer(self.temp_dir) def tearDown(self): shutil.rmtree(self.temp_dir) @patch('orcanet.core.tf.keras.utils.plot_model') def test_load_model_new_training(self, mock_plot_model): def plot_history(model, folder): pass mock_plot_model.side_effect = plot_history # latest epoch is None aka no model has been trained latest_epoch = None self.orga.io.get_latest_epoch = MagicMock(return_value=latest_epoch) # no model given = error target_model = None with self.assertRaises(ValueError): self.orga._get_model(target_model, logging=False) # model given = ok target_model = build_test_model() model = self.orga._get_model(target_model, logging=False) self.assertEqual(target_model.to_json(), model.to_json()) def test_load_model_continue_training(self): # latest epoch is not None aka model has been trained before latest_epoch = (1, 1) self.orga.io.get_latest_epoch = MagicMock(return_value=latest_epoch) # no model given = load model target_model = None model_file = os.path.join(self.temp_dir, "test_model.h5") self.orga.io.get_model_path = MagicMock(return_value=model_file) # no model given = ok saved_model = build_test_model() saved_model.save(model_file) loaded_model = self.orga._get_model(model=target_model, logging=False) self.assertEqual(saved_model.to_json(), loaded_model.to_json()) # model given = ok target_model = 1.2 model = self.orga._get_model(target_model, logging=False) self.assertEqual(target_model, model) def test_val_is_due(self): no_of_files = 3 self.orga.cfg.validate_interval = 2 targets = { (1, 1): False, (1, 2): True, (1, 3): True, (2, 1): False, (2, 2): True, } self.orga.io.get_no_of_files = MagicMock(return_value=no_of_files) for epoch, target in targets.items(): value = self.orga.val_is_due(epoch) self.assertEqual(value, target) def test_check_if_pred_already_done(self): epoch, fileno = 1, 1 # these dont matter # latest_prediction_file_no = None, total_files = 5 -> pred_done = False self.orga.io.get_latest_prediction_file_no = MagicMock(return_value=None) self.orga.io.get_no_of_files = MagicMock(return_value=5) value = self.orga._check_if_pred_already_done(epoch, fileno) target = False self.assertEqual(value, target) # latest_prediction_file_no = 2, total_files = 5 -> pred_done = False self.orga.io.get_latest_prediction_file_no = MagicMock(return_value=2) self.orga.io.get_no_of_files = MagicMock(return_value=5) value = self.orga._check_if_pred_already_done(epoch, fileno) target = False self.assertEqual(value, target) # latest_prediction_file_no = 4, total_files = 5 -> pred_done = True self.orga.io.get_latest_prediction_file_no = MagicMock(return_value=2) self.orga.io.get_no_of_files = MagicMock(return_value=5) value = self.orga._check_if_pred_already_done(epoch, fileno) target = False self.assertEqual(value, target)
def setUp(self): self.orga = Organizer("./") self.orga.cfg.zero_center_folder = "nope/"
def setUp(self): self.temp_dir = os.path.join(os.path.dirname(__file__), ".temp", "core") os.mkdir(self.temp_dir) self.orga = Organizer(self.temp_dir)