class TestZeroCenter(TestCase):
def setUp(self):
self.orga = Organizer("./")
self.orga.cfg.zero_center_folder = "nope/"
def test_get_xs_mean_existing(self):
target = "asd"
self.orga.xs_mean = target
value = self.orga.get_xs_mean(logging=False)
self.assertEqual(target, value)
def test_get_xs_mean_no_folder(self):
self.orga.cfg.zero_center_folder = None
with self.assertRaises(ValueError):
self.orga.get_xs_mean(logging=False)
@patch('orcanet.core.load_zero_center_data')
def test_get_xs_mean_new(self, load_mock):
target = "blub"
def mock_xs_mean():
return target
load_mock.return_value = mock_xs_mean()
xs_mean = self.orga.get_xs_mean(logging=False)
self.assertEqual(target, xs_mean)
self.assertEqual(target, self.orga.xs_mean)
def inference(directory,
list_file=None,
config_file=None,
epoch=None,
fileno=None):
from orcanet.core import Organizer
orga = Organizer(directory, list_file, config_file, tf_log_level=1)
return orga.inference(epoch=epoch, fileno=fileno)
def make_orga():
orga = Organizer(self.output_folder, config_file=config_file)
orga.cfg._files_dict["train"] = self.train_pathes
orga.cfg._files_dict["val"] = self.val_pathes
orga.cfg._list_file = "test.toml"
orga.cfg.zero_center_folder = self.temp_dir
orga.cfg.label_modifier = label_modifier
orga.cfg.custom_objects = get_custom_objects()
return orga
def get_orga(dims):
dimdict = {
2: get_input_shapes_2d,
3: get_input_shapes_3d,
"graph": get_input_shapes_graph
}
orga = Organizer(".")
orga.io.get_input_shapes = dimdict[dims]
return orga
def setUp(self):
self.data_folder = os.path.join(os.path.dirname(__file__), "data")
self.output_folder = self.data_folder + "/dummy_model"
list_file = self.data_folder + "/in_out_test_list.toml"
config_file = None
self.orga = Organizer(self.output_folder, list_file, config_file)
self.batchsize = 2
self.orga.cfg.batchsize = self.batchsize
self.orga.cfg.label_modifier = label_modifier
def orca_pred(output_folder,
list_file,
config_file,
model_file,
epoch=None,
fileno=None):
"""
Run orga.predict with predefined ModelBuilder networks using a parser.
Per default, the most recent saved model will be loaded.
Parameters
----------
output_folder : str
Path to the folder where everything gets saved to, e.g. the summary
log file, the plots, the trained models, etc.
list_file : str
Path to a list file which contains pathes to all the h5 files that
should be used for training and validation.
config_file : str
Path to a .toml file which overwrites some of the default settings
for training and validating a model.
model_file : str
Path to a file with parameters to build a model of a predefined
architecture with OrcaNet.
epoch : int, optional
The epoch of the saved model to predict with.
fileno : int, optional
The filenumber of the saved model to predict with.
"""
# Set up the Organizer with the input data
orga = Organizer(output_folder, list_file, config_file, tf_log_level=1)
# When predicting with a orga model, the right modifiers and custom
# objects need to be given
update_objects(orga, model_file)
# Per default, a prediction will be done for the model with the
# highest epoch and filenumber.
orga.predict(epoch=epoch, fileno=fileno)
def train(directory,
list_file=None,
config_file=None,
model_file=None,
to_epoch=None):
from orcanet.core import Organizer
from orcanet.model_builder import ModelBuilder
from orcanet.misc import find_file
orga = Organizer(directory, list_file, config_file, tf_log_level=1)
if orga.io.get_latest_epoch() is None:
# Start of training
print("Building new model")
if model_file is None:
model_file = find_file(directory, "model.toml")
model = ModelBuilder(model_file).build(orga, verbose=False)
else:
model = None
return orga.train_and_validate(model=model, to_epoch=to_epoch)
def setUp(self):
self.orga = Organizer(self.temp_dir)
self.orga.cfg.batchsize = 9
self.orga.io.get_local_files = MagicMock(return_value=self.filepaths)
self.orga.io.get_file_sizes = MagicMock(return_value=self.file_sizes)
self.orga.cfg.get_list_file = MagicMock(return_value='/path/to/a/listfilename.toml')
self.orga.io.get_next_pred_path = MagicMock(return_value=self.pred_filepath)
self.model = build_dummy_model(self.input_shapes, self.output_shapes)
self.model.compile(loss="mse", optimizer="sgd")
self.orga._auto_label_modifier = lib.label_modifiers.ColumnLabels(self.model)
def setUp(self):
# Size of train files. No actual files get generated (they are mocked).
file_sizes = [100, 66]
self.temp_dir = os.path.join(os.path.dirname(__file__), ".temp")
self.summary_file = os.path.join(self.temp_dir, "summary.txt")
self.orga = Organizer(self.temp_dir)
self.orga.io.get_file_sizes = MagicMock(return_value=file_sizes)
model = build_test_model()
self.smry = SummaryLogger(self.orga, model)
# tf 2.2: train the model to set metric_names
model.train_on_batch(np.ones((2, 1)), np.ones((2, 1)))
self.metrics = model.metrics_names
def setUp(self):
self.temp_dir = os.path.join(os.path.dirname(__file__), ".temp",
"batch_logger")
self.model = build_test_model()
samples_file1 = 40
samples_file2 = 60
self.data_file1 = get_test_data(samples_file1)
self.data_file2 = get_test_data(samples_file2)
self.orga = Organizer(self.temp_dir)
self.orga.io.get_file_sizes = MagicMock(
return_value=[samples_file1, samples_file2])
self.orga.io.get_subfolder("train_log", create=True)
self.orga.cfg.batchsize = 10
self.orga.cfg.train_logger_display = 4
def setUp(self):
# Pathes
# Temporary output folder
self.temp_dir = os.path.join(os.path.dirname(__file__), ".temp",
"nn_utilities")
self.output_folder = os.path.join(self.temp_dir, "model")
self.data_folder = os.path.join(os.path.dirname(__file__), "data")
config_file = os.path.join(self.data_folder, "config_test.toml")
# Make sure temp dir does either not exist or is empty
if os.path.exists(self.temp_dir):
assert len(os.listdir(self.temp_dir)) == 0
else:
os.makedirs(self.temp_dir)
# Make dummy data
self.train_inp_A = (self.temp_dir + "/train1_A.h5",
self.temp_dir + "/train2_A.h5")
self.train_inp_B = (
self.temp_dir + "/train1_B.h5",
self.temp_dir + "/train2_B.h5",
)
self.train_pathes = {
"testing_input_A": self.train_inp_A,
"testing_input_B": self.train_inp_B,
}
self.shape = (3, 3, 3)
make_dummy_data(self.train_inp_A[0],
self.train_inp_A[1],
self.shape,
mode=1)
make_dummy_data(self.train_inp_B[0],
self.train_inp_B[1],
self.shape,
mode=2)
orga = Organizer(self.output_folder, config_file=config_file)
orga.cfg._files_dict["train"] = self.train_pathes
orga.cfg._list_file = "test.toml"
orga.cfg.zero_center_folder = self.temp_dir
self.orga = orga
def setUp(self):
"""
Make a .temp directory in the current working directory, generate
dummy data in it and set up the cfg object.
"""
self.data_folder = os.path.join(os.path.dirname(__file__), "data")
self.model_file = os.path.join(self.data_folder,
"model_builder_test_CNN_cat.toml")
self.input_shapes = {
"input_A": (3, 3, 3, 3),
}
orga = Organizer(".")
orga.io.get_input_shapes = MagicMock(return_value=self.input_shapes)
orga.cfg.custom_objects = get_custom_objects()
self.orga = orga
def use_orcanet():
temp_folder = "output/"
os.mkdir(temp_folder)
make_dummy_data(temp_folder)
list_file = "example_list.toml"
organizer = Organizer(temp_folder + "sum_model", list_file)
organizer.cfg.train_logger_display = 10
model = make_dummy_model()
organizer.train_and_validate(model, epochs=3)
organizer.predict()
def orca_train(output_folder,
list_file,
config_file,
model_file,
recompile_model=False):
"""
Run orga.train with predefined ModelBuilder networks using a parser.
Parameters
----------
output_folder : str
Path to the folder where everything gets saved to, e.g. the summary
log file, the plots, the trained models, etc.
list_file : str
Path to a list file which contains pathes to all the h5 files that
should be used for training and validation.
config_file : str
Path to a .toml file which overwrites some of the default settings
for training and validating a model.
model_file : str
Path to a file with parameters to build a model of a predefined
architecture with OrcaNet.
recompile_model : bool
If the model should be recompiled or not. Necessary, if e.g. the
loss_weights are changed during the training.
"""
# Set up the Organizer with the input data
orga = Organizer(output_folder, list_file, config_file, tf_log_level=1)
# Load in the orga sample-, label-, and dataset-modifiers, as well as
# the custom objects
update_objects(orga, model_file)
# If this is the start of the training, a compiled model needs to be
# handed to the orga.train function
if orga.io.get_latest_epoch() is None:
# The ModelBuilder class allows to construct models from a toml file,
# adapted to the datasets in the orga instance. Its modifiers will
# be taken into account for this
builder = ModelBuilder(model_file)
model = builder.build(orga, log_comp_opts=True)
elif recompile_model is True:
builder = ModelBuilder(model_file)
path_of_model = orga.io.get_model_path(-1, -1)
model = ks.models.load_model(
path_of_model, custom_objects=orga.cfg.get_custom_objects())
print("Recompiling the saved model")
model = builder.compile_model(
model, custom_objects=orga.cfg.get_custom_objects())
builder.log_model_properties(orga)
else:
model = None
try:
# Use a custom LR schedule
user_lr = orga.cfg.learning_rate
lr = orca_learning_rates(user_lr, orga.io.get_no_of_files("train"))
orga.cfg.learning_rate = lr
except NameError:
pass
# start the training
orga.train_and_validate(model=model)
class TestOrganizer(tf.test.TestCase):
"""
Test the organizer by creating a dummy directory .temp/core.
"""
def setUp(self):
self.temp_dir = os.path.join(os.path.dirname(__file__), ".temp", "core")
os.mkdir(self.temp_dir)
self.orga = Organizer(self.temp_dir)
def tearDown(self):
shutil.rmtree(self.temp_dir)
@patch('orcanet.core.tf.keras.utils.plot_model')
def test_load_model_new_training(self, mock_plot_model):
def plot_history(model, folder):
pass
mock_plot_model.side_effect = plot_history
# latest epoch is None aka no model has been trained
latest_epoch = None
self.orga.io.get_latest_epoch = MagicMock(return_value=latest_epoch)
# no model given = error
target_model = None
with self.assertRaises(ValueError):
self.orga._get_model(target_model, logging=False)
# model given = ok
target_model = build_test_model()
model = self.orga._get_model(target_model, logging=False)
self.assertEqual(target_model.to_json(), model.to_json())
def test_load_model_continue_training(self):
# latest epoch is not None aka model has been trained before
latest_epoch = (1, 1)
self.orga.io.get_latest_epoch = MagicMock(return_value=latest_epoch)
# no model given = load model
target_model = None
model_file = os.path.join(self.temp_dir, "test_model.h5")
self.orga.io.get_model_path = MagicMock(return_value=model_file)
# no model given = ok
saved_model = build_test_model()
saved_model.save(model_file)
loaded_model = self.orga._get_model(model=target_model, logging=False)
self.assertEqual(saved_model.to_json(), loaded_model.to_json())
# model given = ok
target_model = 1.2
model = self.orga._get_model(target_model, logging=False)
self.assertEqual(target_model, model)
def test_val_is_due(self):
no_of_files = 3
self.orga.cfg.validate_interval = 2
targets = {
(1, 1): False,
(1, 2): True,
(1, 3): True,
(2, 1): False,
(2, 2): True,
}
self.orga.io.get_no_of_files = MagicMock(return_value=no_of_files)
for epoch, target in targets.items():
value = self.orga.val_is_due(epoch)
self.assertEqual(value, target)
def test_check_if_pred_already_done(self):
epoch, fileno = 1, 1 # these dont matter
# latest_prediction_file_no = None, total_files = 5 -> pred_done = False
self.orga.io.get_latest_prediction_file_no = MagicMock(return_value=None)
self.orga.io.get_no_of_files = MagicMock(return_value=5)
value = self.orga._check_if_pred_already_done(epoch, fileno)
target = False
self.assertEqual(value, target)
# latest_prediction_file_no = 2, total_files = 5 -> pred_done = False
self.orga.io.get_latest_prediction_file_no = MagicMock(return_value=2)
self.orga.io.get_no_of_files = MagicMock(return_value=5)
value = self.orga._check_if_pred_already_done(epoch, fileno)
target = False
self.assertEqual(value, target)
# latest_prediction_file_no = 4, total_files = 5 -> pred_done = True
self.orga.io.get_latest_prediction_file_no = MagicMock(return_value=2)
self.orga.io.get_no_of_files = MagicMock(return_value=5)
value = self.orga._check_if_pred_already_done(epoch, fileno)
target = False
self.assertEqual(value, target)
def setUp(self):
self.orga = Organizer("./")
self.orga.cfg.zero_center_folder = "nope/"
def setUp(self):
self.temp_dir = os.path.join(os.path.dirname(__file__), ".temp", "core")
os.mkdir(self.temp_dir)
self.orga = Organizer(self.temp_dir)
