def test_01_gen_model(self):
path = ROOT_PATH
manager = InstanceManager(path)
model = DummyModel
input_shape = (32, 32, 3)
manager.gen_instance(model, input_shape)
def load_and_sampling(input_shapes=None,
visualizers=None,
metadata_path=None,
env_setting=None):
manager = InstanceManager(env_setting)
manager.load_instance(instance_path=metadata_path,
input_shapes=input_shapes)
manager.load_visualizer(visualizers)
manager.sampling_instance(is_restore=True)
def build_and_train(model=None,
input_shapes=None,
dataset=None,
visualizers=None,
epoch_time=50,
check_point_interval=5000):
manager = InstanceManager()
metadata_path = manager.build_instance(model)
manager.load_instance(metadata_path, input_shapes)
for visualizer, interval in visualizers:
manager.load_visualizer(visualizer, interval)
manager.train_instance(epoch_time,
dataset=dataset,
check_point_interval=check_point_interval)
del manager
def tf_model_train(model=None, dataset=None, visuliziers=None, epoch=None):
dataset = DatasetLoader().load_dataset(dataset)
input_shapes = dataset.train_set.input_shapes
model = ModelClassLoader.load_model_class(model)
manager = InstanceManager()
metadata_path = manager.build_instance(model, input_shapes)
manager.load_instance(metadata_path)
for v_fun, i in visuliziers:
manager.load_visualizer(VisualizerClassLoader.load(v_fun), i)
manager.train_instance(epoch=epoch,
dataset=dataset,
check_point_interval=5000,
with_tensorboard=True)
del manager
def test_03_train_model(self):
path = ROOT_PATH
manager = InstanceManager(path)
model = DummyModel
manager.build_instance(model)
visualizer_1 = DummyVisualizer_1
visualizer_2 = DummyVisualizer_2
visualizers = [visualizer_1, visualizer_2]
manager.load_visualizer(visualizers)
dataset = DummyDataset()
epoch_time = 10
manager.train_instance(dataset, epoch_time)
def test_03_train_model(self):
path = ROOT_PATH
manager = InstanceManager(path)
model = DummyModel
input_shape = (32, 32, 3)
manager.gen_instance(model, input_shape)
visualizer_1 = DummyVisualizer_1
visualizer_2 = DummyVisualizer_2
visualizers = [visualizer_1, visualizer_2]
manager.load_visualizer(visualizers)
dataset = DummyDataset()
epoch_time = 10
manager.train_model(dataset, epoch_time)
def f():
path = ROOT_PATH
manager = InstanceManager(path)
model = DummyModel
input_shape = (32, 32, 3)
manager.gen_instance(model, input_shape)
visualizers = [
(dummy_log, 10),
]
manager.load_visualizer(visualizers)
dataset = DummyDataset()
epoch_time = 1
check_point_interval = 500
manager.train_model(dataset, epoch_time, check_point_interval)
def test_04_save_model(self):
path = ROOT_PATH
manager = InstanceManager(path)
model = DummyModel
manager.build_instance(model)
visualizer_1 = DummyVisualizer_1
visualizer_2 = DummyVisualizer_2
visualizers = [visualizer_1, visualizer_2]
manager.load_visualizer(visualizers)
dataset = DummyDataset()
epoch_time = 10
check_point_interval = 2
manager.train_instance(dataset, epoch_time, check_point_interval)
def test_02_load_visualizer(self):
path = ROOT_PATH
manager = InstanceManager(path)
model = DummyModel
manager.build_instance(model)
visualizer_1 = DummyVisualizer_1
visualizer_2 = DummyVisualizer_2
visualizers = [visualizer_1, visualizer_2]
manager.load_visualizer(visualizers)
def test_02_load_visualizer(self):
path = ROOT_PATH
manager = InstanceManager(path)
model = DummyModel
input_shape = (32, 32, 3)
manager.gen_instance(model, input_shape)
visualizer_1 = DummyVisualizer_1
visualizer_2 = DummyVisualizer_2
visualizers = [visualizer_1, visualizer_2]
manager.load_visualizer(visualizers)
# build model
from model.ModelClassLoader import ModelClassLoader
from InstanceManger import InstanceManager
model = ModelClassLoader.load_model_class("model")
manager = InstanceManager()
model_metadata_path = manager.build_instance(model)
# load model
# after build model
manager.load_instance(model_metadata_path, input_shapes_from_dataset)
# load visualizer
# feed visualizer_class
from visualizer.VisualizerClassLoader import VisualizerClassLoader
visualizer = VisualizerClassLoader.load("visualizer_name")
manager.load_visualizer(visualizer, execute_interval=10)
# train model
# after load instance into manager train model
manager.train_instance(
epoch_time,
dataset=dataset,
check_point_interval=check_point_interval,
# if need to start train from checkpoint
# set is_restore to True
# is_restore=True
# default to start with tensorboard sub process while train instance and after train instance close tensorboard
# if does not need tensorboard set with_tensorboard to False
import os
class dummy_log(AbstractPrintLog):
def task(self, sess=None, iter_num=None, model=None, dataset=None):
super().task(sess, iter_num, model, dataset)
self.log('this is dummy log')
if __name__ == '__main__':
root_path = ROOT_PATH
LLD_PATH = os.path.join(root_path, 'dataset', 'LLD')
lld_data = LLD()
lld_data.load(LLD_PATH)
manager = InstanceManager(ROOT_PATH)
input_shape = [32, 32, 3]
model = GAN
manager.gen_instance(model, input_shape)
metadata_path = manager.metadata_path
iter_cycle = 10
visualizers = [(dummy_log, iter_cycle)]
manager.load_visualizer(visualizers)
epoch_time = 10
check_point_interval_per_iter = 5000
manager.train_model(lld_data, epoch_time, check_point_interval_per_iter)
def test_05_load_model(self):
path = ROOT_PATH
manager = InstanceManager(path)
model = DummyModel
input_shape = (32, 32, 3)
manager.gen_instance(model, input_shape)
meta_data_path = manager.metadata_path
visualizer_1 = DummyVisualizer_1
visualizer_2 = DummyVisualizer_2
visualizers = [visualizer_1, visualizer_2]
manager.load_visualizer(visualizers)
dataset = DummyDataset()
epoch_time = 10
check_point_interval = 2
manager.train_model(dataset, epoch_time, check_point_interval)
#
manager = InstanceManager(path)
manager.load_model(meta_data_path, input_shape)
visualizer_1 = DummyVisualizer_1
visualizer_2 = DummyVisualizer_2
visualizers = [visualizer_1, visualizer_2]
manager.load_visualizer(visualizers)
dataset = DummyDataset()
epoch_time = 10
check_point_interval = 2
manager.train_model(dataset, epoch_time, check_point_interval)
def test_01_gen_model(self):
path = ROOT_PATH
manager = InstanceManager(path)
model = DummyModel
manager.build_instance(model)
visualizer_1 = DummyVisualizer_1
visualizer_2 = DummyVisualizer_2
visualizers = [visualizer_1, visualizer_2]
manager.load_visualizer(visualizers)
dataset = DummyDataset()
epoch_time = 10
check_point_interval = 2
manager.train_instance(dataset, epoch_time, check_point_interval)
if __name__ == '__main__':
path = ROOT_PATH
manager = InstanceManager(path)
model = DummyModel
input_shape = (32, 32, 3)
manager.build_instance(model)
visualizers = [
(dummy_log, 10),
]
manager.load_visualizer(visualizers)
dataset = DummyDataset()
epoch_time = 1
check_point_interval = 500
manager.train_instance(dataset, epoch_time, check_point_interval)
def test_00_init(self):
path = ROOT_PATH
InstanceManager(path)
def test_models(model_list=None, input_shapes=None, env_setting=None):
for model in model_list:
manager = InstanceManager(env_setting)
manager.gen_instance(model, input_shapes)
del manager
def load_model_and_train(input_shapes=None, visualizers=None, metadata_path=None, env_setting=None, dataset=None,
epoch_time=50, check_point_interval_per_iter=5000):
manager = None
try:
manager = InstanceManager(env_setting)
manager.load_model(metadata_path=metadata_path, input_shapes=input_shapes)
manager.load_visualizer(visualizers)
manager.open_tensorboard()
manager.train_model(dataset, epoch_time, check_point_interval_per_iter, is_restore=True)
except Exception as e:
print(e)
finally:
manager.close_tensorboard()
del manager
def build_test_model(model_list=None, env_setting=None):
for model in model_list:
manager = InstanceManager(env_setting)
manager.build_instance(model)
del manager
# usage
from DatasetLoader import DatasetLoader
from InstanceManger import InstanceManager
# load dataset by calling DatasetLoader
# input_shapes is for tensorflow.PlaceHolder's shape
# need to build instanc e
dataset, input_shapes = DatasetLoader().load_dataset("dataset_name")
# apply to train model
instanceManager = InstanceManager()
instanceManager.train_instance(epoch_time,
dataset=dataset,
check_point_interval=check_point_interval)
# 1. add dataset folder path in **env_setting.py**
EXAMPLE_DATASET_PATH = os.path.join(DATA_PATH, 'example_dataset')
# 2. add dataset_batch_keys in **dict_keys.dataset_batch_keys.py**
INPUT_SHAPE_KEY_DATA_X = "INPUT_SHAPE_KEY_DATA_X"
INPUT_SHAPE_KEY_DATA_X = "INPUT_SHAPE_KEY_DATA_X"
INPUT_SHAPE_KEY_LABEL = "INPUT_SHAPE_KEY_LABEL"
INPUT_SHAPE_KEY_LABEL_SIZE = "INPUT_SHAPE_KEY_LABEL_SIZE"
# 3. add input_shapes_keys in **dict_keys.dataset_batch_keys.py**
BATCH_KEY_EXAMPLE_TRAIN_X = "BATCH_KEY_EXAMPLE_TRAIN_X"
BATCH_KEY_EXAMPLE_TEST_X = "BATCH_KEY_EXAMPLE_TEST_X"
BATCH_KEY_EXAMPLE_TRAIN_LABEL = "BATCH_KEY_EXAMPLE_TRAIN_LABEL"
BATCH_KEY_EXAMPLE_TEST_LABEL = "BATCH_KEY_EXAMPLE_TEST_LABEL"
# 4. implement dataset class in **data_handler.dataset_name.py**
def gen_model_and_train(model=None, input_shapes=None, dataset=None, visualizers=None, env_path=None, epoch_time=50,
check_point_interval_per_iter=5000):
manager = InstanceManager(env_path)
manager.gen_instance(model, input_shapes)
manager.load_visualizer(visualizers)
manager.open_tensorboard()
manager.train_model(dataset, epoch_time, check_point_interval_per_iter)
manager.close_tensorboard()
del manager