def test_random_seed_setter():
RandomSeedSetter(seeds=1)
RandomSeedSetter({'random': 1, 'numpy': 2, 'tf-graph': 3, 'tf-global': 4})
pass_on_error(RandomSeedSetter, {'random': 1})
pass_on_error(RandomSeedSetter, seeds=1.)
class TG():
pass
tg = TG()
rss = RandomSeedSetter(freq={
'train:iter': 1, 'train:batch': 1, 'train:epoch': 1,
'val:iter': 1, 'val:batch': 1, 'val:epoch': 1,
'val_end': 1, 'save': 1, 'load': 1})
rss.init_with_traingen(tg)
rss.on_train_iter_end('train:iter')
rss.on_train_batch_end('train:batch')
rss.on_train_epoch_end('train:epoch')
rss.on_val_iter_end('val:iter')
rss.on_val_batch_end('val:batch')
rss.on_val_epoch_end('val:epoch')
rss.on_val_end('val_end')
rss.on_save('save')
rss.on_load('load')
assert all(s == 8 for s in rss.seeds.values()), rss.seeds
def _test_no_supported_file_ext():
C = deepcopy(DATAGEN_CFG)
with tempdir() as dirpath:
plt.plot([0, 1])
plt.gcf().savefig(os.path.join(dirpath, "img.png"))
C['data_path'] = dirpath
pass_on_error(DataGenerator, **C)
def _validate_weighted_slices_range(C): # [util.misc]
C['traingen']['pred_weighted_slices_range'] = (.5, 1.5)
C['traingen']['eval_fn'] = 'evaluate'
pass_on_error(_util_make_autoencoder, C)
C = deepcopy(CONFIGS)
tg = _util_make_autoencoder(C)
tg.pred_weighted_slices_range = (.5, 1.5)
tg._eval_fn_name = 'predict'
tg.datagen.slices_per_batch = None
tg.val_datagen.slices_per_batch = None
pass_on_error(tg._validate_traingen_configs)
C['traingen']['max_is_best'] = True
C['traingen']['eval_fn'] = 'evaluate'
C['traingen']['pred_weighted_slices_range'] = (.1, 1.1)
pass_on_error(_util_make_classifier, C)
C['traingen']['eval_fn'] = 'predict'
pass_on_error(_util_make_classifier, C)
C = deepcopy(CONFIGS)
C['datagen'].pop('slices_per_batch', None)
pass_on_error(_util_make_classifier, C)
C = deepcopy(CONFIGS)
C['traingen']['eval_fn'] = 'predict'
tg = _util_make_classifier(C)
tg.pred_weighted_slices_range = (.1, 1.1)
tg.datagen.slices_per_batch = 1
tg.val_datagen.slices_per_batch = 1
tg._validate_traingen_configs()
def _train_postiter_processing(C): # [train_generator]
tg = _util_make_autoencoder(C)
tg.datagen.batch_exhausted = True
with mock.patch('deeptrain.train_generator.TrainGenerator.'
'_update_train_history') as mock_update:
mock_update.side_effect = Exception
tg._update_temp_history = lambda x: x
pass_on_error(tg._train_postiter_processing, [])
def _validate_metrics(tg):
tg.val_metrics = ['acc', 'loss']
pass_on_error(_validate_traingen_configs, tg)
tg.key_metric_fn = None
tg.custom_metrics = {}
tg._eval_fn_name = 'predict'
pass_on_error(_validate_traingen_configs, tg)
def test_preprocessing(): # [deeptrain.preprocessing]
data = np.random.randn(15, 2)
pass_on_error(preprocessing.numpy2D_to_csv, data, batch_size=16)
lz4f_cache = preprocessing.lz4f
preprocessing.lz4f = None
pass_on_error(preprocessing.numpy_to_lz4f, data)
preprocessing.lz4f = lz4f_cache
def metrics_getattr(C): # [TrainGenerator]
def _raise(Except):
raise Except()
import_err = lambda: _raise(ImportError)
with mock.patch('builtins.__import__', side_effect=import_err):
pass_on_error(getattr, metrics, 'r2_score')
def _validate_key_metric_fn(tg):
tg.custom_metrics = 1
tg.key_metric_fn = None
tg._eval_fn_name = 'predict'
pass_on_error(_validate_traingen_configs, tg)
tg.key_metric_fn = 1
tg._eval_fn_name = 'evaluate'
pass_on_error(_validate_traingen_configs, tg)
def checkpoint(C): # [util.saving]
tg = _util_make_autoencoder(C)
tg.train()
tg.max_checkpoints = -1
with mock.patch('os.remove') as mock_remove:
mock_remove.side_effect = OSError('Permission Denied')
tg.checkpoint(forced=True, overwrite=False)
tg.logdir = None
pass_on_error(tg.checkpoint)
def _test_validate_data_shapes():
tg = TraingenDummy()
tg.model.batch_size = None
tg.batch_size = 16
tg.model.output_shape = (None, 100)
data = {'preds_all': np.random.uniform(0, 1, (3, 16, 100))}
training._validate_data_shapes(tg, data)
data = {'preds_all': np.random.uniform(0, 1, (3, 1, 16, 100))}
pass_on_error(training._validate_data_shapes, tg, data)
def test_main(monkeypatch):
t0 = time()
C = deepcopy(CONFIGS)
with tempdir(C['traingen']['logs_dir']), \
tempdir(C['traingen']['best_models_dir']):
C['traingen'].update(
dict(
val_freq={'batch': 20},
plot_history_freq={'val': 2},
unique_checkpoint_freq={'val': 2},
optimizer_save_configs={'include': ['updates', 'crowbar']},
max_one_best_save=True,
max_checkpoints=3,
))
tg = init_session(C, model=classifier)
tg.train()
_test_load(tg, C)
C['traingen'].update(
dict(
val_freq={'iter': 20},
temp_checkpoint_freq={'val': 3},
optimizer_save_configs={'exclude': ['iterations']},
optimizer_load_configs={'include': ['momentum', 'momentam']},
eval_fn='predict',
key_metric='catco_custom',
custom_metrics={
'catco_custom': metrics.categorical_crossentropy
},
))
tg = init_session(C, model=classifier)
with tempdir() as savedir:
_log_init_state(tg, savedir=savedir, verbose=1)
tg.train()
_test_load(tg, C)
tg = init_session(C, model=classifier)
tg.train()
tg._train_loop_done = True
tg.train()
tg.plot_configs['0']['vhlines']['v'] = 'invalid_vlines'
pass_on_error(tg.get_history_fig)
tg.clear_cache(reset_val_flags=True)
tg._should_do({}, forced=True)
tg.get_last_log('report', best=False)
tg.get_last_log('report', best=True)
tg.get_last_log('init_state')
tg.val_epoch
monkeypatch.setattr('builtins.input', lambda x: 'y')
tg.destroy(confirm=False)
print("\nTime elapsed: {:.3f}".format(time() - t0))
def _test_lz4f_dataset(C):
del C['labels_path']
C['data_path'] = os.path.join(datadir, 'image_lz4f', 'train',
'128batch__1.npy')
pass_on_error(DataGenerator, **C)
C['data_loader'] = 'numpy-lz4f'
pass_on_error(DataGenerator, **C)
C['data_batch_shape'] = (128, 28, 28, 1)
DataGenerator(**C)
def _validate_class_weights(C): # [util.misc]
C['traingen']['class_weights'] = {'0': 1, 1: 2}
pass_on_error(_util_make_classifier, C)
C['traingen']['class_weights'] = {0: 1}
pass_on_error(_util_make_classifier, C)
C = deepcopy(CONFIGS)
tg = _util_make_classifier(C)
tg.model.loss = 'categorical_crossentropy'
tg.class_weights = {0: 1, 2: 5, 3: 6}
tg._validate_traingen_configs()
def _test_init_optimizer():
model = ModelDummy()
model.loss = 'sparse_categorical_crossentropy'
misc._init_optimizer(model)
model.loss = 'hinge'
misc._init_optimizer(model)
model.loss = 'poisson'
misc._init_optimizer(model)
model.loss = 'categorical_crossentropy'
misc._init_optimizer(model, class_weights={0: 1, 1: 5})
misc._init_optimizer(model, class_weights=None)
model.loss = 'goblin'
pass_on_error(misc._init_optimizer, model)
def _traingen_callbacks(C): # [train_generator]
tg = _util_make_autoencoder(C)
tg.callbacks = [1]
pass_on_error(tg._apply_callbacks)
from deeptrain.callbacks import TraingenCallback
tc = TraingenCallback()
def raise_exception(self):
raise NotImplementedError
tc.init_with_traingen = raise_exception
tg.callbacks = [tc]
pass_on_error(tg._init_callbacks)
def _test_make_group_batch_and_labels():
C = deepcopy(DATAGEN_CFG)
dg = DataGenerator(**C)
dg.batch = np.random.randn(128, 10)
dg.labels = np.random.randn(129, 10)
pass_on_error(dg._make_group_batch_and_labels, n_batches=2)
dg.shuffle_group_samples = True
dg.labels = dg.batch.copy()
dg._make_group_batch_and_labels(n_batches=2)
dg.labels_path = None
dg._make_group_batch_and_labels(n_batches=2)
dg.shuffle_group_batches = True
dg.shuffle_group_samples = False
dg._make_group_batch_and_labels(n_batches=2)
def _get_best_subset_val_history(C): # [util.training]
C['traingen']['best_subset_size'] = 2
tg = _util_make_classifier(C)
tg.val_datagen.slices_per_batch = 4
tg._labels_cache = np.random.randint(0, 2, (3, 4, batch_size, 1))
tg._preds_cache = np.random.uniform(0, 1, (3, 4, batch_size, 1))
tg._sw_cache = np.random.randint(0, 2, (3, 4, batch_size, 1))
tg._class_labels_cache = tg._labels_cache.copy()
tg._val_set_name_cache = ['1', '2', '3']
tg.key_metric = 'f1_score'
tg.val_temp_history = {'f1_score': []}
tg.key_metric_fn = metrics.f1_score
tg._eval_fn_name = 'predict'
tg.dynamic_predict_threshold_min_max = None
tg._get_best_subset_val_history()
tg._eval_fn_name = 'superfit'
pass_on_error(tg._get_best_subset_val_history)
def _test_misc():
C = deepcopy(DATAGEN_CFG)
dg = DataGenerator(**C)
dg.superbatch = {'1': 1, '2': 2}
dg.superbatch_set_nums = ['3']
pass_on_error(dg._get_next_batch, set_num='3', warn=True)
dg.all_labels = {}
pass_on_error(dg._get_next_labels, set_num='3')
pass_on_error(setattr, dg, 'load_data', 1)
pass_on_error(setattr, dg, 'load_labels', 1)
with tempdir() as dirpath:
path = os.path.join(dirpath, "arr.npy")
np.save(path, np.array([1]))
C = deepcopy(DATAGEN_CFG)
C['labels_path'] = None
C['data_path'] = path
pass_on_error(DataGenerator, **C)
def _test_infer_and_set_info():
C = deepcopy(DATAGEN_CFG)
with tempdir() as dirpath:
path = os.path.join(dirpath, "arr.npy")
np.save(path, np.array([1]))
C['labels_path'] = None
C['data_loader'] = DataLoader(path, loader='numpy')
DataGenerator(**C)
C['labels_loader'] = DataLoader(path, loader='numpy')
DataGenerator(**C)
C['data_loader'] = DataGenerator
pass_on_error(DataGenerator, **C)
C['labels_loader'] = None
C['data_loader'] = DataLoader
DataGenerator(**C)
C['labels_loader'] = DataGenerator
pass_on_error(DataGenerator, **C)
def _test_generate_report():
tg = TraingenDummy()
tg.report_configs = {'model': dict(stuff='staff'), 'saitama': None}
pass_on_error(logging.generate_report, tg)
tg.report_configs = {'model': {'genos': [1]}}
pass_on_error(logging.generate_report, tg)
tg.report_configs = {'model': {'include': [], 'exclude': []}}
pass_on_error(logging.generate_report, tg)
tg.report_configs = {'model': {'exclude_types': ['etc']}}
tg.model_configs = {'a': 1}
pass_on_error(logging.generate_report, tg)
def _test_validate_args(C):
pass_on_error(DataLoader, 1, 1)
kw = dict(path=C['data_path'], loader=1, filepaths=None)
pass_on_error(DataLoader, **kw)
kw['filepaths'] = ['x']
pass_on_error(DataLoader, **kw)
def _validate_callbacks(C): # [util.misc]
C['traingen']['callbacks'] = {'invalid_stage': 1}
pass_on_error(_util_make_autoencoder, C)
C['traingen']['callbacks'] = {'save': 1}
pass_on_error(_util_make_autoencoder, C)
C['traingen']['callbacks'] = 1
pass_on_error(_util_make_autoencoder, C)
def _test_init():
C = deepcopy(DATAGEN_CFG)
C['superbatch_set_nums'] = 'all'
C['superbatch_path'] = 'x'
pass_on_error(DataGenerator, **C)
C = deepcopy(DATAGEN_CFG)
C['labels_path'] = 1
pass_on_error(DataGenerator, **C)
C['data_path'] = 1
pass_on_error(DataGenerator, **C)
def save(C): # [util.saving]
tg = _util_make_autoencoder(C)
tg.checkpoint()
tg.model.loss = 'mean_squared_error'
tg.train()
tg.final_fig_dir = tg.logdir
pass_on_error(tg.load)
pass_on_error(tg.checkpoint, overwrite="underwrite")
tg.datagen.set_nums_to_process = [9001]
tg.save()
tg._save_history_fig()
tg._save_history_fig()
tg.optimizer_load_configs = {'exclude': ['weights']}
tg.loadskip_list = ['optimizer_load_configs']
tg.datagen.loadskip_list = ['stuff']
tg.load()
tg._history_fig = 1
tg._save_history_fig()
tg.loadskip_list = 'auto'
tg.load()
tg.loadskip_list = 'none'
tg.load()
tg.optimizer_save_configs = {'include': []}
tg.save()
with mock.patch('backend.K.get_value') as mock_get_value:
mock_get_value.side_effect = Exception()
tg.save()
tg.optimizer_save_configs = {'include': ['leaking_rate']}
tg.datagen.group_batch = []
with mock.patch('pickle.dump') as mock_dump:
mock_dump.side_effect = Exception()
tg.save()
tg.logdir = 'abc'
pass_on_error(tg.load)
tg.logdir = None
pass_on_error(tg.load)
def _validate_directories(C): # [util.misc]
C['traingen']['best_models_dir'] = None
pass_on_error(_util_make_classifier, C)
C = deepcopy(CONFIGS)
C['traingen']['logs_dir'] = None
pass_on_error(_util_make_classifier, C)
C = deepcopy(CONFIGS)
C['traingen']['best_models_dir'] = None
C['traingen']['logs_dir'] = None
pass_on_error(_util_make_classifier, C)
def _on_val_end(C): # [train_generator]
tg = _util_make_autoencoder(C)
tg.batch_size = 'x'
pass_on_error(tg._on_val_end)
tg.epochs += 1
tg._train_loop_done = True
def validate(self):
self._train_loop_done = False
tg.validate = validate.__get__(tg)
pass_on_error(tg.train)
tg._train_loop_done = False
tg._fit_fn_name = 'x'
tg.epochs += 1
pass_on_error(tg.train)
tg.batch_size = 'a'
tg._eval_fn_name = 'predict'
pass_on_error(tg._on_val_end, 0, 0, 0)
def test_advance_batch():
C = deepcopy(DATAGEN_CFG)
C['superbatch_path'] = os.path.join(datadir, 'image', 'train')
dg = DataGenerator(**C)
dg.advance_batch()
C['batch_size'] = 31
dg = DataGenerator(**C)
pass_on_error(dg.advance_batch)
C['batch_size'] = 256
dg = DataGenerator(**C)
dg.set_nums_to_process = []
pass_on_error(dg.advance_batch)
C['data_loader'] = 'pigeon'
pass_on_error(DataGenerator, **C)
def _update_temp_history(C): # [util.training]
tg = _util_make_classifier(C)
tg.val_temp_history['loss'] = (1, 2, 3)
tg._update_temp_history(metrics=(4, ), val=True)
tg.val_temp_history['loss'] = []
tg._update_temp_history(metrics=(4, ), val=True)
tg.datagen.slice_idx = 1
tg.datagen.slices_per_batch = 2
tg.temp_history = {'binary_accuracy': []}
tg.train_metrics = ['binary_accuracy']
pass_on_error(tg._update_temp_history, metrics=[1], val=False)
pass_on_error(tg._update_temp_history,
metrics=[dict(a=1, b=2)],
val=False)
# tg._update_temp_history([[1]], val=False) # tests `_handle_non_scalar`
tg.temp_history = {'f1_score': []}
tg.train_metrics = ['f1_score']
pass_on_error(tg._update_temp_history, metrics=[[1, 2]], val=False)
def _layer_hists(model):
pass_on_error(layer_hists, model, '*', mode='gradients')
pass_on_error(layer_hists, model, '*', mode='outputs')
pass_on_error(layer_hists, model, '*', mode='skeletons')
def test_data_to_hdf5(monkeypatch): # [deeptrain.preprocessing]
"""Dedicated test since it uses monkeypatch"""
C = deepcopy(CONFIGS)
# set preemptively in case data.h5 somehow found in dir
monkeypatch.setattr('builtins.input', lambda x: 'y')
with tempdir(C['traingen']['logs_dir']) as loaddir:
with open(os.path.join(loaddir, "data.txt"), 'w') as txt:
txt.write("etc")
savepath = os.path.join(loaddir, "data.h5")
pass_on_error(preprocessing.data_to_hdf5,
savepath.replace('.h5', ''),
batch_size=32,
loaddir=loaddir)
data = np.random.randn(1, 32, 100)
np.save(os.path.join(loaddir, "data.npy"), data)
pass_on_error(preprocessing.data_to_hdf5,
savepath=savepath,
batch_size=32,
loaddir=loaddir)
kw = dict(savepath=savepath, data=data, batch_size=32)
pass_on_error(preprocessing.data_to_hdf5, **kw)
os.remove(os.path.join(loaddir, "data.txt"))
preprocessing.data_to_hdf5(**kw)
monkeypatch.setattr('builtins.input', lambda x: 'y')
preprocessing.data_to_hdf5(**kw)
monkeypatch.setattr('builtins.input', lambda x: 'n')
pass_on_error(preprocessing.data_to_hdf5, **kw)
preprocessing.data_to_hdf5(overwrite=True, **kw)
pass_on_error(preprocessing.data_to_hdf5, overwrite=False, **kw)
pass_on_error(preprocessing.data_to_hdf5,
kw['savepath'],
kw['batch_size'],
loaddir=None,
data=None)
pass_on_error(preprocessing.data_to_hdf5,
kw['savepath'],
kw['batch_size'],
loaddir=loaddir,
data=data)
_data = [data[0], data[0, :31]]
pass_on_error(preprocessing.data_to_hdf5,
kw['savepath'],
kw['batch_size'],
data=_data,
overwrite=True)
_data = [np.vstack([data[0], data[0]])]
pass_on_error(preprocessing.data_to_hdf5,
kw['savepath'],
kw['batch_size'],
data=_data,
overwrite=True)
