def test_random_seed_setter(): RandomSeedSetter(seeds=1) RandomSeedSetter({'random': 1, 'numpy': 2, 'tf-graph': 3, 'tf-global': 4}) pass_on_error(RandomSeedSetter, {'random': 1}) pass_on_error(RandomSeedSetter, seeds=1.) class TG(): pass tg = TG() rss = RandomSeedSetter(freq={ 'train:iter': 1, 'train:batch': 1, 'train:epoch': 1, 'val:iter': 1, 'val:batch': 1, 'val:epoch': 1, 'val_end': 1, 'save': 1, 'load': 1}) rss.init_with_traingen(tg) rss.on_train_iter_end('train:iter') rss.on_train_batch_end('train:batch') rss.on_train_epoch_end('train:epoch') rss.on_val_iter_end('val:iter') rss.on_val_batch_end('val:batch') rss.on_val_epoch_end('val:epoch') rss.on_val_end('val_end') rss.on_save('save') rss.on_load('load') assert all(s == 8 for s in rss.seeds.values()), rss.seeds
def _test_no_supported_file_ext(): C = deepcopy(DATAGEN_CFG) with tempdir() as dirpath: plt.plot([0, 1]) plt.gcf().savefig(os.path.join(dirpath, "img.png")) C['data_path'] = dirpath pass_on_error(DataGenerator, **C)
def _validate_weighted_slices_range(C): # [util.misc] C['traingen']['pred_weighted_slices_range'] = (.5, 1.5) C['traingen']['eval_fn'] = 'evaluate' pass_on_error(_util_make_autoencoder, C) C = deepcopy(CONFIGS) tg = _util_make_autoencoder(C) tg.pred_weighted_slices_range = (.5, 1.5) tg._eval_fn_name = 'predict' tg.datagen.slices_per_batch = None tg.val_datagen.slices_per_batch = None pass_on_error(tg._validate_traingen_configs) C['traingen']['max_is_best'] = True C['traingen']['eval_fn'] = 'evaluate' C['traingen']['pred_weighted_slices_range'] = (.1, 1.1) pass_on_error(_util_make_classifier, C) C['traingen']['eval_fn'] = 'predict' pass_on_error(_util_make_classifier, C) C = deepcopy(CONFIGS) C['datagen'].pop('slices_per_batch', None) pass_on_error(_util_make_classifier, C) C = deepcopy(CONFIGS) C['traingen']['eval_fn'] = 'predict' tg = _util_make_classifier(C) tg.pred_weighted_slices_range = (.1, 1.1) tg.datagen.slices_per_batch = 1 tg.val_datagen.slices_per_batch = 1 tg._validate_traingen_configs()
def _train_postiter_processing(C): # [train_generator] tg = _util_make_autoencoder(C) tg.datagen.batch_exhausted = True with mock.patch('deeptrain.train_generator.TrainGenerator.' '_update_train_history') as mock_update: mock_update.side_effect = Exception tg._update_temp_history = lambda x: x pass_on_error(tg._train_postiter_processing, [])
def _validate_metrics(tg): tg.val_metrics = ['acc', 'loss'] pass_on_error(_validate_traingen_configs, tg) tg.key_metric_fn = None tg.custom_metrics = {} tg._eval_fn_name = 'predict' pass_on_error(_validate_traingen_configs, tg)
def test_preprocessing(): # [deeptrain.preprocessing] data = np.random.randn(15, 2) pass_on_error(preprocessing.numpy2D_to_csv, data, batch_size=16) lz4f_cache = preprocessing.lz4f preprocessing.lz4f = None pass_on_error(preprocessing.numpy_to_lz4f, data) preprocessing.lz4f = lz4f_cache
def metrics_getattr(C): # [TrainGenerator] def _raise(Except): raise Except() import_err = lambda: _raise(ImportError) with mock.patch('builtins.__import__', side_effect=import_err): pass_on_error(getattr, metrics, 'r2_score')
def _validate_key_metric_fn(tg): tg.custom_metrics = 1 tg.key_metric_fn = None tg._eval_fn_name = 'predict' pass_on_error(_validate_traingen_configs, tg) tg.key_metric_fn = 1 tg._eval_fn_name = 'evaluate' pass_on_error(_validate_traingen_configs, tg)
def checkpoint(C): # [util.saving] tg = _util_make_autoencoder(C) tg.train() tg.max_checkpoints = -1 with mock.patch('os.remove') as mock_remove: mock_remove.side_effect = OSError('Permission Denied') tg.checkpoint(forced=True, overwrite=False) tg.logdir = None pass_on_error(tg.checkpoint)
def _test_validate_data_shapes(): tg = TraingenDummy() tg.model.batch_size = None tg.batch_size = 16 tg.model.output_shape = (None, 100) data = {'preds_all': np.random.uniform(0, 1, (3, 16, 100))} training._validate_data_shapes(tg, data) data = {'preds_all': np.random.uniform(0, 1, (3, 1, 16, 100))} pass_on_error(training._validate_data_shapes, tg, data)
def test_main(monkeypatch): t0 = time() C = deepcopy(CONFIGS) with tempdir(C['traingen']['logs_dir']), \ tempdir(C['traingen']['best_models_dir']): C['traingen'].update( dict( val_freq={'batch': 20}, plot_history_freq={'val': 2}, unique_checkpoint_freq={'val': 2}, optimizer_save_configs={'include': ['updates', 'crowbar']}, max_one_best_save=True, max_checkpoints=3, )) tg = init_session(C, model=classifier) tg.train() _test_load(tg, C) C['traingen'].update( dict( val_freq={'iter': 20}, temp_checkpoint_freq={'val': 3}, optimizer_save_configs={'exclude': ['iterations']}, optimizer_load_configs={'include': ['momentum', 'momentam']}, eval_fn='predict', key_metric='catco_custom', custom_metrics={ 'catco_custom': metrics.categorical_crossentropy }, )) tg = init_session(C, model=classifier) with tempdir() as savedir: _log_init_state(tg, savedir=savedir, verbose=1) tg.train() _test_load(tg, C) tg = init_session(C, model=classifier) tg.train() tg._train_loop_done = True tg.train() tg.plot_configs['0']['vhlines']['v'] = 'invalid_vlines' pass_on_error(tg.get_history_fig) tg.clear_cache(reset_val_flags=True) tg._should_do({}, forced=True) tg.get_last_log('report', best=False) tg.get_last_log('report', best=True) tg.get_last_log('init_state') tg.val_epoch monkeypatch.setattr('builtins.input', lambda x: 'y') tg.destroy(confirm=False) print("\nTime elapsed: {:.3f}".format(time() - t0))
def _test_lz4f_dataset(C): del C['labels_path'] C['data_path'] = os.path.join(datadir, 'image_lz4f', 'train', '128batch__1.npy') pass_on_error(DataGenerator, **C) C['data_loader'] = 'numpy-lz4f' pass_on_error(DataGenerator, **C) C['data_batch_shape'] = (128, 28, 28, 1) DataGenerator(**C)
def _validate_class_weights(C): # [util.misc] C['traingen']['class_weights'] = {'0': 1, 1: 2} pass_on_error(_util_make_classifier, C) C['traingen']['class_weights'] = {0: 1} pass_on_error(_util_make_classifier, C) C = deepcopy(CONFIGS) tg = _util_make_classifier(C) tg.model.loss = 'categorical_crossentropy' tg.class_weights = {0: 1, 2: 5, 3: 6} tg._validate_traingen_configs()
def _test_init_optimizer(): model = ModelDummy() model.loss = 'sparse_categorical_crossentropy' misc._init_optimizer(model) model.loss = 'hinge' misc._init_optimizer(model) model.loss = 'poisson' misc._init_optimizer(model) model.loss = 'categorical_crossentropy' misc._init_optimizer(model, class_weights={0: 1, 1: 5}) misc._init_optimizer(model, class_weights=None) model.loss = 'goblin' pass_on_error(misc._init_optimizer, model)
def _traingen_callbacks(C): # [train_generator] tg = _util_make_autoencoder(C) tg.callbacks = [1] pass_on_error(tg._apply_callbacks) from deeptrain.callbacks import TraingenCallback tc = TraingenCallback() def raise_exception(self): raise NotImplementedError tc.init_with_traingen = raise_exception tg.callbacks = [tc] pass_on_error(tg._init_callbacks)
def _test_make_group_batch_and_labels(): C = deepcopy(DATAGEN_CFG) dg = DataGenerator(**C) dg.batch = np.random.randn(128, 10) dg.labels = np.random.randn(129, 10) pass_on_error(dg._make_group_batch_and_labels, n_batches=2) dg.shuffle_group_samples = True dg.labels = dg.batch.copy() dg._make_group_batch_and_labels(n_batches=2) dg.labels_path = None dg._make_group_batch_and_labels(n_batches=2) dg.shuffle_group_batches = True dg.shuffle_group_samples = False dg._make_group_batch_and_labels(n_batches=2)
def _get_best_subset_val_history(C): # [util.training] C['traingen']['best_subset_size'] = 2 tg = _util_make_classifier(C) tg.val_datagen.slices_per_batch = 4 tg._labels_cache = np.random.randint(0, 2, (3, 4, batch_size, 1)) tg._preds_cache = np.random.uniform(0, 1, (3, 4, batch_size, 1)) tg._sw_cache = np.random.randint(0, 2, (3, 4, batch_size, 1)) tg._class_labels_cache = tg._labels_cache.copy() tg._val_set_name_cache = ['1', '2', '3'] tg.key_metric = 'f1_score' tg.val_temp_history = {'f1_score': []} tg.key_metric_fn = metrics.f1_score tg._eval_fn_name = 'predict' tg.dynamic_predict_threshold_min_max = None tg._get_best_subset_val_history() tg._eval_fn_name = 'superfit' pass_on_error(tg._get_best_subset_val_history)
def _test_misc(): C = deepcopy(DATAGEN_CFG) dg = DataGenerator(**C) dg.superbatch = {'1': 1, '2': 2} dg.superbatch_set_nums = ['3'] pass_on_error(dg._get_next_batch, set_num='3', warn=True) dg.all_labels = {} pass_on_error(dg._get_next_labels, set_num='3') pass_on_error(setattr, dg, 'load_data', 1) pass_on_error(setattr, dg, 'load_labels', 1) with tempdir() as dirpath: path = os.path.join(dirpath, "arr.npy") np.save(path, np.array([1])) C = deepcopy(DATAGEN_CFG) C['labels_path'] = None C['data_path'] = path pass_on_error(DataGenerator, **C)
def _test_infer_and_set_info(): C = deepcopy(DATAGEN_CFG) with tempdir() as dirpath: path = os.path.join(dirpath, "arr.npy") np.save(path, np.array([1])) C['labels_path'] = None C['data_loader'] = DataLoader(path, loader='numpy') DataGenerator(**C) C['labels_loader'] = DataLoader(path, loader='numpy') DataGenerator(**C) C['data_loader'] = DataGenerator pass_on_error(DataGenerator, **C) C['labels_loader'] = None C['data_loader'] = DataLoader DataGenerator(**C) C['labels_loader'] = DataGenerator pass_on_error(DataGenerator, **C)
def _test_generate_report(): tg = TraingenDummy() tg.report_configs = {'model': dict(stuff='staff'), 'saitama': None} pass_on_error(logging.generate_report, tg) tg.report_configs = {'model': {'genos': [1]}} pass_on_error(logging.generate_report, tg) tg.report_configs = {'model': {'include': [], 'exclude': []}} pass_on_error(logging.generate_report, tg) tg.report_configs = {'model': {'exclude_types': ['etc']}} tg.model_configs = {'a': 1} pass_on_error(logging.generate_report, tg)
def _test_validate_args(C): pass_on_error(DataLoader, 1, 1) kw = dict(path=C['data_path'], loader=1, filepaths=None) pass_on_error(DataLoader, **kw) kw['filepaths'] = ['x'] pass_on_error(DataLoader, **kw)
def _validate_callbacks(C): # [util.misc] C['traingen']['callbacks'] = {'invalid_stage': 1} pass_on_error(_util_make_autoencoder, C) C['traingen']['callbacks'] = {'save': 1} pass_on_error(_util_make_autoencoder, C) C['traingen']['callbacks'] = 1 pass_on_error(_util_make_autoencoder, C)
def _test_init(): C = deepcopy(DATAGEN_CFG) C['superbatch_set_nums'] = 'all' C['superbatch_path'] = 'x' pass_on_error(DataGenerator, **C) C = deepcopy(DATAGEN_CFG) C['labels_path'] = 1 pass_on_error(DataGenerator, **C) C['data_path'] = 1 pass_on_error(DataGenerator, **C)
def save(C): # [util.saving] tg = _util_make_autoencoder(C) tg.checkpoint() tg.model.loss = 'mean_squared_error' tg.train() tg.final_fig_dir = tg.logdir pass_on_error(tg.load) pass_on_error(tg.checkpoint, overwrite="underwrite") tg.datagen.set_nums_to_process = [9001] tg.save() tg._save_history_fig() tg._save_history_fig() tg.optimizer_load_configs = {'exclude': ['weights']} tg.loadskip_list = ['optimizer_load_configs'] tg.datagen.loadskip_list = ['stuff'] tg.load() tg._history_fig = 1 tg._save_history_fig() tg.loadskip_list = 'auto' tg.load() tg.loadskip_list = 'none' tg.load() tg.optimizer_save_configs = {'include': []} tg.save() with mock.patch('backend.K.get_value') as mock_get_value: mock_get_value.side_effect = Exception() tg.save() tg.optimizer_save_configs = {'include': ['leaking_rate']} tg.datagen.group_batch = [] with mock.patch('pickle.dump') as mock_dump: mock_dump.side_effect = Exception() tg.save() tg.logdir = 'abc' pass_on_error(tg.load) tg.logdir = None pass_on_error(tg.load)
def _validate_directories(C): # [util.misc] C['traingen']['best_models_dir'] = None pass_on_error(_util_make_classifier, C) C = deepcopy(CONFIGS) C['traingen']['logs_dir'] = None pass_on_error(_util_make_classifier, C) C = deepcopy(CONFIGS) C['traingen']['best_models_dir'] = None C['traingen']['logs_dir'] = None pass_on_error(_util_make_classifier, C)
def _on_val_end(C): # [train_generator] tg = _util_make_autoencoder(C) tg.batch_size = 'x' pass_on_error(tg._on_val_end) tg.epochs += 1 tg._train_loop_done = True def validate(self): self._train_loop_done = False tg.validate = validate.__get__(tg) pass_on_error(tg.train) tg._train_loop_done = False tg._fit_fn_name = 'x' tg.epochs += 1 pass_on_error(tg.train) tg.batch_size = 'a' tg._eval_fn_name = 'predict' pass_on_error(tg._on_val_end, 0, 0, 0)
def test_advance_batch(): C = deepcopy(DATAGEN_CFG) C['superbatch_path'] = os.path.join(datadir, 'image', 'train') dg = DataGenerator(**C) dg.advance_batch() C['batch_size'] = 31 dg = DataGenerator(**C) pass_on_error(dg.advance_batch) C['batch_size'] = 256 dg = DataGenerator(**C) dg.set_nums_to_process = [] pass_on_error(dg.advance_batch) C['data_loader'] = 'pigeon' pass_on_error(DataGenerator, **C)
def _update_temp_history(C): # [util.training] tg = _util_make_classifier(C) tg.val_temp_history['loss'] = (1, 2, 3) tg._update_temp_history(metrics=(4, ), val=True) tg.val_temp_history['loss'] = [] tg._update_temp_history(metrics=(4, ), val=True) tg.datagen.slice_idx = 1 tg.datagen.slices_per_batch = 2 tg.temp_history = {'binary_accuracy': []} tg.train_metrics = ['binary_accuracy'] pass_on_error(tg._update_temp_history, metrics=[1], val=False) pass_on_error(tg._update_temp_history, metrics=[dict(a=1, b=2)], val=False) # tg._update_temp_history([[1]], val=False) # tests `_handle_non_scalar` tg.temp_history = {'f1_score': []} tg.train_metrics = ['f1_score'] pass_on_error(tg._update_temp_history, metrics=[[1, 2]], val=False)
def _layer_hists(model): pass_on_error(layer_hists, model, '*', mode='gradients') pass_on_error(layer_hists, model, '*', mode='outputs') pass_on_error(layer_hists, model, '*', mode='skeletons')
def test_data_to_hdf5(monkeypatch): # [deeptrain.preprocessing] """Dedicated test since it uses monkeypatch""" C = deepcopy(CONFIGS) # set preemptively in case data.h5 somehow found in dir monkeypatch.setattr('builtins.input', lambda x: 'y') with tempdir(C['traingen']['logs_dir']) as loaddir: with open(os.path.join(loaddir, "data.txt"), 'w') as txt: txt.write("etc") savepath = os.path.join(loaddir, "data.h5") pass_on_error(preprocessing.data_to_hdf5, savepath.replace('.h5', ''), batch_size=32, loaddir=loaddir) data = np.random.randn(1, 32, 100) np.save(os.path.join(loaddir, "data.npy"), data) pass_on_error(preprocessing.data_to_hdf5, savepath=savepath, batch_size=32, loaddir=loaddir) kw = dict(savepath=savepath, data=data, batch_size=32) pass_on_error(preprocessing.data_to_hdf5, **kw) os.remove(os.path.join(loaddir, "data.txt")) preprocessing.data_to_hdf5(**kw) monkeypatch.setattr('builtins.input', lambda x: 'y') preprocessing.data_to_hdf5(**kw) monkeypatch.setattr('builtins.input', lambda x: 'n') pass_on_error(preprocessing.data_to_hdf5, **kw) preprocessing.data_to_hdf5(overwrite=True, **kw) pass_on_error(preprocessing.data_to_hdf5, overwrite=False, **kw) pass_on_error(preprocessing.data_to_hdf5, kw['savepath'], kw['batch_size'], loaddir=None, data=None) pass_on_error(preprocessing.data_to_hdf5, kw['savepath'], kw['batch_size'], loaddir=loaddir, data=data) _data = [data[0], data[0, :31]] pass_on_error(preprocessing.data_to_hdf5, kw['savepath'], kw['batch_size'], data=_data, overwrite=True) _data = [np.vstack([data[0], data[0]])] pass_on_error(preprocessing.data_to_hdf5, kw['savepath'], kw['batch_size'], data=_data, overwrite=True)