def setUp(self): train_dataset_path = config.ROOT + '/python/test/resource/test_image_dataset_noisy.py' train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.raw_video_h5py_filepath = config.ROOT + '/workspace/workdir/rawvideo.hdf5' self.raw_video_h5py_file = DisYUVRawVideoExtractor.open_h5py_file( self.raw_video_h5py_filepath) optional_dict2 = {'h5py_file': self.raw_video_h5py_file} _, self.features = run_executors_in_parallel( DisYUVRawVideoExtractor, train_assets, fifo_mode=True, delete_workdir=True, parallelize= False, # CAN ONLY USE SERIAL MODE FOR DisYRawVideoExtractor result_store=None, optional_dict=None, optional_dict2=optional_dict2, ) np.random.seed(0) np.random.shuffle(self.features) self.patch_h5py_filepath = config.ROOT + '/workspace/workdir/patch.hdf5' self.model_filename = config.ROOT + "/workspace/model/test_save_load.pkl" NeuralNetTrainTestModel.reset()
def test_explain_train_test_model(self): model_class = SklearnRandomForestTrainTestModel train_dataset_path = config.ROOT + '/python/test/resource/' \ 'test_image_dataset_diffdim.py' train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) _, self.features = run_executors_in_parallel( MomentNorefFeatureExtractor, train_assets, fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, optional_dict=None, optional_dict2=None, ) xys = model_class.get_xys_from_results(self.features[:7]) model = model_class({'norm_type':'normalize', 'random_state':0}, None) model.train(xys) np.random.seed(0) xs = model_class.get_xs_from_results(self.features[7:]) explainer = LocalExplainer(neighbor_samples=1000) exps = explainer.explain(model, xs) self.assertAlmostEqual(exps['feature_weights'][0, 0], -0.12416, places=4) self.assertAlmostEqual(exps['feature_weights'][1, 0], 0.00076, places=4) self.assertAlmostEqual(exps['feature_weights'][0, 1], -0.20931, places=4) self.assertAlmostEqual(exps['feature_weights'][1, 1], -0.01245, places=4) self.assertAlmostEqual(exps['feature_weights'][0, 2], 0.02322, places=4) self.assertAlmostEqual(exps['feature_weights'][1, 2], 0.03673, places=4) self.assertAlmostEqual(exps['features'][0, 0], 107.73501, places=4) self.assertAlmostEqual(exps['features'][1, 0], 35.81638, places=4) self.assertAlmostEqual(exps['features'][0, 1], 13691.23881, places=4) self.assertAlmostEqual(exps['features'][1, 1], 1611.56764, places=4) self.assertAlmostEqual(exps['features'][0, 2], 2084.40542, places=4) self.assertAlmostEqual(exps['features'][1, 2], 328.75389, places=4) self.assertAlmostEqual(exps['features_normalized'][0, 0], -0.65527, places=4) self.assertAlmostEqual(exps['features_normalized'][1, 0], -3.74922, places=4) self.assertAlmostEqual(exps['features_normalized'][0, 1], -0.68872, places=4) self.assertAlmostEqual(exps['features_normalized'][1, 1], -2.79586, places=4) self.assertAlmostEqual(exps['features_normalized'][0, 2], 0.08524, places=4) self.assertAlmostEqual(exps['features_normalized'][1, 2], -1.32625, places=4) self.assertEqual(exps['feature_names'], ['Moment_noref_feature_1st_score', 'Moment_noref_feature_2nd_score', 'Moment_noref_feature_var_score'] )
def test_read_dataset(self): train_dataset_path = config.ROOT + '/python/test/resource/test_image_dataset_diffdim.py' train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.assertEquals(len(train_assets), 9) self.assertTrue('groundtruth' in train_assets[0].asset_dict.keys()) self.assertTrue('os' not in train_assets[0].asset_dict.keys()) self.assertTrue('width' in train_assets[0].asset_dict.keys()) self.assertTrue('height' in train_assets[0].asset_dict.keys()) self.assertTrue('quality_width' not in train_assets[0].asset_dict.keys()) self.assertTrue('quality_height' not in train_assets[0].asset_dict.keys())
def test_read_dataset_diffyuv(self): train_dataset_path = config.ROOT + '/python/test/resource/test_dataset_diffyuv.py' train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.assertEquals(len(train_assets), 4) self.assertEquals(train_assets[0].ref_width_height, (1920, 1080)) self.assertEquals(train_assets[0].dis_width_height, (1920, 1080)) self.assertEquals(train_assets[0].quality_width_height, (1920, 1080)) self.assertEquals(train_assets[0].yuv_type, 'yuv420p') self.assertEquals(train_assets[2].ref_width_height, (1280, 720)) self.assertEquals(train_assets[2].dis_width_height, (1280, 720)) self.assertEquals(train_assets[2].quality_width_height, (1280, 720)) self.assertEquals(train_assets[2].yuv_type, 'yuv420p10le')
def test_read_dataset_qualitywh2(self): train_dataset_path = config.ROOT + '/python/test/resource/test_image_dataset_diffdim_qualitywh2.py' train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.assertTrue('quality_width' in train_assets[0].asset_dict.keys()) self.assertTrue('quality_height' in train_assets[0].asset_dict.keys()) self.assertTrue('resampling_type' in train_assets[0].asset_dict.keys()) self.assertTrue('quality_width' not in train_assets[1].asset_dict.keys()) self.assertTrue('quality_height' not in train_assets[1].asset_dict.keys()) self.assertTrue('resampling_type' not in train_assets[1].asset_dict.keys()) self.assertEqual(train_assets[0].asset_dict['quality_width'], 200) self.assertEqual(train_assets[0].asset_dict['quality_height'], 100) self.assertEqual(train_assets[0].asset_dict['resampling_type'], 'bicubic')
def setUp(self): train_dataset_path = config.ROOT + '/python/test/resource/test_image_dataset_diffdim.py' train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) _, self.features = run_executors_in_parallel( MomentNorefFeatureExtractor, train_assets, fifo_mode=True, delete_workdir=True, parallelize=True, result_store=None, optional_dict=None, optional_dict2=None, )
def test_read_dataset_crop_and_pad(self): train_dataset_path = config.ROOT + '/python/test/resource/example_dataset_crop_pad.py' train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.assertEquals(len(train_assets), 3) self.assertEquals( str(train_assets[0]), "example_0_1_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_576x324_crop288:162:144:81" ) self.assertEquals( str(train_assets[1]), "example_0_2_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_576x324_padiw+100:ih+100:50:50" ) self.assertEquals( str(train_assets[2]), "example_0_3_src01_hrc00_576x324_576x324_vs_src01_hrc01_576x324_576x324_q_576x324_crop288:162:144:81_padiw+288:ih+162:144:81" )
def setUp(self): train_dataset_path = config.ROOT + '/python/test/resource/test_image_dataset_diffdim.py' train_dataset = import_python_file(train_dataset_path) train_assets = read_dataset(train_dataset) self.h5py_filepath = config.ROOT + '/workspace/workdir/test.hdf5' self.h5py_file = DisYUVRawVideoExtractor.open_h5py_file(self.h5py_filepath) optional_dict2 = {'h5py_file': self.h5py_file} _, self.features = run_executors_in_parallel( DisYUVRawVideoExtractor, train_assets, fifo_mode=True, delete_workdir=True, parallelize=False, # CAN ONLY USE SERIAL MODE FOR DisYRawVideoExtractor result_store=None, optional_dict=None, optional_dict2=optional_dict2, )
from core.raw_extractor import DisYUVRawVideoExtractor from core.nn_train_test_model import ToddNoiseClassifierTrainTestModel from routine import read_dataset from tools.misc import import_python_file # parameters num_train = 500 num_test = 50 n_epochs = 30 seed = 0 # None # read input dataset dataset_path = config.ROOT + '/resource/dataset/BSDS500_noisy_dataset.py' dataset = import_python_file(dataset_path) assets = read_dataset(dataset) # shuffle assets np.random.seed(seed) np.random.shuffle(assets) assets = assets[:(num_train + num_test)] raw_video_h5py_filepath = config.ROOT + '/workspace/workdir/rawvideo.hdf5' raw_video_h5py_file = DisYUVRawVideoExtractor.open_h5py_file(raw_video_h5py_filepath) print '======================== Extract raw YUVs ==============================' _, raw_yuvs = run_executors_in_parallel( DisYUVRawVideoExtractor, assets, fifo_mode=True,