def test_should_return_a_valid_summary_with_default_constructor(self):
access_grid_config = AccessGridConfig()
parameter_grid = access_grid_config.get_parameter_grid()
parameter = parameter_grid[0]
summary = access_grid_config.get_config_summary_from_parameters_entry(parameter)
self.assertEqual(summary, "framerate15_duration2000_centered")
def test_should_return_a_valid_summary_with_starting_time_acquisition_list_1000_and_center_video_acquisition_list_false(self):
access_grid_config = AccessGridConfig(starting_time_acquisition_list=[1000],
center_video_acquisition_list=[False])
parameter_grid = access_grid_config.get_parameter_grid()
parameter = parameter_grid[0]
summary = access_grid_config.get_config_summary_from_parameters_entry(parameter)
self.assertEqual(summary, "framerate15_duration2000_startingtime1000")
def test_should_return_a_valid_parameter_grid_with_parametrized_constructor(self):
access_grid_config = AccessGridConfig(framerate_list=[30, 15],
total_time_acquisition_list=[500, 1000],
starting_time_acquisition_list=[100, 200],
center_video_acquisition_list=[False])
parameter_grid = access_grid_config.get_parameter_grid()
self.assertEqual(len(parameter_grid), 8)
def test_should_return_a_valid_parameter_grid_with_default_constructor(self):
access_grid_config = AccessGridConfig()
parameter_grid = access_grid_config.get_parameter_grid()
self.assertEqual(len(parameter_grid), 1)
parameter = parameter_grid[0]
self.assertEqual(parameter["framerate"], 15)
self.assertEqual(parameter["total_time_acquisition"], 2000)
self.assertEqual(parameter["starting_time_acquisition"], -1)
self.assertEqual(parameter["center_video_acquisition"], True)
def test_should_throw_an_exception_if_parameters_are_none_or_not_a_list(self):
parameters_grid = list(ParameterGrid({'framerate_list': [None, "wrong"],
'total_time_acquisition': [None, "wrong"],
'starting_time_acquisition': [None, "wrong"],
'center_video_acquisition': [None, "wrong"]}))
for parameters in parameters_grid:
self.assertRaises(TypeError,
lambda: AccessGridConfig(parameters["framerate_list"],
parameters["total_time_acquisition"],
parameters["starting_time_acquisition"],
parameters["center_video_acquisition"]
)
)
def __init__(self,
type_evaluation,
database_paths_filename,
databases_list,
protocols_list,
feature_extractor,
pipeline,
result_path,
access_grid_config=AccessGridConfig(),
verbose=False,
number_threads=1,
use_data_augmentation=False,
skip_features_extraction=False,
dict_extracted_features_paths=None,
skip_training=False,
skip_scores_prediction=False,
dict_scores_prediction=None,
recreate=False,
configuration_file='\'Configure class\''):
if type_evaluation not in evaluation_long_names.keys():
raise ValueError(
"type_evaluation is not available. Try with {}".format(
evaluation_long_names.keys()))
self.type_evaluation = type_evaluation
self.database_paths_filename = database_paths_filename
self.databases_list = databases_list
self.protocols_list = protocols_list
self.feature_extractor = feature_extractor
self.pipeline = pipeline
self.result_path = result_path
self.features_extractor_manager = FeaturesExtractorManager(
self.feature_extractor, FeaturesSaver(self.result_path))
self.access_grid_config = access_grid_config
self.verbose = verbose
self.number_threads = number_threads
self.use_data_augmentation = use_data_augmentation
self.skip_features_extraction = skip_features_extraction
self.dict_extracted_features_paths = dict_extracted_features_paths
self.skip_training = skip_training
self.skip_scores_prediction = skip_scores_prediction
self.dict_scores_prediction = dict_scores_prediction
self.recreate = recreate
self.configuration_file = configuration_file
self.__set_git_info()
self.__check_configuration()
self.__set_pipeline_key_correspondencies()
if self.database_paths_filename:
export_database_paths_from_file(self.database_paths_filename)
class UnitTestConfiguration(unittest.TestCase):
configuration_file = 'resources/config/config_test.py'
database_paths_filename = 'resources/config/database_paths.json'
databases = [DummyDatabase('resources')]
protocols = ['grandtest']
feature_extractor = DummyFeaturesExtractor()
pipeline = Pipeline(
'pca095_linear_svc',
[Pca(name='Pca', n_components=0.95),
LinearSvc(name='LinearSvc')])
result_path = 'result'
access_grid_config = AccessGridConfig(
framerate_list=[5, 10, 15, 20, 25],
total_time_acquisition_list=[500, 1000, 1500, 2000],
starting_time_acquisition_list=[100],
center_video_acquisition_list=[False])
verbose = True
number_threads = 1
use_data_augmentation = False
skip_features_extraction = False
dict_extracted_features_paths = None
skip_training = False
skip_scores_prediction = False
dict_scores_prediction = None
recreate = False
def tearDown(self):
if os.path.isdir('result'):
shutil.rmtree('result')
def test_init_fromfilename_wrong_path(self):
self.assertRaises(IOError,
lambda: Configuration.fromfilename('ACE', 'WRONG'))
def test_init_fromfilename_wrong_type_evaluation(self):
self.assertRaises(
ValueError, lambda: Configuration.fromfilename(
'WRONG', self.configuration_file))
def test_init_fromfilename_correct_params(self):
Configuration.fromfilename('ACE', self.configuration_file)
def test_init_correct_params_but_database_path_not_defined(self):
replay_path = None
if "REPLAY_ATTACK_PATH" in os.environ:
replay_path = os.environ["REPLAY_ATTACK_PATH"]
del os.environ["REPLAY_ATTACK_PATH"]
self.assertRaises(
EnvironmentError, lambda: Configuration(
'ACE',
self.database_paths_filename, ['replay-attack'],
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
if replay_path:
os.environ["REPLAY_ATTACK_PATH"] = replay_path
def test_init_incorrect_databases_param(self):
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
'WRONG_PARAM',
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
def test_init_incorrect_databases_param_no_exist(self):
self.assertRaises(
ValueError, lambda: Configuration(
'ACE',
self.database_paths_filename, ['no_exist_database'],
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
def test_init_incorrect_protocol_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
'WRONG_PARAM',
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_incorrect_features_extractor_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
"WRONG_PARAM",
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_incorrect_pipeline_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
"WRONG_PARAM",
self.result_path,
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_none_result_path_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
self.pipeline,
None,
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_incorrect_result_path_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
self.pipeline, ['WRONG_PARAM'],
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_none_access_grid_config_list_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=None,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_incorrect_access_grid_config_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config="WRONG_PARAMETER",
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_none_verbose_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
verbose=None,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_wrong_verbose_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
verbose='WRONG',
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_none_number_threads_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=None,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_none_use_data_augmentation_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=None,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_none_skip_features_extraction_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda:
Configuration('ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=None,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_none_skip_training_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=None,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_none_skip_scores_prediction(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=None,
dict_scores_prediction=self.dict_scores_prediction,
recreate=self.recreate))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_none_recreate_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=None))
del os.environ["REPLAY_ATTACK_PATH"]
def test_init_with_not_valid_categorized_scores_plotter_param(self):
os.environ["REPLAY_ATTACK_PATH"] = "resources"
self.assertRaises(
TypeError, lambda: Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
categorized_scores_plotter="not_valid",
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.
dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=None))
del os.environ["REPLAY_ATTACK_PATH"]
def test_print_to_file(self):
filename_result = 'result/configuration.txt'
configuration = Configuration.fromfilename('ACE',
self.configuration_file)
configuration.save_to_file(filename_result)
self.assertTrue(os.path.isfile(filename_result))
def test_should_check_if_database_paths_are_loaded_as_global_env(self):
if "REPLAY_ATTACK_PATH" in os.environ.keys():
del os.environ["REPLAY_ATTACK_PATH"]
_ = Configuration(
'ACE',
self.database_paths_filename,
self.databases,
self.protocols,
self.feature_extractor,
self.pipeline,
self.result_path,
access_grid_config=self.access_grid_config,
verbose=self.verbose,
number_threads=self.number_threads,
use_data_augmentation=self.use_data_augmentation,
skip_features_extraction=self.skip_features_extraction,
dict_extracted_features_paths=self.dict_extracted_features_paths,
skip_training=self.skip_training,
skip_scores_prediction=self.skip_scores_prediction,
dict_scores_prediction=self.dict_scores_prediction,
recreate=True)
self.assertTrue(os.environ["REPLAY_ATTACK_PATH"] == "resources")
del os.environ["REPLAY_ATTACK_PATH"]
feature_extractor = DummyFeaturesExtractor()
# Pipeline:
from bob.gradiant.pipelines import Pipeline, Pca, LinearSvc
pipeline = Pipeline('test_approach_pca095_linear_svc',
[Pca(name='Pca', n_components=0.95), LinearSvc(name='LinearSvc')])
# Result base path:
result_path = 'result'
# Framerate and time parameters:
from bob.gradiant.core import AccessGridConfig
access_grid_config = AccessGridConfig(framerate_list=[15, 20],
total_time_acquisition_list=[1000, 1500],
starting_time_acquisition_list=[100],
center_video_acquisition_list=[False])
# -----------------------------------------------------------------
# OPTIONAL ARGUMENTS:
# Pad evaluation comparative using the framework bob.gradiant.pad.comparative
categorized_scores_plotter = None
# Verbose (only True/False are valid):
verbose = True
# Number of threads for parallelizing the features extraction:
number_threads = 1
# Data augmentation:
"""
DEFAULT_DATABASE_PATHS_CONFIG_FILE = os.path.join(
os.path.abspath(os.path.dirname(__file__)), 'database_paths.json')
"""
This following class contains the configuration in order to setup our experiments.
This class will combine the options (params) to modify the accesses (video)
consistently with the params.
Default parameters will filter the video from the center, using 2000 ms and 15 frames per second.
:param framerate_list: a list with a selected frame rates (default [15])
:param total_time_acquisition_list: a list with selected times to keep (default [2000] <- 2s)
:param starting_time_acquisition_list: a list with selected points to start the video.
Frames before this value will be discarded. (default [-1] <- ignore this parameter)
:param center_video_acquisition_list: Bool value. If True, it will ignore the starting_time value and crop the video
from the central point. (default [True])
"""
DEFAULT_ACCESS_GRID_CONFIG = AccessGridConfig(
framerate_list=[15],
total_time_acquisition_list=[2000],
starting_time_acquisition_list=[-1],
center_video_acquisition_list=[True])
"""
This file help us to maintain the base result path
"""
DEFAULT_BASE_RESULT_PATH = 'resources_bob_paper_icb2019_gradgpad'
"""
Number of threads to be executed extracting features
"""
DEFAULT_NUMBER_THREADS = 4