def test_holdout(self, patch):
call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \
'--fold 0 --folds 1 --seed 1 --mode holdout %s' % \
(self.dataset_string, self.param_string)
sys.argv = shlex.split(call)
HPOlib_interface.main()
self.assertEqual(patch.call_count, 1)
call_args, call_kwargs = patch.call_args
self.assertEqual(call_args,
(self.dataset_string, 'holdout', '1', self.params))
self.assertEqual(call_kwargs, {'mode_args': None})
def test_holdout_iterative_fit(self, patch):
call = (
"autosklearn.cli.HPOlib_interface --dataset %s "
"--fold 0 --folds 1 --seed 1 --mode holdout-iterative-fit %s" % (self.dataset_string, self.param_string)
)
sys.argv = shlex.split(call)
HPOlib_interface.main()
self.assertEqual(patch.call_count, 1)
call_args, call_kwargs = patch.call_args
self.assertEqual(call_args, (self.dataset_string, "holdout-iterative-fit", "1", self.params))
self.assertEqual(call_kwargs, {"mode_args": None})
def test_nested_cv(self, patch):
call = (
"autosklearn.cli.HPOlib_interface --dataset %s "
"--fold 0 --folds 1 --seed 1 "
"--mode 3/3-nested-cv %s" % (self.dataset_string, self.param_string)
)
sys.argv = shlex.split(call)
HPOlib_interface.main()
self.assertEqual(patch.call_count, 1)
call_args, call_kwargs = patch.call_args
self.assertEqual(call_args, (self.dataset_string, "nested-cv", "1", self.params))
self.assertEqual(call_kwargs, {"mode_args": {"outer_folds": 3, "inner_folds": 3}})
def test_holdout(self, patch):
call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \
'--fold 0 --folds 1 --seed 1 --mode holdout %s' % \
(self.dataset_string, self.param_string)
sys.argv = shlex.split(call)
HPOlib_interface.main(output_dir=self.output_directory)
self.assertEqual(patch.call_count, 1)
call_args, call_kwargs = patch.call_args
self.assertEqual(call_args, (self.dataset_string, 'holdout', '1',
self.params))
self.assertEqual(call_kwargs, {'mode_args': None,
'output_dir': self.output_directory})
def test_partial_cv(self, patch):
results = []
for fold in range(3):
call = (
"autosklearn.cli.HPOlib_interface --dataset %s "
"--fold %d --folds 3 --mode cv --seed 1 "
"%s" % (self.dataset_string, fold, self.param_string)
)
sys.argv = shlex.split(call)
HPOlib_interface.main()
self.assertEqual(patch.call_count, fold + 1)
call_args, call_kwargs = patch.call_args
self.assertEqual(call_args, (self.dataset_string, "partial-cv", "1", self.params))
self.assertEqual(call_kwargs, {"mode_args": {"folds": 3, "fold": fold}})
def test_nested_cv(self, patch):
call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \
'--fold 0 --folds 1 --seed 1 ' \
'--mode 3/3-nested-cv %s' % \
(self.dataset_string, self.param_string)
sys.argv = shlex.split(call)
HPOlib_interface.main(output_dir=self.output_directory)
self.assertEqual(patch.call_count, 1)
call_args, call_kwargs = patch.call_args
self.assertEqual(call_args, (self.dataset_string, 'nested-cv', '1',
self.params))
self.assertEqual(call_kwargs, {'mode_args': {'outer_folds': 3,
'inner_folds': 3},
'output_dir': self.output_directory})
def test_holdout_iterative_fit(self, patch):
call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \
'--fold 0 --folds 1 --seed 1 --mode holdout-iterative-fit %s' % \
(self.dataset_string, self.param_string)
sys.argv = shlex.split(call)
HPOlib_interface.main(output_dir=self.output_directory)
self.assertEqual(patch.call_count, 1)
call_args, call_kwargs = patch.call_args
self.assertEqual(
call_args,
(self.dataset_string, 'holdout-iterative-fit', '1', self.params))
self.assertEqual(call_kwargs, {
'mode_args': None,
'output_dir': self.output_directory
})
def test_nested_cv(self, patch):
call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \
'--fold 0 --folds 1 --seed 1 ' \
'--mode 3/3-nested-cv %s' % \
(self.dataset_string, self.param_string)
sys.argv = shlex.split(call)
HPOlib_interface.main()
self.assertEqual(patch.call_count, 1)
call_args, call_kwargs = patch.call_args
self.assertEqual(call_args,
(self.dataset_string, 'nested-cv', '1', self.params))
self.assertEqual(call_kwargs,
{'mode_args': {
'outer_folds': 3,
'inner_folds': 3
}})
def test_partial_cv(self, patch):
results = []
for fold in range(3):
call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \
'--fold %d --folds 3 --mode cv --seed 1 ' \
'%s' % \
(self.dataset_string, fold, self.param_string)
sys.argv = shlex.split(call)
HPOlib_interface.main(output_dir=self.output_directory)
self.assertEqual(patch.call_count, fold+1)
call_args, call_kwargs = patch.call_args
self.assertEqual(call_args, (self.dataset_string, 'partial-cv', '1',
self.params))
self.assertEqual(call_kwargs, {'mode_args': {'folds': 3,
'fold': fold},
'output_dir': self.output_directory})
def test_cv(self, patch):
call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \
'--fold 0 --folds 1 --seed 1 ' \
'--mode 3cv %s' % \
(self.dataset_string, self.param_string)
sys.argv = shlex.split(call)
HPOlib_interface.main(output_dir=self.output_directory)
self.assertEqual(patch.call_count, 1)
call_args, call_kwargs = patch.call_args
self.assertEqual(call_args,
(self.dataset_string, 'cv', '1', self.params))
self.assertEqual(call_kwargs, {
'mode_args': {
'folds': 3
},
'output_dir': self.output_directory
})
def test_partial_cv(self, patch):
results = []
for fold in range(3):
call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \
'--fold %d --folds 3 --mode cv --seed 1 ' \
'%s' % \
(self.dataset_string, fold, self.param_string)
sys.argv = shlex.split(call)
HPOlib_interface.main()
self.assertEqual(patch.call_count, fold + 1)
call_args, call_kwargs = patch.call_args
self.assertEqual(
call_args,
(self.dataset_string, 'partial-cv', '1', self.params))
self.assertEqual(call_kwargs,
{'mode_args': {
'folds': 3,
'fold': fold
}})
