def test_holdout(self, patch): call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \ '--fold 0 --folds 1 --seed 1 --mode holdout %s' % \ (self.dataset_string, self.param_string) sys.argv = shlex.split(call) HPOlib_interface.main() self.assertEqual(patch.call_count, 1) call_args, call_kwargs = patch.call_args self.assertEqual(call_args, (self.dataset_string, 'holdout', '1', self.params)) self.assertEqual(call_kwargs, {'mode_args': None})
def test_holdout_iterative_fit(self, patch): call = ( "autosklearn.cli.HPOlib_interface --dataset %s " "--fold 0 --folds 1 --seed 1 --mode holdout-iterative-fit %s" % (self.dataset_string, self.param_string) ) sys.argv = shlex.split(call) HPOlib_interface.main() self.assertEqual(patch.call_count, 1) call_args, call_kwargs = patch.call_args self.assertEqual(call_args, (self.dataset_string, "holdout-iterative-fit", "1", self.params)) self.assertEqual(call_kwargs, {"mode_args": None})
def test_nested_cv(self, patch): call = ( "autosklearn.cli.HPOlib_interface --dataset %s " "--fold 0 --folds 1 --seed 1 " "--mode 3/3-nested-cv %s" % (self.dataset_string, self.param_string) ) sys.argv = shlex.split(call) HPOlib_interface.main() self.assertEqual(patch.call_count, 1) call_args, call_kwargs = patch.call_args self.assertEqual(call_args, (self.dataset_string, "nested-cv", "1", self.params)) self.assertEqual(call_kwargs, {"mode_args": {"outer_folds": 3, "inner_folds": 3}})
def test_holdout(self, patch): call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \ '--fold 0 --folds 1 --seed 1 --mode holdout %s' % \ (self.dataset_string, self.param_string) sys.argv = shlex.split(call) HPOlib_interface.main(output_dir=self.output_directory) self.assertEqual(patch.call_count, 1) call_args, call_kwargs = patch.call_args self.assertEqual(call_args, (self.dataset_string, 'holdout', '1', self.params)) self.assertEqual(call_kwargs, {'mode_args': None, 'output_dir': self.output_directory})
def test_partial_cv(self, patch): results = [] for fold in range(3): call = ( "autosklearn.cli.HPOlib_interface --dataset %s " "--fold %d --folds 3 --mode cv --seed 1 " "%s" % (self.dataset_string, fold, self.param_string) ) sys.argv = shlex.split(call) HPOlib_interface.main() self.assertEqual(patch.call_count, fold + 1) call_args, call_kwargs = patch.call_args self.assertEqual(call_args, (self.dataset_string, "partial-cv", "1", self.params)) self.assertEqual(call_kwargs, {"mode_args": {"folds": 3, "fold": fold}})
def test_nested_cv(self, patch): call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \ '--fold 0 --folds 1 --seed 1 ' \ '--mode 3/3-nested-cv %s' % \ (self.dataset_string, self.param_string) sys.argv = shlex.split(call) HPOlib_interface.main(output_dir=self.output_directory) self.assertEqual(patch.call_count, 1) call_args, call_kwargs = patch.call_args self.assertEqual(call_args, (self.dataset_string, 'nested-cv', '1', self.params)) self.assertEqual(call_kwargs, {'mode_args': {'outer_folds': 3, 'inner_folds': 3}, 'output_dir': self.output_directory})
def test_holdout_iterative_fit(self, patch): call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \ '--fold 0 --folds 1 --seed 1 --mode holdout-iterative-fit %s' % \ (self.dataset_string, self.param_string) sys.argv = shlex.split(call) HPOlib_interface.main(output_dir=self.output_directory) self.assertEqual(patch.call_count, 1) call_args, call_kwargs = patch.call_args self.assertEqual( call_args, (self.dataset_string, 'holdout-iterative-fit', '1', self.params)) self.assertEqual(call_kwargs, { 'mode_args': None, 'output_dir': self.output_directory })
def test_nested_cv(self, patch): call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \ '--fold 0 --folds 1 --seed 1 ' \ '--mode 3/3-nested-cv %s' % \ (self.dataset_string, self.param_string) sys.argv = shlex.split(call) HPOlib_interface.main() self.assertEqual(patch.call_count, 1) call_args, call_kwargs = patch.call_args self.assertEqual(call_args, (self.dataset_string, 'nested-cv', '1', self.params)) self.assertEqual(call_kwargs, {'mode_args': { 'outer_folds': 3, 'inner_folds': 3 }})
def test_partial_cv(self, patch): results = [] for fold in range(3): call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \ '--fold %d --folds 3 --mode cv --seed 1 ' \ '%s' % \ (self.dataset_string, fold, self.param_string) sys.argv = shlex.split(call) HPOlib_interface.main(output_dir=self.output_directory) self.assertEqual(patch.call_count, fold+1) call_args, call_kwargs = patch.call_args self.assertEqual(call_args, (self.dataset_string, 'partial-cv', '1', self.params)) self.assertEqual(call_kwargs, {'mode_args': {'folds': 3, 'fold': fold}, 'output_dir': self.output_directory})
def test_cv(self, patch): call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \ '--fold 0 --folds 1 --seed 1 ' \ '--mode 3cv %s' % \ (self.dataset_string, self.param_string) sys.argv = shlex.split(call) HPOlib_interface.main(output_dir=self.output_directory) self.assertEqual(patch.call_count, 1) call_args, call_kwargs = patch.call_args self.assertEqual(call_args, (self.dataset_string, 'cv', '1', self.params)) self.assertEqual(call_kwargs, { 'mode_args': { 'folds': 3 }, 'output_dir': self.output_directory })
def test_partial_cv(self, patch): results = [] for fold in range(3): call = 'autosklearn.cli.HPOlib_interface --dataset %s ' \ '--fold %d --folds 3 --mode cv --seed 1 ' \ '%s' % \ (self.dataset_string, fold, self.param_string) sys.argv = shlex.split(call) HPOlib_interface.main() self.assertEqual(patch.call_count, fold + 1) call_args, call_kwargs = patch.call_args self.assertEqual( call_args, (self.dataset_string, 'partial-cv', '1', self.params)) self.assertEqual(call_kwargs, {'mode_args': { 'folds': 3, 'fold': fold }})