def cross_validation(auto_var: AutoVar, grid, valid_eps: float): X = auto_var.get_intermidiate_variable('trnX') y = auto_var.get_intermidiate_variable('trny') # TODO add clone method sess = auto_var.inter_var.pop('sess') val_auto_var = deepcopy(auto_var) auto_var.inter_var['sess'] = sess val_auto_var._read_only = False val_auto_var._no_hooks = True # shuffle kf = KFold(n_splits=3) rets = [] for i, (train, test) in enumerate(kf.split(X)): print(f"{i}-th cross validation.....") fn = partial(exp_fn, trnX=X[train], trny=y[train], valX=X[test], valy=y[test], eps=valid_eps) params, results = val_auto_var.run_grid_params(fn, grid_params=grid) #params, ret = auto_var.run_grid_params(fn, grid) rets.append(results) scores = np.array(rets).mean(axis=0) print("xvalidation results:", params, scores) return params[np.argmax(scores)]
def test_run_grid(self): auto_var = AutoVar() auto_var.add_variable_class(OrdVarClass()) auto_var.add_variable_class(DatasetVarClass()) auto_var.add_variable('random_seed', int) grid_params = { "ord": ['1', '2'], "dataset": ['halfmoon_50', 'halfmoon_10'], "random_seed": [1126], } def fn(auto_var): return { "ord": auto_var.var_value['ord'], "dataset": auto_var.var_value['dataset'], "random_seed": auto_var.var_value['random_seed'], } params, results = auto_var.run_grid_params(fn, grid_params=grid_params, n_jobs=1) self.assertEqual(params[0], { 'ord': '1', 'dataset': 'halfmoon_50', 'random_seed': 1126, }) del results[0]['var_value']['git_hash'] self.assertEqual(params[0], results[0]['var_value']) del results[0]['running_time'] del results[0]['var_value'] self.assertEqual(params[0], results[0])
def test_cmd_args_with_vars(self): auto_var = AutoVar(logging_level=logging.INFO) auto_var.add_variable_class(DatasetVarClass()) args = ['--dataset', 'halfmoon_300'] auto_var.parse_argparse(args=args) with self.assertRaises(argparse.ArgumentTypeError): args = ['--dataset', 'halfmaan_300'] auto_var.parse_argparse(args=args) with self.assertRaises(SystemExit) as cm: args = [] auto_var.parse_argparse(args=args) self.assertEqual(cm.exception.code, 2)
def test_json_file(self): settings = {'file_format': 'json', 'result_file_dir': 'test'} mkdir_p(settings['result_file_dir']) auto_var = AutoVar( settings=settings, after_experiment_hooks=[ partial(save_result_to_file, get_name_fn=default_get_file_name) ], ) auto_var.add_variable_class(OrdVarClass()) auto_var.set_variable_value_by_dict({'ord': '1'}) def experiment(auto_var): return {'test': auto_var.get_var('ord')} _ = auto_var.run_single_experiment(experiment, with_hook=True) with open("test/1.json", 'r') as f: ret = json.load(f) self.assertEqual(ret['test'], auto_var.get_var('ord')) shutil.rmtree(settings['result_file_dir'])
from autovar.base import RegisteringChoiceType, register_var, VariableClass from autovar.hooks import check_result_file_exist, save_result_to_file from autovar.hooks import create_placeholder_file, remove_placeholder_if_error from autovar.hooks import default_get_file_name as get_file_name from .datasets import DatasetVarClass auto_var = AutoVar(logging_level=logging.INFO, before_experiment_hooks=[ partial(check_result_file_exist, get_name_fn=get_file_name), partial(create_placeholder_file, get_name_fn=get_file_name), ], after_experiment_hooks=[ partial(save_result_to_file, get_name_fn=get_file_name), partial(remove_placeholder_if_error, get_name_fn=get_file_name), ], settings={ 'file_format': 'pickle', 'server_url': '', 'result_file_dir': './results/' }) auto_var.add_variable_class(DatasetVarClass()) auto_var.add_variable('random_seed', int) #auto_var.add_variable('eps', float) #auto_var.add_variable('optimizer', str) #auto_var.add_variable('learning_rate', float, default=1e-2) #auto_var.add_variable('batch_size', int, default=64)
class OrdVarClass(VariableClass, metaclass=RegisteringChoiceType): """Defines which distance measure to use for attack.""" var_name = "ord" @register_var() @staticmethod def inf(auto_var): """L infinity norm""" return np.inf @register_var(argument='2') @staticmethod def l2(auto_var): """L2 norm""" return 2 @register_var(argument='1') @staticmethod def l1(auto_var): """L1 norm""" return 1 auto_var = AutoVar(logging_level=logging.CRITICAL) auto_var.add_variable_class(OrdVarClass()) auto_var.add_variable_class(DatasetVarClass()) auto_var.add_variable_class(ModelVarClass()) auto_var.add_variable_class(AttackVarClass()) auto_var.add_variable('random_seed', int)
def test_cmd_args(self): auto_var = AutoVar(logging_level=logging.INFO) self.assertFalse(auto_var._no_hooks) args = ['--no-hooks'] auto_var.parse_argparse(args=args) self.assertTrue(auto_var._no_hooks)
def test_val(self): auto_var = AutoVar(logging_level=logging.INFO) with self.assertRaises(VariableNotRegisteredError): auto_var.get_var('ord') auto_var.add_variable_class(OrdVarClass()) auto_var.add_variable_class(DatasetVarClass()) auto_var.add_variable('random_seed', int) with self.assertRaises(VariableValueNotSetError): auto_var.get_var('ord') auto_var.set_variable_value_by_dict({ 'ord': '1', 'dataset': 'halfmoon_200', 'random_seed': 1126 }) self.assertEqual(auto_var.get_var('ord'), 1) self.assertEqual(auto_var.get_var('random_seed'), 1126) self.assertEqual(len(auto_var.get_var('dataset')[0]), 200) self.assertEqual( len(auto_var.get_var_with_argument('dataset', 'halfmoon_300')[0]), 300) with self.assertRaises(ValueError): auto_var.set_variable_value_by_dict({'ord': 'l2'}) with self.assertRaises(TypeError): auto_var.set_variable_value_by_dict({'random_seed': '1126.0'}) self.assertEqual(auto_var.get_var_shown_name(var_name="dataset"), 'shown_halfmoon') assert_array_equal( auto_var.get_var_with_argument('dataset', 'halfmoon_300')[0], auto_var.get_var_with_argument('dataset', 'moon_300')[0], ) argparse_help = auto_var.get_argparser().format_help() self.assertTrue('halfmoon dataset' in argparse_help) self.assertTrue('Dataset variable class' in argparse_help)
def test_cache_files(self): auto_var = AutoVar(logging_level=logging.INFO) auto_var.add_variable_class(DatasetVarClass()) auto_var.add_variable_class(OrdVarClass()) auto_var.set_variable_value("dataset", "no4_halfmoon_5") auto_var.set_variable_value("ord", "1") X, y = auto_var.get_var("dataset") cacheX, cachey = auto_var.get_var("dataset") assert_array_equal(X, cacheX) assert_array_equal(y, cachey) temp_dir = auto_var.variables['dataset']['cache_dirs'][ 'no4_halfmoon_(?P<n_samples>\\d+)'] cacheX, cachey = joblib.load( os.path.join(temp_dir, "no4_halfmoon_5-1.pkl")) assert_array_equal(X, cacheX) assert_array_equal(y, cachey)
def test_cmd_args_with_vars_2(self): auto_var = AutoVar(logging_level=logging.INFO) auto_var.add_variable_class(OrdVarClass()) auto_var.parse_argparse(args=[]) self.assertEqual(auto_var.get_var("ord"), 2)
def l2(auto_var): """L2 norm""" return 2 @register_var(argument='1') @staticmethod def l1(auto_var): """L1 norm""" return 1 auto_var = AutoVar( logging_level=logging.INFO, before_experiment_hooks=[ partial(check_result_file_exist, get_name_fn=get_file_name), #partial(save_parameter_to_file, get_name_fn=partial(get_file_name, name_only=False)) ], after_experiment_hooks=[ partial(save_result_to_file, get_name_fn=get_file_name) ], settings={ 'server_url': '', 'result_file_dir': './results/' }) auto_var.add_variable_class(OrdVarClass()) auto_var.add_variable_class(DatasetVarClass()) auto_var.add_variable_class(ModelVarClass()) auto_var.add_variable_class(AttackVarClass()) auto_var.add_variable('random_seed', int)