def cross_validation(auto_var: AutoVar, grid, valid_eps: float):
X = auto_var.get_intermidiate_variable('trnX')
y = auto_var.get_intermidiate_variable('trny')
# TODO add clone method
sess = auto_var.inter_var.pop('sess')
val_auto_var = deepcopy(auto_var)
auto_var.inter_var['sess'] = sess
val_auto_var._read_only = False
val_auto_var._no_hooks = True
# shuffle
kf = KFold(n_splits=3)
rets = []
for i, (train, test) in enumerate(kf.split(X)):
print(f"{i}-th cross validation.....")
fn = partial(exp_fn,
trnX=X[train],
trny=y[train],
valX=X[test],
valy=y[test],
eps=valid_eps)
params, results = val_auto_var.run_grid_params(fn, grid_params=grid)
#params, ret = auto_var.run_grid_params(fn, grid)
rets.append(results)
scores = np.array(rets).mean(axis=0)
print("xvalidation results:", params, scores)
return params[np.argmax(scores)]
def test_run_grid(self):
auto_var = AutoVar()
auto_var.add_variable_class(OrdVarClass())
auto_var.add_variable_class(DatasetVarClass())
auto_var.add_variable('random_seed', int)
grid_params = {
"ord": ['1', '2'],
"dataset": ['halfmoon_50', 'halfmoon_10'],
"random_seed": [1126],
}
def fn(auto_var):
return {
"ord": auto_var.var_value['ord'],
"dataset": auto_var.var_value['dataset'],
"random_seed": auto_var.var_value['random_seed'],
}
params, results = auto_var.run_grid_params(fn,
grid_params=grid_params,
n_jobs=1)
self.assertEqual(params[0], {
'ord': '1',
'dataset': 'halfmoon_50',
'random_seed': 1126,
})
del results[0]['var_value']['git_hash']
self.assertEqual(params[0], results[0]['var_value'])
del results[0]['running_time']
del results[0]['var_value']
self.assertEqual(params[0], results[0])
def test_cmd_args_with_vars(self):
auto_var = AutoVar(logging_level=logging.INFO)
auto_var.add_variable_class(DatasetVarClass())
args = ['--dataset', 'halfmoon_300']
auto_var.parse_argparse(args=args)
with self.assertRaises(argparse.ArgumentTypeError):
args = ['--dataset', 'halfmaan_300']
auto_var.parse_argparse(args=args)
with self.assertRaises(SystemExit) as cm:
args = []
auto_var.parse_argparse(args=args)
self.assertEqual(cm.exception.code, 2)
def test_json_file(self):
settings = {'file_format': 'json', 'result_file_dir': 'test'}
mkdir_p(settings['result_file_dir'])
auto_var = AutoVar(
settings=settings,
after_experiment_hooks=[
partial(save_result_to_file, get_name_fn=default_get_file_name)
],
)
auto_var.add_variable_class(OrdVarClass())
auto_var.set_variable_value_by_dict({'ord': '1'})
def experiment(auto_var):
return {'test': auto_var.get_var('ord')}
_ = auto_var.run_single_experiment(experiment, with_hook=True)
with open("test/1.json", 'r') as f:
ret = json.load(f)
self.assertEqual(ret['test'], auto_var.get_var('ord'))
shutil.rmtree(settings['result_file_dir'])
from autovar.base import RegisteringChoiceType, register_var, VariableClass
from autovar.hooks import check_result_file_exist, save_result_to_file
from autovar.hooks import create_placeholder_file, remove_placeholder_if_error
from autovar.hooks import default_get_file_name as get_file_name
from .datasets import DatasetVarClass
auto_var = AutoVar(logging_level=logging.INFO,
before_experiment_hooks=[
partial(check_result_file_exist,
get_name_fn=get_file_name),
partial(create_placeholder_file,
get_name_fn=get_file_name),
],
after_experiment_hooks=[
partial(save_result_to_file, get_name_fn=get_file_name),
partial(remove_placeholder_if_error,
get_name_fn=get_file_name),
],
settings={
'file_format': 'pickle',
'server_url': '',
'result_file_dir': './results/'
})
auto_var.add_variable_class(DatasetVarClass())
auto_var.add_variable('random_seed', int)
#auto_var.add_variable('eps', float)
#auto_var.add_variable('optimizer', str)
#auto_var.add_variable('learning_rate', float, default=1e-2)
#auto_var.add_variable('batch_size', int, default=64)
class OrdVarClass(VariableClass, metaclass=RegisteringChoiceType):
"""Defines which distance measure to use for attack."""
var_name = "ord"
@register_var()
@staticmethod
def inf(auto_var):
"""L infinity norm"""
return np.inf
@register_var(argument='2')
@staticmethod
def l2(auto_var):
"""L2 norm"""
return 2
@register_var(argument='1')
@staticmethod
def l1(auto_var):
"""L1 norm"""
return 1
auto_var = AutoVar(logging_level=logging.CRITICAL)
auto_var.add_variable_class(OrdVarClass())
auto_var.add_variable_class(DatasetVarClass())
auto_var.add_variable_class(ModelVarClass())
auto_var.add_variable_class(AttackVarClass())
auto_var.add_variable('random_seed', int)
def test_cmd_args(self):
auto_var = AutoVar(logging_level=logging.INFO)
self.assertFalse(auto_var._no_hooks)
args = ['--no-hooks']
auto_var.parse_argparse(args=args)
self.assertTrue(auto_var._no_hooks)
def test_val(self):
auto_var = AutoVar(logging_level=logging.INFO)
with self.assertRaises(VariableNotRegisteredError):
auto_var.get_var('ord')
auto_var.add_variable_class(OrdVarClass())
auto_var.add_variable_class(DatasetVarClass())
auto_var.add_variable('random_seed', int)
with self.assertRaises(VariableValueNotSetError):
auto_var.get_var('ord')
auto_var.set_variable_value_by_dict({
'ord': '1',
'dataset': 'halfmoon_200',
'random_seed': 1126
})
self.assertEqual(auto_var.get_var('ord'), 1)
self.assertEqual(auto_var.get_var('random_seed'), 1126)
self.assertEqual(len(auto_var.get_var('dataset')[0]), 200)
self.assertEqual(
len(auto_var.get_var_with_argument('dataset', 'halfmoon_300')[0]),
300)
with self.assertRaises(ValueError):
auto_var.set_variable_value_by_dict({'ord': 'l2'})
with self.assertRaises(TypeError):
auto_var.set_variable_value_by_dict({'random_seed': '1126.0'})
self.assertEqual(auto_var.get_var_shown_name(var_name="dataset"),
'shown_halfmoon')
assert_array_equal(
auto_var.get_var_with_argument('dataset', 'halfmoon_300')[0],
auto_var.get_var_with_argument('dataset', 'moon_300')[0],
)
argparse_help = auto_var.get_argparser().format_help()
self.assertTrue('halfmoon dataset' in argparse_help)
self.assertTrue('Dataset variable class' in argparse_help)
def test_cache_files(self):
auto_var = AutoVar(logging_level=logging.INFO)
auto_var.add_variable_class(DatasetVarClass())
auto_var.add_variable_class(OrdVarClass())
auto_var.set_variable_value("dataset", "no4_halfmoon_5")
auto_var.set_variable_value("ord", "1")
X, y = auto_var.get_var("dataset")
cacheX, cachey = auto_var.get_var("dataset")
assert_array_equal(X, cacheX)
assert_array_equal(y, cachey)
temp_dir = auto_var.variables['dataset']['cache_dirs'][
'no4_halfmoon_(?P<n_samples>\\d+)']
cacheX, cachey = joblib.load(
os.path.join(temp_dir, "no4_halfmoon_5-1.pkl"))
assert_array_equal(X, cacheX)
assert_array_equal(y, cachey)
def test_cmd_args_with_vars_2(self):
auto_var = AutoVar(logging_level=logging.INFO)
auto_var.add_variable_class(OrdVarClass())
auto_var.parse_argparse(args=[])
self.assertEqual(auto_var.get_var("ord"), 2)
def l2(auto_var):
"""L2 norm"""
return 2
@register_var(argument='1')
@staticmethod
def l1(auto_var):
"""L1 norm"""
return 1
auto_var = AutoVar(
logging_level=logging.INFO,
before_experiment_hooks=[
partial(check_result_file_exist, get_name_fn=get_file_name),
#partial(save_parameter_to_file, get_name_fn=partial(get_file_name, name_only=False))
],
after_experiment_hooks=[
partial(save_result_to_file, get_name_fn=get_file_name)
],
settings={
'server_url': '',
'result_file_dir': './results/'
})
auto_var.add_variable_class(OrdVarClass())
auto_var.add_variable_class(DatasetVarClass())
auto_var.add_variable_class(ModelVarClass())
auto_var.add_variable_class(AttackVarClass())
auto_var.add_variable('random_seed', int)