def __init__(self, config_dict, num_samples, seed=1, grid_type='Sobol'):
self.config = HyperparameterConfiguration(config_dict)
self.params = self.config.get_param_list()
sobol = None
if grid_type == 'Sobol':
sobol = SobolGridGenerator(self.params, num_samples, seed)
else:
raise ValueError("Not supported grid type: {}".format(grid_type))
self.grid = np.asarray(sobol.generate())
self.hpvs = self.generate()
def __init__(self, name, grid, hpv, hp_config, one_hot=False):
self.name = name
if type(hp_config) == dict:
self.hp_config = HyperparameterConfiguration(hp_config)
else:
self.hp_config = hp_config
self.hpv = hpv
if one_hot == True:
self.grid = self.get_one_hot_grid()
else:
self.grid = np.asarray(grid)
super(GridParameterSpace, self).__init__(len(hpv))
def create_runner(trainer_url,
space,
run_mode,
target_val,
time_expired,
run_config,
hp_config,
goal_metric="error",
save_internal=False,
num_resume=0,
use_surrogate=None,
early_term_rule="DecaTercet",
id="Runner"):
try:
kwargs = {}
if use_surrogate != None:
kwargs["surrogate"] = use_surrogate
id += "-S_{}".format(use_surrogate)
if isinstance(hp_config, dict):
hp_config = HyperparameterConfiguration(hp_config)
cred = ""
if "credential" in run_config:
cred = run_config["credential"]
else:
raise ValueError("No credential info in run configuration")
rtc = RemoteTrainConnector(trainer_url, hp_config, cred, **kwargs)
t = trainer.get_remote_trainer(rtc, space, run_config)
if run_config and "early_term_rule" in run_config:
early_term_rule = run_config["early_term_rule"]
if early_term_rule != "None":
id = "{}.ETR-{}".format(id, early_term_rule)
machine = HPOBanditMachine(space,
t,
run_mode,
target_val,
time_expired,
run_config,
goal_metric=goal_metric,
num_resume=num_resume,
save_internal=save_internal,
id=id)
return machine
except Exception as ex:
warn("Runner creation failed: {}".format(ex))
def __init__(self, config, spec, use_default=False):
if type(config) == dict:
self.config = HyperparameterConfiguration(config)
else:
self.config = config
self.params = self.config.get_param_names()
self.spec = spec
self.use_default = use_default
if use_default:
default = self.config.get_default_vector()
debug("Default value setting: {}".format(default))
norm_vec = self.config.convert("arr", "norm_arr", default)
hpv = self.config.convert("arr", "list", default)
self.grid = np.array([norm_vec])
self.hpvs = np.array([hpv])
self.schemata = np.zeros(self.hpvs.shape)
self.generations = np.array([0,])
else:
self.grid = np.array([])
self.hpvs = np.array([])
self.schemata = np.array([])
self.generations = np.array([])
def get_status(self):
try:
resp = self.conn.request_get("/", args={}, headers=self.headers)
status = resp['headers']['status']
if status == '200':
space = json.loads(resp['body'])
self.num_samples = space['num_samples']
self.hp_config = HyperparameterConfiguration(
space["hp_config"])
self.space_id = space['space_id']
return space
else:
raise ValueError(
"Connection failed with code {}".format(status))
except Exception as ex:
debug("Getting remote space: {}".format(ex))
return None
class GridParameterSpace(ParameterSpace):
def __init__(self, name, grid, hpv, hp_config, one_hot=False):
self.name = name
if type(hp_config) == dict:
self.hp_config = HyperparameterConfiguration(hp_config)
else:
self.hp_config = hp_config
self.hpv = hpv
if one_hot == True:
self.grid = self.get_one_hot_grid()
else:
self.grid = np.asarray(grid)
super(GridParameterSpace, self).__init__(len(hpv))
def get_size(self):
return len(self.hpv)
def get_name(self):
return self.name
def get_hp_config(self):
return self.hp_config
def get_params(self):
return self.hp_config.get_param_list()
def get_grid_dim(self):
return self.grid.shape[1]
def get_grid(self, type_or_index='all', use_interim=False):
if type_or_index == "completes":
completes = self.get_completes(use_interim)
#debug("index of completes: {}".format(completes))
return self.grid[completes, :]
elif type_or_index == "candidates":
candidates = self.get_candidates(use_interim)
#debug("index of candidates: {}".format(candidates))
return self.grid[candidates, :]
elif type_or_index != 'all':
return self.grid[type_or_index]
else:
return self.grid
def get_hpv(self, index=None):
if index != None:
params = self.hp_config.get_param_list()
args = self.hpv[index]
hpv = {}
for i in range(len(params)):
p = params[i]
hpv[p] = args[i]
return hpv
else:
return self.hpv
def expand(self, hpv):
cvt = VectorGridConverter(self.hpv, self.get_candidates(), self.hp_config)
grid_vec = cvt.to_grid_vector(hpv)
self.hpv = np.append(self.hpv, [hpv], axis=0)
self.grid = np.append(self.grid, [grid_vec], axis=0)
debug("Sampling space expanded: {}".format(len(self.hpv)))
return super(GridParameterSpace, self).expand(hpv)
def get_one_hot_grid(self):
grid = []
num_samples = self.get_size()
for i in range(num_samples):
s = self.get_hpv(i)
c = self.get_hp_config()
t = OneHotVectorTransformer(c)
e = t.transform(s)
grid.append(np.asarray(e))
return np.asarray(grid)
def __init__(self, config):
if type(config) == dict:
config = HyperparameterConfiguration(config)
self.config = config
class HyperparameterVectorGenerator(object):
def __init__(self, config_dict, num_samples, seed=1, grid_type='Sobol'):
self.config = HyperparameterConfiguration(config_dict)
self.params = self.config.get_param_list()
sobol = None
if grid_type == 'Sobol':
sobol = SobolGridGenerator(self.params, num_samples, seed)
else:
raise ValueError("Not supported grid type: {}".format(grid_type))
self.grid = np.asarray(sobol.generate())
self.hpvs = self.generate()
def get_grid(self):
return self.grid
def get_hpv(self):
#debug("HPV-0: {}".format(self.hpvs[0]))
return self.hpvs
def generate(self, return_type='array'):
debug("Sampling hyperparameter configurations...")
hps = self.config.hyperparams
hpv_list = []
if return_type == 'array':
for i in range(len(self.grid)):
vec = self.grid[i]
hpv = []
for j in range(len(vec)):
param_name = self.params[j]
value = vec[j]
hp_cfg = getattr(hps, param_name)
arg = self.to_param_value(hp_cfg, value)
hpv.append(arg)
#debug("hp{}:{}".format(i, hpv))
hpv_list.append(hpv)
return hpv_list
else:
for i in range(len(self.grid)):
vec = self.grid[i]
hpv = {}
for j in range(len(vec)):
param_name = self.params[j]
value = vec[j]
hp_cfg = getattr(hps, param_name)
arg = self.to_param_value(hp_cfg, value)
hpv[param_name] = arg
hpv_list.append(hpv)
if return_type == 'table':
table = pd.DataFrame(data=hpv_list)
return table
elif return_type == 'dict':
return hpv_list
else:
raise ValueError(
"Not supported format: {}".format(return_type))
def to_param_value(self, hp_cfg, value):
result = None
range_list = hp_cfg.range
range_list.sort()
if hp_cfg.value_type == "categorical" or hp_cfg.value_type == 'preordered':
size = len(range_list)
index = int(value * size)
if index == size: # handle terminal condition
index = size - 1
result = range_list[index]
else:
max_value = max(range_list)
min_value = min(range_list)
if hp_cfg.type == 'int':
result = min_value + int(
value *
(max_value - min_value + 1)) #XXX:to include max value
if hasattr(hp_cfg, 'power_of'):
result = int(np.power(hp_cfg.power_of, result))
elif hp_cfg.type == 'float':
result = min_value + (
value * (max_value - min_value)
) #FIXME:float type can't access max_value.
if hasattr(hp_cfg, 'power_of'):
result = np.power(hp_cfg.power_of, result)
if hp_cfg.type == 'int':
result = int(result)
elif hp_cfg.type == 'bool':
result = bool(result)
elif hp_cfg.type == 'str':
result = str(result)
return result
class HyperparameterVectorGenerator(object):
def __init__(self, config, spec, use_default=False):
if type(config) == dict:
self.config = HyperparameterConfiguration(config)
else:
self.config = config
self.params = self.config.get_param_names()
self.spec = spec
self.use_default = use_default
if use_default:
default = self.config.get_default_vector()
debug("Default value setting: {}".format(default))
norm_vec = self.config.convert("arr", "norm_arr", default)
hpv = self.config.convert("arr", "list", default)
self.grid = np.array([norm_vec])
self.hpvs = np.array([hpv])
self.schemata = np.zeros(self.hpvs.shape)
self.generations = np.array([0,])
else:
self.grid = np.array([])
self.hpvs = np.array([])
self.schemata = np.array([])
self.generations = np.array([])
def get_param_vectors(self):
return self.grid
def get_hp_vectors(self):
return self.hpvs
def get_schemata(self):
return self.schemata
def get_generations(self):
return self.generations
def create_generator(self, spec):
seed = spec['seed']
n_s = spec['num_samples']
if 'sample_method' in spec:
if spec['sample_method'] == 'uniform':
return UniformRandomGenerator(self.config, n_s, seed)
elif spec['sample_method'] == 'latin':
return LatinHypercubeGenerator(self.config, n_s, seed)
elif spec['sample_method'] == 'local':
return LocalSearchGenerator(self.config, n_s,
spec['best_candidate'], spec['generation'], seed)
elif spec['sample_method'] == 'genetic':
if not 'mutation_ratio' in spec:
spec['mutation_ratio'] = .1 # default mutation ratio
return EvolutionaryGenerator(self.config, n_s, spec['current_best'],
spec['best_candidate'], seed, spec['mutation_ratio'])
elif spec['sample_method'] != 'Sobol':
warn("Not supported sampling method: {}. We utilize Sobol sequences as default.".format(spec['sample_method']))
if 'num_skips' in spec:
seed += spec['num_skips']
return SobolSequenceGenerator(self.config, n_s, seed)
def generate(self):
#debug("Sampling hyperparameter configurations...")
s_t = time.time()
try:
g = self.create_generator(self.spec)
debug("Candidates will be generated using {}...".format(g.get_name()))
grid = g.generate()
schemata = g.get_schemata()
gen = g.get_generations()
# TODO:speeding up required
if self.use_default:
self.grid = np.concatenate((self.grid, grid))
self.schemata = np.concatenate((self.schemata, schemata))
self.generations = np.concatenate((self.generations, gen))
else:
self.grid = grid
self.schemata = schemata
self.generations = gen
self.hpvs = self.config.convert('grid', 'hpv_list', self.grid)
except Exception as ex:
warn("Failed to generate candidates: {}".format(ex))