def __init__(self, **kwargs):
super(HostState, self).__init__(**kwargs)
self.results_dir = self._register('results_dir', 'results/', True)
self.tmp_dir = self._register('tmp_dir', 'tmp/')
self.export_dir = self._register('export_dir', 'export/', True)
# ensure the user don't shoot himself in the foot
if self.results_dir == self.tmp_dir:
fatal('Result dir and tmp dir must be different')
# create directory if needed
tf_utils.create_directory(self.results_dir)
tf_utils.create_directory(self.tmp_dir, remove_existing=True)
tf_utils.create_directory(self.export_dir)
# init _HOST
config._Host = Host()
status = config._Host.get_status()
tf_version = status['software']['tensorflow']
if tf_version:
major, minor, rev = tf_version.split('.')
if major == '1':
if int(minor) >= 13:
print('ok')
else:
fatal(
"Keras Tuner only work with TensorFlow version >= 1.13\
current version: %s - please upgrade" % tf_version)
else:
warning('Could not determine TensorFlow version.')
def add(self, metric):
""" Add a metric to the collection
Args:
metric (Metric or str): Metric object or metric name
"""
# our own metric object -> direct add
if isinstance(metric, Metric):
# our own metric, do nothing
metric_name = metric.name
else:
if isinstance(metric, str):
# metric by name
metric_name = metric
else:
# keras metric
metric_name = metric.name
metric_name = self._replace_alias(metric_name)
# canonalize metric name (val_metric vs metric)
no_val_name = metric_name.replace('val_', '')
if no_val_name in _METRIC_DIRECTION:
direction = _METRIC_DIRECTION[no_val_name]
else:
fatal('Unknown metric %s' % metric_name)
# create a metric object
metric = Metric(metric_name, direction)
if metric_name in self._objects:
fatal('Duplicate metric:%s' % metric_name)
self._objects[metric_name] = metric
self._last_insert_idx = metric_name
def set_objective(self, name):
"Mark a metric as the tuning objective"
name = self._replace_alias(name)
print(name)
if name not in self._objects:
found = False
# accuracy special case which seems to be only a
# Windows -TF 1.13+ issue
if 'accuracy' in name:
# is it a validation metric
val = False
if 'val' in name:
val = True
for mm in self.get_metric_names():
# none val_* accuracy
if not val and 'accuracy' in mm and 'val' not in mm:
name = mm
found = True
# val_* accuracy
if val and 'accuracy' in mm and 'val' in mm:
name = mm
found = True
if not found:
metrics = ", ".join(list(self._objects.keys()))
fatal("can't find objective: %s in metric list:%s" % (name,
metrics))
if self._objective_name:
fatal("Objective already set to %s" % self._objective_name)
self._objective_name = name
self._objects[name].is_objective = True
return name
def __init__(self, name, direction):
""" Initialize a metric object
Args:
name (str): metric name
direction (str): metric direction. One of {'min', 'max'}
Attributes:
history (list): metric epoch history
is_objective (bool): is this metric the main tuning objectived.
Defaults to False.
start_time (float): when the metric was created
wall_time (list): time it took to reach a given epoch from start
time. Data recorded as float which are delta from start_time.
"""
self.name = name
if direction not in ['min', 'max']:
fatal('invalid direction. must be in {min, max}')
self.direction = direction
self.history = []
self.is_objective = False
self.start_time = time()
self.wall_time = []
def parse_args():
"Parse cmdline options"
parser = argparse.ArgumentParser(description='display tuning results')
parser.add_argument('--input_dir',
'-i',
type=str,
default='results/',
help='Directory containing tuner results')
parser.add_argument('--project',
'-p',
type=str,
default='default',
help='Which project to display result for')
parser.add_argument('--architecture',
'-a',
type=str,
default=None,
help='Restrict results to a given architecture')
parser.add_argument('--num_models',
'-n',
type=int,
default=10,
help='Num models to display')
parser.add_argument('--metric',
'-m',
type=str,
default=None,
help='Metrics to sort by - if None\
use objective')
parser.add_argument('--display_hyper_parameters',
'--hyper',
type=bool,
default=True,
help='Display hyperparameters values')
parser.add_argument('--use_colors',
'-c',
type=bool,
default=True,
help='Use terminal colors.')
args = parser.parse_args()
if not os.path.exists(args.input_dir):
fatal("[Error] Invalid Input directory %s" % args.input_dir,
raise_exception=False)
parser.print_help()
quit()
return args
def results_summary(input_dir='results/',
project='default',
architecture=None,
sort_metric=None,
display_hyper_parameters=True,
num_models=10,
use_colors=True):
"""
Collect kerastuner results and output a summary
"""
ic = InstanceStatesCollection()
ic.load_from_dir(input_dir, project=project, verbose=0)
if sort_metric:
instance_states = ic.sort_by_metric(sort_metric)
else:
# by default sort by objective
instance_states = ic.sort_by_objective()
sort_metric = ic.get_last().objective
other_metrics = ic.get_last().agg_metrics.get_metric_names()
other_metrics.remove(sort_metric) # removing the main metric
sort_metric_values = []
other_metrics_values = defaultdict(list)
hyper_parameters = defaultdict(list)
hyper_parameters_group = defaultdict(set)
for instance in instance_states[:num_models]:
val = instance.agg_metrics.get(sort_metric).get_best_value()
sort_metric_values.append(round(val, 4))
# other metrics
for metric in other_metrics:
val = instance.agg_metrics.get(metric).get_best_value()
other_metrics_values[metric].append(round(val, 4))
# hyper-parameters
for k, v in instance.hyper_parameters.items():
hyper_parameters[k].append(v)
hyper_parameters_group[v['group']].add(k)
if not len(sort_metric_values):
fatal("No models found - wrong dir (-i) or project (-p)?")
num_models = min(len(sort_metric_values), num_models)
section("Result summary")
subsection("Metrics")
display_metrics(sort_metric, sort_metric_values, other_metrics,
other_metrics_values, num_models, use_colors)
if display_hyper_parameters:
subsection("Hyper Parameters")
display_hparams(hyper_parameters, hyper_parameters_group, num_models,
use_colors)
def Linear(self, name, start, stop, num_buckets, precision=0,
group='default'):
"""Return a random value from a range which is linearly divided.
Args:
name (str): name of the parameter
start (int/float): lower bound of the range
stop (int/float): upper bound of the range
num_buckets (int): into how many buckets to divided the range in
precision (int): For float range. Round the result rounded to the
nth decimal if needed. 0 means not rounded
Returns:
an element of the range
"""
if not isinstance(start, float) and not isinstance(start, int):
fatal("start must be a float or an int")
if not isinstance(stop, float) and not isinstance(stop, int):
fatal("stop must be a float or an int")
if not isinstance(num_buckets, int):
fatal("num_bucket must be an integer")
if stop <= start:
fatal("start value:%s larger than stop value:%s" % (start, stop))
my_range = np.linspace(start, stop, num_buckets)
self._record_hyperparameters(name, 'Linear', num_buckets, start, stop,
group, my_range)
return start
def Range(self, name, start, stop, increment=1, group='default'):
"""Return a random value from a range.
Args:
name (str): name of the parameter
start (int/float): lower bound of the range
stop (int/float): upper bound of the range
increment (int/float): incremental step
Returns:
an element of the range
"""
if not isinstance(start, int) or not isinstance(stop, int):
fatal("start, stop must be integers")
if not isinstance(increment, int):
fatal("increment must be an integer")
if stop <= start:
fatal("start value:%s larger than stop value:%s" % (start, stop))
rsize = stop - start
if rsize < increment:
fatal("increment: %s greater than range size:%s" % (increment,
rsize))
my_range = list(range(start, stop, increment))
self._record_hyperparameters(name, 'Range', len(my_range),
start, stop, group, my_range)
return start
def Choice(self, name, selection, group="default"):
"""Return a random value from an explicit list of choice.
Args:
name (str): name of the parameter
selection (list): list of explicit choices
group (str): Optional logical group name this parameter belongs to
Returns:
an element of the list provided
"""
if not isinstance(selection, list):
fatal("list if choice must be a list []")
self._record_hyperparameters(name, 'Choice', len(selection),
selection[0], selection[-1], group,
selection)
return selection[0]
def _register(self, name, default_value, to_report=False):
"""
Register a user value and check its value type match what is expected
Args:
name (str): Arg name
default_value: the default value if not supplied by the user
to_report (bool, optional): Defaults to False. Report as key param?
Returns:
value to use
"""
value = self.kwargs.get(name, default_value)
if not isinstance(value, type(default_value)):
fatal('Invalid type for %s -- expected:%s, got:%s' %
(name, type(default_value), type(value)))
self.user_parameters.append(name)
if to_report:
self.to_report.append(name)
return value
def _record_hyperparameters(self, name, htype, space_size, start, stop,
group, values):
"""
Record a given hyperparameter
Args:
name (str): name of the hyperparameter
htype (str): type of hyperparameter
space_size (int): number of values the param can take
start: lower bound
stop: upper bound
values (list): list of potential values. Truncated to 100
"""
key = self._get_key(name, group)
# check if we have a duplicate
if key in self._hyperparameters_config:
fatal("%s hyperparameter is declared twice" % key)
# making sure values are serializable
serializable_values = []
for v in values[:self.max_reported_values]:
if isinstance(v, np.integer):
serializable_values.append(int(v))
if isinstance(v, np.float):
serializable_values.append(float(v))
else:
serializable_values.append(v)
self._hyperparameters_config[key] = {
"name": name,
"group": group,
"type": htype,
"space_size": space_size,
"start": start,
"stop": stop,
"values": serializable_values
}
def _record_hyperparameter(self, name, value, group):
""" Record hyperparameter value
Args:
name (str): name of the hyperparameter
value: value of the hyperparameter
group (str): which logical group this parameters belongs to
"""
hparam = {"name": name, "value": value, "group": group}
key = self._get_key(name, group)
# new hyper-parameter - makes reporting unstable
if key not in self._hyperparameters_config:
self.dynamic_hyperparameters = True
if self.fatal_on_dynamic_hyperparmeter:
fatal('Parameter %s is dynamic - this is incompatible with\
tuning algorithm' % key)
else:
warning('Parameter %s is dynamic - this will make reporitng\
innacurate. Consider making hyperparameters\
non-conditional' % key)
self._hyperparameters[key] = hparam
def __init__(self, model_fn, objective, name, distributions, **kwargs):
""" Tuner abstract class
Args:
model_fn (function): Function that return a Keras model
name (str): name of the tuner
objective (str): Which objective the tuner optimize for
distributions (Distributions): distributions object
Notes:
All meta data and varialbles are stored into self.state
defined in ../states/tunerstate.py
"""
# hypertuner state init
self.state = TunerState(name, objective, **kwargs)
self.stats = self.state.stats # shorthand access
self.cloudservice = CloudService()
# check model function
if not model_fn:
fatal("Model function can't be empty")
try:
mdl = model_fn()
except:
traceback.print_exc()
fatal("Invalid model function")
if not isinstance(mdl, Model):
t = "tensorflow.keras.models.Model"
fatal("Invalid model function: Doesn't return a %s object" % t)
# function is valid - recording it
self.model_fn = model_fn
# Initializing distributions
hparams = config._DISTRIBUTIONS.get_hyperparameters_config()
if len(hparams) == 0:
warning("No hyperparameters used in model function. Are you sure?")
# set global distribution object to the one requested by tuner
# !MUST be after _eval_model_fn()
config._DISTRIBUTIONS = distributions(hparams)
# instances management
self.max_fail_streak = 5 # how many failure before giving up
self.instance_states = InstanceStatesCollection()
# previous models
print("Loading from %s" % self.state.host.results_dir)
count = self.instance_states.load_from_dir(self.state.host.results_dir,
self.state.project,
self.state.architecture)
self.stats.instance_states_previously_trained = count
info("Tuner initialized")
