def run(self,
exp_function: list or function,
params: dict = None,
artifacts: dict = None,
param_space_iter=None,
max_evals: int = None) -> list:
"""
Runs the given experiment function or list of functions with various parameter configuration
# Arguments
exp_function (function or list): Method that implements the experiment.
params (dict): Dictionary that contains the configuration (e.g. hyperparameters) for the experiment (optional).
artifacts (dict): Dictionary that contains artifacts (any kind of python object) required for the experiment (optional).
param_space_iter (iterator): An iterator of configuration dicts.
max_evals (int): Maximum number of evaluations (optional).
# Returns
List of finished experiments
"""
evals = 0
for selected_params in param_space_iter:
evals += 1
exp = self.create_exp(
name_suffix=text_utils.simplify_dict_to_str(selected_params))
params.update(selected_params)
exp.run_exp(exp_function, params, artifacts)
if max_evals and evals >= max_evals:
# Stop evaluation
break
return self.experiments()
def skopt_objective(**param_selection):
exp = self.create_exp(
name_suffix=text_utils.simplify_dict_to_str(param_selection))
params.update(param_selection)
exp.run_exp(exp_function, params, artifacts)
score = exp.exp_metadata.result
if not score or not isinstance(score, numbers.Number):
exp.log.info(
"Provided exp_function did not return a numeric score/result."
)
return None
if self.maximize_score:
# hyperopt only supports min
score = -score
return score
def plot_exp_metric_comparison(exp_list: list, metric_keys: list or str = None, param_keys: list or str = None,
title="Experiment Results", interactive=False):
"""
Comparison visualization of metrics of various experiments
# Arguments
exp_list (list): List of experiments.
metric_keys (list or str): Metric keys to select for visualization. If no metrics_key are provided, the main result metric will be used (optional).
param_keys (list or str): Param keys to use for label for visualization. If no param keys are provided, the experiment name will be used as label (optional).
title (string): Title of the visualization (optional).
interactive (boolean): If `True`, the visualization will be rendered with plotly (optional).
"""
# TODO sort metric?
labels = []
data_traces = {}
if metric_keys:
if type(metric_keys) is not list:
metric_keys = [metric_keys]
for key in metric_keys:
data_traces[key] = []
else:
data_traces["result"] = []
if param_keys:
if type(param_keys) is not list:
param_keys = [param_keys]
else:
param_dicts = []
for exp in exp_list:
param_dicts.append(exp.params)
try:
# automatically select params that have changed values
param_keys = get_keys_with_diff_values(param_dicts)
except:
# do nothing
pass
for exp in exp_list:
if param_keys:
param_dict = {}
for param in param_keys:
if param in exp.params:
param_dict[param] = exp.params[param]
labels.append(text_utils.simplify_dict_to_str(param_dict))
else:
labels.append(text_utils.truncate_middle(exp.name, 30))
if metric_keys:
for key in metric_keys:
if key in exp.metrics:
data_traces[key].append(exp.metrics[key])
else:
log.warning("Metric " + key + "couldn't be found in experiment: " + str(exp.name))
else:
if exp.exp_metadata.result:
data_traces["result"].append(exp.exp_metadata.result)
else:
log.warning("Result metric wasn't returned for experiment: " + str(exp.name))
if interactive:
import plotly
import plotly.graph_objs as go
plotly.offline.init_notebook_mode()
data = []
for key in data_traces:
trace = go.Scatter(
x=labels,
y=data_traces[key],
mode='lines+markers',
name=key
)
data.append(trace)
plotly.offline.iplot(data, filename=title)
else:
import pylab
for key in data_traces:
y = data_traces[key]
pylab.plot(range(len(y)), y, label=key)
pylab.legend(loc='best')
pylab.xticks(range(len(labels)), labels, size='small', rotation='vertical')
pylab.show()