def __init__(self, id, backend='local', seed=None):
"""
Automatically load my backend if exists
"""
self._backend = init_backend(experiment_id=self.id, backend=backend)
self._backend = self._backend.load()
if self.id != self._backend.experiment_id:
# this should not occur if user instanciates incorrect
# Backend instance by themselves
msg = 'Experiment ID mismatch: instance={}, backend={}'
raise ValueError(msg.format(self.id, self._backend.experiment_id))
self._seed = seed
if self.trial_id != 0:
msg = 'Loaded existing experiment: {}'
logger.info(msg.format(self))
else:
msg = 'Initialized new experiment: {}'
logger.info(msg.format(self))
self._parameters = ParameterManager()
self._codes = CodeManager(backend=self._backend)
self._environment = Environment(backend=self._backend)
# TODO: check code change
# output environment info
self._environment.log_environment_info()
self._check_environment_change()
def test_parameter_manager(self):
p = ParameterManager()
p.define('a')
p.define('b')
assert p.describe() == "a=Undefined, b=Undefined"
p.set(a=2)
assert p.describe() == "a=2<class 'int'>, b=Undefined"
def test_parameter_to_dict(self):
p = ParameterManager()
p.define('a')
p.define('b')
exp = {'a': Undefined(), 'b': Undefined()}
assert p.to_dict() == exp
p.set(a=1)
exp = {'a': 1, 'b': Undefined()}
assert p.to_dict() == exp
def test_parameter_redefine(self):
p = ParameterManager()
a1 = p.define('a')
assert p.describe() == "a=Undefined"
a2 = p.define('a')
assert p.describe() == "a=Undefined"
with pytest.raises(daskperiment.core.errors.ParameterUndefinedError):
a1.resolve()
with pytest.raises(daskperiment.core.errors.ParameterUndefinedError):
a2.resolve()
p.set(a=2)
assert p.describe() == "a=2<class 'int'>"
assert a1.resolve() == 2
assert a2.resolve() == 2
def test_parameter_describe(self):
p = ParameterManager()
a = p.define('a')
b = p.define('b')
p.set(a=2, b=5)
assert p.describe() == "a=2<class 'int'>, b=5<class 'int'>"
assert a.resolve() == 2
assert b.resolve() == 5
def test_parameter_resolve(self):
p = ParameterManager()
a = p.define('a')
b = p.define('b')
p.set(a=2)
assert a.resolve() == 2
with pytest.raises(daskperiment.core.errors.ParameterUndefinedError):
b.resolve()
p.set(b=5)
assert b.resolve() == 5
def test_parameter_check_defined(self):
p = ParameterManager()
p.define('a')
p.define('b')
with pytest.raises(daskperiment.core.errors.ParameterUndefinedError):
p._check_all_defined()
p.set(a=2)
with pytest.raises(daskperiment.core.errors.ParameterUndefinedError):
p._check_all_defined()
p.set(b=5)
p._check_all_defined()
class Experiment(object):
_instance_cache = {}
def __new__(cls, id, backend='local', seed=None):
# return the identical instance based on id
if id in cls._instance_cache:
return cls._instance_cache[id]
obj = super().__new__(cls)
id = validate_key(id, keyname='Experiment ID')
obj.id = id
cls._instance_cache[id] = obj
return obj
def __init__(self, id, backend='local', seed=None):
"""
Automatically load my backend if exists
"""
self._backend = init_backend(experiment_id=self.id, backend=backend)
self._backend = self._backend.load()
if self.id != self._backend.experiment_id:
# this should not occur if user instanciates incorrect
# Backend instance by themselves
msg = 'Experiment ID mismatch: instance={}, backend={}'
raise ValueError(msg.format(self.id, self._backend.experiment_id))
self._seed = seed
if self.trial_id != 0:
msg = 'Loaded existing experiment: {}'
logger.info(msg.format(self))
else:
msg = 'Initialized new experiment: {}'
logger.info(msg.format(self))
self._parameters = ParameterManager()
self._codes = CodeManager(backend=self._backend)
self._environment = Environment(backend=self._backend)
# TODO: check code change
# output environment info
self._environment.log_environment_info()
self._check_environment_change()
def __repr__(self):
msg = 'Experiment(id: {}, trial_id: {}, backend: {})'
return msg.format(self.id, self.trial_id, self._backend)
@property
def trial_id(self):
"""
Return latest trial ID of the experiment.
If the trial is performed IMMEDIATELY after, the trial's ID should be
the ID displayed here.
In database-like backends, trial ID may be changed by other process.
Thus, actual trial ID is fixed after the trial is started.
"""
return self._trials.trial_id
@property
def current_trial_id(self):
"""
Return current trial ID of the trial.
It is accessible during the trial is performing, and specifies the ID
which the trial is stored.
"""
return self._trials.current_trial_id
@property
def _trials(self):
"""
Property for TrialManager
"""
return self._backend.trials
@property
def _metrics(self):
"""
Property for MetricManager
"""
return self._backend.metrics
##########################################################
# Parameter
##########################################################
def parameter(self, name, default=None):
"""
Declare a parameter in the Experiment.
It returns Parameter instance which can be passed as an argument to
experiment functions.
Prameters
---------
name: str
Parameter variable name in the Experiment.
You must specify the name provided here in .set_parameters.
default: object, optional
Default value of the parameter.
Returns
-------
Parameter: parameter
"""
return self._parameters.define(name, default=default)
def set_parameters(self, **kwargs):
"""
Define (set) values to declared parameters.
It is a method to combine parameter name and actual value for
the trial.
Prameters
---------
**kwargs: dict
Provide parameter name and values as key=value format, like
.set_parameters(a=1, b=2)
Returns
-------
None
"""
self._parameters.set(**kwargs)
def get_parameters(self, trial_id=None):
"""
Get parameter values used in the trial.
Prameters
---------
trial_id: int, optional
Trial ID to get parameters. If not provided, current paramters are
returned.
Returns
-------
dict: parameters
"""
if trial_id is None:
return self._parameters.to_dict()
else:
self._check_trial_id(trial_id)
return self._trials.load_parameters(trial_id)
##########################################################
# Testing
##########################################################
def _delete_cache(self):
"""
Delete cache dir. This should be only used in test functions.
"""
self._backend._delete_cache()
##########################################################
# Decorators
##########################################################
def __call__(self, func=None):
"""
A decorator to declare the function is in experiment step.
It returns a ExperimentFunction which inherits Dask.Delayed.
Prameters
---------
func: callable
A function for experiment step
Returns
-------
ExperimentFunction: func
"""
def wrap(func):
return self._build_step(func, persist=False)
if func is None:
return wrap
else:
return wrap(func)
def persist(self, func=None):
"""
A decorator to declare the function is in experiment step, and
persists the function's results in each trials.
It returns a ExperimentFunction which inherits Dask.Delayed.
Prameters
---------
func: callable
A function for experiment step
Returns
-------
ExperimentFunction: func
"""
def wrap(func):
return self._build_step(func, persist=True)
if func is None:
return wrap
else:
return wrap(func)
def _build_step(self, func, persist=False):
"""
Build a single eperiment step
"""
dask_obj = dask.delayed(wrap_result(self, func, persist=persist))
self._codes.register(func)
return ExperimentFunction(self, dask_obj)
def result(self, func=None):
"""
A decorator to declare the function is the last experiment step.
It returns a ResultFunction which inherits Dask.Delayed.
The difference from ExperimentFunction is that ResultFunction updates
experimet's history, but ExperimentFunction doesn't.
Prameters
---------
func: callable
A function for experiment step
Returns
-------
ResultFunction: func
"""
def wrap(func):
dask_obj = dask.delayed(wrap_result(self, func))
self._codes.register(func)
return ResultFunction(self, dask_obj)
if func is None:
return wrap
else:
return wrap(func)
##########################################################
# Run experiment
##########################################################
def _save_experiment_step(self, trial_id):
"""
Save the trial info
"""
self._trials.save_parameters(trial_id, self._parameters)
self._codes.save(trial_id)
self._environment.save(trial_id)
def _save_backend(self):
"""
Save Backend
"""
return self._backend.save()
def _check_trial_id(self, trial_id):
if not isinstance(trial_id, int):
msg = 'Trial id must be integer, given: {}{}'
msg.format(trial_id, type(trial_id))
raise TrialIDNotFoundError(msg.format(trial_id, type(trial_id)))
if trial_id <= 0 or self.trial_id < trial_id:
raise TrialIDNotFoundError(trial_id)
def check_executable(self):
"""
Check whether the current Experiment is executable.
* Parameters are all defined
"""
self._parameters._check_all_defined()
##########################################################
# History management
##########################################################
def get_history(self, verbose=False):
"""
Return a trial history of the experiment.
It stores trial parameters and its results and related information.
Prameters
---------
verbose: bool, optinal
Whether to include detailed info (Seed and Result Type).
Default False.
Returns
-------
DataFrame: history
"""
return self._trials.get_history(verbose=verbose)
def _save_persist(self, step, result):
trial_id = self._trials.current_trial_id
key = self._backend.get_persist_key(step, trial_id)
self._backend.save_object(key, result)
def get_persisted(self, step, trial_id):
"""
Get persisted result.
Prameters
---------
step: str
The name of the function decorated by persist.
trial_id: int
Trial ID to be loaded
Returns
-------
object: persisted_result
"""
self._check_trial_id(trial_id)
key = self._backend.get_persist_key(step, trial_id)
return self._backend.load_object(key)
##########################################################
# Code management
##########################################################
def get_code(self, trial_id=None):
"""
Get code context in the trial.
Prameters
---------
trial_id: int, optional
Trial ID to get code context. If not provided, current code
context is returned.
Returns
-------
str: code_context
"""
if trial_id is not None:
self._check_trial_id(trial_id)
return self._codes.get_code(trial_id=trial_id)
##########################################################
# Metrics management
##########################################################
def save_metric(self, metric_key, epoch, value):
"""
Save metric during the trial (a transition of values during the trial).
Prameters
---------
metric_key: str
A key to distinguish metric
epoch: int
An epoch when the metric is recorded
value: scalar
A value of the distinguish metric
"""
trial_id = self._trials.current_trial_id
self._metrics.save(metric_key=metric_key,
trial_id=trial_id,
epoch=epoch,
value=value)
def load_metric(self, metric_key, trial_id):
"""
Load metric during the trial (a transition of values during the trial).
Prameters
---------
metric_key: str
A key to distinguish metric
trial_id: int, list of int
Trial ID(s) to load metric.
Returns
-------
DataFrame: metrics
"""
if not pd.api.types.is_list_like(trial_id):
trial_id = [trial_id]
for i in trial_id:
self._check_trial_id(i)
return self._metrics.load(metric_key=metric_key, trial_id=trial_id)
##########################################################
# Environment management
##########################################################
def _check_environment_change(self):
trial_id = self.trial_id
if trial_id > 0:
# Code change shouldn't be checked here,
# codes are registered after instanciation
# self._codes.check_code_change(trial_id)
self._environment.check_environment_change(trial_id)
def get_environment(self, trial_id=None, category=None):
"""
Get environment info in the trial.
* Platform information
* Python information
* daskperiment information
Prameters
---------
trial_id: int, optional
Trial ID to get environment information. If not provided, current
environment information is returned.
Returns
-------
list of str: environment
"""
if trial_id is not None:
self._check_trial_id(trial_id)
return self._environment.get_environment(trial_id=trial_id,
category=category)
def get_python_packages(self, trial_id=None):
"""
Get installed Python package information in the trial.
The format is the same as pip freeze.
Prameters
---------
trial_id: int, optional
Trial ID to get code context. If not provided, current code
context is returned.
Returns
-------
list of str: packages
"""
if trial_id is not None:
self._check_trial_id(trial_id)
return self._environment.get_python_packages(trial_id=trial_id)
##########################################################
# Dashboard
##########################################################
def start_dashboard(self, port=5000):
"""
Start DaskperimentBoard web application.
"""
import daskperiment.board.board as board
return board.maybe_start_dashboard(self, port=port)