def test_parent_parent_fetch_trials(create_db_instance):
"""Test that experiment fetch trials from grand parent properly (adapters are muted)"""
experiment_name = 'supernaedo2.3.1'
root_name = 'supernaedo2'
leaf_names = ['supernaedo2.3.1']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.parent.parent.item.name == root_name
assert len(exp_node.children) == 0
# 2
# |
# 2.3
# |
# 2.3.1
assert len(list(exp_node.root)) == 3
experiment.connect_to_version_control_tree(exp_node)
for node in exp_node.root:
node.item._experiment.refers['adapter'] = Adapter.build([])
query = {'status': 'completed'}
assert len(exp_node.parent.parent.item.fetch_trials(query)) == 6
assert len(exp_node.parent.item.fetch_trials(query)) == 4
assert len(exp_node.item.fetch_trials(query)) == 2
assert len(experiment.fetch_trials_tree(query)) == 6 + 4 + 2
def test_children_fetch_trials(create_db_instance):
"""Test that experiment fetch trials from children properly (adapters are muted)"""
experiment_name = "supernaedo2"
root_name = "supernaedo2"
leaf_names = ["supernaedo2.3"]
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.children[0].item.name == leaf_names[0]
assert len(exp_node.children) == 1
# 2
# |
# 2.3
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
for node in exp_node.root:
node.item._experiment.refers["adapter"] = Adapter.build([])
query = {"status": "completed"}
assert len(experiment.fetch_trials(query)) == 6
assert len(
experiment._experiment._node.children[0].item.fetch_trials(query)) == 4
assert len(experiment.fetch_trials_tree(query)) == 10
def _instantiate_config(self, config):
"""Check before dispatching experiment whether configuration corresponds
to a executable experiment environment.
1. Check `refers` and instantiate `Adapter` objects from it.
2. Try to build parameter space from user arguments.
3. Check whether configured algorithms correspond to [known]/valid
implementations of the ``Algorithm`` class. Instantiate these objects.
4. Check if experiment `is_done`, prompt for larger `max_trials` if it is. (TODO)
"""
# Just overwrite everything else given
for section, value in config.items():
if section not in self.__slots__:
log.info(
"Found section '%s' in configuration. Experiments "
"do not support this option. Ignoring.", section)
continue
if section.startswith('_'):
log.info(
"Found section '%s' in configuration. "
"Cannot set private attributes. Ignoring.", section)
continue
# Copy sub configuration to value confusing side-effects
# Only copy at this level, not `config` directly to avoid TypeErrors if config contains
# non-serializable objects (copy.deepcopy complains otherwise).
if isinstance(value, dict):
value = copy.deepcopy(value)
setattr(self, section, value)
try:
space_builder = SpaceBuilder()
space = space_builder.build_from(config['metadata']['user_args'])
if not space:
raise ValueError(
"Parameter space is empty. There is nothing to optimize.")
# Instantiate algorithms
self.algorithms = PrimaryAlgo(space, self.algorithms)
except KeyError:
pass
if self.refers and not isinstance(self.refers.get('adapter'),
BaseAdapter):
self.refers['adapter'] = Adapter.build(self.refers['adapter'])
if not self.producer.get('strategy'):
self.producer = {
'strategy': Strategy(of_type="MaxParallelStrategy")
}
elif not isinstance(self.producer.get('strategy'),
BaseParallelStrategy):
self.producer = {
'strategy': Strategy(of_type=self.producer['strategy'])
}
def _instantiate_adapters(config):
"""Instantiate the adapter object
Parameters
----------
config: list
List of adapter configurations to build a CompositeAdapter for the EVC.
"""
return Adapter.build(config)
def test_algorithm_change_configuration():
"""Test :meth:`orion.core.evc.adapters.AlgorithmChange.configuration`"""
algorithm_change_adaptor = AlgorithmChange()
configuration = algorithm_change_adaptor.configuration[0]
assert configuration['of_type'] == "algorithmchange"
assert Adapter.build([configuration
]).adapters[0].configuration[0] == configuration
def test_composite_configuration(dummy_param):
"""Test :meth:`orion.core.evc.adapters.CompositeAdapter.configuration`"""
new_param = Trial.Param(name='second_normal_prior',
type='integer',
value=1)
dimension_addition_adapter = DimensionAddition(dummy_param)
dimension_deletion_adapter = DimensionDeletion(new_param)
composite_adapter = CompositeAdapter(dimension_addition_adapter,
dimension_deletion_adapter)
configuration = composite_adapter.configuration
assert configuration[0] == dimension_addition_adapter.configuration[0]
assert configuration[1] == dimension_deletion_adapter.configuration[0]
assert Adapter.build(
configuration).adapters[0].configuration[0] == configuration[0]
assert Adapter.build(
configuration).adapters[1].configuration[0] == configuration[1]
def test_code_change_configuration():
"""Test :meth:`orion.core.evc.adapters.CodeChange.configuration`"""
code_change_adaptor = CodeChange(CodeChange.UNSURE)
configuration = code_change_adaptor.configuration[0]
assert configuration['of_type'] == "codechange"
assert configuration['change_type'] == CodeChange.UNSURE
assert Adapter.build([configuration
]).adapters[0].configuration[0] == configuration
def test_dimension_deletion_configuration(dummy_param):
"""Test :meth:`orion.core.evc.adapters.DimensionDeletion.configuration`"""
dimension_deletion_adapter = DimensionDeletion(dummy_param)
configuration = dimension_deletion_adapter.configuration[0]
assert configuration['of_type'] == "dimensiondeletion"
assert configuration['param'] == dummy_param.to_dict()
assert Adapter.build([configuration
]).adapters[0].configuration[0] == configuration
def test_adapter_creation(dummy_param):
"""Test initialization using :meth:`orion.core.evc.adapters.Adapter.build`"""
adapter = Adapter.build([{
'of_type': 'DimensionAddition',
'param': dummy_param.to_dict()
}])
assert isinstance(adapter, CompositeAdapter)
assert len(adapter.adapters) == 1
assert isinstance(adapter.adapters[0], DimensionAddition)
assert adapter.adapters[0].param.to_dict() == dummy_param.to_dict()
def test_dimension_renaming_configuration():
"""Test :meth:`orion.core.evc.adapters.DimensionRenaming.configuration`"""
old_name = 'old_name'
new_name = 'new_name'
dimension_renaming_adapter = DimensionRenaming(old_name, new_name)
configuration = dimension_renaming_adapter.configuration[0]
assert configuration['of_type'] == "dimensionrenaming"
assert configuration['old_name'] == old_name
assert configuration['new_name'] == new_name
assert Adapter.build([configuration
]).adapters[0].configuration[0] == configuration
def test_dimension_prior_change_configuration(small_prior, large_prior):
"""Test :meth:`orion.core.evc.adapters.DimensionPriorChange.configuration`"""
dimension_name = 'small_prior'
dimension_prior_change_adapter = DimensionPriorChange(
dimension_name, small_prior, large_prior)
configuration = dimension_prior_change_adapter.configuration[0]
assert configuration['of_type'] == "dimensionpriorchange"
assert configuration['name'] == dimension_name
assert configuration['old_prior'] == small_prior
assert configuration['new_prior'] == large_prior
assert Adapter.build([configuration
]).adapters[0].configuration[0] == configuration
def __init__(self, name, user=None):
"""Initialize viewed experiment object with primary key (:attr:`name`, :attr:`user`).
Build an experiment from configuration found in `Database` with a key (name, user).
.. note::
A view is fully configured at initialiation. It cannot be reconfigured.
If no experiment is found for the key (name, user), a `ValueError` will be raised.
:param name: Describe a configuration with a unique identifier per :attr:`user`.
:type name: str
"""
self._experiment = Experiment(name, user)
if self._experiment.id is None:
raise ValueError(
"No experiment with given name '%s' for user '%s' inside database, "
"no view can be created." %
(self._experiment.name, self._experiment.metadata['user']))
# TODO: Views are not fully configured until configuration is refactored
# This snippet is to instantiate adapters anyhow, because it is required for
# experiment views in EVC.
self.refers.setdefault('parent_id', None)
self.refers.setdefault('root_id', self._id)
self.refers.setdefault('adapter', [])
if self.refers['adapter'] and not isinstance(
self.refers.get('adapter'), BaseAdapter):
self.refers['adapter'] = Adapter.build(self.refers['adapter'])
# try:
# self._experiment.configure(self._experiment.configuration, enable_branching=False,
# enable_update=False)
# except ValueError as e:
# if "Configuration is different and generates a branching event" in str(e):
# raise RuntimeError(
# "Configuration in the database does not correspond to the one generated by "
# "Experiment object. This is likely due to a backward incompatible update in "
# "Oríon. Please report to https://github.com/epistimio/orion/issues.") from e
# raise
self._experiment._storage = ReadOnlyStorageProtocol(
self._experiment._storage)