def test_parent_parent_fetch_trials(create_db_instance):
"""Test that experiment fetch trials from grand parent properly (adapters are muted)"""
experiment_name = 'supernaedo2.3.1'
root_name = 'supernaedo2'
leaf_names = ['supernaedo2.3.1']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.parent.parent.item.name == root_name
assert len(exp_node.children) == 0
# 2
# |
# 2.3
# |
# 2.3.1
assert len(list(exp_node.root)) == 3
experiment.connect_to_version_control_tree(exp_node)
for node in exp_node.root:
node.item._experiment.refers['adapter'] = Adapter.build([])
query = {'status': 'completed'}
assert len(exp_node.parent.parent.item.fetch_trials(query)) == 6
assert len(exp_node.parent.item.fetch_trials(query)) == 4
assert len(exp_node.item.fetch_trials(query)) == 2
assert len(experiment.fetch_trials_tree(query)) == 6 + 4 + 2
def test_full_forward(create_db_instance):
"""Test that trials are adapted properly down to leaf"""
experiment_name = 'supernaedo2.3.1.1'
root_name = None
leaf_names = []
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.parent.item.name == 'supernaedo2.3.1'
assert exp_node.parent.parent.item.name == 'supernaedo2.3'
assert exp_node.parent.parent.parent.item.name == 'supernaedo2'
# 2
# |
# 2.3
# |
# 2.3.1
# |
# 2.3.1.1
assert len(list(exp_node.root)) == 4
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
assert len(exp_node.item.fetch_trials(query)) == 1
assert len(exp_node.parent.item.fetch_trials(query)) == 2
assert len(exp_node.parent.parent.item.fetch_trials(query)) == 4
assert len(exp_node.parent.parent.parent.item.fetch_trials(query)) == 6
assert len(experiment.fetch_trials_tree(query)) == 2 + 1 + 2 + 1
def test_full_forward_full_backward(create_db_instance):
"""Test that trials are adapted properly forward from parent and backward from leafs"""
experiment_name = 'supernaedo2.3'
root_name = 'supernaedo2'
leaf_names = []
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.parent.name == root_name
assert exp_node.children[0].item.name == 'supernaedo2.3.1'
assert exp_node.children[0].children[0].item.name == 'supernaedo2.3.1.1'
assert exp_node.children[0].children[1].item.name == 'supernaedo2.3.1.2'
assert exp_node.children[0].children[2].item.name == 'supernaedo2.3.1.3'
# 2
# |
# 2.3
# |
# 2.3.1
# | \ \
# 2.3.1.1 2.3.1.2 2.3.1.3
assert len(list(exp_node.root)) == 6
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
assert len(exp_node.parent.item.fetch_trials(query)) == 6
assert len(exp_node.item.fetch_trials(query)) == 4
assert len(exp_node.children[0].item.fetch_trials(query)) == 2
assert len(exp_node.children[0].children[0].item.fetch_trials(query)) == 1
assert len(exp_node.children[0].children[1].item.fetch_trials(query)) == 1
assert len(exp_node.children[0].children[2].item.fetch_trials(query)) == 1
assert len(experiment.fetch_trials_tree(query)) == 6 + 4 + 1 + 1 + 0 + 0
def test_code_change_break_backward(create_db_instance):
"""Test that no trials pass to parent when code change type is 'break'"""
experiment_name = 'supernaedo2.3.1'
root_name = 'supernaedo2.3.1'
leaf_names = ['supernaedo2.3.1.3']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.children[0].item.name == leaf_names[0]
# 2.3
# |
# 2.3.1
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
children_trials = exp_node.children[0].item.fetch_trials(query)
assert len(children_trials) == 1
assert len(exp_node.item.fetch_trials(query)) == 2
adapter = exp_node.children[0].item.refers['adapter']
assert adapter.adapters[0].change_type == CodeChange.BREAK
adapted_children_trials = adapter.backward(children_trials)
assert len(adapted_children_trials) == 0
assert len(experiment.fetch_trials_tree(query)) == 2 + 0
def test_algo_change_backward(create_db_instance):
"""Test that all trials pass to parent when algorithm is changed"""
experiment_name = 'supernaedo2.2'
root_name = 'supernaedo2.2'
leaf_names = ['supernaedo2.2.1']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.children[0].item.name == leaf_names[0]
# 2.2
# |
# 2.2.1
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
children_trials = exp_node.children[0].item.fetch_trials(query)
assert len(children_trials) == 1
assert len(exp_node.item.fetch_trials(query)) == 1
adapter = exp_node.children[0].item.refers['adapter']
adapted_children_trials = adapter.backward(children_trials)
assert len(adapted_children_trials) == 1
assert len(experiment.fetch_trials_tree(query)) == 1 + 1
def test_prior_change_backward(create_db_instance):
"""Test that all encoding are renamed to encoding_layer in parent"""
experiment_name = 'supernaedo2.3'
root_name = 'supernaedo2.3'
leaf_names = ['supernaedo2.3.1']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.children[0].item.name == leaf_names[0]
# 2.3
# |
# 2.3.1
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
children_trials = exp_node.children[0].item.fetch_trials(query)
assert len(children_trials) == 2
assert len(exp_node.item.fetch_trials(query)) == 4
adapter = exp_node.children[0].item.refers['adapter']
adapted_children_trials = adapter.backward(children_trials)
assert len(adapted_children_trials) == 1
assert len(experiment.fetch_trials_tree(query)) == 4 + 1
def test_code_change_unsure_forward(create_db_instance):
"""Test that all trials pass to children when code change type is 'unsure'"""
experiment_name = 'supernaedo2.3.1.2'
root_name = 'supernaedo2.3.1'
leaf_names = ['supernaedo2.3.1.2']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.parent.item.name == root_name
# 2.3.1
# |
# 2.3.1.1
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
parent_trials = exp_node.parent.item.fetch_trials(query)
assert len(parent_trials) == 2
assert len(exp_node.item.fetch_trials(query)) == 1
adapter = experiment.refers['adapter']
assert adapter.adapters[0].change_type == CodeChange.UNSURE
adapted_parent_trials = adapter.forward(parent_trials)
assert len(adapted_parent_trials) == 2
assert len(experiment.fetch_trials_tree(query)) == 2 + 1
def test_prior_change_forward(create_db_instance):
"""Test that trials from parent only pass to children if valid in the new prior"""
experiment_name = 'supernaedo2.3.1'
root_name = 'supernaedo2.3'
leaf_names = ['supernaedo2.3.1']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.parent.item.name == root_name
# 2.3
# |
# 2.3.1
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
parent_trials = exp_node.parent.item.fetch_trials(query)
assert len(parent_trials) == 4
assert len(exp_node.item.fetch_trials(query)) == 2
adapter = experiment.refers['adapter']
adapted_parent_trials = adapter.forward(parent_trials)
assert len(adapted_parent_trials) == 1
assert len(experiment.fetch_trials_tree(query)) == 2 + 1
def test_renaming_forward(create_db_instance):
"""Test that all encoding_layer are renamed to encoding in children"""
experiment_name = 'supernaedo2.3'
root_name = 'supernaedo2'
leaf_names = ['supernaedo2.3']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.parent.item.name == root_name
# 2
# |
# 2.1
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
parent_trials = exp_node.parent.item.fetch_trials(query)
assert len(parent_trials) == 6
assert len(exp_node.item.fetch_trials(query)) == 4
assert all((trial._params[0].name == "/encoding_layer") for trial in parent_trials)
adapter = experiment.refers['adapter']
adapted_parent_trials = adapter.forward(parent_trials)
assert len(adapted_parent_trials) == 6
assert all((trial._params[0].name == "/encoding") for trial in adapted_parent_trials)
assert len(experiment.fetch_trials_tree(query)) == 6 + 4
def test_deletion_adapter_backward(create_db_instance):
"""Test that all decoding_layer are passed with gru to parent"""
experiment_name = 'supernaedo2'
root_name = 'supernaedo2'
leaf_names = ['supernaedo2.1']
experiment = ExperimentView(experiment_name)
exp_node = build_trimmed_tree(experiment, root_name, leaf_names)
assert exp_node.item.name == experiment_name
assert exp_node.children[0].item.name == leaf_names[0]
# 2
# |
# 2.1
assert len(list(exp_node.root)) == 2
experiment.connect_to_version_control_tree(exp_node)
query = {'status': 'completed'}
assert len(exp_node.item.fetch_trials(query)) == 6
assert len(exp_node.children[0].item.fetch_trials(query)) == 1
adapter = exp_node.children[0].item.refers['adapter']
assert len(adapter.backward(exp_node.children[0].item.fetch_trials(query))) == 1
assert len(experiment.fetch_trials_tree(query)) == 6 + 1
class ExperimentNode(TreeNode):
"""Experiment node to connect experiments to EVC tree.
The node carries an experiment in attribute `item`. The node can be instantiated only using the
name of the experiment. The experiment will be created lazily on access to `node.item`.
Attributes
----------
name: str
Name of the experiment
item: None or :class:`orion.core.worker.experiment.Experiment`
None if the experiment is not initialized yet. When initializing lazily, it creates an
`ExperimentView`.
.. seealso::
:py:class:`TreeNode` for tree-specific attributes and methods.
"""
__slots__ = ('name', 'version', '_no_parent_lookup',
'_no_children_lookup') + TreeNode.__slots__
def __init__(self,
name,
version,
experiment=None,
parent=None,
children=tuple()):
"""Initialize experiment node with item, experiment, parent and children
.. seealso::
:class:`orion.core.evc.tree.TreeNode` for information about the attributes
"""
super(ExperimentNode, self).__init__(experiment, parent, children)
self.name = name
self.version = version
self._no_parent_lookup = True
self._no_children_lookup = True
@property
def item(self):
"""Get the experiment associated to the node
Note that accessing `item` may trigger the lazy initialization of the experiment if it was
not done already.
"""
if self._item is None:
self._item = ExperimentView(self.name, version=self.version)
self._item.connect_to_version_control_tree(self)
return self._item
@property
def parent(self):
"""Get parent of the experiment, None if no parent
.. note::
The instantiation of an EVC tree is lazy, which means accessing the parent of a node
may trigger a call to database to build this parent live.
"""
if self._parent is None and self._no_parent_lookup:
self._no_parent_lookup = False
query = {'_id': self.item.refers['parent_id']}
selection = {'name': 1, 'version': 1}
experiments = get_storage().fetch_experiments(query, selection)
if experiments:
parent = experiments[0]
exp_node = ExperimentNode(name=parent['name'],
version=parent.get('version', 1))
self.set_parent(exp_node)
return self._parent
@property
def children(self):
"""Get children of the experiment, empty list if no children
.. note::
The instantiation of an EVC tree is lazy, which means accessing the children of a node
may trigger a call to database to build those children live.
"""
if self._no_children_lookup:
self._children = []
self._no_children_lookup = False
query = {'refers.parent_id': self.item.id}
selection = {'name': 1, 'version': 1}
experiments = get_storage().fetch_experiments(query, selection)
for child in experiments:
self.add_children(
ExperimentNode(name=child['name'],
version=child.get('version', 1)))
return self._children
@property
def adapter(self):
"""Get the adapter of the experiment with respect to its parent"""
return self.item.refers["adapter"]
@property
def tree_name(self):
"""Return a formatted name of the Node for a tree pretty-print."""
if self.item is not None:
return self.name + "-v{}".format(self.item.version)
return self.name
def fetch_lost_trials(self):
"""See :meth:`orion.core.evc.experiment:Experiment._fetch_trials`"""
return self._fetch_trials('fetch_lost_trials')
def fetch_trials(self):
"""See :meth:`orion.core.evc.experiment:Experiment._fetch_trials`"""
return self._fetch_trials('fetch_trials')
def fetch_pending_trials(self):
"""See :meth:`orion.core.evc.experiment:Experiment._fetch_trials`"""
return self._fetch_trials('fetch_pending_trials')
def fetch_noncompleted_trials(self):
"""See :meth:`orion.core.evc.experiment:Experiment._fetch_trials`"""
return self._fetch_trials('fetch_noncompleted_trials')
def fetch_trials_by_status(self, status):
"""See :meth:`orion.core.evc.experiment:Experiment._fetch_trials`"""
return self._fetch_trials('fetch_trials_by_status', status=status)
def _fetch_trials(self, fun_name, *args, **kwargs):
"""Fetch trials recursively in the EVC tree using the fetch function `fun_name`.
Parameters
----------
fun_name: callable
Function name to call to fetch trials. The function must be an attribute of
:class:`orion.core.worker.experiment:Experiment`
*args:
Positional arguments to pass to `fun_name`.
**kwargs
Keyword arguments to pass to `fun_name.
"""
def retrieve_trials(node, parent_or_children):
"""Retrieve the trials of a node/experiment."""
fun = getattr(node.item, fun_name)
# with_evc_tree needs to be False here or we will have an infinite loop
trials = fun(*args, with_evc_tree=False, **kwargs)
return dict(trials=trials,
experiment=node.item), parent_or_children
# get the trials of the parents
parent_trials = None
if self.parent is not None:
parent_trials = self.parent.map(retrieve_trials,
self.parent.parent)
# get the trials of the children
children_trials = self.map(retrieve_trials, self.children)
children_trials.set_parent(parent_trials)
adapt_trials(children_trials)
return sum([node.item['trials'] for node in children_trials.root], [])
class ExperimentNode(TreeNode):
"""Experiment node to connect experiments to EVC tree.
The node carries an experiment in attribute `item`. The node can be instantiated only using the
name of the experiment. The experiment will be created lazily on access to `node.item`.
Attributes
----------
name: str
Name of the experiment
item: None or :class:`orion.core.worker.experiment.Experiment`
None if the experiment is not initialized yet. When initializing lazily, it creates an
`ExperimentView`.
.. seealso::
:py:class:`TreeNode` for tree-specific attributes and methods.
"""
__slots__ = ('name', '_no_parent_lookup', '_no_children_lookup') + TreeNode.__slots__
def __init__(self, name, experiment=None, parent=None, children=tuple()):
"""Initialize experiment node with item, experiment, parent and children
.. seealso::
:class:`orion.core.evc.tree.TreeNode` for information about the attributes
"""
super(ExperimentNode, self).__init__(experiment, parent, children)
self.name = name
self._no_parent_lookup = True
self._no_children_lookup = True
@property
def item(self):
"""Get the experiment associated to the node
Note that accessing `item` may trigger the lazy initialization of the experiment if it was
not done already.
"""
if self._item is None:
self._item = ExperimentView(self.name)
self._item.connect_to_version_control_tree(self)
return self._item
@property
def parent(self):
"""Get parent of the experiment, None if no parent
.. note::
The instantiation of an EVC tree is lazy, which means accessing the parent of a node
may trigger a call to database to build this parent live.
"""
if self._parent is None and self._no_parent_lookup:
self._no_parent_lookup = False
query = {'_id': self.item.refers['parent_id']}
selection = {'name': 1}
experiments = Database().read('experiments', query, selection=selection)
if experiments:
self.set_parent(ExperimentNode(name=experiments[0]['name']))
return self._parent
@property
def children(self):
"""Get children of the experiment, empty list if no children
.. note::
The instantiation of an EVC tree is lazy, which means accessing the children of a node
may trigger a call to database to build those children live.
"""
if not self._children and self._no_children_lookup:
self._no_children_lookup = False
query = {'refers.parent_id': self.item.id}
selection = {'name': 1}
experiments = Database().read('experiments', query, selection=selection)
for child in experiments:
self.add_children(ExperimentNode(name=child['name']))
return self._children
@property
def adapter(self):
"""Get the adapter of the experiment with respect to its parent"""
return self.item.refers["adapter"]
def fetch_trials(self, query, selection=None):
"""Fetch trials recursively in the EVC tree
.. seealso::
:meth:`orion.core.worker.Experiment.fetch_trials` for more information about the
arguments.
"""
trials_tree = fetch_trials_tree(self, query, selection)
adapt_trials(trials_tree)
return sum([node.item['trials'] for node in trials_tree.root], [])
