def __init__(self, child_type):
if not issubclass(child_type, Vertex):
raise TypeError('ListVertex children must inherit from Protocol.')
self.children = None # Ahead of super so the n_history call doesn't trigger the setter and find no children
super(ListVertex, self).__init__()
self.child_type = child_type
self._initialized = False
self.direct = InputDictionary()
self.broadcast = InputDictionary()
self._n_history = None
self.n_history = 1
def __init__(self, **kwargs):
try: # Super magic when the inheritance path is just vertices
super(Vertex, self).__init__()
except TypeError: # Super magic when the inheritance path includes GenericJob (i.e. for a Protocol)
super(Vertex, self).__init__(**kwargs)
self.input = InputDictionary()
self.output = IODictionary()
self.archive = IODictionary()
self.archive.clock = 0
self.archive.output = IODictionary()
self.archive.input = IODictionary()
self.archive.whitelist = IODictionary()
self.archive.whitelist.input = IODictionary()
self.archive.whitelist.output = IODictionary()
self._vertex_state = "next"
self.possible_vertex_states = ["next"]
self.vertex_name = None
self.n_history = 1
self.on = True
self.graph_parent = None
class Vertex(LoggerMixin, ABC):
"""
A parent class for objects which are valid vertices of a directed acyclic graph.
Attributes:
input (InputDictionary): A pointer-capable dictionary for inputs, including a sub-dictionary for defaults.
(Default is a clean dictionary.)
output (IODictionary): A pointer-capable dictionary for outputs. (Default is a clean dictionary.)
archive (IODictionary): A pointer-capable dictionary for sampling the history of inputs and outputs. (Default
is a clean dictionary.)
vertex_state (str): Which edge to follow out of this vertex. (Default is "next".)
possible_vertex_states (list[str]): Allowable exiting edge names. (Default is ["next"], one edge only!)
vertex_name (str): Vertex name. (Default is None, gets set automatically when if the vertex is being added to
a graph, or if the vertex is a Protocol being instantiated.)
n_history (int): The length of each list stored in the output dictionary. (Default is 1, keep only the most
recent output.)
on (bool): Whether to execute the vertex when it is the active vertex of the graph, or simply skip over it.
(default is True -- actually execute!)
graph_parent (Vertex): The object who owns the graph that this vertex resides in. (Default is None.)
Input attributes:
default (IODictionary): A dictionary for fall-back values in case a key is requested that isn't in the main
input dictionary.
Archive attributes:
whitelist (IODictionary): A nested dictionary of periods for archiving input and output values. Stores on
executions where `clock % period = 0`.
clock (int): The timer for whether whether or not input/output should be archived to hdf5.
"""
def __init__(self, **kwargs):
try: # Super magic when the inheritance path is just vertices
super(Vertex, self).__init__()
except TypeError: # Super magic when the inheritance path includes GenericJob (i.e. for a Protocol)
super(Vertex, self).__init__(**kwargs)
self.input = InputDictionary()
self.output = IODictionary()
self.archive = IODictionary()
self.archive.clock = 0
self.archive.output = IODictionary()
self.archive.input = IODictionary()
self.archive.whitelist = IODictionary()
self.archive.whitelist.input = IODictionary()
self.archive.whitelist.output = IODictionary()
self._vertex_state = "next"
self.possible_vertex_states = ["next"]
self.vertex_name = None
self.n_history = 1
self.on = True
self.graph_parent = None
def get_graph_location(self):
return self._get_graph_location()[:-1] # Cut the trailing underscore
def _get_graph_location(self, loc=""):
new_loc = self.vertex_name + "_" + loc
if self.graph_parent is None:
return new_loc
else:
return self.graph_parent._get_graph_location(loc=new_loc)
@property
def vertex_state(self):
return self._vertex_state
@vertex_state.setter
def vertex_state(self, new_state):
if new_state not in self.possible_vertex_states:
raise ValueError("New state not in list of possible states")
self._vertex_state = new_state
@abstractmethod
def execute(self):
"""What to do when this vertex is the active vertex during graph traversal."""
pass
@property
def whitelist(self):
return {
'input': self.archive.whitelist.input,
'output': self.archive.whitelist.output
}
@whitelist.setter
def whitelist(self, value):
self.set_whitelist(value)
def set_whitelist(self, dictionary):
"""
Sets whitelist of the current vertex. Argument defines the form:
```
{'input': 5,
'output': 1}
# sets all keys of input to dump at every fith execution cycle
# sets all keys of output to dump at every execution cycle
{'input': {'structure': None,
'forces': 5}
}
# disables the archiveing of input.structure but keeps forces
```
Args:
dictionary (dict): The whitelist specification.
"""
for k, v in dictionary.items():
if k not in ('input', 'output'):
raise ValueError
if isinstance(v, int):
self._set_archive_period(k, v)
elif isinstance(v, dict):
self._set_archive_whitelist(k, **v)
else:
raise TypeError
def _set_archive_whitelist(self, archive, **kwargs):
"""
Whitelist properties of either "input" or "output" archive and set their dump period.
Args:
archive (str): either 'input' or 'output'.
**kwargs: property names, values should be positive integers, specifies the dump freq, None = inf = < 0.
"""
for k, v in kwargs.items():
whitelist = getattr(self.archive.whitelist, archive)
whitelist[k] = v
def _set_archive_period(self, archive, n, keys=None):
"""
Sets the archive period for each property of to "n" if keys is not specified.
If keys is a list of property names, "n" will be set a s archiving period only for those
Args:
archive (str): Either 'input' or 'output'.
n (int): Dump at every `n` steps
keys (list of str): The affected keys
"""
if keys is None:
keys = list(getattr(self, archive).keys())
self._set_archive_whitelist(archive, **{k: n for k in keys})
def set_input_archive_period(self, n, keys=None):
self._set_archive_period('input', n, keys=keys)
def set_output_archive_period(self, n, keys=None):
self._set_archive_period('output', n, keys=keys)
def set_input_whitelist(self, **kwargs):
self._set_archive_whitelist('input', **kwargs)
def set_output_whitelist(self, **kwargs):
self._set_archive_whitelist('output', **kwargs)
def set_archive_period(self, n):
self.set_input_archive_period(n)
self.set_output_archive_period(n)
def _update_archive(self):
# Update input
history_key = 't_%s' % self.archive.clock
for key, value in self.input.items():
if key in self.archive.whitelist.input:
# the keys there, but it could be explicitly set to < 0 or None
period = self.archive.whitelist.input[key]
if period is not None and period >= 0:
# TODO: Notifaction when whitelist contains items which are not items of input
# check if the period matches that of the key
if self.archive.clock % period == 0:
if key not in self.archive.input:
self.archive.input[key] = TimelineDict()
self.archive.input[key][history_key] = value
else:
# we want to archive it only if there is a change, thus get the last element
last_val = ordered_dict_get_last(
self.archive.input[key])
if not Comparer(last_val) == value:
self.archive.input[key][history_key] = value
self.logger.info(
'Property "{}" did change in input ({} -> {})'
.format(key, last_val, value))
else:
self.logger.info(
'Property "{}" did not change in input'.
format(key))
# Update output
for key, value in self.output.items():
if key in self.archive.whitelist.output:
period = self.archive.whitelist.output[key]
if period is not None and period >= 0:
# TODO: Notifaction when whitelist contains items which are not items of input
# check if the period matches that of the key
if self.archive.clock % period == 0:
val = value[-1]
if key not in self.archive.output:
self.archive.output[key] = TimelineDict()
self.archive.output[key][history_key] = val
else:
# we want to archive it only if there is a change, thus get the last element
last_val = ordered_dict_get_last(
self.archive.output[key])
if not Comparer(last_val) == val:
self.archive.output[key][history_key] = val
else:
self.logger.info(
'Property "{}" did not change in input'.
format(key))
def _update_output(self, output_data):
if output_data is None:
return
for key, value in output_data.items():
if key not in self.output:
self.output[key] = [value]
else:
history = list(self.output[key])
# Roll the list if it is necessary
history.append(value)
if len(history) > self.n_history:
# Remove the head of the queue
history.pop(0)
self.output[key] = history
def update_and_archive(self, output_data):
self._update_output(output_data)
self._update_archive()
def finish(self):
pass
def parallel_setup(self):
"""How to prepare to execute in parallel when there's a list of these vertices together."""
pass
def to_hdf(self, hdf, group_name=None):
"""
Store the Vertex in an HDF5 file.
Args:
hdf (ProjectHDFio): HDF5 group object.
group_name (str): HDF5 subgroup name. (Default is None.)
"""
if group_name is not None:
hdf5_server = hdf.open(group_name)
else:
hdf5_server = hdf
hdf5_server["TYPE"] = str(type(self))
hdf5_server["possiblevertexstates"] = self.possible_vertex_states
hdf5_server["vertexstate"] = self.vertex_state
hdf5_server["vertexname"] = self.vertex_name
hdf5_server["nhistory"] = self.n_history
self.input.to_hdf(hdf=hdf5_server, group_name="input")
self.output.to_hdf(hdf=hdf5_server, group_name="output")
self.archive.to_hdf(hdf=hdf5_server, group_name="archive")
def from_hdf(self, hdf, group_name=None):
"""
Load the Vertex from an HDF5 file.
Args:
hdf (ProjectHDFio): HDF5 group object.
group_name (str): HDF5 subgroup name. (Default is None.)
"""
if group_name is not None:
hdf5_server = hdf.open(group_name)
else:
hdf5_server = hdf
self.possible_vertex_states = hdf5_server["possiblevertexstates"]
self._vertex_state = hdf5_server["vertexstate"]
self.vertex_name = hdf5_server["vertexname"]
self.n_history = hdf5_server["nhistory"]
self.input.from_hdf(hdf=hdf5_server, group_name="input")
self.output.from_hdf(hdf=hdf5_server, group_name="output")
self.archive.from_hdf(hdf=hdf5_server, group_name="archive")
# sort the dictionaries after loading, do it for both input and output dictionaries
for archive_name in ('input', 'output'):
archive = getattr(self.archive, archive_name)
for key in archive.keys():
history = archive[key]
# create an ordered dictionary from it, convert it to integer back again
archive[key] = TimelineDict(
sorted(history.items(),
key=lambda item: int(item[0].replace('t_', ''))))
class ListVertex(PrimitiveVertex):
"""
A base class for making wrappers to run multiple instances of the same protocol. Protocols which inherit from this
class require the sub-protocol being copied to be passed in at initialization as an argument.
Attributes to be assigned to the
Attributes:
child_type (Command): A class inheriting from ``Command`` with which to create child instances. (Passed as an
argument at instantiation.)
children (list): The instances of the child command to execute. These are created automatically (at run-time if
they don't exist already).
broadcast (InputDictionary): Input data to be split element-wise across all the child commands. (Entries here
should always have the same number of entries as there are children, i.e. each child can have its own
value.)
direct (InputDictionary): Input data which is to be copied to each child (i.e. all children have the same
value.)
Input attributes:
n_children (int): How many children to create.
"""
def __init__(self, child_type):
if not issubclass(child_type, Vertex):
raise TypeError('ListVertex children must inherit from Protocol.')
self.children = None # Ahead of super so the n_history call doesn't trigger the setter and find no children
super(ListVertex, self).__init__()
self.child_type = child_type
self._initialized = False
self.direct = InputDictionary()
self.broadcast = InputDictionary()
self._n_history = None
self.n_history = 1
@abstractmethod
def command(self, n_children):
pass
def finish(self):
for child in self.children:
child.finish()
super(ListVertex, self).finish()
def _initialize(self, n_children):
children = [
self.child_type(name="child_{}".format(n))
for n in range(n_children)
]
# Locate children in graph
for n, child in enumerate(children):
child.graph_parent = self
child.vertex_name = "child_{}".format(n)
# Link input to input.direct
for key in list(self.direct.keys()):
for child in children:
setattr(child.input, key, getattr(Pointer(self.direct), key))
# Link input.default to input.direct.default
for key in list(self.direct.default.keys()):
for child in children:
setattr(child.input.default, key,
getattr(Pointer(self.direct.default), key))
# Link input to input.broadcast
for key in list(self.broadcast.keys()):
for n, child in enumerate(children):
setattr(child.input, key,
getattr(Pointer(self.broadcast), key)[n])
# Link input.default to input.broadcast.default
for key in list(self.broadcast.default.keys()):
for n, child in enumerate(children):
setattr(child.input.default, key,
getattr(Pointer(self.broadcast.default), key)[n])
self.children = children
self._initialized = True
@property
def n_history(self):
return self._n_history
@n_history.setter
def n_history(self, n_hist):
self._n_history = n_hist
if self.children is not None:
for child in self.children:
child.n_history = n_hist
def _extract_output_data_from_children(self):
output_keys = list(self.children[0].output.keys()
) # Assumes that all the children are the same...
if len(output_keys) > 0:
output_data = {}
for key in output_keys:
values = []
for child in self.children:
values.append(child.output[key][-1])
output_data[key] = values
else:
output_data = None
return output_data
def to_hdf(self, hdf=None, group_name=None):
super(ListVertex, self).to_hdf(hdf=hdf, group_name=group_name)
hdf[group_name]['initialized'] = self._initialized
if self.children is not None:
with hdf.open(group_name + "/children") as hdf5_server:
for n, child in enumerate(self.children):
child.to_hdf(hdf=hdf5_server, group_name="child" + str(n))
def from_hdf(self, hdf=None, group_name=None):
super(ListVertex, self).from_hdf(hdf=hdf, group_name=group_name)
self._initialized = hdf[group_name]['initialized']
if self._initialized:
with hdf.open(group_name + "/children") as hdf5_server:
children = []
for n in np.arange(self.input.n_children, dtype=int):
child = self.child_type(name="child_{}".format(n))
child.from_hdf(hdf=hdf5_server,
group_name="child" + str(n))
child.graph_parent = self
children.append(child)
self.children = children