def __init__(self, child_type): if not issubclass(child_type, Vertex): raise TypeError('ListVertex children must inherit from Protocol.') self.children = None # Ahead of super so the n_history call doesn't trigger the setter and find no children super(ListVertex, self).__init__() self.child_type = child_type self._initialized = False self.direct = InputDictionary() self.broadcast = InputDictionary() self._n_history = None self.n_history = 1
def __init__(self, **kwargs): try: # Super magic when the inheritance path is just vertices super(Vertex, self).__init__() except TypeError: # Super magic when the inheritance path includes GenericJob (i.e. for a Protocol) super(Vertex, self).__init__(**kwargs) self.input = InputDictionary() self.output = IODictionary() self.archive = IODictionary() self.archive.clock = 0 self.archive.output = IODictionary() self.archive.input = IODictionary() self.archive.whitelist = IODictionary() self.archive.whitelist.input = IODictionary() self.archive.whitelist.output = IODictionary() self._vertex_state = "next" self.possible_vertex_states = ["next"] self.vertex_name = None self.n_history = 1 self.on = True self.graph_parent = None
class Vertex(LoggerMixin, ABC): """ A parent class for objects which are valid vertices of a directed acyclic graph. Attributes: input (InputDictionary): A pointer-capable dictionary for inputs, including a sub-dictionary for defaults. (Default is a clean dictionary.) output (IODictionary): A pointer-capable dictionary for outputs. (Default is a clean dictionary.) archive (IODictionary): A pointer-capable dictionary for sampling the history of inputs and outputs. (Default is a clean dictionary.) vertex_state (str): Which edge to follow out of this vertex. (Default is "next".) possible_vertex_states (list[str]): Allowable exiting edge names. (Default is ["next"], one edge only!) vertex_name (str): Vertex name. (Default is None, gets set automatically when if the vertex is being added to a graph, or if the vertex is a Protocol being instantiated.) n_history (int): The length of each list stored in the output dictionary. (Default is 1, keep only the most recent output.) on (bool): Whether to execute the vertex when it is the active vertex of the graph, or simply skip over it. (default is True -- actually execute!) graph_parent (Vertex): The object who owns the graph that this vertex resides in. (Default is None.) Input attributes: default (IODictionary): A dictionary for fall-back values in case a key is requested that isn't in the main input dictionary. Archive attributes: whitelist (IODictionary): A nested dictionary of periods for archiving input and output values. Stores on executions where `clock % period = 0`. clock (int): The timer for whether whether or not input/output should be archived to hdf5. """ def __init__(self, **kwargs): try: # Super magic when the inheritance path is just vertices super(Vertex, self).__init__() except TypeError: # Super magic when the inheritance path includes GenericJob (i.e. for a Protocol) super(Vertex, self).__init__(**kwargs) self.input = InputDictionary() self.output = IODictionary() self.archive = IODictionary() self.archive.clock = 0 self.archive.output = IODictionary() self.archive.input = IODictionary() self.archive.whitelist = IODictionary() self.archive.whitelist.input = IODictionary() self.archive.whitelist.output = IODictionary() self._vertex_state = "next" self.possible_vertex_states = ["next"] self.vertex_name = None self.n_history = 1 self.on = True self.graph_parent = None def get_graph_location(self): return self._get_graph_location()[:-1] # Cut the trailing underscore def _get_graph_location(self, loc=""): new_loc = self.vertex_name + "_" + loc if self.graph_parent is None: return new_loc else: return self.graph_parent._get_graph_location(loc=new_loc) @property def vertex_state(self): return self._vertex_state @vertex_state.setter def vertex_state(self, new_state): if new_state not in self.possible_vertex_states: raise ValueError("New state not in list of possible states") self._vertex_state = new_state @abstractmethod def execute(self): """What to do when this vertex is the active vertex during graph traversal.""" pass @property def whitelist(self): return { 'input': self.archive.whitelist.input, 'output': self.archive.whitelist.output } @whitelist.setter def whitelist(self, value): self.set_whitelist(value) def set_whitelist(self, dictionary): """ Sets whitelist of the current vertex. Argument defines the form: ``` {'input': 5, 'output': 1} # sets all keys of input to dump at every fith execution cycle # sets all keys of output to dump at every execution cycle {'input': {'structure': None, 'forces': 5} } # disables the archiveing of input.structure but keeps forces ``` Args: dictionary (dict): The whitelist specification. """ for k, v in dictionary.items(): if k not in ('input', 'output'): raise ValueError if isinstance(v, int): self._set_archive_period(k, v) elif isinstance(v, dict): self._set_archive_whitelist(k, **v) else: raise TypeError def _set_archive_whitelist(self, archive, **kwargs): """ Whitelist properties of either "input" or "output" archive and set their dump period. Args: archive (str): either 'input' or 'output'. **kwargs: property names, values should be positive integers, specifies the dump freq, None = inf = < 0. """ for k, v in kwargs.items(): whitelist = getattr(self.archive.whitelist, archive) whitelist[k] = v def _set_archive_period(self, archive, n, keys=None): """ Sets the archive period for each property of to "n" if keys is not specified. If keys is a list of property names, "n" will be set a s archiving period only for those Args: archive (str): Either 'input' or 'output'. n (int): Dump at every `n` steps keys (list of str): The affected keys """ if keys is None: keys = list(getattr(self, archive).keys()) self._set_archive_whitelist(archive, **{k: n for k in keys}) def set_input_archive_period(self, n, keys=None): self._set_archive_period('input', n, keys=keys) def set_output_archive_period(self, n, keys=None): self._set_archive_period('output', n, keys=keys) def set_input_whitelist(self, **kwargs): self._set_archive_whitelist('input', **kwargs) def set_output_whitelist(self, **kwargs): self._set_archive_whitelist('output', **kwargs) def set_archive_period(self, n): self.set_input_archive_period(n) self.set_output_archive_period(n) def _update_archive(self): # Update input history_key = 't_%s' % self.archive.clock for key, value in self.input.items(): if key in self.archive.whitelist.input: # the keys there, but it could be explicitly set to < 0 or None period = self.archive.whitelist.input[key] if period is not None and period >= 0: # TODO: Notifaction when whitelist contains items which are not items of input # check if the period matches that of the key if self.archive.clock % period == 0: if key not in self.archive.input: self.archive.input[key] = TimelineDict() self.archive.input[key][history_key] = value else: # we want to archive it only if there is a change, thus get the last element last_val = ordered_dict_get_last( self.archive.input[key]) if not Comparer(last_val) == value: self.archive.input[key][history_key] = value self.logger.info( 'Property "{}" did change in input ({} -> {})' .format(key, last_val, value)) else: self.logger.info( 'Property "{}" did not change in input'. format(key)) # Update output for key, value in self.output.items(): if key in self.archive.whitelist.output: period = self.archive.whitelist.output[key] if period is not None and period >= 0: # TODO: Notifaction when whitelist contains items which are not items of input # check if the period matches that of the key if self.archive.clock % period == 0: val = value[-1] if key not in self.archive.output: self.archive.output[key] = TimelineDict() self.archive.output[key][history_key] = val else: # we want to archive it only if there is a change, thus get the last element last_val = ordered_dict_get_last( self.archive.output[key]) if not Comparer(last_val) == val: self.archive.output[key][history_key] = val else: self.logger.info( 'Property "{}" did not change in input'. format(key)) def _update_output(self, output_data): if output_data is None: return for key, value in output_data.items(): if key not in self.output: self.output[key] = [value] else: history = list(self.output[key]) # Roll the list if it is necessary history.append(value) if len(history) > self.n_history: # Remove the head of the queue history.pop(0) self.output[key] = history def update_and_archive(self, output_data): self._update_output(output_data) self._update_archive() def finish(self): pass def parallel_setup(self): """How to prepare to execute in parallel when there's a list of these vertices together.""" pass def to_hdf(self, hdf, group_name=None): """ Store the Vertex in an HDF5 file. Args: hdf (ProjectHDFio): HDF5 group object. group_name (str): HDF5 subgroup name. (Default is None.) """ if group_name is not None: hdf5_server = hdf.open(group_name) else: hdf5_server = hdf hdf5_server["TYPE"] = str(type(self)) hdf5_server["possiblevertexstates"] = self.possible_vertex_states hdf5_server["vertexstate"] = self.vertex_state hdf5_server["vertexname"] = self.vertex_name hdf5_server["nhistory"] = self.n_history self.input.to_hdf(hdf=hdf5_server, group_name="input") self.output.to_hdf(hdf=hdf5_server, group_name="output") self.archive.to_hdf(hdf=hdf5_server, group_name="archive") def from_hdf(self, hdf, group_name=None): """ Load the Vertex from an HDF5 file. Args: hdf (ProjectHDFio): HDF5 group object. group_name (str): HDF5 subgroup name. (Default is None.) """ if group_name is not None: hdf5_server = hdf.open(group_name) else: hdf5_server = hdf self.possible_vertex_states = hdf5_server["possiblevertexstates"] self._vertex_state = hdf5_server["vertexstate"] self.vertex_name = hdf5_server["vertexname"] self.n_history = hdf5_server["nhistory"] self.input.from_hdf(hdf=hdf5_server, group_name="input") self.output.from_hdf(hdf=hdf5_server, group_name="output") self.archive.from_hdf(hdf=hdf5_server, group_name="archive") # sort the dictionaries after loading, do it for both input and output dictionaries for archive_name in ('input', 'output'): archive = getattr(self.archive, archive_name) for key in archive.keys(): history = archive[key] # create an ordered dictionary from it, convert it to integer back again archive[key] = TimelineDict( sorted(history.items(), key=lambda item: int(item[0].replace('t_', ''))))
class ListVertex(PrimitiveVertex): """ A base class for making wrappers to run multiple instances of the same protocol. Protocols which inherit from this class require the sub-protocol being copied to be passed in at initialization as an argument. Attributes to be assigned to the Attributes: child_type (Command): A class inheriting from ``Command`` with which to create child instances. (Passed as an argument at instantiation.) children (list): The instances of the child command to execute. These are created automatically (at run-time if they don't exist already). broadcast (InputDictionary): Input data to be split element-wise across all the child commands. (Entries here should always have the same number of entries as there are children, i.e. each child can have its own value.) direct (InputDictionary): Input data which is to be copied to each child (i.e. all children have the same value.) Input attributes: n_children (int): How many children to create. """ def __init__(self, child_type): if not issubclass(child_type, Vertex): raise TypeError('ListVertex children must inherit from Protocol.') self.children = None # Ahead of super so the n_history call doesn't trigger the setter and find no children super(ListVertex, self).__init__() self.child_type = child_type self._initialized = False self.direct = InputDictionary() self.broadcast = InputDictionary() self._n_history = None self.n_history = 1 @abstractmethod def command(self, n_children): pass def finish(self): for child in self.children: child.finish() super(ListVertex, self).finish() def _initialize(self, n_children): children = [ self.child_type(name="child_{}".format(n)) for n in range(n_children) ] # Locate children in graph for n, child in enumerate(children): child.graph_parent = self child.vertex_name = "child_{}".format(n) # Link input to input.direct for key in list(self.direct.keys()): for child in children: setattr(child.input, key, getattr(Pointer(self.direct), key)) # Link input.default to input.direct.default for key in list(self.direct.default.keys()): for child in children: setattr(child.input.default, key, getattr(Pointer(self.direct.default), key)) # Link input to input.broadcast for key in list(self.broadcast.keys()): for n, child in enumerate(children): setattr(child.input, key, getattr(Pointer(self.broadcast), key)[n]) # Link input.default to input.broadcast.default for key in list(self.broadcast.default.keys()): for n, child in enumerate(children): setattr(child.input.default, key, getattr(Pointer(self.broadcast.default), key)[n]) self.children = children self._initialized = True @property def n_history(self): return self._n_history @n_history.setter def n_history(self, n_hist): self._n_history = n_hist if self.children is not None: for child in self.children: child.n_history = n_hist def _extract_output_data_from_children(self): output_keys = list(self.children[0].output.keys() ) # Assumes that all the children are the same... if len(output_keys) > 0: output_data = {} for key in output_keys: values = [] for child in self.children: values.append(child.output[key][-1]) output_data[key] = values else: output_data = None return output_data def to_hdf(self, hdf=None, group_name=None): super(ListVertex, self).to_hdf(hdf=hdf, group_name=group_name) hdf[group_name]['initialized'] = self._initialized if self.children is not None: with hdf.open(group_name + "/children") as hdf5_server: for n, child in enumerate(self.children): child.to_hdf(hdf=hdf5_server, group_name="child" + str(n)) def from_hdf(self, hdf=None, group_name=None): super(ListVertex, self).from_hdf(hdf=hdf, group_name=group_name) self._initialized = hdf[group_name]['initialized'] if self._initialized: with hdf.open(group_name + "/children") as hdf5_server: children = [] for n in np.arange(self.input.n_children, dtype=int): child = self.child_type(name="child_{}".format(n)) child.from_hdf(hdf=hdf5_server, group_name="child" + str(n)) child.graph_parent = self children.append(child) self.children = children