def ensure_atom(self, atom): """Ensure that there is an atomdetail in storage for the given atom. Returns uuid for the atomdetail that is/was created. """ if isinstance(atom, task.BaseTask): return self._ensure_task(atom.name, misc.get_version_string(atom), atom.save_as) elif isinstance(atom, retry.Retry): return self._ensure_retry(atom.name, misc.get_version_string(atom), atom.save_as) else: raise TypeError("Object of type 'atom' expected not" " '%s' (%s)" % (atom, type(atom)))
def ensure_atom(self, atom): """Ensure that there is an atomdetail in storage for the given atom. Returns uuid for the atomdetail that is/was created. """ if isinstance(atom, task.BaseTask): return self._ensure_task(atom.name, misc.get_version_string(atom), atom.save_as) elif isinstance(atom, retry.Retry): return self._ensure_retry(atom.name, misc.get_version_string(atom), atom.save_as) else: raise TypeError("Object of type 'atom' expected." " Got %s, %r." % (type(atom), atom))
def compile(self): """Compiles the contained flow into a structure which the engine can use to run or if this can not be done then an exception is thrown indicating why this compilation could not be achieved. """ if self._root is not None: return assert self._graph_action is not None, ('Graph action class must be' ' specified') self._change_state(states.RESUMING) # does nothing in PENDING state task_graph = flow_utils.flatten(self._flow) if task_graph.number_of_nodes() == 0: raise exc.EmptyFlow("Flow %s is empty." % self._flow.name) self._root = self._graph_action(task_graph) for task in task_graph.nodes_iter(): try: task_id = self.storage.get_uuid_by_name(task.name) except exc.NotFound: task_id = uuidutils.generate_uuid() task_version = misc.get_version_string(task) self.storage.add_task(task_name=task.name, uuid=task_id, task_version=task_version) self.storage.set_result_mapping(task_id, task.save_as) self._root.add(task, task_action.TaskAction(task, task_id)) self._change_state(states.SUSPENDED) # does nothing in PENDING state
def _ensure_storage_for(self, task_graph): # NOTE(harlowja): signal to the tasks that exist that we are about to # resume, if they have a previous state, they will now transition to # a resuming state (and then to suspended). self._change_state(states.RESUMING) # does nothing in PENDING state for task in task_graph.nodes_iter(): task_version = misc.get_version_string(task) self.storage.ensure_task(task.name, task_version, task.save_as) self._change_state(states.SUSPENDED) # does nothing in PENDING state
def pformat(self, indent=0, linesep=os.linesep): """Pretty formats this atom detail into a string.""" cls_name = self.__class__.__name__ lines = ["%s%s: '%s'" % (" " * (indent), cls_name, self.name)] lines.extend(_format_shared(self, indent=indent + 1)) lines.append("%s- version = %s" % (" " * (indent + 1), misc.get_version_string(self))) lines.append("%s- results = %s" % (" " * (indent + 1), self.results)) lines.append("%s- failure = %s" % (" " * (indent + 1), bool(self.failure))) lines.extend(_format_meta(self.meta, indent=indent + 1)) return linesep.join(lines)
def ensure_atoms(self, atoms): """Ensure there is an atomdetail for **each** of the given atoms. Returns list of atomdetail uuids for each atom processed. """ atom_ids = [] missing_ads = [] for i, atom in enumerate(atoms): match = misc.match_type(atom, self._ensure_matchers) if not match: raise TypeError("Unknown atom '%s' (%s) requested to ensure" % (atom, type(atom))) atom_detail_cls, kind = match atom_name = atom.name if not atom_name: raise ValueError("%s name must be non-empty" % (kind)) try: atom_id = self._atom_name_to_uuid[atom_name] except KeyError: missing_ads.append((i, atom, atom_detail_cls)) # This will be later replaced with the uuid that is created... atom_ids.append(None) else: ad = self._flowdetail.find(atom_id) if not isinstance(ad, atom_detail_cls): raise exceptions.Duplicate( "Atom detail '%s' already exists in flow" " detail '%s'" % (atom_name, self._flowdetail.name)) else: atom_ids.append(ad.uuid) self._set_result_mapping(atom_name, atom.save_as) if missing_ads: needs_to_be_created_ads = [] for (i, atom, atom_detail_cls) in missing_ads: ad = self._create_atom_detail( atom.name, atom_detail_cls, atom_version=misc.get_version_string(atom)) needs_to_be_created_ads.append((i, atom, ad)) # Add the atom detail(s) to a clone, which upon success will be # updated into the contained flow detail; if it does not get saved # then no update will happen. source, clone = self._fetch_flowdetail(clone=True) for (_i, _atom, ad) in needs_to_be_created_ads: clone.add(ad) self._with_connection(self._save_flow_detail, source, clone) # Insert the needed data, and get outta here... for (i, atom, ad) in needs_to_be_created_ads: atom_name = atom.name atom_ids[i] = ad.uuid self._atom_name_to_uuid[atom_name] = ad.uuid self._set_result_mapping(atom_name, atom.save_as) self._failures.setdefault(atom_name, {}) return atom_ids
def pformat_task_detail(task_detail, indent=0): """Pretty formats a task detail.""" lines = ["%sTask: '%s'" % (" " * (indent), task_detail.name)] lines.extend(_format_shared(task_detail, indent=indent + 1)) lines.append("%s- version = %s" % (" " * (indent + 1), misc.get_version_string(task_detail))) lines.append("%s- results = %s" % (" " * (indent + 1), task_detail.results)) lines.append("%s- failure = %s" % (" " * (indent + 1), bool(task_detail.failure))) lines.extend(_format_meta(task_detail.meta, indent=indent + 1)) return "\n".join(lines)
def pformat_task_detail(task_detail, indent=0): """Pretty formats a task detail""" lines = ["%sTask: '%s'" % (" " * (indent), task_detail.name)] lines.extend(_format_shared(task_detail, indent=indent + 1)) lines.append("%s- version = %s" % (" " * (indent + 1), misc.get_version_string(task_detail))) lines.append("%s- results = %s" % (" " * (indent + 1), task_detail.results)) lines.append("%s- failure = %s" % (" " * (indent + 1), bool(task_detail.failure))) lines.extend(_format_meta(task_detail.meta, indent=indent + 1)) return "\n".join(lines)
def ensure_atom(self, atom): """Ensure that there is an atomdetail in storage for the given atom. Returns uuid for the atomdetail that is/was created. """ functor = misc.match_type_handler(atom, self._ensure_matchers) if not functor: raise TypeError("Unknown item '%s' (%s) requested to ensure" % (atom, type(atom))) else: return functor(atom.name, misc.get_version_string(atom), atom.save_as)
def pformat_atom_detail(atom_detail, indent=0): """Pretty formats a atom detail.""" detail_type = logbook.atom_detail_type(atom_detail) lines = ["%s%s: '%s'" % (" " * (indent), detail_type, atom_detail.name)] lines.extend(_format_shared(atom_detail, indent=indent + 1)) lines.append("%s- version = %s" % (" " * (indent + 1), misc.get_version_string(atom_detail))) lines.append("%s- results = %s" % (" " * (indent + 1), atom_detail.results)) lines.append("%s- failure = %s" % (" " * (indent + 1), bool(atom_detail.failure))) lines.extend(_format_meta(atom_detail.meta, indent=indent + 1)) return "\n".join(lines)
def _ensure_storage(self): # NOTE(harlowja): signal to the tasks that exist that we are about to # resume, if they have a previous state, they will now transition to # a resuming state (and then to suspended). self._change_state(states.RESUMING) # does nothing in PENDING state for node in self._compilation.execution_graph.nodes_iter(): version = misc.get_version_string(node) if isinstance(node, retry.Retry): self.storage.ensure_retry(node.name, version, node.save_as) else: self.storage.ensure_task(node.name, version, node.save_as) if node.inject: self.storage.inject_atom_args(node.name, node.inject) self._change_state(states.SUSPENDED) # does nothing in PENDING state
def ensure_atom(self, atom): """Ensure that there is an atomdetail in storage for the given atom. Returns uuid for the atomdetail that is/was created. """ match = misc.match_type(atom, self._ensure_matchers) if not match: raise TypeError("Unknown atom '%s' (%s) requested to ensure" % (atom, type(atom))) else: detail_cls, kind = match atom_id = self._ensure_atom_detail(kind, detail_cls, atom.name, misc.get_version_string(atom), atom.save_as) return atom_id
def compile(self): """Compiles the contained flow into a structure which the engine can use to run or if this can not be done then an exception is thrown indicating why this compilation could not be achieved. """ if self._root is not None: return assert self._graph_action is not None, ('Graph action class must be' ' specified') self._change_state(states.RESUMING) # does nothing in PENDING state task_graph = flow_utils.flatten(self._flow) self._root = self._graph_action(task_graph) loaded_failures = {} for task in task_graph.nodes_iter(): try: task_id = self.storage.get_uuid_by_name(task.name) except exc.NotFound: task_id = uuidutils.generate_uuid() task_version = misc.get_version_string(task) self.storage.add_task(task_name=task.name, uuid=task_id, task_version=task_version) try: result = self.storage.get(task_id) except exc.NotFound: result = None if isinstance(result, misc.Failure): # NOTE(imelnikov): old failure may have exc_info which # might get lost during serialization, so we preserve # old failure object if possible. old_failure = self._failures.get(task_id, None) if result.matches(old_failure): loaded_failures[task_id] = old_failure else: loaded_failures[task_id] = result self.storage.set_result_mapping(task_id, task.save_as) self._root.add(task, task_action.TaskAction(task, task_id)) self._failures = loaded_failures self._change_state(states.SUSPENDED) # does nothing in PENDING state
def compile(self): """Compiles the contained flow into a structure which the engine can use to run or if this can not be done then an exception is thrown indicating why this compilation could not be achieved. """ if self._root is not None: return assert self._graph_action_cls is not None, ( 'Graph action class must be specified') self._change_state(states.RESUMING) # does nothing in PENDING state task_graph = flow_utils.flatten(self._flow) if task_graph.number_of_nodes() == 0: raise exc.EmptyFlow("Flow %s is empty." % self._flow.name) self._root = self._graph_action_cls(task_graph) for task in task_graph.nodes_iter(): task_version = misc.get_version_string(task) self.storage.ensure_task(task.name, task_version, task.save_as) self._change_state(states.SUSPENDED) # does nothing in PENDING state
def __str__(self): return "%s==%s" % (self.name, misc.get_version_string(self))