Beispiel #1
0
    def ensure_atom(self, atom):
        """Ensure that there is an atomdetail in storage for the given atom.

        Returns uuid for the atomdetail that is/was created.
        """
        if isinstance(atom, task.BaseTask):
            return self._ensure_task(atom.name, misc.get_version_string(atom),
                                     atom.save_as)
        elif isinstance(atom, retry.Retry):
            return self._ensure_retry(atom.name, misc.get_version_string(atom),
                                      atom.save_as)
        else:
            raise TypeError("Object of type 'atom' expected not"
                            " '%s' (%s)" % (atom, type(atom)))
Beispiel #2
0
    def ensure_atom(self, atom):
        """Ensure that there is an atomdetail in storage for the given atom.

        Returns uuid for the atomdetail that is/was created.
        """
        if isinstance(atom, task.BaseTask):
            return self._ensure_task(atom.name,
                                     misc.get_version_string(atom),
                                     atom.save_as)
        elif isinstance(atom, retry.Retry):
            return self._ensure_retry(atom.name,
                                      misc.get_version_string(atom),
                                      atom.save_as)
        else:
            raise TypeError("Object of type 'atom' expected."
                            " Got %s, %r." % (type(atom), atom))
Beispiel #3
0
    def compile(self):
        """Compiles the contained flow into a structure which the engine can
        use to run or if this can not be done then an exception is thrown
        indicating why this compilation could not be achieved.
        """
        if self._root is not None:
            return

        assert self._graph_action is not None, ('Graph action class must be'
                                                ' specified')
        self._change_state(states.RESUMING)  # does nothing in PENDING state
        task_graph = flow_utils.flatten(self._flow)
        if task_graph.number_of_nodes() == 0:
            raise exc.EmptyFlow("Flow %s is empty." % self._flow.name)
        self._root = self._graph_action(task_graph)
        for task in task_graph.nodes_iter():
            try:
                task_id = self.storage.get_uuid_by_name(task.name)
            except exc.NotFound:
                task_id = uuidutils.generate_uuid()
                task_version = misc.get_version_string(task)
                self.storage.add_task(task_name=task.name, uuid=task_id,
                                      task_version=task_version)

            self.storage.set_result_mapping(task_id, task.save_as)
            self._root.add(task, task_action.TaskAction(task, task_id))
        self._change_state(states.SUSPENDED)  # does nothing in PENDING state
Beispiel #4
0
 def _ensure_storage_for(self, task_graph):
     # NOTE(harlowja): signal to the tasks that exist that we are about to
     # resume, if they have a previous state, they will now transition to
     # a resuming state (and then to suspended).
     self._change_state(states.RESUMING)  # does nothing in PENDING state
     for task in task_graph.nodes_iter():
         task_version = misc.get_version_string(task)
         self.storage.ensure_task(task.name, task_version, task.save_as)
     self._change_state(states.SUSPENDED)  # does nothing in PENDING state
 def _ensure_storage_for(self, task_graph):
     # NOTE(harlowja): signal to the tasks that exist that we are about to
     # resume, if they have a previous state, they will now transition to
     # a resuming state (and then to suspended).
     self._change_state(states.RESUMING)  # does nothing in PENDING state
     for task in task_graph.nodes_iter():
         task_version = misc.get_version_string(task)
         self.storage.ensure_task(task.name, task_version, task.save_as)
     self._change_state(states.SUSPENDED)  # does nothing in PENDING state
Beispiel #6
0
 def pformat(self, indent=0, linesep=os.linesep):
     """Pretty formats this atom detail into a string."""
     cls_name = self.__class__.__name__
     lines = ["%s%s: '%s'" % (" " * (indent), cls_name, self.name)]
     lines.extend(_format_shared(self, indent=indent + 1))
     lines.append("%s- version = %s" % (" " * (indent + 1), misc.get_version_string(self)))
     lines.append("%s- results = %s" % (" " * (indent + 1), self.results))
     lines.append("%s- failure = %s" % (" " * (indent + 1), bool(self.failure)))
     lines.extend(_format_meta(self.meta, indent=indent + 1))
     return linesep.join(lines)
Beispiel #7
0
    def ensure_atoms(self, atoms):
        """Ensure there is an atomdetail for **each** of the given atoms.

        Returns list of atomdetail uuids for each atom processed.
        """
        atom_ids = []
        missing_ads = []
        for i, atom in enumerate(atoms):
            match = misc.match_type(atom, self._ensure_matchers)
            if not match:
                raise TypeError("Unknown atom '%s' (%s) requested to ensure" %
                                (atom, type(atom)))
            atom_detail_cls, kind = match
            atom_name = atom.name
            if not atom_name:
                raise ValueError("%s name must be non-empty" % (kind))
            try:
                atom_id = self._atom_name_to_uuid[atom_name]
            except KeyError:
                missing_ads.append((i, atom, atom_detail_cls))
                # This will be later replaced with the uuid that is created...
                atom_ids.append(None)
            else:
                ad = self._flowdetail.find(atom_id)
                if not isinstance(ad, atom_detail_cls):
                    raise exceptions.Duplicate(
                        "Atom detail '%s' already exists in flow"
                        " detail '%s'" % (atom_name, self._flowdetail.name))
                else:
                    atom_ids.append(ad.uuid)
                    self._set_result_mapping(atom_name, atom.save_as)
        if missing_ads:
            needs_to_be_created_ads = []
            for (i, atom, atom_detail_cls) in missing_ads:
                ad = self._create_atom_detail(
                    atom.name,
                    atom_detail_cls,
                    atom_version=misc.get_version_string(atom))
                needs_to_be_created_ads.append((i, atom, ad))
            # Add the atom detail(s) to a clone, which upon success will be
            # updated into the contained flow detail; if it does not get saved
            # then no update will happen.
            source, clone = self._fetch_flowdetail(clone=True)
            for (_i, _atom, ad) in needs_to_be_created_ads:
                clone.add(ad)
            self._with_connection(self._save_flow_detail, source, clone)
            # Insert the needed data, and get outta here...
            for (i, atom, ad) in needs_to_be_created_ads:
                atom_name = atom.name
                atom_ids[i] = ad.uuid
                self._atom_name_to_uuid[atom_name] = ad.uuid
                self._set_result_mapping(atom_name, atom.save_as)
                self._failures.setdefault(atom_name, {})
        return atom_ids
def pformat_task_detail(task_detail, indent=0):
    """Pretty formats a task detail."""
    lines = ["%sTask: '%s'" % (" " * (indent), task_detail.name)]
    lines.extend(_format_shared(task_detail, indent=indent + 1))
    lines.append("%s- version = %s" %
                 (" " * (indent + 1), misc.get_version_string(task_detail)))
    lines.append("%s- results = %s" % (" " *
                                       (indent + 1), task_detail.results))
    lines.append("%s- failure = %s" %
                 (" " * (indent + 1), bool(task_detail.failure)))
    lines.extend(_format_meta(task_detail.meta, indent=indent + 1))
    return "\n".join(lines)
Beispiel #9
0
    def ensure_atoms(self, atoms):
        """Ensure there is an atomdetail for **each** of the given atoms.

        Returns list of atomdetail uuids for each atom processed.
        """
        atom_ids = []
        missing_ads = []
        for i, atom in enumerate(atoms):
            match = misc.match_type(atom, self._ensure_matchers)
            if not match:
                raise TypeError("Unknown atom '%s' (%s) requested to ensure"
                                % (atom, type(atom)))
            atom_detail_cls, kind = match
            atom_name = atom.name
            if not atom_name:
                raise ValueError("%s name must be non-empty" % (kind))
            try:
                atom_id = self._atom_name_to_uuid[atom_name]
            except KeyError:
                missing_ads.append((i, atom, atom_detail_cls))
                # This will be later replaced with the uuid that is created...
                atom_ids.append(None)
            else:
                ad = self._flowdetail.find(atom_id)
                if not isinstance(ad, atom_detail_cls):
                    raise exceptions.Duplicate(
                        "Atom detail '%s' already exists in flow"
                        " detail '%s'" % (atom_name, self._flowdetail.name))
                else:
                    atom_ids.append(ad.uuid)
                    self._set_result_mapping(atom_name, atom.save_as)
        if missing_ads:
            needs_to_be_created_ads = []
            for (i, atom, atom_detail_cls) in missing_ads:
                ad = self._create_atom_detail(
                    atom.name, atom_detail_cls,
                    atom_version=misc.get_version_string(atom))
                needs_to_be_created_ads.append((i, atom, ad))
            # Add the atom detail(s) to a clone, which upon success will be
            # updated into the contained flow detail; if it does not get saved
            # then no update will happen.
            source, clone = self._fetch_flowdetail(clone=True)
            for (_i, _atom, ad) in needs_to_be_created_ads:
                clone.add(ad)
            self._with_connection(self._save_flow_detail, source, clone)
            # Insert the needed data, and get outta here...
            for (i, atom, ad) in needs_to_be_created_ads:
                atom_name = atom.name
                atom_ids[i] = ad.uuid
                self._atom_name_to_uuid[atom_name] = ad.uuid
                self._set_result_mapping(atom_name, atom.save_as)
                self._failures.setdefault(atom_name, {})
        return atom_ids
Beispiel #10
0
def pformat_task_detail(task_detail, indent=0):
    """Pretty formats a task detail"""
    lines = ["%sTask: '%s'" % (" " * (indent), task_detail.name)]
    lines.extend(_format_shared(task_detail, indent=indent + 1))
    lines.append("%s- version = %s"
                 % (" " * (indent + 1), misc.get_version_string(task_detail)))
    lines.append("%s- results = %s"
                 % (" " * (indent + 1), task_detail.results))
    lines.append("%s- failure = %s" % (" " * (indent + 1),
                                       bool(task_detail.failure)))
    lines.extend(_format_meta(task_detail.meta, indent=indent + 1))
    return "\n".join(lines)
Beispiel #11
0
 def pformat(self, indent=0, linesep=os.linesep):
     """Pretty formats this atom detail into a string."""
     cls_name = self.__class__.__name__
     lines = ["%s%s: '%s'" % (" " * (indent), cls_name, self.name)]
     lines.extend(_format_shared(self, indent=indent + 1))
     lines.append("%s- version = %s" %
                  (" " * (indent + 1), misc.get_version_string(self)))
     lines.append("%s- results = %s" % (" " * (indent + 1), self.results))
     lines.append("%s- failure = %s" % (" " *
                                        (indent + 1), bool(self.failure)))
     lines.extend(_format_meta(self.meta, indent=indent + 1))
     return linesep.join(lines)
Beispiel #12
0
    def ensure_atom(self, atom):
        """Ensure that there is an atomdetail in storage for the given atom.

        Returns uuid for the atomdetail that is/was created.
        """
        functor = misc.match_type_handler(atom, self._ensure_matchers)
        if not functor:
            raise TypeError("Unknown item '%s' (%s) requested to ensure" %
                            (atom, type(atom)))
        else:
            return functor(atom.name, misc.get_version_string(atom),
                           atom.save_as)
def pformat_atom_detail(atom_detail, indent=0):
    """Pretty formats a atom detail."""
    detail_type = logbook.atom_detail_type(atom_detail)
    lines = ["%s%s: '%s'" % (" " * (indent), detail_type, atom_detail.name)]
    lines.extend(_format_shared(atom_detail, indent=indent + 1))
    lines.append("%s- version = %s" %
                 (" " * (indent + 1), misc.get_version_string(atom_detail)))
    lines.append("%s- results = %s" % (" " *
                                       (indent + 1), atom_detail.results))
    lines.append("%s- failure = %s" %
                 (" " * (indent + 1), bool(atom_detail.failure)))
    lines.extend(_format_meta(atom_detail.meta, indent=indent + 1))
    return "\n".join(lines)
Beispiel #14
0
    def ensure_atom(self, atom):
        """Ensure that there is an atomdetail in storage for the given atom.

        Returns uuid for the atomdetail that is/was created.
        """
        functor = misc.match_type_handler(atom, self._ensure_matchers)
        if not functor:
            raise TypeError("Unknown item '%s' (%s) requested to ensure"
                            % (atom, type(atom)))
        else:
            return functor(atom.name,
                           misc.get_version_string(atom),
                           atom.save_as)
def pformat_atom_detail(atom_detail, indent=0):
    """Pretty formats a atom detail."""
    detail_type = logbook.atom_detail_type(atom_detail)
    lines = ["%s%s: '%s'" % (" " * (indent), detail_type, atom_detail.name)]
    lines.extend(_format_shared(atom_detail, indent=indent + 1))
    lines.append("%s- version = %s"
                 % (" " * (indent + 1), misc.get_version_string(atom_detail)))
    lines.append("%s- results = %s"
                 % (" " * (indent + 1), atom_detail.results))
    lines.append("%s- failure = %s" % (" " * (indent + 1),
                                       bool(atom_detail.failure)))
    lines.extend(_format_meta(atom_detail.meta, indent=indent + 1))
    return "\n".join(lines)
Beispiel #16
0
 def _ensure_storage(self):
     # NOTE(harlowja): signal to the tasks that exist that we are about to
     # resume, if they have a previous state, they will now transition to
     # a resuming state (and then to suspended).
     self._change_state(states.RESUMING)  # does nothing in PENDING state
     for node in self._compilation.execution_graph.nodes_iter():
         version = misc.get_version_string(node)
         if isinstance(node, retry.Retry):
             self.storage.ensure_retry(node.name, version, node.save_as)
         else:
             self.storage.ensure_task(node.name, version, node.save_as)
         if node.inject:
             self.storage.inject_atom_args(node.name, node.inject)
     self._change_state(states.SUSPENDED)  # does nothing in PENDING state
Beispiel #17
0
    def ensure_atom(self, atom):
        """Ensure that there is an atomdetail in storage for the given atom.

        Returns uuid for the atomdetail that is/was created.
        """
        match = misc.match_type(atom, self._ensure_matchers)
        if not match:
            raise TypeError("Unknown atom '%s' (%s) requested to ensure"
                            % (atom, type(atom)))
        else:
            detail_cls, kind = match
            atom_id = self._ensure_atom_detail(kind, detail_cls, atom.name,
                                               misc.get_version_string(atom),
                                               atom.save_as)
            return atom_id
Beispiel #18
0
    def compile(self):
        """Compiles the contained flow into a structure which the engine can
        use to run or if this can not be done then an exception is thrown
        indicating why this compilation could not be achieved.
        """
        if self._root is not None:
            return

        assert self._graph_action is not None, ('Graph action class must be'
                                                ' specified')
        self._change_state(states.RESUMING)  # does nothing in PENDING state
        task_graph = flow_utils.flatten(self._flow)
        self._root = self._graph_action(task_graph)
        loaded_failures = {}

        for task in task_graph.nodes_iter():
            try:
                task_id = self.storage.get_uuid_by_name(task.name)
            except exc.NotFound:
                task_id = uuidutils.generate_uuid()
                task_version = misc.get_version_string(task)
                self.storage.add_task(task_name=task.name,
                                      uuid=task_id,
                                      task_version=task_version)
            try:
                result = self.storage.get(task_id)
            except exc.NotFound:
                result = None

            if isinstance(result, misc.Failure):
                # NOTE(imelnikov): old failure may have exc_info which
                # might get lost during serialization, so we preserve
                # old failure object if possible.
                old_failure = self._failures.get(task_id, None)
                if result.matches(old_failure):
                    loaded_failures[task_id] = old_failure
                else:
                    loaded_failures[task_id] = result

            self.storage.set_result_mapping(task_id, task.save_as)
            self._root.add(task, task_action.TaskAction(task, task_id))
        self._failures = loaded_failures
        self._change_state(states.SUSPENDED)  # does nothing in PENDING state
Beispiel #19
0
    def compile(self):
        """Compiles the contained flow into a structure which the engine can
        use to run or if this can not be done then an exception is thrown
        indicating why this compilation could not be achieved.
        """
        if self._root is not None:
            return

        assert self._graph_action is not None, ('Graph action class must be'
                                                ' specified')
        self._change_state(states.RESUMING)  # does nothing in PENDING state
        task_graph = flow_utils.flatten(self._flow)
        self._root = self._graph_action(task_graph)
        loaded_failures = {}

        for task in task_graph.nodes_iter():
            try:
                task_id = self.storage.get_uuid_by_name(task.name)
            except exc.NotFound:
                task_id = uuidutils.generate_uuid()
                task_version = misc.get_version_string(task)
                self.storage.add_task(task_name=task.name, uuid=task_id,
                                      task_version=task_version)
            try:
                result = self.storage.get(task_id)
            except exc.NotFound:
                result = None

            if isinstance(result, misc.Failure):
                # NOTE(imelnikov): old failure may have exc_info which
                # might get lost during serialization, so we preserve
                # old failure object if possible.
                old_failure = self._failures.get(task_id, None)
                if result.matches(old_failure):
                    loaded_failures[task_id] = old_failure
                else:
                    loaded_failures[task_id] = result

            self.storage.set_result_mapping(task_id, task.save_as)
            self._root.add(task, task_action.TaskAction(task, task_id))
        self._failures = loaded_failures
        self._change_state(states.SUSPENDED)  # does nothing in PENDING state
Beispiel #20
0
    def compile(self):
        """Compiles the contained flow into a structure which the engine can
        use to run or if this can not be done then an exception is thrown
        indicating why this compilation could not be achieved.
        """
        if self._root is not None:
            return

        assert self._graph_action_cls is not None, (
            'Graph action class must be specified')
        self._change_state(states.RESUMING)  # does nothing in PENDING state
        task_graph = flow_utils.flatten(self._flow)
        if task_graph.number_of_nodes() == 0:
            raise exc.EmptyFlow("Flow %s is empty." % self._flow.name)
        self._root = self._graph_action_cls(task_graph)
        for task in task_graph.nodes_iter():
            task_version = misc.get_version_string(task)
            self.storage.ensure_task(task.name, task_version, task.save_as)

        self._change_state(states.SUSPENDED)  # does nothing in PENDING state
Beispiel #21
0
 def __str__(self):
     return "%s==%s" % (self.name, misc.get_version_string(self))
Beispiel #22
0
 def __str__(self):
     return "%s==%s" % (self.name, misc.get_version_string(self))