示例#1
0
    def test_add_node(self):
        g = DAG()
        g.add_node(1)
        self.assertListEqual(g.nodes, [1])

        class Foo():
            pass

        bar = Foo()
        g.add_node(bar)

        self.assertListEqual(g.nodes, [1, bar])

        g.add_node(bar)

        self.assertListEqual(g.nodes, [1, bar])
示例#2
0
def orchestrations_full():
    json_data = request.get_json()
    json_steps = json_data.pop('steps')
    generated_version = False
    if 'version' not in json_data:
        generated_version = True
        json_data['version'] = Orchestration.query.filter_by(
            name=json_data['name']).count() + 1
    o = Orchestration(**json_data)
    db.session.add(o)
    resp_data = {'id': o.id}
    if generated_version:
        resp_data.update(version=o.version)

    # reorder steps in order of dependency
    id2step = {s['id']: s for s in json_steps}

    dag = DAG()
    for s in json_steps:
        step_id = s['id']
        if s['undo'] and len(s.get('parent_step_ids', [])) == 0:
            raise errors.UndoStepWithoutParent(step_id)
        dag.add_node(step_id)
        for p_s_id in s.get('parent_step_ids', []):
            dag.add_edge(p_s_id, step_id)

    if dag.is_cyclic():
        raise errors.CycleError

    new_steps = []
    for step_id in dag.ordered_nodes:
        step = id2step[step_id]
        new_steps.append(step)
    # end reorder steps in order of dependency

    rid2step = OrderedDict()
    dependencies = {}
    for json_step in new_steps:
        rid = json_step.pop('id', None)
        json_step['rid'] = rid
        if rid is not None and rid in rid2step.keys():
            raise errors.DuplicatedId(rid)
        if 'action_template_id' in json_step:
            json_step['action_template'] = ActionTemplate.query.get_or_raise(
                json_step.pop('action_template_id'))
        elif 'action_type' in json_step:
            json_step['action_type'] = ActionType[json_step.pop('action_type')]
        dependencies[rid] = {
            'parent_step_ids':
            [p_id for p_id in json_step.pop('parent_step_ids', [])]
        }
        s = o.add_step(**json_step)
        db.session.add(s)
        if rid:
            rid2step[rid] = s

        continue

    # process dependencies
    for rid, dep in dependencies.items():
        step = rid2step[rid]
        parents = []
        for p_s_id in dep['parent_step_ids']:
            if p_s_id in rid2step:
                parents.append(rid2step[p_s_id])
        o.set_parents(step, parents)

    db.session.commit()

    # send new ids in order of appearance at beginning
    new_id_steps = []
    for rid in rid2step.keys():
        new_id_steps.append(rid2step[rid].id)
    resp_data.update({'step_ids': new_id_steps})
    return resp_data, 201
示例#3
0
class Orchestration(UUIDistributedEntityMixin, db.Model):
    __tablename__ = 'D_orchestration'
    order = 20

    name = db.Column(db.String(80), nullable=False)
    version = db.Column(db.Integer, nullable=False)
    description = db.Column(db.Text)
    stop_on_error = db.Column(db.Boolean)
    stop_undo_on_error = db.Column(db.Boolean)
    undo_on_error = db.Column(db.Boolean)
    created_at = db.Column(UtcDateTime(timezone=True), default=get_now)

    steps = db.relationship(
        "Step",
        primaryjoin="Step.orchestration_id==Orchestration.id",
        back_populates="orchestration")

    __table_args__ = (db.UniqueConstraint('name',
                                          'version',
                                          name='D_orchestration_uq01'), )

    def __init__(self,
                 name: str,
                 version: int,
                 description: t.Optional[str] = None,
                 steps: t.List[Step] = None,
                 stop_on_error: bool = True,
                 stop_undo_on_error: bool = True,
                 undo_on_error: bool = True,
                 dependencies: Tdependencies = None,
                 created_at=None,
                 **kwargs):
        super().__init__(**kwargs)

        self.name = name
        self.version = version
        self.description = '\n'.join(description) if is_iterable_not_string(
            description) else description
        self.steps = steps or []
        assert isinstance(stop_on_error, bool)
        self.stop_on_error = stop_on_error
        assert isinstance(stop_undo_on_error, bool)
        self.stop_undo_on_error = stop_undo_on_error
        assert isinstance(undo_on_error, bool)
        self.undo_on_error = undo_on_error
        self.created_at = created_at or get_now()

        if dependencies:
            self.set_dependencies(dependencies)
        else:
            self._graph = DAG()

    @orm.reconstructor
    def init_on_load(self):
        self._graph = DAG()
        for step in self.steps:
            if step.parents:
                for p in step.parents:
                    self._graph.add_edge(p, step)
            else:
                self._graph.add_node(step)

    def set_dependencies(self, dependencies: Tdependencies):
        edges = []
        find = lambda id_: next(
            (step for step in self.steps if step.id == id_))
        if isinstance(dependencies, t.Dict):
            for k, v in dependencies.items():
                try:
                    step_from = find(k)
                except StopIteration:
                    raise ValueError(f'id step {k} not found in steps list')
                for child_id in v:
                    try:
                        step_to = find(child_id)
                    except StopIteration:
                        raise ValueError(
                            f'id step {child_id} not found in steps list')
                    edges.append((step_from, step_to))
        elif isinstance(dependencies, t.Iterable):
            edges = dependencies
        else:
            raise ValueError(
                f'dependencies must be a dict like object or an iterable of tuples. '
                f'See the docs for more information')
        self._graph = DAG(edges)

    @property
    def parents(self) -> t.Dict[Step, t.List[Step]]:
        return self._graph.pred

    @property
    def children(self) -> t.Dict[Step, t.List[Step]]:
        return self._graph.succ

    @property
    def dependencies(self) -> t.Dict[Step, t.List[Step]]:
        return self.children

    @property
    def root(self) -> t.List[Step]:
        return self._graph.root

    @property
    def target(self) -> t.Set[str]:
        target = set()
        for step in self.steps:
            if step.undo is False:
                target.update(step.target)
        return target

    def _step_exists(self, step: t.Union[t.List[Step], Step]):
        """Checks if all the steps belong to the current orchestration

        Parameters
        ----------
        step: list[Step]|Step
            Step or list of steps to be evaluated

        Returns
        -------
        None

        Raises
        ------
        ValueError: if any step passed as argument is not in the orchestration
        """
        if not isinstance(step, list):
            steps = [step]
        else:
            steps = step

        for step in steps:
            if not (step in self._graph.nodes and step.orchestration is self):
                raise ValueError(f'{step} is NOT from this orchestration')

    def _check_dependencies(self, step, parents=None, children=None):
        """
        Checks if the dependencies that are going to be added accomplish the business rules. These rules are:
            1. a 'do' Step cannot be preceded for an 'undo' Step
            2. cannot be cycles inside the orchestration

        Parameters
        ----------
        step: Step
            step to be evaluated
        parents: list[Step]
            parent steps to be added
        children: list[Step]
            children steps to be added

        Raises
        ------
        ValueError
            if the rule 1 is not passed
        CycleError
            if the rule 2 is not passed
        """
        parents = parents or []
        children = children or []
        if parents:
            if any([p.undo for p in parents]) and not step.undo:
                attr = 'rid' if step.rid is not None else 'id'
                raise errors.ParentUndoError(getattr(
                    step, attr), [getattr(s, attr) for s in parents if s.undo])
        if children:
            if any([not c.undo for c in children]) and step.undo:
                attr = 'rid' if step.rid is not None else 'id'
                raise errors.ChildDoError(
                    getattr(step, attr),
                    [getattr(s, attr) for s in children if s.undo])
        g = self._graph.copy()
        g.add_edges_from([(p, step) for p in parents])
        g.add_edges_from([(step, c) for c in children])
        if g.is_cyclic():
            raise errors.CycleError()

    def add_step(self, *args, parents=None, children=None, **kwargs) -> Step:
        """Allows to add step into the orchestration

        :param args: args passed to Step: undo, action_template.
        :param parents: list of parent steps.
        :param children: list of children steps.
        :param kwargs: keyword arguments passed to the Step
        :return: The step created.
        """
        parents = parents or []
        children = children or []
        s = Step(None, *args, **kwargs)
        self._step_exists(parents + children)
        self._check_dependencies(s, parents, children)
        s.orchestration = self
        self._graph.add_node(s)
        self.add_parents(s, parents)
        self.add_children(s, children)
        return s

    def delete_step(self, step: Step) -> 'Orchestration':
        """
        Allows to remove a Step from the orchestration

        Parameters
        ----------
        step Step: step to remove from the orchestration
        """
        self._step_exists(step)
        i = self.steps.index(step)
        self.steps.pop(i)
        self._graph.remove_node(step)
        return self

    def add_parents(self, step: Step,
                    parents: t.List[Step]) -> 'Orchestration':
        """add_parents adds the parents passed into the step. No remove from previous parents

        Parameters
        ----------
        step: Step
            step to add parents
        parents: list
            list of parent steps to add

        See Also
        --------
        set_parents, delete_parents
        add_children, set_children, delete_children

        Examples
        --------
        >>> at = ActionTemplate(name='action', version=1, action_type=ActionType.SHELL, code='code to run',
                                expected_output='',
                                expected_rc=0, system_kwargs={})
        >>> o = Orchestration('Test Orchestration', 1, DAG(), 'description')
        >>> s1 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
        >>> s2 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
        >>> s3 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
        >>> o.add_parents(s2, [s1])
        >>> o.parents[s2]
        [Step1]
        >>> o.add_parents(s2, [s3])
        >>> o.parents[s2]
        [Step1, Step3]
        """
        self._step_exists([step] + list(parents))
        self._check_dependencies(step, parents=parents)
        step._add_parents(parents)
        self._graph.add_edges_from([(p, step) for p in parents])
        return self

    def delete_parents(self, step: Step,
                       parents: t.List[Step]) -> 'Orchestration':
        """delete_parents deletes the parents passed from the step.

        Parameters
        ----------
        step: Step
            step to remove parents
        parents: list
            list of parent steps to remove

        See Also
        --------
        add_parents, set_parents
        add_children, set_children, delete_children

        Examples
        --------
        >>> at = ActionTemplate(name='action', version=1, action_type=ActionType.SHELL, code='code to run',
                                expected_output='',
                                expected_rc=0, system_kwargs={})
        >>> o = Orchestration('Test Orchestration', 1, DAG(), 'description')
        >>> s1 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
        >>> s2 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
        >>> s3 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
        >>> o.add_children(s1, [s2, s3])
        >>> o.children[s1]
        [Step2, Step3]
        >>> o.delete_parents(s3, [s1])
        >>> o.children[s1]
        [Step2]
        """
        self._step_exists([step] + list(parents))
        step._remove_parents(parents)
        self._graph.remove_edges_from([(p, step) for p in parents])
        return self

    def set_parents(self, step: Step,
                    parents: t.List[Step]) -> 'Orchestration':
        """set_parents sets the parents passed on the step, removing the previos ones

        Parameters
        ----------
        step: Step
            step to remove parents
        parents: list
            list of parent steps to set

        See Also
        --------
        add_parents, delete_parents
        add_children, delete_children, set_children,

        Examples
        --------
        >>> at = ActionTemplate(name='action', version=1, action_type=ActionType.SHELL, code='code to run',
                                expected_rc=0, system_kwargs={})
        >>> o = Orchestration('Test Orchestration', 1, DAG(), 'description')
        >>> s1 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
        >>> s2 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
        >>> s3 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
        >>> o.add_parents(s1, [s2])
        >>> o.parents[s1]
        [Step2]
        >>> o.set_parents(s1, [s3])
        >>> o.parents[s1]
        [Step3]
        """
        self.delete_parents(step, self._graph.pred[step])
        self.add_parents(step, parents)
        return self

    def add_children(self, step: Step,
                     children: t.List[Step]) -> 'Orchestration':
        self._step_exists([step] + children)
        self._check_dependencies(step, children=children)
        step._add_children(children)
        self._graph.add_edges_from([(step, c) for c in children])
        return self

    def delete_children(self, step: Step,
                        children: t.List[Step]) -> 'Orchestration':
        self._step_exists([step] + children)
        step._remove_children(children)
        self._graph.remove_edges_from([(step, c) for c in children])
        return self

    def set_children(self, step: Step,
                     children: t.List[Step]) -> 'Orchestration':
        self.delete_children(step, self._graph.succ[step])
        self.add_children(step, children)
        return self

    def eq_imp(self, other: 'Orchestration') -> bool:
        """
        compares if two orchestrations implement same steps with same parameters and dependencies

        Parameters
        ----------
        other: Orchestration

        Returns
        -------
        result: bool
        """
        if isinstance(other, self.__class__):
            if len(self.steps) != len(other.steps):
                return False
            res = []
            for s in self.steps:
                res.append(any(map(lambda x: s.eq_imp(x), other.steps)))

            if all(res):

                matched_steps = []
                res2 = []
                v2 = []
                for k1, v1 in self.children.items():
                    k2 = None
                    for s in filter(lambda x: x not in matched_steps,
                                    other.steps):
                        if k1.eq_imp(s):
                            k2 = s
                            v2 = other.children[k2]
                            break
                    if not k2:
                        raise RuntimeError('Step not found in other')
                    matched_steps.append(k2)
                    if len(v1) != len(v2):
                        return False

                    for s in v1:
                        res2.append(any(map(lambda x: s.eq_imp(x), v2)))
                return all(res2)
            else:
                return False
        else:
            return False

    def subtree(
        self, steps: t.Union[t.List[Step], t.Iterable[Step]]
    ) -> t.Dict[Step, t.List[Step]]:
        return self._graph.subtree(steps)

    def to_json(self,
                add_target=False,
                add_params=False,
                add_steps=False,
                add_action=False,
                split_lines=False,
                add_schema=False,
                **kwargs):
        data = super().to_json(**kwargs)
        data.update(name=self.name,
                    version=self.version,
                    stop_on_error=self.stop_on_error,
                    undo_on_error=self.undo_on_error,
                    stop_undo_on_error=self.stop_undo_on_error)
        if add_target:
            data.update(target=list(self.target))
        if add_steps:
            json_steps = []
            for step in self.steps:
                json_step = step.to_json(add_action=add_action,
                                         split_lines=split_lines)
                json_step.pop('orchestration_id')
                json_steps.append(json_step)
            data['steps'] = json_steps
        if add_schema:
            data['schema'] = self.schema
        return data

    @classmethod
    def from_json(cls, kwargs):
        return super().from_json(kwargs)

    def __str__(self):
        return f"{self.name}.{self.version}"

    @property
    def schema(self):
        from ...use_cases.deployment import reserved_words, extract_container_var
        outer_schema = {'required': set(), 'output': set()}
        level = 1
        while level <= self._graph.depth:
            new_schema = copy.deepcopy(outer_schema)
            for step in self._graph.get_nodes_at_level(level):
                schema = step.schema
                container_names = [
                    k for k in schema.keys() if k not in reserved_words
                ]
                for cn in container_names:
                    for k, v in schema.get(cn, {}).items():
                        if cn not in new_schema:
                            new_schema.update({cn: {}})
                        if not (k in outer_schema['output']
                                or k in schema.get('mapping', {})):
                            new_schema[cn].update({k: v})

                for k in schema.get('required', []):
                    cn, v = extract_container_var(k)
                    nv = f"{cn}.{v}"
                    if cn == 'input':
                        if v not in outer_schema[
                                'output'] and v not in schema.get(
                                    'mapping', {}):
                            new_schema['required'].add(nv)
                    elif cn != 'env':
                        new_schema['required'].add(nv)

                for k, v in schema.get('mapping', {}).items():
                    if isinstance(v, dict) and len(v) == 1 and 'from' in v:
                        action, source = tuple(v.items())[0]
                        d_cn, d_var = extract_container_var(k)
                        d_nv = f"{d_cn}.{d_var}"
                        s_cn, s_var = extract_container_var(source)
                        s_nv = f"{s_cn}.{s_var}"

                        if d_nv in schema.get('required',
                                              {}) or d_var in schema.get(
                                                  'required', {}):
                            if s_cn == 'input':
                                if s_var not in outer_schema[
                                        'output'] and s_var not in outer_schema[
                                            'input']:
                                    raise errors.MappingError(d_nv, step)
                            elif s_cn != 'env':
                                new_schema['required'].add(s_nv)

                [
                    new_schema['output'].add(extract_container_var(k)[1])
                    for k in schema.get('output', [])
                ]

            level += 1
            outer_schema = new_schema
        outer_schema['required'] = sorted(list(outer_schema['required']))
        outer_schema['output'] = sorted(list(outer_schema['output']))
        for c in ('required', 'output', 'input'):
            # clean data
            if not outer_schema.get(c):
                outer_schema.pop(c, None)

        return outer_schema

    @staticmethod
    def get(id_or_name, version=None) -> t.Union['Orchestration', str]:
        if is_valid_uuid(id_or_name):
            orch = Orchestration.query.get(id_or_name)
            if orch is None:
                return str(errors.EntityNotFound('Orchestration', id_or_name))
        else:
            if id_or_name:
                query = Orchestration.query.filter_by(
                    name=id_or_name).order_by(Orchestration.version.desc())
                if version:
                    query.filter_by(version=version)
                orch = query.first()
                if orch is None:
                    return f"No orchestration found for '{id_or_name}'" + (
                        f" version '{version}'" if version else None)
            else:
                return "No orchestration specified"
        return orch
示例#4
0
    def test_nodes(self):
        G = DAG()
        self.assertListEqual(G.nodes, [])

        G.add_node(1)
        self.assertListEqual(G.nodes, [1])
示例#5
0
    def test_succ(self):
        G = DAG()
        self.assertDictEqual(G.succ, {})

        G.add_node(1)
        self.assertDictEqual(G.succ, {1: []})
示例#6
0
    def test_pred(self):
        G = DAG()
        self.assertDictEqual(G.pred, {})

        G.add_node(1)
        self.assertDictEqual(G.pred, {1: []})