class Orchestration(UUIDistributedEntityMixin, db.Model):
__tablename__ = 'D_orchestration'
order = 20
name = db.Column(db.String(80), nullable=False)
version = db.Column(db.Integer, nullable=False)
description = db.Column(db.Text)
stop_on_error = db.Column(db.Boolean)
stop_undo_on_error = db.Column(db.Boolean)
undo_on_error = db.Column(db.Boolean)
created_at = db.Column(UtcDateTime(timezone=True), default=get_now)
steps = db.relationship(
"Step",
primaryjoin="Step.orchestration_id==Orchestration.id",
back_populates="orchestration")
__table_args__ = (db.UniqueConstraint('name',
'version',
name='D_orchestration_uq01'), )
def __init__(self,
name: str,
version: int,
description: t.Optional[str] = None,
steps: t.List[Step] = None,
stop_on_error: bool = True,
stop_undo_on_error: bool = True,
undo_on_error: bool = True,
dependencies: Tdependencies = None,
created_at=None,
**kwargs):
super().__init__(**kwargs)
self.name = name
self.version = version
self.description = '\n'.join(description) if is_iterable_not_string(
description) else description
self.steps = steps or []
assert isinstance(stop_on_error, bool)
self.stop_on_error = stop_on_error
assert isinstance(stop_undo_on_error, bool)
self.stop_undo_on_error = stop_undo_on_error
assert isinstance(undo_on_error, bool)
self.undo_on_error = undo_on_error
self.created_at = created_at or get_now()
if dependencies:
self.set_dependencies(dependencies)
else:
self._graph = DAG()
@orm.reconstructor
def init_on_load(self):
self._graph = DAG()
for step in self.steps:
if step.parents:
for p in step.parents:
self._graph.add_edge(p, step)
else:
self._graph.add_node(step)
def set_dependencies(self, dependencies: Tdependencies):
edges = []
find = lambda id_: next(
(step for step in self.steps if step.id == id_))
if isinstance(dependencies, t.Dict):
for k, v in dependencies.items():
try:
step_from = find(k)
except StopIteration:
raise ValueError(f'id step {k} not found in steps list')
for child_id in v:
try:
step_to = find(child_id)
except StopIteration:
raise ValueError(
f'id step {child_id} not found in steps list')
edges.append((step_from, step_to))
elif isinstance(dependencies, t.Iterable):
edges = dependencies
else:
raise ValueError(
f'dependencies must be a dict like object or an iterable of tuples. '
f'See the docs for more information')
self._graph = DAG(edges)
@property
def parents(self) -> t.Dict[Step, t.List[Step]]:
return self._graph.pred
@property
def children(self) -> t.Dict[Step, t.List[Step]]:
return self._graph.succ
@property
def dependencies(self) -> t.Dict[Step, t.List[Step]]:
return self.children
@property
def root(self) -> t.List[Step]:
return self._graph.root
@property
def target(self) -> t.Set[str]:
target = set()
for step in self.steps:
if step.undo is False:
target.update(step.target)
return target
def _step_exists(self, step: t.Union[t.List[Step], Step]):
"""Checks if all the steps belong to the current orchestration
Parameters
----------
step: list[Step]|Step
Step or list of steps to be evaluated
Returns
-------
None
Raises
------
ValueError: if any step passed as argument is not in the orchestration
"""
if not isinstance(step, list):
steps = [step]
else:
steps = step
for step in steps:
if not (step in self._graph.nodes and step.orchestration is self):
raise ValueError(f'{step} is NOT from this orchestration')
def _check_dependencies(self, step, parents=None, children=None):
"""
Checks if the dependencies that are going to be added accomplish the business rules. These rules are:
1. a 'do' Step cannot be preceded for an 'undo' Step
2. cannot be cycles inside the orchestration
Parameters
----------
step: Step
step to be evaluated
parents: list[Step]
parent steps to be added
children: list[Step]
children steps to be added
Raises
------
ValueError
if the rule 1 is not passed
CycleError
if the rule 2 is not passed
"""
parents = parents or []
children = children or []
if parents:
if any([p.undo for p in parents]) and not step.undo:
attr = 'rid' if step.rid is not None else 'id'
raise errors.ParentUndoError(getattr(
step, attr), [getattr(s, attr) for s in parents if s.undo])
if children:
if any([not c.undo for c in children]) and step.undo:
attr = 'rid' if step.rid is not None else 'id'
raise errors.ChildDoError(
getattr(step, attr),
[getattr(s, attr) for s in children if s.undo])
g = self._graph.copy()
g.add_edges_from([(p, step) for p in parents])
g.add_edges_from([(step, c) for c in children])
if g.is_cyclic():
raise errors.CycleError()
def add_step(self, *args, parents=None, children=None, **kwargs) -> Step:
"""Allows to add step into the orchestration
:param args: args passed to Step: undo, action_template.
:param parents: list of parent steps.
:param children: list of children steps.
:param kwargs: keyword arguments passed to the Step
:return: The step created.
"""
parents = parents or []
children = children or []
s = Step(None, *args, **kwargs)
self._step_exists(parents + children)
self._check_dependencies(s, parents, children)
s.orchestration = self
self._graph.add_node(s)
self.add_parents(s, parents)
self.add_children(s, children)
return s
def delete_step(self, step: Step) -> 'Orchestration':
"""
Allows to remove a Step from the orchestration
Parameters
----------
step Step: step to remove from the orchestration
"""
self._step_exists(step)
i = self.steps.index(step)
self.steps.pop(i)
self._graph.remove_node(step)
return self
def add_parents(self, step: Step,
parents: t.List[Step]) -> 'Orchestration':
"""add_parents adds the parents passed into the step. No remove from previous parents
Parameters
----------
step: Step
step to add parents
parents: list
list of parent steps to add
See Also
--------
set_parents, delete_parents
add_children, set_children, delete_children
Examples
--------
>>> at = ActionTemplate(name='action', version=1, action_type=ActionType.SHELL, code='code to run',
expected_output='',
expected_rc=0, system_kwargs={})
>>> o = Orchestration('Test Orchestration', 1, DAG(), 'description')
>>> s1 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s2 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s3 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> o.add_parents(s2, [s1])
>>> o.parents[s2]
[Step1]
>>> o.add_parents(s2, [s3])
>>> o.parents[s2]
[Step1, Step3]
"""
self._step_exists([step] + list(parents))
self._check_dependencies(step, parents=parents)
step._add_parents(parents)
self._graph.add_edges_from([(p, step) for p in parents])
return self
def delete_parents(self, step: Step,
parents: t.List[Step]) -> 'Orchestration':
"""delete_parents deletes the parents passed from the step.
Parameters
----------
step: Step
step to remove parents
parents: list
list of parent steps to remove
See Also
--------
add_parents, set_parents
add_children, set_children, delete_children
Examples
--------
>>> at = ActionTemplate(name='action', version=1, action_type=ActionType.SHELL, code='code to run',
expected_output='',
expected_rc=0, system_kwargs={})
>>> o = Orchestration('Test Orchestration', 1, DAG(), 'description')
>>> s1 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s2 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s3 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> o.add_children(s1, [s2, s3])
>>> o.children[s1]
[Step2, Step3]
>>> o.delete_parents(s3, [s1])
>>> o.children[s1]
[Step2]
"""
self._step_exists([step] + list(parents))
step._remove_parents(parents)
self._graph.remove_edges_from([(p, step) for p in parents])
return self
def set_parents(self, step: Step,
parents: t.List[Step]) -> 'Orchestration':
"""set_parents sets the parents passed on the step, removing the previos ones
Parameters
----------
step: Step
step to remove parents
parents: list
list of parent steps to set
See Also
--------
add_parents, delete_parents
add_children, delete_children, set_children,
Examples
--------
>>> at = ActionTemplate(name='action', version=1, action_type=ActionType.SHELL, code='code to run',
expected_rc=0, system_kwargs={})
>>> o = Orchestration('Test Orchestration', 1, DAG(), 'description')
>>> s1 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s2 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s3 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> o.add_parents(s1, [s2])
>>> o.parents[s1]
[Step2]
>>> o.set_parents(s1, [s3])
>>> o.parents[s1]
[Step3]
"""
self.delete_parents(step, self._graph.pred[step])
self.add_parents(step, parents)
return self
def add_children(self, step: Step,
children: t.List[Step]) -> 'Orchestration':
self._step_exists([step] + children)
self._check_dependencies(step, children=children)
step._add_children(children)
self._graph.add_edges_from([(step, c) for c in children])
return self
def delete_children(self, step: Step,
children: t.List[Step]) -> 'Orchestration':
self._step_exists([step] + children)
step._remove_children(children)
self._graph.remove_edges_from([(step, c) for c in children])
return self
def set_children(self, step: Step,
children: t.List[Step]) -> 'Orchestration':
self.delete_children(step, self._graph.succ[step])
self.add_children(step, children)
return self
def eq_imp(self, other: 'Orchestration') -> bool:
"""
compares if two orchestrations implement same steps with same parameters and dependencies
Parameters
----------
other: Orchestration
Returns
-------
result: bool
"""
if isinstance(other, self.__class__):
if len(self.steps) != len(other.steps):
return False
res = []
for s in self.steps:
res.append(any(map(lambda x: s.eq_imp(x), other.steps)))
if all(res):
matched_steps = []
res2 = []
v2 = []
for k1, v1 in self.children.items():
k2 = None
for s in filter(lambda x: x not in matched_steps,
other.steps):
if k1.eq_imp(s):
k2 = s
v2 = other.children[k2]
break
if not k2:
raise RuntimeError('Step not found in other')
matched_steps.append(k2)
if len(v1) != len(v2):
return False
for s in v1:
res2.append(any(map(lambda x: s.eq_imp(x), v2)))
return all(res2)
else:
return False
else:
return False
def subtree(
self, steps: t.Union[t.List[Step], t.Iterable[Step]]
) -> t.Dict[Step, t.List[Step]]:
return self._graph.subtree(steps)
def to_json(self,
add_target=False,
add_params=False,
add_steps=False,
add_action=False,
split_lines=False,
add_schema=False,
**kwargs):
data = super().to_json(**kwargs)
data.update(name=self.name,
version=self.version,
stop_on_error=self.stop_on_error,
undo_on_error=self.undo_on_error,
stop_undo_on_error=self.stop_undo_on_error)
if add_target:
data.update(target=list(self.target))
if add_steps:
json_steps = []
for step in self.steps:
json_step = step.to_json(add_action=add_action,
split_lines=split_lines)
json_step.pop('orchestration_id')
json_steps.append(json_step)
data['steps'] = json_steps
if add_schema:
data['schema'] = self.schema
return data
@classmethod
def from_json(cls, kwargs):
return super().from_json(kwargs)
def __str__(self):
return f"{self.name}.{self.version}"
@property
def schema(self):
from ...use_cases.deployment import reserved_words, extract_container_var
outer_schema = {'required': set(), 'output': set()}
level = 1
while level <= self._graph.depth:
new_schema = copy.deepcopy(outer_schema)
for step in self._graph.get_nodes_at_level(level):
schema = step.schema
container_names = [
k for k in schema.keys() if k not in reserved_words
]
for cn in container_names:
for k, v in schema.get(cn, {}).items():
if cn not in new_schema:
new_schema.update({cn: {}})
if not (k in outer_schema['output']
or k in schema.get('mapping', {})):
new_schema[cn].update({k: v})
for k in schema.get('required', []):
cn, v = extract_container_var(k)
nv = f"{cn}.{v}"
if cn == 'input':
if v not in outer_schema[
'output'] and v not in schema.get(
'mapping', {}):
new_schema['required'].add(nv)
elif cn != 'env':
new_schema['required'].add(nv)
for k, v in schema.get('mapping', {}).items():
if isinstance(v, dict) and len(v) == 1 and 'from' in v:
action, source = tuple(v.items())[0]
d_cn, d_var = extract_container_var(k)
d_nv = f"{d_cn}.{d_var}"
s_cn, s_var = extract_container_var(source)
s_nv = f"{s_cn}.{s_var}"
if d_nv in schema.get('required',
{}) or d_var in schema.get(
'required', {}):
if s_cn == 'input':
if s_var not in outer_schema[
'output'] and s_var not in outer_schema[
'input']:
raise errors.MappingError(d_nv, step)
elif s_cn != 'env':
new_schema['required'].add(s_nv)
[
new_schema['output'].add(extract_container_var(k)[1])
for k in schema.get('output', [])
]
level += 1
outer_schema = new_schema
outer_schema['required'] = sorted(list(outer_schema['required']))
outer_schema['output'] = sorted(list(outer_schema['output']))
for c in ('required', 'output', 'input'):
# clean data
if not outer_schema.get(c):
outer_schema.pop(c, None)
return outer_schema
@staticmethod
def get(id_or_name, version=None) -> t.Union['Orchestration', str]:
if is_valid_uuid(id_or_name):
orch = Orchestration.query.get(id_or_name)
if orch is None:
return str(errors.EntityNotFound('Orchestration', id_or_name))
else:
if id_or_name:
query = Orchestration.query.filter_by(
name=id_or_name).order_by(Orchestration.version.desc())
if version:
query.filter_by(version=version)
orch = query.first()
if orch is None:
return f"No orchestration found for '{id_or_name}'" + (
f" version '{version}'" if version else None)
else:
return "No orchestration specified"
return orch
class CompositeCommand(ICommand):
def __init__(self, dict_tree: t.Dict[ICommand, t.List[ICommand]],
stop_on_error: bool, stop_undo_on_error: bool = None,
id_=None, executor: concurrent.futures.Executor = None,
register: 'RegisterStepExecution' = None, var_context: Context = None):
"""
Parameters
----------
dict_tree:
dict tree containing command nodes as keys and it's neighbours as a list of commands. See examples for more
information
undo_on_error:
executes the undo process even if the command ended up with error
stop_on_error:
executes all the commands regardless if a command succeed or fail
stop_undo_on_error:
executes all the undo commands regardless if an undo command succeed or fail
id_:
command identifier
executor:
async call executor. defaults to concurrent.futures.process.ProcessPoolExecutor
"""
self._id = id_
self._dag = DAG().from_dict_of_lists(dict_tree)
self.stop_on_error = stop_on_error
self.stop_undo_on_error = stop_undo_on_error
self.executor = executor or ThreadPoolExecutor(max_workers=4)
self.register = register
self.var_context = var_context
@property
def success(self) -> t.Optional[bool]:
success_list = [n.success for n in self._dag]
if success_list == [None] * len(success_list):
return None
elif success_list == [True] * len(success_list):
return True
else:
return False
def invoke(self, timeout=None) -> bool:
# logging.basicConfig(format='%(threadName)s %(levelname)-8s %(message)s')
# logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
res = []
stop = False
level = 1
start = time.time()
while level <= self._dag.depth:
commands = self._dag.get_nodes_at_level(level)
duration = time.time() - start
left = max(timeout - duration, 0) if timeout else None
if len(commands) == 1:
r = commands[0].invoke(timeout=left)
res.append(r)
stop_on_error = commands[0].stop_on_error if commands[0].stop_on_error is not None \
else self.stop_on_error
if r is False and stop_on_error is True:
break
else:
futures = {}
for command in commands:
futures.update(
{command.id: self.executor.submit(command.invoke, timeout=left)})
stop = None
while len(futures) > 0 and (left is None or (time.time() - start) < left) and stop is not True:
for cmd_id, future in dict(futures).items():
r = None
try:
r = future.result(timeout=0.5)
except concurrent.futures.TimeoutError:
pass
except Exception as e:
logger.exception(f"Error while executing step {cmd_id}")
del futures[cmd_id]
r = False
else:
del futures[cmd_id]
if r is not None:
res.append(r)
cmd = list(filter(lambda c: c.id == cmd_id, commands))[0]
stop_on_error = cmd.stop_on_error if cmd.stop_on_error is not None else self.stop_on_error
if r is False and stop_on_error is True:
stop = True
duration = time.time() - start
left = max(timeout - duration, 0) if timeout else None
# if self.var_context:
# self.var_context.merge_common_variables()
if stop is True or (timeout is not None and (time.time() - start) > timeout):
break
level += 1
return all(res) if len(res) > 0 else None
def undo(self, timeout=None) -> t.Optional[bool]:
"""
Executes undo operation
Returns
-------
bool:
True if all undo commands that run ended up successfully
False if any undo command ended up badly
None if undo operation not executed
"""
if self.stop_undo_on_error is None:
raise RuntimeError(f'stop_undo_on_error not set for command {self.id}')
res = []
level = self._dag.depth
start = time.time()
duration = time.time() - start
left = max(timeout - duration, 0) if timeout else None
while level > 0 and (left is None or (time.time() - start) < left):
commands = self._dag.get_nodes_at_level(level)
duration = time.time() - start
left = max(timeout - duration, 0) if timeout else None
if len(commands) == 1:
r = commands[0].undo(timeout=left)
if r is not None:
res.append(r)
stop_undo_on_error = commands[0].stop_undo_on_error if commands[0].stop_undo_on_error is not None \
else self.stop_undo_on_error
if r is False and stop_undo_on_error is True:
break
else:
futures = {}
for command in commands:
futures.update(
{command.id: self.executor.submit(command.undo, timeout=left)})
stop = None
while len(futures) > 0 and (left is None or (time.time() - start) < left):
for cmd_id, future in dict(futures).items():
r = None
try:
r = future.result(timeout=0.5)
except concurrent.futures.TimeoutError:
pass
except Exception as e:
logger.exception(f"Error while executing step {cmd_id}")
del futures[cmd_id]
r = False
else:
del futures[cmd_id]
if r is not None:
res.append(r)
cmd = list(filter(lambda c: c.id == cmd_id, commands))[0]
stop_undo_on_error = cmd.stop_undo_on_error if cmd.stop_undo_on_error is not None \
else self.stop_undo_on_error
if r is False and cmd.stop_undo_on_error is True:
stop = True
duration = time.time() - start
left = max(timeout - duration, 0) if timeout else None
if stop is True or (timeout is not None and (time.time() - start) > timeout):
break
level -= 1
return all(res) if len(res) > 0 else None
@property
def result(self) -> t.Dict[Id, CompletedProcess]:
e = {}
for n in self._dag.nodes:
if n.success is not None:
e.update(n.result)
return e
def __len__(self):
return len(self._dag)
def __iter__(self):
return self._dag
def test_get_nodes_at_level(self):
g = DAG([(1, 2), (1, 7), (7, 2), (2, 4), (2, 3), (4, 5), (4, 6)])
self.assertListEqual([4, 3], g.get_nodes_at_level(4))