class ServiceOrchestration(EntityReprMixin, DistributedEntityMixin, db.Model):
__tablename__ = 'D_service_orchestration'
order = 50
id = db.Column(UUID, primary_key=True)
service_id = db.Column(UUID, db.ForeignKey('D_service.id'))
orchestration_id = db.Column(UUID, db.ForeignKey('D_orchestration.id'))
execution_time = db.Column(UtcDateTime(), default=get_now)
class FileServerAssociation(DistributedEntityMixin, SoftDeleteMixin, db.Model):
__tablename__ = 'D_file_server_association'
order = 30
file_id = db.Column(typos.UUID, db.ForeignKey('D_file.id'), nullable=False, primary_key=True)
dst_server_id = db.Column(typos.UUID, db.ForeignKey('D_server.id'), nullable=False, primary_key=True)
dest_folder = db.Column(db.Text)
l_mtime = db.Column(db.INTEGER)
file = db.relationship("File")
destination_server = db.relationship("Server")
@property
def destination_folder(self):
return self.dest_folder or self.file.dest_folder or os.path.dirname(self.file.target)
@property
def target(self):
return os.path.join(self.destination_folder, os.path.basename(self.file.target))
def to_json(self, human=False, **kwargs) -> t.Dict:
data = super().to_json(**kwargs)
if human:
data.update({'file': {'target': self.file.target, 'src_server': self.file.source_server.name,
'dst_server': str(self.destination_server.name),
'dest_folder': self.dest_folder}})
else:
data.update({'file_id': str(self.file.id), 'dst_server_id': str(self.destination_server.id),
'dest_folder': self.dest_folder})
return data
@classmethod
def from_json(cls, kwargs) -> 'FileServerAssociation':
kwargs = copy.deepcopy(kwargs)
kwargs['file'] = File.query.get_or_raise(kwargs.get('file_id'))
kwargs['destination_server'] = Server.query.get_or_raise(kwargs.get('dst_server_id'))
super().from_json(kwargs)
try:
o = cls.query.get((kwargs['file_id'], kwargs['dst_server_id']))
except RuntimeError as e:
o = None
if o:
for k, v in kwargs.items():
if getattr(o, k) != v:
setattr(o, k, v)
return o
else:
return cls(**kwargs)
class Vault(DistributedEntityMixin, SoftDeleteMixin, db.Model):
__tablename__ = 'D_vault'
user_id = db.Column(UUID, db.ForeignKey('D_user.id'), primary_key=True)
scope = db.Column(db.String(100), primary_key=True, default='global')
name = db.Column(db.String(100), primary_key=True)
_old_name = db.Column(db.String(100))
value = db.Column(db.PickleType)
user = db.relationship("User")
def to_json(self, human=False, **kwargs):
dto = super().to_json(**kwargs)
dto.update(name=self.name, value=self.value)
if self.scope:
dto.update(scope=self.scope)
if human:
dto.update(user={'id': self.user.id, 'name': self.user.name})
else:
dto.update(user_id=self.user_id or getattr(self.user, 'id', None))
return dto
@classmethod
def from_json(cls, kwargs):
super().from_json(kwargs)
kwargs['user'] = User.query.get_or_raise(kwargs.pop('user_id', None))
o = cls.query.get(
(getattr(kwargs.get('user'),
'id'), kwargs.get('scope', 'global'), kwargs.get('name')))
if o:
for k, v in kwargs.items():
if getattr(o, k) != v:
setattr(o, k, v)
return o
else:
return cls(**kwargs)
@classmethod
def get_variables_from(cls, user: t.Union[Id, User], scope='global'):
if isinstance(user, User):
user_id = user.id
else:
user_id = user
return {
vault.name: vault.value
for vault in cls.query.filter_by(user_id=user_id,
scope=scope).all()
}
def __str__(self):
return f"Vault({self.user}:{self.scope}[{self.name}={self.value}])"
class File(UUIDistributedEntityMixin, SoftDeleteMixin, db.Model):
__tablename__ = 'D_file'
order = 20
src_server_id = db.Column(typos.UUID, db.ForeignKey('D_server.id'), nullable=False)
target = db.Column(db.Text, nullable=False)
dest_folder = db.Column(db.Text)
_old_target = db.Column("$$target", db.Text)
l_mtime = db.Column(db.INTEGER)
source_server = db.relationship("Server")
destinations: t.List[FileServerAssociation] = db.relationship("FileServerAssociation", lazy='joined')
__table_args__ = (db.UniqueConstraint('src_server_id', 'target'),)
query_class = QueryWithSoftDelete
def __init__(self, source_server: Server, target: str,
dest_folder=None, destination_servers: Destination_Servers = None, **kwargs):
super().__init__(**kwargs)
self.source_server = source_server
self.target = target
self.dest_folder = dest_folder
dest = []
for ds in destination_servers or []:
if isinstance(ds, Server):
dest.append(FileServerAssociation(file=self, destination_server=ds))
elif isinstance(ds, dict):
dest.append(
FileServerAssociation(file=self, destination_server=Server.query.get(ds.get('dst_server_id')),
dest_folder=ds.get('dest_folder')))
elif isinstance(ds, tuple):
if isinstance(ds[0], Server):
dest.append(FileServerAssociation(file=self, destination_server=ds[0], dest_folder=ds[1]))
else:
dest.append(
FileServerAssociation(file=self, destination_server=Server.query.get(ds[0]), dest_folder=ds[1]))
self.destinations = dest
def __str__(self):
return f"{self.source_server}:{self.target}"
def to_json(self, human=False, destinations=False, include: t.List[str] = None,
exclude: t.List[str] = None, **kwargs):
data = super().to_json(**kwargs)
if self.source_server.id is None:
raise RuntimeError('Set ids for servers before')
data.update(target=self.target, dest_folder=self.dest_folder)
if human:
data.update(src_server=str(self.source_server.name))
if destinations:
dest = []
for d in self.destinations:
dest.append(dict(dst_server=d.destination_server.name, dest_folder=d.dest_folder))
data.update(destinations=dest)
else:
data.update(src_server_id=str(self.source_server.id))
if destinations:
dest = []
for d in self.destinations:
dest.append(dict(dst_server_id=d.destination_server.id, dest_folder=d.dest_folder))
data.update(destinations=dest)
if include:
data = {k: v for k, v in data.items() if k in include}
if exclude:
data = {k: v for k, v in data.items() if k not in exclude}
return data
@classmethod
def from_json(cls, kwargs) -> 'File':
kwargs = copy.deepcopy(kwargs)
kwargs['source_server'] = Server.query.get_or_raise(kwargs.pop('src_server_id'))
return super().from_json(kwargs)
def delete(self):
for d in self.destinations:
d.delete()
return super().delete()
class Step(UUIDistributedEntityMixin, db.Model):
__tablename__ = "D_step"
order = 30
orchestration_id = db.Column(UUID, db.ForeignKey('D_orchestration.id'), nullable=False)
action_template_id = db.Column(UUID, db.ForeignKey('D_action_template.id'))
undo = db.Column(db.Boolean, nullable=False)
step_stop_on_error = db.Column("stop_on_error", db.Boolean)
step_stop_undo_on_error = db.Column("stop_undo_on_error", db.Boolean)
step_undo_on_error = db.Column("undo_on_error", db.Boolean)
step_expected_stdout = db.Column("expected_stdout", db.Text)
step_expected_stderr = db.Column("expected_stderr", db.Text)
step_expected_rc = db.Column("expected_rc", db.Integer)
step_system_kwargs = db.Column("system_kwargs", db.JSON)
target = db.Column(ScalarListType(str))
created_on = db.Column(UtcDateTime(), nullable=False, default=get_now)
step_action_type = db.Column("action_type", typos.Enum(ActionType))
step_code = db.Column("code", db.Text)
step_post_process = db.Column("post_process", db.Text)
step_pre_process = db.Column("pre_process", db.Text)
step_name = db.Column("name", db.String(40))
step_schema = db.Column("schema", db.JSON)
step_description = db.Column("description", db.Text)
orchestration = db.relationship("Orchestration", primaryjoin="Step.orchestration_id==Orchestration.id",
back_populates="steps")
action_template = db.relationship("ActionTemplate", primaryjoin="Step.action_template_id==ActionTemplate.id",
backref="steps")
parent_steps = db.relationship("Step", secondary="D_step_step",
primaryjoin="D_step.c.id==D_step_step.c.step_id",
secondaryjoin="D_step.c.id==D_step_step.c.parent_step_id",
back_populates="children_steps")
children_steps = db.relationship("Step", secondary="D_step_step",
primaryjoin="D_step.c.id==D_step_step.c.parent_step_id",
secondaryjoin="D_step.c.id==D_step_step.c.step_id",
back_populates="parent_steps")
def __init__(self, orchestration: 'Orchestration', undo: bool, action_template: 'ActionTemplate' = None,
action_type: ActionType = None, code: str = None, pre_process: str = None, post_process: str = None,
stop_on_error: bool = None, stop_undo_on_error: bool = None, undo_on_error: bool = None,
expected_stdout: t.Optional[str] = None,
expected_stderr: t.Optional[str] = None,
expected_rc: t.Optional[int] = None,
schema: t.Dict[str, t.Any] = None,
system_kwargs: t.Dict[str, t.Any] = None,
parent_steps: t.List['Step'] = None, children_steps: t.List['Step'] = None,
target: t.Union[str, t.Iterable[str]] = None, name: str = None, description: str = None, rid=None,
**kwargs):
super().__init__(**kwargs)
assert undo in (False, True)
if action_template is not None:
assert action_type is None
else:
assert action_type is not None
self.undo = undo
self.step_stop_on_error = stop_on_error if stop_on_error is not None else kwargs.pop('step_stop_on_error', None)
self.step_stop_undo_on_error = stop_undo_on_error if stop_undo_on_error is not None else kwargs.pop(
'step_stop_undo_on_error', None)
self.step_undo_on_error = undo_on_error if undo_on_error is not None else kwargs.pop('step_undo_on_error', None)
self.action_template = action_template
self.step_action_type = action_type if action_type is not None else kwargs.pop('step_action_type', None)
expected_stdout = expected_stdout if expected_stdout is not None else kwargs.pop(
'step_expected_stdout', None)
self.step_expected_stdout = '\n'.join(expected_stdout) if is_iterable_not_string(
expected_stdout) else expected_stdout
expected_stderr = expected_stderr if expected_stderr is not None else kwargs.pop(
'step_expected_stderr', None)
self.step_expected_stderr = '\n'.join(expected_stderr) if is_iterable_not_string(
expected_stderr) else expected_stderr
self.step_expected_rc = expected_rc if expected_rc is not None else kwargs.pop('step_expected_rc', None)
self.step_schema = schema if schema is not None else kwargs.pop('step_schema', None) or {}
self.step_system_kwargs = system_kwargs if system_kwargs is not None else kwargs.pop('step_system_kwargs',
None) or {}
code = code if code is not None else kwargs.pop('step_code', None)
self.step_code = '\n'.join(code) if is_iterable_not_string(code) else code
post_process = post_process if post_process is not None else kwargs.pop('step_post_process', None)
self.step_post_process = '\n'.join(post_process) if is_iterable_not_string(post_process) else post_process
pre_process = pre_process if pre_process is not None else kwargs.pop('step_pre_process', None)
self.step_pre_process = '\n'.join(pre_process) if is_iterable_not_string(pre_process) else pre_process
self.orchestration = orchestration
self.parent_steps = parent_steps or []
self.children_steps = children_steps or []
if self.undo is False:
if target is None:
self.target = ['all']
else:
self.target = [target] if isinstance(target, str) else (target if len(target) > 0 else ['all'])
else:
if target:
raise errors.BaseError('target must not be set when creating an UNDO step')
self.created_on = kwargs.get('created_on') or get_now()
self.step_name = name if name is not None else kwargs.pop('step_name', None)
description = description if description is not None else kwargs.pop('step_description', None)
self.step_description = '\n'.join(description) if is_iterable_not_string(description) else description
self.rid = rid # used when creating an Orchestration
@orm.reconstructor
def init_on_load(self):
self.system_kwargs = self.system_kwargs or {}
@property
def parents(self):
return self.parent_steps
@property
def children(self):
return self.children_steps
@property
def parent_undo_steps(self):
return [s for s in self.parent_steps if s.undo == True]
@property
def children_undo_steps(self):
return [s for s in self.children_steps if s.undo == True]
@property
def parent_do_steps(self):
return [s for s in self.parent_steps if s.undo == False]
@property
def children_do_steps(self):
return [s for s in self.children_steps if s.undo == False]
@property
def stop_on_error(self):
return self.step_stop_on_error if self.step_stop_on_error is not None else self.orchestration.stop_on_error
@stop_on_error.setter
def stop_on_error(self, value):
self.step_stop_on_error = value
@property
def stop_undo_on_error(self):
return self.step_stop_undo_on_error if self.step_stop_undo_on_error is not None \
else self.orchestration.stop_undo_on_error
@stop_undo_on_error.setter
def stop_undo_on_error(self, value):
self.step_stop_undo_on_error = value
@property
def undo_on_error(self):
if self.undo:
return None
return self.step_undo_on_error if self.step_undo_on_error is not None else self.orchestration.undo_on_error
@undo_on_error.setter
def undo_on_error(self, value):
self.step_undo_on_error = value
@property
def schema(self):
if self.action_template:
schema = dict(ChainMap(self.step_schema, self.action_template.schema))
else:
schema = dict(self.step_schema)
if self.action_type == ActionType.ORCHESTRATION:
from .orchestration import Orchestration
mapping = schema.get('mapping', {})
o = Orchestration.get(mapping.get('orchestration', None), mapping.get('version', None))
if isinstance(o, Orchestration):
orch_schema = o.schema
i = schema.get('input', {})
i.update(orch_schema.get('input', {}))
schema.update({'input': i})
m = schema.get('mapping', {})
m.update(orch_schema.get('mapping', {}))
schema.update({'mapping': m})
r = schema.get('required', [])
[r.append(k) for k in orch_schema.get('required', []) if k not in r]
schema.update({'required': r})
o = schema.get('output', [])
[o.append(k) for k in orch_schema.get('output', []) if k not in o]
schema.update({'output': o})
return schema
@schema.setter
def schema(self, value):
self.step_schema = value
@property
def system_kwargs(self):
if self.action_template:
return dict(ChainMap(self.step_system_kwargs, self.action_template.system_kwargs))
else:
return dict(self.step_system_kwargs)
@system_kwargs.setter
def system_kwargs(self, value):
self.step_system_kwargs = value
@property
def code(self):
if self.step_code is None and self.action_template:
return self.action_template.code
else:
return self.step_code
@code.setter
def code(self, value):
self.step_code = value
@property
def action_type(self):
if self.step_action_type is None and self.action_template:
return self.action_template.action_type
else:
return self.step_action_type
@action_type.setter
def action_type(self, value):
self.step_action_type = value
@property
def post_process(self):
if self.step_post_process is None and self.action_template:
return self.action_template.post_process
else:
return self.step_post_process
@post_process.setter
def post_process(self, value):
self.step_post_process = value
@property
def pre_process(self):
if self.step_pre_process is None and self.action_template:
return self.action_template.pre_process
else:
return self.step_pre_process
@pre_process.setter
def pre_process(self, value):
self.step_pre_process = value
@property
def expected_stdout(self):
if self.step_expected_stdout is None and self.action_template:
return self.action_template.expected_stdout
else:
return self.step_expected_stdout
@expected_stdout.setter
def expected_stdout(self, value):
self.step_expected_stdout = value
@property
def expected_stderr(self):
if self.step_expected_stderr is None and self.action_template:
return self.action_template.expected_stderr
else:
return self.step_expected_stderr
@expected_stderr.setter
def expected_stderr(self, value):
if value == self.action_template.expected_stderr:
self.step_expected_stderr = None
else:
self.step_expected_stderr = value
@property
def expected_rc(self):
if self.step_expected_rc is None and self.action_template:
return self.action_template.expected_rc
else:
return self.step_expected_rc
@expected_rc.setter
def expected_rc(self, value):
if self.action_template and value == self.action_template.expected_rc:
self.step_expected_rc = None
else:
self.step_expected_rc = value
@property
def name(self):
if self.step_name is None and self.action_template:
return str(self.action_template)
else:
return self.step_name
@name.setter
def name(self, value):
self.step_name = value
@property
def description(self):
if self.step_description is None and self.action_template:
return str(self.action_template)
else:
return self.step_description
@description.setter
def description(self, value):
self.step_description = value
def eq_imp(self, other):
"""
two steps are equal if they execute the same code with the same parameters even if they are from different
orchestrations or they are in the same orchestration with different positions
Parameters
----------
other: Step
Returns
-------
result: bool
Notes
-----
_id and _orchestration are not compared
"""
if isinstance(other, self.__class__):
return all([self.undo == other.undo,
self.stop_on_error == other.stop_on_error,
self.stop_undo_on_error == other.stop_undo_on_error,
self.undo_on_error == other.undo_on_error,
self.expected_stdout == other.expected_stdout,
self.expected_stderr == other.expected_stderr,
self.expected_rc == other.expected_rc,
self.system_kwargs == other.system_kwargs,
self.code == other.code,
self.post_process == other.post_process,
self.pre_process == other.pre_process,
self.action_type == other.action_type,
])
else:
raise NotImplemented
def __str__(self):
return self.name or self.id
def __repr__(self):
return ('Undo ' if self.undo else '') + self.__class__.__name__ + ' ' + str(getattr(self, 'id', ''))
def _add_parents(self, parents):
for step in parents:
if not step in self.parent_steps:
self.parent_steps.append(step)
def _remove_parents(self, parents):
for step in parents:
if step in self.parent_steps:
self.parent_steps.remove(step)
def _add_children(self, children):
for step in children:
if not step in self.children_steps:
self.children_steps.append(step)
def _remove_children(self, children):
for step in children:
if step in self.children_steps:
self.children_steps.remove(step)
def to_json(self, add_action=False, split_lines=False, **kwargs):
data = super().to_json(**kwargs)
if getattr(self.orchestration, 'id', None):
data.update(orchestration_id=str(self.orchestration.id))
if getattr(self.action_template, 'id', None):
if add_action:
data.update(action_template=self.action_template.to_json(split_lines=split_lines))
else:
data.update(action_template_id=str(self.action_template.id))
data.update(undo=self.undo)
data.update(stop_on_error=self.step_stop_on_error) if self.step_stop_on_error is not None else None
data.update(
stop_undo_on_error=self.step_stop_undo_on_error) if self.step_stop_undo_on_error is not None else None
data.update(undo_on_error=self.step_undo_on_error) if self.step_undo_on_error is not None else None
if self.step_schema:
data.update(schema=self.step_schema)
if self.step_expected_stdout is not None:
data.update(
expected_stdout=self.step_expected_stdout.split('\n') if split_lines else self.step_expected_stdout)
if self.step_expected_stderr is not None:
data.update(
expected_stderr=self.step_expected_stderr.split('\n') if split_lines else self.step_expected_stderr)
data.update(expected_rc=self.step_expected_rc) if self.step_expected_rc is not None else None
data.update(system_kwargs=self.step_system_kwargs) if self.step_system_kwargs is not None else None
data.update(parent_step_ids=[str(step.id) for step in self.parents])
if self.step_code is not None:
data.update(code=self.step_code.split('\n') if split_lines else self.step_code)
data.update(action_type=self.step_action_type.name) if self.step_action_type is not None else None
if self.step_post_process is not None:
data.update(post_process=self.step_post_process.split('\n') if split_lines else self.step_post_process)
if self.step_pre_process is not None:
data.update(pre_process=self.step_pre_process.split('\n') if split_lines else self.step_pre_process)
data.update(created_on=self.created_on.strftime(defaults.DATETIME_FORMAT))
if self.step_description is not None:
data.update(description=self.step_description.split('\n') if split_lines else self.step_description)
if self.step_name is not None:
data.update(name=self.step_name)
return data
@classmethod
def from_json(cls, kwargs):
from dimensigon.domain.entities import ActionTemplate, Orchestration
kwargs = dict(kwargs)
if 'orchestration_id' in kwargs:
ident = kwargs.pop('orchestration_id')
kwargs['orchestration'] = db.session.query(Orchestration).get(ident)
if kwargs['orchestration'] is None:
raise errors.EntityNotFound('Orchestration', ident=ident)
if 'action_template_id' in kwargs:
ident = kwargs.pop('action_template_id')
kwargs['action_template'] = db.session.query(ActionTemplate).get(ident)
if kwargs['action_template'] is None:
raise errors.EntityNotFound('ActionTemplate', ident=ident)
if 'action_type' in kwargs:
kwargs['action_type'] = ActionType[kwargs.pop('action_type')]
if 'created_on' in kwargs:
kwargs['created_on'] = datetime.datetime.strptime(kwargs['created_on'], defaults.DATETIME_FORMAT)
kwargs['parent_steps'] = []
for parent_step_id in kwargs.pop('parent_step_ids', []):
ps = Step.query.get(parent_step_id)
if ps:
kwargs['parent_steps'].append(ps)
else:
raise errors.EntityNotFound('Step', parent_step_id)
return super().from_json(kwargs)
from sqlalchemy import orm
from dimensigon import defaults
from dimensigon.domain.entities.base import UUIDistributedEntityMixin
from dimensigon.utils.typos import UUID, ScalarListType, UtcDateTime
from dimensigon.web import db, errors
from .action_template import ActionType
from ...utils import typos
from ...utils.helpers import get_now, is_iterable_not_string, is_valid_uuid
if t.TYPE_CHECKING:
from .action_template import ActionTemplate
from .orchestration import Orchestration
step_step = db.Table('D_step_step',
db.Column('parent_step_id', UUID, db.ForeignKey('D_step.id'), primary_key=True),
db.Column('step_id', UUID, db.ForeignKey('D_step.id'), primary_key=True),
)
class Step(UUIDistributedEntityMixin, db.Model):
__tablename__ = "D_step"
order = 30
orchestration_id = db.Column(UUID, db.ForeignKey('D_orchestration.id'), nullable=False)
action_template_id = db.Column(UUID, db.ForeignKey('D_action_template.id'))
undo = db.Column(db.Boolean, nullable=False)
step_stop_on_error = db.Column("stop_on_error", db.Boolean)
step_stop_undo_on_error = db.Column("stop_undo_on_error", db.Boolean)
step_undo_on_error = db.Column("undo_on_error", db.Boolean)
class Route(db.Model):
__tablename__ = 'L_route'
destination_id = db.Column(UUID,
db.ForeignKey('D_server.id'),
primary_key=True,
nullable=False)
proxy_server_id = db.Column(UUID, db.ForeignKey('D_server.id'))
gate_id = db.Column(UUID, db.ForeignKey('D_gate.id'))
cost = db.Column(db.Integer)
destination = db.relationship("Server",
foreign_keys=[destination_id],
back_populates="route",
lazy='joined')
proxy_server = db.relationship("Server",
foreign_keys=[proxy_server_id],
lazy='joined')
gate = db.relationship("Gate", foreign_keys=[gate_id], lazy='joined')
def __init__(self,
destination: 'Server',
proxy_server_or_gate: t.Union['Server', 'Gate'] = None,
cost: int = None):
# avoid cycle import
from dimensigon.domain.entities import Server
self.destination = destination
if isinstance(proxy_server_or_gate, Server):
proxy_server = proxy_server_or_gate
gate = None
else:
proxy_server = None
gate = proxy_server_or_gate
if proxy_server:
if proxy_server == destination:
raise ValueError(
'You must specify a gate when proxy_server equals destination'
)
else:
if cost is None or cost == 0:
raise ValueError(
"Cost must be specified and greater than 0 when proxy_server"
)
self.proxy_server = proxy_server
self.cost = cost
elif gate:
# check if gate is from neighbour or from a proxy server
if destination == gate.server:
if cost is not None and cost > 0:
raise ValueError(
"Cost must be set to 0 when defining route for a neighbour"
)
self.gate = gate
self.cost = 0
else:
if cost is None or cost <= 0:
raise ValueError(
"Cost must be specified and greater than 0 when gate is from a proxy_server"
)
else:
self.proxy_server = gate.server
self.cost = cost
elif cost == 0:
# find a gateway and set that gateway as default
if len(destination.external_gates) == 1:
self.gate = destination.external_gates[0]
self.cost = 0
else:
for gate in destination.external_gates:
if check_host(gate.dns or str(gate.ip),
gate.port,
timeout=1,
retry=3,
delay=0.5):
self.gate = gate
self.cost = 0
break
# if not (self.gate or self.proxy_server):
# raise ValueError('Not a valid route')
def validate_route(self, rc: RouteContainer):
if rc.proxy_server:
if not (rc.gate is None and rc.cost > 0):
raise errors.InvalidRoute(self.destination, rc)
if rc.proxy_server._me:
raise errors.InvalidRoute(self.destination, rc)
elif rc.gate:
if not rc.cost == 0:
raise errors.InvalidRoute(self.destination, rc)
else:
if rc.cost is not None:
raise errors.InvalidRoute(self.destination, rc)
def set_route(self, rc: RouteContainer):
self.validate_route(rc)
self.proxy_server, self.gate, self.cost = rc
def __str__(self):
return f"{self.destination} -> " \
f"{self.proxy_server or self.gate}, {self.cost}"
def __repr__(self):
return f"Route({self.to_json()})"
def to_json(self, human=False):
if not self.destination.id or (self.proxy_server
and not self.proxy_server.id) or (
self.gate and not self.gate.id):
raise RuntimeError("commit object before dump to json")
if human:
return {
'destination':
str(self.destination) if self.destination else None,
'proxy_server':
str(self.proxy_server) if self.proxy_server else None,
'gate': str(self.gate) if self.gate else None,
'cost': self.cost
}
else:
return {
'destination_id': self.destination_id,
'proxy_server_id': self.proxy_server_id,
'gate_id': self.gate_id,
'cost': self.cost
}
class Log(UUIDistributedEntityMixin, SoftDeleteMixin, db.Model):
__tablename__ = 'D_log'
src_server_id = db.Column(typos.UUID,
db.ForeignKey('D_server.id'),
nullable=False)
target = db.Column(db.Text, nullable=False)
include = db.Column(db.Text)
exclude = db.Column(db.Text)
dst_server_id = db.Column(typos.UUID,
db.ForeignKey('D_server.id'),
nullable=False)
mode = db.Column(typos.Enum(Mode))
dest_folder = db.Column(db.Text)
recursive = db.Column(db.Boolean, default=False)
_old_target = db.Column("$$target", db.Text)
source_server = db.relationship("Server",
foreign_keys=[src_server_id],
back_populates="log_sources")
destination_server = db.relationship("Server",
foreign_keys=[dst_server_id],
back_populates="log_destinations")
__table_args__ = (db.UniqueConstraint('src_server_id', 'target',
'dst_server_id'), )
def __init__(self,
source_server: 'Server',
target: str,
destination_server: 'Server',
dest_folder=None,
include=None,
exclude=None,
recursive=False,
mode=Mode.REPO_MIRROR,
**kwargs):
super().__init__(**kwargs)
self.source_server = source_server
self.target = target
self.destination_server = destination_server
self.dest_folder = dest_folder
if self.dest_folder is None:
self.mode = mode
else:
self.mode = Mode.FOLDER
self.include = include
self._re_include = re.compile(self.include or '')
self.exclude = exclude
self._re_exclude = re.compile(self.exclude or '^$')
self.recursive = recursive
def __str__(self):
return f"{self.source_server}:{self.target} -> {self.destination_server}:{self.dest_folder}"
def to_json(self,
human=False,
include: t.List[str] = None,
exclude: t.List[str] = None,
**kwargs):
data = super().to_json(**kwargs)
if self.source_server.id is None or self.destination_server.id is None:
raise RuntimeError('Set ids for servers before')
data.update(target=self.target,
include=self.include,
exclude=self.exclude,
dest_folder=self.dest_folder,
recursive=self.recursive,
mode=self.mode.name)
if human:
data.update(src_server=str(self.source_server.name),
dst_server=str(self.destination_server.name))
else:
data.update(src_server_id=str(self.source_server.id),
dst_server_id=str(self.destination_server.id))
if include:
data = {k: v for k, v in data.items() if k in include}
if exclude:
data = {k: v for k, v in data.items() if k not in exclude}
return data
@classmethod
def from_json(cls, kwargs) -> 'Log':
from dimensigon.domain.entities import Server
kwargs = copy.deepcopy(kwargs)
kwargs['mode'] = Mode[kwargs.get('mode')]
kwargs['source_server'] = Server.query.get(kwargs.pop('src_server_id'))
kwargs['destination_server'] = Server.query.get(
kwargs.pop('dst_server_id'))
return super().from_json(kwargs)
class Transfer(UUIDEntityMixin, EntityReprMixin, db.Model):
__tablename__ = "L_transfer"
software_id = db.Column(db.ForeignKey('D_software.id'))
dest_path = db.Column(db.Text, nullable=False)
num_chunks = db.Column(db.Integer, nullable=False)
status = db.Column(typos.Enum(Status),
nullable=False,
default=Status.WAITING_CHUNKS)
created_on = db.Column(UtcDateTime(timezone=True))
started_on = db.Column(UtcDateTime(timezone=True))
ended_on = db.Column(UtcDateTime(timezone=True))
_filename = db.Column("filename", db.String(256))
_size = db.Column("size", db.Integer)
_checksum = db.Column("checksum", db.Text())
software = db.relationship("Software", uselist=False)
def __init__(self,
software: t.Union[Software, str],
dest_path: str,
num_chunks: int,
status: Status = None,
size: int = None,
checksum: str = None,
created_on=None,
**kwargs):
super().__init__(**kwargs)
if isinstance(software, Software):
self.software = software
else:
self._filename = software
if size is None:
ValueError("'size' must be specified when sending a file")
self._size = size
if checksum is None:
ValueError("'checksum' must be specified when sending a file")
self._checksum = checksum
self.dest_path = dest_path
self.num_chunks = num_chunks
self.status = status or Status.WAITING_CHUNKS
self.created_on = created_on or get_now()
def to_json(self):
json = dict(id=str(self.id),
dest_path=self.dest_path,
num_chunks=self.num_chunks,
status=self.status.name,
created_on=self.created_on.strftime(
dimensigon.defaults.DATETIME_FORMAT),
file=os.path.join(self.dest_path, self.filename))
if self.software:
json.update(software_id=str(self.software.id))
else:
json.update(size=self._size, checksum=self._checksum)
if self.started_on:
json.update(started_on=self.started_on.strftime(
dimensigon.defaults.DATETIME_FORMAT))
if self.ended_on:
json.update(ended_on=self.ended_on.strftime(
dimensigon.defaults.DATETIME_FORMAT))
return json
@property
def filename(self):
if self.software:
return self.software.filename
else:
return self._filename
@property
def size(self):
if self.software:
return self.software.size
else:
return self._size
@property
def checksum(self):
if self.software:
return self.software.checksum
else:
return self._checksum
def wait_transfer(self,
timeout=None,
refresh_interval: float = 0.02) -> Status:
timeout = timeout or 300
refresh_interval = refresh_interval
start = time.time()
db.session.refresh(self)
delta = 0
while self.status in (Status.IN_PROGRESS,
Status.WAITING_CHUNKS) and delta < timeout:
time.sleep(refresh_interval)
db.session.refresh(self)
delta = time.time() - start
return self.status
class StepExecution(UUIDEntityMixin, EntityReprMixin, db.Model):
__tablename__ = 'L_step_execution'
start_time = db.Column(UtcDateTime(timezone=True), nullable=False)
end_time = db.Column(UtcDateTime(timezone=True))
params = db.Column(db.JSON)
rc = db.Column(db.Integer)
stdout = db.Column(db.Text)
stderr = db.Column(db.Text)
success = db.Column(db.Boolean)
step_id = db.Column(UUID, db.ForeignKey('D_step.id'), nullable=False)
server_id = db.Column(UUID, db.ForeignKey('D_server.id'))
orch_execution_id = db.Column(UUID, db.ForeignKey('L_orch_execution.id'))
pre_process_elapsed_time = db.Column(db.Float)
execution_elapsed_time = db.Column(db.Float)
post_process_elapsed_time = db.Column(db.Float)
child_orch_execution_id = db.Column(UUID)
step = db.relationship("Step")
server = db.relationship("Server", foreign_keys=[server_id])
orch_execution = db.relationship("OrchExecution",
foreign_keys=[orch_execution_id],
uselist=False,
back_populates="step_executions")
child_orch_execution = db.relationship(
"OrchExecution",
uselist=False,
foreign_keys=[child_orch_execution_id],
primaryjoin="StepExecution.child_orch_execution_id==OrchExecution.id")
# def __init__(self, *args, **kwargs):
# UUIDEntityMixin.__init__(self, **kwargs)
def load_completed_result(self, cp: 'CompletedProcess'):
self.success = cp.success
self.stdout = cp.stdout
self.stderr = cp.stderr
self.rc = cp.rc
def to_json(self, human=False, split_lines=False):
data = {}
if self.id:
data.update(id=str(self.id))
if self.start_time:
data.update(
start_time=self.start_time.strftime(defaults.DATETIME_FORMAT))
if self.end_time:
data.update(
end_time=self.end_time.strftime(defaults.DATETIME_FORMAT))
data.update(params=self.params)
data.update(step_id=str(getattr(self.step, 'id', None)
) if getattr(self.step, 'id', None) else None)
data.update(rc=self.rc, success=self.success)
if human:
data.update(server=str(self.server) if self.server else None)
try:
stdout = json.loads(self.stdout)
except:
stdout = self.stdout.split(
'\n'
) if split_lines and self.stdout and '\n' in self.stdout else self.stdout
try:
stderr = json.loads(self.stderr)
except:
stderr = self.stderr.split(
'\n'
) if split_lines and self.stderr and '\n' in self.stderr else self.stderr
else:
data.update(server_id=str(getattr(self.server, 'id', None))
if getattr(self.server, 'id', None) else None)
stdout = self.stdout.split(
'\n'
) if split_lines and self.stdout and '\n' in self.stdout else self.stdout
stderr = self.stderr.split(
'\n'
) if split_lines and self.stderr and '\n' in self.stderr else self.stderr
data.update(stdout=stdout)
data.update(stderr=stderr)
if self.child_orch_execution_id:
data.update(
child_orch_execution_id=str(self.child_orch_execution_id))
data.update(pre_process_elapsed_time=self.pre_process_elapsed_time
) if self.pre_process_elapsed_time is not None else None
data.update(execution_elapsed_time=self.execution_elapsed_time
) if self.execution_elapsed_time is not None else None
data.update(post_process_elapsed_time=self.post_process_elapsed_time
) if self.post_process_elapsed_time is not None else None
return data
class OrchExecution(UUIDEntityMixin, EntityReprMixin, db.Model):
__tablename__ = 'L_orch_execution'
start_time = db.Column(UtcDateTime(timezone=True),
nullable=False,
default=get_now)
end_time = db.Column(UtcDateTime(timezone=True))
orchestration_id = db.Column(UUID,
db.ForeignKey('D_orchestration.id'),
nullable=False)
target = db.Column(db.JSON)
params = db.Column(db.JSON)
executor_id = db.Column(UUID, db.ForeignKey('D_user.id'))
service_id = db.Column(UUID, db.ForeignKey('D_service.id'))
success = db.Column(db.Boolean)
undo_success = db.Column(db.Boolean)
message = db.Column(db.Text)
server_id = db.Column(UUID, db.ForeignKey('D_server.id'))
parent_step_execution_id = db.Column(UUID)
orchestration = db.relationship("Orchestration")
executor = db.relationship("User")
service = db.relationship("Service")
step_executions = db.relationship("StepExecution",
back_populates="orch_execution",
order_by="StepExecution.start_time")
server = db.relationship("Server", foreign_keys=[server_id])
parent_step_execution = db.relationship(
"StepExecution",
uselist=False,
foreign_keys=[parent_step_execution_id],
primaryjoin="OrchExecution.parent_step_execution_id==StepExecution.id")
# def __init__(self, *args, **kwargs):
# UUIDEntityMixin.__init__(self, **kwargs)
def to_json(self, add_step_exec=False, human=False, split_lines=False):
data = {}
if self.id:
data.update(id=str(self.id))
if self.start_time:
data.update(
start_time=self.start_time.strftime(defaults.DATETIME_FORMAT))
if self.end_time:
data.update(
end_time=self.end_time.strftime(defaults.DATETIME_FORMAT))
if human:
# convert target ids to server names
d = {}
if isinstance(self.target, dict):
for k, v in self.target.items():
if is_iterable_not_string(v):
d[k] = [str(Server.query.get(s) or s) for s in v]
else:
d[k] = str(Server.query.get(v) or v)
elif isinstance(self.target, list):
d = [str(Server.query.get(s) or s) for s in self.target]
else:
d = str(Server.query.get(self.target) or self.target)
data.update(target=d)
if self.executor:
data.update(executor=str(self.executor))
if self.service:
data.update(service=str(self.service))
if self.orchestration:
data.update(
orchestration=dict(id=str(self.orchestration.id),
name=self.orchestration.name,
version=self.orchestration.version))
else:
data.update(orchestration=None)
if self.server:
data.update(
server=dict(id=str(self.server.id), name=self.server.name))
else:
data.update(target=self.target)
if self.orchestration_id or getattr(self.orchestration, 'id',
None):
data.update(
orchestration_id=str(self.orchestration_id or getattr(
self.orchestration, 'id', None)))
if self.executor_id or getattr(self.executor, 'id', None):
data.update(executor_id=str(
self.executor_id or getattr(self.executor, 'id', None)))
if self.service_id or getattr(self.service, 'id', None):
data.update(service_id=str(
self.server_id or getattr(self.service, 'id', None)))
if self.server_id or getattr(self.server, 'id', None):
data.update(server_id=str(self.server_id
or getattr(self.server, 'id', None)))
data.update(params=self.params)
data.update(success=self.success)
data.update(undo_success=self.undo_success)
data.update(message=self.message)
if self.parent_step_execution_id and not add_step_exec:
data.update(
parent_step_execution_id=str(self.parent_step_execution_id))
if add_step_exec:
steps = []
for se in self.step_executions:
se: StepExecution
se_json = se.to_json(human, split_lines=split_lines)
if se.child_orch_execution:
se_json[
'orch_execution'] = se.child_orch_execution.to_json(
add_step_exec=add_step_exec,
split_lines=split_lines,
human=human)
elif se.child_orch_execution_id:
from dimensigon.web.network import get, Response
from dimensigon.network.auth import HTTPBearerAuth
from flask_jwt_extended import create_access_token
params = ['steps']
if human:
params.append('human')
try:
resp = get(se.server,
'api_1_0.orchexecutionresource',
view_data=dict(
execution_id=se.child_orch_execution_id,
params=params))
except Exception as e:
current_app.logger.exception(
f"Exception while trying to acquire orch execution "
f"{se.child_orch_execution_id} from {se.server}")
resp = Response(exception=e)
if resp.ok:
se_json['orch_execution'] = resp.msg
se_json.pop('child_orch_execution_id', None)
steps.append(se_json)
# steps.sort(key=lambda x: x.start_time)
data.update(steps=steps)
return data
@classmethod
def from_json(cls, kwargs):
if 'start_time' in kwargs:
kwargs['start_time'] = datetime.strptime(kwargs.get('start_time'),
defaults.DATETIME_FORMAT)
if 'end_time' in kwargs:
kwargs['end_time'] = datetime.strptime(kwargs.get('end_time'),
defaults.DATETIME_FORMAT)
try:
o = cls.query.get(kwargs.get('id'))
except RuntimeError as e:
o = None
if o:
for k, v in kwargs.items():
if getattr(o, k) != v:
setattr(o, k, v)
return o
else:
return cls(**kwargs)