class FileServerAssociation(DistributedEntityMixin, SoftDeleteMixin, db.Model):
__tablename__ = 'D_file_server_association'
order = 30
file_id = db.Column(typos.UUID, db.ForeignKey('D_file.id'), nullable=False, primary_key=True)
dst_server_id = db.Column(typos.UUID, db.ForeignKey('D_server.id'), nullable=False, primary_key=True)
dest_folder = db.Column(db.Text)
l_mtime = db.Column(db.INTEGER)
file = db.relationship("File")
destination_server = db.relationship("Server")
@property
def destination_folder(self):
return self.dest_folder or self.file.dest_folder or os.path.dirname(self.file.target)
@property
def target(self):
return os.path.join(self.destination_folder, os.path.basename(self.file.target))
def to_json(self, human=False, **kwargs) -> t.Dict:
data = super().to_json(**kwargs)
if human:
data.update({'file': {'target': self.file.target, 'src_server': self.file.source_server.name,
'dst_server': str(self.destination_server.name),
'dest_folder': self.dest_folder}})
else:
data.update({'file_id': str(self.file.id), 'dst_server_id': str(self.destination_server.id),
'dest_folder': self.dest_folder})
return data
@classmethod
def from_json(cls, kwargs) -> 'FileServerAssociation':
kwargs = copy.deepcopy(kwargs)
kwargs['file'] = File.query.get_or_raise(kwargs.get('file_id'))
kwargs['destination_server'] = Server.query.get_or_raise(kwargs.get('dst_server_id'))
super().from_json(kwargs)
try:
o = cls.query.get((kwargs['file_id'], kwargs['dst_server_id']))
except RuntimeError as e:
o = None
if o:
for k, v in kwargs.items():
if getattr(o, k) != v:
setattr(o, k, v)
return o
else:
return cls(**kwargs)
class Service(EntityReprMixin, DistributedEntityMixin, db.Model):
__tablename__ = 'D_service'
order = 40
id = db.Column(UUID, primary_key=True, default=uuid.uuid4)
name = db.Column(db.String(255), nullable=False)
details = db.Column(db.JSON)
created_on = db.Column(UtcDateTime(timezone=True), nullable=False, default=get_now)
last_ping = db.Column(UtcDateTime(timezone=True))
status = db.Column(db.String(40))
orchestrations = db.relationship("Orchestration", secondary="D_service_orchestration",
order_by="ServiceOrchestration.execution_time")
def __init__(self, name: str, details: Kwargs, status: str, created_on: datetime = get_now(),
last_ping: datetime = None, id: uuid.UUID = None, **kwargs):
super().__init__(**kwargs)
self.id = id
self.name = name
self.details = details
self.status = status
self.created_on = created_on
self.last_ping = last_ping
def to_json(self):
return {'id': str(self.id), 'name': self.name, 'details': self.details,
'last_ping': self.last_ping.strftime("%d/%m/%Y %H:%M:%S"),
'status': self.status}
class Vault(DistributedEntityMixin, SoftDeleteMixin, db.Model):
__tablename__ = 'D_vault'
user_id = db.Column(UUID, db.ForeignKey('D_user.id'), primary_key=True)
scope = db.Column(db.String(100), primary_key=True, default='global')
name = db.Column(db.String(100), primary_key=True)
_old_name = db.Column(db.String(100))
value = db.Column(db.PickleType)
user = db.relationship("User")
def to_json(self, human=False, **kwargs):
dto = super().to_json(**kwargs)
dto.update(name=self.name, value=self.value)
if self.scope:
dto.update(scope=self.scope)
if human:
dto.update(user={'id': self.user.id, 'name': self.user.name})
else:
dto.update(user_id=self.user_id or getattr(self.user, 'id', None))
return dto
@classmethod
def from_json(cls, kwargs):
super().from_json(kwargs)
kwargs['user'] = User.query.get_or_raise(kwargs.pop('user_id', None))
o = cls.query.get(
(getattr(kwargs.get('user'),
'id'), kwargs.get('scope', 'global'), kwargs.get('name')))
if o:
for k, v in kwargs.items():
if getattr(o, k) != v:
setattr(o, k, v)
return o
else:
return cls(**kwargs)
@classmethod
def get_variables_from(cls, user: t.Union[Id, User], scope='global'):
if isinstance(user, User):
user_id = user.id
else:
user_id = user
return {
vault.name: vault.value
for vault in cls.query.filter_by(user_id=user_id,
scope=scope).all()
}
def __str__(self):
return f"Vault({self.user}:{self.scope}[{self.name}={self.value}])"
class Gate(UUIDistributedEntityMixin, SoftDeleteMixin, db.Model):
__tablename__ = "D_gate"
order = 2
server_id = db.Column(UUID)
dns = db.Column(db.String(100))
ip = db.Column(IPType)
port = db.Column(db.Integer, nullable=False)
hidden = db.Column(db.Boolean, default=False)
_old_server_id = db.Column(UUID)
__table_args__ = (db.UniqueConstraint('server_id', 'ip', 'port'),
db.UniqueConstraint('server_id', 'dns', 'port'))
server = db.relationship("Server",
backref="gates",
foreign_keys=[server_id],
primaryjoin="Gate.server_id==Server.id")
def __init__(self,
server: 'Server',
port: int = defaults.DEFAULT_PORT,
dns: str = None,
ip: t.Union[str, ipaddress.IPv4Address,
ipaddress.IPv6Address] = None,
**kwargs):
super().__init__(**kwargs)
self.server = server
self.port = port
self.ip = ipaddress.ip_address(ip) if isinstance(ip, str) else ip
self.dns = dns
if not (self.dns or self.ip):
self.dns = server.name
self.hidden = kwargs.get('hidden', False) or False
def __str__(self):
return f'{self.dns or self.ip}:{self.port}'
def to_json(self, human=False, **kwargs):
data = super().to_json(**kwargs)
data.update(server_id=str(self.server.id) if self.server.id else None,
ip=str(self.ip) if self.ip else None,
dns=self.dns,
port=self.port,
hidden=self.hidden)
return data
@classmethod
def from_json(cls, kwargs):
from dimensigon.domain.entities import Server
server = kwargs.pop('server', None)
kwargs = copy.deepcopy(kwargs)
if server:
kwargs['server'] = server
kwargs['ip'] = ipaddress.ip_address(kwargs.get('ip')) if isinstance(
kwargs.get('ip'), str) else kwargs.get('ip')
if 'server_id' in kwargs and kwargs['server_id'] is not None:
# through db.session to allow load from removed entities
kwargs['server'] = db.session.query(Server).filter_by(
id=kwargs.get('server_id')).one()
kwargs.pop('server_id')
return super().from_json(kwargs)
class Server(UUIDistributedEntityMixin, SoftDeleteMixin, db.Model):
__tablename__ = 'D_server'
order = 1
name = db.Column(db.String(255), nullable=False, unique=True)
granules = db.Column(ScalarListType())
_me = db.Column("me", db.Boolean, default=False)
_old_name = db.Column("$$name", db.String(255))
l_ignore_on_lock = db.Column(
"ignore_on_lock", db.Boolean,
default=False) # ignore the server for locking when set
created_on = db.Column(UtcDateTime(timezone=True)) # new in version 3
route: t.Optional[Route] = db.relationship(
"Route",
primaryjoin="Route.destination_id==Server.id",
uselist=False,
back_populates="destination",
cascade="all, delete-orphan")
gates: t.List[Gate]
log_sources = db.relationship("Log",
primaryjoin="Server.id==Log.src_server_id",
back_populates="source_server")
log_destinations = db.relationship(
"Log",
primaryjoin="Server.id==Log.dst_server_id",
back_populates="destination_server")
files = db.relationship("File", back_populates="source_server")
file_server_associations = db.relationship(
"FileServerAssociation", back_populates="destination_server")
software_server_associations = db.relationship("SoftwareServerAssociation",
back_populates="server")
# software_list = db.relationship("SoftwareServerAssociation", back_populates="server")
def __init__(self,
name: str,
granules: t.List[str] = None,
dns_or_ip: t.Union[str, ipaddress.IPv4Address,
ipaddress.IPv6Address] = None,
port: int = None,
gates: t.List[t.Union[TGate, t.Dict[str, t.Any]]] = None,
me: bool = False,
created_on=None,
**kwargs):
super().__init__(**kwargs)
self.name = name
if port or dns_or_ip:
self.add_new_gate(dns_or_ip or self.name, port
or defaults.DEFAULT_PORT)
if gates:
for gate in gates:
if isinstance(gate, str):
if ':' in gate:
self.add_new_gate(*gate.split(':'))
else:
self.add_new_gate(gate, defaults.DEFAULT_PORT)
elif isinstance(gate, tuple):
self.add_new_gate(*gate)
elif isinstance(gate, dict):
gate['server'] = self
Gate.from_json(gate)
assert 'all' not in (granules or [])
self.granules = granules or []
self._me = me
self.created_on = created_on or get_now()
# create an empty route
if not me and not self.deleted:
Route(self)
@property
def external_gates(self):
e_g = []
for g in self.gates:
if not g.ip:
try:
ip = ipaddress.ip_address(
socket.getaddrinfo(g.dns,
0,
family=socket.AF_INET,
proto=socket.IPPROTO_TCP)[0][4][0])
except socket.gaierror:
e_g.append(g)
continue
except KeyError:
e_g.append(g)
continue
else:
ip = g.ip
if not ip.is_loopback:
e_g.append(g)
return e_g
@property
def localhost_gates(self):
l_g = []
for g in self.gates:
if not g.ip:
try:
ip = ipaddress.ip_address(
socket.getaddrinfo(g.dns,
0,
family=socket.AF_INET,
proto=socket.IPPROTO_TCP)[0][4][0])
except socket.gaierror:
continue
except KeyError:
continue
else:
ip = g.ip
if ip.is_loopback:
l_g.append(g)
return l_g
@property
def hidden_gates(self):
hg = [g for g in self.gates if g.hidden]
hg.sort(key=lambda x: x.last_modified_at or get_now())
return hg
def add_new_gate(self,
dns_or_ip: t.Union[str, ipaddress.IPv4Address,
ipaddress.IPv6Address],
port: int,
hidden=None):
ip = None
dns = None
if dns_or_ip:
try:
ip = ipaddress.ip_address(dns_or_ip)
except ValueError:
dns = dns_or_ip
return Gate(server=self, port=port, dns=dns, ip=ip, hidden=hidden)
def __str__(self, ):
return f"{self.name}"
def url(self, view: str = None, **values) -> str:
"""
generates the full url to access the server. Uses url_for to generate the full_path.
Parameters
----------
view
values
Raises
-------
ConnectionError:
if server is unreachable
"""
try:
scheme = 'http' if current_app.dm and 'keyfile' not in current_app.dm.config.http_conf else 'https'
except:
scheme = 'https'
gate = None
route = self.route
if self._me and (route is None or route.cost is None):
if len(self.localhost_gates) > 0:
gate = self.localhost_gates[0]
else:
# current_app.logger.warning(
# f"No localhost set for '{self}'. Trying connection through another gate")
if len(self.gates) == 0:
raise RuntimeError(f"No gate set for server '{self}'")
gate = self.gates[0]
elif getattr(route, 'cost', None) == 0:
gate = route.gate
elif getattr(route, 'proxy_server', None):
gate = getattr(getattr(route.proxy_server, 'route', None), 'gate',
None)
if not gate:
raise errors.UnreachableDestination(self,
getattr(g, 'server', None))
root_path = f"{scheme}://{gate}"
if view is None:
return root_path
else:
with current_app.test_request_context():
return root_path + url_for(view, **values)
@classmethod
def get_neighbours(cls, exclude: t.Union[t.Union[Id, 'Server'], t.List[t.Union[Id, 'Server']]] = None,
session=None) -> \
t.List[
'Server']:
"""returns neighbour servers
Args:
alive: if True, returns neighbour servers inside the cluster
Returns:
"""
if session:
query = session.query(cls).filter_by(deleted=0)
else:
query = cls.query
query = query.join(cls.route).filter(Route.cost == 0)
if exclude:
if isinstance(exclude, list):
if isinstance(exclude[0], Server):
query = query.filter(
Server.id.notin_([s.id for s in exclude]))
else:
query = query.filter(Server.id.notin_(exclude))
elif isinstance(exclude, Server):
query = query.filter(Server.id != exclude.id)
else:
query = query.filter(Server.id != exclude)
return query.order_by(cls.name).all()
@classmethod
def get_not_neighbours(cls, session=None) -> t.List['Server']:
if session:
query = session.query(cls).filter_by(deleted=0)
else:
query = cls.query
return query.outerjoin(cls.route).filter(
or_(or_(Route.cost > 0, Route.cost == None),
cls.route == None)).filter(cls._me == False).order_by(
cls.name).all()
@classmethod
def get_reachable_servers(
cls,
alive=False,
exclude: t.Union[t.Union[Id, 'Server'],
t.List[t.Union[Id, 'Server']]] = None
) -> t.List['Server']:
"""returns list of reachable servers
Args:
alive: if True, returns servers inside the cluster
exclude: filter to exclude servers
Returns:
list of all reachable servers
"""
query = cls.query.join(cls.route).filter(Route.cost.isnot(None))
if exclude:
if is_iterable_not_string(exclude):
c_exclude = [
e.id if isinstance(e, Server) else e for e in exclude
]
else:
c_exclude = [
exclude.id if isinstance(exclude, Server) else exclude
]
query = query.filter(Server.id.notin_(c_exclude))
if alive:
query = query.filter(
Server.id.in_([
iden
for iden in current_app.dm.cluster_manager.get_alive()
]))
return query.order_by(Server.name).all()
def to_json(self,
add_gates=False,
human=False,
add_ignore=False,
**kwargs):
data = super().to_json(**kwargs)
data.update({
'name':
self.name,
'granules':
self.granules,
'created_on':
self.created_on.strftime(defaults.DATETIME_FORMAT)
})
if add_gates:
data.update(gates=[])
for g in self.gates:
json_gate = g.to_json(human=human)
json_gate.pop('server_id', None)
json_gate.pop('server', None) # added to remove when human set
data['gates'].append(json_gate)
if add_ignore:
data.update(ignore_on_lock=self.l_ignore_on_lock or False)
return data
@classmethod
def from_json(cls, kwargs) -> 'Server':
kwargs = copy.deepcopy(kwargs)
gates = kwargs.pop('gates', [])
if 'created_on' in kwargs:
kwargs['created_on'] = dt.datetime.strptime(
kwargs['created_on'], defaults.DATETIME_FORMAT)
server = super().from_json(kwargs)
for gate in gates:
gate.update(server=server)
Gate.from_json(gate)
return server
@classmethod
def get_current(cls, session=None) -> 'Server':
if session is None:
session = db.session
return session.query(cls).filter_by(_me=True).filter_by(
deleted=False).one()
@staticmethod
def set_initial(session=None, gates=None) -> Id:
logger = logging.getLogger('dm.db')
if session is None:
session = db.session
server = session.query(Server).filter_by(_me=True).all()
if len(server) == 0:
try:
server_name = current_app.config.get(
'SERVER_NAME') or defaults.HOSTNAME
except:
server_name = defaults.HOSTNAME
if gates is None:
gates = [(ip, defaults.DEFAULT_PORT) for ip in get_ips()]
server = Server(name=server_name, gates=gates, me=True)
logger.info(
f'Creating Server {server.name} with the following gates: {gates}'
)
session.add(server)
return server.id
elif len(server) > 1:
raise ValueError('Multiple servers found as me.')
else:
return server[0].id
def ignore_on_lock(self, value: bool):
if value != self.l_ignore_on_lock:
from dimensigon.domain.entities import bypass_datamark_update
with bypass_datamark_update:
self.l_ignore_on_lock = value
def set_route(self, proxy_route, gate=None, cost=None):
if self.route is None:
self.route = Route(destination=self)
if isinstance(proxy_route, RouteContainer):
self.route.set_route(proxy_route)
elif isinstance(proxy_route, Route):
assert proxy_route.destination == self
self.route.set_route(
RouteContainer(proxy_route.proxy_server, proxy_route.gate,
proxy_route.cost))
else:
self.route.set_route(RouteContainer(proxy_route, gate, cost))
def delete(self):
super().delete()
for g in self.gates:
g.delete()
for l in self.log_sources:
l.delete()
for l in self.log_destinations:
l.delete()
for ssa in self.software_server_associations:
ssa.delete()
for fsa in self.file_server_associations:
fsa.delete()
for f in self.files:
f.delete()
class File(UUIDistributedEntityMixin, SoftDeleteMixin, db.Model):
__tablename__ = 'D_file'
order = 20
src_server_id = db.Column(typos.UUID, db.ForeignKey('D_server.id'), nullable=False)
target = db.Column(db.Text, nullable=False)
dest_folder = db.Column(db.Text)
_old_target = db.Column("$$target", db.Text)
l_mtime = db.Column(db.INTEGER)
source_server = db.relationship("Server")
destinations: t.List[FileServerAssociation] = db.relationship("FileServerAssociation", lazy='joined')
__table_args__ = (db.UniqueConstraint('src_server_id', 'target'),)
query_class = QueryWithSoftDelete
def __init__(self, source_server: Server, target: str,
dest_folder=None, destination_servers: Destination_Servers = None, **kwargs):
super().__init__(**kwargs)
self.source_server = source_server
self.target = target
self.dest_folder = dest_folder
dest = []
for ds in destination_servers or []:
if isinstance(ds, Server):
dest.append(FileServerAssociation(file=self, destination_server=ds))
elif isinstance(ds, dict):
dest.append(
FileServerAssociation(file=self, destination_server=Server.query.get(ds.get('dst_server_id')),
dest_folder=ds.get('dest_folder')))
elif isinstance(ds, tuple):
if isinstance(ds[0], Server):
dest.append(FileServerAssociation(file=self, destination_server=ds[0], dest_folder=ds[1]))
else:
dest.append(
FileServerAssociation(file=self, destination_server=Server.query.get(ds[0]), dest_folder=ds[1]))
self.destinations = dest
def __str__(self):
return f"{self.source_server}:{self.target}"
def to_json(self, human=False, destinations=False, include: t.List[str] = None,
exclude: t.List[str] = None, **kwargs):
data = super().to_json(**kwargs)
if self.source_server.id is None:
raise RuntimeError('Set ids for servers before')
data.update(target=self.target, dest_folder=self.dest_folder)
if human:
data.update(src_server=str(self.source_server.name))
if destinations:
dest = []
for d in self.destinations:
dest.append(dict(dst_server=d.destination_server.name, dest_folder=d.dest_folder))
data.update(destinations=dest)
else:
data.update(src_server_id=str(self.source_server.id))
if destinations:
dest = []
for d in self.destinations:
dest.append(dict(dst_server_id=d.destination_server.id, dest_folder=d.dest_folder))
data.update(destinations=dest)
if include:
data = {k: v for k, v in data.items() if k in include}
if exclude:
data = {k: v for k, v in data.items() if k not in exclude}
return data
@classmethod
def from_json(cls, kwargs) -> 'File':
kwargs = copy.deepcopy(kwargs)
kwargs['source_server'] = Server.query.get_or_raise(kwargs.pop('src_server_id'))
return super().from_json(kwargs)
def delete(self):
for d in self.destinations:
d.delete()
return super().delete()
class Step(UUIDistributedEntityMixin, db.Model):
__tablename__ = "D_step"
order = 30
orchestration_id = db.Column(UUID, db.ForeignKey('D_orchestration.id'), nullable=False)
action_template_id = db.Column(UUID, db.ForeignKey('D_action_template.id'))
undo = db.Column(db.Boolean, nullable=False)
step_stop_on_error = db.Column("stop_on_error", db.Boolean)
step_stop_undo_on_error = db.Column("stop_undo_on_error", db.Boolean)
step_undo_on_error = db.Column("undo_on_error", db.Boolean)
step_expected_stdout = db.Column("expected_stdout", db.Text)
step_expected_stderr = db.Column("expected_stderr", db.Text)
step_expected_rc = db.Column("expected_rc", db.Integer)
step_system_kwargs = db.Column("system_kwargs", db.JSON)
target = db.Column(ScalarListType(str))
created_on = db.Column(UtcDateTime(), nullable=False, default=get_now)
step_action_type = db.Column("action_type", typos.Enum(ActionType))
step_code = db.Column("code", db.Text)
step_post_process = db.Column("post_process", db.Text)
step_pre_process = db.Column("pre_process", db.Text)
step_name = db.Column("name", db.String(40))
step_schema = db.Column("schema", db.JSON)
step_description = db.Column("description", db.Text)
orchestration = db.relationship("Orchestration", primaryjoin="Step.orchestration_id==Orchestration.id",
back_populates="steps")
action_template = db.relationship("ActionTemplate", primaryjoin="Step.action_template_id==ActionTemplate.id",
backref="steps")
parent_steps = db.relationship("Step", secondary="D_step_step",
primaryjoin="D_step.c.id==D_step_step.c.step_id",
secondaryjoin="D_step.c.id==D_step_step.c.parent_step_id",
back_populates="children_steps")
children_steps = db.relationship("Step", secondary="D_step_step",
primaryjoin="D_step.c.id==D_step_step.c.parent_step_id",
secondaryjoin="D_step.c.id==D_step_step.c.step_id",
back_populates="parent_steps")
def __init__(self, orchestration: 'Orchestration', undo: bool, action_template: 'ActionTemplate' = None,
action_type: ActionType = None, code: str = None, pre_process: str = None, post_process: str = None,
stop_on_error: bool = None, stop_undo_on_error: bool = None, undo_on_error: bool = None,
expected_stdout: t.Optional[str] = None,
expected_stderr: t.Optional[str] = None,
expected_rc: t.Optional[int] = None,
schema: t.Dict[str, t.Any] = None,
system_kwargs: t.Dict[str, t.Any] = None,
parent_steps: t.List['Step'] = None, children_steps: t.List['Step'] = None,
target: t.Union[str, t.Iterable[str]] = None, name: str = None, description: str = None, rid=None,
**kwargs):
super().__init__(**kwargs)
assert undo in (False, True)
if action_template is not None:
assert action_type is None
else:
assert action_type is not None
self.undo = undo
self.step_stop_on_error = stop_on_error if stop_on_error is not None else kwargs.pop('step_stop_on_error', None)
self.step_stop_undo_on_error = stop_undo_on_error if stop_undo_on_error is not None else kwargs.pop(
'step_stop_undo_on_error', None)
self.step_undo_on_error = undo_on_error if undo_on_error is not None else kwargs.pop('step_undo_on_error', None)
self.action_template = action_template
self.step_action_type = action_type if action_type is not None else kwargs.pop('step_action_type', None)
expected_stdout = expected_stdout if expected_stdout is not None else kwargs.pop(
'step_expected_stdout', None)
self.step_expected_stdout = '\n'.join(expected_stdout) if is_iterable_not_string(
expected_stdout) else expected_stdout
expected_stderr = expected_stderr if expected_stderr is not None else kwargs.pop(
'step_expected_stderr', None)
self.step_expected_stderr = '\n'.join(expected_stderr) if is_iterable_not_string(
expected_stderr) else expected_stderr
self.step_expected_rc = expected_rc if expected_rc is not None else kwargs.pop('step_expected_rc', None)
self.step_schema = schema if schema is not None else kwargs.pop('step_schema', None) or {}
self.step_system_kwargs = system_kwargs if system_kwargs is not None else kwargs.pop('step_system_kwargs',
None) or {}
code = code if code is not None else kwargs.pop('step_code', None)
self.step_code = '\n'.join(code) if is_iterable_not_string(code) else code
post_process = post_process if post_process is not None else kwargs.pop('step_post_process', None)
self.step_post_process = '\n'.join(post_process) if is_iterable_not_string(post_process) else post_process
pre_process = pre_process if pre_process is not None else kwargs.pop('step_pre_process', None)
self.step_pre_process = '\n'.join(pre_process) if is_iterable_not_string(pre_process) else pre_process
self.orchestration = orchestration
self.parent_steps = parent_steps or []
self.children_steps = children_steps or []
if self.undo is False:
if target is None:
self.target = ['all']
else:
self.target = [target] if isinstance(target, str) else (target if len(target) > 0 else ['all'])
else:
if target:
raise errors.BaseError('target must not be set when creating an UNDO step')
self.created_on = kwargs.get('created_on') or get_now()
self.step_name = name if name is not None else kwargs.pop('step_name', None)
description = description if description is not None else kwargs.pop('step_description', None)
self.step_description = '\n'.join(description) if is_iterable_not_string(description) else description
self.rid = rid # used when creating an Orchestration
@orm.reconstructor
def init_on_load(self):
self.system_kwargs = self.system_kwargs or {}
@property
def parents(self):
return self.parent_steps
@property
def children(self):
return self.children_steps
@property
def parent_undo_steps(self):
return [s for s in self.parent_steps if s.undo == True]
@property
def children_undo_steps(self):
return [s for s in self.children_steps if s.undo == True]
@property
def parent_do_steps(self):
return [s for s in self.parent_steps if s.undo == False]
@property
def children_do_steps(self):
return [s for s in self.children_steps if s.undo == False]
@property
def stop_on_error(self):
return self.step_stop_on_error if self.step_stop_on_error is not None else self.orchestration.stop_on_error
@stop_on_error.setter
def stop_on_error(self, value):
self.step_stop_on_error = value
@property
def stop_undo_on_error(self):
return self.step_stop_undo_on_error if self.step_stop_undo_on_error is not None \
else self.orchestration.stop_undo_on_error
@stop_undo_on_error.setter
def stop_undo_on_error(self, value):
self.step_stop_undo_on_error = value
@property
def undo_on_error(self):
if self.undo:
return None
return self.step_undo_on_error if self.step_undo_on_error is not None else self.orchestration.undo_on_error
@undo_on_error.setter
def undo_on_error(self, value):
self.step_undo_on_error = value
@property
def schema(self):
if self.action_template:
schema = dict(ChainMap(self.step_schema, self.action_template.schema))
else:
schema = dict(self.step_schema)
if self.action_type == ActionType.ORCHESTRATION:
from .orchestration import Orchestration
mapping = schema.get('mapping', {})
o = Orchestration.get(mapping.get('orchestration', None), mapping.get('version', None))
if isinstance(o, Orchestration):
orch_schema = o.schema
i = schema.get('input', {})
i.update(orch_schema.get('input', {}))
schema.update({'input': i})
m = schema.get('mapping', {})
m.update(orch_schema.get('mapping', {}))
schema.update({'mapping': m})
r = schema.get('required', [])
[r.append(k) for k in orch_schema.get('required', []) if k not in r]
schema.update({'required': r})
o = schema.get('output', [])
[o.append(k) for k in orch_schema.get('output', []) if k not in o]
schema.update({'output': o})
return schema
@schema.setter
def schema(self, value):
self.step_schema = value
@property
def system_kwargs(self):
if self.action_template:
return dict(ChainMap(self.step_system_kwargs, self.action_template.system_kwargs))
else:
return dict(self.step_system_kwargs)
@system_kwargs.setter
def system_kwargs(self, value):
self.step_system_kwargs = value
@property
def code(self):
if self.step_code is None and self.action_template:
return self.action_template.code
else:
return self.step_code
@code.setter
def code(self, value):
self.step_code = value
@property
def action_type(self):
if self.step_action_type is None and self.action_template:
return self.action_template.action_type
else:
return self.step_action_type
@action_type.setter
def action_type(self, value):
self.step_action_type = value
@property
def post_process(self):
if self.step_post_process is None and self.action_template:
return self.action_template.post_process
else:
return self.step_post_process
@post_process.setter
def post_process(self, value):
self.step_post_process = value
@property
def pre_process(self):
if self.step_pre_process is None and self.action_template:
return self.action_template.pre_process
else:
return self.step_pre_process
@pre_process.setter
def pre_process(self, value):
self.step_pre_process = value
@property
def expected_stdout(self):
if self.step_expected_stdout is None and self.action_template:
return self.action_template.expected_stdout
else:
return self.step_expected_stdout
@expected_stdout.setter
def expected_stdout(self, value):
self.step_expected_stdout = value
@property
def expected_stderr(self):
if self.step_expected_stderr is None and self.action_template:
return self.action_template.expected_stderr
else:
return self.step_expected_stderr
@expected_stderr.setter
def expected_stderr(self, value):
if value == self.action_template.expected_stderr:
self.step_expected_stderr = None
else:
self.step_expected_stderr = value
@property
def expected_rc(self):
if self.step_expected_rc is None and self.action_template:
return self.action_template.expected_rc
else:
return self.step_expected_rc
@expected_rc.setter
def expected_rc(self, value):
if self.action_template and value == self.action_template.expected_rc:
self.step_expected_rc = None
else:
self.step_expected_rc = value
@property
def name(self):
if self.step_name is None and self.action_template:
return str(self.action_template)
else:
return self.step_name
@name.setter
def name(self, value):
self.step_name = value
@property
def description(self):
if self.step_description is None and self.action_template:
return str(self.action_template)
else:
return self.step_description
@description.setter
def description(self, value):
self.step_description = value
def eq_imp(self, other):
"""
two steps are equal if they execute the same code with the same parameters even if they are from different
orchestrations or they are in the same orchestration with different positions
Parameters
----------
other: Step
Returns
-------
result: bool
Notes
-----
_id and _orchestration are not compared
"""
if isinstance(other, self.__class__):
return all([self.undo == other.undo,
self.stop_on_error == other.stop_on_error,
self.stop_undo_on_error == other.stop_undo_on_error,
self.undo_on_error == other.undo_on_error,
self.expected_stdout == other.expected_stdout,
self.expected_stderr == other.expected_stderr,
self.expected_rc == other.expected_rc,
self.system_kwargs == other.system_kwargs,
self.code == other.code,
self.post_process == other.post_process,
self.pre_process == other.pre_process,
self.action_type == other.action_type,
])
else:
raise NotImplemented
def __str__(self):
return self.name or self.id
def __repr__(self):
return ('Undo ' if self.undo else '') + self.__class__.__name__ + ' ' + str(getattr(self, 'id', ''))
def _add_parents(self, parents):
for step in parents:
if not step in self.parent_steps:
self.parent_steps.append(step)
def _remove_parents(self, parents):
for step in parents:
if step in self.parent_steps:
self.parent_steps.remove(step)
def _add_children(self, children):
for step in children:
if not step in self.children_steps:
self.children_steps.append(step)
def _remove_children(self, children):
for step in children:
if step in self.children_steps:
self.children_steps.remove(step)
def to_json(self, add_action=False, split_lines=False, **kwargs):
data = super().to_json(**kwargs)
if getattr(self.orchestration, 'id', None):
data.update(orchestration_id=str(self.orchestration.id))
if getattr(self.action_template, 'id', None):
if add_action:
data.update(action_template=self.action_template.to_json(split_lines=split_lines))
else:
data.update(action_template_id=str(self.action_template.id))
data.update(undo=self.undo)
data.update(stop_on_error=self.step_stop_on_error) if self.step_stop_on_error is not None else None
data.update(
stop_undo_on_error=self.step_stop_undo_on_error) if self.step_stop_undo_on_error is not None else None
data.update(undo_on_error=self.step_undo_on_error) if self.step_undo_on_error is not None else None
if self.step_schema:
data.update(schema=self.step_schema)
if self.step_expected_stdout is not None:
data.update(
expected_stdout=self.step_expected_stdout.split('\n') if split_lines else self.step_expected_stdout)
if self.step_expected_stderr is not None:
data.update(
expected_stderr=self.step_expected_stderr.split('\n') if split_lines else self.step_expected_stderr)
data.update(expected_rc=self.step_expected_rc) if self.step_expected_rc is not None else None
data.update(system_kwargs=self.step_system_kwargs) if self.step_system_kwargs is not None else None
data.update(parent_step_ids=[str(step.id) for step in self.parents])
if self.step_code is not None:
data.update(code=self.step_code.split('\n') if split_lines else self.step_code)
data.update(action_type=self.step_action_type.name) if self.step_action_type is not None else None
if self.step_post_process is not None:
data.update(post_process=self.step_post_process.split('\n') if split_lines else self.step_post_process)
if self.step_pre_process is not None:
data.update(pre_process=self.step_pre_process.split('\n') if split_lines else self.step_pre_process)
data.update(created_on=self.created_on.strftime(defaults.DATETIME_FORMAT))
if self.step_description is not None:
data.update(description=self.step_description.split('\n') if split_lines else self.step_description)
if self.step_name is not None:
data.update(name=self.step_name)
return data
@classmethod
def from_json(cls, kwargs):
from dimensigon.domain.entities import ActionTemplate, Orchestration
kwargs = dict(kwargs)
if 'orchestration_id' in kwargs:
ident = kwargs.pop('orchestration_id')
kwargs['orchestration'] = db.session.query(Orchestration).get(ident)
if kwargs['orchestration'] is None:
raise errors.EntityNotFound('Orchestration', ident=ident)
if 'action_template_id' in kwargs:
ident = kwargs.pop('action_template_id')
kwargs['action_template'] = db.session.query(ActionTemplate).get(ident)
if kwargs['action_template'] is None:
raise errors.EntityNotFound('ActionTemplate', ident=ident)
if 'action_type' in kwargs:
kwargs['action_type'] = ActionType[kwargs.pop('action_type')]
if 'created_on' in kwargs:
kwargs['created_on'] = datetime.datetime.strptime(kwargs['created_on'], defaults.DATETIME_FORMAT)
kwargs['parent_steps'] = []
for parent_step_id in kwargs.pop('parent_step_ids', []):
ps = Step.query.get(parent_step_id)
if ps:
kwargs['parent_steps'].append(ps)
else:
raise errors.EntityNotFound('Step', parent_step_id)
return super().from_json(kwargs)
class Route(db.Model):
__tablename__ = 'L_route'
destination_id = db.Column(UUID,
db.ForeignKey('D_server.id'),
primary_key=True,
nullable=False)
proxy_server_id = db.Column(UUID, db.ForeignKey('D_server.id'))
gate_id = db.Column(UUID, db.ForeignKey('D_gate.id'))
cost = db.Column(db.Integer)
destination = db.relationship("Server",
foreign_keys=[destination_id],
back_populates="route",
lazy='joined')
proxy_server = db.relationship("Server",
foreign_keys=[proxy_server_id],
lazy='joined')
gate = db.relationship("Gate", foreign_keys=[gate_id], lazy='joined')
def __init__(self,
destination: 'Server',
proxy_server_or_gate: t.Union['Server', 'Gate'] = None,
cost: int = None):
# avoid cycle import
from dimensigon.domain.entities import Server
self.destination = destination
if isinstance(proxy_server_or_gate, Server):
proxy_server = proxy_server_or_gate
gate = None
else:
proxy_server = None
gate = proxy_server_or_gate
if proxy_server:
if proxy_server == destination:
raise ValueError(
'You must specify a gate when proxy_server equals destination'
)
else:
if cost is None or cost == 0:
raise ValueError(
"Cost must be specified and greater than 0 when proxy_server"
)
self.proxy_server = proxy_server
self.cost = cost
elif gate:
# check if gate is from neighbour or from a proxy server
if destination == gate.server:
if cost is not None and cost > 0:
raise ValueError(
"Cost must be set to 0 when defining route for a neighbour"
)
self.gate = gate
self.cost = 0
else:
if cost is None or cost <= 0:
raise ValueError(
"Cost must be specified and greater than 0 when gate is from a proxy_server"
)
else:
self.proxy_server = gate.server
self.cost = cost
elif cost == 0:
# find a gateway and set that gateway as default
if len(destination.external_gates) == 1:
self.gate = destination.external_gates[0]
self.cost = 0
else:
for gate in destination.external_gates:
if check_host(gate.dns or str(gate.ip),
gate.port,
timeout=1,
retry=3,
delay=0.5):
self.gate = gate
self.cost = 0
break
# if not (self.gate or self.proxy_server):
# raise ValueError('Not a valid route')
def validate_route(self, rc: RouteContainer):
if rc.proxy_server:
if not (rc.gate is None and rc.cost > 0):
raise errors.InvalidRoute(self.destination, rc)
if rc.proxy_server._me:
raise errors.InvalidRoute(self.destination, rc)
elif rc.gate:
if not rc.cost == 0:
raise errors.InvalidRoute(self.destination, rc)
else:
if rc.cost is not None:
raise errors.InvalidRoute(self.destination, rc)
def set_route(self, rc: RouteContainer):
self.validate_route(rc)
self.proxy_server, self.gate, self.cost = rc
def __str__(self):
return f"{self.destination} -> " \
f"{self.proxy_server or self.gate}, {self.cost}"
def __repr__(self):
return f"Route({self.to_json()})"
def to_json(self, human=False):
if not self.destination.id or (self.proxy_server
and not self.proxy_server.id) or (
self.gate and not self.gate.id):
raise RuntimeError("commit object before dump to json")
if human:
return {
'destination':
str(self.destination) if self.destination else None,
'proxy_server':
str(self.proxy_server) if self.proxy_server else None,
'gate': str(self.gate) if self.gate else None,
'cost': self.cost
}
else:
return {
'destination_id': self.destination_id,
'proxy_server_id': self.proxy_server_id,
'gate_id': self.gate_id,
'cost': self.cost
}
class Log(UUIDistributedEntityMixin, SoftDeleteMixin, db.Model):
__tablename__ = 'D_log'
src_server_id = db.Column(typos.UUID,
db.ForeignKey('D_server.id'),
nullable=False)
target = db.Column(db.Text, nullable=False)
include = db.Column(db.Text)
exclude = db.Column(db.Text)
dst_server_id = db.Column(typos.UUID,
db.ForeignKey('D_server.id'),
nullable=False)
mode = db.Column(typos.Enum(Mode))
dest_folder = db.Column(db.Text)
recursive = db.Column(db.Boolean, default=False)
_old_target = db.Column("$$target", db.Text)
source_server = db.relationship("Server",
foreign_keys=[src_server_id],
back_populates="log_sources")
destination_server = db.relationship("Server",
foreign_keys=[dst_server_id],
back_populates="log_destinations")
__table_args__ = (db.UniqueConstraint('src_server_id', 'target',
'dst_server_id'), )
def __init__(self,
source_server: 'Server',
target: str,
destination_server: 'Server',
dest_folder=None,
include=None,
exclude=None,
recursive=False,
mode=Mode.REPO_MIRROR,
**kwargs):
super().__init__(**kwargs)
self.source_server = source_server
self.target = target
self.destination_server = destination_server
self.dest_folder = dest_folder
if self.dest_folder is None:
self.mode = mode
else:
self.mode = Mode.FOLDER
self.include = include
self._re_include = re.compile(self.include or '')
self.exclude = exclude
self._re_exclude = re.compile(self.exclude or '^$')
self.recursive = recursive
def __str__(self):
return f"{self.source_server}:{self.target} -> {self.destination_server}:{self.dest_folder}"
def to_json(self,
human=False,
include: t.List[str] = None,
exclude: t.List[str] = None,
**kwargs):
data = super().to_json(**kwargs)
if self.source_server.id is None or self.destination_server.id is None:
raise RuntimeError('Set ids for servers before')
data.update(target=self.target,
include=self.include,
exclude=self.exclude,
dest_folder=self.dest_folder,
recursive=self.recursive,
mode=self.mode.name)
if human:
data.update(src_server=str(self.source_server.name),
dst_server=str(self.destination_server.name))
else:
data.update(src_server_id=str(self.source_server.id),
dst_server_id=str(self.destination_server.id))
if include:
data = {k: v for k, v in data.items() if k in include}
if exclude:
data = {k: v for k, v in data.items() if k not in exclude}
return data
@classmethod
def from_json(cls, kwargs) -> 'Log':
from dimensigon.domain.entities import Server
kwargs = copy.deepcopy(kwargs)
kwargs['mode'] = Mode[kwargs.get('mode')]
kwargs['source_server'] = Server.query.get(kwargs.pop('src_server_id'))
kwargs['destination_server'] = Server.query.get(
kwargs.pop('dst_server_id'))
return super().from_json(kwargs)
class Orchestration(UUIDistributedEntityMixin, db.Model):
__tablename__ = 'D_orchestration'
order = 20
name = db.Column(db.String(80), nullable=False)
version = db.Column(db.Integer, nullable=False)
description = db.Column(db.Text)
stop_on_error = db.Column(db.Boolean)
stop_undo_on_error = db.Column(db.Boolean)
undo_on_error = db.Column(db.Boolean)
created_at = db.Column(UtcDateTime(timezone=True), default=get_now)
steps = db.relationship(
"Step",
primaryjoin="Step.orchestration_id==Orchestration.id",
back_populates="orchestration")
__table_args__ = (db.UniqueConstraint('name',
'version',
name='D_orchestration_uq01'), )
def __init__(self,
name: str,
version: int,
description: t.Optional[str] = None,
steps: t.List[Step] = None,
stop_on_error: bool = True,
stop_undo_on_error: bool = True,
undo_on_error: bool = True,
dependencies: Tdependencies = None,
created_at=None,
**kwargs):
super().__init__(**kwargs)
self.name = name
self.version = version
self.description = '\n'.join(description) if is_iterable_not_string(
description) else description
self.steps = steps or []
assert isinstance(stop_on_error, bool)
self.stop_on_error = stop_on_error
assert isinstance(stop_undo_on_error, bool)
self.stop_undo_on_error = stop_undo_on_error
assert isinstance(undo_on_error, bool)
self.undo_on_error = undo_on_error
self.created_at = created_at or get_now()
if dependencies:
self.set_dependencies(dependencies)
else:
self._graph = DAG()
@orm.reconstructor
def init_on_load(self):
self._graph = DAG()
for step in self.steps:
if step.parents:
for p in step.parents:
self._graph.add_edge(p, step)
else:
self._graph.add_node(step)
def set_dependencies(self, dependencies: Tdependencies):
edges = []
find = lambda id_: next(
(step for step in self.steps if step.id == id_))
if isinstance(dependencies, t.Dict):
for k, v in dependencies.items():
try:
step_from = find(k)
except StopIteration:
raise ValueError(f'id step {k} not found in steps list')
for child_id in v:
try:
step_to = find(child_id)
except StopIteration:
raise ValueError(
f'id step {child_id} not found in steps list')
edges.append((step_from, step_to))
elif isinstance(dependencies, t.Iterable):
edges = dependencies
else:
raise ValueError(
f'dependencies must be a dict like object or an iterable of tuples. '
f'See the docs for more information')
self._graph = DAG(edges)
@property
def parents(self) -> t.Dict[Step, t.List[Step]]:
return self._graph.pred
@property
def children(self) -> t.Dict[Step, t.List[Step]]:
return self._graph.succ
@property
def dependencies(self) -> t.Dict[Step, t.List[Step]]:
return self.children
@property
def root(self) -> t.List[Step]:
return self._graph.root
@property
def target(self) -> t.Set[str]:
target = set()
for step in self.steps:
if step.undo is False:
target.update(step.target)
return target
def _step_exists(self, step: t.Union[t.List[Step], Step]):
"""Checks if all the steps belong to the current orchestration
Parameters
----------
step: list[Step]|Step
Step or list of steps to be evaluated
Returns
-------
None
Raises
------
ValueError: if any step passed as argument is not in the orchestration
"""
if not isinstance(step, list):
steps = [step]
else:
steps = step
for step in steps:
if not (step in self._graph.nodes and step.orchestration is self):
raise ValueError(f'{step} is NOT from this orchestration')
def _check_dependencies(self, step, parents=None, children=None):
"""
Checks if the dependencies that are going to be added accomplish the business rules. These rules are:
1. a 'do' Step cannot be preceded for an 'undo' Step
2. cannot be cycles inside the orchestration
Parameters
----------
step: Step
step to be evaluated
parents: list[Step]
parent steps to be added
children: list[Step]
children steps to be added
Raises
------
ValueError
if the rule 1 is not passed
CycleError
if the rule 2 is not passed
"""
parents = parents or []
children = children or []
if parents:
if any([p.undo for p in parents]) and not step.undo:
attr = 'rid' if step.rid is not None else 'id'
raise errors.ParentUndoError(getattr(
step, attr), [getattr(s, attr) for s in parents if s.undo])
if children:
if any([not c.undo for c in children]) and step.undo:
attr = 'rid' if step.rid is not None else 'id'
raise errors.ChildDoError(
getattr(step, attr),
[getattr(s, attr) for s in children if s.undo])
g = self._graph.copy()
g.add_edges_from([(p, step) for p in parents])
g.add_edges_from([(step, c) for c in children])
if g.is_cyclic():
raise errors.CycleError()
def add_step(self, *args, parents=None, children=None, **kwargs) -> Step:
"""Allows to add step into the orchestration
:param args: args passed to Step: undo, action_template.
:param parents: list of parent steps.
:param children: list of children steps.
:param kwargs: keyword arguments passed to the Step
:return: The step created.
"""
parents = parents or []
children = children or []
s = Step(None, *args, **kwargs)
self._step_exists(parents + children)
self._check_dependencies(s, parents, children)
s.orchestration = self
self._graph.add_node(s)
self.add_parents(s, parents)
self.add_children(s, children)
return s
def delete_step(self, step: Step) -> 'Orchestration':
"""
Allows to remove a Step from the orchestration
Parameters
----------
step Step: step to remove from the orchestration
"""
self._step_exists(step)
i = self.steps.index(step)
self.steps.pop(i)
self._graph.remove_node(step)
return self
def add_parents(self, step: Step,
parents: t.List[Step]) -> 'Orchestration':
"""add_parents adds the parents passed into the step. No remove from previous parents
Parameters
----------
step: Step
step to add parents
parents: list
list of parent steps to add
See Also
--------
set_parents, delete_parents
add_children, set_children, delete_children
Examples
--------
>>> at = ActionTemplate(name='action', version=1, action_type=ActionType.SHELL, code='code to run',
expected_output='',
expected_rc=0, system_kwargs={})
>>> o = Orchestration('Test Orchestration', 1, DAG(), 'description')
>>> s1 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s2 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s3 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> o.add_parents(s2, [s1])
>>> o.parents[s2]
[Step1]
>>> o.add_parents(s2, [s3])
>>> o.parents[s2]
[Step1, Step3]
"""
self._step_exists([step] + list(parents))
self._check_dependencies(step, parents=parents)
step._add_parents(parents)
self._graph.add_edges_from([(p, step) for p in parents])
return self
def delete_parents(self, step: Step,
parents: t.List[Step]) -> 'Orchestration':
"""delete_parents deletes the parents passed from the step.
Parameters
----------
step: Step
step to remove parents
parents: list
list of parent steps to remove
See Also
--------
add_parents, set_parents
add_children, set_children, delete_children
Examples
--------
>>> at = ActionTemplate(name='action', version=1, action_type=ActionType.SHELL, code='code to run',
expected_output='',
expected_rc=0, system_kwargs={})
>>> o = Orchestration('Test Orchestration', 1, DAG(), 'description')
>>> s1 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s2 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s3 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> o.add_children(s1, [s2, s3])
>>> o.children[s1]
[Step2, Step3]
>>> o.delete_parents(s3, [s1])
>>> o.children[s1]
[Step2]
"""
self._step_exists([step] + list(parents))
step._remove_parents(parents)
self._graph.remove_edges_from([(p, step) for p in parents])
return self
def set_parents(self, step: Step,
parents: t.List[Step]) -> 'Orchestration':
"""set_parents sets the parents passed on the step, removing the previos ones
Parameters
----------
step: Step
step to remove parents
parents: list
list of parent steps to set
See Also
--------
add_parents, delete_parents
add_children, delete_children, set_children,
Examples
--------
>>> at = ActionTemplate(name='action', version=1, action_type=ActionType.SHELL, code='code to run',
expected_rc=0, system_kwargs={})
>>> o = Orchestration('Test Orchestration', 1, DAG(), 'description')
>>> s1 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s2 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s3 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> o.add_parents(s1, [s2])
>>> o.parents[s1]
[Step2]
>>> o.set_parents(s1, [s3])
>>> o.parents[s1]
[Step3]
"""
self.delete_parents(step, self._graph.pred[step])
self.add_parents(step, parents)
return self
def add_children(self, step: Step,
children: t.List[Step]) -> 'Orchestration':
self._step_exists([step] + children)
self._check_dependencies(step, children=children)
step._add_children(children)
self._graph.add_edges_from([(step, c) for c in children])
return self
def delete_children(self, step: Step,
children: t.List[Step]) -> 'Orchestration':
self._step_exists([step] + children)
step._remove_children(children)
self._graph.remove_edges_from([(step, c) for c in children])
return self
def set_children(self, step: Step,
children: t.List[Step]) -> 'Orchestration':
self.delete_children(step, self._graph.succ[step])
self.add_children(step, children)
return self
def eq_imp(self, other: 'Orchestration') -> bool:
"""
compares if two orchestrations implement same steps with same parameters and dependencies
Parameters
----------
other: Orchestration
Returns
-------
result: bool
"""
if isinstance(other, self.__class__):
if len(self.steps) != len(other.steps):
return False
res = []
for s in self.steps:
res.append(any(map(lambda x: s.eq_imp(x), other.steps)))
if all(res):
matched_steps = []
res2 = []
v2 = []
for k1, v1 in self.children.items():
k2 = None
for s in filter(lambda x: x not in matched_steps,
other.steps):
if k1.eq_imp(s):
k2 = s
v2 = other.children[k2]
break
if not k2:
raise RuntimeError('Step not found in other')
matched_steps.append(k2)
if len(v1) != len(v2):
return False
for s in v1:
res2.append(any(map(lambda x: s.eq_imp(x), v2)))
return all(res2)
else:
return False
else:
return False
def subtree(
self, steps: t.Union[t.List[Step], t.Iterable[Step]]
) -> t.Dict[Step, t.List[Step]]:
return self._graph.subtree(steps)
def to_json(self,
add_target=False,
add_params=False,
add_steps=False,
add_action=False,
split_lines=False,
add_schema=False,
**kwargs):
data = super().to_json(**kwargs)
data.update(name=self.name,
version=self.version,
stop_on_error=self.stop_on_error,
undo_on_error=self.undo_on_error,
stop_undo_on_error=self.stop_undo_on_error)
if add_target:
data.update(target=list(self.target))
if add_steps:
json_steps = []
for step in self.steps:
json_step = step.to_json(add_action=add_action,
split_lines=split_lines)
json_step.pop('orchestration_id')
json_steps.append(json_step)
data['steps'] = json_steps
if add_schema:
data['schema'] = self.schema
return data
@classmethod
def from_json(cls, kwargs):
return super().from_json(kwargs)
def __str__(self):
return f"{self.name}.{self.version}"
@property
def schema(self):
from ...use_cases.deployment import reserved_words, extract_container_var
outer_schema = {'required': set(), 'output': set()}
level = 1
while level <= self._graph.depth:
new_schema = copy.deepcopy(outer_schema)
for step in self._graph.get_nodes_at_level(level):
schema = step.schema
container_names = [
k for k in schema.keys() if k not in reserved_words
]
for cn in container_names:
for k, v in schema.get(cn, {}).items():
if cn not in new_schema:
new_schema.update({cn: {}})
if not (k in outer_schema['output']
or k in schema.get('mapping', {})):
new_schema[cn].update({k: v})
for k in schema.get('required', []):
cn, v = extract_container_var(k)
nv = f"{cn}.{v}"
if cn == 'input':
if v not in outer_schema[
'output'] and v not in schema.get(
'mapping', {}):
new_schema['required'].add(nv)
elif cn != 'env':
new_schema['required'].add(nv)
for k, v in schema.get('mapping', {}).items():
if isinstance(v, dict) and len(v) == 1 and 'from' in v:
action, source = tuple(v.items())[0]
d_cn, d_var = extract_container_var(k)
d_nv = f"{d_cn}.{d_var}"
s_cn, s_var = extract_container_var(source)
s_nv = f"{s_cn}.{s_var}"
if d_nv in schema.get('required',
{}) or d_var in schema.get(
'required', {}):
if s_cn == 'input':
if s_var not in outer_schema[
'output'] and s_var not in outer_schema[
'input']:
raise errors.MappingError(d_nv, step)
elif s_cn != 'env':
new_schema['required'].add(s_nv)
[
new_schema['output'].add(extract_container_var(k)[1])
for k in schema.get('output', [])
]
level += 1
outer_schema = new_schema
outer_schema['required'] = sorted(list(outer_schema['required']))
outer_schema['output'] = sorted(list(outer_schema['output']))
for c in ('required', 'output', 'input'):
# clean data
if not outer_schema.get(c):
outer_schema.pop(c, None)
return outer_schema
@staticmethod
def get(id_or_name, version=None) -> t.Union['Orchestration', str]:
if is_valid_uuid(id_or_name):
orch = Orchestration.query.get(id_or_name)
if orch is None:
return str(errors.EntityNotFound('Orchestration', id_or_name))
else:
if id_or_name:
query = Orchestration.query.filter_by(
name=id_or_name).order_by(Orchestration.version.desc())
if version:
query.filter_by(version=version)
orch = query.first()
if orch is None:
return f"No orchestration found for '{id_or_name}'" + (
f" version '{version}'" if version else None)
else:
return "No orchestration specified"
return orch
class Transfer(UUIDEntityMixin, EntityReprMixin, db.Model):
__tablename__ = "L_transfer"
software_id = db.Column(db.ForeignKey('D_software.id'))
dest_path = db.Column(db.Text, nullable=False)
num_chunks = db.Column(db.Integer, nullable=False)
status = db.Column(typos.Enum(Status),
nullable=False,
default=Status.WAITING_CHUNKS)
created_on = db.Column(UtcDateTime(timezone=True))
started_on = db.Column(UtcDateTime(timezone=True))
ended_on = db.Column(UtcDateTime(timezone=True))
_filename = db.Column("filename", db.String(256))
_size = db.Column("size", db.Integer)
_checksum = db.Column("checksum", db.Text())
software = db.relationship("Software", uselist=False)
def __init__(self,
software: t.Union[Software, str],
dest_path: str,
num_chunks: int,
status: Status = None,
size: int = None,
checksum: str = None,
created_on=None,
**kwargs):
super().__init__(**kwargs)
if isinstance(software, Software):
self.software = software
else:
self._filename = software
if size is None:
ValueError("'size' must be specified when sending a file")
self._size = size
if checksum is None:
ValueError("'checksum' must be specified when sending a file")
self._checksum = checksum
self.dest_path = dest_path
self.num_chunks = num_chunks
self.status = status or Status.WAITING_CHUNKS
self.created_on = created_on or get_now()
def to_json(self):
json = dict(id=str(self.id),
dest_path=self.dest_path,
num_chunks=self.num_chunks,
status=self.status.name,
created_on=self.created_on.strftime(
dimensigon.defaults.DATETIME_FORMAT),
file=os.path.join(self.dest_path, self.filename))
if self.software:
json.update(software_id=str(self.software.id))
else:
json.update(size=self._size, checksum=self._checksum)
if self.started_on:
json.update(started_on=self.started_on.strftime(
dimensigon.defaults.DATETIME_FORMAT))
if self.ended_on:
json.update(ended_on=self.ended_on.strftime(
dimensigon.defaults.DATETIME_FORMAT))
return json
@property
def filename(self):
if self.software:
return self.software.filename
else:
return self._filename
@property
def size(self):
if self.software:
return self.software.size
else:
return self._size
@property
def checksum(self):
if self.software:
return self.software.checksum
else:
return self._checksum
def wait_transfer(self,
timeout=None,
refresh_interval: float = 0.02) -> Status:
timeout = timeout or 300
refresh_interval = refresh_interval
start = time.time()
db.session.refresh(self)
delta = 0
while self.status in (Status.IN_PROGRESS,
Status.WAITING_CHUNKS) and delta < timeout:
time.sleep(refresh_interval)
db.session.refresh(self)
delta = time.time() - start
return self.status
class StepExecution(UUIDEntityMixin, EntityReprMixin, db.Model):
__tablename__ = 'L_step_execution'
start_time = db.Column(UtcDateTime(timezone=True), nullable=False)
end_time = db.Column(UtcDateTime(timezone=True))
params = db.Column(db.JSON)
rc = db.Column(db.Integer)
stdout = db.Column(db.Text)
stderr = db.Column(db.Text)
success = db.Column(db.Boolean)
step_id = db.Column(UUID, db.ForeignKey('D_step.id'), nullable=False)
server_id = db.Column(UUID, db.ForeignKey('D_server.id'))
orch_execution_id = db.Column(UUID, db.ForeignKey('L_orch_execution.id'))
pre_process_elapsed_time = db.Column(db.Float)
execution_elapsed_time = db.Column(db.Float)
post_process_elapsed_time = db.Column(db.Float)
child_orch_execution_id = db.Column(UUID)
step = db.relationship("Step")
server = db.relationship("Server", foreign_keys=[server_id])
orch_execution = db.relationship("OrchExecution",
foreign_keys=[orch_execution_id],
uselist=False,
back_populates="step_executions")
child_orch_execution = db.relationship(
"OrchExecution",
uselist=False,
foreign_keys=[child_orch_execution_id],
primaryjoin="StepExecution.child_orch_execution_id==OrchExecution.id")
# def __init__(self, *args, **kwargs):
# UUIDEntityMixin.__init__(self, **kwargs)
def load_completed_result(self, cp: 'CompletedProcess'):
self.success = cp.success
self.stdout = cp.stdout
self.stderr = cp.stderr
self.rc = cp.rc
def to_json(self, human=False, split_lines=False):
data = {}
if self.id:
data.update(id=str(self.id))
if self.start_time:
data.update(
start_time=self.start_time.strftime(defaults.DATETIME_FORMAT))
if self.end_time:
data.update(
end_time=self.end_time.strftime(defaults.DATETIME_FORMAT))
data.update(params=self.params)
data.update(step_id=str(getattr(self.step, 'id', None)
) if getattr(self.step, 'id', None) else None)
data.update(rc=self.rc, success=self.success)
if human:
data.update(server=str(self.server) if self.server else None)
try:
stdout = json.loads(self.stdout)
except:
stdout = self.stdout.split(
'\n'
) if split_lines and self.stdout and '\n' in self.stdout else self.stdout
try:
stderr = json.loads(self.stderr)
except:
stderr = self.stderr.split(
'\n'
) if split_lines and self.stderr and '\n' in self.stderr else self.stderr
else:
data.update(server_id=str(getattr(self.server, 'id', None))
if getattr(self.server, 'id', None) else None)
stdout = self.stdout.split(
'\n'
) if split_lines and self.stdout and '\n' in self.stdout else self.stdout
stderr = self.stderr.split(
'\n'
) if split_lines and self.stderr and '\n' in self.stderr else self.stderr
data.update(stdout=stdout)
data.update(stderr=stderr)
if self.child_orch_execution_id:
data.update(
child_orch_execution_id=str(self.child_orch_execution_id))
data.update(pre_process_elapsed_time=self.pre_process_elapsed_time
) if self.pre_process_elapsed_time is not None else None
data.update(execution_elapsed_time=self.execution_elapsed_time
) if self.execution_elapsed_time is not None else None
data.update(post_process_elapsed_time=self.post_process_elapsed_time
) if self.post_process_elapsed_time is not None else None
return data
class OrchExecution(UUIDEntityMixin, EntityReprMixin, db.Model):
__tablename__ = 'L_orch_execution'
start_time = db.Column(UtcDateTime(timezone=True),
nullable=False,
default=get_now)
end_time = db.Column(UtcDateTime(timezone=True))
orchestration_id = db.Column(UUID,
db.ForeignKey('D_orchestration.id'),
nullable=False)
target = db.Column(db.JSON)
params = db.Column(db.JSON)
executor_id = db.Column(UUID, db.ForeignKey('D_user.id'))
service_id = db.Column(UUID, db.ForeignKey('D_service.id'))
success = db.Column(db.Boolean)
undo_success = db.Column(db.Boolean)
message = db.Column(db.Text)
server_id = db.Column(UUID, db.ForeignKey('D_server.id'))
parent_step_execution_id = db.Column(UUID)
orchestration = db.relationship("Orchestration")
executor = db.relationship("User")
service = db.relationship("Service")
step_executions = db.relationship("StepExecution",
back_populates="orch_execution",
order_by="StepExecution.start_time")
server = db.relationship("Server", foreign_keys=[server_id])
parent_step_execution = db.relationship(
"StepExecution",
uselist=False,
foreign_keys=[parent_step_execution_id],
primaryjoin="OrchExecution.parent_step_execution_id==StepExecution.id")
# def __init__(self, *args, **kwargs):
# UUIDEntityMixin.__init__(self, **kwargs)
def to_json(self, add_step_exec=False, human=False, split_lines=False):
data = {}
if self.id:
data.update(id=str(self.id))
if self.start_time:
data.update(
start_time=self.start_time.strftime(defaults.DATETIME_FORMAT))
if self.end_time:
data.update(
end_time=self.end_time.strftime(defaults.DATETIME_FORMAT))
if human:
# convert target ids to server names
d = {}
if isinstance(self.target, dict):
for k, v in self.target.items():
if is_iterable_not_string(v):
d[k] = [str(Server.query.get(s) or s) for s in v]
else:
d[k] = str(Server.query.get(v) or v)
elif isinstance(self.target, list):
d = [str(Server.query.get(s) or s) for s in self.target]
else:
d = str(Server.query.get(self.target) or self.target)
data.update(target=d)
if self.executor:
data.update(executor=str(self.executor))
if self.service:
data.update(service=str(self.service))
if self.orchestration:
data.update(
orchestration=dict(id=str(self.orchestration.id),
name=self.orchestration.name,
version=self.orchestration.version))
else:
data.update(orchestration=None)
if self.server:
data.update(
server=dict(id=str(self.server.id), name=self.server.name))
else:
data.update(target=self.target)
if self.orchestration_id or getattr(self.orchestration, 'id',
None):
data.update(
orchestration_id=str(self.orchestration_id or getattr(
self.orchestration, 'id', None)))
if self.executor_id or getattr(self.executor, 'id', None):
data.update(executor_id=str(
self.executor_id or getattr(self.executor, 'id', None)))
if self.service_id or getattr(self.service, 'id', None):
data.update(service_id=str(
self.server_id or getattr(self.service, 'id', None)))
if self.server_id or getattr(self.server, 'id', None):
data.update(server_id=str(self.server_id
or getattr(self.server, 'id', None)))
data.update(params=self.params)
data.update(success=self.success)
data.update(undo_success=self.undo_success)
data.update(message=self.message)
if self.parent_step_execution_id and not add_step_exec:
data.update(
parent_step_execution_id=str(self.parent_step_execution_id))
if add_step_exec:
steps = []
for se in self.step_executions:
se: StepExecution
se_json = se.to_json(human, split_lines=split_lines)
if se.child_orch_execution:
se_json[
'orch_execution'] = se.child_orch_execution.to_json(
add_step_exec=add_step_exec,
split_lines=split_lines,
human=human)
elif se.child_orch_execution_id:
from dimensigon.web.network import get, Response
from dimensigon.network.auth import HTTPBearerAuth
from flask_jwt_extended import create_access_token
params = ['steps']
if human:
params.append('human')
try:
resp = get(se.server,
'api_1_0.orchexecutionresource',
view_data=dict(
execution_id=se.child_orch_execution_id,
params=params))
except Exception as e:
current_app.logger.exception(
f"Exception while trying to acquire orch execution "
f"{se.child_orch_execution_id} from {se.server}")
resp = Response(exception=e)
if resp.ok:
se_json['orch_execution'] = resp.msg
se_json.pop('child_orch_execution_id', None)
steps.append(se_json)
# steps.sort(key=lambda x: x.start_time)
data.update(steps=steps)
return data
@classmethod
def from_json(cls, kwargs):
if 'start_time' in kwargs:
kwargs['start_time'] = datetime.strptime(kwargs.get('start_time'),
defaults.DATETIME_FORMAT)
if 'end_time' in kwargs:
kwargs['end_time'] = datetime.strptime(kwargs.get('end_time'),
defaults.DATETIME_FORMAT)
try:
o = cls.query.get(kwargs.get('id'))
except RuntimeError as e:
o = None
if o:
for k, v in kwargs.items():
if getattr(o, k) != v:
setattr(o, k, v)
return o
else:
return cls(**kwargs)