class Service(EntityReprMixin, DistributedEntityMixin, db.Model):
__tablename__ = 'D_service'
order = 40
id = db.Column(UUID, primary_key=True, default=uuid.uuid4)
name = db.Column(db.String(255), nullable=False)
details = db.Column(db.JSON)
created_on = db.Column(UtcDateTime(timezone=True), nullable=False, default=get_now)
last_ping = db.Column(UtcDateTime(timezone=True))
status = db.Column(db.String(40))
orchestrations = db.relationship("Orchestration", secondary="D_service_orchestration",
order_by="ServiceOrchestration.execution_time")
def __init__(self, name: str, details: Kwargs, status: str, created_on: datetime = get_now(),
last_ping: datetime = None, id: uuid.UUID = None, **kwargs):
super().__init__(**kwargs)
self.id = id
self.name = name
self.details = details
self.status = status
self.created_on = created_on
self.last_ping = last_ping
def to_json(self):
return {'id': str(self.id), 'name': self.name, 'details': self.details,
'last_ping': self.last_ping.strftime("%d/%m/%Y %H:%M:%S"),
'status': self.status}
class Vault(DistributedEntityMixin, SoftDeleteMixin, db.Model):
__tablename__ = 'D_vault'
user_id = db.Column(UUID, db.ForeignKey('D_user.id'), primary_key=True)
scope = db.Column(db.String(100), primary_key=True, default='global')
name = db.Column(db.String(100), primary_key=True)
_old_name = db.Column(db.String(100))
value = db.Column(db.PickleType)
user = db.relationship("User")
def to_json(self, human=False, **kwargs):
dto = super().to_json(**kwargs)
dto.update(name=self.name, value=self.value)
if self.scope:
dto.update(scope=self.scope)
if human:
dto.update(user={'id': self.user.id, 'name': self.user.name})
else:
dto.update(user_id=self.user_id or getattr(self.user, 'id', None))
return dto
@classmethod
def from_json(cls, kwargs):
super().from_json(kwargs)
kwargs['user'] = User.query.get_or_raise(kwargs.pop('user_id', None))
o = cls.query.get(
(getattr(kwargs.get('user'),
'id'), kwargs.get('scope', 'global'), kwargs.get('name')))
if o:
for k, v in kwargs.items():
if getattr(o, k) != v:
setattr(o, k, v)
return o
else:
return cls(**kwargs)
@classmethod
def get_variables_from(cls, user: t.Union[Id, User], scope='global'):
if isinstance(user, User):
user_id = user.id
else:
user_id = user
return {
vault.name: vault.value
for vault in cls.query.filter_by(user_id=user_id,
scope=scope).all()
}
def __str__(self):
return f"Vault({self.user}:{self.scope}[{self.name}={self.value}])"
class Dimension(UUIDEntityMixin, EntityReprMixin, db.Model):
__tablename__ = 'L_dimension'
name = db.Column(db.String(40), nullable=False, unique=True)
private = db.Column(PrivateKey, unique=True)
public = db.Column(PublicKey, unique=True)
current = db.Column(db.Boolean, nullable=False)
created_at = db.Column(UtcDateTime(timezone=True), default=get_now)
def __init__(self, name: str, private: t.Union[rsa.PrivateKey, bytes] = None,
public: t.Union[rsa.PublicKey, bytes] = None, created_at: datetime = get_now(), current=False,
**kwargs):
super().__init__(**kwargs)
self.name = name
if isinstance(private, bytes):
self.private = rsa.PrivateKey.load_pkcs1(private)
else:
self.private = private
if isinstance(public, bytes):
self.public = rsa.PublicKey.load_pkcs1(public)
else:
self.public = public
self.created_at = created_at
self.current = current
@classmethod
def get_current(cls) -> 'Dimension':
global current
if has_app_context():
app = current_app._get_current_object()
if app not in current:
entity = cls.query.filter_by(current=True).one()
if entity:
db.session.expunge(entity)
current[app] = entity
else:
raise NoResultFound('No row was found for one()')
return db.session.merge(current[app], load=False)
return cls.query.filter_by(current=True).one()
def to_json(self):
return {'id': str(self.id) if self.id else None, 'name': self.name,
'private': self.private.save_pkcs1().decode('ascii'),
'public': self.public.save_pkcs1().decode('ascii'),
'created_at': self.created_at.strftime(defaults.DATETIME_FORMAT)}
@classmethod
def from_json(cls, kwargs):
return cls(id=kwargs.get('id'), name=kwargs.get('name'),
private=rsa.PrivateKey.load_pkcs1(kwargs.get('private')),
public=rsa.PublicKey.load_pkcs1(kwargs.get('public')),
created_at=datetime.strptime(kwargs.get('created_at'), defaults.DATETIME_FORMAT)
)
class Catalog(db.Model):
__tablename__ = 'L_catalog'
entity = db.Column(db.String(40), primary_key=True, unique=True)
last_modified_at = db.Column(UtcDateTime(timezone=True), nullable=False)
def __init__(self, entity: str, last_modified_at: datetime):
self.entity = entity
self.last_modified_at = last_modified_at
def __repr__(self):
return f'<{self.__class__.__name__}({self.entity}, {self.last_modified_at})>'
@classmethod
def max_catalog(cls, out=None) -> t.Union[datetime, str]:
catalog_ver = db.session.query(db.func.max(
Catalog.last_modified_at)).scalar()
if catalog_ver is None:
catalog_ver = defaults.INITIAL_DATEMARK
return catalog_ver.strftime(
defaults.DATEMARK_FORMAT) if out is str else catalog_ver
class Gate(UUIDistributedEntityMixin, SoftDeleteMixin, db.Model):
__tablename__ = "D_gate"
order = 2
server_id = db.Column(UUID)
dns = db.Column(db.String(100))
ip = db.Column(IPType)
port = db.Column(db.Integer, nullable=False)
hidden = db.Column(db.Boolean, default=False)
_old_server_id = db.Column(UUID)
__table_args__ = (db.UniqueConstraint('server_id', 'ip', 'port'),
db.UniqueConstraint('server_id', 'dns', 'port'))
server = db.relationship("Server",
backref="gates",
foreign_keys=[server_id],
primaryjoin="Gate.server_id==Server.id")
def __init__(self,
server: 'Server',
port: int = defaults.DEFAULT_PORT,
dns: str = None,
ip: t.Union[str, ipaddress.IPv4Address,
ipaddress.IPv6Address] = None,
**kwargs):
super().__init__(**kwargs)
self.server = server
self.port = port
self.ip = ipaddress.ip_address(ip) if isinstance(ip, str) else ip
self.dns = dns
if not (self.dns or self.ip):
self.dns = server.name
self.hidden = kwargs.get('hidden', False) or False
def __str__(self):
return f'{self.dns or self.ip}:{self.port}'
def to_json(self, human=False, **kwargs):
data = super().to_json(**kwargs)
data.update(server_id=str(self.server.id) if self.server.id else None,
ip=str(self.ip) if self.ip else None,
dns=self.dns,
port=self.port,
hidden=self.hidden)
return data
@classmethod
def from_json(cls, kwargs):
from dimensigon.domain.entities import Server
server = kwargs.pop('server', None)
kwargs = copy.deepcopy(kwargs)
if server:
kwargs['server'] = server
kwargs['ip'] = ipaddress.ip_address(kwargs.get('ip')) if isinstance(
kwargs.get('ip'), str) else kwargs.get('ip')
if 'server_id' in kwargs and kwargs['server_id'] is not None:
# through db.session to allow load from removed entities
kwargs['server'] = db.session.query(Server).filter_by(
id=kwargs.get('server_id')).one()
kwargs.pop('server_id')
return super().from_json(kwargs)
class ActionTemplate(UUIDistributedEntityMixin, db.Model):
SEND_SOFTWARE = '00000000-0000-0000-000a-000000000001'
WAIT_SERVERS = '00000000-0000-0000-000a-000000000002'
ORCHESTRATION = '00000000-0000-0000-000a-000000000003'
WAIT_ROUTE2SERVERS = '00000000-0000-0000-000a-000000000004'
DELETE_SERVERS = '00000000-0000-0000-000a-000000000005'
__tablename__ = 'D_action_template'
order = 10
name = db.Column(db.String(40), nullable=False)
version = db.Column(db.Integer, nullable=False)
action_type = db.Column(typos.Enum(ActionType), nullable=False)
code = db.Column(db.Text)
expected_stdout = db.Column(db.Text)
expected_stderr = db.Column(db.Text)
expected_rc = db.Column(db.Integer)
system_kwargs = db.Column(db.JSON)
pre_process = db.Column(db.Text)
post_process = db.Column(db.Text)
schema = db.Column(db.JSON)
description = db.Column(db.Text)
def __init__(self,
name: str,
version: int,
action_type: ActionType,
code: MultiLine = None,
expected_stdout: MultiLine = None,
expected_stderr: MultiLine = None,
expected_rc: int = None,
system_kwargs: typos.Kwargs = None,
pre_process: MultiLine = None,
post_process: MultiLine = None,
schema: typos.Kwargs = None,
description: MultiLine = None,
**kwargs):
super().__init__(**kwargs)
self.name = name
self.version = version
self.action_type = action_type
self.code = '\n'.join(code) if is_iterable_not_string(code) else code
self.schema = schema or {}
self.expected_stdout = '\n'.join(
expected_stdout) if is_iterable_not_string(
expected_stdout) else expected_stdout
self.expected_stderr = '\n'.join(
expected_stderr) if is_iterable_not_string(
expected_stderr) else expected_stderr
self.expected_rc = expected_rc
self.system_kwargs = system_kwargs or {}
self.pre_process = '\n'.join(pre_process) if is_iterable_not_string(
pre_process) else pre_process
self.post_process = '\n'.join(post_process) if is_iterable_not_string(
post_process) else post_process
self.description = '\n'.join(description) if is_iterable_not_string(
description) else description
__table_args__ = (db.UniqueConstraint('name', 'version'), )
@orm.reconstructor
def init_on_load(self):
self.system_kwargs = self.system_kwargs or {}
def __str__(self):
return f"{self.name}.ver{self.version}"
def to_json(self, split_lines=False, **kwargs):
data = super().to_json(**kwargs)
data.update(name=self.name,
version=self.version,
action_type=self.action_type.name)
if self.code is not None:
data.update(
code=self.code.split('\n') if split_lines else self.code)
if self.schema:
data.update(schema=self.schema)
if self.system_kwargs:
data.update(system_kwargs=self.system_kwargs)
if self.expected_stdout is not None:
data.update(expected_stdout=self.expected_stdout.
split('\n') if split_lines else self.expected_stdout)
if self.expected_stderr is not None:
data.update(expected_stderr=self.expected_stderr.
split('\n') if split_lines else self.expected_stderr)
if self.expected_rc is not None:
data.update(expected_rc=self.expected_rc)
if self.post_process is not None:
data.update(post_process=self.post_process.
split('\n') if split_lines else self.post_process)
if self.pre_process is not None:
data.update(pre_process=self.pre_process.
split('\n') if split_lines else self.pre_process)
return data
@classmethod
def from_json(cls, kwargs):
kwargs = copy.deepcopy(kwargs)
kwargs['action_type'] = ActionType[kwargs.get('action_type')]
return super().from_json(kwargs)
@classmethod
def set_initial(cls, session=None):
from dimensigon.domain.entities import bypass_datamark_update
if session is None:
session = db.session
with bypass_datamark_update(session):
at = session.query(cls).get(cls.SEND_SOFTWARE)
if at is None:
at = ActionTemplate(
name='send software',
version=1,
action_type=ActionType.NATIVE,
expected_rc=201,
last_modified_at=defaults.INITIAL_DATEMARK,
schema={
"input": {
"software": {
"type":
"string",
"description":
"software name or ID to send. If "
"name specified and version not set, "
"biggest version will be taken"
},
"version": {
"type": "string",
"description": "software version to take"
},
"server": {
"type": "string",
"description": "destination server id"
},
"dest_path": {
"type": "string",
"description":
"destination path to send software"
},
"chunk_size": {
"type": "integer"
},
"max_senders": {
"type": "integer"
},
},
"required": ["software", "server"],
"output": ["file"]
},
id=cls.SEND_SOFTWARE,
post_process=
"import json\nif cp.success:\n json_data=json.loads(cp.stdout)\n vc.set('file', "
"json_data.get('file'))")
session.add(at)
at = session.query(cls).get(cls.WAIT_SERVERS)
if at is None:
at = ActionTemplate(
name='wait servers',
version=1,
action_type=ActionType.NATIVE,
description="waits server_names to join to the dimension",
last_modified_at=defaults.INITIAL_DATEMARK,
schema={
"input": {
"server_names": {
"type": ["array", "string"],
"items": {
"type": "string"
}
},
},
"required": ["server_names"]
},
id=cls.WAIT_SERVERS)
session.add(at)
at = session.query(cls).get(cls.ORCHESTRATION)
if at is None:
at = ActionTemplate(name='orchestration',
version=1,
action_type=ActionType.ORCHESTRATION,
description="launches an orchestration",
schema={
"input": {
"orchestration": {
"type":
"string",
"description":
"orchestration name or ID to "
"execute. If no version "
"specified, the last one will "
"be executed"
},
"version": {
"type": "integer"
},
"hosts": {
"type":
["string", "array", "object"],
"items": {
"type": "string"
},
"minItems":
1,
"patternProperties": {
".*": {
"anyOf": [
{
"type":
"string"
},
{
"type":
"array",
"items": {
"type":
"string"
},
"minItems": 1
},
]
},
},
},
},
"required": ["orchestration", "hosts"]
},
last_modified_at=defaults.INITIAL_DATEMARK,
id=cls.ORCHESTRATION)
session.add(at)
at = session.query(cls).get(cls.WAIT_ROUTE2SERVERS)
if at is None:
at = ActionTemplate(
name='wait route to servers',
version=1,
action_type=ActionType.NATIVE,
description="waits until we have a valid route to a server",
schema={
"input": {
"server_names": {
"type": ["array", "string"],
"items": {
"type": "string"
}
},
},
"required": ["server_names"]
},
last_modified_at=defaults.INITIAL_DATEMARK,
id=cls.WAIT_ROUTE2SERVERS)
session.add(at)
at = session.query(cls).get(cls.DELETE_SERVERS)
if at is None:
at = ActionTemplate(
name='delete servers',
version=1,
action_type=ActionType.NATIVE,
description="deletes server_names from the dimension",
schema={
"input": {
"server_names": {
"type": ["array", "string"],
"items": {
"type": "string"
}
},
},
"required": ["server_names"]
},
last_modified_at=defaults.INITIAL_DATEMARK,
id=cls.DELETE_SERVERS)
session.add(at)
class Server(UUIDistributedEntityMixin, SoftDeleteMixin, db.Model):
__tablename__ = 'D_server'
order = 1
name = db.Column(db.String(255), nullable=False, unique=True)
granules = db.Column(ScalarListType())
_me = db.Column("me", db.Boolean, default=False)
_old_name = db.Column("$$name", db.String(255))
l_ignore_on_lock = db.Column(
"ignore_on_lock", db.Boolean,
default=False) # ignore the server for locking when set
created_on = db.Column(UtcDateTime(timezone=True)) # new in version 3
route: t.Optional[Route] = db.relationship(
"Route",
primaryjoin="Route.destination_id==Server.id",
uselist=False,
back_populates="destination",
cascade="all, delete-orphan")
gates: t.List[Gate]
log_sources = db.relationship("Log",
primaryjoin="Server.id==Log.src_server_id",
back_populates="source_server")
log_destinations = db.relationship(
"Log",
primaryjoin="Server.id==Log.dst_server_id",
back_populates="destination_server")
files = db.relationship("File", back_populates="source_server")
file_server_associations = db.relationship(
"FileServerAssociation", back_populates="destination_server")
software_server_associations = db.relationship("SoftwareServerAssociation",
back_populates="server")
# software_list = db.relationship("SoftwareServerAssociation", back_populates="server")
def __init__(self,
name: str,
granules: t.List[str] = None,
dns_or_ip: t.Union[str, ipaddress.IPv4Address,
ipaddress.IPv6Address] = None,
port: int = None,
gates: t.List[t.Union[TGate, t.Dict[str, t.Any]]] = None,
me: bool = False,
created_on=None,
**kwargs):
super().__init__(**kwargs)
self.name = name
if port or dns_or_ip:
self.add_new_gate(dns_or_ip or self.name, port
or defaults.DEFAULT_PORT)
if gates:
for gate in gates:
if isinstance(gate, str):
if ':' in gate:
self.add_new_gate(*gate.split(':'))
else:
self.add_new_gate(gate, defaults.DEFAULT_PORT)
elif isinstance(gate, tuple):
self.add_new_gate(*gate)
elif isinstance(gate, dict):
gate['server'] = self
Gate.from_json(gate)
assert 'all' not in (granules or [])
self.granules = granules or []
self._me = me
self.created_on = created_on or get_now()
# create an empty route
if not me and not self.deleted:
Route(self)
@property
def external_gates(self):
e_g = []
for g in self.gates:
if not g.ip:
try:
ip = ipaddress.ip_address(
socket.getaddrinfo(g.dns,
0,
family=socket.AF_INET,
proto=socket.IPPROTO_TCP)[0][4][0])
except socket.gaierror:
e_g.append(g)
continue
except KeyError:
e_g.append(g)
continue
else:
ip = g.ip
if not ip.is_loopback:
e_g.append(g)
return e_g
@property
def localhost_gates(self):
l_g = []
for g in self.gates:
if not g.ip:
try:
ip = ipaddress.ip_address(
socket.getaddrinfo(g.dns,
0,
family=socket.AF_INET,
proto=socket.IPPROTO_TCP)[0][4][0])
except socket.gaierror:
continue
except KeyError:
continue
else:
ip = g.ip
if ip.is_loopback:
l_g.append(g)
return l_g
@property
def hidden_gates(self):
hg = [g for g in self.gates if g.hidden]
hg.sort(key=lambda x: x.last_modified_at or get_now())
return hg
def add_new_gate(self,
dns_or_ip: t.Union[str, ipaddress.IPv4Address,
ipaddress.IPv6Address],
port: int,
hidden=None):
ip = None
dns = None
if dns_or_ip:
try:
ip = ipaddress.ip_address(dns_or_ip)
except ValueError:
dns = dns_or_ip
return Gate(server=self, port=port, dns=dns, ip=ip, hidden=hidden)
def __str__(self, ):
return f"{self.name}"
def url(self, view: str = None, **values) -> str:
"""
generates the full url to access the server. Uses url_for to generate the full_path.
Parameters
----------
view
values
Raises
-------
ConnectionError:
if server is unreachable
"""
try:
scheme = 'http' if current_app.dm and 'keyfile' not in current_app.dm.config.http_conf else 'https'
except:
scheme = 'https'
gate = None
route = self.route
if self._me and (route is None or route.cost is None):
if len(self.localhost_gates) > 0:
gate = self.localhost_gates[0]
else:
# current_app.logger.warning(
# f"No localhost set for '{self}'. Trying connection through another gate")
if len(self.gates) == 0:
raise RuntimeError(f"No gate set for server '{self}'")
gate = self.gates[0]
elif getattr(route, 'cost', None) == 0:
gate = route.gate
elif getattr(route, 'proxy_server', None):
gate = getattr(getattr(route.proxy_server, 'route', None), 'gate',
None)
if not gate:
raise errors.UnreachableDestination(self,
getattr(g, 'server', None))
root_path = f"{scheme}://{gate}"
if view is None:
return root_path
else:
with current_app.test_request_context():
return root_path + url_for(view, **values)
@classmethod
def get_neighbours(cls, exclude: t.Union[t.Union[Id, 'Server'], t.List[t.Union[Id, 'Server']]] = None,
session=None) -> \
t.List[
'Server']:
"""returns neighbour servers
Args:
alive: if True, returns neighbour servers inside the cluster
Returns:
"""
if session:
query = session.query(cls).filter_by(deleted=0)
else:
query = cls.query
query = query.join(cls.route).filter(Route.cost == 0)
if exclude:
if isinstance(exclude, list):
if isinstance(exclude[0], Server):
query = query.filter(
Server.id.notin_([s.id for s in exclude]))
else:
query = query.filter(Server.id.notin_(exclude))
elif isinstance(exclude, Server):
query = query.filter(Server.id != exclude.id)
else:
query = query.filter(Server.id != exclude)
return query.order_by(cls.name).all()
@classmethod
def get_not_neighbours(cls, session=None) -> t.List['Server']:
if session:
query = session.query(cls).filter_by(deleted=0)
else:
query = cls.query
return query.outerjoin(cls.route).filter(
or_(or_(Route.cost > 0, Route.cost == None),
cls.route == None)).filter(cls._me == False).order_by(
cls.name).all()
@classmethod
def get_reachable_servers(
cls,
alive=False,
exclude: t.Union[t.Union[Id, 'Server'],
t.List[t.Union[Id, 'Server']]] = None
) -> t.List['Server']:
"""returns list of reachable servers
Args:
alive: if True, returns servers inside the cluster
exclude: filter to exclude servers
Returns:
list of all reachable servers
"""
query = cls.query.join(cls.route).filter(Route.cost.isnot(None))
if exclude:
if is_iterable_not_string(exclude):
c_exclude = [
e.id if isinstance(e, Server) else e for e in exclude
]
else:
c_exclude = [
exclude.id if isinstance(exclude, Server) else exclude
]
query = query.filter(Server.id.notin_(c_exclude))
if alive:
query = query.filter(
Server.id.in_([
iden
for iden in current_app.dm.cluster_manager.get_alive()
]))
return query.order_by(Server.name).all()
def to_json(self,
add_gates=False,
human=False,
add_ignore=False,
**kwargs):
data = super().to_json(**kwargs)
data.update({
'name':
self.name,
'granules':
self.granules,
'created_on':
self.created_on.strftime(defaults.DATETIME_FORMAT)
})
if add_gates:
data.update(gates=[])
for g in self.gates:
json_gate = g.to_json(human=human)
json_gate.pop('server_id', None)
json_gate.pop('server', None) # added to remove when human set
data['gates'].append(json_gate)
if add_ignore:
data.update(ignore_on_lock=self.l_ignore_on_lock or False)
return data
@classmethod
def from_json(cls, kwargs) -> 'Server':
kwargs = copy.deepcopy(kwargs)
gates = kwargs.pop('gates', [])
if 'created_on' in kwargs:
kwargs['created_on'] = dt.datetime.strptime(
kwargs['created_on'], defaults.DATETIME_FORMAT)
server = super().from_json(kwargs)
for gate in gates:
gate.update(server=server)
Gate.from_json(gate)
return server
@classmethod
def get_current(cls, session=None) -> 'Server':
if session is None:
session = db.session
return session.query(cls).filter_by(_me=True).filter_by(
deleted=False).one()
@staticmethod
def set_initial(session=None, gates=None) -> Id:
logger = logging.getLogger('dm.db')
if session is None:
session = db.session
server = session.query(Server).filter_by(_me=True).all()
if len(server) == 0:
try:
server_name = current_app.config.get(
'SERVER_NAME') or defaults.HOSTNAME
except:
server_name = defaults.HOSTNAME
if gates is None:
gates = [(ip, defaults.DEFAULT_PORT) for ip in get_ips()]
server = Server(name=server_name, gates=gates, me=True)
logger.info(
f'Creating Server {server.name} with the following gates: {gates}'
)
session.add(server)
return server.id
elif len(server) > 1:
raise ValueError('Multiple servers found as me.')
else:
return server[0].id
def ignore_on_lock(self, value: bool):
if value != self.l_ignore_on_lock:
from dimensigon.domain.entities import bypass_datamark_update
with bypass_datamark_update:
self.l_ignore_on_lock = value
def set_route(self, proxy_route, gate=None, cost=None):
if self.route is None:
self.route = Route(destination=self)
if isinstance(proxy_route, RouteContainer):
self.route.set_route(proxy_route)
elif isinstance(proxy_route, Route):
assert proxy_route.destination == self
self.route.set_route(
RouteContainer(proxy_route.proxy_server, proxy_route.gate,
proxy_route.cost))
else:
self.route.set_route(RouteContainer(proxy_route, gate, cost))
def delete(self):
super().delete()
for g in self.gates:
g.delete()
for l in self.log_sources:
l.delete()
for l in self.log_destinations:
l.delete()
for ssa in self.software_server_associations:
ssa.delete()
for fsa in self.file_server_associations:
fsa.delete()
for f in self.files:
f.delete()
class User(UUIDistributedEntityMixin, db.Model):
__tablename__ = 'D_user'
order = 3
name = db.Column(db.String(30),
nullable=False) # changed in SCHEMA VERSION 8
_password = db.Column('password', db.String(256))
email = db.Column(db.String)
created_at = db.Column(UtcDateTime)
active = db.Column('is_active', db.Boolean(), nullable=False)
groups = db.Column(ScalarListType(str))
__table_args__ = (db.UniqueConstraint('name'), )
def __init__(self,
name,
password=None,
email=None,
created_at=None,
active=True,
groups: t.Union[str, t.List[str]] = None,
**kwargs):
super().__init__(**kwargs)
self.name = name
self._password = password or kwargs.get('_password', None)
self.email = email
self.created_at = created_at or get_now()
self.active = active
if isinstance(groups, str):
self.groups = groups.split(':')
else:
self.groups = groups or []
@classmethod
def get_by_name(cls, name):
return cls.query.filter_by(name=name).one_or_none()
@classmethod
def get_by_group(cls, group):
return [g for g in cls.query.all() if group in g.groups]
def _hash_password(self, password):
if not self._password:
self._password = sha256_crypt.hash(password)
def verify_password(self, password) -> bool:
return sha256_crypt.verify(password, self._password)
def set_password(self, password):
self._password = None
self._hash_password(password)
def to_json(self, password=False, **kwargs) -> dict:
data = super().to_json(**kwargs)
data.update(name=self.name,
email=self.email,
created_at=self.created_at.strftime(
defaults.DATETIME_FORMAT),
active=self.active,
groups=','.join(self.groups))
if password:
data.update(_password=self._password)
return data
@classmethod
def from_json(cls, kwargs):
kwargs = dict(kwargs)
if 'created_at' in kwargs:
kwargs['created_at'] = datetime.strptime(kwargs.get('created_at'),
defaults.DATETIME_FORMAT)
if 'groups' in kwargs:
kwargs['groups'] = kwargs['groups'].split(':')
return super().from_json(kwargs)
@classmethod
def set_initial(cls, session=None):
"""
Args:
session: used when not in Flask context
"""
from dimensigon.domain.entities import bypass_datamark_update
if session is None:
session = db.session
with bypass_datamark_update(session):
root = session.query(cls).filter_by(name='root').one_or_none()
if not root:
root = User(id=ROOT,
name='root',
groups=['administrator'],
last_modified_at=defaults.INITIAL_DATEMARK)
session.add(root)
ops = session.query(cls).filter_by(name='ops').one_or_none()
if not ops:
ops = User(id=OPS,
name='ops',
groups=['operator', 'deployer'],
last_modified_at=defaults.INITIAL_DATEMARK)
session.add(ops)
reporter = session.query(cls).filter_by(
name='reporter').one_or_none()
if not reporter:
reporter = User(id=REPORTER,
name='reporter',
groups=['readonly'],
last_modified_at=defaults.INITIAL_DATEMARK)
session.add(reporter)
join = session.query(cls).filter_by(name='join').one_or_none()
if not join:
join = User(id=JOIN,
name='join',
groups=[''],
last_modified_at=defaults.INITIAL_DATEMARK)
session.add(join)
@classmethod
def get_current(cls):
return cls.query.get(get_jwt_identity())
def __repr__(self):
return f"{self.id}.{self.name}"
def __str__(self):
return self.name
class Step(UUIDistributedEntityMixin, db.Model):
__tablename__ = "D_step"
order = 30
orchestration_id = db.Column(UUID, db.ForeignKey('D_orchestration.id'), nullable=False)
action_template_id = db.Column(UUID, db.ForeignKey('D_action_template.id'))
undo = db.Column(db.Boolean, nullable=False)
step_stop_on_error = db.Column("stop_on_error", db.Boolean)
step_stop_undo_on_error = db.Column("stop_undo_on_error", db.Boolean)
step_undo_on_error = db.Column("undo_on_error", db.Boolean)
step_expected_stdout = db.Column("expected_stdout", db.Text)
step_expected_stderr = db.Column("expected_stderr", db.Text)
step_expected_rc = db.Column("expected_rc", db.Integer)
step_system_kwargs = db.Column("system_kwargs", db.JSON)
target = db.Column(ScalarListType(str))
created_on = db.Column(UtcDateTime(), nullable=False, default=get_now)
step_action_type = db.Column("action_type", typos.Enum(ActionType))
step_code = db.Column("code", db.Text)
step_post_process = db.Column("post_process", db.Text)
step_pre_process = db.Column("pre_process", db.Text)
step_name = db.Column("name", db.String(40))
step_schema = db.Column("schema", db.JSON)
step_description = db.Column("description", db.Text)
orchestration = db.relationship("Orchestration", primaryjoin="Step.orchestration_id==Orchestration.id",
back_populates="steps")
action_template = db.relationship("ActionTemplate", primaryjoin="Step.action_template_id==ActionTemplate.id",
backref="steps")
parent_steps = db.relationship("Step", secondary="D_step_step",
primaryjoin="D_step.c.id==D_step_step.c.step_id",
secondaryjoin="D_step.c.id==D_step_step.c.parent_step_id",
back_populates="children_steps")
children_steps = db.relationship("Step", secondary="D_step_step",
primaryjoin="D_step.c.id==D_step_step.c.parent_step_id",
secondaryjoin="D_step.c.id==D_step_step.c.step_id",
back_populates="parent_steps")
def __init__(self, orchestration: 'Orchestration', undo: bool, action_template: 'ActionTemplate' = None,
action_type: ActionType = None, code: str = None, pre_process: str = None, post_process: str = None,
stop_on_error: bool = None, stop_undo_on_error: bool = None, undo_on_error: bool = None,
expected_stdout: t.Optional[str] = None,
expected_stderr: t.Optional[str] = None,
expected_rc: t.Optional[int] = None,
schema: t.Dict[str, t.Any] = None,
system_kwargs: t.Dict[str, t.Any] = None,
parent_steps: t.List['Step'] = None, children_steps: t.List['Step'] = None,
target: t.Union[str, t.Iterable[str]] = None, name: str = None, description: str = None, rid=None,
**kwargs):
super().__init__(**kwargs)
assert undo in (False, True)
if action_template is not None:
assert action_type is None
else:
assert action_type is not None
self.undo = undo
self.step_stop_on_error = stop_on_error if stop_on_error is not None else kwargs.pop('step_stop_on_error', None)
self.step_stop_undo_on_error = stop_undo_on_error if stop_undo_on_error is not None else kwargs.pop(
'step_stop_undo_on_error', None)
self.step_undo_on_error = undo_on_error if undo_on_error is not None else kwargs.pop('step_undo_on_error', None)
self.action_template = action_template
self.step_action_type = action_type if action_type is not None else kwargs.pop('step_action_type', None)
expected_stdout = expected_stdout if expected_stdout is not None else kwargs.pop(
'step_expected_stdout', None)
self.step_expected_stdout = '\n'.join(expected_stdout) if is_iterable_not_string(
expected_stdout) else expected_stdout
expected_stderr = expected_stderr if expected_stderr is not None else kwargs.pop(
'step_expected_stderr', None)
self.step_expected_stderr = '\n'.join(expected_stderr) if is_iterable_not_string(
expected_stderr) else expected_stderr
self.step_expected_rc = expected_rc if expected_rc is not None else kwargs.pop('step_expected_rc', None)
self.step_schema = schema if schema is not None else kwargs.pop('step_schema', None) or {}
self.step_system_kwargs = system_kwargs if system_kwargs is not None else kwargs.pop('step_system_kwargs',
None) or {}
code = code if code is not None else kwargs.pop('step_code', None)
self.step_code = '\n'.join(code) if is_iterable_not_string(code) else code
post_process = post_process if post_process is not None else kwargs.pop('step_post_process', None)
self.step_post_process = '\n'.join(post_process) if is_iterable_not_string(post_process) else post_process
pre_process = pre_process if pre_process is not None else kwargs.pop('step_pre_process', None)
self.step_pre_process = '\n'.join(pre_process) if is_iterable_not_string(pre_process) else pre_process
self.orchestration = orchestration
self.parent_steps = parent_steps or []
self.children_steps = children_steps or []
if self.undo is False:
if target is None:
self.target = ['all']
else:
self.target = [target] if isinstance(target, str) else (target if len(target) > 0 else ['all'])
else:
if target:
raise errors.BaseError('target must not be set when creating an UNDO step')
self.created_on = kwargs.get('created_on') or get_now()
self.step_name = name if name is not None else kwargs.pop('step_name', None)
description = description if description is not None else kwargs.pop('step_description', None)
self.step_description = '\n'.join(description) if is_iterable_not_string(description) else description
self.rid = rid # used when creating an Orchestration
@orm.reconstructor
def init_on_load(self):
self.system_kwargs = self.system_kwargs or {}
@property
def parents(self):
return self.parent_steps
@property
def children(self):
return self.children_steps
@property
def parent_undo_steps(self):
return [s for s in self.parent_steps if s.undo == True]
@property
def children_undo_steps(self):
return [s for s in self.children_steps if s.undo == True]
@property
def parent_do_steps(self):
return [s for s in self.parent_steps if s.undo == False]
@property
def children_do_steps(self):
return [s for s in self.children_steps if s.undo == False]
@property
def stop_on_error(self):
return self.step_stop_on_error if self.step_stop_on_error is not None else self.orchestration.stop_on_error
@stop_on_error.setter
def stop_on_error(self, value):
self.step_stop_on_error = value
@property
def stop_undo_on_error(self):
return self.step_stop_undo_on_error if self.step_stop_undo_on_error is not None \
else self.orchestration.stop_undo_on_error
@stop_undo_on_error.setter
def stop_undo_on_error(self, value):
self.step_stop_undo_on_error = value
@property
def undo_on_error(self):
if self.undo:
return None
return self.step_undo_on_error if self.step_undo_on_error is not None else self.orchestration.undo_on_error
@undo_on_error.setter
def undo_on_error(self, value):
self.step_undo_on_error = value
@property
def schema(self):
if self.action_template:
schema = dict(ChainMap(self.step_schema, self.action_template.schema))
else:
schema = dict(self.step_schema)
if self.action_type == ActionType.ORCHESTRATION:
from .orchestration import Orchestration
mapping = schema.get('mapping', {})
o = Orchestration.get(mapping.get('orchestration', None), mapping.get('version', None))
if isinstance(o, Orchestration):
orch_schema = o.schema
i = schema.get('input', {})
i.update(orch_schema.get('input', {}))
schema.update({'input': i})
m = schema.get('mapping', {})
m.update(orch_schema.get('mapping', {}))
schema.update({'mapping': m})
r = schema.get('required', [])
[r.append(k) for k in orch_schema.get('required', []) if k not in r]
schema.update({'required': r})
o = schema.get('output', [])
[o.append(k) for k in orch_schema.get('output', []) if k not in o]
schema.update({'output': o})
return schema
@schema.setter
def schema(self, value):
self.step_schema = value
@property
def system_kwargs(self):
if self.action_template:
return dict(ChainMap(self.step_system_kwargs, self.action_template.system_kwargs))
else:
return dict(self.step_system_kwargs)
@system_kwargs.setter
def system_kwargs(self, value):
self.step_system_kwargs = value
@property
def code(self):
if self.step_code is None and self.action_template:
return self.action_template.code
else:
return self.step_code
@code.setter
def code(self, value):
self.step_code = value
@property
def action_type(self):
if self.step_action_type is None and self.action_template:
return self.action_template.action_type
else:
return self.step_action_type
@action_type.setter
def action_type(self, value):
self.step_action_type = value
@property
def post_process(self):
if self.step_post_process is None and self.action_template:
return self.action_template.post_process
else:
return self.step_post_process
@post_process.setter
def post_process(self, value):
self.step_post_process = value
@property
def pre_process(self):
if self.step_pre_process is None and self.action_template:
return self.action_template.pre_process
else:
return self.step_pre_process
@pre_process.setter
def pre_process(self, value):
self.step_pre_process = value
@property
def expected_stdout(self):
if self.step_expected_stdout is None and self.action_template:
return self.action_template.expected_stdout
else:
return self.step_expected_stdout
@expected_stdout.setter
def expected_stdout(self, value):
self.step_expected_stdout = value
@property
def expected_stderr(self):
if self.step_expected_stderr is None and self.action_template:
return self.action_template.expected_stderr
else:
return self.step_expected_stderr
@expected_stderr.setter
def expected_stderr(self, value):
if value == self.action_template.expected_stderr:
self.step_expected_stderr = None
else:
self.step_expected_stderr = value
@property
def expected_rc(self):
if self.step_expected_rc is None and self.action_template:
return self.action_template.expected_rc
else:
return self.step_expected_rc
@expected_rc.setter
def expected_rc(self, value):
if self.action_template and value == self.action_template.expected_rc:
self.step_expected_rc = None
else:
self.step_expected_rc = value
@property
def name(self):
if self.step_name is None and self.action_template:
return str(self.action_template)
else:
return self.step_name
@name.setter
def name(self, value):
self.step_name = value
@property
def description(self):
if self.step_description is None and self.action_template:
return str(self.action_template)
else:
return self.step_description
@description.setter
def description(self, value):
self.step_description = value
def eq_imp(self, other):
"""
two steps are equal if they execute the same code with the same parameters even if they are from different
orchestrations or they are in the same orchestration with different positions
Parameters
----------
other: Step
Returns
-------
result: bool
Notes
-----
_id and _orchestration are not compared
"""
if isinstance(other, self.__class__):
return all([self.undo == other.undo,
self.stop_on_error == other.stop_on_error,
self.stop_undo_on_error == other.stop_undo_on_error,
self.undo_on_error == other.undo_on_error,
self.expected_stdout == other.expected_stdout,
self.expected_stderr == other.expected_stderr,
self.expected_rc == other.expected_rc,
self.system_kwargs == other.system_kwargs,
self.code == other.code,
self.post_process == other.post_process,
self.pre_process == other.pre_process,
self.action_type == other.action_type,
])
else:
raise NotImplemented
def __str__(self):
return self.name or self.id
def __repr__(self):
return ('Undo ' if self.undo else '') + self.__class__.__name__ + ' ' + str(getattr(self, 'id', ''))
def _add_parents(self, parents):
for step in parents:
if not step in self.parent_steps:
self.parent_steps.append(step)
def _remove_parents(self, parents):
for step in parents:
if step in self.parent_steps:
self.parent_steps.remove(step)
def _add_children(self, children):
for step in children:
if not step in self.children_steps:
self.children_steps.append(step)
def _remove_children(self, children):
for step in children:
if step in self.children_steps:
self.children_steps.remove(step)
def to_json(self, add_action=False, split_lines=False, **kwargs):
data = super().to_json(**kwargs)
if getattr(self.orchestration, 'id', None):
data.update(orchestration_id=str(self.orchestration.id))
if getattr(self.action_template, 'id', None):
if add_action:
data.update(action_template=self.action_template.to_json(split_lines=split_lines))
else:
data.update(action_template_id=str(self.action_template.id))
data.update(undo=self.undo)
data.update(stop_on_error=self.step_stop_on_error) if self.step_stop_on_error is not None else None
data.update(
stop_undo_on_error=self.step_stop_undo_on_error) if self.step_stop_undo_on_error is not None else None
data.update(undo_on_error=self.step_undo_on_error) if self.step_undo_on_error is not None else None
if self.step_schema:
data.update(schema=self.step_schema)
if self.step_expected_stdout is not None:
data.update(
expected_stdout=self.step_expected_stdout.split('\n') if split_lines else self.step_expected_stdout)
if self.step_expected_stderr is not None:
data.update(
expected_stderr=self.step_expected_stderr.split('\n') if split_lines else self.step_expected_stderr)
data.update(expected_rc=self.step_expected_rc) if self.step_expected_rc is not None else None
data.update(system_kwargs=self.step_system_kwargs) if self.step_system_kwargs is not None else None
data.update(parent_step_ids=[str(step.id) for step in self.parents])
if self.step_code is not None:
data.update(code=self.step_code.split('\n') if split_lines else self.step_code)
data.update(action_type=self.step_action_type.name) if self.step_action_type is not None else None
if self.step_post_process is not None:
data.update(post_process=self.step_post_process.split('\n') if split_lines else self.step_post_process)
if self.step_pre_process is not None:
data.update(pre_process=self.step_pre_process.split('\n') if split_lines else self.step_pre_process)
data.update(created_on=self.created_on.strftime(defaults.DATETIME_FORMAT))
if self.step_description is not None:
data.update(description=self.step_description.split('\n') if split_lines else self.step_description)
if self.step_name is not None:
data.update(name=self.step_name)
return data
@classmethod
def from_json(cls, kwargs):
from dimensigon.domain.entities import ActionTemplate, Orchestration
kwargs = dict(kwargs)
if 'orchestration_id' in kwargs:
ident = kwargs.pop('orchestration_id')
kwargs['orchestration'] = db.session.query(Orchestration).get(ident)
if kwargs['orchestration'] is None:
raise errors.EntityNotFound('Orchestration', ident=ident)
if 'action_template_id' in kwargs:
ident = kwargs.pop('action_template_id')
kwargs['action_template'] = db.session.query(ActionTemplate).get(ident)
if kwargs['action_template'] is None:
raise errors.EntityNotFound('ActionTemplate', ident=ident)
if 'action_type' in kwargs:
kwargs['action_type'] = ActionType[kwargs.pop('action_type')]
if 'created_on' in kwargs:
kwargs['created_on'] = datetime.datetime.strptime(kwargs['created_on'], defaults.DATETIME_FORMAT)
kwargs['parent_steps'] = []
for parent_step_id in kwargs.pop('parent_step_ids', []):
ps = Step.query.get(parent_step_id)
if ps:
kwargs['parent_steps'].append(ps)
else:
raise errors.EntityNotFound('Step', parent_step_id)
return super().from_json(kwargs)
class Orchestration(UUIDistributedEntityMixin, db.Model):
__tablename__ = 'D_orchestration'
order = 20
name = db.Column(db.String(80), nullable=False)
version = db.Column(db.Integer, nullable=False)
description = db.Column(db.Text)
stop_on_error = db.Column(db.Boolean)
stop_undo_on_error = db.Column(db.Boolean)
undo_on_error = db.Column(db.Boolean)
created_at = db.Column(UtcDateTime(timezone=True), default=get_now)
steps = db.relationship(
"Step",
primaryjoin="Step.orchestration_id==Orchestration.id",
back_populates="orchestration")
__table_args__ = (db.UniqueConstraint('name',
'version',
name='D_orchestration_uq01'), )
def __init__(self,
name: str,
version: int,
description: t.Optional[str] = None,
steps: t.List[Step] = None,
stop_on_error: bool = True,
stop_undo_on_error: bool = True,
undo_on_error: bool = True,
dependencies: Tdependencies = None,
created_at=None,
**kwargs):
super().__init__(**kwargs)
self.name = name
self.version = version
self.description = '\n'.join(description) if is_iterable_not_string(
description) else description
self.steps = steps or []
assert isinstance(stop_on_error, bool)
self.stop_on_error = stop_on_error
assert isinstance(stop_undo_on_error, bool)
self.stop_undo_on_error = stop_undo_on_error
assert isinstance(undo_on_error, bool)
self.undo_on_error = undo_on_error
self.created_at = created_at or get_now()
if dependencies:
self.set_dependencies(dependencies)
else:
self._graph = DAG()
@orm.reconstructor
def init_on_load(self):
self._graph = DAG()
for step in self.steps:
if step.parents:
for p in step.parents:
self._graph.add_edge(p, step)
else:
self._graph.add_node(step)
def set_dependencies(self, dependencies: Tdependencies):
edges = []
find = lambda id_: next(
(step for step in self.steps if step.id == id_))
if isinstance(dependencies, t.Dict):
for k, v in dependencies.items():
try:
step_from = find(k)
except StopIteration:
raise ValueError(f'id step {k} not found in steps list')
for child_id in v:
try:
step_to = find(child_id)
except StopIteration:
raise ValueError(
f'id step {child_id} not found in steps list')
edges.append((step_from, step_to))
elif isinstance(dependencies, t.Iterable):
edges = dependencies
else:
raise ValueError(
f'dependencies must be a dict like object or an iterable of tuples. '
f'See the docs for more information')
self._graph = DAG(edges)
@property
def parents(self) -> t.Dict[Step, t.List[Step]]:
return self._graph.pred
@property
def children(self) -> t.Dict[Step, t.List[Step]]:
return self._graph.succ
@property
def dependencies(self) -> t.Dict[Step, t.List[Step]]:
return self.children
@property
def root(self) -> t.List[Step]:
return self._graph.root
@property
def target(self) -> t.Set[str]:
target = set()
for step in self.steps:
if step.undo is False:
target.update(step.target)
return target
def _step_exists(self, step: t.Union[t.List[Step], Step]):
"""Checks if all the steps belong to the current orchestration
Parameters
----------
step: list[Step]|Step
Step or list of steps to be evaluated
Returns
-------
None
Raises
------
ValueError: if any step passed as argument is not in the orchestration
"""
if not isinstance(step, list):
steps = [step]
else:
steps = step
for step in steps:
if not (step in self._graph.nodes and step.orchestration is self):
raise ValueError(f'{step} is NOT from this orchestration')
def _check_dependencies(self, step, parents=None, children=None):
"""
Checks if the dependencies that are going to be added accomplish the business rules. These rules are:
1. a 'do' Step cannot be preceded for an 'undo' Step
2. cannot be cycles inside the orchestration
Parameters
----------
step: Step
step to be evaluated
parents: list[Step]
parent steps to be added
children: list[Step]
children steps to be added
Raises
------
ValueError
if the rule 1 is not passed
CycleError
if the rule 2 is not passed
"""
parents = parents or []
children = children or []
if parents:
if any([p.undo for p in parents]) and not step.undo:
attr = 'rid' if step.rid is not None else 'id'
raise errors.ParentUndoError(getattr(
step, attr), [getattr(s, attr) for s in parents if s.undo])
if children:
if any([not c.undo for c in children]) and step.undo:
attr = 'rid' if step.rid is not None else 'id'
raise errors.ChildDoError(
getattr(step, attr),
[getattr(s, attr) for s in children if s.undo])
g = self._graph.copy()
g.add_edges_from([(p, step) for p in parents])
g.add_edges_from([(step, c) for c in children])
if g.is_cyclic():
raise errors.CycleError()
def add_step(self, *args, parents=None, children=None, **kwargs) -> Step:
"""Allows to add step into the orchestration
:param args: args passed to Step: undo, action_template.
:param parents: list of parent steps.
:param children: list of children steps.
:param kwargs: keyword arguments passed to the Step
:return: The step created.
"""
parents = parents or []
children = children or []
s = Step(None, *args, **kwargs)
self._step_exists(parents + children)
self._check_dependencies(s, parents, children)
s.orchestration = self
self._graph.add_node(s)
self.add_parents(s, parents)
self.add_children(s, children)
return s
def delete_step(self, step: Step) -> 'Orchestration':
"""
Allows to remove a Step from the orchestration
Parameters
----------
step Step: step to remove from the orchestration
"""
self._step_exists(step)
i = self.steps.index(step)
self.steps.pop(i)
self._graph.remove_node(step)
return self
def add_parents(self, step: Step,
parents: t.List[Step]) -> 'Orchestration':
"""add_parents adds the parents passed into the step. No remove from previous parents
Parameters
----------
step: Step
step to add parents
parents: list
list of parent steps to add
See Also
--------
set_parents, delete_parents
add_children, set_children, delete_children
Examples
--------
>>> at = ActionTemplate(name='action', version=1, action_type=ActionType.SHELL, code='code to run',
expected_output='',
expected_rc=0, system_kwargs={})
>>> o = Orchestration('Test Orchestration', 1, DAG(), 'description')
>>> s1 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s2 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s3 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> o.add_parents(s2, [s1])
>>> o.parents[s2]
[Step1]
>>> o.add_parents(s2, [s3])
>>> o.parents[s2]
[Step1, Step3]
"""
self._step_exists([step] + list(parents))
self._check_dependencies(step, parents=parents)
step._add_parents(parents)
self._graph.add_edges_from([(p, step) for p in parents])
return self
def delete_parents(self, step: Step,
parents: t.List[Step]) -> 'Orchestration':
"""delete_parents deletes the parents passed from the step.
Parameters
----------
step: Step
step to remove parents
parents: list
list of parent steps to remove
See Also
--------
add_parents, set_parents
add_children, set_children, delete_children
Examples
--------
>>> at = ActionTemplate(name='action', version=1, action_type=ActionType.SHELL, code='code to run',
expected_output='',
expected_rc=0, system_kwargs={})
>>> o = Orchestration('Test Orchestration', 1, DAG(), 'description')
>>> s1 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s2 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s3 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> o.add_children(s1, [s2, s3])
>>> o.children[s1]
[Step2, Step3]
>>> o.delete_parents(s3, [s1])
>>> o.children[s1]
[Step2]
"""
self._step_exists([step] + list(parents))
step._remove_parents(parents)
self._graph.remove_edges_from([(p, step) for p in parents])
return self
def set_parents(self, step: Step,
parents: t.List[Step]) -> 'Orchestration':
"""set_parents sets the parents passed on the step, removing the previos ones
Parameters
----------
step: Step
step to remove parents
parents: list
list of parent steps to set
See Also
--------
add_parents, delete_parents
add_children, delete_children, set_children,
Examples
--------
>>> at = ActionTemplate(name='action', version=1, action_type=ActionType.SHELL, code='code to run',
expected_rc=0, system_kwargs={})
>>> o = Orchestration('Test Orchestration', 1, DAG(), 'description')
>>> s1 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s2 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> s3 = o.add_step(undo=False, action_template=at, parents=[], children=[], stop_on_error=False)
>>> o.add_parents(s1, [s2])
>>> o.parents[s1]
[Step2]
>>> o.set_parents(s1, [s3])
>>> o.parents[s1]
[Step3]
"""
self.delete_parents(step, self._graph.pred[step])
self.add_parents(step, parents)
return self
def add_children(self, step: Step,
children: t.List[Step]) -> 'Orchestration':
self._step_exists([step] + children)
self._check_dependencies(step, children=children)
step._add_children(children)
self._graph.add_edges_from([(step, c) for c in children])
return self
def delete_children(self, step: Step,
children: t.List[Step]) -> 'Orchestration':
self._step_exists([step] + children)
step._remove_children(children)
self._graph.remove_edges_from([(step, c) for c in children])
return self
def set_children(self, step: Step,
children: t.List[Step]) -> 'Orchestration':
self.delete_children(step, self._graph.succ[step])
self.add_children(step, children)
return self
def eq_imp(self, other: 'Orchestration') -> bool:
"""
compares if two orchestrations implement same steps with same parameters and dependencies
Parameters
----------
other: Orchestration
Returns
-------
result: bool
"""
if isinstance(other, self.__class__):
if len(self.steps) != len(other.steps):
return False
res = []
for s in self.steps:
res.append(any(map(lambda x: s.eq_imp(x), other.steps)))
if all(res):
matched_steps = []
res2 = []
v2 = []
for k1, v1 in self.children.items():
k2 = None
for s in filter(lambda x: x not in matched_steps,
other.steps):
if k1.eq_imp(s):
k2 = s
v2 = other.children[k2]
break
if not k2:
raise RuntimeError('Step not found in other')
matched_steps.append(k2)
if len(v1) != len(v2):
return False
for s in v1:
res2.append(any(map(lambda x: s.eq_imp(x), v2)))
return all(res2)
else:
return False
else:
return False
def subtree(
self, steps: t.Union[t.List[Step], t.Iterable[Step]]
) -> t.Dict[Step, t.List[Step]]:
return self._graph.subtree(steps)
def to_json(self,
add_target=False,
add_params=False,
add_steps=False,
add_action=False,
split_lines=False,
add_schema=False,
**kwargs):
data = super().to_json(**kwargs)
data.update(name=self.name,
version=self.version,
stop_on_error=self.stop_on_error,
undo_on_error=self.undo_on_error,
stop_undo_on_error=self.stop_undo_on_error)
if add_target:
data.update(target=list(self.target))
if add_steps:
json_steps = []
for step in self.steps:
json_step = step.to_json(add_action=add_action,
split_lines=split_lines)
json_step.pop('orchestration_id')
json_steps.append(json_step)
data['steps'] = json_steps
if add_schema:
data['schema'] = self.schema
return data
@classmethod
def from_json(cls, kwargs):
return super().from_json(kwargs)
def __str__(self):
return f"{self.name}.{self.version}"
@property
def schema(self):
from ...use_cases.deployment import reserved_words, extract_container_var
outer_schema = {'required': set(), 'output': set()}
level = 1
while level <= self._graph.depth:
new_schema = copy.deepcopy(outer_schema)
for step in self._graph.get_nodes_at_level(level):
schema = step.schema
container_names = [
k for k in schema.keys() if k not in reserved_words
]
for cn in container_names:
for k, v in schema.get(cn, {}).items():
if cn not in new_schema:
new_schema.update({cn: {}})
if not (k in outer_schema['output']
or k in schema.get('mapping', {})):
new_schema[cn].update({k: v})
for k in schema.get('required', []):
cn, v = extract_container_var(k)
nv = f"{cn}.{v}"
if cn == 'input':
if v not in outer_schema[
'output'] and v not in schema.get(
'mapping', {}):
new_schema['required'].add(nv)
elif cn != 'env':
new_schema['required'].add(nv)
for k, v in schema.get('mapping', {}).items():
if isinstance(v, dict) and len(v) == 1 and 'from' in v:
action, source = tuple(v.items())[0]
d_cn, d_var = extract_container_var(k)
d_nv = f"{d_cn}.{d_var}"
s_cn, s_var = extract_container_var(source)
s_nv = f"{s_cn}.{s_var}"
if d_nv in schema.get('required',
{}) or d_var in schema.get(
'required', {}):
if s_cn == 'input':
if s_var not in outer_schema[
'output'] and s_var not in outer_schema[
'input']:
raise errors.MappingError(d_nv, step)
elif s_cn != 'env':
new_schema['required'].add(s_nv)
[
new_schema['output'].add(extract_container_var(k)[1])
for k in schema.get('output', [])
]
level += 1
outer_schema = new_schema
outer_schema['required'] = sorted(list(outer_schema['required']))
outer_schema['output'] = sorted(list(outer_schema['output']))
for c in ('required', 'output', 'input'):
# clean data
if not outer_schema.get(c):
outer_schema.pop(c, None)
return outer_schema
@staticmethod
def get(id_or_name, version=None) -> t.Union['Orchestration', str]:
if is_valid_uuid(id_or_name):
orch = Orchestration.query.get(id_or_name)
if orch is None:
return str(errors.EntityNotFound('Orchestration', id_or_name))
else:
if id_or_name:
query = Orchestration.query.filter_by(
name=id_or_name).order_by(Orchestration.version.desc())
if version:
query.filter_by(version=version)
orch = query.first()
if orch is None:
return f"No orchestration found for '{id_or_name}'" + (
f" version '{version}'" if version else None)
else:
return "No orchestration specified"
return orch
class Transfer(UUIDEntityMixin, EntityReprMixin, db.Model):
__tablename__ = "L_transfer"
software_id = db.Column(db.ForeignKey('D_software.id'))
dest_path = db.Column(db.Text, nullable=False)
num_chunks = db.Column(db.Integer, nullable=False)
status = db.Column(typos.Enum(Status),
nullable=False,
default=Status.WAITING_CHUNKS)
created_on = db.Column(UtcDateTime(timezone=True))
started_on = db.Column(UtcDateTime(timezone=True))
ended_on = db.Column(UtcDateTime(timezone=True))
_filename = db.Column("filename", db.String(256))
_size = db.Column("size", db.Integer)
_checksum = db.Column("checksum", db.Text())
software = db.relationship("Software", uselist=False)
def __init__(self,
software: t.Union[Software, str],
dest_path: str,
num_chunks: int,
status: Status = None,
size: int = None,
checksum: str = None,
created_on=None,
**kwargs):
super().__init__(**kwargs)
if isinstance(software, Software):
self.software = software
else:
self._filename = software
if size is None:
ValueError("'size' must be specified when sending a file")
self._size = size
if checksum is None:
ValueError("'checksum' must be specified when sending a file")
self._checksum = checksum
self.dest_path = dest_path
self.num_chunks = num_chunks
self.status = status or Status.WAITING_CHUNKS
self.created_on = created_on or get_now()
def to_json(self):
json = dict(id=str(self.id),
dest_path=self.dest_path,
num_chunks=self.num_chunks,
status=self.status.name,
created_on=self.created_on.strftime(
dimensigon.defaults.DATETIME_FORMAT),
file=os.path.join(self.dest_path, self.filename))
if self.software:
json.update(software_id=str(self.software.id))
else:
json.update(size=self._size, checksum=self._checksum)
if self.started_on:
json.update(started_on=self.started_on.strftime(
dimensigon.defaults.DATETIME_FORMAT))
if self.ended_on:
json.update(ended_on=self.ended_on.strftime(
dimensigon.defaults.DATETIME_FORMAT))
return json
@property
def filename(self):
if self.software:
return self.software.filename
else:
return self._filename
@property
def size(self):
if self.software:
return self.software.size
else:
return self._size
@property
def checksum(self):
if self.software:
return self.software.checksum
else:
return self._checksum
def wait_transfer(self,
timeout=None,
refresh_interval: float = 0.02) -> Status:
timeout = timeout or 300
refresh_interval = refresh_interval
start = time.time()
db.session.refresh(self)
delta = 0
while self.status in (Status.IN_PROGRESS,
Status.WAITING_CHUNKS) and delta < timeout:
time.sleep(refresh_interval)
db.session.refresh(self)
delta = time.time() - start
return self.status
class Parameter(db.Model):
__tablename__ = 'L_parameter'
parameter = db.Column(db.String(80), primary_key=True)
value = db.Column(db.Text)
dump = db.Column(Dill)
load = db.Column(Dill)
def __init__(self, parameter: str, dump: t.Callable[[t.Any], str] = None, load: t.Callable[[str], t.Any] = None):
self.parameter = parameter
self.dump = dump
self.load = load
@staticmethod
def _convert(param: 'Parameter'):
if param.load:
try:
value = param.load(param.value)
except:
return None
else:
return value
else:
return param.value
@classmethod
def _get_parameter(cls, key: str):
p = cls.query.get(key)
if p:
return p
else:
raise KeyError(key)
@classmethod
def get(cls, key: str, default=_Empty):
try:
p = cls._get_parameter(key)
except KeyError:
if default == _Empty:
raise
else:
return default
value = cls._convert(p)
if value is None and default != _Empty:
return default
else:
return value
def set_functions(self, dump: t.Callable[[t.Any], str] = _Empty, load: t.Callable[[str], t.Any] = _Empty):
if dump != _Empty:
self.dump = dump
if load != _Empty:
self.load = load
@classmethod
def set(cls, key, value):
p = cls.query.get(key)
if p.dump:
p.value = p.dump(value)
else:
p.value = value
@classmethod
def set_initial(cls, session=None):
if session is None:
session = db.session
if not session.query(cls).get('last_graceful_shutdown'):
p = Parameter('last_graceful_shutdown')
session.add(p)
if not session.query(cls).get('fetching_catalog'):
p = Parameter('fetching_catalog')
session.add(p)
if not session.query(cls).get('join_server'):
p = Parameter('join_server')
session.add(p)
if not session.query(cls).get('new_gates_server'):
p = Parameter('new_gates_server')
session.add(p)
