class DbWorkflow(Base):
__tablename__ = "db_dbworkflow"
aiida_query = _QueryProperty(_AiidaQuery)
id = Column(Integer, primary_key=True)
uuid = Column(UUID(as_uuid=True), default=uuid_func)
ctime = Column(DateTime(timezone=True), default=timezone.now)
mtime = Column(DateTime(timezone=True), default=timezone.now)
user_id = Column(Integer, ForeignKey('db_dbuser.id'))
user = relationship('DbUser')
label = Column(String(255), index=True)
description = Column(Text)
nodeversion = Column(Integer)
lastsyncedversion = Column(Integer)
state = Column(ChoiceType((_, _) for _ in wf_states),
default=wf_states.INITIALIZED)
report = Column(Text)
data = relationship("DbWorkflowData", backref='parent')
# XXX the next three attributes have "blank=False", but can be null. It may
# be needed to add some validation for this, but only at commit time.
# To do so: see https://stackoverflow.com/questions/28228766/running-cleaning-validation-code-before-committing-in-sqlalchemy
module = Column(Text)
module_class = Column(Text)
script_path = Column(Text)
# XXX restrict the size of this column, MD5 have a fixed size
script_md5 = Column(String(255)) # Blank = False.
def __init__(self, *args, **kwargs):
super(DbWorkflow, self).__init__(*args, **kwargs)
self.nodeversion = 1
self.lastsyncedversion = 0
@property
def pk(self):
return self.id
def get_aiida_class(self):
"""
Return the corresponding aiida instance of class aiida.worflow
"""
from aiida.orm.workflow import Workflow
return Workflow.get_subclass_from_uuid(self.uuid)
def set_state(self, state):
self.state = state
self.save()
def set_script_md5(self, md5):
self.script_md5 = md5
self.save()
# def add_data(self, dict, d_type):
# for k in dict.keys():
# p, create = self.data.get_or_create(name=k, data_type=d_type)
# p.set_value(dict[k])
def add_data(self, dict, d_type):
for k in dict.keys():
p, create = self._get_or_create_data(name=k, data_type=d_type)
p.set_value(dict[k])
def _get_or_create_data(self, name, data_type):
match_data = {name: _ for _ in self.data if _.name == name}
if not match_data: # create case
dbdata = DbWorkflowData(parent_id=self.id,
name=name,
data_type=data_type)
self.data.append(dbdata)
return dbdata, True
else: # already existing case
return match_data[name], False
def _get_or_create_step(self, name, user):
match_step = [
_ for _ in self.steps if (_.name == name and _.user == user)
]
if not match_step: # create case
dbstep = DbWorkflowStep(parent_id=self.id,
name=name,
user_id=user.id)
self.steps.append(dbstep)
return dbstep, True
else: # already existing case
return match_step[0], False
def get_data(self, d_type):
dict = {}
# for p in self.data.filter(parent=self, data_type=d_type):
for p in [_ for _ in self.data if _.data_type == d_type]:
dict[p.name] = p.get_value()
return dict
def add_parameters(self, _dict, force=False):
if not self.state == wf_states.INITIALIZED and not force:
raise ValueError(
"Cannot add initial parameters to an already initialized workflow"
)
self.add_data(_dict, wf_data_types.PARAMETER)
def add_parameter(self, name, value):
self.add_parameters({name: value})
def get_parameters(self):
return self.get_data(wf_data_types.PARAMETER)
def get_parameter(self, name):
res = self.get_parameters()
if name in res:
return res[name]
else:
raise ValueError("Error retrieving results: {0}".format(name))
def add_results(self, _dict):
self.add_data(_dict, wf_data_types.RESULT)
def add_result(self, name, value):
self.add_results({name: value})
def get_results(self):
return self.get_data(wf_data_types.RESULT)
def get_result(self, name):
res = self.get_results()
if name in res:
return res[name]
else:
raise ValueError("Error retrieving results: {0}".format(name))
def add_attributes(self, _dict):
self.add_data(_dict, wf_data_types.ATTRIBUTE)
def add_attribute(self, name, value):
self.add_attributes({name: value})
def get_attributes(self):
return self.get_data(wf_data_types.ATTRIBUTE)
def get_attribute(self, name):
res = self.get_attributes()
if name in res:
return res[name]
else:
raise ValueError("Error retrieving results: {0}".format(name))
def clear_report(self):
self.report = ''
self.save()
def append_to_report(self, _text):
if self.report == None:
self.report = ''
self.report += str(timezone.now()) + "] " + _text + "\n"
self.save()
def get_calculations(self):
from aiida.orm import JobCalculation
return JobCalculation.query(workflow_step=self.steps)
def get_sub_workflows(self):
return DbWorkflow.objects.filter(parent_workflow_step=self.steps.all())
def is_subworkflow(self):
"""
Return True if this is a subworkflow, False if it is a root workflow,
launched by the user.
"""
return len(self.parent_workflow_step) > 0
def finish(self):
self.state = wf_states.FINISHED
def __str__(self):
simplename = self.module_class
# node pk + type
if self.label:
return "{} workflow [{}]: {}".format(simplename, self.pk,
self.label)
else:
return "{} workflow [{}]".format(simplename, self.pk)
class DbNode(Base):
__tablename__ = "db_dbnode"
aiida_query = _QueryProperty(_AiidaQuery)
id = Column(Integer, primary_key=True)
uuid = Column(UUID(as_uuid=True), default=uuid_func)
type = Column(String(255), index=True)
label = Column(
String(255), index=True, nullable=True,
default="") # Does it make sense to be nullable and have a default?
description = Column(Text(), nullable=True, default="")
ctime = Column(DateTime(timezone=True), default=timezone.now)
mtime = Column(DateTime(timezone=True), default=timezone.now)
nodeversion = Column(Integer, default=1)
public = Column(Boolean, default=False)
attributes = Column(JSONB)
extras = Column(JSONB)
dbcomputer_id = Column(Integer,
ForeignKey('db_dbcomputer.id',
deferrable=True,
initially="DEFERRED",
ondelete="RESTRICT"),
nullable=True)
# This should have the same ondelet behaviour as db_computer_id, right?
user_id = Column(Integer,
ForeignKey('db_dbuser.id',
deferrable=True,
initially="DEFERRED",
ondelete="restrict"),
nullable=False)
# TODO SP: The 'passive_deletes=all' argument here means that SQLAlchemy
# won't take care of automatic deleting in the DbLink table. This still
# isn't exactly the same behaviour than with Django. The solution to
# this is probably a ON DELETE inside the DB. On removing node with id=x,
# we would remove all link with x as an output.
######### RELATIONSSHIPS ################
dbcomputer = relationship('DbComputer',
backref=backref('dbnodes',
passive_deletes='all',
cascade='merge'))
# User
user = relationship('DbUser',
backref=backref(
'dbnodes',
passive_deletes='all',
cascade='merge',
))
# outputs via db_dblink table
outputs_q = relationship("DbNode",
secondary="db_dblink",
primaryjoin="DbNode.id == DbLink.input_id",
secondaryjoin="DbNode.id == DbLink.output_id",
backref=backref("inputs_q",
passive_deletes=True,
lazy='dynamic'),
lazy='dynamic',
passive_deletes=True)
@property
def outputs(self):
return self.outputs_q.all()
@property
def inputs(self):
return self.inputs_q.all()
# children via db_dbpath
# suggest change name to descendants
children_q = relationship("DbNode",
secondary="db_dbpath",
primaryjoin="DbNode.id == DbPath.parent_id",
secondaryjoin="DbNode.id == DbPath.child_id",
backref=backref("parents_q",
passive_deletes=True,
lazy='dynamic'),
lazy='dynamic',
passive_deletes=True)
@property
def children(self):
return self.children_q.all()
@property
def parents(self):
return self.parents_q.all()
def __init__(self, *args, **kwargs):
super(DbNode, self).__init__(*args, **kwargs)
if self.attributes is None:
self.attributes = dict()
if self.extras is None:
self.extras = dict()
# XXX repetition between django/sqlalchemy here.
def get_aiida_class(self):
"""
Return the corresponding aiida instance of class aiida.orm.Node or a
appropriate subclass.
"""
from aiida.common.pluginloader import from_type_to_pluginclassname
from aiida.orm.node import Node
try:
pluginclassname = from_type_to_pluginclassname(self.type)
except DbContentError:
raise DbContentError("The type name of node with pk= {} is "
"not valid: '{}'".format(self.pk, self.type))
try:
PluginClass = load_plugin(Node, 'aiida.orm', pluginclassname)
except MissingPluginError:
aiidalogger.error(
"Unable to find plugin for type '{}' (node= {}), "
"will use base Node class".format(self.type, self.pk))
PluginClass = Node
return PluginClass(dbnode=self)
def get_simple_name(self, invalid_result=None):
"""
Return a string with the last part of the type name.
If the type is empty, use 'Node'.
If the type is invalid, return the content of the input variable
``invalid_result``.
:param invalid_result: The value to be returned if the node type is
not recognized.
"""
thistype = self.type
# Fix for base class
if thistype == "":
thistype = "node.Node."
if not thistype.endswith("."):
return invalid_result
else:
thistype = thistype[:-1] # Strip final dot
return thistype.rpartition('.')[2]
def set_attr(self, key, value):
DbNode._set_attr(self.attributes, key, value)
flag_modified(self, "attributes")
def set_extra(self, key, value):
DbNode._set_attr(self.extras, key, value)
flag_modified(self, "extras")
def reset_extras(self, new_extras):
self.extras.clear()
self.extras.update(new_extras)
flag_modified(self, "extras")
def del_attr(self, key):
DbNode._del_attr(self.attributes, key)
flag_modified(self, "attributes")
def del_extra(self, key):
DbNode._del_attr(self.extras, key)
flag_modified(self, "extras")
@staticmethod
def _set_attr(d, key, value):
if '.' in key:
raise ValueError(
"We don't know how to treat key with dot in it yet")
d[key] = value
@staticmethod
def _del_attr(d, key):
if '.' in key:
raise ValueError(
"We don't know how to treat key with dot in it yet")
if key not in d:
raise ValueError("Key {} does not exists".format(key))
del d[key]
@property
def pk(self):
return self.id
def __str__(self):
simplename = self.get_simple_name(invalid_result="Unknown")
# node pk + type
if self.label:
return "{} node [{}]: {}".format(simplename, self.pk, self.label)
else:
return "{} node [{}]".format(simplename, self.pk)
@hybrid_property
def state(self):
"""
Return the most recent state from DbCalcState
"""
if not self.id:
return None
all_states = DbCalcState.query.filter(
DbCalcState.dbnode_id == self.id).all()
if all_states:
#return max((st.time, st.state) for st in all_states)[1]
return sort_states(((dbcalcstate.state, dbcalcstate.state.value)
for dbcalcstate in all_states),
use_key=True)[0]
else:
return None
@state.expression
def state(cls):
"""
Return the expression to get the 'latest' state from DbCalcState,
to be used in queries, where 'latest' is defined using the state order
defined in _sorted_datastates.
"""
# Sort first the latest states
whens = {
v: idx
for idx, v in enumerate(_sorted_datastates[::-1], start=1)
}
custom_sort_order = case(
value=DbCalcState.state, whens=whens,
else_=100) # else: high value to put it at the bottom
q1 = select([
DbCalcState.id.label('id'),
DbCalcState.dbnode_id.label('dbnode_id'),
DbCalcState.state.label('state'),
func.row_number().over(
partition_by=DbCalcState.dbnode_id,
order_by=custom_sort_order).label('the_row_number')
])
q1 = q1.cte()
subq = select([
q1.c.dbnode_id.label('dbnode_id'),
q1.c.state.label('state')
]).select_from(q1).where(q1.c.the_row_number == 1).alias()
return select([subq.c.state]).\
where(
subq.c.dbnode_id == cls.id,
).\
label('laststate')
class DbNode(Base):
__tablename__ = "db_dbnode"
aiida_query = _QueryProperty(_AiidaQuery)
id = Column(Integer, primary_key=True)
uuid = Column(UUID(as_uuid=True), default=uuid_func)
type = Column(String(255), index=True)
label = Column(String(255), index=True, nullable=True, default="") # Does it make sense to be nullable and have a default?
description = Column(Text(), nullable=True, default="")
ctime = Column(DateTime(timezone=True), default=timezone.now)
mtime = Column(DateTime(timezone=True), default=timezone.now)
nodeversion = Column(Integer, default=1)
public = Column(Boolean, default=False)
dbcomputer_id = Column(
Integer,
ForeignKey('db_dbcomputer.id', deferrable=True, initially="DEFERRED"),
nullable=True
)
user_id = Column(
Integer,
ForeignKey(
'db_dbuser.id',deferrable=True,initially="DEFERRED"
),
nullable=False
)
attributes = Column(JSONB, default={})
extras = Column(JSONB, default={})
# TODO SP: The 'passive_deletes=all' argument here means that SQLAlchemy
# won't take care of automatic deleting in the DbLink table. This still
# isn't exactly the same behaviour than with Django. The solution to
# this is probably a ON DELETE inside the DB. On removing node with id=x,
# we would remove all link with x as an output.
######### RELATIONSSHIPS ################
# Computer & user
dbcomputer = relationship(
'DbComputer',
backref=backref('dbnodes', passive_deletes=True)
)
user = relationship('DbUser', backref='dbnodes')
# outputs via db_dblink table
outputs = relationship(
"DbNode", secondary="db_dblink",
primaryjoin="DbNode.id == DbLink.input_id",
secondaryjoin="DbNode.id == DbLink.output_id",
backref=backref("inputs", passive_deletes=True),
passive_deletes=True
)
# children via db_dbpath
# suggest change name to descendants
children = relationship(
"DbNode", secondary="db_dbpath",
primaryjoin="DbNode.id == DbPath.parent_id",
secondaryjoin="DbNode.id == DbPath.child_id",
backref=backref("parents", passive_deletes=True),
passive_deletes=True,
)
# Appender-query, so one can query the results:
#~ dbcomputer_q = relationship(
#~ 'DbComputer',
#~ backref=backref('dbnodes_q', passive_deletes=True, lazy='dynamic'),
#~ lazy='dynamic'
#~ )
outputs_q = relationship(
"DbNode", secondary="db_dblink",
primaryjoin= "DbNode.id == DbLink.input_id",
secondaryjoin= "DbNode.id == DbLink.output_id",
backref=backref(
"inputs_q",
passive_deletes=True,
lazy='dynamic'
),
passive_deletes=True,
lazy = 'dynamic'
)
children_q = relationship(
"DbNode", secondary="db_dbpath",
primaryjoin="DbNode.id == DbPath.parent_id",
secondaryjoin="DbNode.id == DbPath.child_id",
backref=backref("parents_q", lazy='dynamic', passive_deletes=True),
lazy='dynamic',
passive_deletes=True
)
# XXX repetition between django/sqlalchemy here.
def get_aiida_class(self):
"""
Return the corresponding aiida instance of class aiida.orm.Node or a
appropriate subclass.
"""
from aiida.common.pluginloader import from_type_to_pluginclassname
from aiida.orm.node import Node
try:
pluginclassname = from_type_to_pluginclassname(self.type)
except DbContentError:
raise DbContentError("The type name of node with pk= {} is "
"not valid: '{}'".format(self.pk, self.type))
try:
PluginClass = load_plugin(Node, 'aiida.orm', pluginclassname)
except MissingPluginError:
aiidalogger.error("Unable to find plugin for type '{}' (node= {}), "
"will use base Node class".format(self.type, self.pk))
PluginClass = Node
return PluginClass(dbnode=self)
def get_simple_name(self, invalid_result=None):
"""
Return a string with the last part of the type name.
If the type is empty, use 'Node'.
If the type is invalid, return the content of the input variable
``invalid_result``.
:param invalid_result: The value to be returned if the node type is
not recognized.
"""
thistype = self.type
# Fix for base class
if thistype == "":
thistype = "node.Node."
if not thistype.endswith("."):
return invalid_result
else:
thistype = thistype[:-1] # Strip final dot
return thistype.rpartition('.')[2]
def set_attr(self, key, value):
DbNode._set_attr(self.attributes, key, value)
flag_modified(self, "attributes")
def set_extra(self, key, value):
DbNode._set_attr(self.extras, key, value)
flag_modified(self, "extras")
def del_attr(self, key):
DbNode._del_attr(self.attributes, key)
flag_modified(self, "attributes")
def del_extra(self, key):
DbNode._del_attr(self.extras, key)
flag_modified(self, "extras")
@staticmethod
def _set_attr(d, key, value):
if '.' in key:
raise ValueError(
"We don't know how to treat key with dot in it yet")
d[key] = value
@staticmethod
def _del_attr(d, key):
if '.' in key:
raise ValueError("We don't know how to treat key with dot in it yet")
if key not in d:
raise ValueError("Key {} does not exists".format(key))
del d[key]
@property
def pk(self):
return self.id
def __str__(self):
simplename = self.get_simple_name(invalid_result="Unknown")
# node pk + type
if self.label:
return "{} node [{}]: {}".format(simplename, self.pk, self.label)
else:
return "{} node [{}]".format(simplename, self.pk)