class DbWorkflow(Base): __tablename__ = "db_dbworkflow" aiida_query = _QueryProperty(_AiidaQuery) id = Column(Integer, primary_key=True) uuid = Column(UUID(as_uuid=True), default=uuid_func) ctime = Column(DateTime(timezone=True), default=timezone.now) mtime = Column(DateTime(timezone=True), default=timezone.now) user_id = Column(Integer, ForeignKey('db_dbuser.id')) user = relationship('DbUser') label = Column(String(255), index=True) description = Column(Text) nodeversion = Column(Integer) lastsyncedversion = Column(Integer) state = Column(ChoiceType((_, _) for _ in wf_states), default=wf_states.INITIALIZED) report = Column(Text) data = relationship("DbWorkflowData", backref='parent') # XXX the next three attributes have "blank=False", but can be null. It may # be needed to add some validation for this, but only at commit time. # To do so: see https://stackoverflow.com/questions/28228766/running-cleaning-validation-code-before-committing-in-sqlalchemy module = Column(Text) module_class = Column(Text) script_path = Column(Text) # XXX restrict the size of this column, MD5 have a fixed size script_md5 = Column(String(255)) # Blank = False. def __init__(self, *args, **kwargs): super(DbWorkflow, self).__init__(*args, **kwargs) self.nodeversion = 1 self.lastsyncedversion = 0 @property def pk(self): return self.id def get_aiida_class(self): """ Return the corresponding aiida instance of class aiida.worflow """ from aiida.orm.workflow import Workflow return Workflow.get_subclass_from_uuid(self.uuid) def set_state(self, state): self.state = state self.save() def set_script_md5(self, md5): self.script_md5 = md5 self.save() # def add_data(self, dict, d_type): # for k in dict.keys(): # p, create = self.data.get_or_create(name=k, data_type=d_type) # p.set_value(dict[k]) def add_data(self, dict, d_type): for k in dict.keys(): p, create = self._get_or_create_data(name=k, data_type=d_type) p.set_value(dict[k]) def _get_or_create_data(self, name, data_type): match_data = {name: _ for _ in self.data if _.name == name} if not match_data: # create case dbdata = DbWorkflowData(parent_id=self.id, name=name, data_type=data_type) self.data.append(dbdata) return dbdata, True else: # already existing case return match_data[name], False def _get_or_create_step(self, name, user): match_step = [ _ for _ in self.steps if (_.name == name and _.user == user) ] if not match_step: # create case dbstep = DbWorkflowStep(parent_id=self.id, name=name, user_id=user.id) self.steps.append(dbstep) return dbstep, True else: # already existing case return match_step[0], False def get_data(self, d_type): dict = {} # for p in self.data.filter(parent=self, data_type=d_type): for p in [_ for _ in self.data if _.data_type == d_type]: dict[p.name] = p.get_value() return dict def add_parameters(self, _dict, force=False): if not self.state == wf_states.INITIALIZED and not force: raise ValueError( "Cannot add initial parameters to an already initialized workflow" ) self.add_data(_dict, wf_data_types.PARAMETER) def add_parameter(self, name, value): self.add_parameters({name: value}) def get_parameters(self): return self.get_data(wf_data_types.PARAMETER) def get_parameter(self, name): res = self.get_parameters() if name in res: return res[name] else: raise ValueError("Error retrieving results: {0}".format(name)) def add_results(self, _dict): self.add_data(_dict, wf_data_types.RESULT) def add_result(self, name, value): self.add_results({name: value}) def get_results(self): return self.get_data(wf_data_types.RESULT) def get_result(self, name): res = self.get_results() if name in res: return res[name] else: raise ValueError("Error retrieving results: {0}".format(name)) def add_attributes(self, _dict): self.add_data(_dict, wf_data_types.ATTRIBUTE) def add_attribute(self, name, value): self.add_attributes({name: value}) def get_attributes(self): return self.get_data(wf_data_types.ATTRIBUTE) def get_attribute(self, name): res = self.get_attributes() if name in res: return res[name] else: raise ValueError("Error retrieving results: {0}".format(name)) def clear_report(self): self.report = '' self.save() def append_to_report(self, _text): if self.report == None: self.report = '' self.report += str(timezone.now()) + "] " + _text + "\n" self.save() def get_calculations(self): from aiida.orm import JobCalculation return JobCalculation.query(workflow_step=self.steps) def get_sub_workflows(self): return DbWorkflow.objects.filter(parent_workflow_step=self.steps.all()) def is_subworkflow(self): """ Return True if this is a subworkflow, False if it is a root workflow, launched by the user. """ return len(self.parent_workflow_step) > 0 def finish(self): self.state = wf_states.FINISHED def __str__(self): simplename = self.module_class # node pk + type if self.label: return "{} workflow [{}]: {}".format(simplename, self.pk, self.label) else: return "{} workflow [{}]".format(simplename, self.pk)
class DbNode(Base): __tablename__ = "db_dbnode" aiida_query = _QueryProperty(_AiidaQuery) id = Column(Integer, primary_key=True) uuid = Column(UUID(as_uuid=True), default=uuid_func) type = Column(String(255), index=True) label = Column( String(255), index=True, nullable=True, default="") # Does it make sense to be nullable and have a default? description = Column(Text(), nullable=True, default="") ctime = Column(DateTime(timezone=True), default=timezone.now) mtime = Column(DateTime(timezone=True), default=timezone.now) nodeversion = Column(Integer, default=1) public = Column(Boolean, default=False) attributes = Column(JSONB) extras = Column(JSONB) dbcomputer_id = Column(Integer, ForeignKey('db_dbcomputer.id', deferrable=True, initially="DEFERRED", ondelete="RESTRICT"), nullable=True) # This should have the same ondelet behaviour as db_computer_id, right? user_id = Column(Integer, ForeignKey('db_dbuser.id', deferrable=True, initially="DEFERRED", ondelete="restrict"), nullable=False) # TODO SP: The 'passive_deletes=all' argument here means that SQLAlchemy # won't take care of automatic deleting in the DbLink table. This still # isn't exactly the same behaviour than with Django. The solution to # this is probably a ON DELETE inside the DB. On removing node with id=x, # we would remove all link with x as an output. ######### RELATIONSSHIPS ################ dbcomputer = relationship('DbComputer', backref=backref('dbnodes', passive_deletes='all', cascade='merge')) # User user = relationship('DbUser', backref=backref( 'dbnodes', passive_deletes='all', cascade='merge', )) # outputs via db_dblink table outputs_q = relationship("DbNode", secondary="db_dblink", primaryjoin="DbNode.id == DbLink.input_id", secondaryjoin="DbNode.id == DbLink.output_id", backref=backref("inputs_q", passive_deletes=True, lazy='dynamic'), lazy='dynamic', passive_deletes=True) @property def outputs(self): return self.outputs_q.all() @property def inputs(self): return self.inputs_q.all() # children via db_dbpath # suggest change name to descendants children_q = relationship("DbNode", secondary="db_dbpath", primaryjoin="DbNode.id == DbPath.parent_id", secondaryjoin="DbNode.id == DbPath.child_id", backref=backref("parents_q", passive_deletes=True, lazy='dynamic'), lazy='dynamic', passive_deletes=True) @property def children(self): return self.children_q.all() @property def parents(self): return self.parents_q.all() def __init__(self, *args, **kwargs): super(DbNode, self).__init__(*args, **kwargs) if self.attributes is None: self.attributes = dict() if self.extras is None: self.extras = dict() # XXX repetition between django/sqlalchemy here. def get_aiida_class(self): """ Return the corresponding aiida instance of class aiida.orm.Node or a appropriate subclass. """ from aiida.common.pluginloader import from_type_to_pluginclassname from aiida.orm.node import Node try: pluginclassname = from_type_to_pluginclassname(self.type) except DbContentError: raise DbContentError("The type name of node with pk= {} is " "not valid: '{}'".format(self.pk, self.type)) try: PluginClass = load_plugin(Node, 'aiida.orm', pluginclassname) except MissingPluginError: aiidalogger.error( "Unable to find plugin for type '{}' (node= {}), " "will use base Node class".format(self.type, self.pk)) PluginClass = Node return PluginClass(dbnode=self) def get_simple_name(self, invalid_result=None): """ Return a string with the last part of the type name. If the type is empty, use 'Node'. If the type is invalid, return the content of the input variable ``invalid_result``. :param invalid_result: The value to be returned if the node type is not recognized. """ thistype = self.type # Fix for base class if thistype == "": thistype = "node.Node." if not thistype.endswith("."): return invalid_result else: thistype = thistype[:-1] # Strip final dot return thistype.rpartition('.')[2] def set_attr(self, key, value): DbNode._set_attr(self.attributes, key, value) flag_modified(self, "attributes") def set_extra(self, key, value): DbNode._set_attr(self.extras, key, value) flag_modified(self, "extras") def reset_extras(self, new_extras): self.extras.clear() self.extras.update(new_extras) flag_modified(self, "extras") def del_attr(self, key): DbNode._del_attr(self.attributes, key) flag_modified(self, "attributes") def del_extra(self, key): DbNode._del_attr(self.extras, key) flag_modified(self, "extras") @staticmethod def _set_attr(d, key, value): if '.' in key: raise ValueError( "We don't know how to treat key with dot in it yet") d[key] = value @staticmethod def _del_attr(d, key): if '.' in key: raise ValueError( "We don't know how to treat key with dot in it yet") if key not in d: raise ValueError("Key {} does not exists".format(key)) del d[key] @property def pk(self): return self.id def __str__(self): simplename = self.get_simple_name(invalid_result="Unknown") # node pk + type if self.label: return "{} node [{}]: {}".format(simplename, self.pk, self.label) else: return "{} node [{}]".format(simplename, self.pk) @hybrid_property def state(self): """ Return the most recent state from DbCalcState """ if not self.id: return None all_states = DbCalcState.query.filter( DbCalcState.dbnode_id == self.id).all() if all_states: #return max((st.time, st.state) for st in all_states)[1] return sort_states(((dbcalcstate.state, dbcalcstate.state.value) for dbcalcstate in all_states), use_key=True)[0] else: return None @state.expression def state(cls): """ Return the expression to get the 'latest' state from DbCalcState, to be used in queries, where 'latest' is defined using the state order defined in _sorted_datastates. """ # Sort first the latest states whens = { v: idx for idx, v in enumerate(_sorted_datastates[::-1], start=1) } custom_sort_order = case( value=DbCalcState.state, whens=whens, else_=100) # else: high value to put it at the bottom q1 = select([ DbCalcState.id.label('id'), DbCalcState.dbnode_id.label('dbnode_id'), DbCalcState.state.label('state'), func.row_number().over( partition_by=DbCalcState.dbnode_id, order_by=custom_sort_order).label('the_row_number') ]) q1 = q1.cte() subq = select([ q1.c.dbnode_id.label('dbnode_id'), q1.c.state.label('state') ]).select_from(q1).where(q1.c.the_row_number == 1).alias() return select([subq.c.state]).\ where( subq.c.dbnode_id == cls.id, ).\ label('laststate')
class DbNode(Base): __tablename__ = "db_dbnode" aiida_query = _QueryProperty(_AiidaQuery) id = Column(Integer, primary_key=True) uuid = Column(UUID(as_uuid=True), default=uuid_func) type = Column(String(255), index=True) label = Column(String(255), index=True, nullable=True, default="") # Does it make sense to be nullable and have a default? description = Column(Text(), nullable=True, default="") ctime = Column(DateTime(timezone=True), default=timezone.now) mtime = Column(DateTime(timezone=True), default=timezone.now) nodeversion = Column(Integer, default=1) public = Column(Boolean, default=False) dbcomputer_id = Column( Integer, ForeignKey('db_dbcomputer.id', deferrable=True, initially="DEFERRED"), nullable=True ) user_id = Column( Integer, ForeignKey( 'db_dbuser.id',deferrable=True,initially="DEFERRED" ), nullable=False ) attributes = Column(JSONB, default={}) extras = Column(JSONB, default={}) # TODO SP: The 'passive_deletes=all' argument here means that SQLAlchemy # won't take care of automatic deleting in the DbLink table. This still # isn't exactly the same behaviour than with Django. The solution to # this is probably a ON DELETE inside the DB. On removing node with id=x, # we would remove all link with x as an output. ######### RELATIONSSHIPS ################ # Computer & user dbcomputer = relationship( 'DbComputer', backref=backref('dbnodes', passive_deletes=True) ) user = relationship('DbUser', backref='dbnodes') # outputs via db_dblink table outputs = relationship( "DbNode", secondary="db_dblink", primaryjoin="DbNode.id == DbLink.input_id", secondaryjoin="DbNode.id == DbLink.output_id", backref=backref("inputs", passive_deletes=True), passive_deletes=True ) # children via db_dbpath # suggest change name to descendants children = relationship( "DbNode", secondary="db_dbpath", primaryjoin="DbNode.id == DbPath.parent_id", secondaryjoin="DbNode.id == DbPath.child_id", backref=backref("parents", passive_deletes=True), passive_deletes=True, ) # Appender-query, so one can query the results: #~ dbcomputer_q = relationship( #~ 'DbComputer', #~ backref=backref('dbnodes_q', passive_deletes=True, lazy='dynamic'), #~ lazy='dynamic' #~ ) outputs_q = relationship( "DbNode", secondary="db_dblink", primaryjoin= "DbNode.id == DbLink.input_id", secondaryjoin= "DbNode.id == DbLink.output_id", backref=backref( "inputs_q", passive_deletes=True, lazy='dynamic' ), passive_deletes=True, lazy = 'dynamic' ) children_q = relationship( "DbNode", secondary="db_dbpath", primaryjoin="DbNode.id == DbPath.parent_id", secondaryjoin="DbNode.id == DbPath.child_id", backref=backref("parents_q", lazy='dynamic', passive_deletes=True), lazy='dynamic', passive_deletes=True ) # XXX repetition between django/sqlalchemy here. def get_aiida_class(self): """ Return the corresponding aiida instance of class aiida.orm.Node or a appropriate subclass. """ from aiida.common.pluginloader import from_type_to_pluginclassname from aiida.orm.node import Node try: pluginclassname = from_type_to_pluginclassname(self.type) except DbContentError: raise DbContentError("The type name of node with pk= {} is " "not valid: '{}'".format(self.pk, self.type)) try: PluginClass = load_plugin(Node, 'aiida.orm', pluginclassname) except MissingPluginError: aiidalogger.error("Unable to find plugin for type '{}' (node= {}), " "will use base Node class".format(self.type, self.pk)) PluginClass = Node return PluginClass(dbnode=self) def get_simple_name(self, invalid_result=None): """ Return a string with the last part of the type name. If the type is empty, use 'Node'. If the type is invalid, return the content of the input variable ``invalid_result``. :param invalid_result: The value to be returned if the node type is not recognized. """ thistype = self.type # Fix for base class if thistype == "": thistype = "node.Node." if not thistype.endswith("."): return invalid_result else: thistype = thistype[:-1] # Strip final dot return thistype.rpartition('.')[2] def set_attr(self, key, value): DbNode._set_attr(self.attributes, key, value) flag_modified(self, "attributes") def set_extra(self, key, value): DbNode._set_attr(self.extras, key, value) flag_modified(self, "extras") def del_attr(self, key): DbNode._del_attr(self.attributes, key) flag_modified(self, "attributes") def del_extra(self, key): DbNode._del_attr(self.extras, key) flag_modified(self, "extras") @staticmethod def _set_attr(d, key, value): if '.' in key: raise ValueError( "We don't know how to treat key with dot in it yet") d[key] = value @staticmethod def _del_attr(d, key): if '.' in key: raise ValueError("We don't know how to treat key with dot in it yet") if key not in d: raise ValueError("Key {} does not exists".format(key)) del d[key] @property def pk(self): return self.id def __str__(self): simplename = self.get_simple_name(invalid_result="Unknown") # node pk + type if self.label: return "{} node [{}]: {}".format(simplename, self.pk, self.label) else: return "{} node [{}]".format(simplename, self.pk)