def test_user_node_3(self):
"""Test that when a user and two nodes having that user are created,
storing only ONE NODE induces storage of that node, of the user but
not of the other node
Assert the correct storage of the user and node. Assert the
non-storage of the other node."""
# Create user
dbu1 = DbUser('tests3@schema', 'spam', 'eggs', 'monty')
# Creat node
node_dict = dict(user=dbu1)
dbn_1 = DbNode(**node_dict)
dbn_2 = DbNode(**node_dict)
# Check that the two are neither flushed nor committed
self.assertIsNone(dbu1.id)
self.assertIsNone(dbn_1.id)
self.assertIsNone(dbn_2.id)
session = aiida.backends.sqlalchemy.get_scoped_session()
# Add only first node and commit
session.add(dbn_1)
session.commit()
# Check for which object a pk has been assigned, which means that
# things have been at least flushed into the database
self.assertIsNotNone(dbu1.id)
self.assertIsNotNone(dbn_1.id)
self.assertIsNone(dbn_2.id)
def test_multiple_node_creation(self):
"""
This test checks that a node is not added automatically to the session
(and subsequently committed) when a user is in the session.
It tests the fix for the issue #234
"""
from aiida.backends.sqlalchemy.models.node import DbNode
from aiida.common.utils import get_new_uuid
import aiida.backends.sqlalchemy
backend = self.backend
# Get the automatic user
dbuser = backend.users.create('{}@aiida.net'.format(
self.id())).store().dbmodel
# Create a new node but don't add it to the session
node_uuid = get_new_uuid()
DbNode(user=dbuser, uuid=node_uuid, node_type=None)
session = aiida.backends.sqlalchemy.get_scoped_session()
# Query the session before commit
res = session.query(DbNode.uuid).filter(DbNode.uuid == node_uuid).all()
self.assertEqual(
len(res), 0,
'There should not be any nodes with this UUID in the session/DB.')
# Commit the transaction
session.commit()
# Check again that the node is not in the DB
res = session.query(DbNode.uuid).filter(DbNode.uuid == node_uuid).all()
self.assertEqual(
len(res), 0,
'There should not be any nodes with this UUID in the session/DB.')
# Get the automatic user
dbuser = orm.User.objects.get_default().backend_entity.dbmodel
# Create a new node but now add it to the session
node_uuid = get_new_uuid()
node = DbNode(user=dbuser, uuid=node_uuid, node_type=None)
session.add(node)
# Query the session before commit
res = session.query(DbNode.uuid).filter(DbNode.uuid == node_uuid).all()
self.assertEqual(
len(res), 1,
'There should be a node in the session/DB with the UUID {}'.format(
node_uuid))
# Commit the transaction
session.commit()
# Check again that the node is in the db
res = session.query(DbNode.uuid).filter(DbNode.uuid == node_uuid).all()
self.assertEqual(
len(res), 1,
'There should be a node in the session/DB with the UUID {}'.format(
node_uuid))
def test_multiple_node_creation(self):
"""
This test checks that a node is not added automatically to the session
(and subsequently committed) when a user is in the session.
It tests the fix for the issue #234
"""
from aiida.backends.sqlalchemy.models.node import DbNode
from aiida.common.utils import get_new_uuid
from aiida.backends.utils import get_automatic_user
import aiida.backends.sqlalchemy
# Get the automatic user
user = get_automatic_user()
# Create a new node but don't add it to the session
node_uuid = get_new_uuid()
DbNode(user=user, uuid=node_uuid, type=None)
session = aiida.backends.sqlalchemy.get_scoped_session()
# Query the session before commit
res = session.query(DbNode.uuid).filter(
DbNode.uuid == node_uuid).all()
self.assertEqual(len(res), 0, "There should not be any nodes with this"
"UUID in the session/DB.")
# Commit the transaction
session.commit()
# Check again that the node is not in the DB
res = session.query(DbNode.uuid).filter(
DbNode.uuid == node_uuid).all()
self.assertEqual(len(res), 0, "There should not be any nodes with this"
"UUID in the session/DB.")
# Get the automatic user
user = get_automatic_user()
# Create a new node but now add it to the session
node_uuid = get_new_uuid()
node = DbNode(user=user, uuid=node_uuid, type=None)
session.add(node)
# Query the session before commit
res = session.query(DbNode.uuid).filter(
DbNode.uuid == node_uuid).all()
self.assertEqual(len(res), 1,
"There should be a node in the session/DB with the "
"UUID {}".format(node_uuid))
# Commit the transaction
session.commit()
# Check again that the node is in the db
res = session.query(DbNode.uuid).filter(
DbNode.uuid == node_uuid).all()
self.assertEqual(len(res), 1,
"There should be a node in the session/DB with the "
"UUID {}".format(node_uuid))
def __init__(self, **kwargs):
super(Node, self).__init__()
self._temp_folder = None
dbnode = kwargs.pop('dbnode', None)
# Set the internal parameters
# Can be redefined in the subclasses
self._init_internal_params()
if dbnode is not None:
if not isinstance(dbnode, DbNode):
raise TypeError("dbnode is not a DbNode instance")
if dbnode.id is None:
raise ValueError("If cannot load an aiida.orm.Node instance "
"from an unsaved DbNode object.")
if kwargs:
raise ValueError("If you pass a dbnode, you cannot pass any "
"further parameter")
# If I am loading, I cannot modify it
self._to_be_stored = False
self._dbnode = dbnode
# If this is changed, fix also the importer
self._repo_folder = RepositoryFolder(section=self._section_name,
uuid=self._dbnode.uuid)
else:
# TODO: allow to get the user from the parameters
user = get_automatic_user()
self._dbnode = DbNode(user=user,
uuid=get_new_uuid(),
type=self._plugin_type_string)
self._to_be_stored = True
# As creating the temp folder may require some time on slow
# filesystems, we defer its creation
self._temp_folder = None
# Used only before the first save
self._attrs_cache = {}
# If this is changed, fix also the importer
self._repo_folder = RepositoryFolder(section=self._section_name,
uuid=self.uuid)
# Automatically set all *other* attributes, if possible, otherwise
# stop
self._set_with_defaults(**kwargs)
def test_user_node_1(self):
"""Test that when a user and a node having that user are created,
storing NODE induces storage of the USER
Assert the correct storage of user and node."""
# Create user
dbu1 = DbUser('test1@schema', 'spam', 'eggs', 'monty')
# Creat node
node_dict = dict(user=dbu1)
dbn_1 = DbNode(**node_dict)
# Check that the two are neither flushed nor committed
self.assertIsNone(dbu1.id)
self.assertIsNone(dbn_1.id)
session = aiida.backends.sqlalchemy.get_scoped_session()
# Add only the node and commit
session.add(dbn_1)
session.commit()
# Check that a pk has been assigned, which means that things have
# been flushed into the database
self.assertIsNotNone(dbn_1.id)
self.assertIsNotNone(dbu1.id)
def test_user_node_4(self):
"""Test that when several nodes are created with the same user and each
of them is assigned to the same name, storage of last node object
associated to that node does not trigger storage of all objects.
Assert the correct storage of the user and node. Assert the
non-storage of the other nodes."""
# Create user
dbu1 = DbUser('tests4@schema', 'spam', 'eggs', 'monty')
# Creat node objects assigningd them to the same name
# Check https://docs.python.org/2/tutorial/classes.html subsec. 9.1
for _ in range(5):
# It is important to change the uuid each time (or any other
# variable) so that a different objects (with a different pointer)
# is actually created in this scope.
dbn_1 = DbNode(user=dbu1, uuid=get_new_uuid())
# Check that the two are neither flushed nor committed
self.assertIsNone(dbu1.id)
self.assertIsNone(dbn_1.id)
session = aiida.backends.sqlalchemy.get_scoped_session()
# Add only first node and commit
session.add(dbn_1)
session.commit()
# Check for which object a pk has been assigned, which means that
# things have been at least flushed into the database
self.assertIsNotNone(dbu1.id)
self.assertIsNotNone(dbn_1.id)
def test_user_node_2(self):
"""Test that when a user and a node having that user are created,
storing USER does NOT induce storage of the NODE
Assert the correct storage of user and node."""
import warnings
from sqlalchemy import exc as sa_exc
# Create user
dbu1 = DbUser('tests2@schema', 'spam', 'eggs', 'monty')
# Creat node
node_dict = dict(user=dbu1)
dbn_1 = DbNode(**node_dict)
# Check that the two are neither flushed nor committed
self.assertIsNone(dbu1.id)
self.assertIsNone(dbn_1.id)
session = aiida.backends.sqlalchemy.get_scoped_session()
# Catch all the SQLAlchemy warnings generated by the following code
with warnings.catch_warnings(): # pylint: disable=no-member
warnings.simplefilter('ignore', category=sa_exc.SAWarning) # pylint: disable=no-member
# Add only the user and commit
session.add(dbu1)
session.commit()
# Check that a pk has been assigned (or not), which means that things
# have been flushed into the database
self.assertIsNotNone(dbu1.id)
self.assertIsNone(dbn_1.id)
def test_User_node_2(self):
"""
Test that when a user and a node having that user are created,
storing USER does NOT induce storage of the NODE
Assert the correct storage of user and node
"""
import warnings
from sqlalchemy import exc as sa_exc
# Create user
dbu1 = DbUser('tests2@schema', "spam", "eggs", "monty")
# Creat node
node_dict = dict(user=dbu1)
dbn1 = DbNode(**node_dict)
# Check that the two are neither flushed nor committed
self.assertIsNone(dbu1.id)
self.assertIsNone(dbn1.id)
# Catch all the SQLAlchemy warnings generated by the following code
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=sa_exc.SAWarning)
# Add only the user and commit
aiida.backends.sqlalchemy.session.add(dbu1)
aiida.backends.sqlalchemy.session.commit()
# Check that a pk has been assigned (or not), which means that things
# have been flushed into the database
self.assertIsNotNone(dbu1.id)
self.assertIsNone(dbn1.id)
class Node(AbstractNode):
def __init__(self, **kwargs):
super(Node, self).__init__()
self._temp_folder = None
dbnode = kwargs.pop('dbnode', None)
# Set the internal parameters
# Can be redefined in the subclasses
self._init_internal_params()
if dbnode is not None:
if not isinstance(dbnode, DbNode):
raise TypeError("dbnode is not a DbNode instance")
if dbnode.id is None:
raise ValueError("If cannot load an aiida.orm.Node instance "
"from an unsaved DbNode object.")
if kwargs:
raise ValueError("If you pass a dbnode, you cannot pass any "
"further parameter")
# If I am loading, I cannot modify it
self._to_be_stored = False
self._dbnode = dbnode
# If this is changed, fix also the importer
self._repo_folder = RepositoryFolder(section=self._section_name,
uuid=self._dbnode.uuid)
else:
# TODO: allow to get the user from the parameters
user = get_automatic_user()
self._dbnode = DbNode(user=user,
uuid=get_new_uuid(),
type=self._plugin_type_string)
self._to_be_stored = True
# As creating the temp folder may require some time on slow
# filesystems, we defer its creation
self._temp_folder = None
# Used only before the first save
self._attrs_cache = {}
# If this is changed, fix also the importer
self._repo_folder = RepositoryFolder(section=self._section_name,
uuid=self.uuid)
# Automatically set all *other* attributes, if possible, otherwise
# stop
self._set_with_defaults(**kwargs)
@staticmethod
def get_db_columns():
return get_db_columns(DbNode)
@classmethod
def get_subclass_from_uuid(cls, uuid):
from aiida.orm.querybuilder import QueryBuilder
from sqlalchemy.exc import DatabaseError
try:
qb = QueryBuilder()
qb.append(cls, filters={'uuid': {'==': str(uuid)}})
if qb.count() == 0:
raise NotExistent("No entry with UUID={} found".format(uuid))
node = qb.first()[0]
if not isinstance(node, cls):
raise NotExistent("UUID={} is not an instance of {}".format(
uuid, cls.__name__))
return node
except DatabaseError as de:
raise ValueError(de.message)
@classmethod
def get_subclass_from_pk(cls, pk):
from aiida.orm.querybuilder import QueryBuilder
from sqlalchemy.exc import DatabaseError
# If it is not an int make a final attempt
# to convert to an integer. If you fail,
# raise an exception.
try:
pk = int(pk)
except:
raise ValueError("Incorrect type for int")
try:
qb = QueryBuilder()
qb.append(cls, filters={'id': {'==': pk}})
if qb.count() == 0:
raise NotExistent("No entry with pk= {} found".format(pk))
node = qb.first()[0]
if not isinstance(node, cls):
raise NotExistent("pk= {} is not an instance of {}".format(
pk, cls.__name__))
return node
except DatabaseError as de:
raise ValueError(de.message)
@classmethod
def query(cls, *args, **kwargs):
raise NotImplementedError("The node query method is not supported in "
"SQLAlchemy. Please use QueryBuilder.")
def _update_db_label_field(self, field_value):
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
self.dbnode.label = field_value
if not self._to_be_stored:
session.add(self._dbnode)
self._increment_version_number_db()
def _update_db_description_field(self, field_value):
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
self.dbnode.description = field_value
if not self._to_be_stored:
session.add(self._dbnode)
self._increment_version_number_db()
def _replace_dblink_from(self, src, label, link_type):
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
try:
self._add_dblink_from(src, label)
except UniquenessError:
# I have to replace the link; I do it within a transaction
try:
self._remove_dblink_from(label)
self._add_dblink_from(src, label, link_type)
session.commit()
except:
session.rollback()
raise
def _remove_dblink_from(self, label):
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
link = DbLink.query.filter_by(label=label).first()
if link is not None:
session.delete(link)
def _add_dblink_from(self,
src,
label=None,
link_type=LinkType.UNSPECIFIED):
from aiida.backends.sqlalchemy import get_scoped_session
from aiida.orm.querybuilder import QueryBuilder
session = get_scoped_session()
if not isinstance(src, Node):
raise ValueError("src must be a Node instance")
if self.uuid == src.uuid:
raise ValueError("Cannot link to itself")
if self._to_be_stored:
raise ModificationNotAllowed(
"Cannot call the internal _add_dblink_from if the "
"destination node is not stored")
if src._to_be_stored:
raise ModificationNotAllowed(
"Cannot call the internal _add_dblink_from if the "
"source node is not stored")
# Check for cycles. This works if the transitive closure is enabled; if
# it isn't, this test will never fail, but then having a circular link
# is not meaningful but does not pose a huge threat
#
# I am linking src->self; a loop would be created if a DbPath exists
# already in the TC table from self to src
if link_type is LinkType.CREATE or link_type is LinkType.INPUT:
if QueryBuilder().append(
Node, filters={
'id': self.pk
}, tag='parent').append(
Node,
filters={
'id': src.pk
},
tag='child',
descendant_of='parent').count() > 0:
raise ValueError(
"The link you are attempting to create would generate a loop"
)
if label is None:
autolabel_idx = 1
existing_from_autolabels = session.query(DbLink.label).filter(
DbLink.output_id == self.dbnode.id, DbLink.label.like("link%"))
while "link_{}".format(autolabel_idx) in existing_from_autolabels:
autolabel_idx += 1
safety_counter = 0
while True:
safety_counter += 1
if safety_counter > 100:
# Well, if you have more than 100 concurrent addings
# to the same node, you are clearly doing something wrong...
raise InternalError(
"Hey! We found more than 100 concurrent"
" adds of links "
"to the same nodes! Are you really doing that??")
try:
self._do_create_link(src, "link_{}".format(autolabel_idx),
link_type)
break
except UniquenessError:
# Retry loop until you find a new loop
autolabel_idx += 1
else:
self._do_create_link(src, label, link_type)
def _do_create_link(self, src, label, link_type):
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
try:
with session.begin_nested():
link = DbLink(input_id=src.dbnode.id,
output_id=self.dbnode.id,
label=label,
type=link_type.value)
session.add(link)
except SQLAlchemyError as e:
raise UniquenessError("There is already a link with the same "
"name (raw message was {})"
"".format(e))
def _get_db_input_links(self, link_type):
link_filter = {'output': self.dbnode}
if link_type is not None:
link_filter['type'] = link_type.value
return [(i.label, i.input.get_aiida_class())
for i in DbLink.query.filter_by(
output=self.dbnode).distinct().all()]
def _get_db_output_links(self, link_type):
link_filter = {'input': self.dbnode}
if link_type is not None:
link_filter['type'] = link_type.value
return ((i.label, i.output.get_aiida_class())
for i in DbLink.query.filter_by(
**link_filter).distinct().all())
def _set_db_computer(self, computer):
self.dbnode.dbcomputer = DbComputer.get_dbcomputer(computer)
def _set_db_attr(self, key, value):
"""
Set the value directly in the DB, without checking if it is stored, or
using the cache.
DO NOT USE DIRECTLY.
:param str key: key name
:param value: its value
"""
try:
self.dbnode.set_attr(key, value)
self._increment_version_number_db()
except:
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
session.rollback()
raise
def _del_db_attr(self, key):
try:
self.dbnode.del_attr(key)
self._increment_version_number_db()
except:
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
session.rollback()
raise
def _get_db_attr(self, key):
try:
return get_attr(self.dbnode.attributes, key)
except (KeyError, IndexError):
raise AttributeError("Attribute '{}' does not exist".format(key))
def _set_db_extra(self, key, value, exclusive=False):
if exclusive:
raise NotImplementedError(
"exclusive=True not implemented yet in SQLAlchemy backend")
try:
self.dbnode.set_extra(key, value)
self._increment_version_number_db()
except:
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
session.rollback()
raise
def _reset_db_extras(self, new_extras):
try:
self.dbnode.reset_extras(new_extras)
self._increment_version_number_db()
except:
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
session.rollback()
raise
def _get_db_extra(self, key, default=None):
try:
return get_attr(self.dbnode.extras, key)
except (KeyError, AttributeError):
raise AttributeError("DbExtra {} does not exist".format(key))
def _del_db_extra(self, key):
try:
self.dbnode.del_extra(key)
self._increment_version_number_db()
except:
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
session.rollback()
raise
def _db_iterextras(self):
if self.dbnode.extras is None:
return dict().iteritems()
return self.dbnode.extras.iteritems()
def _db_iterattrs(self):
for k, v in self.dbnode.attributes.iteritems():
yield (k, v)
def _db_attrs(self):
for k in self.dbnode.attributes.iterkeys():
yield k
def add_comment(self, content, user=None):
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
if self._to_be_stored:
raise ModificationNotAllowed("Comments can be added only after "
"storing the node")
comment = DbComment(dbnode=self._dbnode, user=user, content=content)
session.add(comment)
try:
session.commit()
except:
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
session.rollback()
raise
def get_comment_obj(self, id=None, user=None):
dbcomments_query = DbComment.query.filter_by(dbnode=self._dbnode)
if id is not None:
dbcomments_query = dbcomments_query.filter_by(id=id)
if user is not None:
dbcomments_query = dbcomments_query.filter_by(user=user)
dbcomments = dbcomments_query.all()
comments = []
from aiida.orm.implementation.sqlalchemy.comment import Comment
for dbcomment in dbcomments:
comments.append(Comment(dbcomment=dbcomment))
return comments
def get_comments(self, pk=None):
comments = self._get_dbcomments(pk)
return [{
"pk": c.id,
"user__email": c.user.email,
"ctime": c.ctime,
"mtime": c.mtime,
"content": c.content
} for c in comments]
def _get_dbcomments(self, pk=None, with_user=False):
comments = DbComment.query.filter_by(dbnode=self._dbnode)
if pk is not None:
try:
correct = all([isinstance(_, int) for _ in pk])
if not correct:
raise ValueError(
'id must be an integer or a list of integers')
comments = comments.filter(DbComment.id.in_(pk))
except TypeError:
if not isinstance(pk, int):
raise ValueError(
'id must be an integer or a list of integers')
comments = comments.filter_by(id=pk)
if with_user:
comments.join(DbUser)
comments = comments.order_by('id').all()
return comments
def _update_comment(self, new_field, comment_pk, user):
comment = DbComment.query.filter_by(dbnode=self._dbnode,
id=comment_pk,
user=user).first()
if not isinstance(new_field, basestring):
raise ValueError("Non string comments are not accepted")
if not comment:
raise NotExistent("Found no comment for user {} and id {}".format(
user, comment_pk))
comment.content = new_field
try:
comment.save()
except:
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
session.rollback()
raise
def _remove_comment(self, comment_pk, user):
comment = DbComment.query.filter_by(dbnode=self._dbnode,
id=comment_pk).first()
if comment:
try:
comment.delete()
except:
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
session.rollback()
raise
def _increment_version_number_db(self):
self._dbnode.nodeversion = DbNode.nodeversion + 1
try:
self._dbnode.save()
except:
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
session.rollback()
raise
def copy(self):
newobject = self.__class__()
newobject.dbnode.type = self.dbnode.type # Inherit type
newobject.dbnode.label = self.dbnode.label # Inherit label
# TODO: add to the description the fact that this was a copy?
newobject.dbnode.description = self.dbnode.description # Inherit description
newobject.dbnode.dbcomputer = self.dbnode.dbcomputer # Inherit computer
for k, v in self.iterattrs():
if k != Sealable.SEALED_KEY:
newobject._set_attr(k, v)
for path in self.get_folder_list():
newobject.add_path(self.get_abs_path(path), path)
return newobject
@property
def id(self):
return self.dbnode.id
@property
def dbnode(self):
return self._dbnode
def store_all(self, with_transaction=True):
"""
Store the node, together with all input links, if cached, and also the
linked nodes, if they were not stored yet.
:parameter with_transaction: if False, no transaction is used. This
is meant to be used ONLY if the outer calling function has already
a transaction open!
"""
if not self._to_be_stored:
raise ModificationNotAllowed(
"Node with pk= {} was already stored".format(self.id))
# For each parent, check that all its inputs are stored
for link in self._inputlinks_cache:
try:
parent_node = self._inputlinks_cache[link][0]
parent_node._check_are_parents_stored()
except ModificationNotAllowed:
raise ModificationNotAllowed(
"Parent node (UUID={}) has "
"unstored parents, cannot proceed (only direct parents "
"can be unstored and will be stored by store_all, not "
"grandparents or other ancestors".format(parent_node.uuid))
self._store_input_nodes()
self.store(with_transaction=False)
self._store_cached_input_links(with_transaction=False)
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
if with_transaction:
try:
session.commit()
except SQLAlchemyError as e:
session.rollback()
raise
return self
def _store_cached_input_links(self, with_transaction=True):
"""
Store all input links that are in the local cache, transferring them
to the DB.
:note: This can be called only if all parents are already stored.
:note: Links are stored only after the input nodes are stored. Moreover,
link storage is done in a transaction, and if one of the links
cannot be stored, an exception is raised and *all* links will remain
in the cache.
:note: This function can be called only after the node is stored.
After that, it can be called multiple times, and nothing will be
executed if no links are still in the cache.
:parameter with_transaction: if False, no transaction is used. This
is meant to be used ONLY if the outer calling function has already
a transaction open!
"""
if self._to_be_stored:
raise ModificationNotAllowed(
"Node with pk= {} is not stored yet".format(self.id))
# This raises if there is an unstored node.
self._check_are_parents_stored()
# I have to store only those links where the source is already
# stored
links_to_store = list(self._inputlinks_cache.keys())
for label in links_to_store:
src, link_type = self._inputlinks_cache[label]
self._add_dblink_from(src, label, link_type)
# If everything went smoothly, clear the entries from the cache.
# I do it here because I delete them all at once if no error
# occurred; otherwise, links will not be stored and I
# should not delete them from the cache (but then an exception
# would have been raised, and the following lines are not executed)
self._inputlinks_cache.clear()
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
if with_transaction:
try:
session.commit()
except SQLAlchemyError as e:
session.rollback()
raise
def store(self, with_transaction=True):
"""
Store a new node in the DB, also saving its repository directory
and attributes.
After being called attributes cannot be
changed anymore! Instead, extras can be changed only AFTER calling
this store() function.
:note: After successful storage, those links that are in the cache, and
for which also the parent node is already stored, will be
automatically stored. The others will remain unstored.
:parameter with_transaction: if False, no transaction is used. This
is meant to be used ONLY if the outer calling function has already
a transaction open!
"""
from aiida.backends.sqlalchemy import get_scoped_session
session = get_scoped_session()
# TODO: This needs to be generalized, allowing for flexible methods
# for storing data and its attributes.
if self._to_be_stored:
self._validate()
self._check_are_parents_stored()
# I save the corresponding django entry
# I set the folder
# NOTE: I first store the files, then only if this is successful,
# I store the DB entry. In this way,
# I assume that if a node exists in the DB, its folder is in place.
# On the other hand, periodically the user might need to run some
# bookkeeping utility to check for lone folders.
self._repository_folder.replace_with_folder(
self._get_temp_folder().abspath, move=True, overwrite=True)
# import aiida.backends.sqlalchemy
try:
# aiida.backends.sqlalchemy.get_scoped_session().add(self._dbnode)
session.add(self._dbnode)
# Save its attributes 'manually' without incrementing
# the version for each add.
self.dbnode.attributes = self._attrs_cache
flag_modified(self.dbnode, "attributes")
# This should not be used anymore: I delete it to
# possibly free memory
del self._attrs_cache
self._temp_folder = None
self._to_be_stored = False
# Here, I store those links that were in the cache and
# that are between stored nodes.
self._store_cached_input_links(with_transaction=False)
if with_transaction:
try:
# aiida.backends.sqlalchemy.get_scoped_session().commit()
session.commit()
except SQLAlchemyError as e:
#print "Cannot store the node. Original exception: {" \
# "}".format(e)
session.rollback()
raise
# This is one of the few cases where it is ok to do a 'global'
# except, also because I am re-raising the exception
except:
# I put back the files in the sandbox folder since the
# transaction did not succeed
self._get_temp_folder().replace_with_folder(
self._repository_folder.abspath, move=True, overwrite=True)
raise
# Set up autogrouping used be verdi run
autogroup = aiida.orm.autogroup.current_autogroup
grouptype = aiida.orm.autogroup.VERDIAUTOGROUP_TYPE
if autogroup is not None:
if not isinstance(autogroup, aiida.orm.autogroup.Autogroup):
raise ValidationError(
"current_autogroup is not an AiiDA Autogroup")
if autogroup.is_to_be_grouped(self):
group_name = autogroup.get_group_name()
if group_name is not None:
g = Group.get_or_create(name=group_name,
type_string=grouptype)[0]
g.add_nodes(self)
return self
@property
def uuid(self):
return unicode(self.dbnode.uuid)
class Node(AbstractNode):
def __init__(self, **kwargs):
super(Node, self).__init__()
self._temp_folder = None
dbnode = kwargs.pop('dbnode', None)
# Set the internal parameters
# Can be redefined in the subclasses
self._init_internal_params()
if dbnode is not None:
if not isinstance(dbnode, DbNode):
raise TypeError("dbnode is not a DbNode instance")
if dbnode.id is None:
raise ValueError("If cannot load an aiida.orm.Node instance "
"from an unsaved DbNode object.")
if kwargs:
raise ValueError("If you pass a dbnode, you cannot pass any "
"further parameter")
# If I am loading, I cannot modify it
self._to_be_stored = False
self._dbnode = dbnode
# If this is changed, fix also the importer
self._repo_folder = RepositoryFolder(section=self._section_name,
uuid=self._dbnode.uuid)
else:
# TODO: allow to get the user from the parameters
user = get_automatic_user()
self._dbnode = DbNode(user=user,
uuid=get_new_uuid(),
type=self._plugin_type_string)
self._to_be_stored = True
# As creating the temp folder may require some time on slow
# filesystems, we defer its creation
self._temp_folder = None
# Used only before the first save
self._attrs_cache = {}
# If this is changed, fix also the importer
self._repo_folder = RepositoryFolder(section=self._section_name,
uuid=self.uuid)
# Automatically set all *other* attributes, if possible, otherwise
# stop
self._set_with_defaults(**kwargs)
@classmethod
def get_subclass_from_uuid(cls, uuid):
try:
node = cls.query(uuid=uuid).one()
except (NoResultFound, ProgrammingError):
raise NotExistent("No entry with UUID={} found".format(uuid))
if not isinstance(node, cls):
raise NotExistent("UUID={} is not an instance of {}".format(
uuid, cls.__name__))
return node
@classmethod
def get_subclass_from_pk(cls, pk):
if not isinstance(pk, int):
raise ValueError("Incorrect type for int")
try:
from aiida.backends.sqlalchemy import session
node = cls.query(id=pk).one()
except NoResultFound:
# DataError is thrown when you pass a string instead of an int for
# instance
raise NotExistent("No entry with pk= {} found".format(pk))
if not isinstance(node, cls):
raise NotExistent("pk= {} is not an instance of {}".format(
pk, cls.__name__))
return node
def __int__(self):
if self._to_be_store:
return None
else:
return self._dbnode.id
@classmethod
def query(cls, *args, **kwargs):
q = django_filter(DbNode.aiida_query, **kwargs)
q = q.reset_joinpoint()
if cls._plugin_type_string:
if not cls._plugin_type_string.endswith('.'):
raise InternalError("The plugin type string does not "
"finish with a dot??")
# If it is 'calculation.Calculation.', we want to filter
# for things that start with 'calculation.' and so on
plug_type = cls._plugin_type_string
# Remove the implementation.django or sqla part.
if plug_type.startswith('implementation.'):
plug_type = '.'.join(plug_type.split('.')[2:])
pre, sep, _ = plug_type[:-1].rpartition('.')
superclass_string = "".join([pre, sep])
q = q.filter(DbNode.type.like("{}%".format(superclass_string)))
return q
def _update_db_label_field(self, field_value):
from aiida.backends.sqlalchemy import session
self.dbnode.label = field_value
if not self._to_be_stored:
self._dbnode.save(commit=False)
self._increment_version_number_db()
def _update_db_description_field(self, field_value):
from aiida.backends.sqlalchemy import session
self.dbnode.description = field_value
if not self._to_be_stored:
self._dbnode.save(commit=False)
self._increment_version_number_db()
def _replace_dblink_from(self, src, label, link_type):
from aiida.backends.sqlalchemy import session
try:
self._add_dblink_from(src, label)
except UniquenessError:
# I have to replace the link; I do it within a transaction
self._remove_dblink_from(label)
self._add_dblink_from(src, label, link_type)
def _remove_dblink_from(self, label):
from aiida.backends.sqlalchemy import session
link = self.dbnode.outputs.filter_by(label=label).first()
session.delete(link)
session.commit()
def _add_dblink_from(self,
src,
label=None,
link_type=LinkType.UNSPECIFIED):
from aiida.backends.sqlalchemy import session
if not isinstance(src, Node):
raise ValueError("src must be a Node instance")
if self.uuid == src.uuid:
raise ValueError("Cannot link to itself")
if self._to_be_stored:
raise ModificationNotAllowed(
"Cannot call the internal _add_dblink_from if the "
"destination node is not stored")
if src._to_be_stored:
raise ModificationNotAllowed(
"Cannot call the internal _add_dblink_from if the "
"source node is not stored")
# Check for cycles. This works if the transitive closure is enabled; if
# it isn't, this test will never fail, but then having a circular link
# is not meaningful but does not pose a huge threat
#
# I am linking src->self; a loop would be created if a DbPath exists
# already in the TC table from self to src
c = session.query(literal(True)).filter(
DbPath.query.filter_by(parent_id=self.dbnode.id,
child_id=src.dbnode.id).exists()).scalar()
if c:
raise ValueError(
"The link you are attempting to create would generate a loop")
if label is None:
autolabel_idx = 1
existing_from_autolabels = session.query(DbLink.label).filter(
DbLink.output_id == self.dbnode.id, DbLink.label.like("link%"))
while "link_{}".format(autolabel_idx) in existing_from_autolabels:
autolabel_idx += 1
safety_counter = 0
while True:
safety_counter += 1
if safety_counter > 3:
# Well, if you have more than 100 concurrent addings
# to the same node, you are clearly doing something wrong...
raise InternalError(
"Hey! We found more than 100 concurrent"
" adds of links "
"to the same nodes! Are you really doing that??")
try:
self._do_create_link(src, "link_{}".format(autolabel_idx),
link_type)
break
except UniquenessError:
# Retry loop until you find a new loop
autolabel_idx += 1
else:
self._do_create_link(src, label, link_type)
def _do_create_link(self, src, label, link_type):
from aiida.backends.sqlalchemy import session
try:
link = DbLink(input_id=src.dbnode.id,
output_id=self.dbnode.id,
label=label,
type=link_type.value)
session.add(link)
session.commit()
except SQLAlchemyError as e:
session.rollback()
raise UniquenessError("There is already a link with the same "
"name (raw message was {})"
"".format(e))
def get_inputs(self,
node_type=None,
also_labels=False,
only_in_db=False,
link_type=None):
link_filter = {'output': self.dbnode}
if link_type is not None:
link_filter['type'] = link_type.value
inputs_list = [(i.label, i.input.get_aiida_class())
for i in DbLink.query.filter_by(
output=self.dbnode).distinct().all()]
if not only_in_db:
# Needed for the check
input_list_keys = [i[0] for i in inputs_list]
for k, v in self._inputlinks_cache.iteritems():
src = v[0]
if k in input_list_keys:
raise InternalError(
"There exist a link with the same name "
"'{}' both in the DB and in the internal "
"cache for node pk= {}!".format(k, self.id))
inputs_list.append((k, src))
if node_type is None:
filtered_list = inputs_list
else:
filtered_list = [
i for i in inputs_list if isinstance(i[1], node_type)
]
if also_labels:
return list(filtered_list)
else:
return [i[1] for i in filtered_list]
@override
def get_outputs(self, type=None, also_labels=False, link_type=None):
link_filter = {'input': self.dbnode}
if link_type is not None:
link_filter['type'] = link_type.value
outputs_list = ((i.label, i.output.get_aiida_class())
for i in DbLink.query.filter_by(
**link_filter).distinct().all())
if type is None:
if also_labels:
return list(outputs_list)
else:
return [i[1] for i in outputs_list]
else:
filtered_list = (i for i in outputs_list if isinstance(i[1], type))
if also_labels:
return list(filtered_list)
else:
return [i[1] for i in filtered_list]
def set_computer(self, computer):
if self._to_be_stored:
computer = DbComputer.get_dbcomputer(computer)
self.dbnode.dbcomputer = computer
else:
raise ModificationNotAllowed(
"Node with uuid={} was already stored".format(self.uuid))
def _set_attr(self, key, value):
if self._to_be_stored:
self._attrs_cache[key] = copy.deepcopy(value)
else:
self.dbnode.set_attr(key, value)
self._increment_version_number_db()
def _del_attr(self, key):
if self._to_be_stored:
try:
del self._attrs_cache[key]
except KeyError:
raise AttributeError("Attribute {} does not exist".format(key))
else:
self.dbnode.del_attr(key)
self._increment_version_number_db()
def get_attr(self, key, default=_NO_DEFAULT):
exception = AttributeError("Attribute '{}' does not exist".format(key))
has_default = default is not _NO_DEFAULT
if self._to_be_stored:
try:
return self._attrs_cache[key]
except KeyError:
if has_default:
return default
raise exception
else:
try:
return get_attr(self.dbnode.attributes, key)
except (KeyError, IndexError):
if has_default:
return default
else:
raise exception
def set_extra(self, key, value, exclusive=False):
# TODO SP: validate key
# TODO SP: handle exclusive (what to do in case the key already exist
# ?)
if self._to_be_stored:
raise ModificationNotAllowed(
"The extras of a node can be set only after "
"storing the node")
self.dbnode.set_extra(key, value)
self._increment_version_number_db()
def get_extra(self, key, default=None):
# TODO SP: in the Django implementation, if the node is not stored,
# we can't get an extra. In the SQLA one, because this is simply a
# column, we could still return one if it exists.
try:
return get_attr(self.dbnode.extras, key)
except (KeyError, IndexError) as e:
if default:
return default
else:
raise AttributeError
def get_extras(self):
return self.dbnode.extras
def del_extra(self, key):
self.dbnode.del_extra(key)
def extras(self):
return self.dbnode.extras
def iterextras(self):
return self.dbnode.extras.iteritems()
def iterattrs(self):
# TODO: check what happens if someone stores the object while
# the iterator is being used!
if self._to_be_stored:
it_items = self._attrs_cache.iteritems()
else:
it_items = self.dbnode.attributes.iteritems()
for k, v in it_items:
yield (k, v)
def get_attrs(self):
return self.dbnode.attributes
def attrs(self):
if self._to_be_stored:
it = self._attrs_cache.iterkeys()
else:
it = self.dbnode.attributes.iterkeys()
for k in it:
yield k
def add_comment(self, content, user=None):
from aiida.backends.sqlalchemy import session
if self._to_be_stored:
raise ModificationNotAllowed("Comments can be added only after "
"storing the node")
comment = DbComment(dbnode=self._dbnode, user=user, content=content)
session.add(comment)
session.commit()
def get_comment_obj(self, id=None, user=None):
dbcomments_query = DbComment.query.filter_by(dbnode=self._dbnode)
if id is not None:
dbcomments_query = dbcomments_query.filter_by(id=id)
if user is not None:
dbcomments_query = dbcomments_query.filter_by(user=user)
dbcomments = dbcomments_query.all()
comments = []
from aiida.orm.implementation.sqlalchemy.comment import Comment
for dbcomment in dbcomments:
comments.append(Comment(dbcomment=dbcomment))
return comments
def get_comments(self, pk=None):
comments = self._get_dbcomments(pk)
return [{
"pk": c.id,
"user__email": c.user.email,
"ctime": c.ctime,
"mtime": c.mtime,
"content": c.content
} for c in comments]
def _get_dbcomments(self, pk=None, with_user=False):
comments = DbComment.query.filter_by(dbnode=self._dbnode)
if pk is not None:
try:
correct = all([isinstance(_, int) for _ in pk])
if not correct:
raise ValueError(
'id must be an integer or a list of integers')
comments = comments.filter(DbComment.id.in_(pk))
except TypeError:
if not isinstance(pk, int):
raise ValueError(
'id must be an integer or a list of integers')
comments = comments.filter_by(id=pk)
if with_user:
comments.join(DbUser)
comments = comments.order_by('id').all()
return comments
def _update_comment(self, new_field, comment_pk, user):
comment = DbComment.query.filter_by(dbnode=self._dbnode,
id=comment_pk,
user=user).first()
if not isinstance(new_field, basestring):
raise ValueError("Non string comments are not accepted")
if not comment:
raise NotExistent("Found no comment for user {} and id {}".format(
user, comment_pk))
comment.content = new_field
comment.save()
def _remove_comment(self, comment_pk, user):
comment = DbComment.query.filter_by(dbnode=self._dbnode,
id=comment_pk).first()
if comment:
comment.delete()
def _increment_version_number_db(self):
self._dbnode.nodeversion = DbNode.nodeversion + 1
self._dbnode.save()
def copy(self):
newobject = self.__class__()
newobject.dbnode.type = self.dbnode.type # Inherit type
newobject.dbnode.label = self.dbnode.label # Inherit label
# TODO: add to the description the fact that this was a copy?
newobject.dbnode.description = self.dbnode.description # Inherit description
newobject.dbnode.dbcomputer = self.dbnode.dbcomputer # Inherit computer
for k, v in self.iterattrs():
newobject._set_attr(k, v)
for path in self.get_folder_list():
newobject.add_path(self.get_abs_path(path), path)
return newobject
@property
def pk(self):
return self.dbnode.id
@property
def id(self):
return self.dbnode.id
@property
def dbnode(self):
return self._dbnode
def store_all(self, with_transaction=True):
"""
Store the node, together with all input links, if cached, and also the
linked nodes, if they were not stored yet.
:parameter with_transaction: if False, no transaction is used. This
is meant to be used ONLY if the outer calling function has already
a transaction open!
"""
if not self._to_be_stored:
raise ModificationNotAllowed(
"Node with pk= {} was already stored".format(self.id))
# For each parent, check that all its inputs are stored
for link in self._inputlinks_cache:
try:
parent_node = self._inputlinks_cache[link][0]
parent_node._check_are_parents_stored()
except ModificationNotAllowed:
raise ModificationNotAllowed(
"Parent node (UUID={}) has "
"unstored parents, cannot proceed (only direct parents "
"can be unstored and will be stored by store_all, not "
"grandparents or other ancestors".format(parent_node.uuid))
self._store_input_nodes()
self.store(with_transaction=False)
self._store_cached_input_links(with_transaction=False)
from aiida.backends.sqlalchemy import session
if with_transaction:
try:
session.commit()
except SQLAlchemyError as e:
session.rollback()
return self
def _store_input_nodes(self):
"""
Find all input nodes, and store them, checking that they do not
have unstored inputs in turn.
:note: this function stores all nodes without transactions; always
call it from within a transaction!
"""
if not self._to_be_stored:
raise ModificationNotAllowed(
"_store_input_nodes can be called only if the node is "
"unstored (node {} is stored, instead)".format(self.id))
for link in self._inputlinks_cache:
parent = self._inputlinks_cache[link][0]
if not parent.is_stored:
parent.store(with_transaction=False)
def _check_are_parents_stored(self):
"""
Check if all parents are already stored, otherwise raise.
:raise ModificationNotAllowed: if one of the input nodes in not already
stored.
"""
# Preliminary check to verify that inputs are stored already
for link in self._inputlinks_cache:
if not self._inputlinks_cache[link][0].is_stored:
raise ModificationNotAllowed(
"Cannot store the input link '{}' because the "
"source node is not stored. Either store it first, "
"or call _store_input_links with the store_parents "
"parameter set to True".format(link))
def _store_cached_input_links(self, with_transaction=True):
"""
Store all input links that are in the local cache, transferring them
to the DB.
:note: This can be called only if all parents are already stored.
:note: Links are stored only after the input nodes are stored. Moreover,
link storage is done in a transaction, and if one of the links
cannot be stored, an exception is raised and *all* links will remain
in the cache.
:note: This function can be called only after the node is stored.
After that, it can be called multiple times, and nothing will be
executed if no links are still in the cache.
:parameter with_transaction: if False, no transaction is used. This
is meant to be used ONLY if the outer calling function has already
a transaction open!
"""
if self._to_be_stored:
raise ModificationNotAllowed(
"Node with pk= {} is not stored yet".format(self.id))
# This raises if there is an unstored node.
self._check_are_parents_stored()
# I have to store only those links where the source is already
# stored
links_to_store = list(self._inputlinks_cache.keys())
for label in links_to_store:
src, link_type = self._inputlinks_cache[label]
self._add_dblink_from(src, label, link_type)
# If everything went smoothly, clear the entries from the cache.
# I do it here because I delete them all at once if no error
# occurred; otherwise, links will not be stored and I
# should not delete them from the cache (but then an exception
# would have been raised, and the following lines are not executed)
self._inputlinks_cache.clear()
from aiida.backends.sqlalchemy import session
if with_transaction:
try:
session.commit()
except SQLAlchemyError as e:
session.rollback()
def store(self, with_transaction=True):
"""
Store a new node in the DB, also saving its repository directory
and attributes.
After being called attributes cannot be
changed anymore! Instead, extras can be changed only AFTER calling
this store() function.
:note: After successful storage, those links that are in the cache, and
for which also the parent node is already stored, will be
automatically stored. The others will remain unstored.
:parameter with_transaction: if False, no transaction is used. This
is meant to be used ONLY if the outer calling function has already
a transaction open!
"""
# TODO: This needs to be generalized, allowing for flexible methods
# for storing data and its attributes.
if self._to_be_stored:
self._validate()
self._check_are_parents_stored()
# I save the corresponding django entry
# I set the folder
# NOTE: I first store the files, then only if this is successful,
# I store the DB entry. In this way,
# I assume that if a node exists in the DB, its folder is in place.
# On the other hand, periodically the user might need to run some
# bookkeeping utility to check for lone folders.
self._repository_folder.replace_with_folder(
self._get_temp_folder().abspath, move=True, overwrite=True)
try:
self._dbnode.save(commit=False)
# Save its attributes 'manually' without incrementing
# the version for each add.
self.dbnode.attributes = self._attrs_cache
flag_modified(self.dbnode, "attributes")
# This should not be used anymore: I delete it to
# possibly free memory
del self._attrs_cache
self._temp_folder = None
self._to_be_stored = False
# Here, I store those links that were in the cache and
# that are between stored nodes.
self._store_cached_input_links(with_transaction=False)
if with_transaction:
self.dbnode.session.commit()
# This is one of the few cases where it is ok to do a 'global'
# except, also because I am re-raising the exception
except:
# I put back the files in the sandbox folder since the
# transaction did not succeed
self._get_temp_folder().replace_with_folder(
self._repository_folder.abspath, move=True, overwrite=True)
raise
# Set up autogrouping used be verdi run
autogroup = aiida.orm.autogroup.current_autogroup
grouptype = aiida.orm.autogroup.VERDIAUTOGROUP_TYPE
if autogroup is not None:
if not isinstance(autogroup, aiida.orm.autogroup.Autogroup):
raise ValidationError(
"current_autogroup is not an AiiDA Autogroup")
if autogroup.is_to_be_grouped(self):
group_name = autogroup.get_group_name()
if group_name is not None:
g = Group.get_or_create(name=group_name,
type_string=grouptype)[0]
g.add_nodes(self)
return self
@property
def has_children(self):
#~ from aiida.backends.sqlalchemy.model.node import DbNode
return self.dbnode.outputs_q.all()
#~ return session.query(literal(True)).filter(
#~ self.dbnode.child_paths.exists()).scalar() or False
@property
def has_parents(self):
return session.query(literal(True)).filter(
self.dbnode.parent_paths.exists()).scalar() or False
@property
def uuid(self):
return unicode(self.dbnode.uuid)
