def fill_repo(self):
"""Utility function to create repository nodes"""
from aiida.orm import CalcJobNode, Data, Dict
extra_name = self.__class__.__name__ + '/test_with_subclasses'
resources = {'num_machines': 1, 'num_mpiprocs_per_machine': 1}
a1 = CalcJobNode(computer=self.computer)
a1.set_option('resources', resources)
a1.store()
# To query only these nodes later
a1.set_extra(extra_name, True)
a3 = Data().store()
a3.set_extra(extra_name, True)
a4 = Dict(dict={'a': 'b'}).store()
a4.set_extra(extra_name, True)
a5 = Data().store()
a5.set_extra(extra_name, True)
# I don't set the extras, just to be sure that the filtering works
# The filtering is needed because other tests will put stuff int he DB
a6 = CalcJobNode(computer=self.computer)
a6.set_option('resources', resources)
a6.store()
a7 = Data()
a7.store()
def test_ordering_limits_offsets_sqla(self):
"""Test ordering limits offsets of SQLA query results."""
# Creating 10 nodes with an attribute that can be ordered
for i in range(10):
node = Data()
node.set_attribute('foo', i)
node.store()
q_b = QueryBuilder().append(Node, project='attributes.foo').order_by(
{Node: {
'attributes.foo': {
'cast': 'i'
}
}})
res = next(zip(*q_b.all()))
self.assertEqual(res, tuple(range(10)))
# Now applying an offset:
q_b.offset(5)
res = next(zip(*q_b.all()))
self.assertEqual(res, tuple(range(5, 10)))
# Now also applying a limit:
q_b.limit(3)
res = next(zip(*q_b.all()))
self.assertEqual(res, tuple(range(5, 8)))
def test_ordering_limits_offsets_of_results_for_SQLA(self):
from aiida.orm import Node, Data
from aiida.orm.querybuilder import QueryBuilder
# Creating 10 nodes with an attribute that can be ordered
for i in range(10):
n = Data()
n.set_attribute('foo', i)
n.store()
qb = QueryBuilder().append(
Node, project='attributes.foo'
).order_by(
{Node: {'attributes.foo': {'cast': 'i'}}}
)
res = next(zip(*qb.all()))
self.assertEqual(res, tuple(range(10)))
# Now applying an offset:
qb.offset(5)
res = next(zip(*qb.all()))
self.assertEqual(res, tuple(range(5, 10)))
# Now also applying a limit:
qb.limit(3)
res = next(zip(*qb.all()))
self.assertEqual(res, tuple(range(5, 8)))
def test_node_outdegree_unique_triple(self):
"""Test that the validation of links with outdegree `unique_triple` works correctly
The example here is a `CalculationNode` that has two outgoing CREATE links with the same label, but to different
target nodes. This is legal and should pass validation.
"""
creator = CalculationNode().store()
data_one = Data()
data_two = Data()
# Verify that adding two create links with the same link label but to different target is allowed from the
# perspective of the source node (the CalculationNode in this case)
data_one.add_incoming(creator,
link_type=LinkType.CREATE,
link_label='create')
data_two.add_incoming(creator,
link_type=LinkType.CREATE,
link_label='create')
data_one.store()
data_two.store()
uuids_outgoing = set(node.uuid
for node in creator.get_outgoing().all_nodes())
uuids_expected = set([data_one.uuid, data_two.uuid])
self.assertEqual(uuids_outgoing, uuids_expected)
def get_data_node(store=True):
"""A function to create a simple data node."""
data = Data()
data.set_attribute_many({str(i): i for i in range(10)})
if store:
data.store()
return (), {'node': data}
def get_data_node_and_object(store=True):
"""A function to create a simple data node, with an object."""
data = Data()
data.set_attribute_many({str(i): i for i in range(10)})
data.put_object_from_filelike(StringIO('a' * 10000), 'key')
if store:
data.store()
return (), {'node': data}
def test_delete_collection_outgoing_link(self):
"""Test deletion through objects collection raises when there are outgoing links."""
calculation = CalculationNode().store()
data = Data()
data.add_incoming(calculation, LinkType.CREATE, 'output')
data.store()
with pytest.raises(exceptions.InvalidOperation):
Node.objects.delete(calculation.pk)
def test_load_nodes(self):
"""
Test for load_node() function.
"""
from aiida.orm import load_node
from aiida.backends.sqlalchemy import get_scoped_session
a = Data()
a.store()
self.assertEquals(a.pk, load_node(identifier=a.pk).pk)
self.assertEquals(a.pk, load_node(identifier=a.uuid).pk)
self.assertEquals(a.pk, load_node(pk=a.pk).pk)
self.assertEquals(a.pk, load_node(uuid=a.uuid).pk)
session = get_scoped_session()
try:
session.begin_nested()
with self.assertRaises(ValueError):
load_node(identifier=a.pk, pk=a.pk)
finally:
session.rollback()
try:
session.begin_nested()
with self.assertRaises(ValueError):
load_node(pk=a.pk, uuid=a.uuid)
finally:
session.rollback()
try:
session.begin_nested()
with self.assertRaises(TypeError):
load_node(pk=a.uuid)
finally:
session.rollback()
try:
session.begin_nested()
with self.assertRaises(TypeError):
load_node(uuid=a.pk)
finally:
session.rollback()
try:
session.begin_nested()
with self.assertRaises(ValueError):
load_node()
finally:
session.rollback()
def test_erase_stored_raise(self):
"""
Test that trying to erase the repository content of a stored
Data node without the force flag raises.
"""
node = Data()
node.put_object_from_tree(self.tempdir, '')
node.store()
self.assertEqual(sorted(node.list_object_names()), ['c.txt', 'subdir'])
self.assertEqual(sorted(node.list_object_names('subdir')),
['a.txt', 'b.txt', 'nested'])
self.assertRaises(ModificationNotAllowed, node._repository.erase) # pylint: disable=protected-access
def test_detect_invalid_nodes_unknown_node_type(self):
"""Test `verdi database integrity detect-invalid-nodes` when node type is invalid."""
result = self.cli_runner.invoke(cmd_database.detect_invalid_nodes, [])
self.assertEqual(result.exit_code, 0)
self.assertClickResultNoException(result)
# Create a node with invalid type: since there are a lot of validation rules that prevent us from creating an
# invalid node type normally, we have to do it manually on the database model instance before storing
node = Data()
node.backend_entity.dbmodel.node_type = '__main__.SubClass.'
node.store()
result = self.cli_runner.invoke(cmd_database.detect_invalid_nodes, [])
self.assertNotEqual(result.exit_code, 0)
self.assertIsNotNone(result.exception)
def test_erase_stored_force(self):
"""
Test that _repository.erase removes the content of an stored
Data node when passing force=True.
"""
node = Data()
node.put_object_from_tree(self.tempdir, '')
node.store()
self.assertEqual(sorted(node.list_object_names()), ['c.txt', 'subdir'])
self.assertEqual(sorted(node.list_object_names('subdir')),
['a.txt', 'b.txt', 'nested'])
node._repository.erase(force=True) # pylint: disable=protected-access
self.assertEqual(node.list_object_names(), [])
def test_store_from_cache():
"""Regression test for storing a Node with (nested) repository content with caching."""
data = Data()
with tempfile.TemporaryDirectory() as tmpdir:
dir_path = os.path.join(tmpdir, 'directory')
os.makedirs(dir_path)
with open(os.path.join(dir_path, 'file'), 'w') as file:
file.write('content')
data.put_object_from_tree(tmpdir)
data.store()
clone = data.clone()
clone._store_from_cache(data, with_transaction=True) # pylint: disable=protected-access
assert clone.is_stored
assert clone.get_cache_source() == data.uuid
assert data.get_hash() == clone.get_hash()
def test_subclassing(self):
from aiida.orm.data.structure import StructureData
from aiida.orm.data.parameter import ParameterData
from aiida.orm import Node, Data
from aiida.orm.querybuilder import QueryBuilder
s = StructureData()
s._set_attr('cat', 'miau')
s.store()
d = Data()
d._set_attr('cat', 'miau')
d.store()
p = ParameterData(dict=dict(cat='miau'))
p.store()
n = Node()
n._set_attr('cat', 'miau')
n.store()
# Now when asking for a node with attr.cat==miau, I want 4 esults:
qb = QueryBuilder().append(Node, filters={'attributes.cat': 'miau'})
self.assertEqual(qb.count(), 4)
qb = QueryBuilder().append(Data, filters={'attributes.cat': 'miau'})
self.assertEqual(qb.count(), 3)
# If I'm asking for the specific lowest subclass, I want one result
for cls in (StructureData, ParameterData):
qb = QueryBuilder().append(cls, filters={'attributes.cat': 'miau'})
self.assertEqual(qb.count(), 1)
# Now I am not allow the subclassing, which should give 1 result for each
for cls in (StructureData, ParameterData, Node, Data):
qb = QueryBuilder().append(cls,
filters={'attributes.cat': 'miau'},
subclassing=False)
self.assertEqual(qb.count(), 1)
def test_simple_query_django_1(self):
"""
Testing a simple query
"""
from aiida.orm.querybuilder import QueryBuilder
from aiida.orm.calculation.job import JobCalculation
from aiida.orm import Node, Data, Calculation
from datetime import datetime
from aiida.common.links import LinkType
n1 = Data()
n1.label = 'node1'
n1._set_attr('foo', ['hello', 'goodbye'])
n1.store()
n2 = Calculation()
n2.label = 'node2'
n2._set_attr('foo', 1)
n2.store()
n3 = Data()
n3.label = 'node3'
n3._set_attr('foo', 1.0000) # Stored as fval
n3.store()
n4 = Calculation()
n4.label = 'node4'
n4._set_attr('foo', 'bar')
n4.store()
n5 = Data()
n5.label = 'node5'
n5._set_attr('foo', None)
n5.store()
n2.add_link_from(n1, link_type=LinkType.INPUT)
n3.add_link_from(n2, link_type=LinkType.CREATE)
n4.add_link_from(n3, link_type=LinkType.INPUT)
n5.add_link_from(n4, link_type=LinkType.CREATE)
qb1 = QueryBuilder()
qb1.append(Node, filters={'attributes.foo': 1.000})
self.assertEqual(len(qb1.all()), 2)
qb2 = QueryBuilder()
qb2.append(Data)
self.assertEqual(qb2.count(), 3)
qb2 = QueryBuilder()
qb2.append(type='data.Data.')
self.assertEqual(qb2.count(), 3)
qb3 = QueryBuilder()
qb3.append(Node, project='label', tag='node1')
qb3.append(Node, project='label', tag='node2')
self.assertEqual(qb3.count(), 4)
qb4 = QueryBuilder()
qb4.append(Calculation, tag='node1')
qb4.append(Data, tag='node2')
self.assertEqual(qb4.count(), 2)
qb5 = QueryBuilder()
qb5.append(Data, tag='node1')
qb5.append(Calculation, tag='node2')
self.assertEqual(qb5.count(), 2)
qb6 = QueryBuilder()
qb6.append(Data, tag='node1')
qb6.append(Data, tag='node2')
self.assertEqual(qb6.count(), 0)
class TestNodeAttributesExtras(AiidaTestCase):
"""Test for node attributes and extras."""
def setUp(self):
super(TestNodeAttributesExtras, self).setUp()
self.node = Data()
def test_attributes(self):
"""Test the `Node.attributes` property."""
original_attribute = {'nested': {'a': 1}}
self.node.set_attribute('key', original_attribute)
node_attributes = self.node.attributes
self.assertEqual(node_attributes['key'], original_attribute)
node_attributes['key']['nested']['a'] = 2
self.assertEqual(original_attribute['nested']['a'], 2)
# Now store the node and verify that `attributes` then returns a deep copy
self.node.store()
node_attributes = self.node.attributes
# We change the returned node attributes but the original attribute should remain unchanged
node_attributes['key']['nested']['a'] = 3
self.assertEqual(original_attribute['nested']['a'], 2)
def test_get_attribute(self):
"""Test the `Node.get_attribute` method."""
original_attribute = {'nested': {'a': 1}}
self.node.set_attribute('key', original_attribute)
node_attribute = self.node.get_attribute('key')
self.assertEqual(node_attribute, original_attribute)
node_attribute['nested']['a'] = 2
self.assertEqual(original_attribute['nested']['a'], 2)
default = 'default'
self.assertEqual(self.node.get_attribute('not_existing', default=default), default)
with self.assertRaises(AttributeError):
self.node.get_attribute('not_existing')
# Now store the node and verify that `get_attribute` then returns a deep copy
self.node.store()
node_attribute = self.node.get_attribute('key')
# We change the returned node attributes but the original attribute should remain unchanged
node_attribute['nested']['a'] = 3
self.assertEqual(original_attribute['nested']['a'], 2)
default = 'default'
self.assertEqual(self.node.get_attribute('not_existing', default=default), default)
with self.assertRaises(AttributeError):
self.node.get_attribute('not_existing')
def test_get_attribute_many(self):
"""Test the `Node.get_attribute_many` method."""
original_attribute = {'nested': {'a': 1}}
self.node.set_attribute('key', original_attribute)
node_attribute = self.node.get_attribute_many(['key'])[0]
self.assertEqual(node_attribute, original_attribute)
node_attribute['nested']['a'] = 2
self.assertEqual(original_attribute['nested']['a'], 2)
# Now store the node and verify that `get_attribute` then returns a deep copy
self.node.store()
node_attribute = self.node.get_attribute_many(['key'])[0]
# We change the returned node attributes but the original attribute should remain unchanged
node_attribute['nested']['a'] = 3
self.assertEqual(original_attribute['nested']['a'], 2)
def test_set_attribute(self):
"""Test the `Node.set_attribute` method."""
with self.assertRaises(exceptions.ValidationError):
self.node.set_attribute('illegal.key', 'value')
self.node.set_attribute('valid_key', 'value')
self.node.store()
with self.assertRaises(exceptions.ModificationNotAllowed):
self.node.set_attribute('valid_key', 'value')
def test_set_attribute_many(self):
"""Test the `Node.set_attribute` method."""
with self.assertRaises(exceptions.ValidationError):
self.node.set_attribute_many({'illegal.key': 'value', 'valid_key': 'value'})
self.node.set_attribute_many({'valid_key': 'value'})
self.node.store()
with self.assertRaises(exceptions.ModificationNotAllowed):
self.node.set_attribute_many({'valid_key': 'value'})
def test_reset_attribute(self):
"""Test the `Node.reset_attribute` method."""
attributes_before = {'attribute_one': 'value', 'attribute_two': 'value'}
attributes_after = {'attribute_three': 'value', 'attribute_four': 'value'}
attributes_illegal = {'attribute.illegal': 'value', 'attribute_four': 'value'}
self.node.set_attribute_many(attributes_before)
self.assertEqual(self.node.attributes, attributes_before)
self.node.reset_attributes(attributes_after)
self.assertEqual(self.node.attributes, attributes_after)
with self.assertRaises(exceptions.ValidationError):
self.node.reset_attributes(attributes_illegal)
self.node.store()
with self.assertRaises(exceptions.ModificationNotAllowed):
self.node.reset_attributes(attributes_after)
def test_delete_attribute(self):
"""Test the `Node.delete_attribute` method."""
self.node.set_attribute('valid_key', 'value')
self.assertEqual(self.node.get_attribute('valid_key'), 'value')
self.node.delete_attribute('valid_key')
with self.assertRaises(AttributeError):
self.node.delete_attribute('valid_key')
# Repeat with stored node
self.node.set_attribute('valid_key', 'value')
self.node.store()
with self.assertRaises(exceptions.ModificationNotAllowed):
self.node.delete_attribute('valid_key')
def test_delete_attribute_many(self):
"""Test the `Node.delete_attribute_many` method."""
def test_clear_attributes(self):
"""Test the `Node.clear_attributes` method."""
attributes = {'attribute_one': 'value', 'attribute_two': 'value'}
self.node.set_attribute_many(attributes)
self.assertEqual(self.node.attributes, attributes)
self.node.clear_attributes()
self.assertEqual(self.node.attributes, {})
# Repeat for stored node
self.node.store()
with self.assertRaises(exceptions.ModificationNotAllowed):
self.node.clear_attributes()
def test_attributes_items(self):
"""Test the `Node.attributes_items` generator."""
attributes = {'attribute_one': 'value', 'attribute_two': 'value'}
self.node.set_attribute_many(attributes)
self.assertEqual(dict(self.node.attributes_items()), attributes)
def test_attributes_keys(self):
"""Test the `Node.attributes_keys` generator."""
attributes = {'attribute_one': 'value', 'attribute_two': 'value'}
self.node.set_attribute_many(attributes)
self.assertEqual(set(self.node.attributes_keys()), set(attributes))
def test_extras(self):
"""Test the `Node.extras` property."""
original_extra = {'nested': {'a': 1}}
self.node.set_extra('key', original_extra)
node_extras = self.node.extras
self.assertEqual(node_extras['key'], original_extra)
node_extras['key']['nested']['a'] = 2
self.assertEqual(original_extra['nested']['a'], 2)
# Now store the node and verify that `extras` then returns a deep copy
self.node.store()
node_extras = self.node.extras
# We change the returned node extras but the original extra should remain unchanged
node_extras['key']['nested']['a'] = 3
self.assertEqual(original_extra['nested']['a'], 2)
def test_get_extra(self):
"""Test the `Node.get_extra` method."""
original_extra = {'nested': {'a': 1}}
self.node.set_extra('key', original_extra)
node_extra = self.node.get_extra('key')
self.assertEqual(node_extra, original_extra)
node_extra['nested']['a'] = 2
self.assertEqual(original_extra['nested']['a'], 2)
default = 'default'
self.assertEqual(self.node.get_extra('not_existing', default=default), default)
with self.assertRaises(AttributeError):
self.node.get_extra('not_existing')
# Now store the node and verify that `get_extra` then returns a deep copy
self.node.store()
node_extra = self.node.get_extra('key')
# We change the returned node extras but the original extra should remain unchanged
node_extra['nested']['a'] = 3
self.assertEqual(original_extra['nested']['a'], 2)
default = 'default'
self.assertEqual(self.node.get_extra('not_existing', default=default), default)
with self.assertRaises(AttributeError):
self.node.get_extra('not_existing')
def test_get_extra_many(self):
"""Test the `Node.get_extra_many` method."""
original_extra = {'nested': {'a': 1}}
self.node.set_extra('key', original_extra)
node_extra = self.node.get_extra_many(['key'])[0]
self.assertEqual(node_extra, original_extra)
node_extra['nested']['a'] = 2
self.assertEqual(original_extra['nested']['a'], 2)
# Now store the node and verify that `get_extra` then returns a deep copy
self.node.store()
node_extra = self.node.get_extra_many(['key'])[0]
# We change the returned node extras but the original extra should remain unchanged
node_extra['nested']['a'] = 3
self.assertEqual(original_extra['nested']['a'], 2)
def test_set_extra(self):
"""Test the `Node.set_extra` method."""
with self.assertRaises(exceptions.ValidationError):
self.node.set_extra('illegal.key', 'value')
self.node.set_extra('valid_key', 'value')
self.node.store()
self.node.set_extra('valid_key', 'changed')
self.assertEqual(load_node(self.node.pk).get_extra('valid_key'), 'changed')
def test_set_extra_many(self):
"""Test the `Node.set_extra` method."""
with self.assertRaises(exceptions.ValidationError):
self.node.set_extra_many({'illegal.key': 'value', 'valid_key': 'value'})
self.node.set_extra_many({'valid_key': 'value'})
self.node.store()
self.node.set_extra_many({'valid_key': 'changed'})
self.assertEqual(load_node(self.node.pk).get_extra('valid_key'), 'changed')
def test_reset_extra(self):
"""Test the `Node.reset_extra` method."""
extras_before = {'extra_one': 'value', 'extra_two': 'value'}
extras_after = {'extra_three': 'value', 'extra_four': 'value'}
extras_illegal = {'extra.illegal': 'value', 'extra_four': 'value'}
self.node.set_extra_many(extras_before)
self.assertEqual(self.node.extras, extras_before)
self.node.reset_extras(extras_after)
self.assertEqual(self.node.extras, extras_after)
with self.assertRaises(exceptions.ValidationError):
self.node.reset_extras(extras_illegal)
self.node.store()
self.node.reset_extras(extras_after)
self.assertEqual(load_node(self.node.pk).extras, extras_after)
def test_delete_extra(self):
"""Test the `Node.delete_extra` method."""
self.node.set_extra('valid_key', 'value')
self.assertEqual(self.node.get_extra('valid_key'), 'value')
self.node.delete_extra('valid_key')
with self.assertRaises(AttributeError):
self.node.delete_extra('valid_key')
# Repeat with stored node
self.node.set_extra('valid_key', 'value')
self.node.store()
self.node.delete_extra('valid_key')
with self.assertRaises(AttributeError):
load_node(self.node.pk).get_extra('valid_key')
def test_delete_extra_many(self):
"""Test the `Node.delete_extra_many` method."""
def test_clear_extras(self):
"""Test the `Node.clear_extras` method."""
extras = {'extra_one': 'value', 'extra_two': 'value'}
self.node.set_extra_many(extras)
self.assertEqual(self.node.extras, extras)
self.node.clear_extras()
self.assertEqual(self.node.extras, {})
# Repeat for stored node
self.node.store()
self.node.clear_extras()
self.assertEqual(load_node(self.node.pk).extras, {})
def test_extras_items(self):
"""Test the `Node.extras_items` generator."""
extras = {'extra_one': 'value', 'extra_two': 'value'}
self.node.set_extra_many(extras)
self.assertEqual(dict(self.node.extras_items()), extras)
def test_extras_keys(self):
"""Test the `Node.extras_keys` generator."""
extras = {'extra_one': 'value', 'extra_two': 'value'}
self.node.set_extra_many(extras)
self.assertEqual(set(self.node.extras_keys()), set(extras))
