def test_get_node_info_multiple_call_links(self):
"""Test the `get_node_info` utility.
Regression test for #2868:
Verify that all `CALL` links are included in the formatted string even if link labels are identical.
"""
from aiida.cmdline.utils.common import get_node_info
workflow = orm.WorkflowNode().store()
node_one = orm.CalculationNode()
node_two = orm.CalculationNode()
node_one.add_incoming(workflow,
link_type=LinkType.CALL_CALC,
link_label='CALL_IDENTICAL')
node_two.add_incoming(workflow,
link_type=LinkType.CALL_CALC,
link_label='CALL_IDENTICAL')
node_one.store()
node_two.store()
node_info = get_node_info(workflow)
self.assertTrue('CALL_IDENTICAL' in node_info)
self.assertTrue(str(node_one.pk) in node_info)
self.assertTrue(str(node_two.pk) in node_info)
def test_delete(self):
"""Test `verdi group delete` command."""
orm.Group(label='group_test_delete_01').store()
orm.Group(label='group_test_delete_02').store()
result = self.cli_runner.invoke(cmd_group.group_delete,
['--force', 'group_test_delete_01'])
self.assertClickResultNoException(result)
# Verify that removed group is not present in list
result = self.cli_runner.invoke(cmd_group.group_list)
self.assertClickResultNoException(result)
self.assertNotIn('group_test_delete_01', result.output)
node_01 = orm.CalculationNode().store()
node_02 = orm.CalculationNode().store()
# Add some nodes and then use `verdi group delete` to delete a group that contains nodes
group = orm.load_group(label='group_test_delete_02')
group.add_nodes([node_01, node_02])
self.assertEqual(group.count(), 2)
result = self.cli_runner.invoke(cmd_group.group_delete,
['--force', 'group_test_delete_02'])
self.assertClickResultNoException(result)
with self.assertRaises(exceptions.NotExistent):
orm.load_group(label='group_test_delete_02')
def test_delete(self):
"""Test `verdi group delete` command."""
orm.Group(label='group_test_delete_01').store()
orm.Group(label='group_test_delete_02').store()
result = self.cli_runner.invoke(cmd_group.group_delete,
['--force', 'group_test_delete_01'])
self.assertClickResultNoException(result)
# Verify that removed group is not present in list
result = self.cli_runner.invoke(cmd_group.group_list)
self.assertClickResultNoException(result)
self.assertNotIn('group_test_delete_01', result.output)
node_01 = orm.CalculationNode().store()
node_02 = orm.CalculationNode().store()
# Add some nodes and then use `verdi group delete --clear` to delete a node even when it contains nodes
group = orm.load_group(label='group_test_delete_02')
group.add_nodes([node_01, node_02])
self.assertEqual(group.count(), 2)
# Calling delete on a group without the `--clear` option should raise
result = self.cli_runner.invoke(cmd_group.group_delete,
['--force', 'group_test_delete_02'])
self.assertIsNotNone(result.exception, result.output)
# With `--clear` option should delete group and nodes
result = self.cli_runner.invoke(
cmd_group.group_delete,
['--force', '--clear', 'group_test_delete_02'])
self.assertClickResultNoException(result)
with self.assertRaises(exceptions.NotExistent):
group = orm.load_group(label='group_test_delete_02')
def test_delete(self):
"""Test `verdi group delete` command."""
orm.Group(label='group_test_delete_01').store()
orm.Group(label='group_test_delete_02').store()
orm.Group(label='group_test_delete_03').store()
# dry run
result = self.cli_runner.invoke(cmd_group.group_delete,
['--dry-run', 'group_test_delete_01'])
self.assertClickResultNoException(result)
orm.load_group(label='group_test_delete_01')
result = self.cli_runner.invoke(cmd_group.group_delete,
['--force', 'group_test_delete_01'])
self.assertClickResultNoException(result)
# Verify that removed group is not present in list
result = self.cli_runner.invoke(cmd_group.group_list)
self.assertClickResultNoException(result)
self.assertNotIn('group_test_delete_01', result.output)
node_01 = orm.CalculationNode().store()
node_02 = orm.CalculationNode().store()
node_pks = {node_01.pk, node_02.pk}
# Add some nodes and then use `verdi group delete` to delete a group that contains nodes
group = orm.load_group(label='group_test_delete_02')
group.add_nodes([node_01, node_02])
self.assertEqual(group.count(), 2)
result = self.cli_runner.invoke(cmd_group.group_delete,
['--force', 'group_test_delete_02'])
with self.assertRaises(exceptions.NotExistent):
orm.load_group(label='group_test_delete_02')
# check nodes still exist
for pk in node_pks:
orm.load_node(pk)
# delete the group and the nodes it contains
group = orm.load_group(label='group_test_delete_03')
group.add_nodes([node_01, node_02])
result = self.cli_runner.invoke(
cmd_group.group_delete,
['--force', '--delete-nodes', 'group_test_delete_03'])
self.assertClickResultNoException(result)
# check group and nodes no longer exist
with self.assertRaises(exceptions.NotExistent):
orm.load_group(label='group_test_delete_03')
for pk in node_pks:
with self.assertRaises(exceptions.NotExistent):
orm.load_node(pk)
def _create_branchy_graph():
"""
Creates a basic branchy graph which has two concatenated calculations:
* calc_1 takes data_0 as an input and returns data_1 and data_o.
* calc_2 takes data_1 and data_i as inputs and returns data_2.
This graph looks like this::
+--------+ +--------+
| data_o | | data_i |
+--------+ +--------+
^ |
| v
+--------+ +--------+ +--------+ +--------+ +--------+
| data_0 | --> | calc_1 | --> | data_1 | --> | calc_2 | --> | data_2 |
+--------+ +--------+ +--------+ +--------+ +--------+
"""
data_0 = orm.Data().store()
calc_1 = orm.CalculationNode()
calc_1.add_incoming(data_0, link_type=LinkType.INPUT_CALC, link_label='inpcalc_data_0')
calc_1.store()
data_1 = orm.Data()
data_o = orm.Data()
data_1.add_incoming(calc_1, link_type=LinkType.CREATE, link_label='create_data_1')
data_o.add_incoming(calc_1, link_type=LinkType.CREATE, link_label='create_data_o')
data_1.store()
data_o.store()
data_i = orm.Data().store()
calc_2 = orm.CalculationNode()
calc_2.add_incoming(data_1, link_type=LinkType.INPUT_CALC, link_label='inpcalc_data_1')
calc_2.add_incoming(data_i, link_type=LinkType.INPUT_CALC, link_label='inpcalc_data_i')
calc_2.store()
data_2 = orm.Data()
data_2.add_incoming(calc_2, link_type=LinkType.CREATE, link_label='create_data_2')
data_2.store()
output_dict = {
'data_i': data_i,
'data_0': data_0,
'data_1': data_1,
'data_2': data_2,
'data_o': data_o,
'calc_1': calc_1,
'calc_2': calc_2,
}
return output_dict
def test_exclude_logs_flag(self, temp_dir):
"""Test that the `include_logs` argument for `export` works."""
log_msg = 'Testing logging of critical failure'
# Create node
calc = orm.CalculationNode()
calc.store()
calc.seal()
# Create log message
calc.logger.critical(log_msg)
# Save uuids prior to export
calc_uuid = calc.uuid
# Export, excluding logs
export_file = os.path.join(temp_dir, 'export.tar.gz')
export([calc], outfile=export_file, silent=True, include_logs=False)
# Clean database and reimport exported data
self.reset_database()
import_data(export_file, silent=True)
# Finding all the log messages
import_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid']).all()
import_logs = orm.QueryBuilder().append(orm.Log,
project=['uuid']).all()
# There should be exactly: 1 orm.CalculationNode, 0 Logs
self.assertEqual(len(import_calcs), 1)
self.assertEqual(len(import_logs), 0)
# Check it's the correct node
self.assertEqual(str(import_calcs[0][0]), calc_uuid)
def test_input_and_create_links(self, temp_dir):
"""
Simple test that will verify that INPUT and CREATE links are properly exported and
correctly recreated upon import.
"""
node_work = orm.CalculationNode()
node_input = orm.Int(1).store()
node_output = orm.Int(2).store()
node_work.add_incoming(node_input, LinkType.INPUT_CALC, 'input')
node_work.store()
node_output.add_incoming(node_work, LinkType.CREATE, 'output')
node_work.seal()
export_links = get_all_node_links()
export_file = os.path.join(temp_dir, 'export.aiida')
export([node_output], filename=export_file)
self.clean_db()
import_data(export_file)
import_links = get_all_node_links()
export_set = [tuple(_) for _ in export_links]
import_set = [tuple(_) for _ in import_links]
self.assertSetEqual(set(export_set), set(import_set))
def test_link_manager_with_nested_namespaces(clear_database_before_test):
"""Test the ``LinkManager`` works with nested namespaces."""
inp1 = orm.Data()
inp1.store()
calc = orm.CalculationNode()
calc.add_incoming(inp1, link_type=LinkType.INPUT_CALC, link_label='nested__sub__namespace')
calc.store()
# Attach outputs
out1 = orm.Data()
out1.add_incoming(calc, link_type=LinkType.CREATE, link_label='nested__sub__namespace')
out1.store()
# Check that the recommended way of dereferencing works
assert calc.inputs.nested.sub.namespace.uuid == inp1.uuid
assert calc.outputs.nested.sub.namespace.uuid == out1.uuid
# Leafs will return an ``AttributeDict`` instance
assert isinstance(calc.outputs.nested.sub, AttributeDict)
# Check the legacy way still works
with pytest.warns(Warning):
assert calc.inputs.nested__sub__namespace.uuid == inp1.uuid
assert calc.outputs.nested__sub__namespace.uuid == out1.uuid
# Must raise a AttributeError, otherwise tab competion will not work
with pytest.raises(AttributeError):
_ = calc.outputs.nested.not_existent
# Must raise a KeyError
with pytest.raises(KeyError):
_ = calc.outputs.nested['not_existent']
def test_missing_node_repo_folder_export(self, temp_dir):
"""
Make sure `~aiida.tools.importexport.common.exceptions.ArchiveExportError` is raised during export when missing
Node repository folder.
Create and store a new Node and manually remove its repository folder.
Attempt to export it and make sure `~aiida.tools.importexport.common.exceptions.ArchiveExportError` is raised,
due to the missing folder.
"""
node = orm.CalculationNode().store()
node.seal()
node_uuid = node.uuid
node_repo = RepositoryFolder(section=Repository._section_name, uuid=node_uuid) # pylint: disable=protected-access
self.assertTrue(
node_repo.exists(), msg='Newly created and stored Node should have had an existing repository folder'
)
# Removing the Node's local repository folder
shutil.rmtree(node_repo.abspath, ignore_errors=True)
self.assertFalse(
node_repo.exists(), msg='Newly created and stored Node should have had its repository folder removed'
)
# Try to export, check it raises and check the raise message
filename = os.path.join(temp_dir, 'export.aiida')
with self.assertRaises(exceptions.ArchiveExportError) as exc:
export([node], filename=filename)
self.assertIn(f'Unable to find the repository folder for Node with UUID={node_uuid}', str(exc.exception))
self.assertFalse(os.path.exists(filename), msg='The archive file should not exist')
def test_critical_log_msg_and_metadata(self, temp_dir):
""" Testing logging of critical message """
message = 'Testing logging of critical failure'
calc = orm.CalculationNode()
# Firing a log for an unstored node should not end up in the database
calc.logger.critical(message)
# There should be no log messages for the unstored object
self.assertEqual(len(orm.Log.objects.all()), 0)
# After storing the node, logs above log level should be stored
calc.store()
calc.seal()
calc.logger.critical(message)
# Store Log metadata
log_metadata = orm.Log.objects.get(dbnode_id=calc.id).metadata
export_file = os.path.join(temp_dir, 'export.tar.gz')
export([calc], outfile=export_file, silent=True)
self.reset_database()
import_data(export_file, silent=True)
# Finding all the log messages
logs = orm.Log.objects.all()
self.assertEqual(len(logs), 1)
self.assertEqual(logs[0].message, message)
self.assertEqual(logs[0].metadata, log_metadata)
def test_missing_node_repo_folder_import(self, temp_dir):
"""
Make sure `~aiida.tools.importexport.common.exceptions.CorruptArchive` is raised during import when missing
Node repository folder.
Create and export a Node and manually remove its repository folder in the export file.
Attempt to import it and make sure `~aiida.tools.importexport.common.exceptions.CorruptArchive` is raised,
due to the missing folder.
"""
import tarfile
from aiida.common.folders import SandboxFolder
from aiida.tools.importexport.common.archive import extract_tar
from aiida.tools.importexport.common.config import NODES_EXPORT_SUBFOLDER
from aiida.tools.importexport.common.utils import export_shard_uuid
node = orm.CalculationNode().store()
node.seal()
node_uuid = node.uuid
node_repo = RepositoryFolder(section=Repository._section_name, uuid=node_uuid) # pylint: disable=protected-access
self.assertTrue(
node_repo.exists(), msg='Newly created and stored Node should have had an existing repository folder'
)
# Export and reset db
filename = os.path.join(temp_dir, 'export.aiida')
export([node], filename=filename, file_format='tar.gz', silent=True)
self.reset_database()
# Untar export file, remove repository folder, re-tar
node_shard_uuid = export_shard_uuid(node_uuid)
node_top_folder = node_shard_uuid.split('/')[0]
with SandboxFolder() as folder:
extract_tar(filename, folder, silent=True, nodes_export_subfolder=NODES_EXPORT_SUBFOLDER)
node_folder = folder.get_subfolder(os.path.join(NODES_EXPORT_SUBFOLDER, node_shard_uuid))
self.assertTrue(
node_folder.exists(), msg="The Node's repository folder should still exist in the export file"
)
# Removing the Node's repository folder from the export file
shutil.rmtree(
folder.get_subfolder(os.path.join(NODES_EXPORT_SUBFOLDER, node_top_folder)).abspath, ignore_errors=True
)
self.assertFalse(
node_folder.exists(),
msg="The Node's repository folder should now have been removed in the export file"
)
filename_corrupt = os.path.join(temp_dir, 'export_corrupt.aiida')
with tarfile.open(filename_corrupt, 'w:gz', format=tarfile.PAX_FORMAT, dereference=True) as tar:
tar.add(folder.abspath, arcname='')
# Try to import, check it raises and check the raise message
with self.assertRaises(exceptions.CorruptArchive) as exc:
import_data(filename_corrupt, silent=True)
self.assertIn(
'Unable to find the repository folder for Node with UUID={}'.format(node_uuid), str(exc.exception)
)
def test_dangling_link_to_existing_db_node(self, temp_dir):
"""A dangling link that references a Node that is not included in the archive should `not` be importable"""
struct = orm.StructureData()
struct.store()
struct_uuid = struct.uuid
calc = orm.CalculationNode()
calc.add_incoming(struct, LinkType.INPUT_CALC, 'input')
calc.store()
calc.seal()
calc_uuid = calc.uuid
filename = os.path.join(temp_dir, 'export.aiida')
export([struct], filename=filename, file_format='tar.gz')
unpack = SandboxFolder()
with tarfile.open(filename, 'r:gz', format=tarfile.PAX_FORMAT) as tar:
tar.extractall(unpack.abspath)
with open(unpack.get_abs_path('data.json'), 'r',
encoding='utf8') as fhandle:
data = json.load(fhandle)
data['links_uuid'].append({
'output': calc.uuid,
'input': struct.uuid,
'label': 'input',
'type': LinkType.INPUT_CALC.value
})
with open(unpack.get_abs_path('data.json'), 'wb') as fhandle:
json.dump(data, fhandle)
with tarfile.open(filename, 'w:gz', format=tarfile.PAX_FORMAT) as tar:
tar.add(unpack.abspath, arcname='')
# Make sure the CalculationNode is still in the database
builder = orm.QueryBuilder().append(orm.CalculationNode,
project='uuid')
self.assertEqual(
builder.count(),
1,
msg=
f'There should be a single CalculationNode, instead {builder.count()} has been found'
)
self.assertEqual(builder.all()[0][0], calc_uuid)
with self.assertRaises(DanglingLinkError):
import_data(filename)
# Using the flag `ignore_unknown_nodes` should import it without problems
import_data(filename, ignore_unknown_nodes=True)
builder = orm.QueryBuilder().append(orm.StructureData, project='uuid')
self.assertEqual(
builder.count(),
1,
msg=
f'There should be a single StructureData, instead {builder.count()} has been found'
)
self.assertEqual(builder.all()[0][0], struct_uuid)
def test_mtime_of_imported_comments(self, temp_dir):
"""
Test mtime does not change for imported comments
This is related to correct usage of `comment_mode` when importing.
"""
# Get user
user = orm.User.objects.get_default()
comment_content = 'You get what you give'
# Create node
calc = orm.CalculationNode().store()
calc.seal()
# Create comment
orm.Comment(calc, user, comment_content).store()
calc.store()
# Save UUIDs and mtime
calc_uuid = calc.uuid
builder = orm.QueryBuilder().append(orm.Comment, project=['uuid', 'mtime']).all()
comment_uuid = str(builder[0][0])
comment_mtime = builder[0][1]
builder = orm.QueryBuilder().append(orm.CalculationNode, project=['uuid', 'mtime']).all()
calc_uuid = str(builder[0][0])
calc_mtime = builder[0][1]
# Export, reset database and reimport
export_file = os.path.join(temp_dir, 'export.aiida')
export([calc], filename=export_file, silent=True)
self.reset_database()
import_data(export_file, silent=True)
# Retrieve node and comment
builder = orm.QueryBuilder().append(orm.CalculationNode, tag='calc', project=['uuid', 'mtime'])
builder.append(orm.Comment, with_node='calc', project=['uuid', 'mtime'])
import_entities = builder.all()[0]
self.assertEqual(len(import_entities), 4) # Check we have the correct amount of returned values
import_calc_uuid = str(import_entities[0])
import_calc_mtime = import_entities[1]
import_comment_uuid = str(import_entities[2])
import_comment_mtime = import_entities[3]
# Check we have the correct UUIDs
self.assertEqual(import_calc_uuid, calc_uuid)
self.assertEqual(import_comment_uuid, comment_uuid)
# Make sure the mtime is the same after import as it was before export
self.assertEqual(import_comment_mtime, comment_mtime)
self.assertEqual(import_calc_mtime, calc_mtime)
def test_add_remove_nodes(self):
"""Test `verdi group remove-nodes` command."""
node_01 = orm.CalculationNode().store()
node_02 = orm.CalculationNode().store()
node_03 = orm.CalculationNode().store()
result = self.cli_runner.invoke(
cmd_group.group_add_nodes,
['--force', '--group=dummygroup1', node_01.uuid])
self.assertClickResultNoException(result)
# Check if node is added in group using group show command
result = self.cli_runner.invoke(cmd_group.group_show, ['dummygroup1'])
self.assertClickResultNoException(result)
self.assertIn('CalculationNode', result.output)
self.assertIn(str(node_01.pk), result.output)
# Remove same node
result = self.cli_runner.invoke(
cmd_group.group_remove_nodes,
['--force', '--group=dummygroup1', node_01.uuid])
self.assertIsNone(result.exception, result.output)
# Check if node is added in group using group show command
result = self.cli_runner.invoke(cmd_group.group_show,
['-r', 'dummygroup1'])
self.assertClickResultNoException(result)
self.assertNotIn('CalculationNode', result.output)
self.assertNotIn(str(node_01.pk), result.output)
# Add all three nodes and then use `verdi group remove-nodes --clear` to remove them all
group = orm.load_group(label='dummygroup1')
group.add_nodes([node_01, node_02, node_03])
self.assertEqual(group.count(), 3)
result = self.cli_runner.invoke(
cmd_group.group_remove_nodes,
['--force', '--clear', '--group=dummygroup1'])
self.assertClickResultNoException(result)
self.assertEqual(group.count(), 0)
def test_copy_existing_group(self):
"""Test user is prompted to continue if destination group exists and is not empty"""
source_label = 'source_copy_existing_group'
dest_label = 'dest_copy_existing_group'
# Create source group with nodes
calc_s1 = orm.CalculationNode().store()
calc_s2 = orm.CalculationNode().store()
nodes_source_group = {str(node.uuid) for node in [calc_s1, calc_s2]}
source_group = orm.Group(label=source_label).store()
source_group.add_nodes([calc_s1, calc_s2])
# Copy using `verdi group copy` - making sure all is successful
options = [source_label, dest_label]
result = self.cli_runner.invoke(cmd_group.group_copy, options)
self.assertClickResultNoException(result)
self.assertIn(
'Success: Nodes copied from group<{}> to group<{}>'.format(
source_label, dest_label), result.output, result.exception)
# Check destination group exists with source group's nodes
dest_group = orm.load_group(label=dest_label)
self.assertEqual(dest_group.count(), 2)
nodes_dest_group = {str(node.uuid) for node in dest_group.nodes}
self.assertSetEqual(nodes_source_group, nodes_dest_group)
# Copy again, making sure an abort error is raised, since no user input can be made and default is abort
result = self.cli_runner.invoke(cmd_group.group_copy, options)
self.assertIsNotNone(result.exception, result.output)
self.assertIn(
'Warning: Destination group<{}> already exists and is not empty.'.
format(dest_label), result.output, result.exception)
# Check destination group is unchanged
dest_group = orm.load_group(label=dest_label)
self.assertEqual(dest_group.count(), 2)
nodes_dest_group = {str(node.uuid) for node in dest_group.nodes}
self.assertSetEqual(nodes_source_group, nodes_dest_group)
def test_calc_and_data_nodes_with_comments(self, temp_dir):
""" Test comments for CalculatioNode and Data node are correctly ex-/imported """
# Create user, nodes, and comments
user = orm.User.objects.get_default()
calc_node = orm.CalculationNode().store()
calc_node.seal()
data_node = orm.Data().store()
comment_one = orm.Comment(calc_node, user, self.comments[0]).store()
comment_two = orm.Comment(calc_node, user, self.comments[1]).store()
comment_three = orm.Comment(data_node, user, self.comments[2]).store()
comment_four = orm.Comment(data_node, user, self.comments[3]).store()
# Get values prior to export
calc_uuid = calc_node.uuid
data_uuid = data_node.uuid
calc_comments_uuid = [c.uuid for c in [comment_one, comment_two]]
data_comments_uuid = [c.uuid for c in [comment_three, comment_four]]
# Export nodes
export_file = os.path.join(temp_dir, 'export.tar.gz')
export([calc_node, data_node], outfile=export_file, silent=True)
# Clean database and reimport exported file
self.reset_database()
import_data(export_file, silent=True)
# Get nodes and comments
builder = orm.QueryBuilder()
builder.append(orm.Node, tag='node', project=['uuid'])
builder.append(orm.Comment, with_node='node', project=['uuid'])
nodes_and_comments = builder.all()
self.assertEqual(len(nodes_and_comments), len(self.comments))
for entry in nodes_and_comments:
self.assertEqual(len(entry), 2) # 1 Node + 1 Comment
import_node_uuid = str(entry[0])
import_comment_uuid = str(entry[1])
self.assertIn(import_node_uuid, [calc_uuid, data_uuid])
if import_node_uuid == calc_uuid:
# Calc node comments
self.assertIn(import_comment_uuid, calc_comments_uuid)
else:
# Data node comments
self.assertIn(import_comment_uuid, data_comments_uuid)
def test_get_node_summary(self):
"""Test the `get_node_summary` utility."""
from aiida.cmdline.utils.common import get_node_summary
computer_label = self.computer.name # pylint: disable=no-member
code = orm.Code(
input_plugin_name='arithmetic.add',
remote_computer_exec=[self.computer, '/remote/abs/path'],
)
code.store()
node = orm.CalculationNode()
node.computer = self.computer
node.add_incoming(code,
link_type=LinkType.INPUT_CALC,
link_label='code')
node.store()
summary = get_node_summary(node)
self.assertIn(node.uuid, summary)
self.assertIn(computer_label, summary)
def test_create_log_unserializable_metadata(self):
"""Test that unserializable data will be removed before reaching the database causing an error."""
import functools
def unbound_method(argument):
return argument
partial = functools.partial(unbound_method, 'argument')
node = orm.CalculationNode().store()
# An unbound method in the `args` of the metadata
node.logger.error('problem occurred in method %s', unbound_method)
# A partial in the `args` of the metadata
node.logger.error('problem occurred in partial %s', partial)
# An exception which will include an `exc_info` object
try:
raise ValueError
except ValueError:
node.logger.exception('caught an exception')
self.assertEqual(len(Log.objects.all()), 3)
def test_db_log_handler(self):
"""
Verify that the db log handler is attached correctly
by firing a log message through the regular logging module
attached to a calculation node
"""
from aiida.orm.logs import OrderSpecifier, ASCENDING
message = 'Testing logging of critical failure'
node = orm.CalculationNode()
# Firing a log for an unstored should not end up in the database
node.logger.critical(message)
logs = Log.objects.find()
self.assertEqual(len(logs), 0)
# After storing the node, logs above log level should be stored
node.store()
node.logger.critical(message)
logs = Log.objects.find()
self.assertEqual(len(logs), 1)
self.assertEqual(logs[0].message, message)
# Launching a second log message ensuring that both messages are correctly stored
message2 = message + ' - Second message'
node.logger.critical(message2)
order_by = [OrderSpecifier('time', ASCENDING)]
logs = Log.objects.find(order_by=order_by)
self.assertEqual(len(logs), 2)
self.assertEqual(logs[0].message, message)
self.assertEqual(logs[1].message, message2)
def test_reimport_of_comments_for_single_node(self, temp_dir):
"""
When a node with comments already exist in the DB, and more comments are
imported for the same node (same UUID), test that only new comment-entries
are added.
Part I:
Create CalculationNode and 1 Comment for it.
Export CalculationNode with its 1 Comment to export file #1 "EXISTING database".
Add 3 Comments to CalculationNode.
Export CalculationNode with its 4 Comments to export file #2 "FULL database".
Reset database.
Part II:
Reimport export file #1 "EXISTING database".
Add 3 Comments to CalculationNode.
Export CalculationNode with its 4 Comments to export file #3 "NEW database".
Reset database.
Part III:
Reimport export file #1 "EXISTING database" (1 CalculationNode, 1 Comment).
Import export file #2 "FULL database" (1 CalculationNode, 4 Comments).
Check the database EXACTLY contains 1 CalculationNode and 4 Comments,
with matching UUIDS all the way through all previous Parts.
Part IV:
Import export file #3 "NEW database" (1 CalculationNode, 4 Comments).
Check the database EXACTLY contains 1 CalculationNode and 7 Comments,
with matching UUIDS all the way through all previous Parts.
NB! There should now be 7 Comments in the database. 6 of which are identical
in pairs, except for their UUID.
"""
export_filenames = {
'EXISTING': 'export_EXISTING_db.tar.gz',
'FULL': 'export_FULL_db.tar.gz',
'NEW': 'export_NEW_db.tar.gz'
}
# Get user
# Will have to do this again after resetting the DB
user = orm.User.objects.get_default()
## Part I
# Create node and save UUID
calc = orm.CalculationNode()
calc.store()
calc.seal()
calc_uuid = calc.uuid
# Create first comment
orm.Comment(calc, user, self.comments[0]).store()
# There should be exactly: 1 CalculationNode, 1 Comment
export_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid'])
export_comments = orm.QueryBuilder().append(orm.Comment,
project=['uuid'])
self.assertEqual(export_calcs.count(), 1)
self.assertEqual(export_comments.count(), 1)
# Save Comment UUID before export
existing_comment_uuids = [str(export_comments.all()[0][0])]
# Export "EXISTING" DB
export_file_existing = os.path.join(temp_dir,
export_filenames['EXISTING'])
export([calc], outfile=export_file_existing, silent=True)
# Add remaining Comments
for comment in self.comments[1:]:
orm.Comment(calc, user, comment).store()
# There should be exactly: 1 CalculationNode, 3 Comments (len(self.comments))
export_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid'])
export_comments = orm.QueryBuilder().append(orm.Comment,
project=['uuid'])
self.assertEqual(export_calcs.count(), 1)
self.assertEqual(export_comments.count(), len(self.comments))
# Save Comment UUIDs before export, there should be 4 UUIDs in total (len(self.comments))
full_comment_uuids = set(existing_comment_uuids)
for comment_uuid in export_comments.all():
full_comment_uuids.add(str(comment_uuid[0]))
self.assertEqual(len(full_comment_uuids), len(self.comments))
# Export "FULL" DB
export_file_full = os.path.join(temp_dir, export_filenames['FULL'])
export([calc], outfile=export_file_full, silent=True)
# Clean database
self.reset_database()
## Part II
# Reimport "EXISTING" DB
import_data(export_file_existing, silent=True)
# Check the database is correctly imported.
# There should be exactly: 1 CalculationNode, 1 Comment
import_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid'])
import_comments = orm.QueryBuilder().append(orm.Comment,
project=['uuid'])
self.assertEqual(import_calcs.count(), 1)
self.assertEqual(import_comments.count(), 1)
# Furthermore, the UUIDs should be the same
self.assertEqual(str(import_calcs.all()[0][0]), calc_uuid)
self.assertIn(str(import_comments.all()[0][0]), existing_comment_uuids)
# Add remaining Comments (again)
calc = orm.load_node(
import_calcs.all()[0][0]) # Reload CalculationNode
user = orm.User.objects.get_default() # Get user - again
for comment in self.comments[1:]:
orm.Comment(calc, user, comment).store()
# There should be exactly: 1 CalculationNode, 4 Comments (len(self.comments))
export_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid'])
export_comments = orm.QueryBuilder().append(orm.Comment,
project=['uuid'])
self.assertEqual(export_calcs.count(), 1)
self.assertEqual(export_comments.count(), len(self.comments))
# Save Comment UUIDs before export, there should be 4 UUIDs in total (len(self.comments))
new_comment_uuids = set(existing_comment_uuids)
for comment_uuid in export_comments.all():
new_comment_uuids.add(str(comment_uuid[0]))
self.assertEqual(len(new_comment_uuids), len(self.comments))
# Export "NEW" DB
export_file_new = os.path.join(temp_dir, export_filenames['NEW'])
export([calc], outfile=export_file_new, silent=True)
# Clean database
self.reset_database()
## Part III
# Reimport "EXISTING" DB
import_data(export_file_existing, silent=True)
# Check the database is correctly imported.
# There should be exactly: 1 CalculationNode, 1 Comment
import_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid'])
import_comments = orm.QueryBuilder().append(orm.Comment,
project=['uuid'])
self.assertEqual(import_calcs.count(), 1)
self.assertEqual(import_comments.count(), 1)
# Furthermore, the UUIDs should be the same
self.assertEqual(str(import_calcs.all()[0][0]), calc_uuid)
self.assertIn(str(import_comments.all()[0][0]), existing_comment_uuids)
# Import "FULL" DB
import_data(export_file_full, silent=True)
# Check the database is correctly imported.
# There should be exactly: 1 CalculationNode, 4 Comments (len(self.comments))
import_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid'])
import_comments = orm.QueryBuilder().append(orm.Comment,
project=['uuid'])
self.assertEqual(import_calcs.count(), 1)
self.assertEqual(import_comments.count(), len(self.comments))
# Furthermore, the UUIDs should be the same
self.assertEqual(str(import_calcs.all()[0][0]), calc_uuid)
for comment in import_comments.all():
comment_uuid = str(comment[0])
self.assertIn(comment_uuid, full_comment_uuids)
## Part IV
# Import "NEW" DB
import_data(export_file_new, silent=True)
# Check the database is correctly imported.
# There should be exactly: 1 CalculationNode, 7 Comments (org. (1) + 2 x added (3) Comments)
# 4 of the comments are identical in pairs, except for the UUID.
import_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid'])
import_comments = orm.QueryBuilder().append(
orm.Comment, project=['uuid', 'content'])
self.assertEqual(import_calcs.count(), 1)
self.assertEqual(import_comments.count(), 7)
# Furthermore, the UUIDs should be the same
self.assertEqual(str(import_calcs.all()[0][0]), calc_uuid)
total_comment_uuids = full_comment_uuids.copy()
total_comment_uuids.update(new_comment_uuids)
for comment in import_comments.all():
comment_uuid = str(comment[0])
comment_content = str(comment[1])
self.assertIn(comment_uuid, total_comment_uuids)
self.assertIn(comment_content, self.comments)
def test_reexport(self, temp_dir):
"""
Export something, import and reexport and check if everything is valid.
The export is rather easy::
___ ___ ___
| | INP | | CREATE | |
| p | --> | c | -----> | a |
|___| |___| |___|
"""
import numpy as np
import string
import random
from datetime import datetime
from aiida.common.hashing import make_hash
def get_hash_from_db_content(grouplabel):
"""Helper function to get hash"""
builder = orm.QueryBuilder()
builder.append(orm.Dict, tag='param', project='*')
builder.append(orm.CalculationNode,
tag='calc',
project='*',
edge_tag='p2c',
edge_project=('label', 'type'))
builder.append(orm.ArrayData,
tag='array',
project='*',
edge_tag='c2a',
edge_project=('label', 'type'))
builder.append(orm.Group,
filters={'label': grouplabel},
project='*',
tag='group',
with_node='array')
# I want the query to contain something!
self.assertTrue(builder.count() > 0)
# The hash is given from the preservable entries in an export-import cycle,
# uuids, attributes, labels, descriptions, arrays, link-labels, link-types:
hash_ = make_hash([(
item['param']['*'].attributes,
item['param']['*'].uuid,
item['param']['*'].label,
item['param']['*'].description,
item['calc']['*'].uuid,
item['calc']['*'].attributes,
item['array']['*'].attributes,
[
item['array']['*'].get_array(name).tolist()
for name in item['array']['*'].get_arraynames()
],
item['array']['*'].uuid,
item['group']['*'].uuid,
item['group']['*'].label,
item['p2c']['label'],
item['p2c']['type'],
item['c2a']['label'],
item['c2a']['type'],
item['group']['*'].label,
) for item in builder.dict()])
return hash_
# Creating a folder for the import/export files
chars = string.ascii_uppercase + string.digits
size = 10
grouplabel = 'test-group'
nparr = np.random.random((4, 3, 2)) # pylint: disable=no-member
trial_dict = {}
# give some integers:
trial_dict.update({str(k): np.random.randint(100) for k in range(10)})
# give some floats:
trial_dict.update({str(k): np.random.random() for k in range(10, 20)}) # pylint: disable=no-member
# give some booleans:
trial_dict.update(
{str(k): bool(np.random.randint(1))
for k in range(20, 30)})
# give some text:
trial_dict.update({
str(k): ''.join(random.choice(chars) for _ in range(size))
for k in range(20, 30)
})
param = orm.Dict(dict=trial_dict)
param.label = str(datetime.now())
param.description = 'd_' + str(datetime.now())
param.store()
calc = orm.CalculationNode()
# setting also trial dict as attributes, but randomizing the keys)
for key, value in trial_dict.items():
calc.set_attribute(str(int(key) + np.random.randint(10)), value)
array = orm.ArrayData()
array.set_array('array', nparr)
array.store()
# LINKS
# the calculation has input the parameters-instance
calc.add_incoming(param,
link_type=LinkType.INPUT_CALC,
link_label='input_parameters')
calc.store()
# I want the array to be an output of the calculation
array.add_incoming(calc,
link_type=LinkType.CREATE,
link_label='output_array')
group = orm.Group(label='test-group')
group.store()
group.add_nodes(array)
calc.seal()
hash_from_dbcontent = get_hash_from_db_content(grouplabel)
# I export and reimport 3 times in a row:
for i in range(3):
# Always new filename:
filename = os.path.join(temp_dir, 'export-{}.aiida'.format(i))
# Loading the group from the string
group = orm.Group.get(label=grouplabel)
# exporting based on all members of the group
# this also checks if group memberships are preserved!
export([group] + list(group.nodes), filename=filename, silent=True)
# cleaning the DB!
self.clean_db()
self.create_user()
# reimporting the data from the file
import_data(filename, silent=True, ignore_unknown_nodes=True)
# creating the hash from db content
new_hash = get_hash_from_db_content(grouplabel)
# I check for equality against the first hash created, which implies that hashes
# are equal in all iterations of this process
self.assertEqual(hash_from_dbcontent, new_hash)
def test_link_manager(clear_database_before_test):
"""Test the LinkManager via .inputs and .outputs from a ProcessNode."""
# I first create a calculation with two inputs and two outputs
# Create inputs
inp1 = orm.Data()
inp1.store()
inp2 = orm.Data()
inp2.store()
inp3 = orm.Data()
inp3.store()
# Create calc with inputs
calc = orm.CalculationNode()
calc.add_incoming(inp1, link_type=LinkType.INPUT_CALC, link_label='inp1label')
calc.add_incoming(inp2, link_type=LinkType.INPUT_CALC, link_label='inp2label')
calc.store()
# Attach outputs
out1 = orm.Data()
out2 = orm.Data()
out1.add_incoming(calc, link_type=LinkType.CREATE, link_label='out1label')
out1.store()
out2.add_incoming(calc, link_type=LinkType.CREATE, link_label='out2label')
out2.store()
expected_inputs = {'inp1label': inp1.uuid, 'inp2label': inp2.uuid}
expected_outputs = {'out1label': out1.uuid, 'out2label': out2.uuid}
#### Check the 'inputs' manager ###
# Check that dir() return all keys and nothing else, important
# for tab competion (we skip anything that starts with an underscore)
assert len([key for key in dir(calc.inputs) if not key.startswith('_')]) == len(expected_inputs)
assert set(key for key in dir(calc.inputs) if not key.startswith('_')) == set(expected_inputs)
# Check that it works also as an iterator
assert len(list(calc.inputs)) == len(expected_inputs)
assert set(calc.inputs) == set(expected_inputs)
for key, val in expected_inputs.items():
# calc.inputs.a.uuid == ..., ...
assert getattr(calc.inputs, key).uuid == val
# calc.inputs['a'].uuid == ..., ...
assert calc.inputs[key].uuid == val
# I check the attribute fetching directly
assert calc.inputs.inp1label.uuid == expected_inputs['inp1label']
## Check for not-existing links
# - Must raise a AttributeError, otherwise tab competion will not work
# - Actually raises a NotExistentAttributeError
# - NotExistentAttributeError should also be caught by NotExistent,
# for backwards-compatibility for AiiDA 1.0, 1.1, 1.2
for exception in [AttributeError, NotExistent, NotExistentAttributeError]:
with pytest.raises(exception):
getattr(calc.inputs, 'NotExistentLabel')
# - Must raise a KeyError to behave like a dictionary
# - Actually raises a NotExistentKeyError
# - NotExistentKeyError should also be caught by NotExistent,
# for backwards-compatibility for AiiDA 1.0, 1.1, 1.2
for exception in [KeyError, NotExistent, NotExistentKeyError]:
with pytest.raises(exception):
_ = calc.inputs['NotExistentLabel']
#### Check the 'outputs' manager ###
# Check that dir() return all keys and nothing else, important
# for tab competion (we skip anything that starts with an underscore)
assert len([key for key in dir(calc.outputs) if not key.startswith('_')]) == len(expected_outputs)
assert set(key for key in dir(calc.outputs) if not key.startswith('_')) == set(expected_outputs)
# Check that it works also as an iterator
assert len(list(calc.outputs)) == len(expected_outputs)
assert set(calc.outputs) == set(expected_outputs)
for key, val in expected_outputs.items():
# calc.outputs.a.uuid == ..., ...
assert getattr(calc.outputs, key).uuid == val
# calc.outputs['a'].uuid == ..., ...
assert calc.outputs[key].uuid == val
# I check the attribute fetching directly
assert calc.outputs.out1label.uuid == expected_outputs['out1label']
# Must raise a AttributeError, otherwise tab competion will not work
with pytest.raises(AttributeError):
getattr(calc.outputs, 'NotExistentLabel')
# Must raise a KeyError
with pytest.raises(KeyError):
_ = calc.outputs['NotExistentLabel']
def test_cycle_structure_data(self):
"""
Create an export with some orm.CalculationNode and Data nodes and import it after having
cleaned the database. Verify that the nodes and their attributes are restored
properly after importing the created export archive
"""
from aiida.common.links import LinkType
test_label = 'Test structure'
test_cell = [[8.34, 0.0, 0.0],
[0.298041701839357, 8.53479766274308, 0.0],
[0.842650688117053, 0.47118495164127, 10.6965192730702]]
test_kinds = [{
'symbols': ['Fe'],
'weights': [1.0],
'mass': 55.845,
'name': 'Fe'
}, {
'symbols': ['S'],
'weights': [1.0],
'mass': 32.065,
'name': 'S'
}]
structure = orm.StructureData(cell=test_cell)
structure.append_atom(symbols=['Fe'], position=[0, 0, 0])
structure.append_atom(symbols=['S'], position=[2, 2, 2])
structure.label = test_label
structure.store()
parent_process = orm.CalculationNode()
parent_process.set_attribute('key', 'value')
parent_process.store()
child_calculation = orm.CalculationNode()
child_calculation.set_attribute('key', 'value')
remote_folder = orm.RemoteData(computer=self.computer,
remote_path='/').store()
remote_folder.add_incoming(parent_process,
link_type=LinkType.CREATE,
link_label='link')
child_calculation.add_incoming(remote_folder,
link_type=LinkType.INPUT_CALC,
link_label='link')
child_calculation.store()
structure.add_incoming(child_calculation,
link_type=LinkType.CREATE,
link_label='link')
parent_process.seal()
child_calculation.seal()
with tempfile.NamedTemporaryFile() as handle:
nodes = [
structure, child_calculation, parent_process, remote_folder
]
export(nodes, outfile=handle.name, overwrite=True, silent=True)
# Check that we have the expected number of nodes in the database
self.assertEqual(orm.QueryBuilder().append(orm.Node).count(),
len(nodes))
# Clean the database and verify there are no nodes left
self.clean_db()
self.create_user()
self.assertEqual(orm.QueryBuilder().append(orm.Node).count(), 0)
# After importing we should have the original number of nodes again
import_data(handle.name, silent=True)
self.assertEqual(orm.QueryBuilder().append(orm.Node).count(),
len(nodes))
# Verify that orm.CalculationNodes have non-empty attribute dictionaries
builder = orm.QueryBuilder().append(orm.CalculationNode)
for [calculation] in builder.iterall():
self.assertIsInstance(calculation.attributes, dict)
self.assertNotEqual(len(calculation.attributes), 0)
# Verify that the structure data maintained its label, cell and kinds
builder = orm.QueryBuilder().append(orm.StructureData)
for [structure] in builder.iterall():
self.assertEqual(structure.label, test_label)
# Check that they are almost the same, within numerical precision
self.assertTrue(
np.abs(np.array(structure.cell) -
np.array(test_cell)).max() < 1.e-12)
builder = orm.QueryBuilder().append(orm.StructureData,
project=['attributes.kinds'])
for [kinds] in builder.iterall():
self.assertEqual(len(kinds), 2)
for kind in kinds:
self.assertIn(kind, test_kinds)
# Check that there is a StructureData that is an output of a orm.CalculationNode
builder = orm.QueryBuilder()
builder.append(orm.CalculationNode,
project=['uuid'],
tag='calculation')
builder.append(orm.StructureData, with_incoming='calculation')
self.assertGreater(len(builder.all()), 0)
# Check that there is a RemoteData that is a child and parent of a orm.CalculationNode
builder = orm.QueryBuilder()
builder.append(orm.CalculationNode, tag='parent')
builder.append(orm.RemoteData,
project=['uuid'],
with_incoming='parent',
tag='remote')
builder.append(orm.CalculationNode, with_incoming='remote')
self.assertGreater(len(builder.all()), 0)
def _create_basic_graph():
"""
Creates a basic graph which has one parent workflow (work_2) that calls
a child workflow (work_1) which calls a calculation function (calc_0).
There is one input (data_i) and one output (data_o). It has at least one
link of each class:
* CALL_WORK from work_2 to work_1.
* CALL_CALC from work_1 to calc_0.
* INPUT_CALC from data_i to calc_0 and CREATE from calc_0 to data_o.
* INPUT_WORK from data_i to work_1 and RETURN from work_1 to data_o.
* INPUT_WORK from data_i to work_2 and RETURN from work_2 to data_o.
This graph looks like this::
input_work +--------+ return
+-------------------> | work_2 | --------------------+
| +--------+ |
| | |
| | call_work |
| | |
| v |
| input_work +--------+ return |
| +----------------> | work_1 | -----------------+ |
| | +--------+ | |
| | | | |
| | | call_calc | |
| | | | |
| | v v v
+--------+ input_calc +--------+ create +--------+
| data_i | ------------> | calc_0 | ------------> | data_o |
+--------+ +--------+ +--------+
"""
data_i = orm.Data().store()
work_2 = orm.WorkflowNode()
work_2.add_incoming(data_i, link_type=LinkType.INPUT_WORK, link_label='inpwork2')
work_2.store()
work_1 = orm.WorkflowNode()
work_1.add_incoming(data_i, link_type=LinkType.INPUT_WORK, link_label='inpwork1')
work_1.add_incoming(work_2, link_type=LinkType.CALL_WORK, link_label='callwork')
work_1.store()
calc_0 = orm.CalculationNode()
calc_0.add_incoming(data_i, link_type=LinkType.INPUT_CALC, link_label='inpcalc0')
calc_0.add_incoming(work_1, link_type=LinkType.CALL_CALC, link_label='callcalc')
calc_0.store()
data_o = orm.Data()
data_o.add_incoming(calc_0, link_type=LinkType.CREATE, link_label='create0')
data_o.store()
data_o.add_incoming(work_2, link_type=LinkType.RETURN, link_label='return2')
data_o.add_incoming(work_1, link_type=LinkType.RETURN, link_label='return1')
output_dict = {
'data_i': data_i,
'data_o': data_o,
'calc_0': calc_0,
'work_1': work_1,
'work_2': work_2,
}
return output_dict
def test_multiple_post_return_links(self, temp_dir): # pylint: disable=too-many-locals
"""Check extra RETURN links can be added to existing Nodes, when label is not unique"""
data = orm.Int(1).store()
calc = orm.CalculationNode().store()
work = orm.WorkflowNode().store()
link_label = 'output_data'
data.add_incoming(calc, LinkType.CREATE, link_label)
data.add_incoming(work, LinkType.RETURN, link_label)
calc.seal()
work.seal()
data_uuid = data.uuid
calc_uuid = calc.uuid
work_uuid = work.uuid
before_links = get_all_node_links()
data_provenance = os.path.join(temp_dir, 'data.aiida')
all_provenance = os.path.join(temp_dir, 'all.aiida')
export([data], filename=data_provenance, return_backward=False)
export([data], filename=all_provenance, return_backward=True)
self.clean_db()
# import data provenance
import_data(data_provenance)
no_of_work = orm.QueryBuilder().append(orm.WorkflowNode).count()
self.assertEqual(
no_of_work,
0,
msg=
f'{no_of_work} WorkflowNode(s) was/were found, however, none should be present'
)
nodes = orm.QueryBuilder().append(orm.Node, project='uuid')
self.assertEqual(
nodes.count(),
2,
msg=
f'{no_of_work} Node(s) was/were found, however, exactly two should be present'
)
for node in nodes.iterall():
self.assertIn(node[0], [data_uuid, calc_uuid])
links = get_all_node_links()
self.assertEqual(
len(links),
1,
msg='Only a single Link (from Calc. to Data) is expected, '
'instead {} were found (in, out, label, type): {}'.format(
len(links), links))
for from_uuid, to_uuid, found_label, found_type in links:
self.assertEqual(from_uuid, calc_uuid)
self.assertEqual(to_uuid, data_uuid)
self.assertEqual(found_label, link_label)
self.assertEqual(found_type, LinkType.CREATE.value)
# import data+logic provenance
import_data(all_provenance)
no_of_work = orm.QueryBuilder().append(orm.WorkflowNode).count()
self.assertEqual(
no_of_work,
1,
msg=
f'{no_of_work} WorkflowNode(s) was/were found, however, exactly one should be present'
)
nodes = orm.QueryBuilder().append(orm.Node, project='uuid')
self.assertEqual(
nodes.count(),
3,
msg=
f'{no_of_work} Node(s) was/were found, however, exactly three should be present'
)
for node in nodes.iterall():
self.assertIn(node[0], [data_uuid, calc_uuid, work_uuid])
links = get_all_node_links()
self.assertEqual(
len(links),
2,
msg=
f'Exactly two Links are expected, instead {len(links)} were found (in, out, label, type): {links}'
)
self.assertListEqual(sorted(links), sorted(before_links))
def test_reimport_of_logs_for_single_node(self, temp_dir):
"""
When a node with logs already exist in the DB, and more logs are imported
for the same node (same UUID), test that only new log-entries are added.
Part I:
Create CalculationNode and 1 Log for it.
Export CalculationNode with its 1 Log to export file #1 "EXISTING database".
Add 2 Logs to CalculationNode.
Export CalculationNode with its 3 Logs to export file #2 "FULL database".
Reset database.
Part II:
Reimport export file #1 "EXISTING database".
Add 2 Logs to CalculationNode (different UUID than for "FULL database").
Export CalculationNode with its 3 Logs to export file #3 "NEW database".
Reset database.
Part III:
Reimport export file #1 "EXISTING database" (1 CalculationNode, 1 Log).
Import export file #2 "FULL database" (1 CalculationNode, 3 Logs).
Check the database EXACTLY contains 1 CalculationNode and 3 Logs,
with matching UUIDS all the way through all previous Parts.
Part IV:
Import export file #3 "NEW database" (1 CalculationNode, 3 Logs).
Check the database EXACTLY contains 1 CalculationNode and 5 Logs,
with matching UUIDS all the way through all previous Parts.
NB! There should now be 5 Logs in the database. 4 of which are identical
in pairs, except for their UUID.
"""
export_filenames = {
'EXISTING': 'export_EXISTING_db.tar.gz',
'FULL': 'export_FULL_db.tar.gz',
'NEW': 'export_NEW_db.tar.gz'
}
log_msgs = [
'Life is like riding a bicycle.', 'To keep your balance,',
'you must keep moving.'
]
## Part I
# Create node and save UUID
calc = orm.CalculationNode()
calc.store()
calc.seal()
calc_uuid = calc.uuid
# Create first log message
calc.logger.critical(log_msgs[0])
# There should be exactly: 1 CalculationNode, 1 Log
export_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid'])
export_logs = orm.QueryBuilder().append(orm.Log, project=['uuid'])
self.assertEqual(export_calcs.count(), 1)
self.assertEqual(export_logs.count(), 1)
# Save Log UUID before export
existing_log_uuids = [str(export_logs.all()[0][0])]
# Export "EXISTING" DB
export_file_existing = os.path.join(temp_dir,
export_filenames['EXISTING'])
export([calc], outfile=export_file_existing, silent=True)
# Add remaining Log messages
for log_msg in log_msgs[1:]:
calc.logger.critical(log_msg)
# There should be exactly: 1 CalculationNode, 3 Logs (len(log_msgs))
export_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid'])
export_logs = orm.QueryBuilder().append(orm.Log, project=['uuid'])
self.assertEqual(export_calcs.count(), 1)
self.assertEqual(export_logs.count(), len(log_msgs))
# Save Log UUIDs before export, there should be 3 UUIDs in total (len(log_msgs))
full_log_uuids = set(existing_log_uuids)
for log_uuid in export_logs.all():
full_log_uuids.add(str(log_uuid[0]))
self.assertEqual(len(full_log_uuids), len(log_msgs))
# Export "FULL" DB
export_file_full = os.path.join(temp_dir, export_filenames['FULL'])
export([calc], outfile=export_file_full, silent=True)
# Clean database
self.reset_database()
## Part II
# Reimport "EXISTING" DB
import_data(export_file_existing, silent=True)
# Check the database is correctly imported.
# There should be exactly: 1 CalculationNode, 1 Log
import_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid'])
import_logs = orm.QueryBuilder().append(orm.Log, project=['uuid'])
self.assertEqual(import_calcs.count(), 1)
self.assertEqual(import_logs.count(), 1)
# Furthermore, the UUIDs should be the same
self.assertEqual(str(import_calcs.all()[0][0]), calc_uuid)
self.assertIn(str(import_logs.all()[0][0]), existing_log_uuids)
# Add remaining Log messages (again)
calc = orm.load_node(import_calcs.all()[0][0])
for log_msg in log_msgs[1:]:
calc.logger.critical(log_msg)
# There should be exactly: 1 CalculationNode, 3 Logs (len(log_msgs))
export_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid'])
export_logs = orm.QueryBuilder().append(orm.Log, project=['uuid'])
self.assertEqual(export_calcs.count(), 1)
self.assertEqual(export_logs.count(), len(log_msgs))
# Save Log UUIDs before export, there should be 3 UUIDs in total (len(log_msgs))
new_log_uuids = set(existing_log_uuids)
for log_uuid in export_logs.all():
new_log_uuids.add(str(log_uuid[0]))
self.assertEqual(len(new_log_uuids), len(log_msgs))
# Export "NEW" DB
export_file_new = os.path.join(temp_dir, export_filenames['NEW'])
export([calc], outfile=export_file_new, silent=True)
# Clean database
self.reset_database()
## Part III
# Reimport "EXISTING" DB
import_data(export_file_existing, silent=True)
# Check the database is correctly imported.
# There should be exactly: 1 CalculationNode, 1 Log
import_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid'])
import_logs = orm.QueryBuilder().append(orm.Log, project=['uuid'])
self.assertEqual(import_calcs.count(), 1)
self.assertEqual(import_logs.count(), 1)
# Furthermore, the UUIDs should be the same
self.assertEqual(str(import_calcs.all()[0][0]), calc_uuid)
self.assertIn(str(import_logs.all()[0][0]), existing_log_uuids)
# Import "FULL" DB
import_data(export_file_full, silent=True)
# Check the database is correctly imported.
# There should be exactly: 1 CalculationNode, 3 Logs (len(log_msgs))
import_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid'])
import_logs = orm.QueryBuilder().append(orm.Log, project=['uuid'])
self.assertEqual(import_calcs.count(), 1)
self.assertEqual(import_logs.count(), len(log_msgs))
# Furthermore, the UUIDs should be the same
self.assertEqual(str(import_calcs.all()[0][0]), calc_uuid)
for log in import_logs.all():
log_uuid = str(log[0])
self.assertIn(log_uuid, full_log_uuids)
## Part IV
# Import "NEW" DB
import_data(export_file_new, silent=True)
# Check the database is correctly imported.
# There should be exactly: 1 CalculationNode, 5 Logs (len(log_msgs))
# 4 of the logs are identical in pairs, except for the UUID.
import_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid'])
import_logs = orm.QueryBuilder().append(orm.Log,
project=['uuid', 'message'])
self.assertEqual(import_calcs.count(), 1)
self.assertEqual(import_logs.count(), 5)
# Furthermore, the UUIDs should be the same
self.assertEqual(str(import_calcs.all()[0][0]), calc_uuid)
total_log_uuids = full_log_uuids.copy()
total_log_uuids.update(new_log_uuids)
for log in import_logs.all():
log_uuid = str(log[0])
log_message = str(log[1])
self.assertIn(log_uuid, total_log_uuids)
self.assertIn(log_message, log_msgs)
def test_multiple_imports_for_single_node(self, temp_dir):
"""Test multiple imports for single node with different logs are imported correctly"""
log_msgs = [
'Life is like riding a bicycle.', 'To keep your balance,',
'you must keep moving.'
]
# Create Node and initial log message and save UUIDs prior to export
node = orm.CalculationNode().store()
node.seal()
node.logger.critical(log_msgs[0])
node_uuid = node.uuid
log_uuid_existing = orm.QueryBuilder().append(orm.Log,
project=['uuid']).all()
log_uuid_existing = str(log_uuid_existing[0][0])
# Export as "EXISTING" DB
export_file_existing = os.path.join(temp_dir, 'export_EXISTING.tar.gz')
export([node], outfile=export_file_existing, silent=True)
# Add 2 more Logs and save UUIDs for all three Logs prior to export
node.logger.critical(log_msgs[1])
node.logger.critical(log_msgs[2])
log_uuids_full = orm.QueryBuilder().append(orm.Log,
project=['uuid']).all()
log_uuids_full = [str(log[0]) for log in log_uuids_full]
# Export as "FULL" DB
export_file_full = os.path.join(temp_dir, 'export_FULL.tar.gz')
export([node], outfile=export_file_full, silent=True)
# Clean database and reimport "EXISTING" DB
self.reset_database()
import_data(export_file_existing, silent=True)
# Check correct import
builder = orm.QueryBuilder().append(orm.Node,
tag='node',
project=['uuid'])
builder.append(orm.Log, with_node='node', project=['uuid', 'message'])
builder = builder.all()
self.assertEqual(len(builder), 1) # There is 1 Log in "EXISTING" DB
imported_node_uuid = builder[0][0]
self.assertEqual(imported_node_uuid, node_uuid)
imported_log_uuid = builder[0][1]
imported_log_message = builder[0][2]
self.assertEqual(imported_log_uuid, log_uuid_existing)
self.assertEqual(imported_log_message, log_msgs[0])
# Import "FULL" DB
import_data(export_file_full, silent=True)
# Since the UUID of the node is identical with the node already in the DB,
# the Logs should be added to the existing node, avoiding the addition of
# the single Log already present.
# Check this by retrieving all Logs for the node.
builder = orm.QueryBuilder().append(orm.Node,
tag='node',
project=['uuid'])
builder.append(orm.Log, with_node='node', project=['uuid', 'message'])
builder = builder.all()
self.assertEqual(len(builder),
len(log_msgs)) # There should now be 3 Logs
imported_node_uuid = builder[0][0]
self.assertEqual(imported_node_uuid, node_uuid)
for log in builder:
imported_log_uuid = log[1]
imported_log_content = log[2]
self.assertIn(imported_log_uuid, log_uuids_full)
self.assertIn(imported_log_content, log_msgs)
def test_export_of_imported_logs(self, temp_dir):
"""Test export of imported Log"""
log_msg = 'Testing export of imported log'
# Create node
calc = orm.CalculationNode()
calc.store()
calc.seal()
# Create log message
calc.logger.critical(log_msg)
# Save uuids prior to export
calc_uuid = calc.uuid
log_uuid = orm.QueryBuilder().append(orm.Log, project=['uuid']).all()
log_uuid = str(log_uuid[0][0])
# Export
export_file = os.path.join(temp_dir, 'export.tar.gz')
export([calc], outfile=export_file, silent=True)
# Clean database and reimport exported data
self.reset_database()
import_data(export_file, silent=True)
# Finding all the log messages
import_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid']).all()
import_logs = orm.QueryBuilder().append(orm.Log,
project=['uuid']).all()
# There should be exactly: 1 CalculationNode, 1 Log
self.assertEqual(len(import_calcs), 1)
self.assertEqual(len(import_logs), 1)
# Check the UUIDs are the same
self.assertEqual(str(import_calcs[0][0]), calc_uuid)
self.assertEqual(str(import_logs[0][0]), log_uuid)
# Re-export
calc = orm.load_node(import_calcs[0][0])
re_export_file = os.path.join(temp_dir, 're_export.tar.gz')
export([calc], outfile=re_export_file, silent=True)
# Clean database and reimport exported data
self.reset_database()
import_data(re_export_file, silent=True)
# Finding all the log messages
import_calcs = orm.QueryBuilder().append(orm.CalculationNode,
project=['uuid']).all()
import_logs = orm.QueryBuilder().append(orm.Log,
project=['uuid']).all()
# There should be exactly: 1 CalculationNode, 1 Log
self.assertEqual(len(import_calcs), 1)
self.assertEqual(len(import_logs), 1)
# Check the UUIDs are the same
self.assertEqual(str(import_calcs[0][0]), calc_uuid)
self.assertEqual(str(import_logs[0][0]), log_uuid)
def test_comment_querybuilder(self):
# pylint: disable=too-many-locals
"""Test querying for comments by joining on nodes in the QueryBuilder."""
user_one = self.user
user_two = orm.User(email='[email protected]').store()
node_one = orm.Data().store()
comment_one = Comment(node_one, user_one, 'comment_one').store()
node_two = orm.Data().store()
comment_two = Comment(node_two, user_one, 'comment_two').store()
comment_three = Comment(node_two, user_one, 'comment_three').store()
node_three = orm.CalculationNode().store()
comment_four = Comment(node_three, user_two,
'new_user_comment').store()
node_four = orm.CalculationNode().store()
comment_five = Comment(node_four, user_one, 'user one comment').store()
comment_six = Comment(node_four, user_two, 'user two comment').store()
# Retrieve a node by joining on a specific comment
builder = orm.QueryBuilder()
builder.append(Comment, tag='comment', filters={'id': comment_one.id})
builder.append(orm.Node, with_comment='comment', project=['uuid'])
nodes = builder.all()
self.assertEqual(len(nodes), 1)
for node in nodes:
self.assertIn(str(node[0]), [node_one.uuid])
# Retrieve a comment by joining on a specific node
builder = orm.QueryBuilder()
builder.append(orm.Node, tag='node', filters={'id': node_two.id})
builder.append(Comment, with_node='node', project=['uuid'])
comments = builder.all()
self.assertEqual(len(comments), 2)
for comment in comments:
self.assertIn(str(comment[0]),
[comment_two.uuid, comment_three.uuid])
# Retrieve a user by joining on a specific comment
builder = orm.QueryBuilder()
builder.append(Comment, tag='comment', filters={'id': comment_four.id})
builder.append(orm.User, with_comment='comment', project=['email'])
users = builder.all()
self.assertEqual(len(users), 1)
for user in users:
self.assertEqual(str(user[0]), user_two.email)
# Retrieve a comment by joining on a specific user
builder = orm.QueryBuilder()
builder.append(orm.User, tag='user', filters={'email': user_one.email})
builder.append(Comment, with_user='******', project=['uuid'])
comments = builder.all()
self.assertEqual(len(comments), 5)
for comment in comments:
self.assertIn(str(comment[0]), [
self.comment.uuid, comment_one.uuid, comment_two.uuid,
comment_three.uuid, comment_five.uuid
])
# Retrieve users from comments of a single node by joining specific node
builder = orm.QueryBuilder()
builder.append(orm.Node, tag='node', filters={'id': node_four.id})
builder.append(Comment,
tag='comments',
with_node='node',
project=['uuid'])
builder.append(orm.User, with_comment='comments', project=['email'])
comments_and_users = builder.all()
self.assertEqual(len(comments_and_users), 2)
for entry in comments_and_users:
self.assertEqual(len(entry), 2)
comment_uuid = str(entry[0])
user_email = str(entry[1])
self.assertIn(comment_uuid, [comment_five.uuid, comment_six.uuid])
self.assertIn(user_email, [user_one.email, user_two.email])
def test_group_name_and_type_change(self, temp_dir):
""" Group's name and type columns have changed
Change for columns:
“name” --> “label”
"type" --> "type_string"
Furthermore, type_strings have been updated to:
"" --> "user"
"data.upf.family" --> "data.upf"
"aiida.import" --> "auto.import"
"autogroup.run" --> "auto.run"
The new columns are called on group instances, and will fail if not present.
A user created Group is validated to have the "user" value as a type_string.
A UPF file is created and imported/uploaded as a UPF family,
in order to create a Group with type_string="data.upf".
Any import will create a Group with type_string "auto.import", which is checked.
The type_string="auto.run" is not directly checked, but if the three checks
above succeed, it is understood that "auto.run" is also correctly ex-/imported
as the type_string content for the relevant Groups.
"""
from aiida.orm.nodes.data.upf import upload_upf_family
# To be saved
groups_label = ['Users', 'UpfData']
upf_filename = 'Al.test_file.UPF'
# regular upf file version 2 header
upf_contents = '\n'.join([
"<UPF version=\"2.0.1\">",
'Human readable section is completely irrelevant for parsing!',
'<PP_HEADER',
'contents before element tag',
"element=\"Al\"",
'contents following element tag',
'>',
])
path_to_upf = os.path.join(temp_dir, upf_filename)
with open(path_to_upf, 'w') as upf_file:
upf_file.write(upf_contents)
# Create Groups
node1 = orm.CalculationNode().store()
node2 = orm.CalculationNode().store()
node1.seal()
node2.seal()
group_user = orm.Group(label=groups_label[0]).store()
group_user.add_nodes([node1, node2])
upload_upf_family(temp_dir, groups_label[1], '')
group_upf = orm.load_group(groups_label[1])
# Save uuids and type
groups_uuid = [str(g.uuid) for g in [group_user, group_upf]]
groups_type_string = [g.type_string for g in [group_user, group_upf]]
# Assert correct type strings exists prior to export
self.assertListEqual(groups_type_string, ['core', 'core.upf'])
# Export node
filename = os.path.join(temp_dir, 'export.aiida')
export([group_user, group_upf], filename=filename)
# Clean the database and reimport
self.clean_db()
import_data(filename)
# Retrieve Groups and make sure exactly 3 are retrieved (including the "import group")
builder = orm.QueryBuilder()
builder.append(orm.Group, project=['uuid'])
imported_groups = builder.all()
self.assertEqual(builder.count(), 3)
# Check uuids are the same after import
imported_groups_uuid = [str(g[0]) for g in imported_groups]
# We do not know the "import group"'s uuid, so go through known uuids
for group_uuid in groups_uuid:
self.assertIn(group_uuid, imported_groups_uuid)
# Pop known uuid from imported_groups_uuid, eventually leaving
# only the "import group"
imported_groups_uuid.remove(group_uuid)
# Load group
imported_group = orm.load_group(group_uuid)
# Check whether types are correctly imported
self.assertIn(imported_group.type_string, groups_type_string)
# Assert labels are imported correctly
self.assertIn(imported_group.label, groups_label)
# Check type_string content of "import group"
import_group = orm.load_group(imported_groups_uuid[0])
self.assertEqual(import_group.type_string, 'core.import')
