def test_import_to_group(self):
"""
Test import to existing Group and that Nodes are added correctly for multiple imports of the same,
as well as separate, archives.
"""
archives = [
get_archive_file('arithmetic.add.aiida', filepath='calcjob'),
get_archive_file(self.newest_archive, filepath=self.archive_path)
]
group_label = 'import_madness'
group = Group(group_label).store()
self.assertTrue(group.is_empty, msg='The Group should be empty.')
# Invoke `verdi import`, making sure there are no exceptions
options = ['-G', group.label] + [archives[0]]
result = self.cli_runner.invoke(cmd_import.cmd_import, options)
self.assertIsNone(result.exception, msg=result.output)
self.assertEqual(result.exit_code, 0, msg=result.output)
self.assertFalse(group.is_empty,
msg='The Group should no longer be empty.')
nodes_in_group = group.count()
# Invoke `verdi import` again, making sure Group count doesn't change
options = ['-G', group.label] + [archives[0]]
result = self.cli_runner.invoke(cmd_import.cmd_import, options)
self.assertIsNone(result.exception, msg=result.output)
self.assertEqual(result.exit_code, 0, msg=result.output)
self.assertEqual(
group.count(),
nodes_in_group,
msg=
'The Group count should not have changed from {}. Instead it is now {}'
.format(nodes_in_group, group.count()))
# Invoke `verdi import` again with new archive, making sure Group count is upped
options = ['-G', group.label] + [archives[1]]
result = self.cli_runner.invoke(cmd_import.cmd_import, options)
self.assertIsNone(result.exception, msg=result.output)
self.assertEqual(result.exit_code, 0, msg=result.output)
self.assertGreater(
group.count(),
nodes_in_group,
msg=
'There should now be more than {} nodes in group {} , instead there are {}'
.format(nodes_in_group, group_label, group.count()))
def test_import_url_and_local_archives(self):
"""Test import of both a remote and local archive"""
url_archive = 'export_v0.4_no_UPF.aiida'
local_archive = self.newest_archive
options = [
get_archive_file(local_archive, filepath=self.archive_path),
self.url_path + url_archive,
get_archive_file(local_archive, filepath=self.archive_path)
]
result = self.cli_runner.invoke(cmd_import.cmd_import, options)
self.assertIsNone(result.exception, result.output)
self.assertEqual(result.exit_code, 0, result.output)
def test_import_archive(self):
"""
Test import for archive files from disk
"""
archives = [
get_archive_file('arithmetic.add.aiida', filepath='calcjob'),
get_archive_file(self.newest_archive, filepath=self.archive_path)
]
options = [] + archives
result = self.cli_runner.invoke(cmd_import.cmd_import, options)
self.assertIsNone(result.exception, result.output)
self.assertEqual(result.exit_code, 0, result.output)
def test_import_make_new_group(self):
"""Make sure imported entities are saved in new Group"""
# Initialization
group_label = 'new_group_for_verdi_import'
archives = [
get_archive_file(self.newest_archive, filepath=self.archive_path)
]
# Check Group does not already exist
group_search = Group.objects.find(filters={'label': group_label})
self.assertEqual(
len(group_search),
0,
msg="A Group with label '{}' already exists, this shouldn't be.".
format(group_label))
# Invoke `verdi import`, making sure there are no exceptions
options = ['-G', group_label] + archives
result = self.cli_runner.invoke(cmd_import.cmd_import, options)
self.assertIsNone(result.exception, msg=result.output)
self.assertEqual(result.exit_code, 0, msg=result.output)
# Make sure new Group was created
(group, new_group) = Group.objects.get_or_create(group_label)
self.assertFalse(
new_group,
msg=
'The Group should not have been created now, but instead when it was imported.'
)
self.assertFalse(group.is_empty, msg='The Group should not be empty.')
def test_no_node_export(self, temp_dir):
"""Test migration of export file that has no Nodes"""
input_file = get_archive_file('export_v0.3_no_Nodes.aiida', **self.external_archive)
output_file = os.path.join(temp_dir, 'output_file.aiida')
# Known entities
computer_uuids = [self.computer.uuid] # pylint: disable=no-member
user_emails = [orm.User.objects.get_default().email]
# Known export file content used for checks
node_count = 0
computer_count = 1 + 1 # localhost is always present
computer_uuids.append('4f33c6fd-b624-47df-9ffb-a58f05d323af')
user_emails.append('aiida@localhost')
# Perform the migration
migrate_archive(input_file, output_file)
# Load the migrated file
import_data(output_file, silent=True)
# Check known number of entities is present
self.assertEqual(orm.QueryBuilder().append(orm.Node).count(), node_count)
self.assertEqual(orm.QueryBuilder().append(orm.Computer).count(), computer_count)
# Check unique identifiers
computers = orm.QueryBuilder().append(orm.Computer, project=['uuid']).all()[0][0]
users = orm.QueryBuilder().append(orm.User, project=['email']).all()[0][0]
self.assertIn(computers, computer_uuids)
self.assertIn(users, user_emails)
def test_migrate_v3_to_v4(self):
"""Test function migrate_v3_to_v4"""
from aiida import get_version
# Get metadata.json and data.json as dicts from v0.4 file archive
metadata_v4, data_v4 = get_json_files('export_v0.4_simple.aiida', **self.core_archive)
verify_metadata_version(metadata_v4, version='0.4')
# Get metadata.json and data.json as dicts from v0.3 file archive
# Cannot use 'get_json_files' for 'export_v0.3_simple.aiida',
# because we need to pass the SandboxFolder to 'migrate_v3_to_v4'
dirpath_archive = get_archive_file('export_v0.3_simple.aiida', **self.core_archive)
with SandboxFolder(sandbox_in_repo=False) as folder:
if zipfile.is_zipfile(dirpath_archive):
extract_zip(dirpath_archive, folder, silent=True)
elif tarfile.is_tarfile(dirpath_archive):
extract_tar(dirpath_archive, folder, silent=True)
else:
raise ValueError('invalid file format, expected either a zip archive or gzipped tarball')
try:
with io.open(folder.get_abs_path('data.json'), 'r', encoding='utf8') as fhandle:
data_v3 = jsonload(fhandle)
with io.open(folder.get_abs_path('metadata.json'), 'r', encoding='utf8') as fhandle:
metadata_v3 = jsonload(fhandle)
except IOError:
raise NotExistent('export archive does not contain the required file {}'.format(fhandle.filename))
verify_metadata_version(metadata_v3, version='0.3')
# Migrate to v0.4
migrate_v3_to_v4(metadata_v3, data_v3, folder)
verify_metadata_version(metadata_v3, version='0.4')
# Remove AiiDA version, since this may change irregardless of the migration function
metadata_v3.pop('aiida_version')
metadata_v4.pop('aiida_version')
# Assert conversion message in `metadata.json` is correct and then remove it for later assertions
self.maxDiff = None # pylint: disable=invalid-name
conversion_message = 'Converted from version 0.3 to 0.4 with AiiDA v{}'.format(get_version())
self.assertEqual(
metadata_v3.pop('conversion_info')[-1],
conversion_message,
msg='The conversion message after migration is wrong'
)
metadata_v4.pop('conversion_info')
# Assert changes were performed correctly
self.assertDictEqual(
metadata_v3,
metadata_v4,
msg='After migration, metadata.json should equal intended metadata.json from archives'
)
self.assertDictEqual(
data_v3, data_v4, msg='After migration, data.json should equal intended data.json from archives'
)
def test_v02_to_newest(self, temp_dir):
"""Test migration of exported files from v0.2 to newest export version"""
# Get export file with export version 0.2
input_file = get_archive_file('export_v0.2.aiida',
**self.external_archive)
output_file = os.path.join(temp_dir, 'output_file.aiida')
# Perform the migration
migrate_archive(input_file, output_file)
metadata, _ = get_json_files(output_file)
verify_metadata_version(metadata, version=newest_version)
# Load the migrated file
import_data(output_file, silent=True)
# Do the necessary checks
self.assertEqual(orm.QueryBuilder().append(orm.Node).count(),
self.node_count)
# Verify that CalculationNodes have non-empty attribute dictionaries
builder = orm.QueryBuilder().append(orm.CalculationNode)
for [calculation] in builder.iterall():
self.assertIsInstance(calculation.attributes, dict)
self.assertNotEqual(len(calculation.attributes), 0)
# Verify that the StructureData nodes maintained their (same) label, cell, and kinds
builder = orm.QueryBuilder().append(orm.StructureData)
self.assertEqual(
builder.count(),
self.struct_count,
msg='There should be {} StructureData, instead {} were/was found'.
format(self.struct_count, builder.count()))
for structures in builder.all():
structure = structures[0]
self.assertEqual(structure.label, self.known_struct_label)
self.assertEqual(structure.cell, self.known_cell)
builder = orm.QueryBuilder().append(orm.StructureData,
project=['attributes.kinds'])
for [kinds] in builder.iterall():
self.assertEqual(len(kinds), len(self.known_kinds))
for kind in kinds:
self.assertIn(kind,
self.known_kinds,
msg="Kind '{}' not found in: {}".format(
kind, self.known_kinds))
# Check that there is a StructureData that is an input of a CalculationNode
builder = orm.QueryBuilder()
builder.append(orm.StructureData, tag='structure')
builder.append(orm.CalculationNode, with_incoming='structure')
self.assertGreater(len(builder.all()), 0)
# Check that there is a RemoteData that is the output of a CalculationNode
builder = orm.QueryBuilder()
builder.append(orm.CalculationNode, tag='parent')
builder.append(orm.RemoteData, with_incoming='parent')
self.assertGreater(len(builder.all()), 0)
def test_inspect_empty_archive(self):
"""Test the functionality of `verdi export inspect` for an empty archive."""
filename_input = get_archive_file('empty.aiida',
filepath=self.fixture_archive)
options = [filename_input]
result = self.cli_runner.invoke(cmd_export.inspect, options)
self.assertIsNotNone(result.exception, result.output)
self.assertIn('corrupt archive', result.output)
def test_non_interactive_and_migration(self):
"""Test options `--non-interactive` and `--migration`/`--no-migration`
`migration` = True (default), `non_interactive` = False (default), Expected: Query user, migrate
`migration` = True (default), `non_interactive` = True, Expected: No query, migrate
`migration` = False, `non_interactive` = False (default), Expected: No query, no migrate
`migration` = False, `non_interactive` = True, Expected: No query, no migrate
"""
archive = get_archive_file('export_v0.1_simple.aiida',
filepath=self.archive_path)
confirm_message = 'Do you want to try and migrate {} to the newest export file version?'.format(
archive)
success_message = 'Success: imported archive {}'.format(archive)
# Import "normally", but explicitly specifying `--migration`, make sure confirm message is present
# `migration` = True (default), `non_interactive` = False (default), Expected: Query user, migrate
options = ['--migration', archive]
result = self.cli_runner.invoke(cmd_import.cmd_import, options)
self.assertIsNone(result.exception, msg=result.output)
self.assertEqual(result.exit_code, 0, msg=result.output)
self.assertIn(confirm_message, result.output, msg=result.exception)
self.assertIn(success_message, result.output, msg=result.exception)
# Import using non-interactive, make sure confirm message has gone
# `migration` = True (default), `non_interactive` = True, Expected: No query, migrate
options = ['--non-interactive', archive]
result = self.cli_runner.invoke(cmd_import.cmd_import, options)
self.assertIsNone(result.exception, msg=result.output)
self.assertEqual(result.exit_code, 0, msg=result.output)
self.assertNotIn(confirm_message, result.output, msg=result.exception)
self.assertIn(success_message, result.output, msg=result.exception)
# Import using `--no-migration`, make sure confirm message has gone
# `migration` = False, `non_interactive` = False (default), Expected: No query, no migrate
options = ['--no-migration', archive]
result = self.cli_runner.invoke(cmd_import.cmd_import, options)
self.assertIsNotNone(result.exception, msg=result.output)
self.assertNotEqual(result.exit_code, 0, msg=result.output)
self.assertNotIn(confirm_message, result.output, msg=result.exception)
self.assertNotIn(success_message, result.output, msg=result.exception)
# Import using `--no-migration` and `--non-interactive`, make sure confirm message has gone
# `migration` = False, `non_interactive` = True, Expected: No query, no migrate
options = ['--no-migration', '--non-interactive', archive]
result = self.cli_runner.invoke(cmd_import.cmd_import, options)
self.assertIsNotNone(result.exception, msg=result.output)
self.assertNotEqual(result.exit_code, 0, msg=result.output)
self.assertNotIn(confirm_message, result.output, msg=result.exception)
self.assertNotIn(success_message, result.output, msg=result.exception)
def test_migrate_versions_recent(self):
"""Migrating an archive with the current version should exit with non-zero status."""
filename_input = get_archive_file(self.newest_archive,
filepath=self.fixture_archive)
filename_output = next(tempfile._get_candidate_names()) # pylint: disable=protected-access
try:
options = [filename_input, filename_output]
result = self.cli_runner.invoke(cmd_export.migrate, options)
self.assertIsNotNone(result.exception)
finally:
delete_temporary_file(filename_output)
def test_comment_mode(self):
"""Test toggling comment mode flag"""
archives = [
get_archive_file(self.newest_archive, filepath=self.archive_path)
]
for mode in {'newest', 'overwrite'}:
options = ['--comment-mode', mode] + archives
result = self.cli_runner.invoke(cmd_import.cmd_import, options)
self.assertIsNone(result.exception, result.output)
self.assertIn('Comment mode: {}'.format(mode), result.output)
self.assertEqual(result.exit_code, 0, result.output)
def test_migrate_tar_gz(self):
"""Test that -F/--archive-format option can be used to write a tar.gz instead."""
filename_input = get_archive_file(self.penultimate_archive,
filepath=self.fixture_archive)
filename_output = next(tempfile._get_candidate_names()) # pylint: disable=protected-access
for option in ['-F', '--archive-format']:
try:
options = [option, 'tar.gz', filename_input, filename_output]
result = self.cli_runner.invoke(cmd_export.migrate, options)
self.assertIsNone(result.exception, result.output)
self.assertTrue(os.path.isfile(filename_output))
self.assertTrue(tarfile.is_tarfile(filename_output))
finally:
delete_temporary_file(filename_output)
def test_migrate_recursively(self):
"""Test function 'migrate_recursively'"""
import io
import tarfile
import zipfile
from aiida.common.exceptions import NotExistent
from aiida.common.folders import SandboxFolder
from aiida.common.json import load as jsonload
from aiida.tools.importexport.common.archive import extract_tar, extract_zip
# Get metadata.json and data.json as dicts from v0.1 file archive
# Cannot use 'get_json_files' for 'export_v0.1_simple.aiida',
# because we need to pass the SandboxFolder to 'migrate_recursively'
dirpath_archive = get_archive_file('export_v0.1_simple.aiida',
**self.core_archive)
with SandboxFolder(sandbox_in_repo=False) as folder:
if zipfile.is_zipfile(dirpath_archive):
extract_zip(dirpath_archive, folder, silent=True)
elif tarfile.is_tarfile(dirpath_archive):
extract_tar(dirpath_archive, folder, silent=True)
else:
raise ValueError(
'invalid file format, expected either a zip archive or gzipped tarball'
)
try:
with io.open(folder.get_abs_path('data.json'),
'r',
encoding='utf8') as fhandle:
data = jsonload(fhandle)
with io.open(folder.get_abs_path('metadata.json'),
'r',
encoding='utf8') as fhandle:
metadata = jsonload(fhandle)
except IOError:
raise NotExistent(
'export archive does not contain the required file {}'.
format(fhandle.filename))
verify_metadata_version(metadata, version='0.1')
# Migrate to newest version
new_version = migrate_recursively(metadata, data, folder)
verify_metadata_version(metadata, version=newest_version)
self.assertEqual(new_version, newest_version)
def test_migrate_silent(self):
"""Test that the captured output is an empty string when the -s/--silent option is passed."""
filename_input = get_archive_file(self.penultimate_archive,
filepath=self.fixture_archive)
filename_output = next(tempfile._get_candidate_names()) # pylint: disable=protected-access
for option in ['-s', '--silent']:
try:
options = [option, filename_input, filename_output]
result = self.cli_runner.invoke(cmd_export.migrate, options)
self.assertEqual(result.output, '')
self.assertIsNone(result.exception, result.output)
self.assertTrue(os.path.isfile(filename_output))
self.assertEqual(
zipfile.ZipFile(filename_output).testzip(), None)
finally:
delete_temporary_file(filename_output)
def test_import_old_local_archives(self):
""" Test import of old local archives
Expected behavior: Automatically migrate to newest version and import correctly.
"""
archives = []
for version in range(1, int(EXPORT_VERSION.split('.')[-1]) - 1):
archives.append(('export_v0.{}_simple.aiida'.format(version),
'0.{}'.format(version)))
for archive, version in archives:
options = [get_archive_file(archive, filepath=self.archive_path)]
result = self.cli_runner.invoke(cmd_import.cmd_import, options)
self.assertIsNone(result.exception, msg=result.output)
self.assertEqual(result.exit_code, 0, msg=result.output)
self.assertIn(version, result.output, msg=result.exception)
self.assertIn('Success: imported archive {}'.format(options[0]),
result.output,
msg=result.exception)
def test_migrate_versions_old(self):
"""Migrating archives with a version older than the current should work."""
archives = []
for version in range(1, int(EXPORT_VERSION.split('.')[-1]) - 1):
archives.append('export_v0.{}_simple.aiida'.format(version))
for archive in archives:
filename_input = get_archive_file(archive,
filepath=self.fixture_archive)
filename_output = next(tempfile._get_candidate_names()) # pylint: disable=protected-access
try:
options = [filename_input, filename_output]
result = self.cli_runner.invoke(cmd_export.migrate, options)
self.assertIsNone(result.exception, result.output)
self.assertTrue(os.path.isfile(filename_output))
self.assertEqual(
zipfile.ZipFile(filename_output).testzip(), None)
finally:
delete_temporary_file(filename_output)
def test_migrate_force(self):
"""Test that passing the -f/--force option will overwrite the output file even if it exists."""
filename_input = get_archive_file(self.penultimate_archive,
filepath=self.fixture_archive)
# Using the context manager will create the file and so the command should fail
with tempfile.NamedTemporaryFile() as file_output:
options = [filename_input, file_output.name]
result = self.cli_runner.invoke(cmd_export.migrate, options)
self.assertIsNotNone(result.exception)
for option in ['-f', '--force']:
# Using the context manager will create the file, but we pass the force flag so it should work
with tempfile.NamedTemporaryFile() as file_output:
filename_output = file_output.name
options = [option, filename_input, filename_output]
result = self.cli_runner.invoke(cmd_export.migrate, options)
self.assertIsNone(result.exception, result.output)
self.assertTrue(os.path.isfile(filename_output))
self.assertEqual(
zipfile.ZipFile(filename_output).testzip(), None)
def test_inspect(self):
"""Test the functionality of `verdi export inspect`."""
archives = []
for version in range(1, int(EXPORT_VERSION.split('.')[-1])):
archives.append(('export_v0.{}_simple.aiida'.format(version),
'0.{}'.format(version)))
for archive, version_number in archives:
filename_input = get_archive_file(archive,
filepath=self.fixture_archive)
# Testing the options that will print the meta data and data respectively
for option in ['-m', '-d']:
options = [option, filename_input]
result = self.cli_runner.invoke(cmd_export.inspect, options)
self.assertIsNone(result.exception, result.output)
# Test the --version option which should print the archive format version
options = ['--version', filename_input]
result = self.cli_runner.invoke(cmd_export.inspect, options)
self.assertIsNone(result.exception, result.output)
self.assertEqual(result.output.strip(), version_number)
def test_import_folder(self):
"""Verify a pre-extracted archive (aka. a folder with the archive structure) can be imported.
It is important to check that the source directory or any of its contents are not deleted after import.
"""
from aiida.common.folders import SandboxFolder
from aiida.backends.tests.utils.archives import get_archive_file
from aiida.tools.importexport.common.archive import extract_zip
archive = get_archive_file('arithmetic.add.aiida', filepath='calcjob')
with SandboxFolder() as temp_dir:
extract_zip(archive, temp_dir, silent=True)
# Make sure the JSON files and the nodes subfolder was correctly extracted (is present),
# then try to import it by passing the extracted folder to the import function.
for name in {'metadata.json', 'data.json', 'nodes'}:
self.assertTrue(
os.path.exists(os.path.join(temp_dir.abspath, name)))
# Get list of all folders in extracted archive
org_folders = []
for dirpath, dirnames, _ in os.walk(temp_dir.abspath):
org_folders += [
os.path.join(dirpath, dirname) for dirname in dirnames
]
import_data(temp_dir.abspath, silent=True)
# Check nothing from the source was deleted
src_folders = []
for dirpath, dirnames, _ in os.walk(temp_dir.abspath):
src_folders += [
os.path.join(dirpath, dirname) for dirname in dirnames
]
self.maxDiff = None # pylint: disable=invalid-name
self.assertListEqual(org_folders, src_folders)
def test_migrate_v3_to_v4_complete(self):
"""Test migration for file containing complete v0.3 era possibilities"""
# Get metadata.json and data.json as dicts from v0.3 file archive
dirpath_archive = get_archive_file('export_v0.3.aiida', **self.external_archive)
# Migrate
with SandboxFolder(sandbox_in_repo=False) as folder:
if zipfile.is_zipfile(dirpath_archive):
extract_zip(dirpath_archive, folder, silent=True)
elif tarfile.is_tarfile(dirpath_archive):
extract_tar(dirpath_archive, folder, silent=True)
else:
raise ValueError('invalid file format, expected either a zip archive or gzipped tarball')
try:
with io.open(folder.get_abs_path('data.json'), 'r', encoding='utf8') as fhandle:
data = jsonload(fhandle)
with io.open(folder.get_abs_path('metadata.json'), 'r', encoding='utf8') as fhandle:
metadata = jsonload(fhandle)
except IOError:
raise NotExistent('export archive does not contain the required file {}'.format(fhandle.filename))
verify_metadata_version(metadata, version='0.3')
# Save pre-migration info
links_count_org = len(data['links_uuid'])
work_uuids = {
value['uuid']
for value in data['export_data']['Node'].values()
if value['type'].startswith('calculation.function') or value['type'].startswith('calculation.work')
}
illegal_links = []
for link in data['links_uuid']:
if link['input'] in work_uuids and link['type'] == 'createlink':
illegal_links.append(link)
# Migrate to v0.4
migrate_v3_to_v4(metadata, data, folder)
verify_metadata_version(metadata, version='0.4')
## Following checks are based on the archive-file
## Which means there are more legal entities, they are simply not relevant here.
self.maxDiff = None # pylint: disable=invalid-name
# Check schema-changes
new_node_attrs = {'node_type', 'process_type'}
for change in new_node_attrs:
# data.json
for node in data['export_data']['Node'].values():
self.assertIn(change, node, msg="'{}' not found for {}".format(change, node))
# metadata.json
self.assertIn(
change,
metadata['all_fields_info']['Node'],
msg="'{}' not found in metadata.json for Node".format(change)
)
# Check Node types
legal_node_types = {
'data.float.Float.', 'data.int.Int.', 'data.dict.Dict.', 'data.code.Code.', 'data.structure.StructureData.',
'data.folder.FolderData.', 'data.remote.RemoteData.', 'data.upf.UpfData.', 'data.array.ArrayData.',
'data.array.bands.BandsData.', 'data.array.kpoints.KpointsData.', 'data.array.trajectory.TrajectoryData.',
'process.workflow.workchain.WorkChainNode.', 'process.calculation.calcjob.CalcJobNode.'
}
legal_process_types = {'', 'aiida.calculations:quantumespresso.pw'}
for node in data['export_data']['Node'].values():
self.assertIn(
node['node_type'],
legal_node_types,
msg='{} is not a legal node_type. Legal node types: {}'.format(node['node_type'], legal_node_types)
)
self.assertIn(
node['process_type'],
legal_process_types,
msg='{} is not a legal process_type. Legal process types: {}'.format(
node['process_type'], legal_node_types
)
)
# Check links
# Make sure the two illegal create links were removed during the migration
self.assertEqual(
len(data['links_uuid']),
links_count_org - 2,
msg='Two of the org. {} links should have been removed during the migration, '
'instead there are now {} links'.format(links_count_org, len(data['links_uuid']))
)
legal_link_types = {'unspecified', 'create', 'return', 'input_calc', 'input_work', 'call_calc', 'call_work'}
for link in data['links_uuid']:
self.assertIn(link['type'], legal_link_types)
for link in illegal_links:
self.assertNotIn(link, data['links_uuid'], msg='{} should not be in the migrated export file'.format(link))
# Check Groups
# There is one Group in the export file, it is a user group
updated_attrs = {'label', 'type_string'}
legal_group_type = {'user'}
for attr in updated_attrs:
# data.json
for group in data['export_data']['Group'].values():
self.assertIn(attr, group, msg='{} not found in Group {}'.format(attr, group))
self.assertIn(
group['type_string'],
legal_group_type,
msg='{} is not a legal Group type_string'.format(group['type_string'])
)
# metadata.json
self.assertIn(attr, metadata['all_fields_info']['Group'], msg='{} not found in metadata.json'.format(attr))
# Check node_attributes*
calcjob_nodes = []
process_nodes = []
for node_id, content in data['export_data']['Node'].items():
if content['node_type'] == 'process.calculation.calcjob.CalcJobNode.':
calcjob_nodes.append(node_id)
elif content['node_type'].startswith('process.'):
process_nodes.append(node_id)
mandatory_updated_calcjob_attrs = {'resources', 'parser_name'}
optional_updated_calcjob_attrs = {'custom_environment_variables': 'environment_variables'}
updated_process_attrs = {'process_label'}
fields = {'node_attributes', 'node_attributes_conversion'}
for field in fields:
for node_id in calcjob_nodes:
for attr in mandatory_updated_calcjob_attrs:
self.assertIn(
attr,
data[field][node_id],
msg="Updated attribute name '{}' not found in {} for node_id: {}".format(attr, field, node_id)
)
for old, new in optional_updated_calcjob_attrs.items():
self.assertNotIn(
old,
data[field][node_id],
msg="Old attribute '{}' found in {} for node_id: {}. "
"It should now be updated to '{}' or not exist".format(old, field, node_id, new)
)
for node_id in process_nodes:
for attr in updated_process_attrs:
self.assertIn(
attr,
data[field][node_id],
msg="Updated attribute name '{}' not found in {} for node_id: {}".format(attr, field, node_id)
)
# Check TrajectoryData
# There should be minimum one TrajectoryData in the export file
trajectorydata_nodes = []
for node_id, content in data['export_data']['Node'].items():
if content['node_type'] == 'data.array.trajectory.TrajectoryData.':
trajectorydata_nodes.append(node_id)
updated_attrs = {'symbols'}
fields = {'node_attributes', 'node_attributes_conversion'}
for field in fields:
for node_id in trajectorydata_nodes:
for attr in updated_attrs:
self.assertIn(
attr,
data[field][node_id],
msg="Updated attribute name '{}' not found in {} for TrajecteoryData node_id: {}".format(
attr, field, node_id
)
)
# Check Computer
removed_attrs = {'enabled'}
for attr in removed_attrs:
# data.json
for computer in data['export_data']['Computer'].values():
self.assertNotIn(
attr, computer, msg="'{}' should have been removed from Computer {}".format(attr, computer['name'])
)
# metadata.json
self.assertNotIn(
attr,
metadata['all_fields_info']['Computer'],
msg="'{}' should have been removed from Computer in metadata.json".format(attr)
)
# Check new entities
new_entities = {'Log', 'Comment'}
fields = {'all_fields_info', 'unique_identifiers'}
for entity in new_entities:
for field in fields:
self.assertIn(entity, metadata[field], msg='{} not found in {} in metadata.json'.format(entity, field))
# Check extras
# Dicts with key, vales equal to node_id, {} should be present
# This means they should be same length as data['export_data']['Node'] or 'node_attributes*'
attrs_count = len(data['node_attributes'])
new_fields = {'node_extras', 'node_extras_conversion'}
for field in new_fields:
self.assertIn(field, list(data.keys()), msg="New field '{}' not found in data.json".format(field))
self.assertEqual(
len(data[field]),
attrs_count,
msg="New field '{}' found to have only {} entries, but should have had {} entries".format(
field, len(data[field]), attrs_count
)
)
def test_illegal_create_links(self):
"""Test illegal create links from workchain are detected and removed from exports using v0.3"""
# Initialization
dirpath_archive = get_archive_file('export_v0.3.aiida', **self.external_archive)
known_illegal_links = 2
# Unpack archive, check data.json, and migrate to v0.4
with SandboxFolder(sandbox_in_repo=False) as folder:
if zipfile.is_zipfile(dirpath_archive):
extract_zip(dirpath_archive, folder, silent=True)
elif tarfile.is_tarfile(dirpath_archive):
extract_tar(dirpath_archive, folder, silent=True)
else:
raise ValueError('invalid file format, expected either a zip archive or gzipped tarball')
try:
with io.open(folder.get_abs_path('data.json'), 'r', encoding='utf8') as fhandle:
data = jsonload(fhandle)
with io.open(folder.get_abs_path('metadata.json'), 'r', encoding='utf8') as fhandle:
metadata = jsonload(fhandle)
except IOError:
raise NotExistent('export archive does not contain the required file {}'.format(fhandle.filename))
# Check illegal create links are present in org. export file
links_count = len(data['links_uuid'])
links_count_migrated = links_count - known_illegal_links
workfunc_uuids = {
value['uuid']
for value in data['export_data']['Node'].values()
if value['type'].startswith('calculation.function') or value['type'].startswith('calculation.work')
}
violations = []
for link in data['links_uuid']:
if link['input'] in workfunc_uuids and link['type'] == 'createlink':
violations.append(link)
self.assertEqual(
len(violations),
known_illegal_links,
msg='{} illegal create links were expected, instead {} was/were found'.format(
known_illegal_links, len(violations)
)
)
# Migrate to v0.4
migrate_v3_to_v4(metadata, data, folder)
# Check illegal create links were removed
self.assertEqual(
len(data['links_uuid']),
links_count_migrated,
msg='{} links were expected, instead {} was/were found'.format(
links_count_migrated, len(data['links_uuid'])
)
)
workfunc_uuids = {
value['uuid']
for value in data['export_data']['Node'].values()
if value['node_type'].find('WorkFunctionNode') != -1 or value['node_type'].find('WorkChainNode') != -1
}
violations = []
for link in data['links_uuid']:
if link['input'] in workfunc_uuids and link['type'] == 'create':
violations.append(link)
self.assertEqual(
len(violations), 0, msg='0 illegal links were expected, instead {} was/were found'.format(len(violations))
)
def test_context_required(self):
"""Verify that accessing a property of an Archive outside of a context manager raises."""
with self.assertRaises(InvalidOperation):
filepath = get_archive_file('export_v0.1_simple.aiida', filepath='export/migrate')
archive = Archive(filepath)
archive.version_format # pylint: disable=pointless-statement
def test_compare_migration_with_aiida_made(self):
"""
Compare the migration of a Workflow made and exported with version 0.3 to version 0.4,
and the same Workflow made and exported with version 0.4.
(AiiDA versions 0.12.3 versus 1.0.0b2)
NB: Since PKs and UUIDs will have changed, comparisons between 'data.json'-files will be made indirectly
"""
# Get metadata.json and data.json as dicts from v0.3 file archive and migrate
dirpath_archive = get_archive_file('export_v0.3.aiida', **self.external_archive)
# Migrate
with SandboxFolder(sandbox_in_repo=False) as folder:
if zipfile.is_zipfile(dirpath_archive):
extract_zip(dirpath_archive, folder, silent=True)
elif tarfile.is_tarfile(dirpath_archive):
extract_tar(dirpath_archive, folder, silent=True)
else:
raise ValueError('invalid file format, expected either a zip archive or gzipped tarball')
try:
with io.open(folder.get_abs_path('data.json'), 'r', encoding='utf8') as fhandle:
data_v3 = jsonload(fhandle)
with io.open(folder.get_abs_path('metadata.json'), 'r', encoding='utf8') as fhandle:
metadata_v3 = jsonload(fhandle)
except IOError:
raise NotExistent('export archive does not contain the required file {}'.format(fhandle.filename))
# Migrate to v0.4
migrate_v3_to_v4(metadata_v3, data_v3, folder)
# Get metadata.json and data.json as dicts from v0.4 file archive
metadata_v4, data_v4 = get_json_files('export_v0.4.aiida', **self.external_archive)
# Compare 'metadata.json'
self.maxDiff = None
metadata_v3.pop('conversion_info')
metadata_v3.pop('aiida_version')
metadata_v4.pop('aiida_version')
self.assertDictEqual(metadata_v3, metadata_v4)
self.maxDiff = None
# Compare 'data.json'
self.assertEqual(len(data_v3), len(data_v4))
entities = {
'Node': {
'migrated': [],
'made': []
},
'Computer': {
'migrated': [],
'made': []
},
'Group': {
'migrated': [],
'made': []
}
} # User is special, see below
for entity, details in entities.items():
for node in data_v3['export_data'][entity].values():
add = node.get('node_type', None) # Node
if not add:
add = node.get('hostname', None) # Computer
if not add:
add = node.get('type_string', None) # Group
self.assertIsNotNone(add, msg="Helper variable 'add' should never be None")
details['migrated'].append(add)
for node in data_v4['export_data'][entity].values():
add = node.get('node_type', None) # Node
if not add:
add = node.get('hostname', None) # Computer
if not add:
add = node.get('type_string', None) # Group
self.assertIsNotNone(add, msg="Helper variable 'add' should never be None")
details['made'].append(add)
#### Two extra Dicts are present for AiiDA made export 0.4 file ####
if entity == 'Node':
details['migrated'].extend(2 * ['data.dict.Dict.'])
self.assertListEqual(
sorted(details['migrated']),
sorted(details['made']),
msg='Number of {}-entities differ, see diff for details'.format(entity)
)
fields = {
'groups_uuid', 'node_attributes_conversion', 'node_attributes', 'node_extras', 'node_extras_conversion'
} # 'export_data' is special, see below
for field in fields:
if field != 'groups_uuid':
correction = 2 # Two extra Dicts in AiiDA made export v0.4 file
else:
correction = 0
self.assertEqual(
len(data_v3[field]),
len(data_v4[field]) - correction,
msg='Number of entities in {} differs for the export files'.format(field)
)
number_of_links_v3 = {
'unspecified': 0,
'create': 0,
'return': 0,
'input_calc': 0,
'input_work': 0,
'call_calc': 0,
'call_work': 0
}
for link in data_v3['links_uuid']:
number_of_links_v3[link['type']] += 1
number_of_links_v4 = {
'unspecified': 0,
'create': 0,
'return': 0,
'input_calc': -2, # Two extra Dict inputs to CalcJobNodes
'input_work': 0,
'call_calc': 0,
'call_work': 0
}
for link in data_v4['links_uuid']:
number_of_links_v4[link['type']] += 1
self.assertDictEqual(
number_of_links_v3,
number_of_links_v4,
msg='There are a different number of specific links in the migrated export file than the AiiDA made one.'
)
self.assertEqual(number_of_links_v3['unspecified'], 0)
self.assertEqual(number_of_links_v4['unspecified'], 0)
# Special for data['export_data']['User']
# There is an extra user in the migrated export v0.3 file
self.assertEqual(len(data_v3['export_data']['User']), len(data_v4['export_data']['User']) + 1)
# Special for data['export_data']
# There are Logs exported in the AiiDA made export v0.4 file
self.assertEqual(len(data_v3['export_data']) + 1, len(data_v4['export_data']))
def test_version_format(self):
"""Verify that `version_format` return the correct archive format version."""
filepath = get_archive_file('export_v0.1_simple.aiida', filepath='export/migrate')
with Archive(filepath) as archive:
self.assertEqual(archive.version_format, '0.1')
def test_empty_archive(self):
"""Verify that attempting to unpack an empty archive raises a `CorruptArchive` exception."""
filepath = get_archive_file('empty.aiida', filepath='export/migrate')
with self.assertRaises(CorruptArchive):
with Archive(filepath) as archive:
archive.version_format # pylint: disable=pointless-statement
