def test_import_to_group(self): """ Test import to existing Group and that Nodes are added correctly for multiple imports of the same, as well as separate, archives. """ archives = [ get_archive_file('arithmetic.add.aiida', filepath='calcjob'), get_archive_file(self.newest_archive, filepath=self.archive_path) ] group_label = 'import_madness' group = Group(group_label).store() self.assertTrue(group.is_empty, msg='The Group should be empty.') # Invoke `verdi import`, making sure there are no exceptions options = ['-G', group.label] + [archives[0]] result = self.cli_runner.invoke(cmd_import.cmd_import, options) self.assertIsNone(result.exception, msg=result.output) self.assertEqual(result.exit_code, 0, msg=result.output) self.assertFalse(group.is_empty, msg='The Group should no longer be empty.') nodes_in_group = group.count() # Invoke `verdi import` again, making sure Group count doesn't change options = ['-G', group.label] + [archives[0]] result = self.cli_runner.invoke(cmd_import.cmd_import, options) self.assertIsNone(result.exception, msg=result.output) self.assertEqual(result.exit_code, 0, msg=result.output) self.assertEqual( group.count(), nodes_in_group, msg= 'The Group count should not have changed from {}. Instead it is now {}' .format(nodes_in_group, group.count())) # Invoke `verdi import` again with new archive, making sure Group count is upped options = ['-G', group.label] + [archives[1]] result = self.cli_runner.invoke(cmd_import.cmd_import, options) self.assertIsNone(result.exception, msg=result.output) self.assertEqual(result.exit_code, 0, msg=result.output) self.assertGreater( group.count(), nodes_in_group, msg= 'There should now be more than {} nodes in group {} , instead there are {}' .format(nodes_in_group, group_label, group.count()))
def test_import_url_and_local_archives(self): """Test import of both a remote and local archive""" url_archive = 'export_v0.4_no_UPF.aiida' local_archive = self.newest_archive options = [ get_archive_file(local_archive, filepath=self.archive_path), self.url_path + url_archive, get_archive_file(local_archive, filepath=self.archive_path) ] result = self.cli_runner.invoke(cmd_import.cmd_import, options) self.assertIsNone(result.exception, result.output) self.assertEqual(result.exit_code, 0, result.output)
def test_import_archive(self): """ Test import for archive files from disk """ archives = [ get_archive_file('arithmetic.add.aiida', filepath='calcjob'), get_archive_file(self.newest_archive, filepath=self.archive_path) ] options = [] + archives result = self.cli_runner.invoke(cmd_import.cmd_import, options) self.assertIsNone(result.exception, result.output) self.assertEqual(result.exit_code, 0, result.output)
def test_import_make_new_group(self): """Make sure imported entities are saved in new Group""" # Initialization group_label = 'new_group_for_verdi_import' archives = [ get_archive_file(self.newest_archive, filepath=self.archive_path) ] # Check Group does not already exist group_search = Group.objects.find(filters={'label': group_label}) self.assertEqual( len(group_search), 0, msg="A Group with label '{}' already exists, this shouldn't be.". format(group_label)) # Invoke `verdi import`, making sure there are no exceptions options = ['-G', group_label] + archives result = self.cli_runner.invoke(cmd_import.cmd_import, options) self.assertIsNone(result.exception, msg=result.output) self.assertEqual(result.exit_code, 0, msg=result.output) # Make sure new Group was created (group, new_group) = Group.objects.get_or_create(group_label) self.assertFalse( new_group, msg= 'The Group should not have been created now, but instead when it was imported.' ) self.assertFalse(group.is_empty, msg='The Group should not be empty.')
def test_no_node_export(self, temp_dir): """Test migration of export file that has no Nodes""" input_file = get_archive_file('export_v0.3_no_Nodes.aiida', **self.external_archive) output_file = os.path.join(temp_dir, 'output_file.aiida') # Known entities computer_uuids = [self.computer.uuid] # pylint: disable=no-member user_emails = [orm.User.objects.get_default().email] # Known export file content used for checks node_count = 0 computer_count = 1 + 1 # localhost is always present computer_uuids.append('4f33c6fd-b624-47df-9ffb-a58f05d323af') user_emails.append('aiida@localhost') # Perform the migration migrate_archive(input_file, output_file) # Load the migrated file import_data(output_file, silent=True) # Check known number of entities is present self.assertEqual(orm.QueryBuilder().append(orm.Node).count(), node_count) self.assertEqual(orm.QueryBuilder().append(orm.Computer).count(), computer_count) # Check unique identifiers computers = orm.QueryBuilder().append(orm.Computer, project=['uuid']).all()[0][0] users = orm.QueryBuilder().append(orm.User, project=['email']).all()[0][0] self.assertIn(computers, computer_uuids) self.assertIn(users, user_emails)
def test_migrate_v3_to_v4(self): """Test function migrate_v3_to_v4""" from aiida import get_version # Get metadata.json and data.json as dicts from v0.4 file archive metadata_v4, data_v4 = get_json_files('export_v0.4_simple.aiida', **self.core_archive) verify_metadata_version(metadata_v4, version='0.4') # Get metadata.json and data.json as dicts from v0.3 file archive # Cannot use 'get_json_files' for 'export_v0.3_simple.aiida', # because we need to pass the SandboxFolder to 'migrate_v3_to_v4' dirpath_archive = get_archive_file('export_v0.3_simple.aiida', **self.core_archive) with SandboxFolder(sandbox_in_repo=False) as folder: if zipfile.is_zipfile(dirpath_archive): extract_zip(dirpath_archive, folder, silent=True) elif tarfile.is_tarfile(dirpath_archive): extract_tar(dirpath_archive, folder, silent=True) else: raise ValueError('invalid file format, expected either a zip archive or gzipped tarball') try: with io.open(folder.get_abs_path('data.json'), 'r', encoding='utf8') as fhandle: data_v3 = jsonload(fhandle) with io.open(folder.get_abs_path('metadata.json'), 'r', encoding='utf8') as fhandle: metadata_v3 = jsonload(fhandle) except IOError: raise NotExistent('export archive does not contain the required file {}'.format(fhandle.filename)) verify_metadata_version(metadata_v3, version='0.3') # Migrate to v0.4 migrate_v3_to_v4(metadata_v3, data_v3, folder) verify_metadata_version(metadata_v3, version='0.4') # Remove AiiDA version, since this may change irregardless of the migration function metadata_v3.pop('aiida_version') metadata_v4.pop('aiida_version') # Assert conversion message in `metadata.json` is correct and then remove it for later assertions self.maxDiff = None # pylint: disable=invalid-name conversion_message = 'Converted from version 0.3 to 0.4 with AiiDA v{}'.format(get_version()) self.assertEqual( metadata_v3.pop('conversion_info')[-1], conversion_message, msg='The conversion message after migration is wrong' ) metadata_v4.pop('conversion_info') # Assert changes were performed correctly self.assertDictEqual( metadata_v3, metadata_v4, msg='After migration, metadata.json should equal intended metadata.json from archives' ) self.assertDictEqual( data_v3, data_v4, msg='After migration, data.json should equal intended data.json from archives' )
def test_v02_to_newest(self, temp_dir): """Test migration of exported files from v0.2 to newest export version""" # Get export file with export version 0.2 input_file = get_archive_file('export_v0.2.aiida', **self.external_archive) output_file = os.path.join(temp_dir, 'output_file.aiida') # Perform the migration migrate_archive(input_file, output_file) metadata, _ = get_json_files(output_file) verify_metadata_version(metadata, version=newest_version) # Load the migrated file import_data(output_file, silent=True) # Do the necessary checks self.assertEqual(orm.QueryBuilder().append(orm.Node).count(), self.node_count) # Verify that CalculationNodes have non-empty attribute dictionaries builder = orm.QueryBuilder().append(orm.CalculationNode) for [calculation] in builder.iterall(): self.assertIsInstance(calculation.attributes, dict) self.assertNotEqual(len(calculation.attributes), 0) # Verify that the StructureData nodes maintained their (same) label, cell, and kinds builder = orm.QueryBuilder().append(orm.StructureData) self.assertEqual( builder.count(), self.struct_count, msg='There should be {} StructureData, instead {} were/was found'. format(self.struct_count, builder.count())) for structures in builder.all(): structure = structures[0] self.assertEqual(structure.label, self.known_struct_label) self.assertEqual(structure.cell, self.known_cell) builder = orm.QueryBuilder().append(orm.StructureData, project=['attributes.kinds']) for [kinds] in builder.iterall(): self.assertEqual(len(kinds), len(self.known_kinds)) for kind in kinds: self.assertIn(kind, self.known_kinds, msg="Kind '{}' not found in: {}".format( kind, self.known_kinds)) # Check that there is a StructureData that is an input of a CalculationNode builder = orm.QueryBuilder() builder.append(orm.StructureData, tag='structure') builder.append(orm.CalculationNode, with_incoming='structure') self.assertGreater(len(builder.all()), 0) # Check that there is a RemoteData that is the output of a CalculationNode builder = orm.QueryBuilder() builder.append(orm.CalculationNode, tag='parent') builder.append(orm.RemoteData, with_incoming='parent') self.assertGreater(len(builder.all()), 0)
def test_inspect_empty_archive(self): """Test the functionality of `verdi export inspect` for an empty archive.""" filename_input = get_archive_file('empty.aiida', filepath=self.fixture_archive) options = [filename_input] result = self.cli_runner.invoke(cmd_export.inspect, options) self.assertIsNotNone(result.exception, result.output) self.assertIn('corrupt archive', result.output)
def test_non_interactive_and_migration(self): """Test options `--non-interactive` and `--migration`/`--no-migration` `migration` = True (default), `non_interactive` = False (default), Expected: Query user, migrate `migration` = True (default), `non_interactive` = True, Expected: No query, migrate `migration` = False, `non_interactive` = False (default), Expected: No query, no migrate `migration` = False, `non_interactive` = True, Expected: No query, no migrate """ archive = get_archive_file('export_v0.1_simple.aiida', filepath=self.archive_path) confirm_message = 'Do you want to try and migrate {} to the newest export file version?'.format( archive) success_message = 'Success: imported archive {}'.format(archive) # Import "normally", but explicitly specifying `--migration`, make sure confirm message is present # `migration` = True (default), `non_interactive` = False (default), Expected: Query user, migrate options = ['--migration', archive] result = self.cli_runner.invoke(cmd_import.cmd_import, options) self.assertIsNone(result.exception, msg=result.output) self.assertEqual(result.exit_code, 0, msg=result.output) self.assertIn(confirm_message, result.output, msg=result.exception) self.assertIn(success_message, result.output, msg=result.exception) # Import using non-interactive, make sure confirm message has gone # `migration` = True (default), `non_interactive` = True, Expected: No query, migrate options = ['--non-interactive', archive] result = self.cli_runner.invoke(cmd_import.cmd_import, options) self.assertIsNone(result.exception, msg=result.output) self.assertEqual(result.exit_code, 0, msg=result.output) self.assertNotIn(confirm_message, result.output, msg=result.exception) self.assertIn(success_message, result.output, msg=result.exception) # Import using `--no-migration`, make sure confirm message has gone # `migration` = False, `non_interactive` = False (default), Expected: No query, no migrate options = ['--no-migration', archive] result = self.cli_runner.invoke(cmd_import.cmd_import, options) self.assertIsNotNone(result.exception, msg=result.output) self.assertNotEqual(result.exit_code, 0, msg=result.output) self.assertNotIn(confirm_message, result.output, msg=result.exception) self.assertNotIn(success_message, result.output, msg=result.exception) # Import using `--no-migration` and `--non-interactive`, make sure confirm message has gone # `migration` = False, `non_interactive` = True, Expected: No query, no migrate options = ['--no-migration', '--non-interactive', archive] result = self.cli_runner.invoke(cmd_import.cmd_import, options) self.assertIsNotNone(result.exception, msg=result.output) self.assertNotEqual(result.exit_code, 0, msg=result.output) self.assertNotIn(confirm_message, result.output, msg=result.exception) self.assertNotIn(success_message, result.output, msg=result.exception)
def test_migrate_versions_recent(self): """Migrating an archive with the current version should exit with non-zero status.""" filename_input = get_archive_file(self.newest_archive, filepath=self.fixture_archive) filename_output = next(tempfile._get_candidate_names()) # pylint: disable=protected-access try: options = [filename_input, filename_output] result = self.cli_runner.invoke(cmd_export.migrate, options) self.assertIsNotNone(result.exception) finally: delete_temporary_file(filename_output)
def test_comment_mode(self): """Test toggling comment mode flag""" archives = [ get_archive_file(self.newest_archive, filepath=self.archive_path) ] for mode in {'newest', 'overwrite'}: options = ['--comment-mode', mode] + archives result = self.cli_runner.invoke(cmd_import.cmd_import, options) self.assertIsNone(result.exception, result.output) self.assertIn('Comment mode: {}'.format(mode), result.output) self.assertEqual(result.exit_code, 0, result.output)
def test_migrate_tar_gz(self): """Test that -F/--archive-format option can be used to write a tar.gz instead.""" filename_input = get_archive_file(self.penultimate_archive, filepath=self.fixture_archive) filename_output = next(tempfile._get_candidate_names()) # pylint: disable=protected-access for option in ['-F', '--archive-format']: try: options = [option, 'tar.gz', filename_input, filename_output] result = self.cli_runner.invoke(cmd_export.migrate, options) self.assertIsNone(result.exception, result.output) self.assertTrue(os.path.isfile(filename_output)) self.assertTrue(tarfile.is_tarfile(filename_output)) finally: delete_temporary_file(filename_output)
def test_migrate_recursively(self): """Test function 'migrate_recursively'""" import io import tarfile import zipfile from aiida.common.exceptions import NotExistent from aiida.common.folders import SandboxFolder from aiida.common.json import load as jsonload from aiida.tools.importexport.common.archive import extract_tar, extract_zip # Get metadata.json and data.json as dicts from v0.1 file archive # Cannot use 'get_json_files' for 'export_v0.1_simple.aiida', # because we need to pass the SandboxFolder to 'migrate_recursively' dirpath_archive = get_archive_file('export_v0.1_simple.aiida', **self.core_archive) with SandboxFolder(sandbox_in_repo=False) as folder: if zipfile.is_zipfile(dirpath_archive): extract_zip(dirpath_archive, folder, silent=True) elif tarfile.is_tarfile(dirpath_archive): extract_tar(dirpath_archive, folder, silent=True) else: raise ValueError( 'invalid file format, expected either a zip archive or gzipped tarball' ) try: with io.open(folder.get_abs_path('data.json'), 'r', encoding='utf8') as fhandle: data = jsonload(fhandle) with io.open(folder.get_abs_path('metadata.json'), 'r', encoding='utf8') as fhandle: metadata = jsonload(fhandle) except IOError: raise NotExistent( 'export archive does not contain the required file {}'. format(fhandle.filename)) verify_metadata_version(metadata, version='0.1') # Migrate to newest version new_version = migrate_recursively(metadata, data, folder) verify_metadata_version(metadata, version=newest_version) self.assertEqual(new_version, newest_version)
def test_migrate_silent(self): """Test that the captured output is an empty string when the -s/--silent option is passed.""" filename_input = get_archive_file(self.penultimate_archive, filepath=self.fixture_archive) filename_output = next(tempfile._get_candidate_names()) # pylint: disable=protected-access for option in ['-s', '--silent']: try: options = [option, filename_input, filename_output] result = self.cli_runner.invoke(cmd_export.migrate, options) self.assertEqual(result.output, '') self.assertIsNone(result.exception, result.output) self.assertTrue(os.path.isfile(filename_output)) self.assertEqual( zipfile.ZipFile(filename_output).testzip(), None) finally: delete_temporary_file(filename_output)
def test_import_old_local_archives(self): """ Test import of old local archives Expected behavior: Automatically migrate to newest version and import correctly. """ archives = [] for version in range(1, int(EXPORT_VERSION.split('.')[-1]) - 1): archives.append(('export_v0.{}_simple.aiida'.format(version), '0.{}'.format(version))) for archive, version in archives: options = [get_archive_file(archive, filepath=self.archive_path)] result = self.cli_runner.invoke(cmd_import.cmd_import, options) self.assertIsNone(result.exception, msg=result.output) self.assertEqual(result.exit_code, 0, msg=result.output) self.assertIn(version, result.output, msg=result.exception) self.assertIn('Success: imported archive {}'.format(options[0]), result.output, msg=result.exception)
def test_migrate_versions_old(self): """Migrating archives with a version older than the current should work.""" archives = [] for version in range(1, int(EXPORT_VERSION.split('.')[-1]) - 1): archives.append('export_v0.{}_simple.aiida'.format(version)) for archive in archives: filename_input = get_archive_file(archive, filepath=self.fixture_archive) filename_output = next(tempfile._get_candidate_names()) # pylint: disable=protected-access try: options = [filename_input, filename_output] result = self.cli_runner.invoke(cmd_export.migrate, options) self.assertIsNone(result.exception, result.output) self.assertTrue(os.path.isfile(filename_output)) self.assertEqual( zipfile.ZipFile(filename_output).testzip(), None) finally: delete_temporary_file(filename_output)
def test_migrate_force(self): """Test that passing the -f/--force option will overwrite the output file even if it exists.""" filename_input = get_archive_file(self.penultimate_archive, filepath=self.fixture_archive) # Using the context manager will create the file and so the command should fail with tempfile.NamedTemporaryFile() as file_output: options = [filename_input, file_output.name] result = self.cli_runner.invoke(cmd_export.migrate, options) self.assertIsNotNone(result.exception) for option in ['-f', '--force']: # Using the context manager will create the file, but we pass the force flag so it should work with tempfile.NamedTemporaryFile() as file_output: filename_output = file_output.name options = [option, filename_input, filename_output] result = self.cli_runner.invoke(cmd_export.migrate, options) self.assertIsNone(result.exception, result.output) self.assertTrue(os.path.isfile(filename_output)) self.assertEqual( zipfile.ZipFile(filename_output).testzip(), None)
def test_inspect(self): """Test the functionality of `verdi export inspect`.""" archives = [] for version in range(1, int(EXPORT_VERSION.split('.')[-1])): archives.append(('export_v0.{}_simple.aiida'.format(version), '0.{}'.format(version))) for archive, version_number in archives: filename_input = get_archive_file(archive, filepath=self.fixture_archive) # Testing the options that will print the meta data and data respectively for option in ['-m', '-d']: options = [option, filename_input] result = self.cli_runner.invoke(cmd_export.inspect, options) self.assertIsNone(result.exception, result.output) # Test the --version option which should print the archive format version options = ['--version', filename_input] result = self.cli_runner.invoke(cmd_export.inspect, options) self.assertIsNone(result.exception, result.output) self.assertEqual(result.output.strip(), version_number)
def test_import_folder(self): """Verify a pre-extracted archive (aka. a folder with the archive structure) can be imported. It is important to check that the source directory or any of its contents are not deleted after import. """ from aiida.common.folders import SandboxFolder from aiida.backends.tests.utils.archives import get_archive_file from aiida.tools.importexport.common.archive import extract_zip archive = get_archive_file('arithmetic.add.aiida', filepath='calcjob') with SandboxFolder() as temp_dir: extract_zip(archive, temp_dir, silent=True) # Make sure the JSON files and the nodes subfolder was correctly extracted (is present), # then try to import it by passing the extracted folder to the import function. for name in {'metadata.json', 'data.json', 'nodes'}: self.assertTrue( os.path.exists(os.path.join(temp_dir.abspath, name))) # Get list of all folders in extracted archive org_folders = [] for dirpath, dirnames, _ in os.walk(temp_dir.abspath): org_folders += [ os.path.join(dirpath, dirname) for dirname in dirnames ] import_data(temp_dir.abspath, silent=True) # Check nothing from the source was deleted src_folders = [] for dirpath, dirnames, _ in os.walk(temp_dir.abspath): src_folders += [ os.path.join(dirpath, dirname) for dirname in dirnames ] self.maxDiff = None # pylint: disable=invalid-name self.assertListEqual(org_folders, src_folders)
def test_migrate_v3_to_v4_complete(self): """Test migration for file containing complete v0.3 era possibilities""" # Get metadata.json and data.json as dicts from v0.3 file archive dirpath_archive = get_archive_file('export_v0.3.aiida', **self.external_archive) # Migrate with SandboxFolder(sandbox_in_repo=False) as folder: if zipfile.is_zipfile(dirpath_archive): extract_zip(dirpath_archive, folder, silent=True) elif tarfile.is_tarfile(dirpath_archive): extract_tar(dirpath_archive, folder, silent=True) else: raise ValueError('invalid file format, expected either a zip archive or gzipped tarball') try: with io.open(folder.get_abs_path('data.json'), 'r', encoding='utf8') as fhandle: data = jsonload(fhandle) with io.open(folder.get_abs_path('metadata.json'), 'r', encoding='utf8') as fhandle: metadata = jsonload(fhandle) except IOError: raise NotExistent('export archive does not contain the required file {}'.format(fhandle.filename)) verify_metadata_version(metadata, version='0.3') # Save pre-migration info links_count_org = len(data['links_uuid']) work_uuids = { value['uuid'] for value in data['export_data']['Node'].values() if value['type'].startswith('calculation.function') or value['type'].startswith('calculation.work') } illegal_links = [] for link in data['links_uuid']: if link['input'] in work_uuids and link['type'] == 'createlink': illegal_links.append(link) # Migrate to v0.4 migrate_v3_to_v4(metadata, data, folder) verify_metadata_version(metadata, version='0.4') ## Following checks are based on the archive-file ## Which means there are more legal entities, they are simply not relevant here. self.maxDiff = None # pylint: disable=invalid-name # Check schema-changes new_node_attrs = {'node_type', 'process_type'} for change in new_node_attrs: # data.json for node in data['export_data']['Node'].values(): self.assertIn(change, node, msg="'{}' not found for {}".format(change, node)) # metadata.json self.assertIn( change, metadata['all_fields_info']['Node'], msg="'{}' not found in metadata.json for Node".format(change) ) # Check Node types legal_node_types = { 'data.float.Float.', 'data.int.Int.', 'data.dict.Dict.', 'data.code.Code.', 'data.structure.StructureData.', 'data.folder.FolderData.', 'data.remote.RemoteData.', 'data.upf.UpfData.', 'data.array.ArrayData.', 'data.array.bands.BandsData.', 'data.array.kpoints.KpointsData.', 'data.array.trajectory.TrajectoryData.', 'process.workflow.workchain.WorkChainNode.', 'process.calculation.calcjob.CalcJobNode.' } legal_process_types = {'', 'aiida.calculations:quantumespresso.pw'} for node in data['export_data']['Node'].values(): self.assertIn( node['node_type'], legal_node_types, msg='{} is not a legal node_type. Legal node types: {}'.format(node['node_type'], legal_node_types) ) self.assertIn( node['process_type'], legal_process_types, msg='{} is not a legal process_type. Legal process types: {}'.format( node['process_type'], legal_node_types ) ) # Check links # Make sure the two illegal create links were removed during the migration self.assertEqual( len(data['links_uuid']), links_count_org - 2, msg='Two of the org. {} links should have been removed during the migration, ' 'instead there are now {} links'.format(links_count_org, len(data['links_uuid'])) ) legal_link_types = {'unspecified', 'create', 'return', 'input_calc', 'input_work', 'call_calc', 'call_work'} for link in data['links_uuid']: self.assertIn(link['type'], legal_link_types) for link in illegal_links: self.assertNotIn(link, data['links_uuid'], msg='{} should not be in the migrated export file'.format(link)) # Check Groups # There is one Group in the export file, it is a user group updated_attrs = {'label', 'type_string'} legal_group_type = {'user'} for attr in updated_attrs: # data.json for group in data['export_data']['Group'].values(): self.assertIn(attr, group, msg='{} not found in Group {}'.format(attr, group)) self.assertIn( group['type_string'], legal_group_type, msg='{} is not a legal Group type_string'.format(group['type_string']) ) # metadata.json self.assertIn(attr, metadata['all_fields_info']['Group'], msg='{} not found in metadata.json'.format(attr)) # Check node_attributes* calcjob_nodes = [] process_nodes = [] for node_id, content in data['export_data']['Node'].items(): if content['node_type'] == 'process.calculation.calcjob.CalcJobNode.': calcjob_nodes.append(node_id) elif content['node_type'].startswith('process.'): process_nodes.append(node_id) mandatory_updated_calcjob_attrs = {'resources', 'parser_name'} optional_updated_calcjob_attrs = {'custom_environment_variables': 'environment_variables'} updated_process_attrs = {'process_label'} fields = {'node_attributes', 'node_attributes_conversion'} for field in fields: for node_id in calcjob_nodes: for attr in mandatory_updated_calcjob_attrs: self.assertIn( attr, data[field][node_id], msg="Updated attribute name '{}' not found in {} for node_id: {}".format(attr, field, node_id) ) for old, new in optional_updated_calcjob_attrs.items(): self.assertNotIn( old, data[field][node_id], msg="Old attribute '{}' found in {} for node_id: {}. " "It should now be updated to '{}' or not exist".format(old, field, node_id, new) ) for node_id in process_nodes: for attr in updated_process_attrs: self.assertIn( attr, data[field][node_id], msg="Updated attribute name '{}' not found in {} for node_id: {}".format(attr, field, node_id) ) # Check TrajectoryData # There should be minimum one TrajectoryData in the export file trajectorydata_nodes = [] for node_id, content in data['export_data']['Node'].items(): if content['node_type'] == 'data.array.trajectory.TrajectoryData.': trajectorydata_nodes.append(node_id) updated_attrs = {'symbols'} fields = {'node_attributes', 'node_attributes_conversion'} for field in fields: for node_id in trajectorydata_nodes: for attr in updated_attrs: self.assertIn( attr, data[field][node_id], msg="Updated attribute name '{}' not found in {} for TrajecteoryData node_id: {}".format( attr, field, node_id ) ) # Check Computer removed_attrs = {'enabled'} for attr in removed_attrs: # data.json for computer in data['export_data']['Computer'].values(): self.assertNotIn( attr, computer, msg="'{}' should have been removed from Computer {}".format(attr, computer['name']) ) # metadata.json self.assertNotIn( attr, metadata['all_fields_info']['Computer'], msg="'{}' should have been removed from Computer in metadata.json".format(attr) ) # Check new entities new_entities = {'Log', 'Comment'} fields = {'all_fields_info', 'unique_identifiers'} for entity in new_entities: for field in fields: self.assertIn(entity, metadata[field], msg='{} not found in {} in metadata.json'.format(entity, field)) # Check extras # Dicts with key, vales equal to node_id, {} should be present # This means they should be same length as data['export_data']['Node'] or 'node_attributes*' attrs_count = len(data['node_attributes']) new_fields = {'node_extras', 'node_extras_conversion'} for field in new_fields: self.assertIn(field, list(data.keys()), msg="New field '{}' not found in data.json".format(field)) self.assertEqual( len(data[field]), attrs_count, msg="New field '{}' found to have only {} entries, but should have had {} entries".format( field, len(data[field]), attrs_count ) )
def test_illegal_create_links(self): """Test illegal create links from workchain are detected and removed from exports using v0.3""" # Initialization dirpath_archive = get_archive_file('export_v0.3.aiida', **self.external_archive) known_illegal_links = 2 # Unpack archive, check data.json, and migrate to v0.4 with SandboxFolder(sandbox_in_repo=False) as folder: if zipfile.is_zipfile(dirpath_archive): extract_zip(dirpath_archive, folder, silent=True) elif tarfile.is_tarfile(dirpath_archive): extract_tar(dirpath_archive, folder, silent=True) else: raise ValueError('invalid file format, expected either a zip archive or gzipped tarball') try: with io.open(folder.get_abs_path('data.json'), 'r', encoding='utf8') as fhandle: data = jsonload(fhandle) with io.open(folder.get_abs_path('metadata.json'), 'r', encoding='utf8') as fhandle: metadata = jsonload(fhandle) except IOError: raise NotExistent('export archive does not contain the required file {}'.format(fhandle.filename)) # Check illegal create links are present in org. export file links_count = len(data['links_uuid']) links_count_migrated = links_count - known_illegal_links workfunc_uuids = { value['uuid'] for value in data['export_data']['Node'].values() if value['type'].startswith('calculation.function') or value['type'].startswith('calculation.work') } violations = [] for link in data['links_uuid']: if link['input'] in workfunc_uuids and link['type'] == 'createlink': violations.append(link) self.assertEqual( len(violations), known_illegal_links, msg='{} illegal create links were expected, instead {} was/were found'.format( known_illegal_links, len(violations) ) ) # Migrate to v0.4 migrate_v3_to_v4(metadata, data, folder) # Check illegal create links were removed self.assertEqual( len(data['links_uuid']), links_count_migrated, msg='{} links were expected, instead {} was/were found'.format( links_count_migrated, len(data['links_uuid']) ) ) workfunc_uuids = { value['uuid'] for value in data['export_data']['Node'].values() if value['node_type'].find('WorkFunctionNode') != -1 or value['node_type'].find('WorkChainNode') != -1 } violations = [] for link in data['links_uuid']: if link['input'] in workfunc_uuids and link['type'] == 'create': violations.append(link) self.assertEqual( len(violations), 0, msg='0 illegal links were expected, instead {} was/were found'.format(len(violations)) )
def test_context_required(self): """Verify that accessing a property of an Archive outside of a context manager raises.""" with self.assertRaises(InvalidOperation): filepath = get_archive_file('export_v0.1_simple.aiida', filepath='export/migrate') archive = Archive(filepath) archive.version_format # pylint: disable=pointless-statement
def test_compare_migration_with_aiida_made(self): """ Compare the migration of a Workflow made and exported with version 0.3 to version 0.4, and the same Workflow made and exported with version 0.4. (AiiDA versions 0.12.3 versus 1.0.0b2) NB: Since PKs and UUIDs will have changed, comparisons between 'data.json'-files will be made indirectly """ # Get metadata.json and data.json as dicts from v0.3 file archive and migrate dirpath_archive = get_archive_file('export_v0.3.aiida', **self.external_archive) # Migrate with SandboxFolder(sandbox_in_repo=False) as folder: if zipfile.is_zipfile(dirpath_archive): extract_zip(dirpath_archive, folder, silent=True) elif tarfile.is_tarfile(dirpath_archive): extract_tar(dirpath_archive, folder, silent=True) else: raise ValueError('invalid file format, expected either a zip archive or gzipped tarball') try: with io.open(folder.get_abs_path('data.json'), 'r', encoding='utf8') as fhandle: data_v3 = jsonload(fhandle) with io.open(folder.get_abs_path('metadata.json'), 'r', encoding='utf8') as fhandle: metadata_v3 = jsonload(fhandle) except IOError: raise NotExistent('export archive does not contain the required file {}'.format(fhandle.filename)) # Migrate to v0.4 migrate_v3_to_v4(metadata_v3, data_v3, folder) # Get metadata.json and data.json as dicts from v0.4 file archive metadata_v4, data_v4 = get_json_files('export_v0.4.aiida', **self.external_archive) # Compare 'metadata.json' self.maxDiff = None metadata_v3.pop('conversion_info') metadata_v3.pop('aiida_version') metadata_v4.pop('aiida_version') self.assertDictEqual(metadata_v3, metadata_v4) self.maxDiff = None # Compare 'data.json' self.assertEqual(len(data_v3), len(data_v4)) entities = { 'Node': { 'migrated': [], 'made': [] }, 'Computer': { 'migrated': [], 'made': [] }, 'Group': { 'migrated': [], 'made': [] } } # User is special, see below for entity, details in entities.items(): for node in data_v3['export_data'][entity].values(): add = node.get('node_type', None) # Node if not add: add = node.get('hostname', None) # Computer if not add: add = node.get('type_string', None) # Group self.assertIsNotNone(add, msg="Helper variable 'add' should never be None") details['migrated'].append(add) for node in data_v4['export_data'][entity].values(): add = node.get('node_type', None) # Node if not add: add = node.get('hostname', None) # Computer if not add: add = node.get('type_string', None) # Group self.assertIsNotNone(add, msg="Helper variable 'add' should never be None") details['made'].append(add) #### Two extra Dicts are present for AiiDA made export 0.4 file #### if entity == 'Node': details['migrated'].extend(2 * ['data.dict.Dict.']) self.assertListEqual( sorted(details['migrated']), sorted(details['made']), msg='Number of {}-entities differ, see diff for details'.format(entity) ) fields = { 'groups_uuid', 'node_attributes_conversion', 'node_attributes', 'node_extras', 'node_extras_conversion' } # 'export_data' is special, see below for field in fields: if field != 'groups_uuid': correction = 2 # Two extra Dicts in AiiDA made export v0.4 file else: correction = 0 self.assertEqual( len(data_v3[field]), len(data_v4[field]) - correction, msg='Number of entities in {} differs for the export files'.format(field) ) number_of_links_v3 = { 'unspecified': 0, 'create': 0, 'return': 0, 'input_calc': 0, 'input_work': 0, 'call_calc': 0, 'call_work': 0 } for link in data_v3['links_uuid']: number_of_links_v3[link['type']] += 1 number_of_links_v4 = { 'unspecified': 0, 'create': 0, 'return': 0, 'input_calc': -2, # Two extra Dict inputs to CalcJobNodes 'input_work': 0, 'call_calc': 0, 'call_work': 0 } for link in data_v4['links_uuid']: number_of_links_v4[link['type']] += 1 self.assertDictEqual( number_of_links_v3, number_of_links_v4, msg='There are a different number of specific links in the migrated export file than the AiiDA made one.' ) self.assertEqual(number_of_links_v3['unspecified'], 0) self.assertEqual(number_of_links_v4['unspecified'], 0) # Special for data['export_data']['User'] # There is an extra user in the migrated export v0.3 file self.assertEqual(len(data_v3['export_data']['User']), len(data_v4['export_data']['User']) + 1) # Special for data['export_data'] # There are Logs exported in the AiiDA made export v0.4 file self.assertEqual(len(data_v3['export_data']) + 1, len(data_v4['export_data']))
def test_version_format(self): """Verify that `version_format` return the correct archive format version.""" filepath = get_archive_file('export_v0.1_simple.aiida', filepath='export/migrate') with Archive(filepath) as archive: self.assertEqual(archive.version_format, '0.1')
def test_empty_archive(self): """Verify that attempting to unpack an empty archive raises a `CorruptArchive` exception.""" filepath = get_archive_file('empty.aiida', filepath='export/migrate') with self.assertRaises(CorruptArchive): with Archive(filepath) as archive: archive.version_format # pylint: disable=pointless-statement