def migrate_archive(input_file, output_file, silent=True):
"""Migrate contents using `migrate_recursively`
This is essentially similar to `verdi export migrate`.
However, since this command may be disabled, this function simulates it and keeps the tests working.
:param input_file: filename with full path for archive to be migrated
:param output_file: filename with full path for archive to be created after migration
"""
from aiida.tools.importexport.migration import migrate_recursively
# Unpack archive, migrate, and re-pack archive
with SandboxFolder(sandbox_in_repo=False) as folder:
if zipfile.is_zipfile(input_file):
extract_zip(input_file, folder, silent=silent)
elif tarfile.is_tarfile(input_file):
extract_tar(input_file, folder, silent=silent)
else:
raise ValueError(
'invalid file format, expected either a zip archive or gzipped tarball'
)
try:
with open(folder.get_abs_path('data.json'), 'r',
encoding='utf8') as fhandle:
data = json.load(fhandle)
with open(folder.get_abs_path('metadata.json'),
'r',
encoding='utf8') as fhandle:
metadata = json.load(fhandle)
except IOError:
raise NotExistent(
'export archive does not contain the required file {}'.format(
fhandle.filename))
# Migrate
migrate_recursively(metadata, data, folder)
# Write json files
with open(folder.get_abs_path('data.json'), 'wb') as fhandle:
json.dump(data, fhandle, indent=4)
with open(folder.get_abs_path('metadata.json'), 'wb') as fhandle:
json.dump(metadata, fhandle, indent=4)
# Pack archive
compression = zipfile.ZIP_DEFLATED
with zipfile.ZipFile(output_file,
mode='w',
compression=compression,
allowZip64=True) as archive:
src = folder.abspath
for dirpath, dirnames, filenames in os.walk(src):
relpath = os.path.relpath(dirpath, src)
for filename in dirnames + filenames:
real_src = os.path.join(dirpath, filename)
real_dest = os.path.join(relpath, filename)
archive.write(real_src, real_dest)
def test_wrong_versions(self):
"""Test correct errors are raised if export files have wrong version numbers"""
from aiida.tools.importexport.migration import MIGRATE_FUNCTIONS
wrong_versions = ['0.0', '0.1.0', '0.99']
old_versions = list(MIGRATE_FUNCTIONS.keys())
legal_versions = old_versions + [newest_version]
wrong_version_metadatas = []
for version in wrong_versions:
metadata = {'export_version': version}
wrong_version_metadatas.append(metadata)
# Make sure the "wrong_versions" are wrong
for version in wrong_versions:
self.assertNotIn(
version,
legal_versions,
msg=
"'{}' was not expected to be a legal version, legal version: {}"
.format(version, legal_versions))
# Make sure migrate_recursively throws an ArchiveMigrationError
for metadata in wrong_version_metadatas:
with self.assertRaises(ArchiveMigrationError):
new_version = migrate_recursively(metadata, {}, None)
self.assertIsNone(
new_version,
msg='migrate_recursively should not return anything, '
"hence the 'return' should be None, but instead it is {}".
format(new_version))
def test_migrate_newest_version(self):
"""
Test critical message and SystemExit is raised, when an export file with the newest export version is migrated
"""
# Initialization
metadata = {'export_version': newest_version}
# Check
with self.assertRaises(SystemExit) as exception:
with Capturing(capture_stderr=True):
new_version = migrate_recursively(metadata, {}, None)
self.assertIn(
'Critical: Your export file is already at the newest export version {}'.format(
metadata['export_version']
),
exception.exception,
msg="Expected a critical statement that the export version '{}' is the newest export version '{}', "
'instead got {}'.format(metadata['export_version'], newest_version, exception.exception)
)
self.assertIsNone(
new_version,
msg='migrate_recursively should not return anything, '
"hence the 'return' should be None, but instead it is {}".format(new_version)
)
def test_migrate_recursively_specific_version(self):
"""Test the `version` argument of the `migrate_recursively` function."""
filepath_archive = get_archive_file('export_v0.3_simple.aiida',
**self.core_archive)
with Archive(filepath_archive) as archive:
# Incorrect type
with self.assertRaises(TypeError):
migrate_recursively(archive.meta_data,
archive.data,
None,
version=0.2)
# Backward migrations are not supported
with self.assertRaises(ArchiveMigrationError):
migrate_recursively(archive.meta_data,
archive.data,
None,
version='0.2')
migrate_recursively(archive.meta_data,
archive.data,
None,
version='0.3')
migrated_version = '0.5'
version = migrate_recursively(archive.meta_data,
archive.data,
None,
version=migrated_version)
self.assertEqual(version, migrated_version)
def migrate(input_file, output_file, force, silent, archive_format):
# pylint: disable=too-many-locals,too-many-statements,too-many-branches
"""
Migrate an old export archive file to the most recent format.
"""
import tarfile
import zipfile
from aiida.common import json
from aiida.common.folders import SandboxFolder
from aiida.tools.importexport import migration, extract_zip, extract_tar
if os.path.exists(output_file) and not force:
echo.echo_critical('the output file already exists')
with SandboxFolder(sandbox_in_repo=False) as folder:
if zipfile.is_zipfile(input_file):
extract_zip(input_file, folder, silent=silent)
elif tarfile.is_tarfile(input_file):
extract_tar(input_file, folder, silent=silent)
else:
echo.echo_critical('invalid file format, expected either a zip archive or gzipped tarball')
try:
with io.open(folder.get_abs_path('data.json'), 'r', encoding='utf8') as fhandle:
data = json.load(fhandle)
with io.open(folder.get_abs_path('metadata.json'), 'r', encoding='utf8') as fhandle:
metadata = json.load(fhandle)
except IOError:
echo.echo_critical('export archive does not contain the required file {}'.format(fhandle.filename))
old_version = migration.verify_metadata_version(metadata)
new_version = migration.migrate_recursively(metadata, data, folder)
with io.open(folder.get_abs_path('data.json'), 'wb') as fhandle:
json.dump(data, fhandle, indent=4)
with io.open(folder.get_abs_path('metadata.json'), 'wb') as fhandle:
json.dump(metadata, fhandle)
if archive_format in ['zip', 'zip-uncompressed']:
compression = zipfile.ZIP_DEFLATED if archive_format == 'zip' else zipfile.ZIP_STORED
with zipfile.ZipFile(output_file, mode='w', compression=compression, allowZip64=True) as archive:
src = folder.abspath
for dirpath, dirnames, filenames in os.walk(src):
relpath = os.path.relpath(dirpath, src)
for filename in dirnames + filenames:
real_src = os.path.join(dirpath, filename)
real_dest = os.path.join(relpath, filename)
archive.write(real_src, real_dest)
elif archive_format == 'tar.gz':
with tarfile.open(output_file, 'w:gz', format=tarfile.PAX_FORMAT, dereference=True) as archive:
archive.add(folder.abspath, arcname='')
if not silent:
echo.echo_success('migrated the archive from version {} to {}'.format(old_version, new_version))
def test_migrate_recursively(self):
"""Test function 'migrate_recursively'"""
import io
import tarfile
import zipfile
from aiida.common.exceptions import NotExistent
from aiida.common.folders import SandboxFolder
from aiida.common.json import load as jsonload
from aiida.tools.importexport.common.archive import extract_tar, extract_zip
# Get metadata.json and data.json as dicts from v0.1 file archive
# Cannot use 'get_json_files' for 'export_v0.1_simple.aiida',
# because we need to pass the SandboxFolder to 'migrate_recursively'
dirpath_archive = get_archive_file('export_v0.1_simple.aiida',
**self.core_archive)
with SandboxFolder(sandbox_in_repo=False) as folder:
if zipfile.is_zipfile(dirpath_archive):
extract_zip(dirpath_archive, folder, silent=True)
elif tarfile.is_tarfile(dirpath_archive):
extract_tar(dirpath_archive, folder, silent=True)
else:
raise ValueError(
'invalid file format, expected either a zip archive or gzipped tarball'
)
try:
with io.open(folder.get_abs_path('data.json'),
'r',
encoding='utf8') as fhandle:
data = jsonload(fhandle)
with io.open(folder.get_abs_path('metadata.json'),
'r',
encoding='utf8') as fhandle:
metadata = jsonload(fhandle)
except IOError:
raise NotExistent(
'export archive does not contain the required file {}'.
format(fhandle.filename))
verify_metadata_version(metadata, version='0.1')
# Migrate to newest version
new_version = migrate_recursively(metadata, data, folder)
verify_metadata_version(metadata, version=newest_version)
self.assertEqual(new_version, newest_version)
def test_wrong_versions(self):
"""Test correct errors are raised if export files have wrong version numbers"""
from aiida.tools.importexport.migration import MIGRATE_FUNCTIONS
# Initialization
wrong_versions = ['0.0', '0.1.0', '0.99']
old_versions = list(MIGRATE_FUNCTIONS.keys())
legal_versions = old_versions + [newest_version]
wrong_version_metadatas = []
for version in wrong_versions:
metadata = {'export_version': version}
wrong_version_metadatas.append(metadata)
# Checks
# Make sure the "wrong_versions" are wrong
for version in wrong_versions:
self.assertNotIn(
version,
legal_versions,
msg=
"'{}' was not expected to be a legal version, legal version: {}"
.format(version, legal_versions))
# Make sure migrate_recursively throws a critical message and raises SystemExit
for metadata in wrong_version_metadatas:
with self.assertRaises(SystemExit) as exception:
with Capturing(capture_stderr=True):
new_version = migrate_recursively(metadata, {}, None)
self.assertIn(
'Critical: Cannot migrate from version {}'.format(
metadata['export_version']),
exception.exception,
msg=
"Expected a critical statement for the wrong export version '{}', "
'instead got {}'.format(metadata['export_version'],
exception.exception))
self.assertIsNone(
new_version,
msg='migrate_recursively should not return anything, '
"hence the 'return' should be None, but instead it is {}".
format(new_version))
def test_migrate_newest_version(self):
"""Test that migrating the latest version runs without complaints."""
metadata = {'export_version': newest_version}
new_version = migrate_recursively(metadata, {}, None)
self.assertEqual(new_version, newest_version)
def migrate(input_file, output_file, force, silent, in_place, archive_format, version):
# pylint: disable=too-many-locals,too-many-statements,too-many-branches
"""Migrate an export archive to a more recent format version."""
import tarfile
import zipfile
from aiida.common import json
from aiida.common.folders import SandboxFolder
from aiida.tools.importexport import migration, extract_zip, extract_tar, ArchiveMigrationError, EXPORT_VERSION
if version is None:
version = EXPORT_VERSION
if in_place:
if output_file:
echo.echo_critical('output file specified together with --in-place flag')
tempdir = tempfile.TemporaryDirectory()
output_file = os.path.join(tempdir.name, 'archive.aiida')
elif not output_file:
echo.echo_critical(
'no output file specified. Please add --in-place flag if you would like to migrate in place.'
)
if os.path.exists(output_file) and not force:
echo.echo_critical('the output file already exists')
with SandboxFolder(sandbox_in_repo=False) as folder:
if zipfile.is_zipfile(input_file):
extract_zip(input_file, folder, silent=silent)
elif tarfile.is_tarfile(input_file):
extract_tar(input_file, folder, silent=silent)
else:
echo.echo_critical('invalid file format, expected either a zip archive or gzipped tarball')
try:
with open(folder.get_abs_path('data.json'), 'r', encoding='utf8') as fhandle:
data = json.load(fhandle)
with open(folder.get_abs_path('metadata.json'), 'r', encoding='utf8') as fhandle:
metadata = json.load(fhandle)
except IOError:
echo.echo_critical('export archive does not contain the required file {}'.format(fhandle.filename))
old_version = migration.verify_metadata_version(metadata)
if version <= old_version:
echo.echo_success('nothing to be done - archive already at version {} >= {}'.format(old_version, version))
return
try:
new_version = migration.migrate_recursively(metadata, data, folder, version)
except ArchiveMigrationError as exception:
echo.echo_critical(str(exception))
with open(folder.get_abs_path('data.json'), 'wb') as fhandle:
json.dump(data, fhandle, indent=4)
with open(folder.get_abs_path('metadata.json'), 'wb') as fhandle:
json.dump(metadata, fhandle)
if archive_format in ['zip', 'zip-uncompressed']:
compression = zipfile.ZIP_DEFLATED if archive_format == 'zip' else zipfile.ZIP_STORED
with zipfile.ZipFile(output_file, mode='w', compression=compression, allowZip64=True) as archive:
src = folder.abspath
for dirpath, dirnames, filenames in os.walk(src):
relpath = os.path.relpath(dirpath, src)
for filename in dirnames + filenames:
real_src = os.path.join(dirpath, filename)
real_dest = os.path.join(relpath, filename)
archive.write(real_src, real_dest)
elif archive_format == 'tar.gz':
with tarfile.open(output_file, 'w:gz', format=tarfile.PAX_FORMAT, dereference=True) as archive:
archive.add(folder.abspath, arcname='')
if in_place:
os.rename(output_file, input_file)
tempdir.cleanup()
if not silent:
echo.echo_success('migrated the archive from version {} to {}'.format(old_version, new_version))