def migrate(input_file, output_file, force, silent, in_place, archive_format,
version, verbosity):
"""Migrate an export archive to a more recent format version.
.. deprecated:: 1.5.0
Support for the --silent flag, replaced by --verbosity
"""
from aiida.common.log import override_log_formatter_context
from aiida.common.progress_reporter import set_progress_bar_tqdm, set_progress_reporter
from aiida.tools.importexport import detect_archive_type, EXPORT_VERSION
from aiida.tools.importexport.archive.migrators import get_migrator, MIGRATE_LOGGER
if silent is True:
echo.echo_deprecated(
'the --silent option is deprecated, use --verbosity')
if in_place:
if output_file:
echo.echo_critical(
'output file specified together with --in-place flag')
output_file = input_file
force = True
elif not output_file:
echo.echo_critical(
'no output file specified. Please add --in-place flag if you would like to migrate in place.'
)
if verbosity in ['DEBUG', 'INFO']:
set_progress_bar_tqdm(leave=(verbosity == 'DEBUG'))
else:
set_progress_reporter(None)
MIGRATE_LOGGER.setLevel(verbosity)
if version is None:
version = EXPORT_VERSION
migrator_cls = get_migrator(detect_archive_type(input_file))
migrator = migrator_cls(input_file)
try:
with override_log_formatter_context('%(message)s'):
migrator.migrate(version,
output_file,
force=force,
out_compression=archive_format)
except Exception as error: # pylint: disable=broad-except
if verbosity == 'DEBUG':
raise
echo.echo_critical(
'failed to migrate the archive file (use `--verbosity DEBUG` to see traceback): '
f'{error.__class__.__name__}:{error}')
if verbosity in ['DEBUG', 'INFO']:
echo.echo_success(f'migrated the archive to version {version}')
def cmd_import(
ctx, archives, webpages, group, extras_mode_existing, extras_mode_new, comment_mode, migration, non_interactive,
verbosity
):
"""Import data from an AiiDA archive file.
The archive can be specified by its relative or absolute file path, or its HTTP URL.
"""
# pylint: disable=unused-argument
from aiida.common.log import override_log_formatter_context
from aiida.common.progress_reporter import set_progress_bar_tqdm, set_progress_reporter
from aiida.tools.importexport.dbimport.utils import IMPORT_LOGGER
from aiida.tools.importexport.archive.migrators import MIGRATE_LOGGER
if verbosity in ['DEBUG', 'INFO']:
set_progress_bar_tqdm(leave=(verbosity == 'DEBUG'))
else:
set_progress_reporter(None)
IMPORT_LOGGER.setLevel(verbosity)
MIGRATE_LOGGER.setLevel(verbosity)
all_archives = _gather_imports(archives, webpages)
# Preliminary sanity check
if not all_archives:
echo.echo_critical('no valid exported archives were found')
# Shared import key-word arguments
import_kwargs = {
'group': group,
'extras_mode_existing': ExtrasImportCode[extras_mode_existing].value,
'extras_mode_new': extras_mode_new,
'comment_mode': comment_mode,
}
with override_log_formatter_context('%(message)s'):
for archive, web_based in all_archives:
_import_archive(archive, web_based, import_kwargs, migration)
def create(output_file, codes, computers, groups, nodes, archive_format, force,
input_calc_forward, input_work_forward, create_backward,
return_backward, call_calc_backward, call_work_backward,
include_comments, include_logs, verbosity):
"""
Export subsets of the provenance graph to file for sharing.
Besides Nodes of the provenance graph, you can export Groups, Codes, Computers, Comments and Logs.
By default, the archive file will include not only the entities explicitly provided via the command line but also
their provenance, according to the rules outlined in the documentation.
You can modify some of those rules using options of this command.
"""
# pylint: disable=too-many-branches
from aiida.common.log import override_log_formatter_context
from aiida.common.progress_reporter import set_progress_bar_tqdm, set_progress_reporter
from aiida.tools.importexport import export, ExportFileFormat, EXPORT_LOGGER
from aiida.tools.importexport.common.exceptions import ArchiveExportError
entities = []
if codes:
entities.extend(codes)
if computers:
entities.extend(computers)
if groups:
entities.extend(groups)
if nodes:
entities.extend(nodes)
kwargs = {
'input_calc_forward': input_calc_forward,
'input_work_forward': input_work_forward,
'create_backward': create_backward,
'return_backward': return_backward,
'call_calc_backward': call_calc_backward,
'call_work_backward': call_work_backward,
'include_comments': include_comments,
'include_logs': include_logs,
'overwrite': force,
}
if archive_format == 'zip':
export_format = ExportFileFormat.ZIP
kwargs.update({'writer_init': {'use_compression': True}})
elif archive_format == 'zip-uncompressed':
export_format = ExportFileFormat.ZIP
kwargs.update({'writer_init': {'use_compression': False}})
elif archive_format == 'zip-lowmemory':
export_format = ExportFileFormat.ZIP
kwargs.update({'writer_init': {'cache_zipinfo': True}})
elif archive_format == 'tar.gz':
export_format = ExportFileFormat.TAR_GZIPPED
elif archive_format == 'null':
export_format = 'null'
if verbosity in ['DEBUG', 'INFO']:
set_progress_bar_tqdm(leave=(verbosity == 'DEBUG'))
else:
set_progress_reporter(None)
EXPORT_LOGGER.setLevel(verbosity)
try:
with override_log_formatter_context('%(message)s'):
export(entities,
filename=output_file,
file_format=export_format,
**kwargs)
except ArchiveExportError as exception:
echo.echo_critical(
f'failed to write the archive file. Exception: {exception}')
else:
echo.echo_success(f'wrote the export archive file to {output_file}')
def extract_zip(infile,
folder,
nodes_export_subfolder=None,
check_files=('data.json', 'metadata.json'),
**kwargs):
"""Extract the nodes to be imported from a zip file.
:param infile: file path
:type infile: str
:param folder: a temporary folder used to extract the file tree
:type folder: :py:class:`~aiida.common.folders.SandboxFolder`
:param nodes_export_subfolder: name of the subfolder for AiiDA nodes
:type nodes_export_subfolder: str
:param check_files: list of files to check are present
:param silent: suppress progress bar
:type silent: bool
:raises TypeError: if parameter types are not respected
:raises `~aiida.tools.importexport.common.exceptions.CorruptArchive`: if the archive misses files or files have
incorrect formats
"""
warnings.warn(
'extract_zip function is deprecated and will be removed in AiiDA v2.0.0, '
'use extract_tree in the archive-path package instead',
AiidaDeprecationWarning) # pylint: disable=no-member
if nodes_export_subfolder:
if not isinstance(nodes_export_subfolder, str):
raise TypeError('nodes_export_subfolder must be a string')
else:
nodes_export_subfolder = NODES_EXPORT_SUBFOLDER
if not kwargs.get('silent', False):
set_progress_bar_tqdm(unit='files')
else:
set_progress_reporter(None)
data_files = set()
try:
with zipfile.ZipFile(infile, 'r', allowZip64=True) as handle:
members = handle.namelist()
if not members:
raise CorruptArchive('no files detected in archive')
with get_progress_reporter()(total=len(members)) as progress:
for membername in members:
progress.update()
# Check that we are only exporting nodes within the subfolder!
# better check such that there are no .. in the
# path; use probably the folder limit checks
if membername in check_files:
data_files.add(membername)
elif not membername.startswith(nodes_export_subfolder +
os.sep):
continue
_update_description(membername, progress)
handle.extract(path=folder.abspath, member=membername)
except zipfile.BadZipfile:
raise ValueError(
'The input file format for import is not valid (not a zip file)')
for name in check_files:
if name not in data_files:
raise CorruptArchive('Archive missing required file:f {name}')
def extract_tar(infile,
folder,
nodes_export_subfolder=None,
check_files=('data.json', 'metadata.json'),
**kwargs):
"""
Extract the nodes to be imported from a (possibly zipped) tar file.
:param infile: file path
:type infile: str
:param folder: a temporary fodler used to extract the file tree
:type folder: :py:class:`~aiida.common.folders.SandboxFolder`
:param nodes_export_subfolder: name of the subfolder for AiiDA nodes
:type nodes_export_subfolder: str
:param check_files: list of files to check are present
:param silent: suppress progress bar
:type silent: bool
:raises TypeError: if parameter types are not respected
:raises `~aiida.tools.importexport.common.exceptions.CorruptArchive`: if the archive misses files or files have
incorrect formats
"""
warnings.warn(
'extract_tar function is deprecated and will be removed in AiiDA v2.0.0, '
'use extract_tree in the archive-path package instead',
AiidaDeprecationWarning) # pylint: disable=no-member
if nodes_export_subfolder:
if not isinstance(nodes_export_subfolder, str):
raise TypeError('nodes_export_subfolder must be a string')
else:
nodes_export_subfolder = NODES_EXPORT_SUBFOLDER
if not kwargs.get('silent', False):
set_progress_bar_tqdm(unit='files')
else:
set_progress_reporter(None)
data_files = set()
try:
with tarfile.open(infile, 'r:*', format=tarfile.PAX_FORMAT) as handle:
members = handle.getmembers()
if len(members) == 1 and members[0].size == 0:
raise CorruptArchive('no files detected in archive')
with get_progress_reporter()(total=len(members)) as progress:
for member in members:
progress.update()
if member.isdev():
# safety: skip if character device, block device or FIFO
print(
f'WARNING, device found inside the import file: {member.name}',
file=sys.stderr)
continue
if member.issym() or member.islnk():
# safety: in export, I set dereference=True therefore
# there should be no symbolic or hard links.
print(
f'WARNING, symlink found inside the import file: {member.name}',
file=sys.stderr)
continue
# Check that we are only exporting nodes within the subfolder!
# better check such that there are no .. in the
# path; use probably the folder limit checks
if member.name in check_files:
data_files.add(member.name)
elif not member.name.startswith(nodes_export_subfolder +
os.sep):
continue
_update_description(member.name, progress)
handle.extract(path=folder.abspath, member=member)
except tarfile.ReadError:
raise ValueError(
'The input file format for import is not valid (not a tar file)')
for name in check_files:
if name not in data_files:
raise CorruptArchive('Archive missing required file:f {name}')