def test_upload_local_copy_list(fixture_sandbox, aiida_localhost,
aiida_local_code_factory):
"""Test the ``local_copy_list`` functionality in ``upload_calculation``.
Specifically, verify that files in the ``local_copy_list`` do not end up in the repository of the node.
"""
from aiida.common.datastructures import CalcInfo, CodeInfo
from aiida.orm import CalcJobNode, SinglefileData
inputs = {
'file_a': SinglefileData(io.BytesIO(b'content_a')).store(),
'file_b': SinglefileData(io.BytesIO(b'content_b')).store(),
}
node = CalcJobNode(computer=aiida_localhost)
node.store()
code = aiida_local_code_factory('arithmetic.add', '/bin/bash').store()
code_info = CodeInfo()
code_info.code_uuid = code.uuid
calc_info = CalcInfo()
calc_info.uuid = node.uuid
calc_info.codes_info = [code_info]
calc_info.local_copy_list = [
(inputs['file_a'].uuid, inputs['file_a'].filename, './files/file_a'),
(inputs['file_a'].uuid, inputs['file_a'].filename, './files/file_b'),
]
with LocalTransport() as transport:
execmanager.upload_calculation(node, transport, calc_info,
fixture_sandbox)
assert node.list_object_names() == []
def submit_calculation(calculation: CalcJobNode, transport: Transport) -> str:
"""Submit a previously uploaded `CalcJob` to the scheduler.
:param calculation: the instance of CalcJobNode to submit.
:param transport: an already opened transport to use to submit the calculation.
:return: the job id as returned by the scheduler `submit_from_script` call
"""
job_id = calculation.get_job_id()
# If the `job_id` attribute is already set, that means this function was already executed once and the scheduler
# submit command was successful as the job id it returned was set on the node. This scenario can happen when the
# daemon runner gets shutdown right after accomplishing the submission task, but before it gets the chance to
# finalize the state transition of the `CalcJob` to the `UPDATE` transport task. Since the job is already submitted
# we do not want to submit it a second time, so we simply return the existing job id here.
if job_id is not None:
return job_id
scheduler = calculation.computer.get_scheduler()
scheduler.set_transport(transport)
submit_script_filename = calculation.get_option('submit_script_filename')
workdir = calculation.get_remote_workdir()
job_id = scheduler.submit_from_script(workdir, submit_script_filename)
calculation.set_job_id(job_id)
return job_id
def test_get_scheduler_stderr(self):
"""Verify that the repository sandbox folder is cleaned after the node instance is garbage collected."""
option_key = 'scheduler_stderr'
option_value = '_scheduler-error.txt'
stderr = 'some\nstandard error'
# Note: cannot use pytest.mark.parametrize in unittest classes, so I just do a loop here
for with_file in [True, False]:
for with_option in [True, False]:
node = CalcJobNode(computer=self.computer, )
node.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
retrieved = FolderData()
if with_file:
retrieved.put_object_from_filelike(io.StringIO(stderr),
option_value)
if with_option:
node.set_option(option_key, option_value)
node.store()
retrieved.store()
retrieved.add_incoming(node,
link_type=LinkType.CREATE,
link_label='retrieved')
# It should return `None` if no scheduler output is there (file not there, or option not set),
# while it should return the content if both are set
self.assertEqual(node.get_scheduler_stderr(),
stderr if with_file and with_option else None)
def setUpClass(cls, *args, **kwargs):
"""Define a useful CalcJobNode to test the CalcJobResultManager.
We emulate a node for the `TemplateReplacer` calculation job class. To do this we have to make sure the
process type is set correctly and an output parameter node is created.
"""
super(TestCalcJobResultManager, cls).setUpClass(*args, **kwargs)
cls.process_class = CalculationFactory('templatereplacer')
cls.process_type = get_entry_point_string_from_class(cls.process_class.__module__, cls.process_class.__name__)
cls.node = CalcJobNode(computer=cls.computer, process_type=cls.process_type)
cls.node.set_option('resources', {'num_machines': 1, 'num_mpiprocs_per_machine': 1})
cls.node.store()
cls.key_one = 'key_one'
cls.key_two = 'key_two'
cls.val_one = 'val_one'
cls.val_two = 'val_two'
cls.keys = [cls.key_one, cls.key_two]
cls.result_node = Dict(dict={
cls.key_one: cls.val_one,
cls.key_two: cls.val_two,
}).store()
cls.result_node.add_incoming(cls.node, LinkType.CREATE, cls.process_class.spec().default_output_node)
def test_no_process_type(self):
"""`get_results` should raise `ValueError` if `CalcJobNode` has no `process_type`"""
node = CalcJobNode(computer=self.computer)
manager = CalcJobResultManager(node)
with self.assertRaises(ValueError):
manager.get_results()
def test_invalid_process_type(self):
"""`get_results` should raise `ValueError` if `CalcJobNode` has invalid `process_type`"""
node = CalcJobNode(computer=self.computer, process_type='aiida.calculations:not_existent')
manager = CalcJobResultManager(node)
with self.assertRaises(ValueError):
manager.get_results()
def kill_calculation(calculation: CalcJobNode, transport: Transport) -> None:
"""
Kill the calculation through the scheduler
:param calculation: the instance of CalcJobNode to kill.
:param transport: an already opened transport to use to address the scheduler
"""
job_id = calculation.get_job_id()
if job_id is None:
# the calculation has not yet been submitted to the scheduler
return
# Get the scheduler plugin class and initialize it with the correct transport
scheduler = calculation.computer.get_scheduler()
scheduler.set_transport(transport)
# Call the proper kill method for the job ID of this calculation
result = scheduler.kill(job_id)
if result is not True:
# Failed to kill because the job might have already been completed
running_jobs = scheduler.get_jobs(jobs=[job_id], as_dict=True)
job = running_jobs.get(job_id, None)
# If the job is returned it is still running and the kill really failed, so we raise
if job is not None and job.job_state != JobState.DONE:
raise exceptions.RemoteOperationError(
f'scheduler.kill({job_id}) was unsuccessful')
else:
EXEC_LOGGER.warning(
'scheduler.kill() failed but job<{%s}> no longer seems to be running regardless',
job_id)
def test_deletion(self):
from aiida.orm import CalcJobNode
from aiida.common.exceptions import InvalidOperation
import aiida.backends.sqlalchemy
newcomputer = self.backend.computers.create(
name='testdeletioncomputer',
hostname='localhost',
transport_type='local',
scheduler_type='pbspro')
newcomputer.store()
# This should be possible, because nothing is using this computer
self.backend.computers.delete(newcomputer.id)
node = CalcJobNode()
node.computer = self.computer
node.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
node.store()
session = aiida.backends.sqlalchemy.get_scoped_session()
# This should fail, because there is at least a calculation
# using this computer (the one created just above)
try:
session.begin_nested()
with self.assertRaises(InvalidOperation):
self.backend.computers.delete(self.computer.id)
finally:
session.rollback()
def test_process_class_no_default_node(self):
"""`get_results` should raise `ValueError` if process class does not define default output node."""
# This is a valid process class however ArithmeticAddCalculation does define a default output node
process_class = CalculationFactory('arithmetic.add')
process_type = get_entry_point_string_from_class(process_class.__module__, process_class.__name__)
node = CalcJobNode(computer=self.computer, process_type=process_type)
manager = CalcJobResultManager(node)
with self.assertRaises(ValueError):
manager.get_results()
def test_deletion(self):
"""Test computer deletion."""
from aiida.orm import CalcJobNode, Computer
from aiida.common.exceptions import InvalidOperation
newcomputer = Computer(name='testdeletioncomputer',
hostname='localhost',
transport_type='local',
scheduler_type='pbspro',
workdir='/tmp/aiida').store()
# This should be possible, because nothing is using this computer
orm.Computer.objects.delete(newcomputer.id)
calc = CalcJobNode(computer=self.computer)
calc.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
calc.store()
# This should fail, because there is at least a calculation
# using this computer (the one created just above)
with self.assertRaises(InvalidOperation):
orm.Computer.objects.delete(self.computer.id) # pylint: disable=no-member
def _generate_remote_data(computer, remote_path, entry_point_name=None):
"""Generate a RemoteData node, loctated at remote_path"""
from aiida.common.links import LinkType
from aiida.orm import CalcJobNode, RemoteData
from aiida.plugins.entry_point import format_entry_point_string
entry_point = format_entry_point_string('aiida.calculations',
entry_point_name)
remote = RemoteData(remote_path=remote_path)
remote.computer = computer
if entry_point_name is not None:
creator = CalcJobNode(computer=computer, process_type=entry_point)
creator.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
#creator.set_attribute('prefix', 'aiida')
remote.add_incoming(creator,
link_type=LinkType.CREATE,
link_label='remote_folder')
creator.store()
return remote
def test_store_indirect():
"""Test the `BasisData.store` method when called indirectly because its is an input."""
basis = BasisData(io.BytesIO(b'basis'))
basis.element = 'Ar'
node = CalcJobNode()
node.add_incoming(basis, link_type=LinkType.INPUT_CALC, link_label='basis')
node.store_all()
def setUpClass(cls, *args, **kwargs):
super().setUpClass(*args, **kwargs)
cls.computer.configure() # pylint: disable=no-member
cls.construction_options = {
'resources': {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
}
}
cls.calcjob = CalcJobNode()
cls.calcjob.computer = cls.computer
cls.calcjob.set_options(cls.construction_options)
cls.calcjob.store()
def test_store_indirect():
"""Test the `PseudoPotentialData.store` method when called indirectly because its is an input."""
pseudo = PseudoPotentialData(io.BytesIO(b'pseudo'))
pseudo.element = 'Ar'
node = CalcJobNode()
node.add_incoming(pseudo,
link_type=LinkType.INPUT_CALC,
link_label='pseudo')
node.store_all()
def setUpClass(cls, *args, **kwargs):
"""
Create some code to test the CalculationParamType parameter type for the command line infrastructure
We create an initial code with a random name and then on purpose create two code with a name
that matches exactly the ID and UUID, respectively, of the first one. This allows us to test
the rules implemented to solve ambiguities that arise when determing the identifier type
"""
super().setUpClass(*args, **kwargs)
cls.param = CalculationParamType()
cls.entity_01 = CalculationNode().store()
cls.entity_02 = CalculationNode().store()
cls.entity_03 = CalculationNode().store()
cls.entity_04 = WorkFunctionNode()
cls.entity_05 = CalcFunctionNode()
cls.entity_06 = CalcJobNode()
cls.entity_07 = WorkChainNode()
cls.entity_01.label = 'calculation_01'
cls.entity_02.label = str(cls.entity_01.pk)
cls.entity_03.label = str(cls.entity_01.uuid)
def _generate_remote_data(computer,
remote_path,
entry_point_name=None,
extras_root=[]):
"""Return a `KpointsData` with a mesh of npoints in each direction."""
from aiida.common.links import LinkType
from aiida.orm import CalcJobNode, RemoteData, Dict
from aiida.plugins.entry_point import format_entry_point_string
entry_point = format_entry_point_string('aiida.calculations',
entry_point_name)
remote = RemoteData(remote_path=remote_path)
remote.computer = computer
if entry_point_name is not None:
creator = CalcJobNode(computer=computer, process_type=entry_point)
creator.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
remote.add_incoming(creator,
link_type=LinkType.CREATE,
link_label='remote_folder')
for extra in extras_root:
to_link = extra[0]
if isinstance(to_link, dict):
to_link = Dict(dict=to_link)
to_link.store()
creator.add_incoming(to_link,
link_type=LinkType.INPUT_CALC,
link_label=extra[1])
creator.store()
return remote
def test_process_show(self):
"""Test verdi process show"""
workchain_one = WorkChainNode()
workchain_two = WorkChainNode()
workchains = [workchain_one, workchain_two]
workchain_two.set_attribute('process_label', 'workchain_one_caller')
workchain_two.store()
workchain_one.add_incoming(workchain_two, link_type=LinkType.CALL_WORK, link_label='called')
workchain_one.store()
calcjob_one = CalcJobNode()
calcjob_two = CalcJobNode()
calcjob_one.set_attribute('process_label', 'process_label_one')
calcjob_two.set_attribute('process_label', 'process_label_two')
calcjob_one.add_incoming(workchain_one, link_type=LinkType.CALL_CALC, link_label='one')
calcjob_two.add_incoming(workchain_one, link_type=LinkType.CALL_CALC, link_label='two')
calcjob_one.store()
calcjob_two.store()
# Running without identifiers should not except and not print anything
options = []
result = self.cli_runner.invoke(cmd_process.process_show, options)
self.assertIsNone(result.exception, result.output)
self.assertEqual(len(get_result_lines(result)), 0)
# Giving a single identifier should print a non empty string message
options = [str(workchain_one.pk)]
result = self.cli_runner.invoke(cmd_process.process_show, options)
lines = get_result_lines(result)
self.assertClickResultNoException(result)
self.assertTrue(len(lines) > 0)
self.assertIn('workchain_one_caller', result.output)
self.assertIn('process_label_one', lines[-2])
self.assertIn('process_label_two', lines[-1])
# Giving multiple identifiers should print a non empty string message
options = [str(node.pk) for node in workchains]
result = self.cli_runner.invoke(cmd_process.process_show, options)
self.assertIsNone(result.exception, result.output)
self.assertTrue(len(get_result_lines(result)) > 0)
def fill_repo(self):
"""Utility function to create repository nodes"""
from aiida.orm import CalcJobNode, Data, Dict
extra_name = self.__class__.__name__ + '/test_with_subclasses'
resources = {'num_machines': 1, 'num_mpiprocs_per_machine': 1}
a1 = CalcJobNode(computer=self.computer)
a1.set_option('resources', resources)
a1.store()
# To query only these nodes later
a1.set_extra(extra_name, True)
a3 = Data().store()
a3.set_extra(extra_name, True)
a4 = Dict(dict={'a': 'b'}).store()
a4.set_extra(extra_name, True)
a5 = Data().store()
a5.set_extra(extra_name, True)
# I don't set the extras, just to be sure that the filtering works
# The filtering is needed because other tests will put stuff int he DB
a6 = CalcJobNode(computer=self.computer)
a6.set_option('resources', resources)
a6.store()
a7 = Data()
a7.store()
def stash_calculation(calculation: CalcJobNode, transport: Transport) -> None:
"""Stash files from the working directory of a completed calculation to a permanent remote folder.
After a calculation has been completed, optionally stash files from the work directory to a storage location on the
same remote machine. This is useful if one wants to keep certain files from a completed calculation to be removed
from the scratch directory, because they are necessary for restarts, but that are too heavy to retrieve.
Instructions of which files to copy where are retrieved from the `stash.source_list` option.
:param calculation: the calculation job node.
:param transport: an already opened transport.
"""
from aiida.common.datastructures import StashMode
from aiida.orm import RemoteStashFolderData
logger_extra = get_dblogger_extra(calculation)
stash_options = calculation.get_option('stash')
stash_mode = stash_options.get('mode', StashMode.COPY.value)
source_list = stash_options.get('source_list', [])
if not source_list:
return
if stash_mode != StashMode.COPY.value:
EXEC_LOGGER.warning(
f'stashing mode {stash_mode} is not implemented yet.')
return
cls = RemoteStashFolderData
EXEC_LOGGER.debug(
f'stashing files for calculation<{calculation.pk}>: {source_list}',
extra=logger_extra)
uuid = calculation.uuid
target_basepath = os.path.join(stash_options['target_base'], uuid[:2],
uuid[2:4], uuid[4:])
for source_filename in source_list:
source_filepath = os.path.join(calculation.get_remote_workdir(),
source_filename)
target_filepath = os.path.join(target_basepath, source_filename)
# If the source file is in a (nested) directory, create those directories first in the target directory
target_dirname = os.path.dirname(target_filepath)
transport.makedirs(target_dirname, ignore_existing=True)
try:
transport.copy(source_filepath, target_filepath)
except (IOError, ValueError) as exception:
EXEC_LOGGER.warning(
f'failed to stash {source_filepath} to {target_filepath}: {exception}'
)
else:
EXEC_LOGGER.debug(
f'stashed {source_filepath} to {target_filepath}')
remote_stash = cls(
computer=calculation.computer,
target_basepath=target_basepath,
stash_mode=StashMode(stash_mode),
source_list=source_list,
).store()
remote_stash.add_incoming(calculation,
link_type=LinkType.CREATE,
link_label='remote_stash')
def test_get_scheduler_stderr(self):
"""Verify that the repository sandbox folder is cleaned after the node instance is garbage collected."""
option_key = 'scheduler_stderr'
option_value = '_scheduler-error.txt'
stderr = 'some\nstandard error'
node = CalcJobNode(computer=self.computer, )
node.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
retrieved = FolderData()
# No scheduler error filename option so should return `None`
self.assertEqual(node.get_scheduler_stderr(), None)
# No retrieved folder so should return `None`
node.set_option(option_key, option_value)
self.assertEqual(node.get_scheduler_stderr(), None)
# Now it has retrieved folder, but file does not actually exist in it, should not except but return `None
node.store()
retrieved.store()
retrieved.add_incoming(node,
link_type=LinkType.CREATE,
link_label='retrieved')
self.assertEqual(node.get_scheduler_stderr(), None)
# Add the file to the retrieved folder
with tempfile.NamedTemporaryFile(mode='w+') as handle:
handle.write(stderr)
handle.flush()
handle.seek(0)
retrieved.put_object_from_filelike(handle,
option_value,
force=True)
self.assertEqual(node.get_scheduler_stderr(), stderr)
def retrieve_calculation(calculation: CalcJobNode, transport: Transport,
retrieved_temporary_folder: str) -> None:
"""Retrieve all the files of a completed job calculation using the given transport.
If the job defined anything in the `retrieve_temporary_list`, those entries will be stored in the
`retrieved_temporary_folder`. The caller is responsible for creating and destroying this folder.
:param calculation: the instance of CalcJobNode to update.
:param transport: an already opened transport to use for the retrieval.
:param retrieved_temporary_folder: the absolute path to a directory in which to store the files
listed, if any, in the `retrieved_temporary_folder` of the jobs CalcInfo
"""
logger_extra = get_dblogger_extra(calculation)
workdir = calculation.get_remote_workdir()
EXEC_LOGGER.debug(f'Retrieving calc {calculation.pk}', extra=logger_extra)
EXEC_LOGGER.debug(f'[retrieval of calc {calculation.pk}] chdir {workdir}',
extra=logger_extra)
# If the calculation already has a `retrieved` folder, simply return. The retrieval was apparently already completed
# before, which can happen if the daemon is restarted and it shuts down after retrieving but before getting the
# chance to perform the state transition. Upon reloading this calculation, it will re-attempt the retrieval.
link_label = calculation.link_label_retrieved
if calculation.get_outgoing(FolderData,
link_label_filter=link_label).first():
EXEC_LOGGER.warning(
f'CalcJobNode<{calculation.pk}> already has a `{link_label}` output folder: skipping retrieval'
)
return
# Create the FolderData node into which to store the files that are to be retrieved
retrieved_files = FolderData()
with transport:
transport.chdir(workdir)
# First, retrieve the files of folderdata
retrieve_list = calculation.get_retrieve_list()
retrieve_temporary_list = calculation.get_retrieve_temporary_list()
retrieve_singlefile_list = calculation.get_retrieve_singlefile_list()
with SandboxFolder() as folder:
retrieve_files_from_list(calculation, transport, folder.abspath,
retrieve_list)
# Here I retrieved everything; now I store them inside the calculation
retrieved_files.put_object_from_tree(folder.abspath)
# Second, retrieve the singlefiles, if any files were specified in the 'retrieve_temporary_list' key
if retrieve_singlefile_list:
with SandboxFolder() as folder:
_retrieve_singlefiles(calculation, transport, folder,
retrieve_singlefile_list, logger_extra)
# Retrieve the temporary files in the retrieved_temporary_folder if any files were
# specified in the 'retrieve_temporary_list' key
if retrieve_temporary_list:
retrieve_files_from_list(calculation, transport,
retrieved_temporary_folder,
retrieve_temporary_list)
# Log the files that were retrieved in the temporary folder
for filename in os.listdir(retrieved_temporary_folder):
EXEC_LOGGER.debug(
f"[retrieval of calc {calculation.pk}] Retrieved temporary file or folder '{filename}'",
extra=logger_extra)
# Store everything
EXEC_LOGGER.debug(
f'[retrieval of calc {calculation.pk}] Storing retrieved_files={retrieved_files.pk}',
extra=logger_extra)
retrieved_files.store()
# Make sure that attaching the `retrieved` folder with a link is the last thing we do. This gives the biggest chance
# of making this method idempotent. That is to say, if a runner gets interrupted during this action, it will simply
# retry the retrieval, unless we got here and managed to link it up, in which case we move to the next task.
retrieved_files.add_incoming(calculation,
link_type=LinkType.CREATE,
link_label=calculation.link_label_retrieved)
def upload_calculation(node: CalcJobNode,
transport: Transport,
calc_info: CalcInfo,
folder: SandboxFolder,
inputs: Optional[MappingType[str, Any]] = None,
dry_run: bool = False) -> None:
"""Upload a `CalcJob` instance
:param node: the `CalcJobNode`.
:param transport: an already opened transport to use to submit the calculation.
:param calc_info: the calculation info datastructure returned by `CalcJob.presubmit`
:param folder: temporary local file system folder containing the inputs written by `CalcJob.prepare_for_submission`
"""
# pylint: disable=too-many-locals,too-many-branches,too-many-statements
# If the calculation already has a `remote_folder`, simply return. The upload was apparently already completed
# before, which can happen if the daemon is restarted and it shuts down after uploading but before getting the
# chance to perform the state transition. Upon reloading this calculation, it will re-attempt the upload.
link_label = 'remote_folder'
if node.get_outgoing(RemoteData, link_label_filter=link_label).first():
EXEC_LOGGER.warning(
f'CalcJobNode<{node.pk}> already has a `{link_label}` output: skipping upload'
)
return calc_info
computer = node.computer
codes_info = calc_info.codes_info
input_codes = [
load_node(_.code_uuid, sub_classes=(Code, )) for _ in codes_info
]
logger_extra = get_dblogger_extra(node)
transport.set_logger_extra(logger_extra)
logger = LoggerAdapter(logger=EXEC_LOGGER, extra=logger_extra)
if not dry_run and node.has_cached_links():
raise ValueError(
'Cannot submit calculation {} because it has cached input links! If you just want to test the '
'submission, set `metadata.dry_run` to True in the inputs.'.format(
node.pk))
# If we are performing a dry-run, the working directory should actually be a local folder that should already exist
if dry_run:
workdir = transport.getcwd()
else:
remote_user = transport.whoami()
remote_working_directory = computer.get_workdir().format(
username=remote_user)
if not remote_working_directory.strip():
raise exceptions.ConfigurationError(
"[submission of calculation {}] No remote_working_directory configured for computer '{}'"
.format(node.pk, computer.label))
# If it already exists, no exception is raised
try:
transport.chdir(remote_working_directory)
except IOError:
logger.debug(
'[submission of calculation {}] Unable to chdir in {}, trying to create it'
.format(node.pk, remote_working_directory))
try:
transport.makedirs(remote_working_directory)
transport.chdir(remote_working_directory)
except EnvironmentError as exc:
raise exceptions.ConfigurationError(
'[submission of calculation {}] '
'Unable to create the remote directory {} on '
"computer '{}': {}".format(node.pk,
remote_working_directory,
computer.label, exc))
# Store remotely with sharding (here is where we choose
# the folder structure of remote jobs; then I store this
# in the calculation properties using _set_remote_dir
# and I do not have to know the logic, but I just need to
# read the absolute path from the calculation properties.
transport.mkdir(calc_info.uuid[:2], ignore_existing=True)
transport.chdir(calc_info.uuid[:2])
transport.mkdir(calc_info.uuid[2:4], ignore_existing=True)
transport.chdir(calc_info.uuid[2:4])
try:
# The final directory may already exist, most likely because this function was already executed once, but
# failed and as a result was rescheduled by the eninge. In this case it would be fine to delete the folder
# and create it from scratch, except that we cannot be sure that this the actual case. Therefore, to err on
# the safe side, we move the folder to the lost+found directory before recreating the folder from scratch
transport.mkdir(calc_info.uuid[4:])
except OSError:
# Move the existing directory to lost+found, log a warning and create a clean directory anyway
path_existing = os.path.join(transport.getcwd(),
calc_info.uuid[4:])
path_lost_found = os.path.join(remote_working_directory,
REMOTE_WORK_DIRECTORY_LOST_FOUND)
path_target = os.path.join(path_lost_found, calc_info.uuid)
logger.warning(
f'tried to create path {path_existing} but it already exists, moving the entire folder to {path_target}'
)
# Make sure the lost+found directory exists, then copy the existing folder there and delete the original
transport.mkdir(path_lost_found, ignore_existing=True)
transport.copytree(path_existing, path_target)
transport.rmtree(path_existing)
# Now we can create a clean folder for this calculation
transport.mkdir(calc_info.uuid[4:])
finally:
transport.chdir(calc_info.uuid[4:])
# I store the workdir of the calculation for later file retrieval
workdir = transport.getcwd()
node.set_remote_workdir(workdir)
# I first create the code files, so that the code can put
# default files to be overwritten by the plugin itself.
# Still, beware! The code file itself could be overwritten...
# But I checked for this earlier.
for code in input_codes:
if code.is_local():
# Note: this will possibly overwrite files
for filename in code.list_object_names():
# Note, once #2579 is implemented, use the `node.open` method instead of the named temporary file in
# combination with the new `Transport.put_object_from_filelike`
# Since the content of the node could potentially be binary, we read the raw bytes and pass them on
with NamedTemporaryFile(mode='wb+') as handle:
handle.write(code.get_object_content(filename, mode='rb'))
handle.flush()
transport.put(handle.name, filename)
transport.chmod(code.get_local_executable(), 0o755) # rwxr-xr-x
# local_copy_list is a list of tuples, each with (uuid, dest_rel_path)
# NOTE: validation of these lists are done inside calculation.presubmit()
local_copy_list = calc_info.local_copy_list or []
remote_copy_list = calc_info.remote_copy_list or []
remote_symlink_list = calc_info.remote_symlink_list or []
provenance_exclude_list = calc_info.provenance_exclude_list or []
for uuid, filename, target in local_copy_list:
logger.debug(
f'[submission of calculation {node.uuid}] copying local file/folder to {target}'
)
try:
data_node = load_node(uuid=uuid)
except exceptions.NotExistent:
data_node = _find_data_node(inputs, uuid) if inputs else None
if data_node is None:
logger.warning(
f'failed to load Node<{uuid}> specified in the `local_copy_list`'
)
else:
dirname = os.path.dirname(target)
if dirname:
os.makedirs(os.path.join(folder.abspath, dirname),
exist_ok=True)
with folder.open(target, 'wb') as handle:
with data_node.open(filename, 'rb') as source:
shutil.copyfileobj(source, handle)
provenance_exclude_list.append(target)
# In a dry_run, the working directory is the raw input folder, which will already contain these resources
if not dry_run:
for filename in folder.get_content_list():
logger.debug(
f'[submission of calculation {node.pk}] copying file/folder {filename}...'
)
transport.put(folder.get_abs_path(filename), filename)
for (remote_computer_uuid, remote_abs_path,
dest_rel_path) in remote_copy_list:
if remote_computer_uuid == computer.uuid:
logger.debug(
'[submission of calculation {}] copying {} remotely, directly on the machine {}'
.format(node.pk, dest_rel_path, computer.label))
try:
transport.copy(remote_abs_path, dest_rel_path)
except (IOError, OSError):
logger.warning(
'[submission of calculation {}] Unable to copy remote resource from {} to {}! '
'Stopping.'.format(node.pk, remote_abs_path,
dest_rel_path))
raise
else:
raise NotImplementedError(
'[submission of calculation {}] Remote copy between two different machines is '
'not implemented yet'.format(node.pk))
for (remote_computer_uuid, remote_abs_path,
dest_rel_path) in remote_symlink_list:
if remote_computer_uuid == computer.uuid:
logger.debug(
'[submission of calculation {}] copying {} remotely, directly on the machine {}'
.format(node.pk, dest_rel_path, computer.label))
try:
transport.symlink(remote_abs_path, dest_rel_path)
except (IOError, OSError):
logger.warning(
'[submission of calculation {}] Unable to create remote symlink from {} to {}! '
'Stopping.'.format(node.pk, remote_abs_path,
dest_rel_path))
raise
else:
raise IOError(
f'It is not possible to create a symlink between two different machines for calculation {node.pk}'
)
else:
if remote_copy_list:
with open(os.path.join(workdir, '_aiida_remote_copy_list.txt'),
'w') as handle:
for remote_computer_uuid, remote_abs_path, dest_rel_path in remote_copy_list:
handle.write(
'would have copied {} to {} in working directory on remote {}'
.format(remote_abs_path, dest_rel_path,
computer.label))
if remote_symlink_list:
with open(os.path.join(workdir, '_aiida_remote_symlink_list.txt'),
'w') as handle:
for remote_computer_uuid, remote_abs_path, dest_rel_path in remote_symlink_list:
handle.write(
'would have created symlinks from {} to {} in working directory on remote {}'
.format(remote_abs_path, dest_rel_path,
computer.label))
# Loop recursively over content of the sandbox folder copying all that are not in `provenance_exclude_list`. Note
# that directories are not created explicitly. The `node.put_object_from_filelike` call will create intermediate
# directories for nested files automatically when needed. This means though that empty folders in the sandbox or
# folders that would be empty when considering the `provenance_exclude_list` will *not* be copied to the repo. The
# advantage of this explicit copying instead of deleting the files from `provenance_exclude_list` from the sandbox
# first before moving the entire remaining content to the node's repository, is that in this way we are guaranteed
# not to accidentally move files to the repository that should not go there at all cost. Note that all entries in
# the provenance exclude list are normalized first, just as the paths that are in the sandbox folder, otherwise the
# direct equality test may fail, e.g.: './path/file.txt' != 'path/file.txt' even though they reference the same file
provenance_exclude_list = [
os.path.normpath(entry) for entry in provenance_exclude_list
]
for root, _, filenames in os.walk(folder.abspath):
for filename in filenames:
filepath = os.path.join(root, filename)
relpath = os.path.normpath(
os.path.relpath(filepath, folder.abspath))
if relpath not in provenance_exclude_list:
with open(filepath, 'rb') as handle:
node._repository.put_object_from_filelike(handle,
relpath,
'wb',
force=True) # pylint: disable=protected-access
if not dry_run:
# Make sure that attaching the `remote_folder` with a link is the last thing we do. This gives the biggest
# chance of making this method idempotent. That is to say, if a runner gets interrupted during this action, it
# will simply retry the upload, unless we got here and managed to link it up, in which case we move to the next
# task. Because in that case, the check for the existence of this link at the top of this function will exit
# early from this command.
remotedata = RemoteData(computer=computer, remote_path=workdir)
remotedata.add_incoming(node,
link_type=LinkType.CREATE,
link_label='remote_folder')
remotedata.store()
def test_get_set_state(self):
"""Test the `get_state` and `set_state` method."""
node = CalcJobNode(computer=self.computer, )
self.assertEqual(node.get_state(), None)
with self.assertRaises(ValueError):
node.set_state('INVALID')
node.set_state(CalcJobState.UPLOADING)
self.assertEqual(node.get_state(), CalcJobState.UPLOADING)
# Setting an illegal calculation job state, the `get_state` should not fail but return `None`
node.set_attribute(node.CALC_JOB_STATE_KEY, 'INVALID')
self.assertEqual(node.get_state(), None)
def generate_calcjob_node(self,
entry_point_name,
retrieved,
computer_name="localhost",
attributes=None):
"""Fixture to generate a mock `CalcJobNode` for testing parsers.
Parameters
----------
entry_point_name : str
entry point name of the calculation class
retrieved : aiida.orm.FolderData
containing the file(s) to be parsed
computer_name : str
used to get or create a ``Computer``, by default 'localhost'
attributes : None or dict
any additional attributes to set on the node
Returns
-------
aiida.orm.CalcJobNode
instance with the `retrieved` node linked as outgoing
"""
from aiida.common.links import LinkType
from aiida.orm import CalcJobNode
from aiida.plugins.entry_point import format_entry_point_string
process = self.get_calc_cls(entry_point_name)
computer = self.get_or_create_computer(computer_name)
entry_point = format_entry_point_string("aiida.calculations",
entry_point_name)
node = CalcJobNode(computer=computer, process_type=entry_point)
node.set_options({
k: v.default() if callable(v.default) else v.default
for k, v in process.spec_options.items() if v.has_default()
})
node.set_option("resources", {
"num_machines": 1,
"num_mpiprocs_per_machine": 1
})
node.set_option("max_wallclock_seconds", 1800)
if attributes:
node.set_attributes(attributes)
node.store()
retrieved.add_incoming(node,
link_type=LinkType.CREATE,
link_label="retrieved")
retrieved.store()
return node
def _fixture_calc_job_node(entry_point_name, computer, test_name, attributes=None):
"""Fixture to generate a mock `CalcJobNode` for testing parsers.
:param entry_point_name: entry point name of the calculation class
:param computer: a `Computer` instance
:param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder
:param attributes: any optional attributes to set on the node
:return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node
"""
from aiida.common.links import LinkType
from aiida.orm import CalcJobNode, FolderData
from aiida.plugins.entry_point import format_entry_point_string
entry_point = format_entry_point_string('aiida.calculations', entry_point_name)
node = CalcJobNode(computer=computer, process_type=entry_point)
node.set_attribute('input_filename', 'aiida.in')
node.set_attribute('output_filename', 'aiida.out')
node.set_attribute('error_filename', 'aiida.err')
node.set_option('resources', {'num_machines': 1, 'num_mpiprocs_per_machine': 1})
node.set_option('max_wallclock_seconds', 1800)
if attributes:
node.set_attribute_many(attributes)
node.store()
basepath = os.path.dirname(os.path.abspath(__file__))
filepath = os.path.join(basepath, 'parsers', 'fixtures', entry_point_name[len('codtools.'):], test_name)
retrieved = FolderData()
retrieved.put_object_from_tree(filepath)
retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved')
retrieved.store()
return node
def _inner(file_path, input_settings=None):
# Create a test computer
computer = localhost
process_type = 'aiida.calculations:{}'.format('vasp.vasp')
node = CalcJobNode(computer=computer, process_type=process_type)
node.set_attribute('input_filename', 'INCAR')
node.set_attribute('output_filename', 'OUTCAR')
node.set_attribute('error_filename', 'aiida.err')
node.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
node.set_option('max_wallclock_seconds', 1800)
if input_settings is None:
input_settings = {}
settings = Dict(dict=input_settings)
node.add_incoming(settings,
link_type=LinkType.INPUT_CALC,
link_label='settings')
settings.store()
node.store()
# Create a `FolderData` that will represent the `retrieved` folder. Store the test
# output fixture in there and link it.
retrieved = FolderData()
retrieved.put_object_from_tree(file_path)
retrieved.add_incoming(node,
link_type=LinkType.CREATE,
link_label='retrieved')
retrieved.store()
return node
def _generate_calc_job_node(
entry_point_name,
results_folder,
inputs=None,
computer=None,
outputs=None,
outfile_override=None,
):
"""
Generate a CalcJob node with fake retrieved node in the
tests/data
"""
calc_class = CalculationFactory(entry_point_name)
entry_point = format_entry_point_string('aiida.calculations',
entry_point_name)
builder = calc_class.get_builder()
if not computer:
computer = db_test_app.localhost
node = CalcJobNode(computer=computer, process_type=entry_point)
# Monkypatch the inputs
if inputs is not None:
inputs = AttributeDict(inputs)
node.__dict__['inputs'] = inputs
# Add direct inputs, pseudos are omitted
for k, v in inputs.items():
if isinstance(v, Node):
if not v.is_stored:
v.store()
node.add_incoming(v,
link_type=LinkType.INPUT_CALC,
link_label=k)
options = builder.metadata.options
options.update(inputs.metadata.options)
node.set_attribute('input_filename', options.input_filename)
node.set_attribute('seedname', options.seedname)
node.set_attribute('output_filename', options.output_filename)
node.set_attribute('error_filename', 'aiida.err')
node.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
node.set_option('max_wallclock_seconds', 1800)
node.store()
filepath = this_folder.parent / 'data' / results_folder
retrieved = FolderData()
retrieved.put_object_from_tree(str(filepath.resolve()))
# Apply overriding output files
if outfile_override is not None:
for key, content in outfile_override.items():
if content is None:
retrieved.delete_object(key)
continue
buf = BytesIO(content.encode())
retrieved.put_object_from_filelike(buf, key)
retrieved.add_incoming(node,
link_type=LinkType.CREATE,
link_label='retrieved')
retrieved.store()
if outputs is not None:
for label, out_node in outputs.items():
out_node.add_incoming(node,
link_type=LinkType.CREATE,
link_label=label)
if not out_node.is_stored:
out_node.store()
return node
def generate_calcjob_node(
self,
entry_point_name,
retrieved=None,
computer_name="localhost",
options=None,
mark_completed=False,
remote_path=None,
input_nodes=None,
):
"""Fixture to generate a mock `CalcJobNode` for testing parsers.
Parameters
----------
entry_point_name : str
entry point name of the calculation class
retrieved : aiida.orm.FolderData
containing the file(s) to be parsed
computer_name : str
used to get or create a ``Computer``, by default 'localhost'
options : None or dict
any additional metadata options to set on the node
remote_path : str
path to a folder on the computer
mark_completed : bool
if True, set the process state to finished, and the exit_status = 0
input_nodes: dict
mapping of link label to node
Returns
-------
aiida.orm.CalcJobNode
instance with the `retrieved` node linked as outgoing
"""
from aiida.common.links import LinkType
from aiida.engine import ExitCode, ProcessState
from aiida.orm import CalcJobNode, Node, RemoteData
from aiida.plugins.entry_point import format_entry_point_string
process = self.get_calc_cls(entry_point_name)
computer = self.get_or_create_computer(computer_name)
entry_point = format_entry_point_string("aiida.calculations",
entry_point_name)
calc_node = CalcJobNode(computer=computer, process_type=entry_point)
calc_node.set_options({
k: v.default() if callable(v.default) else v.default
for k, v in process.spec_options.items() if v.has_default()
})
calc_node.set_option("resources", {
"num_machines": 1,
"num_mpiprocs_per_machine": 1
})
calc_node.set_option("max_wallclock_seconds", 1800)
if options:
calc_node.set_options(options)
if mark_completed:
calc_node.set_process_state(ProcessState.FINISHED)
calc_node.set_exit_status(ExitCode().status)
if input_nodes is not None:
for label, in_node in input_nodes.items():
in_node_map = in_node
if isinstance(in_node, Node):
in_node_map = {None: in_node_map}
for sublabel, in_node in in_node_map.items():
in_node.store()
link_label = (label if sublabel is None else
"{}__{}".format(label, sublabel))
calc_node.add_incoming(in_node,
link_type=LinkType.INPUT_CALC,
link_label=link_label)
calc_node.store()
if retrieved is not None:
retrieved.add_incoming(calc_node,
link_type=LinkType.CREATE,
link_label="retrieved")
retrieved.store()
if remote_path is not None:
remote = RemoteData(remote_path=remote_path, computer=computer)
remote.add_incoming(calc_node,
link_type=LinkType.CREATE,
link_label="remote_folder")
remote.store()
return calc_node
