def test_parse_insufficient_data(generate_process):
"""Test the scheduler output parsing logic in `CalcJob.parse`.
Here we check explicitly that the parsing does not except even if the required information is not available.
"""
process = generate_process()
retrieved = orm.FolderData().store()
retrieved.add_incoming(process.node,
link_label='retrieved',
link_type=LinkType.CREATE)
process.parse()
filename_stderr = process.node.get_option('scheduler_stderr')
filename_stdout = process.node.get_option('scheduler_stdout')
# The scheduler parsing requires three resources of information, the `detailed_job_info` dictionary which is
# stored as an attribute on the calculation job node and the output of the stdout and stderr which are both
# stored in the repository. In this test, we haven't created these on purpose. This should not except the
# process but should log a warning, so here we check that those expected warnings are attached to the node
logs = [log.message for log in orm.Log.objects.get_logs_for(process.node)]
expected_logs = [
'could not parse scheduler output: the `detailed_job_info` attribute is missing',
f'could not parse scheduler output: the `{filename_stderr}` file is missing',
f'could not parse scheduler output: the `{filename_stdout}` file is missing'
]
for log in expected_logs:
assert log in logs
def test_parser_get_outputs_for_parsing(self):
"""Make sure that the `get_output_for_parsing` method returns the correct output nodes."""
ArithmeticAddCalculation.define = CustomCalcJob.define
node = orm.CalcJobNode(computer=self.computer,
process_type=CustomCalcJob.build_process_type())
node.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
node.set_option('max_wallclock_seconds', 1800)
node.store()
retrieved = orm.FolderData().store()
retrieved.add_incoming(node,
link_type=LinkType.CREATE,
link_label='retrieved')
output = orm.Data().store()
output.add_incoming(node,
link_type=LinkType.CREATE,
link_label='output')
parser = ArithmeticAddParser(node)
outputs_for_parsing = parser.get_outputs_for_parsing()
self.assertIn('retrieved', outputs_for_parsing)
self.assertEqual(outputs_for_parsing['retrieved'].uuid, retrieved.uuid)
self.assertIn('output', outputs_for_parsing)
self.assertEqual(outputs_for_parsing['output'].uuid, output.uuid)
def test_parse_not_implemented(generate_process):
"""Test the scheduler output parsing logic in `CalcJob.parse`.
Here we check explicitly that the parsing does not except even if the scheduler does not implement the method.
"""
process = generate_process()
filename_stderr = process.node.get_option('scheduler_stderr')
filename_stdout = process.node.get_option('scheduler_stdout')
retrieved = orm.FolderData()
retrieved.put_object_from_filelike(io.StringIO('\n'),
filename_stderr,
mode='w')
retrieved.put_object_from_filelike(io.StringIO('\n'),
filename_stdout,
mode='w')
retrieved.store()
retrieved.add_incoming(process.node,
link_label='retrieved',
link_type=LinkType.CREATE)
process.node.set_attribute('detailed_job_info', {})
process.parse()
# The `DirectScheduler` at this point in time does not implement the `parse_output` method. Instead of raising
# a warning message should be logged. We verify here that said message is present.
logs = [log.message for log in orm.Log.objects.get_logs_for(process.node)]
expected_logs = [
'`DirectScheduler` does not implement scheduler output parsing'
]
for log in expected_logs:
assert log in logs
def test_parse_scheduler_excepted(process, monkeypatch):
"""Test the scheduler output parsing logic in `CalcJob.parse`.
Here we check explicitly the case where the `Scheduler.parse_output` method excepts
"""
from aiida.schedulers.plugins.direct import DirectScheduler
retrieved = orm.FolderData().store()
retrieved.add_incoming(process.node, link_label='retrieved', link_type=LinkType.CREATE)
process.node.set_attribute('detailed_job_info', {})
filename_stderr = process.node.get_option('scheduler_stderr')
filename_stdout = process.node.get_option('scheduler_stdout')
with retrieved.open(filename_stderr, 'w') as handle:
handle.write('\n')
with retrieved.open(filename_stdout, 'w') as handle:
handle.write('\n')
msg = 'crash'
def raise_exception(*args, **kwargs):
raise RuntimeError(msg)
# Monkeypatch the `DirectScheduler.parse_output` to raise an exception
monkeypatch.setattr(DirectScheduler, 'parse_output', raise_exception)
process.parse()
logs = [log.message for log in orm.Log.objects.get_logs_for(process.node)]
expected_logs = [f'the `parse_output` method of the scheduler excepted: {msg}']
for log in expected_logs:
assert log in logs
def setUpClass(cls, *args, **kwargs):
super().setUpClass(*args, **kwargs)
node = orm.Data()
cls.ATTR_KEY_ONE = 'a'
cls.ATTR_VAL_ONE = '1'
cls.ATTR_KEY_TWO = 'b'
cls.ATTR_VAL_TWO = 'test'
node.set_attribute_many({cls.ATTR_KEY_ONE: cls.ATTR_VAL_ONE, cls.ATTR_KEY_TWO: cls.ATTR_VAL_TWO})
cls.EXTRA_KEY_ONE = 'x'
cls.EXTRA_VAL_ONE = '2'
cls.EXTRA_KEY_TWO = 'y'
cls.EXTRA_VAL_TWO = 'other'
node.set_extra_many({cls.EXTRA_KEY_ONE: cls.EXTRA_VAL_ONE, cls.EXTRA_KEY_TWO: cls.EXTRA_VAL_TWO})
node.store()
cls.node = node
# Set up a FolderData for the node repo cp tests.
folder_node = orm.FolderData()
cls.content_file1 = 'nobody expects'
cls.content_file2 = 'the minister of silly walks'
cls.key_file1 = 'some/nested/folder/filename.txt'
cls.key_file2 = 'some_other_file.txt'
folder_node.put_object_from_filelike(io.StringIO(cls.content_file1), cls.key_file1)
folder_node.put_object_from_filelike(io.StringIO(cls.content_file2), cls.key_file2)
folder_node.store()
cls.folder_node = folder_node
def generate_inputs():
"""Minimal input for pw2wannier90 calculations."""
basepath = os.path.dirname(os.path.abspath(__file__))
nnkp_filepath = os.path.join(basepath, 'fixtures', 'pw2wannier90', 'inputs', 'aiida.nnkp')
parameters = {
'inputpp': {
'write_amn': False,
'write_mmn': False,
'write_unk': False,
'scdm_proj': True,
'scdm_entanglement': 'isolated',
}
}
settings = {'ADDITIONAL_RETRIEVE_LIST': ['*.amn', '*.mmn', '*.eig']}
# Since we don't actually run pw2wannier.x, we only pretend to have the output folder
# of a parent pw.x calculation. The nnkp file, instead, is real.
inputs = {
'parent_folder': orm.FolderData().store(),
'nnkp_file': orm.SinglefileData(file=nnkp_filepath).store(),
'parameters': orm.Dict(dict=parameters),
'settings': orm.Dict(dict=settings),
}
return AttributeDict(inputs)
def populate_restapi_database(clear_database_before_test):
"""Populates the database with a considerable set of nodes to test the restAPI"""
# pylint: disable=unused-argument
from aiida import orm
struct_forcif = orm.StructureData().store()
orm.StructureData().store()
orm.StructureData().store()
orm.Dict().store()
orm.Dict().store()
orm.CifData(ase=struct_forcif.get_ase()).store()
orm.KpointsData().store()
orm.FolderData().store()
orm.CalcFunctionNode().store()
orm.CalcJobNode().store()
orm.CalcJobNode().store()
orm.WorkFunctionNode().store()
orm.WorkFunctionNode().store()
orm.WorkChainNode().store()
def dos_inputs():
# The DosParser doesn't really need to access the parent folder, but we'd like to make the inputs as realistic
# as possible, so we create an empty FolderData and attach it as an input to the current CalcJobNode.
inputs = {
'parent_folder': orm.FolderData().store(),
}
return AttributeDict(inputs)
def test_validate_transfer_inputs(aiida_localhost, tmp_path, temp_dir):
"""Test the `TransferCalculation` validators."""
from aiida.orm import Computer
from aiida.calculations.transfer import check_node_type, validate_transfer_inputs
fake_localhost = Computer(
label='localhost-fake',
description='extra localhost computer set up by test',
hostname='localhost-fake',
workdir=temp_dir,
transport_type='local',
scheduler_type='direct'
)
fake_localhost.store()
fake_localhost.set_minimum_job_poll_interval(0.)
fake_localhost.configure()
inputs = {
'source_nodes': {
'unused_node': orm.RemoteData(computer=aiida_localhost, remote_path=str(tmp_path)),
},
'instructions':
orm.Dict(
dict={
'local_files': [('inexistent_node', None, None)],
'remote_files': [('inexistent_node', None, None)],
'symlink_files': [('inexistent_node', None, None)],
}
),
'metadata': {
'computer': fake_localhost
},
}
expected_list = []
expected_list.append((
f' > remote node `unused_node` points to computer `{aiida_localhost}`, '
f'not the one being used (`{fake_localhost}`)'
))
expected_list.append(check_node_type('local_files', 'inexistent_node', None, orm.FolderData))
expected_list.append(check_node_type('remote_files', 'inexistent_node', None, orm.RemoteData))
expected_list.append(check_node_type('symlink_files', 'inexistent_node', None, orm.RemoteData))
expected_list.append(' > node `unused_node` provided as inputs is not being used')
expected = '\n\n'
for addition in expected_list:
expected = expected + addition + '\n'
result = validate_transfer_inputs(inputs, None)
assert result == expected
result = check_node_type('list_name', 'node_label', None, orm.RemoteData)
expected = ' > node `node_label` requested on list `list_name` not found among inputs'
assert result == expected
result = check_node_type('list_name', 'node_label', orm.FolderData(), orm.RemoteData)
expected_type = orm.RemoteData.class_node_type
expected = f' > node `node_label`, requested on list `list_name` should be of type `{expected_type}`'
assert result == expected
def test_parse_exit_code_priority(
exit_status_scheduler,
exit_status_retrieved,
final,
generate_calc_job,
fixture_sandbox,
aiida_local_code_factory,
monkeypatch,
): # pylint: disable=too-many-arguments
"""Test the logic around exit codes in the `CalcJob.parse` method.
The `parse` method will first call the `Scheduler.parse_output` method, which if implemented by the relevant
scheduler plugin, will parse the scheduler output and potentially return an exit code. Next, the output parser
plugin is called if defined in the inputs that can also optionally return an exit code. This test is designed
to make sure the right logic is implemented in terms of which exit code should be dominant.
Scheduler result | Retrieved result | Final result | Scenario
-----------------|------------------|-----------------|-----------------------------------------
`None` | `None` | `ExitCode(0)` | Neither parser found any problem
`ExitCode(100)` | `None` | `ExitCode(100)` | Scheduler found issue, output parser does not override
`None` | `ExitCode(400)` | `ExitCode(400)` | Only output parser found a problem
`ExitCode(100)` | `ExitCode(400)` | `ExitCode(400)` | Scheduler found issue, but output parser overrides
| | | with a more specific error code
`ExitCode(100)` | `ExitCode(0)` | `ExitCode(0)` | Scheduler found issue but output parser overrides saying
| | | that despite that the calculation should be considered
| | | finished successfully.
To test this, we just need to test the `CalcJob.parse` method and the easiest way is to simply mock the scheduler
parser and output parser calls called `parse_scheduler_output` and `parse_retrieved_output`, respectively. We will
just mock them by a simple method that returns `None` or an `ExitCode`. We then check that the final exit code
returned by `CalcJob.parse` is the one we expect according to the table above.
"""
from aiida.orm import Int
def parse_scheduler_output(_, __):
if exit_status_scheduler is not None:
return ExitCode(exit_status_scheduler)
def parse_retrieved_output(_, __):
if exit_status_retrieved is not None:
return ExitCode(exit_status_retrieved)
monkeypatch.setattr(CalcJob, 'parse_scheduler_output', parse_scheduler_output)
monkeypatch.setattr(CalcJob, 'parse_retrieved_output', parse_retrieved_output)
inputs = {
'code': aiida_local_code_factory('arithmetic.add', '/bin/bash'),
'x': Int(1),
'y': Int(2),
}
process = generate_calc_job(fixture_sandbox, 'arithmetic.add', inputs, return_process=True)
retrieved = orm.FolderData().store()
retrieved.add_incoming(process.node, link_label='retrieved', link_type=LinkType.CREATE)
result = process.parse()
assert isinstance(result, ExitCode)
assert result.status == final
def test_parse_retrieved_folder(self):
"""Test the `CalcJob.parse` method when there is a retrieved folder."""
process = self.instantiate_process()
retrieved = orm.FolderData().store()
retrieved.add_incoming(process.node, link_label='retrieved', link_type=LinkType.CREATE)
exit_code = process.parse()
# The following exit code is specific to the `ArithmeticAddCalculation` we are testing here and is returned
# because the retrieved folder does not contain the output file it expects
assert exit_code == process.exit_codes.ERROR_READING_OUTPUT_FILE
def test_get_fleurinp_from_folder_data(folderpath, expected_files):
from aiida import orm
from aiida_fleur.data.fleurinp import get_fleurinp_from_folder_data
folder = orm.FolderData()
folder.put_object_from_tree(folderpath)
fleurinp = get_fleurinp_from_folder_data(folder)
assert set(fleurinp.files) == expected_files
def _generate_work_chain_node(entry_point_name,
computer,
test_name=None,
inputs=None,
attributes=None):
"""Fixture to generate a mock `WorkChainNode` for testing parsers.
:param entry_point_name: entry point name of the calculation class
:param computer: a `Computer` instance
:param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder.
:param inputs: any optional nodes to add as input links to the corrent CalcJobNode
:param attributes: any optional attributes to set on the node
:return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node
"""
from aiida import orm
from aiida.common import LinkType
from aiida.plugins.entry_point import format_entry_point_string
entry_point = format_entry_point_string('aiida.workchains',
entry_point_name)
node = orm.WorkChainNode(computer=computer, process_type=entry_point)
if attributes:
node.set_attribute_many(attributes)
if inputs:
for link_label, input_node in flatten_inputs(inputs):
input_node.store()
node.add_incoming(input_node,
link_type=LinkType.INPUT_WORK,
link_label=link_label)
if test_name is not None:
basepath = os.path.dirname(os.path.abspath(__file__))
filepath = os.path.join(basepath, 'parsers', 'fixtures',
entry_point_name[len('quantumespresso.'):],
test_name)
retrieved = orm.FolderData()
retrieved.put_object_from_tree(filepath)
retrieved.add_incoming(node,
link_type=LinkType.CREATE,
link_label='retrieved')
retrieved.store()
remote_folder = orm.RemoteData(computer=computer,
remote_path='/tmp')
remote_folder.add_incoming(node,
link_type=LinkType.CREATE,
link_label='remote_folder')
remote_folder.store()
return node
def test_process(deepmd_code):
"""Test running a calculation
note this does not test that the expected outputs are created of output parsing"""
from aiida.plugins import DataFactory, CalculationFactory
from aiida.engine import run
# Prepare input parameters
DiffParameters = DataFactory('deepmd')
parameters = DiffParameters({'ignore-case': True})
from aiida.orm import SinglefileData
file1 = SinglefileData(
file=os.path.join(tests.TEST_DIR, "input_files", 'file1.txt'))
file2 = SinglefileData(
file=os.path.join(tests.TEST_DIR, "input_files", 'file2.txt'))
# data_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'input_folder')
data_folder = orm.FolderData(
tree=os.path.join(tests.TEST_DIR, "input_files"))
# data_folder = './input_folder'
# set up calculation
inputs = {
'code': deepmd_code,
'model': orm.Dict(dict={}),
'learning_rate': orm.Dict(dict={}),
'loss': orm.Dict(dict={}),
'training': orm.Dict(dict={}),
'file': {
'box_raw':
SinglefileData(
file=os.path.join(tests.TEST_DIR, "input_files", 'file1.txt')),
'coord_raw':
SinglefileData(
file=os.path.join(tests.TEST_DIR, "input_files", 'file2.txt'))
},
'metadata': {
'dry_run': True,
'options': {
'max_wallclock_seconds': 30,
'resources': {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
}
},
},
}
result = run(CalculationFactory('dptrain'), **inputs)
computed_diff = result['deepmd'].get_content()
assert 'content1' in computed_diff
assert 'content2' in computed_diff
def test_parse_non_zero_retval(process):
"""Test the scheduler output parsing logic in `CalcJob.parse`.
This is testing the case where the `detailed_job_info` is incomplete because the call failed. This is checked
through the return value that is stored within the attribute dictionary.
"""
retrieved = orm.FolderData().store()
retrieved.add_incoming(process.node, link_label='retrieved', link_type=LinkType.CREATE)
process.node.set_attribute('detailed_job_info', {'retval': 1, 'stderr': 'accounting disabled', 'stdout': ''})
process.parse()
logs = [log.message for log in orm.Log.objects.get_logs_for(process.node)]
assert 'could not parse scheduler output: return value of `detailed_job_info` is non-zero' in logs
def get_unstored_folder_node(cls):
"""Get a "default" folder node with some data.
The node is unstored so one can add more content to it before storing it.
"""
folder_node = orm.FolderData()
cls.content_file1 = 'nobody expects'
cls.content_file2 = 'the minister of silly walks'
cls.key_file1 = 'some/nested/folder/filename.txt'
cls.key_file2 = 'some_other_file.txt'
folder_node.put_object_from_filelike(io.StringIO(cls.content_file1),
cls.key_file1)
folder_node.put_object_from_filelike(io.StringIO(cls.content_file2),
cls.key_file2)
return folder_node
def _generate_calc_job_node(entry_point_name, computer, test_name=None, inputs=None, attributes=None):
"""Fixture to generate a mock `CalcJobNode` for testing parsers.
:param entry_point_name: entry point name of the calculation class
:param computer: a `Computer` instance
:param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder.
:param inputs: any optional nodes to add as input links to the corrent CalcJobNode
:param attributes: any optional attributes to set on the node
:return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node
"""
import os
from aiida import orm
from aiida.common import LinkType
from aiida.plugins.entry_point import format_entry_point_string
entry_point = format_entry_point_string('aiida.calculations', entry_point_name)
node = orm.CalcJobNode(computer=computer, process_type=entry_point)
node.set_attribute('input_filename', 'aiida.in')
node.set_attribute('output_filename', 'aiida.out')
node.set_attribute('error_filename', 'aiida.err')
node.set_option('resources', {'num_machines': 1, 'num_mpiprocs_per_machine': 1})
node.set_option('max_wallclock_seconds', 1800)
if attributes:
node.set_attributes(attributes)
if inputs:
for link_label, input_node in flatten_inputs(inputs):
input_node.store()
node.add_incoming(input_node, link_type=LinkType.INPUT_CALC, link_label=link_label)
node.store()
if test_name is not None:
basepath = os.path.dirname(os.path.abspath(__file__))
filepath = os.path.join(basepath, 'parsers', 'fixtures', entry_point_name[len('quantumespresso.'):], test_name)
retrieved = orm.FolderData()
retrieved.put_object_from_tree(filepath)
retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved')
retrieved.store()
remote_folder = orm.RemoteData(computer=computer, remote_path='/tmp')
remote_folder.add_incoming(node, link_type=LinkType.CREATE, link_label='remote_folder')
remote_folder.store()
return node
def test_integration_transfer(aiida_localhost, tmp_path):
"""Test a default `TransferCalculation`."""
from aiida.calculations.transfer import TransferCalculation
from aiida.engine import run
content_local = 'Content of local file'
srcfile_local = tmp_path / 'file_local.txt'
srcfile_local.write_text(content_local)
srcnode_local = orm.FolderData(tree=str(tmp_path))
content_remote = 'Content of remote file'
srcfile_remote = tmp_path / 'file_remote.txt'
srcfile_remote.write_text(content_remote)
srcnode_remote = orm.RemoteData(computer=aiida_localhost, remote_path=str(tmp_path))
list_of_nodes = {}
list_of_nodes['source_local'] = srcnode_local
list_for_local = [('source_local', 'file_local.txt', 'file_local.txt')]
list_of_nodes['source_remote'] = srcnode_remote
list_for_remote = [('source_remote', 'file_remote.txt', 'file_remote.txt')]
instructions = orm.Dict(
dict={
'retrieve_files': True,
'local_files': list_for_local,
'remote_files': list_for_remote,
}
)
inputs = {'instructions': instructions, 'source_nodes': list_of_nodes, 'metadata': {'computer': aiida_localhost}}
output_nodes = run(TransferCalculation, **inputs)
output_remotedir = output_nodes['remote_folder']
output_retrieved = output_nodes['retrieved']
# Check the retrieved folder
assert sorted(output_retrieved.list_object_names()) == sorted(['file_local.txt', 'file_remote.txt'])
assert output_retrieved.get_object_content('file_local.txt') == content_local
assert output_retrieved.get_object_content('file_remote.txt') == content_remote
# Check the remote folder
assert 'file_local.txt' in output_remotedir.listdir()
assert 'file_remote.txt' in output_remotedir.listdir()
output_remotedir.getfile(relpath='file_local.txt', destpath=str(tmp_path / 'retrieved_local.txt'))
output_remotedir.getfile(relpath='file_remote.txt', destpath=str(tmp_path / 'retrieved_remote.txt'))
assert (tmp_path / 'retrieved_local.txt').read_text() == content_local
assert (tmp_path / 'retrieved_remote.txt').read_text() == content_remote
def generate_inputs_3d():
"""Return inputs that parser will expect for the 3D case."""
inputs = {
'parent_folder':
orm.FolderData().store(),
'parameters':
orm.Dict(dict={
'INPUTPP': {
'plot_num': 11
},
'PLOT': {
'iflag': 3,
}
})
}
return AttributeDict(inputs)
def test_parser_retrieved(self):
"""Verify that the `retrieved` property returns the retrieved `FolderData` node."""
node = orm.CalcJobNode(
computer=self.computer,
process_type=ArithmeticAddCalculation.build_process_type())
node.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
node.set_option('max_wallclock_seconds', 1800)
node.store()
retrieved = orm.FolderData().store()
retrieved.add_incoming(node,
link_type=LinkType.CREATE,
link_label='retrieved')
parser = ArithmeticAddParser(node)
self.assertEqual(parser.node.uuid, node.uuid)
self.assertEqual(parser.retrieved.uuid, retrieved.uuid)
def generate_inputs_polar():
"""Return inputs that parser will expect for the polar case."""
inputs = {
'parent_folder':
orm.FolderData().store(),
'parameters':
orm.Dict(
dict={
'INPUTPP': {
'plot_num': 11
},
'PLOT': {
'iflag': 4,
'e1': [[1, 1.0], [2, 0.0], [3, 0.0]],
'x0': [[1, 0.], [2, 0.], [3, 0.]],
'nx': 100
}
})
}
return AttributeDict(inputs)
def test_parse_from_node(self):
"""Test that the `parse_from_node` returns a tuple of the parsed output nodes and a calculation node.
The calculation node represents the parsing process
"""
summed = 3
output_filename = 'aiida.out'
# Mock the `CalcJobNode` which should have the `retrieved` folder containing the sum in the outputfile file
# This is the value that should be parsed into the `sum` output node
node = orm.CalcJobNode(
computer=self.computer,
process_type=ArithmeticAddCalculation.build_process_type())
node.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
node.set_option('max_wallclock_seconds', 1800)
node.set_option('output_filename', output_filename)
node.store()
retrieved = orm.FolderData()
retrieved.put_object_from_filelike(io.StringIO('{}'.format(summed)),
output_filename)
retrieved.store()
retrieved.add_incoming(node,
link_type=LinkType.CREATE,
link_label='retrieved')
for cls in [ArithmeticAddParser, SimpleArithmeticAddParser]:
result, calcfunction = cls.parse_from_node(node)
self.assertIsInstance(result['sum'], orm.Int)
self.assertEqual(result['sum'].value, summed)
self.assertIsInstance(calcfunction, orm.CalcFunctionNode)
self.assertEqual(calcfunction.exit_status, 0)
# Verify that the `retrieved_temporary_folder` keyword can be passed, there is no validation though
result, calcfunction = ArithmeticAddParser.parse_from_node(
node, retrieved_temporary_folder='/some/path')
def test_put_transfer(fixture_sandbox, aiida_localhost, generate_calc_job, tmp_path):
"""Test a default `TransferCalculation`."""
file1 = tmp_path / 'file1.txt'
file1.write_text('file 1 content')
folder = tmp_path / 'folder'
folder.mkdir()
file2 = folder / 'file2.txt'
file2.write_text('file 2 content')
data_source = orm.FolderData(tree=str(tmp_path))
entry_point_name = 'core.transfer'
list_of_files = [
('data_source', 'file1.txt', 'folder/file1.txt'),
('data_source', 'folder/file2.txt', 'file2.txt'),
]
list_of_nodes = {'data_source': data_source}
instructions = orm.Dict(dict={'retrieve_files': False, 'local_files': list_of_files})
inputs = {'instructions': instructions, 'source_nodes': list_of_nodes, 'metadata': {'computer': aiida_localhost}}
# Generate calc_info and verify basics
calc_info = generate_calc_job(fixture_sandbox, entry_point_name, inputs)
assert isinstance(calc_info, datastructures.CalcInfo)
assert isinstance(calc_info.codes_info, list)
assert len(calc_info.codes_info) == 0
assert calc_info.skip_submit
# Check that the lists were set correctly
copy_list = [
(data_source.uuid, 'file1.txt', 'folder/file1.txt'),
(data_source.uuid, 'folder/file2.txt', 'file2.txt'),
]
assert sorted(calc_info.remote_symlink_list) == sorted(list())
assert sorted(calc_info.remote_copy_list) == sorted(list())
assert sorted(calc_info.local_copy_list) == sorted(copy_list)
assert sorted(calc_info.retrieve_list) == sorted(list())
def test_cif_structure_roundtrip(self):
from aiida.tools.dbexporters.tcod import export_cif, export_values
from aiida.common.folders import SandboxFolder
import tempfile
with tempfile.NamedTemporaryFile(mode='w+') as tmpf:
tmpf.write('''
data_test
_cell_length_a 10
_cell_length_b 10
_cell_length_c 10
_cell_angle_alpha 90
_cell_angle_beta 90
_cell_angle_gamma 90
loop_
_atom_site_label
_atom_site_fract_x
_atom_site_fract_y
_atom_site_fract_z
C 0 0 0
O 0.5 0.5 0.5
''')
tmpf.flush()
a = orm.CifData(filepath=tmpf.name)
c = a.get_structure()
c.store()
pd = orm.Dict()
code = orm.Code(local_executable='test.sh')
with tempfile.NamedTemporaryFile(mode='w+') as tmpf:
tmpf.write("#/bin/bash\n\necho test run\n")
tmpf.flush()
code.put_object_from_filelike(tmpf, 'test.sh')
code.store()
calc = orm.CalcJobNode(computer=self.computer)
calc.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
calc.add_incoming(code, LinkType.INPUT_CALC, "code")
calc.set_option('environment_variables', {
'PATH': '/dev/null',
'USER': '******'
})
with tempfile.NamedTemporaryFile(mode='w+', prefix="Fe") as tmpf:
tmpf.write("<UPF version=\"2.0.1\">\nelement=\"Fe\"\n")
tmpf.flush()
upf = orm.UpfData(filepath=tmpf.name)
upf.store()
calc.add_incoming(upf, LinkType.INPUT_CALC, "upf")
with tempfile.NamedTemporaryFile(mode='w+') as tmpf:
tmpf.write("data_test")
tmpf.flush()
cif = orm.CifData(filepath=tmpf.name)
cif.store()
calc.add_incoming(cif, LinkType.INPUT_CALC, "cif")
with SandboxFolder() as fhandle:
calc.put_object_from_tree(fhandle.abspath)
calc.store()
fd = orm.FolderData()
with fd.open('_scheduler-stdout.txt', 'w') as fhandle:
fhandle.write(u"standard output")
with fd.open('_scheduler-stderr.txt', 'w') as fhandle:
fhandle.write(u"standard error")
fd.store()
fd.add_incoming(calc, LinkType.CREATE, calc.link_label_retrieved)
pd.add_incoming(calc, LinkType.CREATE, "create1")
pd.store()
with self.assertRaises(ValueError):
export_cif(c, parameters=pd)
c.add_incoming(calc, LinkType.CREATE, "create2")
export_cif(c, parameters=pd)
values = export_values(c, parameters=pd)
values = values['0']
self.assertEquals(values['_tcod_computation_environment'],
['PATH=/dev/null\nUSER=unknown'])
self.assertEquals(values['_tcod_computation_command'],
['cd 1; ./_aiidasubmit.sh'])
def create_provenance(self):
"""create an example provenance graph
"""
pd0 = orm.Dict()
pd0.label = 'pd0'
pd0.store()
pd1 = orm.Dict()
pd1.label = 'pd1'
pd1.store()
wc1 = orm.WorkChainNode()
wc1.set_process_state(ProcessState.RUNNING)
wc1.add_incoming(pd0,
link_type=LinkType.INPUT_WORK,
link_label='input1')
wc1.add_incoming(pd1,
link_type=LinkType.INPUT_WORK,
link_label='input2')
wc1.store()
calc1 = orm.CalcJobNode()
calc1.computer = self.computer
calc1.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
calc1.label = 'calc1'
calc1.set_process_state(ProcessState.FINISHED)
calc1.set_exit_status(0)
calc1.add_incoming(pd0,
link_type=LinkType.INPUT_CALC,
link_label='input1')
calc1.add_incoming(pd1,
link_type=LinkType.INPUT_CALC,
link_label='input2')
calc1.add_incoming(wc1,
link_type=LinkType.CALL_CALC,
link_label='call1')
calc1.store()
rd1 = orm.RemoteData()
rd1.label = 'rd1'
rd1.set_remote_path('/x/y.py')
rd1.computer = self.computer
rd1.store()
rd1.add_incoming(calc1, link_type=LinkType.CREATE, link_label='output')
pd2 = orm.Dict()
pd2.label = 'pd2'
pd2.store()
calcf1 = orm.CalcFunctionNode()
calcf1.label = 'calcf1'
calcf1.set_process_state(ProcessState.FINISHED)
calcf1.set_exit_status(200)
calcf1.add_incoming(rd1,
link_type=LinkType.INPUT_CALC,
link_label='input1')
calcf1.add_incoming(pd2,
link_type=LinkType.INPUT_CALC,
link_label='input2')
calcf1.add_incoming(wc1,
link_type=LinkType.CALL_CALC,
link_label='call2')
calcf1.store()
pd3 = orm.Dict()
pd3.label = 'pd3'
fd1 = orm.FolderData()
fd1.label = 'fd1'
pd3.add_incoming(calcf1,
link_type=LinkType.CREATE,
link_label='output1')
pd3.store()
fd1.add_incoming(calcf1,
link_type=LinkType.CREATE,
link_label='output2')
fd1.store()
pd3.add_incoming(wc1, link_type=LinkType.RETURN, link_label='output1')
fd1.add_incoming(wc1, link_type=LinkType.RETURN, link_label='output2')
return AttributeDict({
'pd0': pd0,
'pd1': pd1,
'calc1': calc1,
'rd1': rd1,
'pd2': pd2,
'calcf1': calcf1,
'pd3': pd3,
'fd1': fd1,
'wc1': wc1
})
def _generate_calc_job_node(
entry_point_name='base', computer=None, test_name=None, inputs=None, attributes=None, retrieve_temporary=None
):
"""Fixture to generate a mock `CalcJobNode` for testing parsers.
:param entry_point_name: entry point name of the calculation class
:param computer: a `Computer` instance
:param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder.
:param inputs: any optional nodes to add as input links to the corrent CalcJobNode
:param attributes: any optional attributes to set on the node
:param retrieve_temporary: optional tuple of an absolute filepath of a temporary directory and a list of
filenames that should be written to this directory, which will serve as the `retrieved_temporary_folder`.
For now this only works with top-level files and does not support files nested in directories.
:return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node.
"""
from aiida import orm
from aiida.common import LinkType
from aiida.plugins.entry_point import format_entry_point_string
if computer is None:
computer = fixture_localhost
filepath_folder = None
if test_name is not None:
basepath = os.path.dirname(os.path.abspath(__file__))
filename = os.path.join(entry_point_name[len('quantumespresso.'):], test_name)
filepath_folder = os.path.join(basepath, 'parsers', 'fixtures', filename)
filepath_input = os.path.join(filepath_folder, 'aiida.in')
entry_point = format_entry_point_string('aiida.calculations', entry_point_name)
node = orm.CalcJobNode(computer=computer, process_type=entry_point)
node.set_attribute('input_filename', 'aiida.in')
node.set_attribute('output_filename', 'aiida.out')
node.set_attribute('error_filename', 'aiida.err')
node.set_option('resources', {'num_machines': 1, 'num_mpiprocs_per_machine': 1})
node.set_option('max_wallclock_seconds', 1800)
if attributes:
node.set_attribute_many(attributes)
if filepath_folder:
from qe_tools.utils.exceptions import ParsingError
from aiida_quantumespresso.tools.pwinputparser import PwInputFile
try:
parsed_input = PwInputFile(filepath_input)
except ParsingError:
pass
else:
inputs['structure'] = parsed_input.get_structuredata()
inputs['parameters'] = orm.Dict(dict=parsed_input.namelists)
if inputs:
metadata = inputs.pop('metadata', {})
options = metadata.get('options', {})
for name, option in options.items():
node.set_option(name, option)
for link_label, input_node in flatten_inputs(inputs):
input_node.store()
node.add_incoming(input_node, link_type=LinkType.INPUT_CALC, link_label=link_label)
node.store()
if retrieve_temporary:
dirpath, filenames = retrieve_temporary
for filename in filenames:
shutil.copy(os.path.join(filepath_folder, filename), os.path.join(dirpath, filename))
if filepath_folder:
retrieved = orm.FolderData()
retrieved.put_object_from_tree(filepath_folder)
# Remove files that are supposed to be only present in the retrieved temporary folder
if retrieve_temporary:
for filename in filenames:
retrieved.delete_object(filename)
retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved')
retrieved.store()
remote_folder = orm.RemoteData(computer=computer, remote_path='/tmp')
remote_folder.add_incoming(node, link_type=LinkType.CREATE, link_label='remote_folder')
remote_folder.store()
return node
def test_complex_graph_import_export(self, temp_dir):
"""
This test checks that a small and bit complex graph can be correctly
exported and imported.
It will create the graph, store it to the database, export it to a file
and import it. In the end it will check if the initial nodes are present
at the imported graph.
"""
from aiida.common.exceptions import NotExistent
calc1 = orm.CalcJobNode()
calc1.computer = self.computer
calc1.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
calc1.label = 'calc1'
calc1.store()
pd1 = orm.Dict()
pd1.label = 'pd1'
pd1.store()
pd2 = orm.Dict()
pd2.label = 'pd2'
pd2.store()
rd1 = orm.RemoteData()
rd1.label = 'rd1'
rd1.set_remote_path('/x/y.py')
rd1.computer = self.computer
rd1.store()
rd1.add_incoming(calc1, link_type=LinkType.CREATE, link_label='link')
calc2 = orm.CalcJobNode()
calc2.computer = self.computer
calc2.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
calc2.label = 'calc2'
calc2.add_incoming(pd1,
link_type=LinkType.INPUT_CALC,
link_label='link1')
calc2.add_incoming(pd2,
link_type=LinkType.INPUT_CALC,
link_label='link2')
calc2.add_incoming(rd1,
link_type=LinkType.INPUT_CALC,
link_label='link3')
calc2.store()
fd1 = orm.FolderData()
fd1.label = 'fd1'
fd1.store()
fd1.add_incoming(calc2, link_type=LinkType.CREATE, link_label='link')
calc1.seal()
calc2.seal()
node_uuids_labels = {
calc1.uuid: calc1.label,
pd1.uuid: pd1.label,
pd2.uuid: pd2.label,
rd1.uuid: rd1.label,
calc2.uuid: calc2.label,
fd1.uuid: fd1.label
}
filename = os.path.join(temp_dir, 'export.aiida')
export([fd1], filename=filename, silent=True)
self.clean_db()
self.create_user()
import_data(filename, silent=True, ignore_unknown_nodes=True)
for uuid, label in node_uuids_labels.items():
try:
orm.load_node(uuid)
except NotExistent:
self.fail(
'Node with UUID {} and label {} was not found.'.format(
uuid, label))
def window_search_builder(test_data_dir, code_wannier90): # pylint: disable=too-many-locals,useless-suppression
"""
Sets up the process builder for window_search tests, and adds the inputs.
"""
builder = WindowSearch.get_builder()
input_folder = orm.FolderData()
input_folder_path = test_data_dir / 'wannier_input_folder'
for filename in os.listdir(input_folder_path):
input_folder.put_object_from_file(
path=str((input_folder_path / filename).resolve()), key=filename
)
builder.wannier.local_input_folder = input_folder
builder.wannier.code = code_wannier90
builder.code_tbmodels = orm.Code.get_from_string('tbmodels')
builder.model_evaluation_workflow = BandDifferenceModelEvaluation
# print(builder.model_evaluation.dynamic)
builder.model_evaluation = {
'code_bands_inspect': orm.Code.get_from_string('bands_inspect'),
}
builder.reference_bands = read(test_data_dir / 'bands.hdf5')
initial_window = orm.List()
initial_window.extend([-4.5, -4, 6.5, 16])
builder.initial_window = initial_window
builder.window_tol = orm.Float(1.5)
a = 3.2395 # pylint: disable=invalid-name
structure = orm.StructureData()
structure.set_pymatgen_structure(
pymatgen.Structure(
lattice=[[0, a, a], [a, 0, a], [a, a, 0]],
species=['In', 'Sb'],
coords=[[0] * 3, [0.25] * 3]
)
)
builder.structure = structure
wannier_parameters = orm.Dict(
dict=dict(
num_wann=14,
num_bands=36,
dis_num_iter=1000,
num_iter=0,
spinors=True,
mp_grid=[6, 6, 6],
)
)
builder.wannier.parameters = wannier_parameters
builder.wannier.metadata.options = {
'resources': {
'num_machines': 1,
'tot_num_mpiprocs': 1
},
'withmpi': False
}
builder.symmetries = orm.SinglefileData(
file=str((test_data_dir / 'symmetries.hdf5').resolve())
)
slice_idx = orm.List()
slice_idx.extend([0, 2, 3, 1, 5, 6, 4, 7, 9, 10, 8, 12, 13, 11])
builder.slice_idx = slice_idx
k_values = [
x if x <= 0.5 else -1 + x
for x in np.linspace(0, 1, 6, endpoint=False)
]
k_points = [
list(reversed(k)) for k in itertools.product(k_values, repeat=3)
]
wannier_bands = orm.BandsData()
wannier_bands.set_kpoints(k_points)
# Just let every energy window be valid.
wannier_bands.set_bands(np.array([[0] * 14] * len(k_points)))
builder.wannier_bands = wannier_bands
return builder
def inner(window_values, slice_, symmetries):
builder = RunWindow.get_builder()
input_folder = orm.FolderData()
input_folder_path = test_data_dir / 'wannier_input_folder'
for filename in os.listdir(input_folder_path):
input_folder.put_object_from_file(path=str(
(input_folder_path / filename).resolve()),
key=filename)
builder.wannier.local_input_folder = input_folder
builder.wannier.code = code_wannier90
builder.code_tbmodels = orm.Code.get_from_string('tbmodels')
builder.model_evaluation_workflow = BandDifferenceModelEvaluation
builder.reference_bands = read(test_data_dir / 'bands.hdf5')
builder.model_evaluation = {
'code_bands_inspect': orm.Code.get_from_string('bands_inspect'),
}
window = orm.List(list=window_values)
builder.window = window
k_values = [
x if x <= 0.5 else -1 + x
for x in np.linspace(0, 1, 6, endpoint=False)
]
k_points = [
list(reversed(k)) for k in itertools.product(k_values, repeat=3)
]
wannier_kpoints = orm.KpointsData()
wannier_kpoints.set_kpoints(k_points)
builder.wannier.kpoints = wannier_kpoints
wannier_bands = orm.BandsData()
wannier_bands.set_kpoints(k_points)
# Just let every energy window be valid.
wannier_bands.set_bands(
np.array([[-20] * 10 + [-0.5] * 7 + [0.5] * 7 + [20] * 12] *
len(k_points)))
builder.wannier_bands = wannier_bands
a = 3.2395 # pylint: disable=invalid-name
structure = orm.StructureData()
structure.set_pymatgen_structure(
pymatgen.Structure(lattice=[[0, a, a], [a, 0, a], [a, a, 0]],
species=['In', 'Sb'],
coords=[[0] * 3, [0.25] * 3]))
builder.structure = structure
wannier_parameters = orm.Dict(dict=dict(
num_wann=14,
num_bands=36,
dis_num_iter=1000,
num_iter=0,
spinors=True,
mp_grid=[6, 6, 6],
))
builder.wannier.parameters = wannier_parameters
builder.wannier.metadata.options = {
'resources': {
'num_machines': 1,
'tot_num_mpiprocs': 1
},
'withmpi': False
}
if symmetries:
builder.symmetries = orm.SinglefileData(
file=str(test_data_dir / 'symmetries.hdf5'))
if slice_:
slice_idx = orm.List()
slice_idx.extend([0, 2, 3, 1, 5, 6, 4, 7, 9, 10, 8, 12, 13, 11])
builder.slice_idx = slice_idx
return builder
def setUpClass(cls, *args, **kwargs): # pylint: disable=too-many-locals, too-many-statements
"""
Basides the standard setup we need to add few more objects in the
database to be able to explore different requests/filters/orderings etc.
"""
# call parent setUpClass method
super(RESTApiTestCase, cls).setUpClass()
# connect the app and the api
# Init the api by connecting it the the app (N.B. respect the following
# order, api.__init__)
kwargs = dict(PREFIX=cls._url_prefix,
PERPAGE_DEFAULT=cls._PERPAGE_DEFAULT,
LIMIT_DEFAULT=cls._LIMIT_DEFAULT)
cls.app = App(__name__)
cls.app.config['TESTING'] = True
AiidaApi(cls.app, **kwargs)
# create test inputs
cell = ((2., 0., 0.), (0., 2., 0.), (0., 0., 2.))
structure = orm.StructureData(cell=cell)
structure.append_atom(position=(0., 0., 0.), symbols=['Ba'])
structure.store()
structure.add_comment('This is test comment.')
structure.add_comment('Add another comment.')
cif = orm.CifData(ase=structure.get_ase())
cif.store()
parameter1 = orm.Dict(dict={'a': 1, 'b': 2})
parameter1.store()
parameter2 = orm.Dict(dict={'c': 3, 'd': 4})
parameter2.store()
kpoint = orm.KpointsData()
kpoint.set_kpoints_mesh([4, 4, 4])
kpoint.store()
resources = {'num_machines': 1, 'num_mpiprocs_per_machine': 1}
calcfunc = orm.CalcFunctionNode(computer=cls.computer)
calcfunc.store()
calc = orm.CalcJobNode(computer=cls.computer)
calc.set_option('resources', resources)
calc.set_attribute('attr1', 'OK')
calc.set_attribute('attr2', 'OK')
calc.set_extra('extra1', False)
calc.set_extra('extra2', 'extra_info')
calc.add_incoming(structure,
link_type=LinkType.INPUT_CALC,
link_label='link_structure')
calc.add_incoming(parameter1,
link_type=LinkType.INPUT_CALC,
link_label='link_parameter')
aiida_in = 'The input file\nof the CalcJob node'
# Add the calcjob_inputs folder with the aiida.in file to the CalcJobNode repository
with tempfile.NamedTemporaryFile(mode='w+') as handle:
handle.write(aiida_in)
handle.flush()
handle.seek(0)
calc.put_object_from_filelike(handle,
key='calcjob_inputs/aiida.in',
force=True)
calc.store()
# create log message for calcjob
import logging
from aiida.common.log import LOG_LEVEL_REPORT
from aiida.common.timezone import now
from aiida.orm import Log
log_record = {
'time': now(),
'loggername': 'loggername',
'levelname': logging.getLevelName(LOG_LEVEL_REPORT),
'dbnode_id': calc.id,
'message': 'This is a template record message',
'metadata': {
'content': 'test'
},
}
Log(**log_record)
aiida_out = 'The output file\nof the CalcJob node'
retrieved_outputs = orm.FolderData()
# Add the calcjob_outputs folder with the aiida.out file to the FolderData node
with tempfile.NamedTemporaryFile(mode='w+') as handle:
handle.write(aiida_out)
handle.flush()
handle.seek(0)
retrieved_outputs.put_object_from_filelike(
handle, key='calcjob_outputs/aiida.out', force=True)
retrieved_outputs.store()
retrieved_outputs.add_incoming(calc,
link_type=LinkType.CREATE,
link_label='retrieved')
kpoint.add_incoming(calc,
link_type=LinkType.CREATE,
link_label='create')
calc1 = orm.CalcJobNode(computer=cls.computer)
calc1.set_option('resources', resources)
calc1.store()
dummy_computers = [{
'name': 'test1',
'hostname': 'test1.epfl.ch',
'transport_type': 'ssh',
'scheduler_type': 'pbspro',
}, {
'name': 'test2',
'hostname': 'test2.epfl.ch',
'transport_type': 'ssh',
'scheduler_type': 'torque',
}, {
'name': 'test3',
'hostname': 'test3.epfl.ch',
'transport_type': 'local',
'scheduler_type': 'slurm',
}, {
'name': 'test4',
'hostname': 'test4.epfl.ch',
'transport_type': 'ssh',
'scheduler_type': 'slurm',
}]
for dummy_computer in dummy_computers:
computer = orm.Computer(**dummy_computer)
computer.store()
# Prepare typical REST responses
cls.process_dummy_data()