def _generate_wc_job_node(entry_point_name, computer, inputs=None):
"""Fixture to generate a mock `WorkChainNode` for testing parsers.
:param entry_point_name: entry point name of the workchain class
:param computer: a `Computer` instance
:param inputs: any optional nodes to add as input links to the corrent CalcJobNode
:return: `WorkChainNode` instance attached inputs if `inputs` is defined
"""
from aiida import orm
from aiida.common import LinkType
from aiida.plugins.entry_point import format_entry_point_string
entry_point = format_entry_point_string('aiida.workflows',
entry_point_name)
node = orm.WorkChainNode(computer=computer, process_type=entry_point)
if inputs:
for link_label, input_node in flatten_inputs(inputs):
input_node.store()
node.add_incoming(input_node,
link_type=LinkType.INPUT_WORK,
link_label=link_label)
node.store()
return node
def get_possibilities(self, incomplete=''):
"""
Return a list of plugins starting with incomplete
"""
if incomplete == '':
return self.get_valid_arguments()
# If there is a chance of ambiguity we always return the entry point string in FULL format, otherwise
# return the possibilities in the same format as the incomplete. Note that this may have some unexpected
# effects. For example if incomplete equals `aiida.` or `calculations` it will be detected as the MINIMAL
# format, even though they would also be the valid beginnings of a FULL or PARTIAL format, except that we
# cannot know that for sure at this time
if self.has_potential_ambiguity:
possibilites = [
eps for eps in self.get_valid_arguments()
if eps.startswith(incomplete)
]
else:
possibilites = []
fmt = get_entry_point_string_format(incomplete)
for group, entry_point in self._entry_points:
entry_point_string = format_entry_point_string(
group, entry_point.name, fmt=fmt)
if entry_point_string.startswith(incomplete):
possibilites.append(entry_point_string)
return possibilites
def _generate_remote_data(computer, remote_path, entry_point_name=None):
"""Generate a RemoteData node, loctated at remote_path"""
from aiida.common.links import LinkType
from aiida.orm import CalcJobNode, RemoteData
from aiida.plugins.entry_point import format_entry_point_string
entry_point = format_entry_point_string('aiida.calculations',
entry_point_name)
remote = RemoteData(remote_path=remote_path)
remote.computer = computer
if entry_point_name is not None:
creator = CalcJobNode(computer=computer, process_type=entry_point)
creator.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
#creator.set_attribute('prefix', 'aiida')
remote.add_incoming(creator,
link_type=LinkType.CREATE,
link_label='remote_folder')
creator.store()
return remote
def generate_calcjob_node(
self,
entry_point_name,
retrieved,
computer_name='localhost',
attributes=None,
):
"""Fixture to generate a mock `CalcJobNode` for testing parsers.
Parameters
----------
entry_point_name : str
entry point name of the calculation class
retrieved : aiida.orm.FolderData
containing the file(s) to be parsed
computer_name : str
used to get or create a ``Computer``, by default 'localhost'
attributes : None or dict
any additional attributes to set on the node
Returns
-------
aiida.orm.CalcJobNode
instance with the `retrieved` node linked as outgoing
"""
process = self.get_calc_cls(entry_point_name)
computer = self.get_or_create_computer(computer_name)
entry_point = format_entry_point_string('aiida.calculations',
entry_point_name)
node = CalcJobNode(computer=computer, process_type=entry_point)
node.set_options({
k: v.default() if callable(v.default) else v.default
for k, v in process.spec_options.items() if v.has_default()
})
node.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
node.set_option('max_wallclock_seconds', 1800)
if attributes:
node.set_attributes(attributes) # pylint: disable=no-member
node.store()
retrieved.add_incoming(
node,
link_type=LinkType.CREATE,
link_label='retrieved',
)
retrieved.store()
return node
def _generate_work_chain_node(entry_point_name,
computer,
test_name=None,
inputs=None,
attributes=None):
"""Fixture to generate a mock `WorkChainNode` for testing parsers.
:param entry_point_name: entry point name of the calculation class
:param computer: a `Computer` instance
:param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder.
:param inputs: any optional nodes to add as input links to the corrent CalcJobNode
:param attributes: any optional attributes to set on the node
:return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node
"""
from aiida import orm
from aiida.common import LinkType
from aiida.plugins.entry_point import format_entry_point_string
entry_point = format_entry_point_string('aiida.workchains',
entry_point_name)
node = orm.WorkChainNode(computer=computer, process_type=entry_point)
if attributes:
node.set_attribute_many(attributes)
if inputs:
for link_label, input_node in flatten_inputs(inputs):
input_node.store()
node.add_incoming(input_node,
link_type=LinkType.INPUT_WORK,
link_label=link_label)
if test_name is not None:
basepath = os.path.dirname(os.path.abspath(__file__))
filepath = os.path.join(basepath, 'parsers', 'fixtures',
entry_point_name[len('quantumespresso.'):],
test_name)
retrieved = orm.FolderData()
retrieved.put_object_from_tree(filepath)
retrieved.add_incoming(node,
link_type=LinkType.CREATE,
link_label='retrieved')
retrieved.store()
remote_folder = orm.RemoteData(computer=computer,
remote_path='/tmp')
remote_folder.add_incoming(node,
link_type=LinkType.CREATE,
link_label='remote_folder')
remote_folder.store()
return node
def get_valid_arguments(self):
"""
Return a list of all available plugins for the groups configured for this PluginParamType instance.
If the entry point names are not unique, because there are multiple groups that contain an entry
point that has an identical name, we need to prefix the names with the full group name
:returns: list of valid entry point strings
"""
if self.has_potential_ambiguity:
fmt = EntryPointFormat.FULL
return sorted([format_entry_point_string(group, ep.name, fmt=fmt) for group, ep in self._entry_points])
return sorted(self._entry_point_names)
def _generate_calc_job_node(entry_point_name, computer, test_name=None, inputs=None, attributes=None):
"""Fixture to generate a mock `CalcJobNode` for testing parsers.
:param entry_point_name: entry point name of the calculation class
:param computer: a `Computer` instance
:param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder.
:param inputs: any optional nodes to add as input links to the corrent CalcJobNode
:param attributes: any optional attributes to set on the node
:return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node
"""
import os
from aiida import orm
from aiida.common import LinkType
from aiida.plugins.entry_point import format_entry_point_string
entry_point = format_entry_point_string('aiida.calculations', entry_point_name)
node = orm.CalcJobNode(computer=computer, process_type=entry_point)
node.set_attribute('input_filename', 'aiida.in')
node.set_attribute('output_filename', 'aiida.out')
node.set_attribute('error_filename', 'aiida.err')
node.set_option('resources', {'num_machines': 1, 'num_mpiprocs_per_machine': 1})
node.set_option('max_wallclock_seconds', 1800)
if attributes:
node.set_attributes(attributes)
if inputs:
for link_label, input_node in flatten_inputs(inputs):
input_node.store()
node.add_incoming(input_node, link_type=LinkType.INPUT_CALC, link_label=link_label)
node.store()
if test_name is not None:
basepath = os.path.dirname(os.path.abspath(__file__))
filepath = os.path.join(basepath, 'parsers', 'fixtures', entry_point_name[len('quantumespresso.'):], test_name)
retrieved = orm.FolderData()
retrieved.put_object_from_tree(filepath)
retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved')
retrieved.store()
remote_folder = orm.RemoteData(computer=computer, remote_path='/tmp')
remote_folder.add_incoming(node, link_type=LinkType.CREATE, link_label='remote_folder')
remote_folder.store()
return node
def _generate_remote_data(computer,
remote_path,
entry_point_name=None,
extras_root=[]):
"""Return a `KpointsData` with a mesh of npoints in each direction."""
from aiida.common.links import LinkType
from aiida.orm import CalcJobNode, RemoteData, Dict
from aiida.plugins.entry_point import format_entry_point_string
entry_point = format_entry_point_string('aiida.calculations',
entry_point_name)
remote = RemoteData(remote_path=remote_path)
remote.computer = computer
if entry_point_name is not None:
creator = CalcJobNode(computer=computer, process_type=entry_point)
creator.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
remote.add_incoming(creator,
link_type=LinkType.CREATE,
link_label='remote_folder')
for extra in extras_root:
to_link = extra[0]
if isinstance(to_link, dict):
to_link = Dict(dict=to_link)
to_link.store()
creator.add_incoming(to_link,
link_type=LinkType.INPUT_CALC,
link_label=extra[1])
creator.store()
return remote
def _fixture_calc_job_node(entry_point_name, computer, test_name, attributes=None):
"""Fixture to generate a mock `CalcJobNode` for testing parsers.
:param entry_point_name: entry point name of the calculation class
:param computer: a `Computer` instance
:param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder
:param attributes: any optional attributes to set on the node
:return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node
"""
from aiida.common.links import LinkType
from aiida.orm import CalcJobNode, FolderData
from aiida.plugins.entry_point import format_entry_point_string
entry_point = format_entry_point_string('aiida.calculations', entry_point_name)
node = CalcJobNode(computer=computer, process_type=entry_point)
node.set_attribute('input_filename', 'aiida.in')
node.set_attribute('output_filename', 'aiida.out')
node.set_attribute('error_filename', 'aiida.err')
node.set_option('resources', {'num_machines': 1, 'num_mpiprocs_per_machine': 1})
node.set_option('max_wallclock_seconds', 1800)
if attributes:
node.set_attribute_many(attributes)
node.store()
basepath = os.path.dirname(os.path.abspath(__file__))
filepath = os.path.join(basepath, 'parsers', 'fixtures', entry_point_name[len('codtools.'):], test_name)
retrieved = FolderData()
retrieved.put_object_from_tree(filepath)
retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved')
retrieved.store()
return node
def _generate_calc_job_node(
entry_point_name='base', computer=None, test_name=None, inputs=None, attributes=None, retrieve_temporary=None
):
"""Fixture to generate a mock `CalcJobNode` for testing parsers.
:param entry_point_name: entry point name of the calculation class
:param computer: a `Computer` instance
:param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder.
:param inputs: any optional nodes to add as input links to the corrent CalcJobNode
:param attributes: any optional attributes to set on the node
:param retrieve_temporary: optional tuple of an absolute filepath of a temporary directory and a list of
filenames that should be written to this directory, which will serve as the `retrieved_temporary_folder`.
For now this only works with top-level files and does not support files nested in directories.
:return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node.
"""
from aiida import orm
from aiida.common import LinkType
from aiida.plugins.entry_point import format_entry_point_string
if computer is None:
computer = fixture_localhost
filepath_folder = None
if test_name is not None:
basepath = os.path.dirname(os.path.abspath(__file__))
filename = os.path.join(entry_point_name[len('quantumespresso.'):], test_name)
filepath_folder = os.path.join(basepath, 'parsers', 'fixtures', filename)
filepath_input = os.path.join(filepath_folder, 'aiida.in')
entry_point = format_entry_point_string('aiida.calculations', entry_point_name)
node = orm.CalcJobNode(computer=computer, process_type=entry_point)
node.set_attribute('input_filename', 'aiida.in')
node.set_attribute('output_filename', 'aiida.out')
node.set_attribute('error_filename', 'aiida.err')
node.set_option('resources', {'num_machines': 1, 'num_mpiprocs_per_machine': 1})
node.set_option('max_wallclock_seconds', 1800)
if attributes:
node.set_attribute_many(attributes)
if filepath_folder:
from qe_tools.utils.exceptions import ParsingError
from aiida_quantumespresso.tools.pwinputparser import PwInputFile
try:
parsed_input = PwInputFile(filepath_input)
except ParsingError:
pass
else:
inputs['structure'] = parsed_input.get_structuredata()
inputs['parameters'] = orm.Dict(dict=parsed_input.namelists)
if inputs:
metadata = inputs.pop('metadata', {})
options = metadata.get('options', {})
for name, option in options.items():
node.set_option(name, option)
for link_label, input_node in flatten_inputs(inputs):
input_node.store()
node.add_incoming(input_node, link_type=LinkType.INPUT_CALC, link_label=link_label)
node.store()
if retrieve_temporary:
dirpath, filenames = retrieve_temporary
for filename in filenames:
shutil.copy(os.path.join(filepath_folder, filename), os.path.join(dirpath, filename))
if filepath_folder:
retrieved = orm.FolderData()
retrieved.put_object_from_tree(filepath_folder)
# Remove files that are supposed to be only present in the retrieved temporary folder
if retrieve_temporary:
for filename in filenames:
retrieved.delete_object(filename)
retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved')
retrieved.store()
remote_folder = orm.RemoteData(computer=computer, remote_path='/tmp')
remote_folder.add_incoming(node, link_type=LinkType.CREATE, link_label='remote_folder')
remote_folder.store()
return node
def generate_calcjob_node(
self,
entry_point_name,
retrieved=None,
computer_name="localhost",
options=None,
mark_completed=False,
remote_path=None,
input_nodes=None,
):
"""Fixture to generate a mock `CalcJobNode` for testing parsers.
Parameters
----------
entry_point_name : str
entry point name of the calculation class
retrieved : aiida.orm.FolderData
containing the file(s) to be parsed
computer_name : str
used to get or create a ``Computer``, by default 'localhost'
options : None or dict
any additional metadata options to set on the node
remote_path : str
path to a folder on the computer
mark_completed : bool
if True, set the process state to finished, and the exit_status = 0
input_nodes: dict
mapping of link label to node
Returns
-------
aiida.orm.CalcJobNode
instance with the `retrieved` node linked as outgoing
"""
from aiida.common.links import LinkType
from aiida.engine import ExitCode, ProcessState
from aiida.orm import CalcJobNode, Node, RemoteData
from aiida.plugins.entry_point import format_entry_point_string
process = self.get_calc_cls(entry_point_name)
computer = self.get_or_create_computer(computer_name)
entry_point = format_entry_point_string("aiida.calculations",
entry_point_name)
calc_node = CalcJobNode(computer=computer, process_type=entry_point)
calc_node.set_options({
k: v.default() if callable(v.default) else v.default
for k, v in process.spec_options.items() if v.has_default()
})
calc_node.set_option("resources", {
"num_machines": 1,
"num_mpiprocs_per_machine": 1
})
calc_node.set_option("max_wallclock_seconds", 1800)
if options:
calc_node.set_options(options)
if mark_completed:
calc_node.set_process_state(ProcessState.FINISHED)
calc_node.set_exit_status(ExitCode().status)
if input_nodes is not None:
for label, in_node in input_nodes.items():
in_node_map = in_node
if isinstance(in_node, Node):
in_node_map = {None: in_node_map}
for sublabel, in_node in in_node_map.items():
in_node.store()
link_label = (label if sublabel is None else
"{}__{}".format(label, sublabel))
calc_node.add_incoming(in_node,
link_type=LinkType.INPUT_CALC,
link_label=link_label)
calc_node.store()
if retrieved is not None:
retrieved.add_incoming(calc_node,
link_type=LinkType.CREATE,
link_label="retrieved")
retrieved.store()
if remote_path is not None:
remote = RemoteData(remote_path=remote_path, computer=computer)
remote.add_incoming(calc_node,
link_type=LinkType.CREATE,
link_label="remote_folder")
remote.store()
return calc_node
def _generate_calc_job_node(entry_point_name,
computer,
test_name=None,
inputs=None,
attributes=None):
"""Fixture to generate a mock `CalcJobNode` for testing parsers.
:param entry_point_name: entry point name of the calculation class
:param computer: a `Computer` instance
:param test_name: relative path of directory
:param inputs: any optional nodes to add as input links to the corrent CalcJobNode
:param attributes: any optional attributes to set on the node
:return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node
"""
# pylint: disable=too-many-locals
import os
from aiida.common import LinkType
from aiida.plugins.entry_point import format_entry_point_string
entry_point = format_entry_point_string('aiida.calculations',
entry_point_name)
node = orm.CalcJobNode(computer=computer, process_type=entry_point)
node.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
node.set_option('max_wallclock_seconds', 1800)
if attributes:
node.set_attribute_many(attributes)
if inputs:
metadata = inputs.pop('metadata', {})
options = metadata.get('options', {})
for name, option in options.items():
node.set_option(name, option)
for link_label, input_node in flatten_inputs(inputs):
input_node.store()
node.add_incoming(input_node,
link_type=LinkType.INPUT_CALC,
link_label=link_label)
node.store()
if test_name is not None:
basepath = os.path.dirname(os.path.abspath(__file__))
filepath = os.path.join(basepath, 'parsers', 'fixtures', 'catmap',
test_name)
retrieved = orm.FolderData()
retrieved.put_object_from_tree(filepath)
retrieved.add_incoming(node,
link_type=LinkType.CREATE,
link_label='retrieved')
retrieved.store()
remote_folder = orm.RemoteData(computer=computer,
remote_path='/tmp')
remote_folder.add_incoming(node,
link_type=LinkType.CREATE,
link_label='remote_folder')
remote_folder.store()
return node
def _generate_calc_job_node( # pylint: disable=too-many-arguments,too-many-locals
entry_point_name,
computer,
seedname=None,
test_name=None,
inputs=None,
attributes=None,
):
"""Fixture to generate a mock `CalcJobNode` for testing parsers.
:param entry_point_name: entry point name of the calculation class
:param computer: a `Computer` instance
:param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder.
:param inputs: any optional nodes to add as input links to the corrent CalcJobNode
:param attributes: any optional attributes to set on the node
:return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node
"""
from aiida import orm
from aiida.common import LinkType
from aiida.plugins.entry_point import format_entry_point_string
entry_point = format_entry_point_string('aiida.calculations',
entry_point_name)
# If no seedname is specified, use the default 'aiida'
evaluated_seedname = seedname or 'aiida'
node = orm.CalcJobNode(computer=computer, process_type=entry_point)
node.set_attribute('input_filename',
'{}.win'.format(evaluated_seedname))
node.set_attribute('output_filename',
'{}.wout'.format(evaluated_seedname))
node.set_attribute('error_filename',
'{}.werr'.format(evaluated_seedname))
node.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
node.set_option('max_wallclock_seconds', 1800)
node.set_option('seedname', evaluated_seedname)
if attributes:
node.set_attribute_many(attributes)
if inputs:
for link_label, input_node in flatten_inputs(inputs):
input_node.store()
node.add_incoming(input_node,
link_type=LinkType.INPUT_CALC,
link_label=link_label)
node.store()
if test_name is not None:
# TODO: remove cast to 'str' when Python2 support is dropped
filepath = str(shared_datadir / test_name)
retrieved = orm.FolderData()
retrieved.put_object_from_tree(filepath)
retrieved.add_incoming(node,
link_type=LinkType.CREATE,
link_label='retrieved')
retrieved.store()
remote_folder = orm.RemoteData(computer=computer,
remote_path='/tmp')
remote_folder.add_incoming(node,
link_type=LinkType.CREATE,
link_label='remote_folder')
remote_folder.store()
return node
def _generate_calc_job_node(
entry_point_name,
results_folder,
inputs=None,
computer=None,
outputs=None,
outfile_override=None,
):
"""
Generate a CalcJob node with fake retrieved node in the
tests/data
"""
calc_class = CalculationFactory(entry_point_name)
entry_point = format_entry_point_string('aiida.calculations',
entry_point_name)
builder = calc_class.get_builder()
if not computer:
computer = db_test_app.localhost
node = CalcJobNode(computer=computer, process_type=entry_point)
# Monkypatch the inputs
if inputs is not None:
inputs = AttributeDict(inputs)
node.__dict__['inputs'] = inputs
# Add direct inputs, pseudos are omitted
for k, v in inputs.items():
if isinstance(v, Node):
if not v.is_stored:
v.store()
node.add_incoming(v,
link_type=LinkType.INPUT_CALC,
link_label=k)
options = builder.metadata.options
options.update(inputs.metadata.options)
node.set_attribute('input_filename', options.input_filename)
node.set_attribute('seedname', options.seedname)
node.set_attribute('output_filename', options.output_filename)
node.set_attribute('error_filename', 'aiida.err')
node.set_option('resources', {
'num_machines': 1,
'num_mpiprocs_per_machine': 1
})
node.set_option('max_wallclock_seconds', 1800)
node.store()
filepath = this_folder.parent / 'data' / results_folder
retrieved = FolderData()
retrieved.put_object_from_tree(str(filepath.resolve()))
# Apply overriding output files
if outfile_override is not None:
for key, content in outfile_override.items():
if content is None:
retrieved.delete_object(key)
continue
buf = BytesIO(content.encode())
retrieved.put_object_from_filelike(buf, key)
retrieved.add_incoming(node,
link_type=LinkType.CREATE,
link_label='retrieved')
retrieved.store()
if outputs is not None:
for label, out_node in outputs.items():
out_node.add_incoming(node,
link_type=LinkType.CREATE,
link_label=label)
if not out_node.is_stored:
out_node.store()
return node
