def _generate_wc_job_node(entry_point_name, computer, inputs=None): """Fixture to generate a mock `WorkChainNode` for testing parsers. :param entry_point_name: entry point name of the workchain class :param computer: a `Computer` instance :param inputs: any optional nodes to add as input links to the corrent CalcJobNode :return: `WorkChainNode` instance attached inputs if `inputs` is defined """ from aiida import orm from aiida.common import LinkType from aiida.plugins.entry_point import format_entry_point_string entry_point = format_entry_point_string('aiida.workflows', entry_point_name) node = orm.WorkChainNode(computer=computer, process_type=entry_point) if inputs: for link_label, input_node in flatten_inputs(inputs): input_node.store() node.add_incoming(input_node, link_type=LinkType.INPUT_WORK, link_label=link_label) node.store() return node
def get_possibilities(self, incomplete=''): """ Return a list of plugins starting with incomplete """ if incomplete == '': return self.get_valid_arguments() # If there is a chance of ambiguity we always return the entry point string in FULL format, otherwise # return the possibilities in the same format as the incomplete. Note that this may have some unexpected # effects. For example if incomplete equals `aiida.` or `calculations` it will be detected as the MINIMAL # format, even though they would also be the valid beginnings of a FULL or PARTIAL format, except that we # cannot know that for sure at this time if self.has_potential_ambiguity: possibilites = [ eps for eps in self.get_valid_arguments() if eps.startswith(incomplete) ] else: possibilites = [] fmt = get_entry_point_string_format(incomplete) for group, entry_point in self._entry_points: entry_point_string = format_entry_point_string( group, entry_point.name, fmt=fmt) if entry_point_string.startswith(incomplete): possibilites.append(entry_point_string) return possibilites
def _generate_remote_data(computer, remote_path, entry_point_name=None): """Generate a RemoteData node, loctated at remote_path""" from aiida.common.links import LinkType from aiida.orm import CalcJobNode, RemoteData from aiida.plugins.entry_point import format_entry_point_string entry_point = format_entry_point_string('aiida.calculations', entry_point_name) remote = RemoteData(remote_path=remote_path) remote.computer = computer if entry_point_name is not None: creator = CalcJobNode(computer=computer, process_type=entry_point) creator.set_option('resources', { 'num_machines': 1, 'num_mpiprocs_per_machine': 1 }) #creator.set_attribute('prefix', 'aiida') remote.add_incoming(creator, link_type=LinkType.CREATE, link_label='remote_folder') creator.store() return remote
def generate_calcjob_node( self, entry_point_name, retrieved, computer_name='localhost', attributes=None, ): """Fixture to generate a mock `CalcJobNode` for testing parsers. Parameters ---------- entry_point_name : str entry point name of the calculation class retrieved : aiida.orm.FolderData containing the file(s) to be parsed computer_name : str used to get or create a ``Computer``, by default 'localhost' attributes : None or dict any additional attributes to set on the node Returns ------- aiida.orm.CalcJobNode instance with the `retrieved` node linked as outgoing """ process = self.get_calc_cls(entry_point_name) computer = self.get_or_create_computer(computer_name) entry_point = format_entry_point_string('aiida.calculations', entry_point_name) node = CalcJobNode(computer=computer, process_type=entry_point) node.set_options({ k: v.default() if callable(v.default) else v.default for k, v in process.spec_options.items() if v.has_default() }) node.set_option('resources', { 'num_machines': 1, 'num_mpiprocs_per_machine': 1 }) node.set_option('max_wallclock_seconds', 1800) if attributes: node.set_attributes(attributes) # pylint: disable=no-member node.store() retrieved.add_incoming( node, link_type=LinkType.CREATE, link_label='retrieved', ) retrieved.store() return node
def _generate_work_chain_node(entry_point_name, computer, test_name=None, inputs=None, attributes=None): """Fixture to generate a mock `WorkChainNode` for testing parsers. :param entry_point_name: entry point name of the calculation class :param computer: a `Computer` instance :param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder. :param inputs: any optional nodes to add as input links to the corrent CalcJobNode :param attributes: any optional attributes to set on the node :return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node """ from aiida import orm from aiida.common import LinkType from aiida.plugins.entry_point import format_entry_point_string entry_point = format_entry_point_string('aiida.workchains', entry_point_name) node = orm.WorkChainNode(computer=computer, process_type=entry_point) if attributes: node.set_attribute_many(attributes) if inputs: for link_label, input_node in flatten_inputs(inputs): input_node.store() node.add_incoming(input_node, link_type=LinkType.INPUT_WORK, link_label=link_label) if test_name is not None: basepath = os.path.dirname(os.path.abspath(__file__)) filepath = os.path.join(basepath, 'parsers', 'fixtures', entry_point_name[len('quantumespresso.'):], test_name) retrieved = orm.FolderData() retrieved.put_object_from_tree(filepath) retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved') retrieved.store() remote_folder = orm.RemoteData(computer=computer, remote_path='/tmp') remote_folder.add_incoming(node, link_type=LinkType.CREATE, link_label='remote_folder') remote_folder.store() return node
def get_valid_arguments(self): """ Return a list of all available plugins for the groups configured for this PluginParamType instance. If the entry point names are not unique, because there are multiple groups that contain an entry point that has an identical name, we need to prefix the names with the full group name :returns: list of valid entry point strings """ if self.has_potential_ambiguity: fmt = EntryPointFormat.FULL return sorted([format_entry_point_string(group, ep.name, fmt=fmt) for group, ep in self._entry_points]) return sorted(self._entry_point_names)
def _generate_calc_job_node(entry_point_name, computer, test_name=None, inputs=None, attributes=None): """Fixture to generate a mock `CalcJobNode` for testing parsers. :param entry_point_name: entry point name of the calculation class :param computer: a `Computer` instance :param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder. :param inputs: any optional nodes to add as input links to the corrent CalcJobNode :param attributes: any optional attributes to set on the node :return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node """ import os from aiida import orm from aiida.common import LinkType from aiida.plugins.entry_point import format_entry_point_string entry_point = format_entry_point_string('aiida.calculations', entry_point_name) node = orm.CalcJobNode(computer=computer, process_type=entry_point) node.set_attribute('input_filename', 'aiida.in') node.set_attribute('output_filename', 'aiida.out') node.set_attribute('error_filename', 'aiida.err') node.set_option('resources', {'num_machines': 1, 'num_mpiprocs_per_machine': 1}) node.set_option('max_wallclock_seconds', 1800) if attributes: node.set_attributes(attributes) if inputs: for link_label, input_node in flatten_inputs(inputs): input_node.store() node.add_incoming(input_node, link_type=LinkType.INPUT_CALC, link_label=link_label) node.store() if test_name is not None: basepath = os.path.dirname(os.path.abspath(__file__)) filepath = os.path.join(basepath, 'parsers', 'fixtures', entry_point_name[len('quantumespresso.'):], test_name) retrieved = orm.FolderData() retrieved.put_object_from_tree(filepath) retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved') retrieved.store() remote_folder = orm.RemoteData(computer=computer, remote_path='/tmp') remote_folder.add_incoming(node, link_type=LinkType.CREATE, link_label='remote_folder') remote_folder.store() return node
def _generate_remote_data(computer, remote_path, entry_point_name=None, extras_root=[]): """Return a `KpointsData` with a mesh of npoints in each direction.""" from aiida.common.links import LinkType from aiida.orm import CalcJobNode, RemoteData, Dict from aiida.plugins.entry_point import format_entry_point_string entry_point = format_entry_point_string('aiida.calculations', entry_point_name) remote = RemoteData(remote_path=remote_path) remote.computer = computer if entry_point_name is not None: creator = CalcJobNode(computer=computer, process_type=entry_point) creator.set_option('resources', { 'num_machines': 1, 'num_mpiprocs_per_machine': 1 }) remote.add_incoming(creator, link_type=LinkType.CREATE, link_label='remote_folder') for extra in extras_root: to_link = extra[0] if isinstance(to_link, dict): to_link = Dict(dict=to_link) to_link.store() creator.add_incoming(to_link, link_type=LinkType.INPUT_CALC, link_label=extra[1]) creator.store() return remote
def _fixture_calc_job_node(entry_point_name, computer, test_name, attributes=None): """Fixture to generate a mock `CalcJobNode` for testing parsers. :param entry_point_name: entry point name of the calculation class :param computer: a `Computer` instance :param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder :param attributes: any optional attributes to set on the node :return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node """ from aiida.common.links import LinkType from aiida.orm import CalcJobNode, FolderData from aiida.plugins.entry_point import format_entry_point_string entry_point = format_entry_point_string('aiida.calculations', entry_point_name) node = CalcJobNode(computer=computer, process_type=entry_point) node.set_attribute('input_filename', 'aiida.in') node.set_attribute('output_filename', 'aiida.out') node.set_attribute('error_filename', 'aiida.err') node.set_option('resources', {'num_machines': 1, 'num_mpiprocs_per_machine': 1}) node.set_option('max_wallclock_seconds', 1800) if attributes: node.set_attribute_many(attributes) node.store() basepath = os.path.dirname(os.path.abspath(__file__)) filepath = os.path.join(basepath, 'parsers', 'fixtures', entry_point_name[len('codtools.'):], test_name) retrieved = FolderData() retrieved.put_object_from_tree(filepath) retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved') retrieved.store() return node
def _generate_calc_job_node( entry_point_name='base', computer=None, test_name=None, inputs=None, attributes=None, retrieve_temporary=None ): """Fixture to generate a mock `CalcJobNode` for testing parsers. :param entry_point_name: entry point name of the calculation class :param computer: a `Computer` instance :param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder. :param inputs: any optional nodes to add as input links to the corrent CalcJobNode :param attributes: any optional attributes to set on the node :param retrieve_temporary: optional tuple of an absolute filepath of a temporary directory and a list of filenames that should be written to this directory, which will serve as the `retrieved_temporary_folder`. For now this only works with top-level files and does not support files nested in directories. :return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node. """ from aiida import orm from aiida.common import LinkType from aiida.plugins.entry_point import format_entry_point_string if computer is None: computer = fixture_localhost filepath_folder = None if test_name is not None: basepath = os.path.dirname(os.path.abspath(__file__)) filename = os.path.join(entry_point_name[len('quantumespresso.'):], test_name) filepath_folder = os.path.join(basepath, 'parsers', 'fixtures', filename) filepath_input = os.path.join(filepath_folder, 'aiida.in') entry_point = format_entry_point_string('aiida.calculations', entry_point_name) node = orm.CalcJobNode(computer=computer, process_type=entry_point) node.set_attribute('input_filename', 'aiida.in') node.set_attribute('output_filename', 'aiida.out') node.set_attribute('error_filename', 'aiida.err') node.set_option('resources', {'num_machines': 1, 'num_mpiprocs_per_machine': 1}) node.set_option('max_wallclock_seconds', 1800) if attributes: node.set_attribute_many(attributes) if filepath_folder: from qe_tools.utils.exceptions import ParsingError from aiida_quantumespresso.tools.pwinputparser import PwInputFile try: parsed_input = PwInputFile(filepath_input) except ParsingError: pass else: inputs['structure'] = parsed_input.get_structuredata() inputs['parameters'] = orm.Dict(dict=parsed_input.namelists) if inputs: metadata = inputs.pop('metadata', {}) options = metadata.get('options', {}) for name, option in options.items(): node.set_option(name, option) for link_label, input_node in flatten_inputs(inputs): input_node.store() node.add_incoming(input_node, link_type=LinkType.INPUT_CALC, link_label=link_label) node.store() if retrieve_temporary: dirpath, filenames = retrieve_temporary for filename in filenames: shutil.copy(os.path.join(filepath_folder, filename), os.path.join(dirpath, filename)) if filepath_folder: retrieved = orm.FolderData() retrieved.put_object_from_tree(filepath_folder) # Remove files that are supposed to be only present in the retrieved temporary folder if retrieve_temporary: for filename in filenames: retrieved.delete_object(filename) retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved') retrieved.store() remote_folder = orm.RemoteData(computer=computer, remote_path='/tmp') remote_folder.add_incoming(node, link_type=LinkType.CREATE, link_label='remote_folder') remote_folder.store() return node
def generate_calcjob_node( self, entry_point_name, retrieved=None, computer_name="localhost", options=None, mark_completed=False, remote_path=None, input_nodes=None, ): """Fixture to generate a mock `CalcJobNode` for testing parsers. Parameters ---------- entry_point_name : str entry point name of the calculation class retrieved : aiida.orm.FolderData containing the file(s) to be parsed computer_name : str used to get or create a ``Computer``, by default 'localhost' options : None or dict any additional metadata options to set on the node remote_path : str path to a folder on the computer mark_completed : bool if True, set the process state to finished, and the exit_status = 0 input_nodes: dict mapping of link label to node Returns ------- aiida.orm.CalcJobNode instance with the `retrieved` node linked as outgoing """ from aiida.common.links import LinkType from aiida.engine import ExitCode, ProcessState from aiida.orm import CalcJobNode, Node, RemoteData from aiida.plugins.entry_point import format_entry_point_string process = self.get_calc_cls(entry_point_name) computer = self.get_or_create_computer(computer_name) entry_point = format_entry_point_string("aiida.calculations", entry_point_name) calc_node = CalcJobNode(computer=computer, process_type=entry_point) calc_node.set_options({ k: v.default() if callable(v.default) else v.default for k, v in process.spec_options.items() if v.has_default() }) calc_node.set_option("resources", { "num_machines": 1, "num_mpiprocs_per_machine": 1 }) calc_node.set_option("max_wallclock_seconds", 1800) if options: calc_node.set_options(options) if mark_completed: calc_node.set_process_state(ProcessState.FINISHED) calc_node.set_exit_status(ExitCode().status) if input_nodes is not None: for label, in_node in input_nodes.items(): in_node_map = in_node if isinstance(in_node, Node): in_node_map = {None: in_node_map} for sublabel, in_node in in_node_map.items(): in_node.store() link_label = (label if sublabel is None else "{}__{}".format(label, sublabel)) calc_node.add_incoming(in_node, link_type=LinkType.INPUT_CALC, link_label=link_label) calc_node.store() if retrieved is not None: retrieved.add_incoming(calc_node, link_type=LinkType.CREATE, link_label="retrieved") retrieved.store() if remote_path is not None: remote = RemoteData(remote_path=remote_path, computer=computer) remote.add_incoming(calc_node, link_type=LinkType.CREATE, link_label="remote_folder") remote.store() return calc_node
def _generate_calc_job_node(entry_point_name, computer, test_name=None, inputs=None, attributes=None): """Fixture to generate a mock `CalcJobNode` for testing parsers. :param entry_point_name: entry point name of the calculation class :param computer: a `Computer` instance :param test_name: relative path of directory :param inputs: any optional nodes to add as input links to the corrent CalcJobNode :param attributes: any optional attributes to set on the node :return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node """ # pylint: disable=too-many-locals import os from aiida.common import LinkType from aiida.plugins.entry_point import format_entry_point_string entry_point = format_entry_point_string('aiida.calculations', entry_point_name) node = orm.CalcJobNode(computer=computer, process_type=entry_point) node.set_option('resources', { 'num_machines': 1, 'num_mpiprocs_per_machine': 1 }) node.set_option('max_wallclock_seconds', 1800) if attributes: node.set_attribute_many(attributes) if inputs: metadata = inputs.pop('metadata', {}) options = metadata.get('options', {}) for name, option in options.items(): node.set_option(name, option) for link_label, input_node in flatten_inputs(inputs): input_node.store() node.add_incoming(input_node, link_type=LinkType.INPUT_CALC, link_label=link_label) node.store() if test_name is not None: basepath = os.path.dirname(os.path.abspath(__file__)) filepath = os.path.join(basepath, 'parsers', 'fixtures', 'catmap', test_name) retrieved = orm.FolderData() retrieved.put_object_from_tree(filepath) retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved') retrieved.store() remote_folder = orm.RemoteData(computer=computer, remote_path='/tmp') remote_folder.add_incoming(node, link_type=LinkType.CREATE, link_label='remote_folder') remote_folder.store() return node
def _generate_calc_job_node( # pylint: disable=too-many-arguments,too-many-locals entry_point_name, computer, seedname=None, test_name=None, inputs=None, attributes=None, ): """Fixture to generate a mock `CalcJobNode` for testing parsers. :param entry_point_name: entry point name of the calculation class :param computer: a `Computer` instance :param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder. :param inputs: any optional nodes to add as input links to the corrent CalcJobNode :param attributes: any optional attributes to set on the node :return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node """ from aiida import orm from aiida.common import LinkType from aiida.plugins.entry_point import format_entry_point_string entry_point = format_entry_point_string('aiida.calculations', entry_point_name) # If no seedname is specified, use the default 'aiida' evaluated_seedname = seedname or 'aiida' node = orm.CalcJobNode(computer=computer, process_type=entry_point) node.set_attribute('input_filename', '{}.win'.format(evaluated_seedname)) node.set_attribute('output_filename', '{}.wout'.format(evaluated_seedname)) node.set_attribute('error_filename', '{}.werr'.format(evaluated_seedname)) node.set_option('resources', { 'num_machines': 1, 'num_mpiprocs_per_machine': 1 }) node.set_option('max_wallclock_seconds', 1800) node.set_option('seedname', evaluated_seedname) if attributes: node.set_attribute_many(attributes) if inputs: for link_label, input_node in flatten_inputs(inputs): input_node.store() node.add_incoming(input_node, link_type=LinkType.INPUT_CALC, link_label=link_label) node.store() if test_name is not None: # TODO: remove cast to 'str' when Python2 support is dropped filepath = str(shared_datadir / test_name) retrieved = orm.FolderData() retrieved.put_object_from_tree(filepath) retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved') retrieved.store() remote_folder = orm.RemoteData(computer=computer, remote_path='/tmp') remote_folder.add_incoming(node, link_type=LinkType.CREATE, link_label='remote_folder') remote_folder.store() return node
def _generate_calc_job_node( entry_point_name, results_folder, inputs=None, computer=None, outputs=None, outfile_override=None, ): """ Generate a CalcJob node with fake retrieved node in the tests/data """ calc_class = CalculationFactory(entry_point_name) entry_point = format_entry_point_string('aiida.calculations', entry_point_name) builder = calc_class.get_builder() if not computer: computer = db_test_app.localhost node = CalcJobNode(computer=computer, process_type=entry_point) # Monkypatch the inputs if inputs is not None: inputs = AttributeDict(inputs) node.__dict__['inputs'] = inputs # Add direct inputs, pseudos are omitted for k, v in inputs.items(): if isinstance(v, Node): if not v.is_stored: v.store() node.add_incoming(v, link_type=LinkType.INPUT_CALC, link_label=k) options = builder.metadata.options options.update(inputs.metadata.options) node.set_attribute('input_filename', options.input_filename) node.set_attribute('seedname', options.seedname) node.set_attribute('output_filename', options.output_filename) node.set_attribute('error_filename', 'aiida.err') node.set_option('resources', { 'num_machines': 1, 'num_mpiprocs_per_machine': 1 }) node.set_option('max_wallclock_seconds', 1800) node.store() filepath = this_folder.parent / 'data' / results_folder retrieved = FolderData() retrieved.put_object_from_tree(str(filepath.resolve())) # Apply overriding output files if outfile_override is not None: for key, content in outfile_override.items(): if content is None: retrieved.delete_object(key) continue buf = BytesIO(content.encode()) retrieved.put_object_from_filelike(buf, key) retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved') retrieved.store() if outputs is not None: for label, out_node in outputs.items(): out_node.add_incoming(node, link_type=LinkType.CREATE, link_label=label) if not out_node.is_stored: out_node.store() return node