def _inner(file_path, input_settings=None): # Create a test computer computer = localhost process_type = 'aiida.calculations:{}'.format('vasp.vasp') node = CalcJobNode(computer=computer, process_type=process_type) node.set_attribute('input_filename', 'INCAR') node.set_attribute('output_filename', 'OUTCAR') #node.set_attribute('error_filename', 'aiida.err') node.set_attribute('scheduler_stderr', '_scheduler-stderr.txt') node.set_attribute('scheduler_stdout', '_scheduler-stdout.txt') node.set_option('resources', { 'num_machines': 1, 'num_mpiprocs_per_machine': 1 }) node.set_option('max_wallclock_seconds', 1800) if input_settings is None: input_settings = {} settings = Dict(dict=input_settings) node.add_incoming(settings, link_type=LinkType.INPUT_CALC, link_label='settings') settings.store() node.store() # Create a `FolderData` that will represent the `retrieved` folder. Store the test # output fixture in there and link it. retrieved = FolderData() retrieved.put_object_from_tree(file_path) retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved') retrieved.store() return node
def _fixture_calc_job_node(entry_point_name, computer, test_name, attributes=None): """Fixture to generate a mock `CalcJobNode` for testing parsers. :param entry_point_name: entry point name of the calculation class :param computer: a `Computer` instance :param test_name: relative path of directory with test output files in the `fixtures/{entry_point_name}` folder :param attributes: any optional attributes to set on the node :return: `CalcJobNode` instance with an attached `FolderData` as the `retrieved` node """ from aiida.common.links import LinkType from aiida.orm import CalcJobNode, FolderData from aiida.plugins.entry_point import format_entry_point_string entry_point = format_entry_point_string('aiida.calculations', entry_point_name) node = CalcJobNode(computer=computer, process_type=entry_point) node.set_attribute('input_filename', 'aiida.in') node.set_attribute('output_filename', 'aiida.out') node.set_attribute('error_filename', 'aiida.err') node.set_option('resources', {'num_machines': 1, 'num_mpiprocs_per_machine': 1}) node.set_option('max_wallclock_seconds', 1800) if attributes: node.set_attribute_many(attributes) node.store() basepath = os.path.dirname(os.path.abspath(__file__)) filepath = os.path.join(basepath, 'parsers', 'fixtures', entry_point_name[len('codtools.'):], test_name) retrieved = FolderData() retrieved.put_object_from_tree(filepath) retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved') retrieved.store() return node
def retrieve_calculation(calculation, transport, retrieved_temporary_folder): """Retrieve all the files of a completed job calculation using the given transport. If the job defined anything in the `retrieve_temporary_list`, those entries will be stored in the `retrieved_temporary_folder`. The caller is responsible for creating and destroying this folder. :param calculation: the instance of CalcJobNode to update. :param transport: an already opened transport to use for the retrieval. :param retrieved_temporary_folder: the absolute path to a directory in which to store the files listed, if any, in the `retrieved_temporary_folder` of the jobs CalcInfo """ logger_extra = get_dblogger_extra(calculation) workdir = calculation.get_remote_workdir() execlogger.debug('Retrieving calc {}'.format(calculation.pk), extra=logger_extra) execlogger.debug('[retrieval of calc {}] chdir {}'.format( calculation.pk, workdir), extra=logger_extra) # If the calculation already has a `retrieved` folder, simply return. The retrieval was apparently already completed # before, which can happen if the daemon is restarted and it shuts down after retrieving but before getting the # chance to perform the state transition. Upon reloading this calculation, it will re-attempt the retrieval. link_label = calculation.link_label_retrieved if calculation.get_outgoing(FolderData, link_label_filter=link_label).first(): execlogger.warning( 'CalcJobNode<{}> already has a `{}` output folder: skipping retrieval' .format(calculation.pk, link_label)) return # Create the FolderData node into which to store the files that are to be retrieved retrieved_files = FolderData() with transport: transport.chdir(workdir) # First, retrieve the files of folderdata retrieve_list = calculation.get_retrieve_list() retrieve_temporary_list = calculation.get_retrieve_temporary_list() retrieve_singlefile_list = calculation.get_retrieve_singlefile_list() with SandboxFolder() as folder: retrieve_files_from_list(calculation, transport, folder.abspath, retrieve_list) # Here I retrieved everything; now I store them inside the calculation retrieved_files.put_object_from_tree(folder.abspath) # Second, retrieve the singlefiles, if any files were specified in the 'retrieve_temporary_list' key if retrieve_singlefile_list: with SandboxFolder() as folder: _retrieve_singlefiles(calculation, transport, folder, retrieve_singlefile_list, logger_extra) # Retrieve the temporary files in the retrieved_temporary_folder if any files were # specified in the 'retrieve_temporary_list' key if retrieve_temporary_list: retrieve_files_from_list(calculation, transport, retrieved_temporary_folder, retrieve_temporary_list) # Log the files that were retrieved in the temporary folder for filename in os.listdir(retrieved_temporary_folder): execlogger.debug( "[retrieval of calc {}] Retrieved temporary file or folder '{}'" .format(calculation.pk, filename), extra=logger_extra) # Store everything execlogger.debug('[retrieval of calc {}] ' 'Storing retrieved_files={}'.format( calculation.pk, retrieved_files.pk), extra=logger_extra) retrieved_files.store() # Make sure that attaching the `retrieved` folder with a link is the last thing we do. This gives the biggest chance # of making this method idempotent. That is to say, if a runner gets interrupted during this action, it will simply # retry the retrieval, unless we got here and managed to link it up, in which case we move to the next task. retrieved_files.add_incoming(calculation, link_type=LinkType.CREATE, link_label=calculation.link_label_retrieved)
def _generate_calc_job_node( entry_point_name, results_folder, inputs=None, computer=None, outputs=None, outfile_override=None, ): """ Generate a CalcJob node with fake retrieved node in the tests/data """ calc_class = CalculationFactory(entry_point_name) entry_point = format_entry_point_string('aiida.calculations', entry_point_name) builder = calc_class.get_builder() if not computer: computer = db_test_app.localhost node = CalcJobNode(computer=computer, process_type=entry_point) # Monkypatch the inputs if inputs is not None: inputs = AttributeDict(inputs) node.__dict__['inputs'] = inputs # Add direct inputs, pseudos are omitted for k, v in inputs.items(): if isinstance(v, Node): if not v.is_stored: v.store() node.add_incoming(v, link_type=LinkType.INPUT_CALC, link_label=k) options = builder.metadata.options options.update(inputs.metadata.options) node.set_attribute('input_filename', options.input_filename) node.set_attribute('seedname', options.seedname) node.set_attribute('output_filename', options.output_filename) node.set_attribute('error_filename', 'aiida.err') node.set_option('resources', { 'num_machines': 1, 'num_mpiprocs_per_machine': 1 }) node.set_option('max_wallclock_seconds', 1800) node.store() filepath = this_folder.parent / 'data' / results_folder retrieved = FolderData() retrieved.put_object_from_tree(str(filepath.resolve())) # Apply overriding output files if outfile_override is not None: for key, content in outfile_override.items(): if content is None: retrieved.delete_object(key) continue buf = BytesIO(content.encode()) retrieved.put_object_from_filelike(buf, key) retrieved.add_incoming(node, link_type=LinkType.CREATE, link_label='retrieved') retrieved.store() if outputs is not None: for label, out_node in outputs.items(): out_node.add_incoming(node, link_type=LinkType.CREATE, link_label=label) if not out_node.is_stored: out_node.store() return node