def test_create_file_from_filelike_py3(self): """Test `aiida.common.folders.Folder.create_file_from_filelike` for python 3.""" unicode_string = 'unicode_string' byte_string = b'byte_string' try: tempdir = tempfile.mkdtemp() folder = Folder(tempdir) folder.create_file_from_filelike(six.StringIO(unicode_string), 'random.dat', mode='w', encoding='utf-8') folder.create_file_from_filelike(six.BytesIO(byte_string), 'random.dat', mode='wb', encoding=None) # For python three we make no exceptions, if you pass a unicode stream with binary mode, one should expect # a TypeError. Same for the inverse case of wanting to write in unicode mode but passing a byte stream with self.assertRaises(TypeError): folder.create_file_from_filelike(six.StringIO(unicode_string), 'random.dat', mode='wb') with self.assertRaises(TypeError): folder.create_file_from_filelike(six.BytesIO(byte_string), 'random.dat', mode='w') finally: shutil.rmtree(tempdir)
def test_get_abs_path_without_limit(self): from aiida.common.folders import Folder fd = Folder('/tmp') # Should not raise any exception self.assertEquals(fd.get_abs_path('test_file.txt'), '/tmp/test_file.txt')
def _collect_files(base, path=''): """ Recursively collects files from the tree, starting at a given path. """ from aiida.common.folders import Folder from aiida.common.utils import md5_file,sha1_file import os if os.path.isdir(os.path.join(base,path)): folder = Folder(os.path.join(base,path)) files_now = [] if path != '': if not path.endswith(os.sep): path = "{}{}".format(path,os.sep) if path != '': files_now.append({ 'name': path, 'type': 'folder', }) for f in sorted(folder.get_content_list()): files = _collect_files(base,path=os.path.join(path,f)) files_now.extend(files) return files_now else: with open(os.path.join(base,path)) as f: return [{ 'name': path, 'contents': f.read(), 'md5': md5_file(os.path.join(base,path)), 'sha1': sha1_file(os.path.join(base,path)), 'type': 'file', }]
def test_create_file_from_filelike(self): """Test `aiida.common.folders.Folder.create_file_from_filelike`.""" unicode_string = 'unicode_string' byte_string = b'byte_string' try: tempdir = tempfile.mkdtemp() folder = Folder(tempdir) folder.create_file_from_filelike(io.StringIO(unicode_string), 'random.dat', mode='w', encoding='utf-8') folder.create_file_from_filelike(io.BytesIO(byte_string), 'random.dat', mode='wb', encoding=None) with self.assertRaises(TypeError): folder.create_file_from_filelike(io.StringIO(unicode_string), 'random.dat', mode='wb') with self.assertRaises(TypeError): folder.create_file_from_filelike(io.BytesIO(byte_string), 'random.dat', mode='w') finally: shutil.rmtree(tempdir)
def test_create_file_from_filelike_py2(): """Test `aiida.common.folders.Folder.create_file_from_filelike` for python 2.""" unicode_string = u'unicode_string' byte_string = 'byte_string' try: tempdir = tempfile.mkdtemp() folder = Folder(tempdir) # Passing a stream with matching file mode should work ofcourse folder.create_file_from_filelike(six.StringIO(unicode_string), 'random.dat', mode='w', encoding='utf-8') folder.create_file_from_filelike(six.StringIO(byte_string), 'random.dat', mode='wb', encoding=None) # For python 2 the `create_file_from_filelike` should be able to deal with incoherent arguments, such as # the examples below where a unicode string is passed with a binary mode, or a byte stream in unicode mode. folder.create_file_from_filelike(six.StringIO(unicode_string), 'random.dat', mode='wb', encoding=None) folder.create_file_from_filelike(six.StringIO(byte_string), 'random.dat', mode='w', encoding='utf-8') finally: shutil.rmtree(tempdir)
def test_submit_test_function(clear_database_before_test, sto_calc_inputs): """ Test the ``submit_test`` method """ from aiida_castep.calculations.castep import CastepCalculation, submit_test from aiida.common.folders import Folder # Test with process class and inputs res = submit_test(CastepCalculation, **sto_calc_inputs) fcontent = Folder(res[1]).get_content_list() assert 'aiida.cell' in fcontent assert 'aiida.param' in fcontent # Nothing should change for the nested dic assert sto_calc_inputs['metadata'].get('dry_run') is not True assert sto_calc_inputs['metadata'].get('store_provenance') is not False # Test with builder builder = CastepCalculation.get_builder() builder._data = sto_calc_inputs res = submit_test(builder) fcontent = Folder(res[1]).get_content_list() assert 'aiida.cell' in fcontent assert 'aiida.param' in fcontent # Nothing should change in the process builder assert builder.metadata.get('dry_run') is not True assert builder.metadata.get('store_provenance') is not False
def prepare_for_submission(self, folder: Folder) -> CalcInfo: """Prepare the calculation for submission. :param folder: a temporary folder on the local file system. :returns: the `CalcInfo` instance """ echo_value = "fail" if self.node.get_option( "fail_calcjob") else "success" with folder.open(self.options.input_filename, "w", encoding="utf8") as handle: handle.write(f"sleep {self.inputs.time.value}\n") handle.write(f'echo "{echo_value}"\n') with folder.open(self.options.payload_filename, "wb") as handle: json.dump(self.inputs.payload.get_dict(), handle) codeinfo = CodeInfo() codeinfo.code_uuid = self.inputs.code.uuid codeinfo.stdin_name = self.options.input_filename codeinfo.stdout_name = self.options.output_filename calcinfo = CalcInfo() calcinfo.codes_info = [codeinfo] calcinfo.retrieve_list = [ self.options.output_filename, self.options.payload_filename, ] return calcinfo
def _collect_files(base, path=''): """ Recursively collects files from the tree, starting at a given path. """ from aiida.common.folders import Folder from aiida.common.utils import md5_file, sha1_file import os def get_filename(file_dict): return file_dict['name'] if os.path.isdir(os.path.join(base, path)): folder = Folder(os.path.join(base, path)) files_now = [] if path != '': if not path.endswith(os.sep): path = "{}{}".format(path, os.sep) if path != '': files_now.append({ 'name': path, 'type': 'folder', }) for f in folder.get_content_list(): files = _collect_files(base, path=os.path.join(path, f)) files_now.extend(files) return sorted(files_now, key=get_filename) elif path == '.aiida/calcinfo.json': files = [] with open(os.path.join(base, path)) as f: files.append({ 'name': path, 'contents': f.read(), 'md5': md5_file(os.path.join(base, path)), 'sha1': sha1_file(os.path.join(base, path)), 'type': 'file', }) import json with open(os.path.join(base, path)) as f: calcinfo = json.load(f) if 'local_copy_list' in calcinfo: for local_copy in calcinfo['local_copy_list']: with open(local_copy[0]) as f: files.append({ 'name': os.path.normpath(local_copy[1]), 'contents': f.read(), 'md5': md5_file(local_copy[0]), 'sha1': sha1_file(local_copy[0]), 'type': 'file', }) return files else: with open(os.path.join(base, path)) as f: return [{ 'name': path, 'contents': f.read(), 'md5': md5_file(os.path.join(base, path)), 'sha1': sha1_file(os.path.join(base, path)), 'type': 'file', }]
def test_get_abs_path_without_limit(self): """ Check that the absolute path function can get an absolute path """ folder = Folder('/tmp') # Should not raise any exception self.assertEqual(folder.get_abs_path('test_file.txt'), '/tmp/test_file.txt')
def test_prepare(vasp_calc, vasp_chgcar, vasp_wavecar, vasp_inputs, localhost_dir): """Check that preparing creates all necessary files.""" from aiida.common.folders import Folder from aiida_vasp.calcs.vasp import VaspCalculation wavecar, _ = vasp_wavecar chgcar, _ = vasp_chgcar inputs_dict = { 'gga': 'PE', 'gga_compat': False, 'lorbit': 11, 'sigma': 0.5, 'magmom': '30 * 2*0.', 'icharg': 11 } inputs = vasp_inputs(parameters=inputs_dict) inputs.charge_density = chgcar inputs.wavefunctions = wavecar calc = vasp_calc(inputs=inputs) temp_folder = Folder(str(localhost_dir.parent)) calcinfo = calc.prepare_for_submission(temp_folder) input_files = temp_folder.get_content_list() for file_name in ['INCAR', 'KPOINTS', 'POSCAR', 'POTCAR']: assert file_name in input_files assert 'EIGENVAL' in calcinfo.retrieve_list assert 'DOSCAR' in calcinfo.retrieve_list assert 'wannier90*' in calcinfo.retrieve_list assert calcinfo.codes_info[0].stdout_name == VaspCalculation._VASP_OUTPUT assert calcinfo.codes_info[0].join_files is True inputs_dict.update({'icharg': 2}) inputs = vasp_inputs(parameters=inputs_dict) inputs.charge_density = chgcar inputs.wavefunctions = wavecar calc = vasp_calc(inputs=inputs) temp_folder = Folder(str(localhost_dir.parent)) calcinfo = calc.prepare_for_submission(temp_folder) assert 'WAVECAR' in [item[1] for item in calcinfo.local_copy_list]
def output_test(pk, testname, skip_uuids_from_inputs=[]): """ This is the function that should be used to create a new test from an existing calculation. It is possible to simplify the file removing unwanted nodes. :param pk: PK of Calculation, used for test :param testname: the name of this test, used to create a new folder. The folder name will be of the form test_PLUGIN_TESTNAME, with PLUGIN substituted by the plugin name, with dots replaced by underscores. Testname can contain only digits, letters and underscores. :param skip_uuids_from_inputs: a list of UUIDs of input nodes to be skipped """ import os import json from aiida.common.folders import Folder from aiida.orm import JobCalculation from aiida.orm.utils import load_node from aiida.orm.importexport import export_tree c = load_node(pk, parent_class=JobCalculation) outfolder = "test_{}_{}".format(c.get_parser_name().replace('.', '_'), testname) if not is_valid_folder_name(outfolder): raise ValueError("The testname is invalid; it can contain only " "letters, digits or underscores") if os.path.exists(outfolder): raise ValueError("Out folder '{}' already exists".format(outfolder)) inputs = [] for node in c.get_inputs(): if node.uuid not in skip_uuids_from_inputs: inputs.append(node.dbnode) folder = Folder(outfolder) to_export = [c.dbnode] + inputs try: to_export.append(c.out.retrieved.dbnode) except AttributeError: raise ValueError("No output retrieved node; without it, we cannot " "test the parser!") export_tree(to_export, folder=folder, also_parents=False, also_calc_outputs=False) # Create an empty checks file with open(os.path.join(outfolder, '_aiida_checks.json'), 'w') as f: json.dump({}, f) for path, dirlist, filelist in os.walk(outfolder): if len(dirlist) == 0 and len(filelist) == 0: with open("{}/.gitignore".format(path), 'w') as f: f.write("# This is a placeholder file, used to make git " "store an empty folder") f.flush()
def get_temp_folder(): """Returns AiiDA folder object. Useful for calculation.submit_test() """ return Folder(tempfile.mkdtemp())
def prepare_for_submission(self, folder: Folder) -> CalcInfo: """Prepare the calculation for submission. Convert the input nodes into the corresponding input files in the format that the code will expect. In addition, define and return a `CalcInfo` instance, which is a simple data structure that contains information for the engine, for example, on what files to copy to the remote machine, what files to retrieve once it has completed, specific scheduler settings and more. :param folder: a temporary folder on the local file system. :returns: the `CalcInfo` instance """ with folder.open(self.options.input_filename, 'w', encoding='utf8') as handle: handle.write('echo $(({x} + {y}))\n'.format(x=self.inputs.x.value, y=self.inputs.y.value)) codeinfo = CodeInfo() codeinfo.code_uuid = self.inputs.code.uuid codeinfo.stdin_name = self.options.input_filename codeinfo.stdout_name = self.options.output_filename calcinfo = CalcInfo() calcinfo.codes_info = [codeinfo] calcinfo.retrieve_list = [self.options.output_filename] return calcinfo
def get_temp_folder(): """Returns AiiDA folder object. Useful for calculation.submit_test() """ from aiida.common.folders import Folder import tempfile return Folder(tempfile.mkdtemp())
def test_empty_repo_folder_export(self, temp_dir): """Check a Node's empty repository folder is exported properly""" from aiida.common.folders import Folder from aiida.tools.importexport.dbexport import export_tree node = orm.Dict().store() node_uuid = node.uuid node_repo = RepositoryFolder(section=Repository._section_name, uuid=node_uuid) # pylint: disable=protected-access self.assertTrue( node_repo.exists(), msg='Newly created and stored Node should have had an existing repository folder' ) for filename, is_file in node_repo.get_content_list(only_paths=False): abspath_filename = os.path.join(node_repo.abspath, filename) if is_file: os.remove(abspath_filename) else: shutil.rmtree(abspath_filename, ignore_errors=False) self.assertFalse( node_repo.get_content_list(), msg='Repository folder should be empty, instead the following was found: {}'.format( node_repo.get_content_list() ) ) archive_variants = { 'archive folder': os.path.join(temp_dir, 'export_tree'), 'tar archive': os.path.join(temp_dir, 'export.tar.gz'), 'zip archive': os.path.join(temp_dir, 'export.zip') } export_tree([node], folder=Folder(archive_variants['archive folder']), silent=True) export([node], filename=archive_variants['tar archive'], file_format='tar.gz', silent=True) export([node], filename=archive_variants['zip archive'], file_format='zip', silent=True) for variant, filename in archive_variants.items(): self.reset_database() node_count = orm.QueryBuilder().append(orm.Dict, project='uuid').count() self.assertEqual(node_count, 0, msg='After DB reset {} Dict Nodes was (wrongly) found'.format(node_count)) import_data(filename, silent=True) builder = orm.QueryBuilder().append(orm.Dict, project='uuid') imported_node_count = builder.count() self.assertEqual( imported_node_count, 1, msg='After {} import a single Dict Node should have been found, ' 'instead {} was/were found'.format(variant, imported_node_count) ) imported_node_uuid = builder.all()[0][0] self.assertEqual( imported_node_uuid, node_uuid, msg='The wrong UUID was found for the imported {}: ' '{}. It should have been: {}'.format(variant, imported_node_uuid, node_uuid) )
def test_prepare_for_submission(vasp2w90_calc_and_ref, tmp_path): """Test that the lwannier90 flag is written at the prepare for submission step.""" from aiida.common.folders import Folder vasp_calc, reference = vasp2w90_calc_and_ref folder = Folder(os.fspath(tmp_path)) vasp_calc.prepare_for_submission(folder) with managed_temp_file() as temp_file: vasp_calc.write_incar(temp_file) with open(temp_file, 'r') as result_incar_fo: assert result_incar_fo.readlines() == reference['incar']
def test_submit_test(clear_database_before_test, sto_calc_inputs): """ Test the ``submit_test`` method """ from aiida_castep.calculations.castep import CastepCalculation from aiida.common.folders import Folder builder = CastepCalculation.get_builder() builder._update(sto_calc_inputs) res = CastepCalculation.submit_test(builder) fcontent = Folder(res[1]).get_content_list() assert 'aiida.cell' in fcontent assert 'aiida.param' in fcontent
def test_incar_validate(vasp_calc, vasp_inputs, localhost_dir): """Test incar with invaid tags raises exception""" from aiida.common import InputValidationError from aiida.common.folders import Folder inputs_dict = { 'gga': 'PE', 'smear': 3 # <- Invalid tag } inputs = vasp_inputs(parameters=inputs_dict) calc = vasp_calc(inputs=inputs) temp_folder = Folder(str(localhost_dir.parent)) with pytest.raises(InputValidationError): calc.prepare_for_submission(temp_folder)
def prepare_for_submission(self, folder: folders.Folder): """Create input files from the input nodes passed to this instance of the `CalcJob`. :param folder: an `aiida.common.folders.Folder` to temporarily write files on disk :return: `aiida.common.datastructures.CalcInfo` instance """ self._validate_inputs() dos_filenames = OpenmxCalculation.dos_filenames remote_symlink_list = [ (self.inputs.openmx_output_folder.computer.uuid, os.path.join(self.inputs.openmx_output_folder.get_remote_path(), dos_filenames['val']), os.path.join(self._DATA_PATH, dos_filenames['val'])), (self.inputs.openmx_output_folder.computer.uuid, os.path.join(self.inputs.openmx_output_folder.get_remote_path(), dos_filenames['vec']), os.path.join(self._DATA_PATH, dos_filenames['vec'])), ] retrieve_list = self._generate_retrieve_list() input_file_content = self._write_input_file() with folder.open(self._INPUT_FILE, 'w') as handle: handle.write(input_file_content) # Fill out the `CodeInfo` codeinfo = datastructures.CodeInfo() codeinfo.cmdline_params = ([ dos_filenames['val'], dos_filenames['vec'] ]) codeinfo.stdin_name = self._INPUT_FILE codeinfo.stdout_name = self._OUTPUT_FILE codeinfo.code_uuid = self.inputs.code.uuid codeinfo.with_mpi = True # Fill out the `CalcInfo` calcinfo = datastructures.CalcInfo() calcinfo.uuid = str(self.uuid) calcinfo.codes_info = [codeinfo] calcinfo.remote_symlink_list = remote_symlink_list calcinfo.retrieve_list = retrieve_list calcinfo.retrieve_list.append(self._OUTPUT_FILE) return calcinfo
def test_write_wavecar(localhost_dir, vasp_calc, vasp_inputs, vasp_wavecar): """Test that WAVECAR file is written correctly.""" from aiida.common.folders import Folder wavecar, _ = vasp_wavecar inputs = vasp_inputs( parameters={ 'gga': 'PE', 'gga_compat': False, 'lorbit': 11, 'sigma': 0.5, 'magmom': '30 * 2*0.', 'istart': 1 }) inputs.wavefunctions = wavecar calc = vasp_calc(inputs=inputs) temp_folder = Folder(str(localhost_dir.parent)) calcinfo = calc.prepare_for_submission(temp_folder) assert 'WAVECAR' in [item[1] for item in calcinfo.local_copy_list]
def test_write_chgcar(localhost_dir, vasp_calc, vasp_inputs, vasp_chgcar): """Test that CHGAR file is written correctly.""" from aiida.common.folders import Folder chgcar, _ = vasp_chgcar inputs = vasp_inputs( parameters={ 'gga': 'PE', 'gga_compat': False, 'lorbit': 11, 'sigma': 0.5, 'magmom': '30 * 2*0.', 'icharg': 1 }) inputs.charge_density = chgcar calc = vasp_calc(inputs=inputs) temp_folder = Folder(str(localhost_dir.dirpath())) calcinfo = calc.prepare_for_submission(temp_folder) assert 'CHGCAR' in [item[1] for item in calcinfo.local_copy_list]
def prepare_for_submission(self, folder: folders.Folder): """Create input files from the input nodes passed to this instance of the `CalcJob`. :param folder: an `aiida.common.folders.Folder` to temporarily write files on disk :return: `aiida.common.datastructures.CalcInfo` instance """ # To be filled out below local_copy_list = [] remote_copy_list = [] remote_symlink_list = [] # Create the subfolders for pseudopotentials and orbitals folder.get_subfolder(self._PSEUDO_SUBFOLDER, create=True) folder.get_subfolder(self._ORBITAL_SUBFOLDER, create=True) # Get an uppercase-key-only version of the settings dictionary (also check for case-insensitive duplicates) if 'settings' in self.inputs: settings = uppercase_dict_keys(self.inputs.settings.get_dict(), dict_name='settings') else: settings = {} # Get an uppercase-key-only verion of the parameters dictionary (also check for case-insensitive duplicates) parameters = uppercase_dict_keys(self.inputs.parameters.get_dict(), dict_name='parameters') # No reserved parameter keywords should be provided self._check_reserved_keywords(parameters) # Load parameter schema with open(self._INPUT_SCHEMA, 'r') as stream: schema = json.load(stream) # Automatically generate input parameters for derived fields, e.g. structure -> Atoms.Unitvectors, etc. parameters = self._generate_input_parameters( self.inputs.structure, self.inputs.kpoints, parameters, self.inputs.pseudos, self.inputs.orbitals, self.inputs.orbital_configurations) # Get a lowercase-value-only version of the parameters dictionary parameters = lowercase_dict_values(parameters) # Validate input parameters self._validate_inputs(self.inputs.structure, self.inputs.kpoints, parameters, self.inputs.pseudos, self.inputs.orbitals, self.inputs.orbital_configurations, schema) # Get input file contents and lists of the pseudopotential and orbital files which need to be copied input_file_content = write_input_file(parameters, schema) local_copy_pseudo_list, local_copy_orbital_list = self._generate_local_copy_lists( self.inputs.pseudos, self.inputs.orbitals) local_copy_list += local_copy_pseudo_list local_copy_list += local_copy_orbital_list # Add output files to retrieve which have been specified to write in the input parameters retrieve_list = [] if parameters.get('BAND_NKPATH', 0) > 0 and parameters.get( 'SCF_EIGENVALUESOLVER', 'band') == 'band': retrieve_list.append(self._DATAFILE_BAND_FILE) if parameters.get('MD_TYPE', 'nomd') != 'nomd': retrieve_list.append(self._DATAFILE_MD_FILE) retrieve_list.append(self._DATAFILE_MD2_FILE) # Write input file with folder.open(self._INPUT_FILE, 'w') as handle: handle.write(input_file_content) # Fill out the `CodeInfo` codeinfo = datastructures.CodeInfo() codeinfo.code_uuid = self.inputs.code.uuid codeinfo.withmpi = True codeinfo.cmdline_params = ([self._INPUT_FILE] + list(settings.pop('CMDLINE', []))) codeinfo.stdout_name = self._OUTPUT_FILE # Fill out the `CalcInfo` calcinfo = datastructures.CalcInfo() calcinfo.uuid = str(self.uuid) calcinfo.codes_info = [codeinfo] calcinfo.local_copy_list = local_copy_list calcinfo.remote_copy_list = remote_copy_list calcinfo.remote_symlink_list = remote_symlink_list calcinfo.retrieve_list = retrieve_list calcinfo.retrieve_list.append(self._OUTPUT_FILE) calcinfo.retrieve_list += settings.pop('ADDITIONAL_RETRIEVE_LIST', []) # TODO: pop parser settings and report remaining unknown settings return calcinfo
def test_unicode(cls): """ Check that there are no exceptions raised when using unicode folders. """ tmpsource = tempfile.mkdtemp() tmpdest = tempfile.mkdtemp() with io.open(os.path.join(tmpsource, 'sąžininga'), 'w', encoding='utf8') as fhandle: fhandle.write(u'test') with io.open(os.path.join(tmpsource, 'žąsis'), 'w', encoding='utf8') as fhandle: fhandle.write(u'test') folder = Folder(tmpdest) folder.insert_path(tmpsource, 'destination') folder.insert_path(tmpsource, u'šaltinis') folder = Folder(os.path.join(tmpsource, u'šaltinis')) folder.insert_path(tmpsource, 'destination') folder.insert_path(tmpdest, u'kitas-šaltinis')
#~ vc = VaspCalculation() incar = { 'SYSTEM': 'TestSystem', 'ediff': 1E-5, 'GGA_COMPAT': False, } poscar = pmg.io.vasp.Poscar.from_file(sys.argv[1]).as_dict() potcar = {} kpoints = {} ParameterData = DataFactory('parameter') code = Code.get_from_string('vasp') vc = code.new_calc() vc.use_settings(ParameterData(dict=incar)) vc.use_structure(ParameterData(dict=poscar)) vc.use_potentials(ParameterData(dict=potcar)) vc.use_kpoints(ParameterData(dict=kpoints)) vc.label = 'Test Vasp Calculation' vc.description = 'Test Vasp Calculation Plugin (with invalid data)' vc.set_max_wallclock_seconds(1) #should not run at all vc.set_withmpi(True) vc.set_resources({'num_machines': 2, 'num_mpiprocs_per_machine': 12}) vc.set_queue_name('bogus_queue_to_prevent_accidental_queuing') fldr = Folder(sys.argv[2]) vc.submit_test(fldr)
def test_unicode(self): """ Check that there are no exceptions raised when using unicode folders. """ from aiida.common.folders import Folder import os, tempfile tmpsource = tempfile.mkdtemp() tmpdest = tempfile.mkdtemp() with open(os.path.join(tmpsource, "sąžininga"), 'w') as f: f.write("test") with open(os.path.join(tmpsource, "žąsis"), 'w') as f: f.write("test") fd = Folder(tmpdest) fd.insert_path(tmpsource, "destination") fd.insert_path(tmpsource, u"šaltinis") fd = Folder(os.path.join(tmpsource, u"šaltinis")) fd.insert_path(tmpsource, "destination") fd.insert_path(tmpdest, u"kitas-šaltinis")
def presubmit(self, folder: Folder) -> CalcInfo: """Prepares the calculation folder with all inputs, ready to be copied to the cluster. :param folder: a SandboxFolder that can be used to write calculation input files and the scheduling script. :return calcinfo: the CalcInfo object containing the information needed by the daemon to handle operations. """ # pylint: disable=too-many-locals,too-many-statements,too-many-branches from aiida.common.exceptions import PluginInternalError, ValidationError, InvalidOperation, InputValidationError from aiida.common import json from aiida.common.utils import validate_list_of_string_tuples from aiida.common.datastructures import CodeInfo, CodeRunMode from aiida.orm import load_node, Code, Computer from aiida.plugins import DataFactory from aiida.schedulers.datastructures import JobTemplate computer = self.node.computer inputs = self.node.get_incoming(link_type=LinkType.INPUT_CALC) if not self.inputs.metadata.dry_run and self.node.has_cached_links( ): # type: ignore[union-attr] raise InvalidOperation( 'calculation node has unstored links in cache') codes = [_ for _ in inputs.all_nodes() if isinstance(_, Code)] for code in codes: if not code.can_run_on(computer): raise InputValidationError( 'The selected code {} for calculation {} cannot run on computer {}' .format(code.pk, self.node.pk, computer.label)) if code.is_local() and code.get_local_executable( ) in folder.get_content_list(): raise PluginInternalError( f'The plugin created a file {code.get_local_executable()} that is also the executable name!' ) calc_info = self.prepare_for_submission(folder) calc_info.uuid = str(self.node.uuid) scheduler = computer.get_scheduler() # I create the job template to pass to the scheduler job_tmpl = JobTemplate() job_tmpl.shebang = computer.get_shebang() job_tmpl.submit_as_hold = False job_tmpl.rerunnable = False job_tmpl.job_environment = {} # 'email', 'email_on_started', 'email_on_terminated', job_tmpl.job_name = f'aiida-{self.node.pk}' job_tmpl.sched_output_path = self.options.scheduler_stdout if self.options.scheduler_stderr == self.options.scheduler_stdout: job_tmpl.sched_join_files = True else: job_tmpl.sched_error_path = self.options.scheduler_stderr job_tmpl.sched_join_files = False # Set retrieve path, add also scheduler STDOUT and STDERR retrieve_list = calc_info.retrieve_list or [] if (job_tmpl.sched_output_path is not None and job_tmpl.sched_output_path not in retrieve_list): retrieve_list.append(job_tmpl.sched_output_path) if not job_tmpl.sched_join_files: if (job_tmpl.sched_error_path is not None and job_tmpl.sched_error_path not in retrieve_list): retrieve_list.append(job_tmpl.sched_error_path) retrieve_list.extend( self.node.get_option('additional_retrieve_list') or []) self.node.set_retrieve_list(retrieve_list) retrieve_singlefile_list = calc_info.retrieve_singlefile_list or [] # a validation on the subclasses of retrieve_singlefile_list for _, subclassname, _ in retrieve_singlefile_list: file_sub_class = DataFactory(subclassname) if not issubclass(file_sub_class, orm.SinglefileData): raise PluginInternalError( '[presubmission of calc {}] retrieve_singlefile_list subclass problem: {} is ' 'not subclass of SinglefileData'.format( self.node.pk, file_sub_class.__name__)) if retrieve_singlefile_list: self.node.set_retrieve_singlefile_list(retrieve_singlefile_list) # Handle the retrieve_temporary_list retrieve_temporary_list = calc_info.retrieve_temporary_list or [] self.node.set_retrieve_temporary_list(retrieve_temporary_list) # the if is done so that if the method returns None, this is # not added. This has two advantages: # - it does not add too many \n\n if most of the prepend_text are empty # - most importantly, skips the cases in which one of the methods # would return None, in which case the join method would raise # an exception prepend_texts = [computer.get_prepend_text()] + \ [code.get_prepend_text() for code in codes] + \ [calc_info.prepend_text, self.node.get_option('prepend_text')] job_tmpl.prepend_text = '\n\n'.join(prepend_text for prepend_text in prepend_texts if prepend_text) append_texts = [self.node.get_option('append_text'), calc_info.append_text] + \ [code.get_append_text() for code in codes] + \ [computer.get_append_text()] job_tmpl.append_text = '\n\n'.join(append_text for append_text in append_texts if append_text) # Set resources, also with get_default_mpiprocs_per_machine resources = self.node.get_option('resources') scheduler.preprocess_resources( resources, computer.get_default_mpiprocs_per_machine()) job_tmpl.job_resource = scheduler.create_job_resource(**resources) subst_dict = { 'tot_num_mpiprocs': job_tmpl.job_resource.get_tot_num_mpiprocs() } for key, value in job_tmpl.job_resource.items(): subst_dict[key] = value mpi_args = [ arg.format(**subst_dict) for arg in computer.get_mpirun_command() ] extra_mpirun_params = self.node.get_option( 'mpirun_extra_params') # same for all codes in the same calc # set the codes_info if not isinstance(calc_info.codes_info, (list, tuple)): raise PluginInternalError( 'codes_info passed to CalcInfo must be a list of CalcInfo objects' ) codes_info = [] for code_info in calc_info.codes_info: if not isinstance(code_info, CodeInfo): raise PluginInternalError( 'Invalid codes_info, must be a list of CodeInfo objects') if code_info.code_uuid is None: raise PluginInternalError( 'CalcInfo should have the information of the code to be launched' ) this_code = load_node(code_info.code_uuid, sub_classes=(Code, )) this_withmpi = code_info.withmpi # to decide better how to set the default if this_withmpi is None: if len(calc_info.codes_info) > 1: raise PluginInternalError( 'For more than one code, it is necessary to set withmpi in codes_info' ) else: this_withmpi = self.node.get_option('withmpi') if this_withmpi: this_argv = (mpi_args + extra_mpirun_params + [this_code.get_execname()] + (code_info.cmdline_params if code_info.cmdline_params is not None else [])) else: this_argv = [this_code.get_execname() ] + (code_info.cmdline_params if code_info.cmdline_params is not None else []) # overwrite the old cmdline_params and add codename and mpirun stuff code_info.cmdline_params = this_argv codes_info.append(code_info) job_tmpl.codes_info = codes_info # set the codes execution mode if len(codes) > 1: try: job_tmpl.codes_run_mode = calc_info.codes_run_mode except KeyError as exc: raise PluginInternalError( 'Need to set the order of the code execution (parallel or serial?)' ) from exc else: job_tmpl.codes_run_mode = CodeRunMode.SERIAL ######################################################################## custom_sched_commands = self.node.get_option( 'custom_scheduler_commands') if custom_sched_commands: job_tmpl.custom_scheduler_commands = custom_sched_commands job_tmpl.import_sys_environment = self.node.get_option( 'import_sys_environment') job_tmpl.job_environment = self.node.get_option( 'environment_variables') queue_name = self.node.get_option('queue_name') account = self.node.get_option('account') qos = self.node.get_option('qos') if queue_name is not None: job_tmpl.queue_name = queue_name if account is not None: job_tmpl.account = account if qos is not None: job_tmpl.qos = qos priority = self.node.get_option('priority') if priority is not None: job_tmpl.priority = priority max_memory_kb = self.node.get_option('max_memory_kb') if max_memory_kb is not None: job_tmpl.max_memory_kb = max_memory_kb max_wallclock_seconds = self.node.get_option('max_wallclock_seconds') if max_wallclock_seconds is not None: job_tmpl.max_wallclock_seconds = max_wallclock_seconds max_memory_kb = self.node.get_option('max_memory_kb') if max_memory_kb is not None: job_tmpl.max_memory_kb = max_memory_kb submit_script_filename = self.node.get_option('submit_script_filename') script_content = scheduler.get_submit_script(job_tmpl) folder.create_file_from_filelike(io.StringIO(script_content), submit_script_filename, 'w', encoding='utf8') subfolder = folder.get_subfolder('.aiida', create=True) subfolder.create_file_from_filelike(io.StringIO(json.dumps(job_tmpl)), 'job_tmpl.json', 'w', encoding='utf8') subfolder.create_file_from_filelike(io.StringIO(json.dumps(calc_info)), 'calcinfo.json', 'w', encoding='utf8') if calc_info.local_copy_list is None: calc_info.local_copy_list = [] if calc_info.remote_copy_list is None: calc_info.remote_copy_list = [] # Some validation this_pk = self.node.pk if self.node.pk is not None else '[UNSTORED]' local_copy_list = calc_info.local_copy_list try: validate_list_of_string_tuples(local_copy_list, tuple_length=3) except ValidationError as exception: raise PluginInternalError( f'[presubmission of calc {this_pk}] local_copy_list format problem: {exception}' ) from exception remote_copy_list = calc_info.remote_copy_list try: validate_list_of_string_tuples(remote_copy_list, tuple_length=3) except ValidationError as exception: raise PluginInternalError( f'[presubmission of calc {this_pk}] remote_copy_list format problem: {exception}' ) from exception for (remote_computer_uuid, _, dest_rel_path) in remote_copy_list: try: Computer.objects.get(uuid=remote_computer_uuid) # pylint: disable=unused-variable except exceptions.NotExistent as exception: raise PluginInternalError( '[presubmission of calc {}] ' 'The remote copy requires a computer with UUID={}' 'but no such computer was found in the ' 'database'.format(this_pk, remote_computer_uuid)) from exception if os.path.isabs(dest_rel_path): raise PluginInternalError( '[presubmission of calc {}] ' 'The destination path of the remote copy ' 'is absolute! ({})'.format(this_pk, dest_rel_path)) return calc_info
'generation_style': 'Gamma', 'kpoints': [[8,8,8]], 'usershift': [0,0,0], } code = Code.get_from_string('vasp') calc = code.new_calc() ParameterData = DataFactory('parameter') SinglefileData = DataFactory('singlefile') calc.use_settings(ParameterData(dict=incar)) calc.use_structure(ParameterData(dict=poscar)) #~ calc.use_potentials(ParameterData(dict=potcar)) calc.use_potentials(SinglefileData(file=potcar)) calc.use_kpoints(ParameterData(dict=kpoints)) calc.label = 'Vasp Selfconsistent Test Run' calc.description = 'Test the Vasp Plugin with a simple selfconsistent run for fcc InAs' calc.set_max_wallclock_seconds(60) calc.set_withmpi(False) calc.set_resources({'num_machines':1, 'num_mpiprocs_per_machine':1}) calc.set_queue_name('dphys_compute') if sys.argv[1] == '--submit': calc.store_all() print "created calculation; with uuid='{}' and PK={}".format(calc.uuid,calc.pk) calc.submit() if sys.argv[1] == '--test': from aiida.common.folders import Folder calc.submit_test(Folder(sys.argv[3]))
def create_builder_from_file(input_folder, input_file_name, code, metadata, pseudo_folder_path=None, use_first=False): """Create a populated process builder for a `PwCalculation` from a standard QE input file and pseudo (upf) files :param input_folder: the folder containing the input file :type input_folder: aiida.common.folders.Folder or str :param input_file_name: the name of the input file :type input_file_name: str :param code: the code associated with the calculation :type code: aiida.orm.Code or str :param metadata: metadata values for the calculation (e.g. resources) :type metadata: dict :param pseudo_folder_path: the folder containing the upf files (if None, then input_folder is used) :type pseudo_folder_path: aiida.common.folders.Folder or str or None :param use_first: passed to UpfData.get_or_create :type use_first: bool :raises NotImplementedError: if the structure is not ibrav=0 :return: a builder instance for PwCalculation """ PwCalculation = CalculationFactory('quantumespresso.pw') builder = PwCalculation.get_builder() builder.metadata = metadata if isinstance(code, six.string_types): code = Code.get_from_string(code) builder.code = code # read input_file if isinstance(input_folder, six.string_types): input_folder = Folder(input_folder) with input_folder.open(input_file_name) as input_file: parsed_file = PwInputFile(input_file) builder.structure = parsed_file.get_structuredata() builder.kpoints = parsed_file.get_kpointsdata() if parsed_file.namelists['SYSTEM']['ibrav'] != 0: raise NotImplementedError('Found ibrav != 0: `aiida-quantumespresso` currently only supports ibrav = 0.') # Then, strip the namelist items that the plugin doesn't allow or sets later. # NOTE: If any of the position or cell units are in alat or crystal # units, that will be taken care of by the input parsing tools, and # we are safe to fake that they were never there in the first place. parameters_dict = copy.deepcopy(parsed_file.namelists) for namelist, blocked_key in PwCalculation._blocked_keywords: # pylint: disable=protected-access for key in list(parameters_dict[namelist].keys()): # take into account that celldm and celldm(*) must be blocked if re.sub('[(0-9)]', '', key) == blocked_key: parameters_dict[namelist].pop(key, None) builder.parameters = Dict(dict=parameters_dict) # Get or create a UpfData node for the pseudopotentials used for the calculation. pseudos_map = {} if pseudo_folder_path is None: pseudo_folder_path = input_folder if isinstance(pseudo_folder_path, six.string_types): pseudo_folder_path = Folder(pseudo_folder_path) names = parsed_file.atomic_species['names'] pseudo_file_names = parsed_file.atomic_species['pseudo_file_names'] pseudo_file_map = {} for name, fname in zip(names, pseudo_file_names): if fname not in pseudo_file_map: local_path = pseudo_folder_path.get_abs_path(fname) upf_node, _ = UpfData.get_or_create(local_path, use_first=use_first, store_upf=False) pseudo_file_map[fname] = upf_node pseudos_map[name] = pseudo_file_map[fname] builder.pseudos = pseudos_map settings_dict = {} if parsed_file.k_points['type'] == 'gamma': settings_dict['gamma_only'] = True # If there are any fixed coordinates (i.e. force modification) present in the input file, specify in settings fixed_coords = parsed_file.atomic_positions['fixed_coords'] # Function ``any()`` only works for 1-dimensional lists so we have to call it twice manually. if any((any(fc_xyz) for fc_xyz in fixed_coords)): settings_dict['FIXED_COORDS'] = fixed_coords if settings_dict: builder.settings = settings_dict return builder
def dryrun_test(cls, inputs, castep_exe='castep.serial', verbose=True): """ Do a dryrun test in a folder with prepared builder or inputs """ if isinstance(inputs, ProcessBuilder): res = cls.submit_test(inputs) else: res = cls.submit_test(cls, **inputs) folder = Folder(res[1]) dry_run_node = res[0] seedname = dry_run_node.get_option('seedname') def _print(inp): if verbose: print(inp) # Do a dryrun try: output = check_output([castep_exe, "-v"], universal_newlines=True) except OSError: _print("CASTEP executable '{}' is not found".format(castep_exe)) return None # Now start dryrun _print("Running with {}".format( check_output(["which", castep_exe], universal_newlines=True))) _print(output) _print("Starting dryrun...") call([castep_exe, "--dryrun", seedname], cwd=folder.abspath) # Check if any *err files contents = folder.get_content_list() for fname in contents: if fnmatch(fname, "*.err"): with folder.open(fname) as fhandle: _print("Error found in {}:\fname".format(fname)) _print(fhandle.read()) raise InputValidationError("Error found during dryrun") # Gather information from the dryrun file dryrun_results = {} out_file = seedname + '.castep' with folder.open(out_file) as fhandle: for line in fhandle: mth = re.match(r"\s*k-Points For SCF Sampling:\s+(\d+)\s*", line) if mth: dryrun_results["num_kpoints"] = int(mth.group(1)) _print("Number of k-points: {}".format(mth.group(1))) mth = None continue mth = re.match( r"\| Approx\. total storage required" r" per process\s+([0-9.]+)\sMB\s+([0-9.]+)", line) if mth: dryrun_results["memory_MB"] = (float(mth.group(1))) dryrun_results["disk_MB"] = (float(mth.group(2))) _print("RAM: {} MB, DISK: {} MB".format( mth.group(1), mth.group(2))) mth = None continue return folder, dryrun_results
def test_unicode(self): """Check that there are no exceptions raised when using unicode folders.""" tmpsource = tempfile.mkdtemp() tmpdest = tempfile.mkdtemp() with open(os.path.join(tmpsource, 'sąžininga'), 'w', encoding='utf8') as fhandle: fhandle.write('test') with open(os.path.join(tmpsource, 'žąsis'), 'w', encoding='utf8') as fhandle: fhandle.write('test') folder = Folder(tmpdest) folder.insert_path(tmpsource, 'destination') folder.insert_path(tmpsource, 'šaltinis') self.assertEqual(sorted(folder.get_content_list()), sorted(['destination', 'šaltinis'])) self.assertEqual( sorted(folder.get_subfolder('destination').get_content_list()), sorted(['sąžininga', 'žąsis'])) self.assertEqual( sorted(folder.get_subfolder('šaltinis').get_content_list()), sorted(['sąžininga', 'žąsis'])) folder = Folder(os.path.join(tmpsource, 'šaltinis')) folder.insert_path(tmpdest, 'destination') folder.insert_path(tmpdest, 'kitas-šaltinis') self.assertEqual(sorted(folder.get_content_list()), sorted(['destination', 'kitas-šaltinis']))