def test_dont_restart_completed_calc(self): """ Set a relaxation up to fail. """ shutil.copy( REAL_PATH + "data/no_steps_left_todo/cache/NaP_intermediates_stopped_early.res", ".", ) shutil.copy( REAL_PATH + "data/no_steps_left_todo/cache/NaP_intermediates_stopped_early.castep", ".", ) cell_dict, s = cell2dict( REAL_PATH + "data/no_steps_left_todo/NaP.cell", verbosity=VERBOSITY, db=False, ) self.assertTrue(s) param_dict, s = param2dict( REAL_PATH + "data/no_steps_left_todo/NaP.param", verbosity=VERBOSITY, db=False, ) self.assertTrue(s) executable = "castep" node = None seed = "NaP_intermediates_stopped_early.res" with self.assertRaises(CalculationError): ComputeTask( ncores=NCORES, nnodes=None, node=node, res=seed, param_dict=param_dict, cell_dict=cell_dict, verbosity=VERBOSITY, killcheck=True, memcheck=False, exec_test=False, reopt=True, executable=executable, max_walltime=5, start=True, ) print("Process completed!") bad_exists = [] bad_exists.append( isfile("bad_castep/NaP_intermediates_stopped_early.res")) bad_exists.append( isfile("bad_castep/NaP_intermediates_stopped_early.castep")) bad_exists = all(bad_exists) good_exists = all( isdir(path) for path in ["input", "bad_castep", "logs"]) self.assertTrue(bad_exists) self.assertTrue(good_exists)
def test_missing_exec(self): """ Ensure failure if exec misses. """ cell_dict, s = cell2dict(REAL_PATH + "/data/LiAs_tests/LiAs.cell", verbosity=VERBOSITY, db=False) assert s param_dict, s = param2dict(REAL_PATH + "/data/LiAs_tests/LiAs.param", verbosity=VERBOSITY, db=False) assert s node = None nnodes = None seed = REAL_PATH + "/data/structures/LiAs_testcase.res" fall_over = False try: ComputeTask( ncores=NCORES, nnodes=nnodes, node=node, res=seed, param_dict=param_dict, cell_dict=cell_dict, verbosity=VERBOSITY, killcheck=True, reopt=False, executable="THIS WAS MEANT TO FAIL, DON'T WORRY", start=True, ) except CriticalError: fall_over = True self.assertTrue(fall_over)
def test_benchmark_dual_core_scf(self): """ Test the time taken to perform a set number of SCF steps on 2 cores. CASTEP prints no total timing data for single core jobs. """ from os import makedirs seed = "_LiC.res" shutil.copy(REAL_PATH + "data/structures/LiC.res", "_LiC.res") cell_dict, s = cell2dict( REAL_PATH + "/data/benchmark/LiC_scf/LiC_scf.cell", verbosity=VERBOSITY, db=False, ) self.assertTrue(s) param_dict, s = param2dict( REAL_PATH + "/data/benchmark/LiC_scf/LiC_scf.param", verbosity=VERBOSITY, db=False, ) self.assertTrue(s) shutil.copy(REAL_PATH + "data/pspots/Li_00PBE.usp", ".") shutil.copy(REAL_PATH + "data/pspots/C_00PBE.usp", ".") with self.assertRaises(CalculationError): ComputeTask( ncores=2, nnodes=None, node=None, res=seed, param_dict=param_dict, cell_dict=cell_dict, verbosity=0, executable=EXECUTABLE, start=True, ) outputs_exist = [ isfile("bad_castep/_LiC.res"), isfile("bad_castep/_LiC.castep"), ] results, s = castep2dict("bad_castep/_LiC.castep", db=False) makedirs(REAL_PATH + "/data/benchmark/results", exist_ok=True) shutil.copy( "bad_castep/_LiC.castep", REAL_PATH + "/data/benchmark/results/_LiC_2core_castep{}.castep".format( results.get("castep_version", "xxx") ), ) self.assertTrue(all(outputs_exist), "couldn't find output files!") self.assertTrue(s, "couldn't read output files!") self.assertLess(results["_time_estimated"], 8)
def test_faked_error_recovery(self): """ Run a calculation that *should* throw a symmetry error, and try to recover from the error. If CASTEP is not present, monkey patch such that ComputeTask copies the output files it would have expected. """ seed = REAL_PATH + "data/symmetry_failure/Sb.res" cell_dict, s = cell2dict(REAL_PATH + "/data/symmetry_failure/KSb.cell", verbosity=VERBOSITY, db=False) assert s param_dict, s = param2dict( REAL_PATH + "/data/symmetry_failure/KSb.param", verbosity=VERBOSITY, db=False, ) assert s ncores = 1 executable = REAL_PATH + "data/symmetry_failure/monkey_patch_move.sh" node = None relaxer = ComputeTask( ncores=ncores, nnodes=None, node=node, res=seed, param_dict=param_dict, cell_dict=cell_dict, verbosity=VERBOSITY, executable=executable, exec_test=False, polltime=1, start=False, ) with self.assertRaises(CalculationError): relaxer.begin() bad_castep_exists = isdir("bad_castep") completed_exists = isdir("completed") self.assertTrue(bad_castep_exists) self.assertFalse(completed_exists) self.assertTrue(relaxer.final_result is None) self.assertEqual(relaxer._num_retries, 3) self.assertTrue("symmetry_generate" not in relaxer.calc_doc) self.assertTrue("snap_to_symmetry" not in relaxer.calc_doc) self.assertTrue("symmetry_tol" not in relaxer.calc_doc)
def test_failed_relaxation(self): """ Set a relaxation up to fail. """ seed = "_LiAs_testcase.res" shutil.copy( REAL_PATH + "data/structures/LiAs_testcase_bad.res", "_LiAs_testcase.res" ) cell_dict, s = cell2dict( REAL_PATH + "/data/LiAs_tests/LiAs.cell", verbosity=VERBOSITY, db=False ) assert s param_dict, s = param2dict( REAL_PATH + "/data/LiAs_tests/LiAs.param", verbosity=VERBOSITY, db=False ) assert s param_dict["geom_max_iter"] = 3 executable = "castep" node = None shutil.copy(REAL_PATH + "data/pspots/Li_00PBE.usp", ".") shutil.copy(REAL_PATH + "data/pspots/As_00PBE.usp", ".") relaxer = ComputeTask( ncores=NCORES, nnodes=None, node=node, res=seed, param_dict=param_dict, cell_dict=cell_dict, verbosity=VERBOSITY, killcheck=True, memcheck=False, reopt=True, executable=executable, rough=0, fine_iter=3, start=False, ) with self.assertRaises(CalculationError): relaxer.begin() bad_exists = isfile("bad_castep/_LiAs_testcase.res") num = reset_job_folder() self.assertTrue(bad_exists, "couldn't find output file!") self.assertEqual(num, 0)
def test_scf(self): """ Perform SCF on structure from file. """ seed = "_LiC.res" shutil.copy(REAL_PATH + "data/structures/LiC.res", "_LiC.res") cell_dict, s = cell2dict(REAL_PATH + "/data/LiC_tests/LiC_scf.cell", verbosity=VERBOSITY, db=False) assert s param_dict, s = param2dict(REAL_PATH + "/data/LiC_tests/LiC_scf.param", verbosity=VERBOSITY, db=False) assert s executable = "castep" node = None shutil.copy(REAL_PATH + "data/pspots/Li_00PBE.usp", ".") shutil.copy(REAL_PATH + "data/pspots/C_00PBE.usp", ".") ComputeTask( ncores=NCORES, nnodes=None, node=node, res=seed, param_dict=param_dict, cell_dict=cell_dict, verbosity=VERBOSITY, killcheck=True, reopt=True, executable=executable, compute_dir="/tmp/compute_test", start=True, ) completed_exists = [ isfile("completed/_LiC.res"), isfile("completed/_LiC.castep"), isfile("completed/_LiC-out.cell"), ] base_file_exists = [ isfile("_LiC.res"), isfile("_LiC.castep"), isfile("_LiC.res.lock"), ] self.assertFalse(any(base_file_exists), "failed to clean up files!") self.assertTrue(all(completed_exists), "couldn't find output files!")
def test_memcheck(self): """ Test the memory checker will not proceed with huge jobs. """ shutil.copy( REAL_PATH + "data/structures/LiAs_testcase.res", "_LiAs_testcase.res" ) shutil.copy(REAL_PATH + "data/pspots/Li_00PBE.usp", ".") shutil.copy(REAL_PATH + "data/pspots/As_00PBE.usp", ".") cell_dict, s = cell2dict( REAL_PATH + "data/LiAs_tests/LiAs.cell", verbosity=VERBOSITY, db=False ) self.assertTrue(s) param_dict, s = param2dict( REAL_PATH + "data/LiAs_tests/LiAs.param", verbosity=VERBOSITY, db=False ) self.assertTrue(s) with self.assertRaises(MaxMemoryEstimateExceeded): ComputeTask( ncores=NCORES, nnodes=None, node=None, res="_LiAs_testcase.res", param_dict=param_dict, cell_dict=cell_dict, verbosity=VERBOSITY, killcheck=True, memcheck=True, maxmem=1, start=True, ) files_that_should_not_exist = ["_LiAs_testcase.res.lock", "jobs.txt"] folders_that_should_exist = ["logs"] folders_that_should_not_exist = ["bad_castep", "input", "completed"] correct_files = all( [not isfile(_file) for _file in files_that_should_not_exist] ) correct_folders = all([isdir(folder) for folder in folders_that_should_exist]) correct_folders *= all( [not isdir(folder) for folder in folders_that_should_not_exist] ) self.assertTrue(correct_folders) self.assertTrue(correct_files)
def test_relax_to_file(self): """ Relax structure from file to file. """ seed = "_Li.res" shutil.copy(REAL_PATH + "data/structures/Li.res", "_Li.res") cell_dict, s = cell2dict(REAL_PATH + "/data/LiAs_tests/LiAs.cell", verbosity=VERBOSITY, db=False) assert s param_dict, s = param2dict(REAL_PATH + "/data/LiAs_tests/LiAs.param", verbosity=VERBOSITY, db=False) assert s executable = "castep" node = None shutil.copy(REAL_PATH + "data/pspots/Li_00PBE.usp", ".") shutil.copy(REAL_PATH + "data/pspots/As_00PBE.usp", ".") ComputeTask( ncores=NCORES, nnodes=None, node=node, res=seed, param_dict=param_dict, cell_dict=cell_dict, verbosity=VERBOSITY, killcheck=True, reopt=True, executable=executable, exec_test=False, start=True, ) print("Process completed!") completed_exists = isfile("completed/_Li.res") base_files_exist = [ isfile("_Li.res"), isfile("_Li.res.lock"), isfile("_Li.castep"), ] self.assertTrue(completed_exists, "couldn't find output file!") self.assertFalse(any(base_files_exist), "couldn't clean input files")
def test_scf_max_walltime(self): """ Perform SCF on structure from file. """ seed = "_LiC.res" shutil.copy(REAL_PATH + "data/structures/LiC.res", "_LiC.res") cell_dict, s = cell2dict(REAL_PATH + "/data/LiC_tests/LiC_scf.cell", verbosity=VERBOSITY, db=False) assert s param_dict, s = param2dict(REAL_PATH + "/data/LiC_tests/LiC_scf.param", verbosity=VERBOSITY, db=False) assert s executable = "castep" node = None shutil.copy(REAL_PATH + "data/pspots/Li_00PBE.usp", ".") shutil.copy(REAL_PATH + "data/pspots/C_00PBE.usp", ".") with self.assertRaises(WalltimeError): ComputeTask( ncores=NCORES, nnodes=None, node=node, res=seed, param_dict=param_dict, cell_dict=cell_dict, verbosity=VERBOSITY, timings=(5, time.time()), polltime=2, executable=executable, compute_dir="/tmp/compute_test", start=True, ) base_file_exists = [ isfile("_LiC.res"), isfile("_LiC.castep"), ] self.assertFalse(isfile("_LiC.res.lock"), "failed to clean up lock") self.assertTrue(all(base_file_exists), "failed to keep valid files!")
def test_old_file(self): """ Run a calculation with an executable that only does "sleep", in the presence of a file that was written previouisly, and check that run3 will stop the calculation early as no file is written. """ seed = REAL_PATH + "data/symmetry_failure/Sb.res" with open("Sb.castep", "w") as f: f.write("I am a CASTEP file, for sure.") cell_dict, s = cell2dict(REAL_PATH + "/data/symmetry_failure/KSb.cell", verbosity=VERBOSITY, db=False) assert s param_dict, s = param2dict( REAL_PATH + "/data/symmetry_failure/KSb.param", verbosity=VERBOSITY, db=False, ) assert s executable = REAL_PATH + "data/missing_file_test/monkey_patch_sleep.sh" node = None relaxer = ComputeTask( ncores=NCORES, nnodes=None, node=node, res=seed, param_dict=param_dict, cell_dict=cell_dict, verbosity=VERBOSITY, executable=executable, exec_test=False, polltime=1, start=False, ) with self.assertRaises(CalculationError): relaxer.begin() self.assertTrue(relaxer.final_result is None)
def _scrape_multi_file_results(self, file_lists, root): """ Add structures to database by parsing .res or .castep files., with DFT data scraped from .castep/.cell/.param files in the same folder, i.e. data from multiple files. Parameters: file_lists: dictionary containing counts of file types in each sub-directory. root: name of sub-directory. Returns: int: number of structures successfully imported. """ multi = False # are there multiple param/cell files? cell = False # was the cell file successfully scraped? param = False # was the param file successfully scraped? import_count = 0 # how many files have been successfully imported? success = False if file_lists[root]['param_count'] == 1: param_dict, success = param2dict(file_lists[root]['param'][0], debug=self.debug, noglob=True, verbosity=self.verbosity) param = success if not success: self.logfile.write(param_dict) elif file_lists[root]['param_count'] > 1: self.log.warning('Multiple param files found: {}'.format( file_lists[root]['param'])) multi = True if file_lists[root]['cell_count'] == 1: cell_dict, success = cell2dict(file_lists[root]['cell'][0], db=True, debug=self.debug, noglob=True, verbosity=self.verbosity) cell = success if not success: self.logfile.write(str(cell_dict)) elif file_lists[root]['cell_count'] > 1: multi = True self.log.warning('Multiple param files found: {}'.format( file_lists[root]['cell'])) if multi: found_multi = False for param_name in file_lists[root]['param']: for cell_name in file_lists[root]['cell']: if param_name.split('.')[0] in cell_name: cell_dict, success = cell2dict( cell_name, debug=self.debug, db=True, verbosity=self.verbosity) cell = success if not success: self.logfile.write(str(cell_dict)) continue param_dict, success = param2dict( param_name, debug=self.debug, verbosity=self.verbosity) param = success if not success: self.logfile.write(param_dict) if success: found_multi = True self.log.info( 'Found matching cell and param files: {}'. format(param_name)) break if not found_multi: self.log.warning( "Unable to find matching cell and param files for {}". format(root)) # combine cell and param dicts for folder input_dict = dict() if cell and param: input_dict.update(cell_dict) input_dict.update(param_dict) input_dict['source'] = cell_dict['source'] + param_dict['source'] else: self.logfile.write( '! {} failed to scrape any cell and param \n'.format(root)) # create res dicts and combine them with input_dict for _, _file in enumerate( loading_bar(file_lists[root]['res'], verbosity=self.verbosity)): exts_with_precedence = ['.castep', '.history', 'history.gz'] # check if a castep-like file exists instead of scraping res if any([ _file.replace('.res', ext) in file_lists[root]['castep'] for ext in exts_with_precedence ]): for ext in exts_with_precedence: if _file.replace('.res', ext) in file_lists[root]['castep']: struct_dict, success = castep2dict( _file.replace('.res', ext), debug=False, noglob=True, dryrun=self.args.get('dryrun'), verbosity=self.verbosity) break # otherwise, scrape res file else: struct_dict, success = res2dict(_file, verbosity=self.verbosity, noglob=True) if not success: self.logfile.write('! {}'.format(struct_dict)) else: try: final_struct = copy.deepcopy(input_dict) final_struct.update(struct_dict) # calculate kpoint spacing if not found if 'lattice_cart' not in final_struct and 'lattice_abc' not in final_struct: msg = '! {} missing lattice'.format(_file) self.logfile.write(msg) if 'kpoints_mp_spacing' not in final_struct and 'kpoints_mp_grid' in final_struct: final_struct['kpoints_mp_spacing'] = calc_mp_spacing( final_struct['lattice_cart'], final_struct['mp_grid']) final_struct['source'] = struct_dict['source'] if 'source' in input_dict: final_struct['source'] += input_dict['source'] if not self.dryrun: final_struct.update(self.tag_dict) import_count += self._struct2db(final_struct) except Exception as exc: self.log.error('Unexpected error for {}, {}'.format( _file, final_struct)) raise exc return import_count
def test_relax_to_queue(self): """ Mimic GA and test Queue relaxations. """ newborn, s = res2dict( REAL_PATH + "/data/structures/LiAs_testcase.res", verbosity=VERBOSITY, db=False, ) assert s cell_dict, s = cell2dict(REAL_PATH + "/data/LiAs_tests/LiAs.cell", verbosity=VERBOSITY, db=False) assert s param_dict, s = param2dict(REAL_PATH + "/data/LiAs_tests/LiAs.param", verbosity=VERBOSITY, db=False) assert s node = None executable = "castep" newborn["source"] = [REAL_PATH + "/data/GA_TESTCASE.res"] shutil.copy(REAL_PATH + "data/pspots/Li_00PBE.usp", ".") shutil.copy(REAL_PATH + "data/pspots/As_00PBE.usp", ".") queue = mp.Queue() relaxer = ComputeTask( ncores=NCORES, nnodes=None, node=node, res=newborn, param_dict=param_dict, cell_dict=cell_dict, verbosity=VERBOSITY, killcheck=True, reopt=False, executable=executable, output_queue=queue, start=True, ) # store proc object with structure ID, node name, output queue and number of cores proc = (1, node, mp.Process(target=relaxer.relax), NCORES) proc[2].start() while proc[2].is_alive(): time.sleep(1) result, success = castep2dict("completed/GA_TESTCASE.castep") queue_result = queue.get() match_dict = dict() for key in queue_result: if key in ["source", "site_occupancy", "geom_iter"]: continue match_dict[key] = queue_result[key] == result[key] if not match_dict[key]: print(key, queue_result[key], result[key]) completed_exists = isfile("completed/GA_TESTCASE.res") input_exists = isfile("input/GA_TESTCASE.res") self.assertTrue(completed_exists, "couldn't find output file!") self.assertTrue(input_exists, "couldn't find shutil.copy of input file!") self.assertTrue(success, "couldn't parse output file!") self.assertTrue(all([match_dict[key] for key in match_dict]))
def test_benchmark_manycore_scf(self): """ Test the time taken to perform a set number of SCF steps on many cores. """ from os import makedirs seed = "_LiC.res" shutil.copy(REAL_PATH + "data/structures/LiC.res", "_LiC.res") cell_dict, s = cell2dict( REAL_PATH + "/data/benchmark/LiC_scf/LiC_scf.cell", verbosity=VERBOSITY, db=False, ) self.assertTrue(s) param_dict, s = param2dict( REAL_PATH + "/data/benchmark/LiC_scf/LiC_scf.param", verbosity=VERBOSITY, db=False, ) self.assertTrue(s) shutil.copy(REAL_PATH + "data/pspots/Li_00PBE.usp", ".") shutil.copy(REAL_PATH + "data/pspots/C_00PBE.usp", ".") with self.assertRaises(CalculationError): ComputeTask( ncores=NCORES, nnodes=None, node=None, res=seed, param_dict=param_dict, cell_dict=cell_dict, verbosity=0, executable=EXECUTABLE, start=True, ) outputs_exist = [ isfile("bad_castep/_LiC.res"), isfile("bad_castep/_LiC.castep"), ] results, s = castep2dict("bad_castep/_LiC.castep", db=False) makedirs(REAL_PATH + "/data/benchmark/results", exist_ok=True) shutil.copy( "bad_castep/_LiC.castep", REAL_PATH + "/data/benchmark/results/_LiC_{}core_castep{}.castep".format( results.get("num_mpi_processes", 0), results.get("castep_version", "xxx"), ), ) self.assertTrue(all(outputs_exist), "couldn't find output files!") self.assertTrue(s, "couldn't read output files!") print(results["_time_estimated"]) benchmark_data = {2: 2 * 7.4, 4: 4 * 4.0, 12: 16.8, 14: 22.4, 18: 23.4} warnings.warn( RuntimeWarning( "Run took {} s with {} MPI processes, with cumulative CPU time of {:.1f} s. Benchmark data\n = {}" .format( results["_time_estimated"], results["num_mpi_processes"], results["_time_estimated"] * results["num_mpi_processes"], benchmark_data, )))
def castep_setup(self): """ Set up CASTEP jobs from res files, and $seed.cell/param. """ # read cell/param files exts = ['cell', 'param'] for ext in exts: if not os.path.isfile('{}.{}'.format(self.seed, ext)): raise InputError( 'Failed to find {ext} file, {seed}.{ext}'.format( ext=ext, seed=self.seed)) self.cell_dict, cell_success = cell2dict(self.seed + '.cell', db=False, lattice=False, positions=True) if not cell_success: print(self.cell_dict) raise InputError('Failed to parse cell file') self.param_dict, param_success = param2dict(self.seed + '.param', db=False) if not param_success: print(self.param_dict) raise InputError('Failed to parse param file') # scan directory for files to run self.file_lists = defaultdict(list) self.file_lists['res'] = glob.glob('*.res') if any(self.seed == file.replace('.res', '') for file in self.file_lists['res']): error = ( "Found .res file with same name as seed: {}.res. This will wreak havoc on your calculations!\n" .format(self.seed) + "Please rename either your seed.cell/seed.param files, or rename the offending {}.res" .format(self.seed)) raise InputError(error) if not self.file_lists['res']: error = ( 'run3 in CASTEP mode requires at least 1 res file in folder, found {}' .format(len(self.file_lists['res']))) raise InputError(error) if (len(self.file_lists['res']) < self.nprocesses and not any( [self.args.get('conv_cutoff'), self.args.get('conv_kpt')])): raise InputError( 'Requested more processes than there are jobs to run!') # do some prelim checks of parameters if self.param_dict['task'].upper() in [ 'GEOMETRYOPTIMISATION', 'GEOMETRYOPTIMIZATION' ]: if 'geom_max_iter' not in self.param_dict: raise InputError('geom_max_iter is unset, please fix this.') if int(self.param_dict['geom_max_iter']) <= 0: raise InputError('geom_max_iter is only {}!'.format( self.param_dict['geom_max_iter'])) # parse convergence args and set them up self.convergence_run_setup() # delete source from cell and param del self.cell_dict['source'] del self.param_dict['source']