def resub_scf(outfile_path): """Resubmits a job that's having trouble converging the scf with different level shifts (1.0 and 0.1). Parameters ---------- outfile_path : str The name of an output file. Returns ------- Resub_flag : bool True if resubmitted. """ # Resubmits a job that's having trouble converging the scf with different level shifts (1.0 and 0.1) history = resub_history() history.read(outfile_path) resubbed_before = False if 'SCF convergence error, level shifts adjusted to aid convergence' in history.notes: resubbed_before = True history.status = os.path.split(outfile_path)[ -1] + ' has been submitted with levels shifted and is still encountering an scf error' history.save() if 'Needs clean resub' in history.notes: resubbed_before = True history.status = os.path.split(outfile_path)[ -1] + ' job recovery has failed - requesting resub_scf() after clean resubmission round' history.save() if not resubbed_before: save_run(outfile_path, rewrite_inscr=False) history = resub_history() history.read(outfile_path) history.resub_number += 1 history.status = 'Level shifts adjusted to assist convergence' history.needs_resub = True history.notes.append('SCF convergence error, level shifts adjusted to aid convergence') history.save() machine=tools.get_machine() root = outfile_path.rsplit('.', 1)[0] name = os.path.split(root)[-1] directory = os.path.split(outfile_path)[0] infile_dict = manager_io.read_infile(outfile_path) home = os.getcwd() if len(directory) > 0: # if the string is blank, then we're already in the correct directory os.chdir(directory) infile_dict['levelshifta'], infile_dict['levelshiftb'] = 1.0, 0.1 infile_dict['machine'] = machine manager_io.write_input(infile_dict) manager_io.write_jobscript(name, machine=machine) os.chdir(home) tools.qsub(root + '_jobscript') return True else: return False
def resub_spin(outfile_path): # resubmits a spin contaminated job with blyp to help convergence to a non-spin contaminated solution history = resub_history() history.read(outfile_path) resubbed_before = False if 'Spin contaminated, lowering HFX to aid convergence' in history.notes: resubbed_before = True history.status = os.path.split( outfile_path )[-1] + ' has been submitted with lower HFX and still converges to a spin contaminated solution' history.save() if 'Needs clean resub' in history.notes: resubbed_before = True history.status = os.path.split( outfile_path )[-1] + ' job recovery has failed - requesting resub_spin() after clean resubmission round' history.save() if 'HFXresampling' in outfile_path: resubbed_before = True history.status = os.path.split( outfile_path )[-1] + ' is spin contaminated, but submitting with lower HFX does not make sense for HFX resampling jobs' history.save() if not resubbed_before: save_run(outfile_path, rewrite_inscr=False) history = resub_history() history.read(outfile_path) history.resub_number += 1 history.status = 'HFX altered to assist convergence' history.needs_resub = True history.notes.append( 'Spin contaminated, lowering HFX to aid convergence') history.save() root = outfile_path.rsplit('.', 1)[0] name = os.path.split(root)[-1] directory = os.path.split(outfile_path)[0] infile_dict = manager_io.read_infile(outfile_path) home = os.getcwd() if len( directory ) > 0: # if the string is blank, then we're already in the correct directory os.chdir(directory) infile_dict['method'] = 'blyp' infile_dict['machine'] = machine manager_io.write_input(infile_dict) manager_io.write_jobscript(name, machine=machine) os.chdir(home) tools.qsub(root + '_jobscript') return True else: return False
def resub_oscillating_scf(outfile_path): # Resubmits a job that's having trouble converging the scf with different level shifts (1.0 and 0.1) history = resub_history() history.read(outfile_path) resubbed_before = False if 'SCF convergence error, precision and grid adjusted to aid convergence' in history.notes: resubbed_before = True history.status = os.path.split( outfile_path )[-1] + ' has been submitted with higher precision and grid and is still encountering an scf error' history.save() if 'Needs clean resub' in history.notes: resubbed_before = True history.status = os.path.split( outfile_path )[-1] + ' job recovery has failed - requesting resub_oscillating_scf() after clean resubmission round' history.save() if not resubbed_before: save_run(outfile_path, rewrite_inscr=False) history = resub_history() history.read(outfile_path) history.resub_number += 1 history.status = 'precision and grid adjusted to assist convergence' history.notes.append( 'SCF convergence error, precision and grid adjusted to aid convergence' ) history.save() root = outfile_path.rsplit('.', 1)[0] name = os.path.split(root)[-1] directory = os.path.split(outfile_path)[0] infile_dict = manager_io.read_infile(outfile_path) home = os.getcwd() if len( directory ) > 0: # if the string is blank, then we're already in the correct directory os.chdir(directory) infile_dict['precision'], infile_dict['dftgrid'], infile_dict[ 'dynamicgrid'] = "double", 5, "no" infile_dict['machine'] = machine manager_io.write_input(infile_dict) manager_io.write_jobscript(name, machine=machine) os.chdir(home) tools.qsub(root + '_jobscript') return True else: return False
def resub_tighter(outfile_path): """Resubmits a thermo job with the gradient error problem. Finds the parent job and resubmits it with a tighter scf convergence criteria. Parameters ---------- outfile_path : str The name of an output file. Returns ------- Resub_flag : bool True if resubmitted. """ # Takes the path to the outfile of a thermo job with the gradient error problem # Finds the parent job and resubmits it with a tighter scf convergence criteria machine=tools.get_machine() name = os.path.split(outfile_path)[-1].rsplit('.', 1)[0] parent_name = name.rsplit('_', 1)[0] parent_directory = os.path.split(os.path.split(outfile_path)[0])[0] parent_path = os.path.join(parent_directory, parent_name + '.out') ultratight_path = os.path.join(parent_directory, parent_name + '_ultratight', parent_name + '_ultratight.out') scr_needs_to_be_saved = False if os.path.exists(ultratight_path): # This ultratight resubmission has happend before, need to archive the results save_run(ultratight_path, rewrite_inscr=False, save_scr_flag=False) scr_needs_to_be_saved = True # Need to save the scr AFTER prepping the new ultratight run. This helps keep compatibility with other functions history = resub_history() history.read(ultratight_path) history.resub_number += 1 history.status = 'Running with tightened convergence thresholds' history.needs_resub = False history.notes.append('Further tightening convergence thresholds') history.save() jobscript = tools.prep_ultratight(parent_path) # Prep tighter convergence run if scr_needs_to_be_saved: save_scr(ultratight_path, rewrite_inscr=False) tools.qsub(jobscript) # Submit tighter convergence run # Set the original thermo run to wait for the ultratight run to finish history = resub_history() history.read(outfile_path) history.waiting = ultratight_path history.save() return True
def grab_waiting(path): if os.path.isfile(path.rsplit('.', 1)[0] + '.pickle'): history = resub_history() history.read(path) return history.waiting raise Exception( 'Attempting to grab a "waiting" criteria that does not exist')
def load_history(PATH): # takes the path to either an outfile or the resub_history pickle # returns the resub_history class object history = resub_history() history.read(PATH) return history
def check_needs_resub(path): if os.path.isfile(path.rsplit('.', 1)[0] + '.pickle'): history = resub_history() history.read(path) return history.needs_resub else: return False
def check_waiting(path): if os.path.isfile(path.rsplit('.', 1)[0] + '.pickle'): history = resub_history() history.read(path) if history.waiting: return True return False
def simple_resub(outfile_path): """Resubmits a job without changing parameters. Particularly useful for CUDA errors. Parameters ---------- outfile_path : str The name of an output file. Returns ------- Resub_flag : bool True if resubmitted. """ # Resubmits a job without changing parameters. Particularly useful for CUDA errors. save_run(outfile_path, rewrite_inscr=False) history = resub_history() history.read(outfile_path) history.resub_number += 1 history.notes.append('Resubbed for unknown error') history.save() root = outfile_path.rsplit('.', 1)[0] tools.qsub(root + '_jobscript') return True
def check_chronic_failure(path): if os.path.isfile(path.rsplit('.', 1)[0] + '.pickle'): history = resub_history() history.read(path) if history.resub_number >= max_resub: return True else: return False
def resub_thermo(outfile_path): # Similar to simple resub, but specific for addressing thermo gradient errors # Checks for the existance of an ultratight version of this run. If it exists, uses the most up to date version for the new thermo run save_run(outfile_path, rewrite_inscr=False) history = resub_history() history.read(outfile_path) history.resub_number += 1 history.status = 'Normal' history.notes.append( 'Resubmitting thermo, possibly with a better initial geo') history.needs_resub = False history.save() name = os.path.split(outfile_path)[-1] name = name.rsplit('.', 1)[0] directory = os.path.split(outfile_path)[0] parent_name = name.rsplit('_', 1)[0] parent_directory = os.path.split(os.path.split(outfile_path)[0])[0] ultratight_dir = os.path.join(parent_directory, parent_name + '_ultratight') infile_dict = manager_io.read_infile(outfile_path) if os.path.exists(ultratight_dir): if os.path.exists(os.path.join(ultratight_dir, 'scr', 'optim.xyz')): tools.extract_optimized_geo( os.path.join(ultratight_dir, 'scr', 'optim.xyz')) shutil.copy(os.path.join(ultratight_dir, 'scr', 'optimized.xyz'), outfile_path.rsplit('.', 1)[0] + '.xyz') else: raise Exception( 'Unable to identify the ultratight geometry for run: ' + outfile_path) if infile_dict['spinmult'] == 1 and os.path.exists( os.path.join(ultratight_dir, 'scr', 'c0')): shutil.copy(os.path.join(ultratight_dir, 'scr', 'c0'), os.path.join(directory, 'c0')) elif infile_dict['spinmult'] != 1 and os.path.exists( os.path.join(ultratight_dir, 'scr', 'ca0')) and os.path.exists( os.path.join(ultratight_dir, 'scr', 'cb0')): shutil.copy(os.path.join(ultratight_dir, 'scr', 'ca0'), os.path.join(directory, 'ca0')) shutil.copy(os.path.join(ultratight_dir, 'scr', 'cb0'), os.path.join(directory, 'cb0')) else: raise Exception( 'Unable to find wavefunction files for ultratight geometry for run: ' + outfile_path) else: raise Exception( 'An ultratight run does not exist for this thermo file. Consider calling simple_resub() or resub_tighter() instead of resub_thermo()' ) jobscript = outfile_path.rsplit('.', 1)[0] + '_jobscript' tools.qsub(jobscript) return True
def clean_resub(outfile_path): # Resubmits a job with default parameters, useful for undoing level shift or hfx alterations save_run(outfile_path) history = resub_history() history.read(outfile_path) history.resub_number += 1 history.status = 'Normal' history.notes.append('Needs clean resub') history.needs_resub = False history.save() root = outfile_path.rsplit('.', 1)[0] name = os.path.split(root)[-1] directory = os.path.split(outfile_path)[0] infile_dict = manager_io.read_infile(outfile_path) home = os.getcwd() if len( directory ) > 0: # if the string is blank, then we're already in the correct directory os.chdir(directory) if os.path.isfile('inscr/optimized.xyz'): coordinates = 'inscr/optimized.xyz' # Should trigger for optimization runs elif os.path.isfile(name + '.xyz'): coordinates = name + '.xyz' # Should trigger for single point runs else: raise ValueError( 'No coordinates idenfied for clean in resubmission in directory ' + os.getcwd()) configure_dict = manager_io.read_configure('in_place', outfile_path) infile_dict['coordinates'] = coordinates infile_dict['method'] = configure_dict['method'] infile_dict['levelshifta'], infile_dict['levelshiftb'] = configure_dict[ 'levela'], configure_dict['levelb'] infile_dict['dispersion'] = configure_dict['dispersion'] infile_dict['constraints'] = False infile_dict['machine'] = machine if infile_dict['spinmult'] == 1: infile_dict['guess'] = 'inscr/c0' manager_io.write_input(infile_dict) else: infile_dict['guess'] = 'inscr/ca0 inscr/cb0' manager_io.write_input(infile_dict) manager_io.write_jobscript(name, custom_line='# -fin inscr/', machine=machine) os.chdir(home) tools.qsub(root + '_jobscript') return True
def save_run(outfile_path, rewrite_inscr=True, save_scr_flag=True): def write(list_of_lines, path): fil = open(path, 'w') for i in list_of_lines: fil.write(i) fil.close() if save_scr_flag: scr_path = save_scr(outfile_path, rewrite_inscr=rewrite_inscr) else: scr_path = os.path.join(os.path.split(outfile_path)[0], 'scr') history = resub_history() history.read(outfile_path) f = open(outfile_path, 'r') out_lines = f.readlines() f.close() history.outfiles.append(out_lines) infile_path = outfile_path.rsplit('.', 1)[0] + '.in' f = open(infile_path, 'r') in_lines = f.readlines() f.close() history.infiles.append(in_lines) jobscript_path = outfile_path.rsplit('.', 1)[0] + '_jobscript' f = open(jobscript_path, 'r') job_lines = f.readlines() f.close() history.jobscripts.append(job_lines) xyz_path = outfile_path.rsplit('.', 1)[0] + '.xyz' f = open(xyz_path, 'r') xyz_lines = f.readlines() f.close() history.xyzs.append(xyz_lines) history.save() if scr_path: # Additionally, write this information to textfile so it's earch to find home = os.getcwd() os.chdir(scr_path) write(out_lines, 'old_outfile') write(in_lines, 'old_infile') write(job_lines, 'old_job') write(xyz_lines, 'old_xyz') os.chdir(home)
def load_history(PATH): """Load a resub history object. Parameters ---------- PATH : str The name of an output file or history object pickle file. Returns ------- history : resub_history resub_history class instance for history object of job. """ # takes the path to either an outfile or the resub_history pickle # returns the resub_history class object history = resub_history() history.read(PATH) return history
def resub_bad_geo(outfile_path, home_directory): """Resubmits a job that's converged to a bad geometry with additional contraints. Parameters ---------- outfile_path : str The name of an output file. home_directory : str Path to the base directory of the run. Returns ------- Resub_flag : bool True if resubmitted. """ # Resubmits a job that's converged to a bad geometry with additional contraints history = resub_history() history.read(outfile_path) resubbed_before = False if 'Bad geometry detected, adding constraints and trying again' in history.notes: resubbed_before = True history.status = os.path.split(outfile_path)[ -1] + " has been submitted with additional constraints and still isn't a good geometry" history.save() if 'Needs clean resub' in history.notes: resubbed_before = True history.status = os.path.split(outfile_path)[ -1] + ' job recovery has failed - requesting resub_bad_geo after clean resubmission round' history.save() if not resubbed_before: save_run(outfile_path, rewrite_inscr=True) history = resub_history() history.read(outfile_path) history.resub_number += 1 history.status = 'Constraints added to help convergence' history.needs_resub = True history.notes.append('Bad geometry detected, adding constraints and trying again') history.save() machine=tools.get_machine() root = outfile_path.rsplit('.', 1)[0] name = os.path.split(root)[-1] directory = os.path.split(outfile_path)[0] infile_dict = manager_io.read_infile(outfile_path) if infile_dict['constraints']: raise Exception( 'resub.py does not currently support the use of external atom constraints. These will be overwritten by clean_resub() during job recovery') goal_geo = manager_io.read_configure(home_directory, outfile_path)['geo_check'] if not goal_geo: raise Exception( 'Goal geometry not specified, job ' + outfile_path + ' should not have been labelled bad geo!') else: metal_index, bonded_atom_indices = moltools.get_metal_and_bonded_atoms(outfile_path, goal_geo) # convert indexes from zero-indexed to one-indexed metal_index += 1 bonded_atom_indices = [index + 1 for index in bonded_atom_indices] # Convert to TeraChem input syntax constraints = ['bond ' + str(metal_index) + '_' + str(index) + '\n' for index in bonded_atom_indices] home = os.getcwd() if len(directory) > 0: # if the string is blank, then we're already in the correct directory os.chdir(directory) infile_dict['constraints'] = constraints infile_dict['machine'] = machine manager_io.write_input(infile_dict) manager_io.write_jobscript(name, machine=machine) os.chdir(home) tools.qsub(root + '_jobscript') return True else: return False
def reset(outfile_path): """Returns the run to the state it was after the first run, before job recovery acted on it. Parameters ---------- outfile_path : str The name of an output file. """ # Returns the run to the state it was after the first run, before job recovery acted on it pickle_path = outfile_path.rsplit('.', 1)[0] + '.pickle' if os.path.isfile(pickle_path): print('Resetting run: ' + os.path.split(outfile_path)[-1].rsplit('.', 1)[0]) old_path = os.path.join(os.path.split(outfile_path)[0], 'pre_reset') if not os.path.isdir(old_path): os.mkdir(old_path) # Find all the stdout and stderr files related to previous runs. queue_output = glob.glob(outfile_path.rsplit('.', 1)[0] + '.e*') queue_output.extend(glob.glob(outfile_path.rsplit('.', 1)[0] + '.pe*')) queue_output.extend(glob.glob(outfile_path.rsplit('.', 1)[0] + '.po*')) queue_output.extend(glob.glob(outfile_path.rsplit('.', 1)[0] + '.o*')) queue_output = [i for i in queue_output if i[-1] in ['1', '2', '3', '4', '5', '6', '7', '8', '9', '0']] # remove all files from states after the specified state move = [] identifier = 1 while True: move.extend(glob.glob(os.path.join(os.path.split(outfile_path)[0], 'scr_' + str(identifier)))) identifier += 1 if len(glob.glob(os.path.join(os.path.split(outfile_path)[0], 'scr_' + str(identifier)))) == 0: break # break when all scr_? files are found. # remove all files for derivative jobs spawned based on this job derivative_types = ['solvent', 'vertEA', 'vertIP', 'thermo', 'kp', 'rm', 'ultratight', 'HFXresampling', 'functional'] possible = [i for i in glob.glob(os.path.join(os.path.split(outfile_path)[0],'*')) if os.path.isdir(i)] for folder in possible: if os.path.split(outfile_path)[1].rsplit('.',1)[0] in folder: derivative = False for typ in derivative_types: if typ in folder: derivative = True if derivative: shutil.rmtree(folder) #rename outfile and jobscript files shutil.move(outfile_path, outfile_path[:-4] + '.old') # rename old out so it isn't found in .out searches shutil.move(outfile_path[:-4] + '_jobscript', outfile_path[ :-4] + '_oldjob') # rename old jobscript so it isn't thought to be job that hasn't started yet move.append(outfile_path[:-4] + '.old') move.append(outfile_path[:-4] + '.xyz') move.append(outfile_path[:-4] + '.in') move.append(outfile_path[:-4] + '_oldjob') if os.path.isdir(os.path.join(os.path.split(outfile_path)[0], 'inscr')): move.append(os.path.join(os.path.split(outfile_path)[0], 'inscr')) move.extend(queue_output) scr_path = os.path.join(os.path.split(outfile_path)[0], 'scr') move.append(scr_path) for path in move: # move the paths to their new location, Random numbers prevent clashes try: shutil.move(path, os.path.join(old_path, str(np.random.randint(999999999)) + '_' + os.path.split(path)[-1])) except: print('No file found for: ' + path) # Rewrite the .xyz, .in, jobscript, and .out file to be the same as they were after the first run history = resub_history() history.read(pickle_path) outfile = history.outfiles[0] infile = history.infiles[0] jobscript = history.jobscripts[0] xyz = history.xyzs[0] writer = open(outfile_path, 'w') for i in outfile: writer.write(i) writer.close() writer = open(outfile_path.rsplit('.', 1)[0] + '.in', 'w') for i in infile: writer.write(i) writer.close() writer = open(outfile_path.rsplit('.', 1)[0] + '.xyz', 'w') for i in xyz: writer.write(i) writer.close() writer = open(outfile_path.rsplit('.', 1)[0] + '_jobscript', 'w') for i in jobscript: writer.write(i) writer.close() shutil.move(scr_path + '_0', scr_path) shutil.move(pickle_path, os.path.join(old_path, str(np.random.randint(999999999)) + '_resub_history'))
def save_run(outfile_path, rewrite_inscr=True, save_scr_flag=True): """Save the outfile within the resub_history pickle object. Parameters ---------- outfile_path : str The name of an output file. rewrite_inscr : bool, optional Determines whether to copy this runs wfn and optimized geometry to the inscr directory. Default is True. save_scr_flag : bool, optional Determine wheether to store scr. Default is True. """ def write(list_of_lines, path): fil = open(path, 'w') for i in list_of_lines: fil.write(i) fil.close() if save_scr_flag: scr_path = save_scr(outfile_path, rewrite_inscr=rewrite_inscr) else: scr_path = os.path.join(os.path.split(outfile_path)[0], 'scr') history = resub_history() history.read(outfile_path) f = open(outfile_path, 'r') out_lines = f.readlines() f.close() history.outfiles.append(out_lines) infile_path = outfile_path.rsplit('.', 1)[0] + '.in' f = open(infile_path, 'r') in_lines = f.readlines() f.close() history.infiles.append(in_lines) jobscript_path = outfile_path.rsplit('.', 1)[0] + '_jobscript' f = open(jobscript_path, 'r') job_lines = f.readlines() f.close() history.jobscripts.append(job_lines) xyz_path = outfile_path.rsplit('.', 1)[0] + '.xyz' f = open(xyz_path, 'r') xyz_lines = f.readlines() f.close() history.xyzs.append(xyz_lines) history.save() if scr_path: # Additionally, write this information to textfile so it's earch to find home = os.getcwd() os.chdir(scr_path) write(out_lines, 'old_outfile') write(in_lines, 'old_infile') write(job_lines, 'old_job') write(xyz_lines, 'old_xyz') os.chdir(home)