def main(_cores: int = None,
_time: int = None,
_nodes: int = 1,
_sub: bool = True):
# Load in the logger
utilities.load_logger_config()
logger = logging.getLogger(__name__)
# Load in the configuration for this user
logger.debug("Calling load_phd_config")
config = utilities.load_phd_config()
# Parse the configuration!
logger.debug("Assigning variables for users queueing")
max_nodes = config['QUEUING']['max_nodes']
node_types = config['QUEUING']['node_types']
max_time = config['QUEUING']['max_time']
high_priority_nodes = config['QUEUING']['high_priority_nodes']
# Creates a list of (num_cores, max_time)
avail_nodes = list(zip(node_types, max_time))
# Assign the vars passed
logger.debug("Checking for any passed args")
cores = _cores
time = _time
nodes = _nodes
sub = _sub
# If no args passed, check for command line args
if cores is None or time is None:
# Arg Parser Here
logger.debug("Parsing args")
parser = argparse.ArgumentParser(
description="Submits a DMD simulation to a queuing system")
parser.add_argument(
"-N",
dest="nodes",
type=int,
required=False,
default=1,
help=f"The number of nodes to run the DMD simulation on.")
parser.add_argument(
"-n",
dest="cores",
type=int,
required=True,
help=f"The number of cores per node to run the DMD simulation with."
)
parser.add_argument(
"-t",
dest="time",
type=int,
required=True,
help="The amount of time to submit the job for (hours).")
parser.add_argument("-sub",
dest="sub",
action='store_true',
help="Submit the job to the queueing system.")
args = parser.parse_args()
if args.nodes > max_nodes or args.cores > max(
node_types) or args.time > max(max_time):
logger.error(
"Invalid args parsed based off of what user has specified in config file"
)
parser.print_help()
sys.exit(1)
cores = args.cores
time = args.time
nodes = args.nodes
sub = args.sub
# Finds the appropriate node to use
logger.debug("Checking to ensure user has enough resources")
big_enough_nodes = list(filter(lambda i: i[0] >= cores, avail_nodes))
enough_time_nodes = list(filter(lambda i: i[1] >= time, big_enough_nodes))
possible_nodes = sorted(enough_time_nodes, key=lambda x: x[0])
if not possible_nodes:
logger.error("User does not have access to resources")
raise ValueError
# Grab the minimum processing node
node_type = min(possible_nodes)[0]
logger.debug(f"Using node type: {node_type}")
high_priority = True if node_type in high_priority_nodes else False
# Create the jinja2 dictionary
job_params = {
"date": datetime.datetime.now(),
"submit_dir": os.path.realpath(os.getcwd()),
"node_type": node_type,
"cores": cores,
"nodes": nodes,
"high_priority": ",highp" if high_priority else "",
"user": os.environ["USER"],
"job_name": os.path.basename(os.getcwd()),
"run_script": os.path.join(os.path.dirname(__file__), "rundmd.py"),
"time": time
}
# Make the submit script for the queuing system
home = os.path.expanduser("~")
path_to_template = os.path.join(home, ".config/phd3")
if not os.path.isdir(path_to_template):
path_to_template = os.path.join(home, ".phd3")
logger.debug(f"Finding file in: {path_to_template} ")
templateLoader = jinja2.FileSystemLoader(searchpath=path_to_template)
templateEnv = jinja2.Environment(loader=templateLoader)
TEMPLATE_FILE = "submit.j2"
try:
template = templateEnv.get_template(TEMPLATE_FILE)
except jinja2.exceptions.TemplateNotFound:
logger.error(f"Template file not found in {path_to_template}")
raise
logger.debug("Found template file!")
# Now we dump text to a file
outputText = template.render(job_params)
try:
logger.debug(f"Writing out submit file: {constants.SUBMIT_FILE_NAME}")
with open(constants.SUBMIT_FILE_NAME, "w+") as submit_file:
submit_file.write(outputText)
except IOError:
logger.exception("Could not write out submit file!")
sys.exit(1)
if not os.path.isfile("dmdinput.json"):
logger.warn("[WARNING] ==>> No dmdinput.json file found")
# Submit the Job!!
if sub:
logger.info("Submitting job to queue")
with Popen(
f"{config['QUEUING']['submit']} {constants.SUBMIT_FILE_NAME}",
shell=True,
universal_newlines=True,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
bufsize=1,
env=os.environ) as shell:
while shell.poll() is None:
logger.info(shell.stdout.readline().strip())
logger.info(shell.stderr.readline().strip())
def __init__(self,
cores,
run_dir: str = './',
time=-1,
coord=None,
parameters: dict = None):
logger.debug("Beginning Calculation")
logger.debug("Initializing variables")
self._submit_directory = os.path.abspath(os.getcwd())
self._scratch_directory = os.path.abspath(run_dir)
self._config = utilities.load_phd_config()
self._turbodir = self._config["PATHS"]['TURBODIR']
self._cores = cores
self._time_to_run = time
self._para_arch = "SMP"
self._resub = False
self._timer_went_off = False
self._coord_file = ""
#Start the general signal for sigUSR1
#This will call function calculation_alarm_handler if SIGUSR1 is sent to the node
signal.signal(signal.SIGUSR1, self.calculation_alarm_handler)
# We need to make sure that we have some parameters for this job!
if parameters is None:
logger.debug("Checking for definput.json file")
if os.path.isfile("definput.json"):
self._parameter_file = os.path.join(self._submit_directory,
"definput.json")
else:
logger.error("No parameters specified for the job!")
raise FileNotFoundError("definput.json")
# read in the parameters here to the dictionary!!
try:
# Read in the user supplied parameters from the input file
logger.debug("Loading in parameters")
with open(self._parameter_file, 'r') as inputfile:
self._raw_parameters = json.load(inputfile)
except IOError:
logger.exception("Could not open the parameter file correctly")
raise
else:
logger.debug("Using parameters passed")
self._raw_parameters = parameters
# we want to make sure that turbomole environment is setup properly first now in case we have to run define or x2t!
logger.debug("Setting up TURBOMOLE Environment")
self.setup_turbomole_env_parallel(self._cores, self._para_arch,
self._scratch_directory,
self._turbodir)
#Check to make sure that we have valid parameters before continuing!
try:
logger.debug("Ensuring proper parameters")
utilities.valid_qm_parameters(self._raw_parameters)
except exceptions.ParameterError:
logger.error("Invalid parameters provided!")
raise
# Now we are in the submit directory, check for a coord file first!
if coord is None:
logger.debug("Looking for a coord file")
if os.path.isfile("coord"):
logger.debug(
f"Found a coord file: {os.path.join(self._submit_directory, 'coord')}"
)
self._coord_file = os.path.join(self._submit_directory,
"coord")
#Check to see if there is a .xyz to use instead
else:
logger.debug(
f"Looking for an .xyz file in: {self._submit_directory}")
files = [
f for f in os.listdir(self._submit_directory)
if os.path.isfile(os.path.join(self._submit_directory, f))
]
for f in files:
if os.path.splitext(f)[-1].lower() == ".xyz":
logger.info(
f"Found an .xyz file to use: {f}, will convert to coord"
)
utilities.xyz_to_coord(f)
self._coord_file = os.path.join(
self._submit_directory, "coord")
break
#No such luck, can't do the job then!
if self._coord_file == "":
logger.error(
"You don't seem to have provided a coord file, cannot proceed!"
)
raise FileNotFoundError("coord")
else:
self._coords = coord
# At this point we will set up the turbomole job if it is not already setup (or more importantly, the control file!)
if not os.path.isfile("control"):
logger.debug(
"Control file does not exist. Attempting to run setupturbomole.py"
)
stm = setupTMjob(self._raw_parameters)
#Remove the stop file if it exists right now, otherwise we will have issues
if os.path.isfile(os.path.join(self._submit_directory, "stop")):
logger.warning("Stop file exists before job has run")
logger.warning("Removing stop file and continuing")
os.remove(os.path.join(self._submit_directory, "stop"))
#We now want to transfer all of the files from this directory to the scratch directory if a diff. directory
if self._scratch_directory != self._submit_directory:
logger.info(
f"Copying file from {self._submit_directory} to {self._scratch_directory}"
)
self._scratch_directory = os.path.join(
self._scratch_directory,
os.path.basename(self._submit_directory))
utilities.copy_directories(self._submit_directory,
self._scratch_directory)
os.chdir(self._scratch_directory)
# Check for MOs in tar.gz format
for mo_file in constants.MO_FILES:
if os.path.isfile(f"{mo_file}.tar.gz"):
logger.info(f"[Untaring] ==>> {f}")
with tarfile.open(f"{mo_file}.tar.gz", "r:gz") as tar:
tar.extractall()
os.remove(f"{mo_file}.tar.gz")
switcher = {
"numforce": self._numforce,
"nf": self._numforce,
"forceopt": self._forceopt,
"geo": self._geo,
"singlepoint": self._singlepoint,
"sp": self._singlepoint,
"trans": self._trans,
"escf": self._escf,
"woelfling": self._woelfling,
"egeo": self._egeo,
"eforceopt": self._eforceopt,
"enumforce": self._enumforce,
"enf": self._enumforce
}
calc = switcher.get(self._raw_parameters["calculation"].lower(), None)
try:
logger.debug("Beginning the TURBOMOLE calulcation")
# Arm the timer
if self._time_to_run != -1:
logger.info("Starting the timer")
signal.signal(signal.SIGALRM, self.calculation_alarm_handler)
signal.alarm(int((self._time_to_run * 60 - 45) * 60))
start = timer()
calc()
end = timer()
logger.info(
f"Time elapsed: {datetime.timedelta(seconds = int(end -start))}"
)
if self._time_to_run != -1:
# Turn off the timer now!
logger.info("Turning off timer")
signal.alarm(0)
except:
logger.error(
f"Failed executing the commands for the job type provided: {self._raw_parameters['calculation']}"
)
raise
#Now we copy the files back!
if self._scratch_directory != self._submit_directory:
utilities.copy_directories(self._scratch_directory,
self._submit_directory)
os.chdir(self._submit_directory)
shutil.rmtree(self._scratch_directory)
self.clean_up()
logger.debug("Finished Calculation")
if self._resub:
logger.info("Resubmitting the job!")
submitturbomole.main(_cores=self._cores, _time=self._time_to_run)
#!/usr/bin/env python3
"""
Author ==>> David J. Reilley
Author ==>> Matthew R. Hennefarth
Date ==>> April 16, 2020
"""
#Utility Paths
import phd3.utility.utilities as utilities
config = utilities.load_phd_config()
#Standard Library Imports
import logging
import os
import shutil
import pkg_resources
#3rd Party Libraries
import propka
from propka import run
#Titrate/PHD3
from . import montecarlo
from ..utility import constants, exceptions
logger = logging.getLogger(__name__)
#Turn off the propka logging
propka_logger = logging.getLogger("propka")
propka_logger.setLevel(logging.CRITICAL)
def free_energy_correction(temp: float = 298.15):
vibscale = 1
gcorr = 0
frozen_coords = False
if not os.path.isfile("coord"):
logger.exception("No coord file!")
raise ValueError("coord")
try:
with open('coord') as coordFile:
numatoms = -1
for line in coordFile:
if line[0] == '$' and "$coord" not in line:
logger.debug("Finished in the coordinate section")
break
if len(line.split()) > 4:
logger.info(
"Frozen Coordinates detected; calculating vibrational free energy correction"
)
frozen_coords = True
gcorr = gcorrvib(temp)
break
numatoms += 1
except:
logger.exception("Error reading the coord file!")
raise
if not frozen_coords:
if numatoms == 1:
logger.info(
"Single atom detected; calculating translational free energy correction"
)
gcorr = gcorrtrans()
else:
with open("vibspectrum") as vibfile:
for line in vibfile:
if not line.startswith(
('$', '#')) and float(line[20:].split()[0]) < 0:
logger.warning(
f"Imaginary frequency detected: {line[20:].split()[0]}"
)
logger.debug("Setting up environment to call freeh")
config = utilities.load_phd_config()
utilities.setup_turbomole_env(config["PATHS"]["TURBODIR"])
try:
logger.debug("Calling freeh")
freehrun = subprocess.run(['freeh'],
input=f'\n{vibscale}\n\nq\n',
stdout=subprocess.PIPE,
universal_newlines=True)
except:
logger.exception("Error in calling 'freeh' program!")
raise ValueError
chempotline = False
linenum = 0
for line in freehrun.stdout.split('\n'):
if chempotline:
linenum += 1
elif 'chem.pot.' in line.split():
chempotline = True
if linenum == 3:
try:
logger.debug(
"Trying to grab the gcorr value from freeh output")
gcorr = float(line.split()[5]) / 2625.50
except ValueError:
logger.exception(
"Error in converting str to float in freeh output")
raise
break
if not gcorr:
raise ValueError("Could not get a free energy correction")
return gcorr
def __init__(self, cores: int = 1, run_dir: str='./', time=-1, pro: protein.Protein=None, parameters: dict=None):
logger.debug("Initializing variables")
self._submit_directory = os.path.abspath(os.getcwd())
self._scratch_directory = os.path.abspath(run_dir)
self._config = utilities.load_phd_config()
self._cores = cores
self._time_to_run = time
self._timer_went_off = False
self._start_time = 0
self._resub = False
utilities.setup_dmd_environ()
if parameters is None:
logger.debug("Checking for a dmdinput.json file")
if os.path.isfile("dmdinput.json"):
self._parameter_file = os.path.join(self._submit_directory, "dmdinput.json")
else:
logger.error("No parameters specified for the job!")
raise FileNotFoundError("dmdinput.json")
#Now we read in the parameters here to a dictionary
try:
logger.debug("Loading in parameters")
with open(self._parameter_file, 'r') as inputfile:
self._raw_parameters = json.load(inputfile)
except IOError:
logger.exception("Could not open the parameter file correctly!")
raise
else:
logger.debug("Using parameters passed")
self._raw_parameters = parameters
# Now we check to see if the parameters are indeed valid
try:
utilities.valid_dmd_parameters(self._raw_parameters)
except ValueError:
logger.exception("Missing a parameter definition!")
raise
except ParameterError:
logger.exception("Invalid parameter specification")
raise
if self._raw_parameters["titr"]["titr on"]:
self._titration = titrate_protein(self._raw_parameters["titr"], self._raw_parameters["Custom protonation states"])
self._raw_parameters = self._titration.expand_commands(self._raw_parameters)
else:
self._titration = None
if pro is None:
if not os.path.isfile("initial.pdb"):
logger.debug("initial.pdb not found, will try setting up from scratch")
sj = setupDMDjob(parameters=self._raw_parameters)
sj.full_setup()
else:
logger.debug("Will setup the protein for DMD")
sj = setupDMDjob(parameters=self._raw_parameters, pro=pro)
sj.full_setup()
self.update_start_time()
self.update_commands()
if self._scratch_directory != self._submit_directory:
self._scratch_directory = os.path.join(self._scratch_directory, os.path.basename(self._submit_directory))
utilities.copy_directories(self._submit_directory, self._scratch_directory)
os.chdir(self._scratch_directory)
# We can arm the timer
if self._time_to_run != -1:
logger.info("Starting the timer")
signal.signal(signal.SIGALRM, self.calculation_alarm_handler)
signal.alarm((self._time_to_run * 60 - 55) * 60)
# We loop over the steps here and will pop elements off the beginning of the dictionary
while len(self._commands.values()) != 0:
logger.info("")
repeat = False
if self._timer_went_off:
logger.info("Timer went off, not continuing onto next command")
break
# Grab the next step dictionary to do
steps = self._commands[list(self._commands.keys())[0]]
logger.debug(f"On step: {steps}")
updated_parameters = self._raw_parameters.copy()
for changes in steps:
logger.debug(f"Updating {changes}: changing {updated_parameters[changes]} to {steps[changes]}")
updated_parameters[changes] = steps[changes]
start = timer()
if updated_parameters["titr"]["titr on"]:
if self._titration is None:
logger.warning("Titration feature cannot be turned on in the middle of a run")
raise ValueError("Titration turned on")
if os.path.isfile(updated_parameters["Movie File"]):
utilities.make_movie("initial.pdb", updated_parameters["Movie File"], "_tmpMovie.pdb")
#Append to movie
with open("_tmpMovie.pdb", 'r') as tmpMovie, open("movie.pdb", 'a') as movie:
for line in tmpMovie:
movie.write(line)
try:
last_frame = utilities.last_frame("_tmpMovie.pdb")
except IndexError:
#grab last initial.pdb, echo and movie.pdb and place over current initial, echo, and movie.pdb and
#add updated_parameters to list
logger.warning("Going back one iteration")
if not os.path.isfile("_older_echo") or not os.path.isfile("_older_movie.pdb"):
logger.error("Cannot go back a step!")
raise
shutil.move("_older_echo", updated_parameters["Echo File"])
last_frame = utilities.last_frame("_older_movie.pdb")
shutil.move("_older_movie.pdb", "movie.pdb")
if os.path.isfile("_last_movie.pdb"):
os.remove("_last_movie.pdb")
if os.path.isfile("_last_echo"):
os.remove("_last_echo")
self._titration._step -=2
repeat = True
self._start_time -= (2*updated_parameters["Time"])
else:
#Clean up our mess
logger.debug("Removing _tmpMovie.pdb file")
os.remove("_tmpMovie.pdb")
logger.debug(f"Removing {updated_parameters['Movie File']} file")
os.remove(updated_parameters["Movie File"])
#TODO check to see if any of the protonation states are invalids (ie, they affect statically held protonation
#states defined by the user)
#TODO, check if this raises any exceptions, and if so, bo back a step
try:
updated_parameters["Custom protonation states"] = self._titration.evaluate_pkas(last_frame)
except Propka_Error:
#grab last initial.pdb, echo and movie.pdb and place over current initial, echo, and movie.pdb and
#add updated_parameters to list
logger.warning("Going back one iteration")
if not os.path.isfile("_last_echo") or not os.path.isfile("_last_movie.pdb"):
logger.error("Cannot go back a step!")
raise
shutil.move("_last_echo", updated_parameters["Echo File"])
last_frame = utilities.last_frame("_last_movie.pdb")
shutil.move("_last_movie.pdb", "movie.pdb")
self._titration._step -=1
self._start_time -= updated_parameters["Time"]
repeat = True
updated_parameters["Custom protonation states"] = self._titration.evaluate_pkas(last_frame)
else:
if not repeat:
if os.path.isfile("_older_movie.pdb"):
os.remove("_older_movie.pdb")
if os.path.isfile("_last_movie.pdb"):
shutil.copy("_last_movie.pdb", "_older_movie.pdb")
if os.path.isfile("movie.pdb"):
shutil.copy("movie.pdb", "_last_movie.pdb")
if os.path.isfile("_older_echo"):
os.remove("_older_echo")
if os.path.isfile("_last_echo"):
shutil.copy("_last_echo", "_older_echo")
if os.path.isfile(updated_parameters["Echo File"]):
shutil.copy(updated_parameters["Echo File"], "_last_echo")
else:
last_frame = utilities.load_pdb("initial.pdb")
try:
updated_parameters["Custom protonation states"] = self._titration.evaluate_pkas(last_frame)
except Propka_Error:
logger.warn("Propka run weirdly, though hopefully it doesn't matter since we skip!")
if os.path.isfile(updated_parameters['Restart File']):
logger.debug(f"Removing {updated_parameters['Restart File']} file")
os.remove(updated_parameters['Restart File'])
sj = setupDMDjob(parameters=updated_parameters, pro=last_frame)
#This will not do a quick dmd setup, so we should be able to expedite that slightly. Also no topparam file either
#creates state, start and outConstr, inConstr
sj.titrate_setup()
#We don't want to use the restart file, so set last var to False
self.run_dmd(updated_parameters, self._start_time, False)
elif "Custom protonation states" in steps.keys():
logger.warning("Why are you trying to change the protonation state in the middle of DMD?")
elif "Frozen atoms" in steps.keys() or "Restrict Displacement" in steps.keys():
logger.warning("Cannot freeze atoms or change displacement between atoms in the middle of a run.")
logger.warning("This does not make any...ignoring these")
else:
# We can just run the job with no issues other than those raised from the above
self.run_dmd(updated_parameters, self._start_time, True)
end = timer()
self.print_summary(updated_parameters['Time'], end-start)
if not repeat:
self._commands.pop(list(self._commands.keys())[0])
# Update the new start time!
self._start_time += updated_parameters["Time"]
if self._commands:
logger.info("Did not finish all of the commands, will save the remaining commands")
if "Resubmit" in self._raw_parameters.keys():
if self._raw_parameters["Resubmit"]:
self._resub = True
else:
logger.debug("Finished all commands...writing final dmdinput.json")
logger.debug("Setting remaining commands to the rest of the commands")
self._raw_parameters["Remaining Commands"] = self._commands
if self._titration is not None and self._commands:
logger.debug("Condensing any commands remaining from the titratable feature")
self._raw_parameters = self._titration.condense_commands(self._raw_parameters)
elif self._titration is not None:
self._raw_parameters["Commands"].clear()
with open("dmdinput.json", 'w') as dmdinput:
logger.debug("Dumping to json")
json.dump(self._raw_parameters, dmdinput, indent=4)
if self._scratch_directory != self._submit_directory:
utilities.copy_directories(self._scratch_directory, self._submit_directory)
os.chdir(self._submit_directory)
shutil.rmtree(self._scratch_directory)
if self._resub:
logger.info("Resubmitting the job!")
submitdmd.main(_cores=self._cores, _time=self._time_to_run)
if os.path.isdir("dmd_backup"):
shutil.rmtree("dmd_backup")
if os.path.isfile("remaining_commands.json"):
os.remove("remaining_commands.json")