def submit_to_queue(self, sub_job: Submission_job) -> int:
"""
submitt a local job
"""
orig_dir = os.getcwd()
if isinstance(sub_job.submit_from_dir, str) and os.path.isdir(
sub_job.submit_from_dir):
os.chdir(sub_job.submit_from_dir)
command_file_path = sub_job.submit_from_dir + "/job_" + str(
sub_job.jobName) + ".sh"
else:
command_file_path = "./job_" + str(sub_job.jobName) + ".sh"
sub_job.command = sub_job.command.strip() # remove trailing linebreaks
if self._nomp >= 1:
command = "export OMP_NUM_THREADS=" + str(
self._nomp) + ";\n " + sub_job.command + ""
else:
command = sub_job.command
if sub_job.sumbit_from_file:
command_file = open(command_file_path, "w")
command_file.write("#!/bin/bash\n")
command_file.write(command.replace("&& ", ";\n") + ";\n")
command_file.close()
command = command_file_path
bash.execute("chmod +x " + command_file_path, env=self.environment)
# finalize string
if self.verbose:
print("Submission Command: \t", " ".join(command))
if self.submission:
try:
process = bash.execute(command=command,
catch_STD=True,
env=self.environment)
std_out_buff = map(str, process.stdout.readlines())
std_out = "\t" + "\n\t".join(std_out_buff)
# next sopt_job is queued with id:
if self.verbose:
print("STDOUT: \n\t" + std_out + "\nEND\n")
if os.path.exists(orig_dir):
os.chdir(orig_dir)
return 0
except ChildProcessError:
try:
print(process)
except ChildProcessError:
pass
raise ChildProcessError("command failed: \n" + str(command))
else:
print("Did not submit: ", command)
return -1
def make_topolog(input_arg, build, param, seq, solve="H2O"):
# define python command
command = "make_top " + input_arg + " " + param + " " + seq + " " + solve + " \n"
# execute command
try:
bash.execute(command=command)
except Exception as err:
bash.increment_error_level(err_prefix="Could not make_topology due to: ", old_err=err)
return command
def _make_clean(build_dir: str,
nCore: int = 1,
verbose: bool = True,
_timing_dict: dict = {},
_timing_prefix: str = ""):
"""
This function triggers make clean and removes the build_dir.
Parameters
----------
build_dir : str
directory prepared for make.
nCore : int, optional
how many cores for each make command?, by default 1
verbose: bool, optional
cleaning, cleaning, every thing gets so shiny and I can sing!, by default True
_timing_dict : dict, optional
structure for storing timings of process, by default {}
_timing_prefix : str, optional
prefix for timing keys, by default ""
"""
os.chdir(build_dir)
cmd = "make -j" + str(nCore) + " clean"
_timing_dict[_timing_prefix + "make_clean_start"] = datetime.now()
if verbose:
print("command: ", cmd)
print("start time: ",
_timing_dict[_timing_prefix + "make_clean_start"])
try:
bash.execute(cmd)
except Exception:
pass
if verbose:
print("remove dir: ", build_dir)
bash.remove_file(build_dir, recursive=True)
_timing_dict[_timing_prefix + "make_clean_end"] = datetime.now()
if verbose:
print(
"duration: ",
str(_timing_dict[_timing_prefix + "make_clean_end"] -
_timing_dict[_timing_prefix + "make_clean_start"]),
"\n",
)
def kill_jobs(self,
job_name: str = None,
regex: bool = False,
job_ids: Union[List[int], int] = None):
"""
this function can be used to terminate or remove pending jobs from the queue.
Parameters
----------
job_name : str
name of the job to be killed
regex : bool
if true, all jobs matching job_name get killed!
job_ids : Union[List[int], int]
job Ids to be killed
"""
if job_name is not None:
job_ids = list(
self.search_queue_for_jobname(job_name, regex=regex).index)
elif job_ids is not None:
if isinstance(job_ids, int):
job_ids = [job_ids]
else:
raise ValueError("Please provide either job_name or job_ids!")
if self.verbose:
print("Stopping: " + ", ".join(map(str, job_ids)))
try:
bash.execute("bkill " + " ".join(map(str, job_ids)))
except Exception as err:
if any(["Job has already finished" in x for x in err.args]):
print("Job has already finished")
else:
raise ChildProcessError("could not execute this command: \n" +
str(err.args))
def submit_jobAarray_to_queue(self, sub_job: Submission_job) -> int:
"""
submitt a local job array
"""
# generate submission_string:
submission_string = ""
if isinstance(sub_job.submit_from_dir, str) and os.path.isdir(
sub_job.submit_from_dir):
submission_string += "cd " + sub_job.submit_from_dir + " && "
if self._nomp > 1:
command = submission_string + " export OMP_NUM_THREADS=" + str(
self._nomp) + " && " + sub_job.command
else:
command = submission_string + sub_job.command
if self.verbose:
print("Submission Command: \t", " ".join(submission_string))
if self.submission:
try:
for jobID in range(sub_job.start_job, sub_job.end_job + 1):
std_out_buff = bash.execute(command="export JOBID=" +
str(jobID) + " && " + command,
env=self.environment)
std_out = "\n".join(std_out_buff.readlines())
if self.verbose:
print("sdtout : " + str(std_out))
return 0
except ChildProcessError:
raise ChildProcessError("could not submit this command: \n" +
submission_string)
else:
print("Did note submit: ", command)
return -1
def md_run(
self,
in_topo_path: str,
in_coord_path: str,
in_imd_path: str,
out_prefix: str,
in_pert_topo_path: str = None,
in_disres_path: str = None,
in_posresspec_path: str = None,
in_refpos_path: str = None,
in_qmmm_path: str = None,
nomp: int = 1,
nmpi: int = 1,
out_trc: bool = False,
out_tre: bool = False,
out_trv: bool = False,
out_trf: bool = False,
out_trs: bool = False,
out_trg: bool = False,
verbose: bool = False,
_binary_name: str = "md",
) -> str:
"""
This function is a wrapper for gromosXX md_mpi. You can directly execute the gromosXX md_mpi in a bash enviroment here.
Warnings
--------
Hybrid jobs are possible, but very difficult to implement correctly to Euler and performance gain is questionable.
If OMP should be used, I suggest the md_run - function.
Parameters
----------
in_topo_path : str
This is the path to the input topology file (x.top)
in_coord_path : str
This is the path to the input coordinate file (x.cnf)
in_imd_path : str
This is the path to the input simulation parameter file (x.imd)
out_prefix : str
This prefix, define the output name.
in_pert_topo_path : str, optional
This is the path to the pertubation file (x.ptp)
in_disres_path : str, optional
This is the path to the distance restraint file (x.dat)
in_posresspec_path : str, optional
This is the path to the position restraint file (x.pos)
in_refpos_path : str, optional
This is the path to the reference position file (x.rpf)
nomp : int, optional
How many omp cores shall be used? Prerequesite, gromos was compiled with -enableOMP
nmpi : int, optional
How many mpi cores shall be used? Prerequesite, gromos was compiled with -enableMPI (and suggested to have -disableOMP)
out_trc : bool, optional
do you want to output a coordinate trajectory (x.trc) file? (needs also an output number in write block of imd!)
out_tre : bool, optional
do you want to output a coordinate trajectory (x.trc) file? (needs also an output number in write block of imd!)
out_trs : bool, optional
do you want to output a coordinate trajectory (x.trc) file? (needs also an output number in write block of imd!)
out_trg : bool, optional
do you want to output the free energy trajectory (x.trg) file? (needs also an output number in write block of imd!)
Returns
-------
str
returns the log_file_path of the simulation.
Raises
------
ChildProcessError
If the execution of the simulation fails, this error is raised.
See Also
--------
md_run, repex_mpi
"""
# BUILD UP str. Command
command = []
if nmpi > 1 and nomp > 1:
raise ValueError(
"There are no Hybrid NMPI and NOMP jobs possible with gromos!")
elif nmpi > 1:
command += ["mpirun -n " + str(nmpi * nomp) + " "
] # --loadbalance " --cpus-per-proc " + + " "
command += [self._bin + _binary_name + "_mpi"]
elif nomp >= 1:
command += ["export OMP_NUM_THREADS=" + str(nomp) + " && "]
command += [self._bin + _binary_name]
else:
command += [self._bin + _binary_name]
command += ["@topo", str(in_topo_path)]
command += ["@conf", str(in_coord_path)]
command += ["@input", str(in_imd_path)]
if in_pert_topo_path is not None:
command += ["@pttopo", str(in_pert_topo_path)]
if in_disres_path is not None:
command += ["@distrest", str(in_disres_path)]
if in_posresspec_path is not None:
print("POSRES", in_posresspec_path)
command += ["@posresspec", str(in_posresspec_path)]
if in_refpos_path is not None:
command += ["@refpos", str(in_refpos_path)]
if in_qmmm_path is not None:
command += ["@qmmm", str(in_qmmm_path)]
if isinstance(out_prefix, str):
command += ["@fin", str(out_prefix) + ".cnf"]
log_file_path = out_prefix + ".omd"
if out_trc:
command += ["@trc", str(out_prefix) + ".trc"]
if out_trv:
command += ["@trv", str(out_prefix) + ".trv"]
if out_trf:
command += ["@trf", str(out_prefix) + ".trf"]
if out_tre:
command += ["@tre", str(out_prefix) + ".tre"]
if out_trs:
command += ["@trs", str(out_prefix) + ".trs"]
if out_trg:
command += ["@trg", str(out_prefix) + ".trg"]
else:
raise ValueError("Outprefix needs to be string got: " +
type(out_prefix) + " - " + str(out_prefix))
command_text = " ".join(command) + " > " + log_file_path + "\n"
if verbose:
print("COMMAND: ", command_text)
start_time = datetime.datetime.now()
process = bash.execute(command_text)
md_run_return = process.poll()
# bash.wait_for_fileSystem(out_prefix+".cnf")
end_time = datetime.datetime.now()
duration = end_time - start_time
time.sleep(time_wait_s_for_filesystem)
log_file = open(log_file_path, "a")
log_file.write("\n\nREMARKS\n")
failed = False
if md_run_return == 0:
log_file.write("\tRUN:\tSUCESSFUL\n")
else:
log_file.write("\tRUN:\tFAILED\n")
omd_file_content = open(log_file_path, "r").read_lines()
if len(omd_file_content) > 0:
print("\t" + "\n\t".join(omd_file_content))
else:
print("\t None")
failed = True
log_file.write("\tTIME:\n\tstart: " + str(start_time) + "\tend: " +
str(end_time) + "\n")
log_file.write("\tDuration:\t " + str(duration) + " d:hh:s.ms\n")
log_file.write("END\n")
log_file.close()
if failed:
raise ChildProcessError(
"GromosXX MD Run Failed!\n\nLOG:" +
"\n".join(open(log_file_path, "r").readlines()))
return log_file_path
def calculate_solvation_free_energy(self, n_points: int = None):
"""
Function to Calculate solvation free energy by integrating over lambda points
Parameters
----------
n_points : int
Number of Lambda points used
Returns
-------
solv_energy : float
Solvation Free Energy
error : float
Error Estimate
"""
if n_points is None:
n_points = self.n_points
# Get dhdl information
Metrics = pd.DataFrame()
# Setup Storage
lambdas = []
averages = []
rmsds = []
ees = []
# Set Lambda Points
lambda_points = n_points
for lambda_point in range(lambda_points):
rlam = lambda_point / (lambda_points - 1)
rlam = np.round(rlam, 3)
# Get system Name
iteration = 2
system_name = self.system_name + "_L" + str(rlam) + "_" + str(iteration)
path = self.groSys_liq.work_folder + "/ti/" + system_name + "/" + system_name + "/"
if os.path.isfile(path + "simulation.tar"):
sim_tar_exists = True
# Extract files
command = "tar" + " "
command += "-xf" + " "
command += path + "simulation.tar "
command += "-C " + path
bash.execute(command, verbose=False)
else:
sim_tar_exists = False
# get trg info
if os.path.isfile(path + "simulation/" + system_name + "_1/" + system_name + "_1.trg.gz"):
trg = Trg(path + "simulation/" + system_name + "_1/" + system_name + "_1.trg.gz")
elif os.path.isfile(path + "simulation/" + system_name + "_1/" + system_name + "_1.trg"):
warnings.warn("Simulation did not finish correctly using limited data!")
trg = Trg(path + "simulation/" + system_name + "_1/" + system_name + "_1.trg")
else:
print(path + "simulation/" + system_name + "_1/" + system_name)
exit("No Files have been found")
dhdls = [trg.database.totals[i][2] for i in range(len(trg.database.totals))]
# Remove simulation directory
if sim_tar_exists:
bash.remove_file(path + "simulation", recursive=True)
# Get values
lambdas.append(rlam)
averages.append(np.mean(dhdls))
rmsds.append(error_estimator(dhdls).calculate_rmsd())
ees.append(error_estimator(dhdls).calculate_error_estimate())
Metrics["Lambda"] = lambdas
Metrics["avg"] = averages
Metrics["rmsd"] = rmsds
Metrics["ee"] = ees
# Integrate over lambda points for value and error
solv_energy = integrate.simpson(Metrics["avg"], Metrics["Lambda"]) * -1
error = integrate.simpson(Metrics["ee"], Metrics["Lambda"]) * -1
if self.verbose:
print(Metrics)
print("Solvation Free Energy:", solv_energy, "pm", error)
return solv_energy, error
def _make_compile(build_dir: str,
nCore: int = 1,
verbose: bool = True,
_timing_dict: dict = {},
_timing_prefix: str = ""):
"""
This function triggers make and make install in the build_dir.
Parameters
----------
build_dir : str
directory prepared for make.
nCore : int, optional
how many cores for each make command?, by default 1
verbose : bool, optional
make, make things and talk loudly about it! , by default True
_timing_dict : dict, optional
structure for storing timings of process, by default {}
_timing_prefix : str, optional
prefix for timing keys, by default ""
"""
os.chdir(build_dir)
_timing_dict[_timing_prefix + "make_start"] = datetime.now()
log_file = build_dir + "/make.log"
cmd = "make -j" + str(nCore)
# Compile
if verbose:
print(spacer2 + "\t\t> MAKE \n" + spacer2)
print("start time: ", _timing_dict[_timing_prefix + "make_start"])
print("log_file: ", log_file)
print("command: ", cmd, "\n")
bash.execute(cmd, catch_STD=log_file)
_timing_dict[_timing_prefix + "make_end"] = datetime.now()
if verbose:
print(
"duration: ",
str(_timing_dict[_timing_prefix + "make_end"] -
_timing_dict[_timing_prefix + "make_start"]),
"\n",
)
# Create Binaries
log_file = build_dir + "/makeInstall.log"
_timing_dict[_timing_prefix + "make_install_start"] = datetime.now()
log_file = build_dir + "/make_install.log"
cmd = "make -j" + str(nCore) + " install"
if verbose:
print(spacer2 + "\t\t> INSTALL \n" + spacer2)
print("start time: ", _timing_dict[_timing_prefix + "make_start"])
print("log_file: ", log_file)
print("command: ", cmd)
bash.execute(cmd, catch_STD=log_file)
_timing_dict[_timing_prefix + "make_install_end"] = datetime.now()
if verbose:
print(
"duration: ",
str(_timing_dict[_timing_prefix + "make_install_end"] -
_timing_dict[_timing_prefix + "make_install_start"]),
"\n",
)
def _configure_gromosXX_autotools(
build_dir: str,
binary_dir: str = None,
with_cuda_dir: str = None,
with_omp: bool = False,
with_mpi: bool = False,
with_debug: bool = False,
verbose: bool = True,
_timing_dict: dict = {},
):
"""
Setting the configurations for the compiling gromosXX process. (uses autotools)
Parameters
----------
build_dir : str
directory, that should be used for building
binary_dir : str, optional
directory in which the binaries should be written to, by default None
with_cuda_dir : str, optional
use the following cuda path and activate cuda support, by default None
with_omp : bool, optional
should gromosXX be compiled with omp - Warning dont combine wiht mpi!, by default False
with_mpi : bool, optional
should gromosXX be compiled wiht mpi - Warning dont combine with omp!, by default False
with_debug : bool, optional
set gromos debug flag, by default False
verbose : bool, optional
compiling is fun, I can tell you more!, by default True
_timing_dict : dict, optional
structure for storing timings of process, by default {}
Raises
------
ValueError
if wrong value was passed
"""
root_dir = os.path.dirname(build_dir)
os.chdir(root_dir)
# Initial config
_timing_dict["gromosXX_init_start"] = datetime.now()
if verbose:
print(spacer2 + "\t\t> INIT \n" + spacer2)
print("start time: ", _timing_dict["gromosXX_init_start"])
print("workdir:", os.getcwd())
bash.execute("./Config.sh")
_timing_dict["gromosXX_init_end"] = datetime.now()
if verbose:
print(
"duration: ",
str(_timing_dict["gromosXX_init_end"] -
_timing_dict["gromosXX_init_start"]), "\n")
# Configure
bash.make_folder(build_dir)
os.chdir(build_dir)
log_file = build_dir + "/configure.log"
_timing_dict["gromosXX_conf_start"] = datetime.now()
if verbose:
print(spacer2 + "\t\t> CONFIGURE \n" + spacer2)
print("start time: ", _timing_dict["gromosXX_conf_start"])
print("workdir:", os.getcwd())
print("log_file: ", log_file)
options = {}
if binary_dir is not None:
options.update({"--bindir": binary_dir})
if with_cuda_dir is not None:
options.update({"--with-cuda": with_cuda_dir})
flags = []
if with_omp and with_mpi:
raise ValueError("Can not use with_omp and with_mpi at the same time!")
if with_omp:
flags.append("--enable-openmp")
if with_mpi:
flags.append("--enable-mpi")
if with_debug:
flags.append("--enable-debug")
cmd = ("../configure " +
" ".join([key + "=" + val for key, val in options.items()]) + " " +
" ".join([flag for flag in flags]))
if verbose:
print("command: ", cmd)
bash.execute(cmd, catch_STD=log_file)
_timing_dict["gromosXX_conf_end"] = datetime.now()
if verbose:
print(
"duration: ",
str(_timing_dict["gromosXX_conf_end"] -
_timing_dict["gromosXX_conf_start"]), "\n")
def _configure_gromosPP_autotools(
build_dir: str,
binary_dir: str = None,
with_omp: bool = False,
with_debug: bool = False,
verbose: bool = True,
_timing_dict: dict = {},
):
"""
Setting the configurations for the compiling gromosPP process. (uses autotools)
Parameters
----------
build_dir : str
directory, that should be used for building
binary_dir : str, optional
directory in which the binaries should be written to, by default None
with_omp : bool, optional
should gromosPP be compiled with omp, by default False
with_debug : bool, optional
set gromos debug flag, by default False
verbose : bool, optional
compiling is fun, I can tell you more!, by default True
_timing_dict : dict, optional
structure for storing timings of process, by default {}
"""
root_dir = os.path.dirname(build_dir)
os.chdir(root_dir)
# Initial config
_timing_dict["gromosPP_init_start"] = datetime.now()
if verbose:
print(spacer2 + "\t\t> INIT \n" + spacer2)
print("start time: ", _timing_dict["gromosPP_init_start"])
print("workdir:", os.getcwd())
bash.execute("./Config.sh")
_timing_dict["gromosPP_init_end"] = datetime.now()
if verbose:
print(
"duration: ",
str(_timing_dict["gromosPP_init_end"] -
_timing_dict["gromosPP_init_start"]), "\n")
# Configure
bash.make_folder(build_dir)
os.chdir(build_dir)
_timing_dict["gromosPP_conf_start"] = datetime.now()
log_file = build_dir + "/configure.log"
if verbose:
print(spacer2 + "\t\t> CONFIGURE \n" + spacer2)
print("start time: ", _timing_dict["gromosPP_conf_start"])
print("log_file: ", log_file)
options = {}
if binary_dir is not None:
options.update({"--bindir": binary_dir})
flags = []
if with_omp:
flags.append("--enable-openmp")
if with_debug:
flags.append("--enable-debug")
flags.append(
" --with-gsl=$(gsl-config --prefix) ") # this is required for gromosPP
cmd = ("../configure " +
" ".join([key + "=" + val for key, val in options.items()]) + " " +
" ".join([flag for flag in flags]))
if verbose:
print("command: ", cmd)
bash.execute(cmd, catch_STD=log_file)
_timing_dict["gromosPP_conf_end"] = datetime.now()
if verbose:
print(
"duration: ",
str(_timing_dict["gromosPP_conf_end"] -
_timing_dict["gromosPP_conf_start"]), "\n")
def submit_to_queue(self, sub_job: Submission_job) -> int:
"""
This function submits the given command to the LSF QUEUE
Parameters
----------
submission_job : Submission_job
the job to be submitted
-------
"""
# job_properties:Job_properties=None, <- currently not usd
orig_dir = os.getcwd()
# generate submission_string:
submission_string = ""
# QUEUE checking to not double submit
if self._block_double_submission and self._submission:
if self.verbose:
print("check queue")
ids = list(self.search_queue_for_jobname(sub_job.jobName).index)
if len(ids) > 0:
if self.verbose:
print("\tSKIP - FOUND JOB: \t\t" +
"\n\t\t".join(map(str, ids)) +
"\n\t\t with jobname: " + sub_job.jobName)
return ids[0]
if isinstance(sub_job.submit_from_dir, str) and os.path.isdir(
sub_job.submit_from_dir):
os.chdir(sub_job.submit_from_dir)
command_file_path = sub_job.submit_from_dir + "/job_" + str(
sub_job.jobName) + ".sh"
else:
command_file_path = "./job_" + str(sub_job.jobName) + ".sh"
submission_string += "bsub "
submission_string += " -J" + sub_job.jobName + " "
submission_string += " -W " + str(self._job_duration) + " "
if not isinstance(sub_job.post_execution_command, type(None)):
submission_string += '-Ep "' + sub_job.post_execution_command + '" '
if not isinstance(sub_job.outLog, str) and not isinstance(
sub_job.errLog, str):
outLog = sub_job.jobName + ".out"
submission_string += " -o " + outLog
elif isinstance(sub_job.outLog, str):
submission_string += " -o " + sub_job.outLog
if isinstance(sub_job.errLog, str):
submission_string += " -e " + sub_job.errLog
nCPU = self._nmpi * self._nomp
submission_string += " -n " + str(nCPU) + " "
# TODO: add GPU support
# add_string = ""
# add_string= "-R \"select[model==XeonGold_5118 || model==XeonGold_6150 || model==XeonE3_1585Lv5 || model==XeonE3_1284Lv4 || model==XeonE7_8867v3 || model == XeonGold_6140 || model==XeonGold_6150 ]\""
if isinstance(self._max_storage, int):
submission_string += " -R rusage[mem=" + str(
self._max_storage) + "] "
if isinstance(sub_job.queue_after_jobID,
(int, str)) and (sub_job.queue_after_jobID != 0
or sub_job.queue_after_jobID != "0"):
submission_string += ' -w "' + self._chain_prefix + "(" + str(
sub_job.queue_after_jobID) + ')" '
if self._begin_mail:
submission_string += " -B "
if self._end_mail:
submission_string += " -N "
sub_job.command = sub_job.command.strip(
) # remove trailing line breaks
if self._nomp >= 1:
command = "export OMP_NUM_THREADS=" + str(
self._nomp) + ";\n " + sub_job.command + " "
else:
command = "\n " + sub_job.command + ""
if sub_job.sumbit_from_file:
if self.verbose:
print("writing tmp-submission-file to: ", command_file_path)
command_file = open(command_file_path, "w")
command_file.write("#!/bin/bash\n")
command_file.write(command + ";\n")
command_file.close()
command = command_file_path
bash.execute("chmod +x " + command_file_path,
env=self._environment)
# finalize string
submission_string = list(
map(lambda x: x.strip(), submission_string.split())) + [command]
if self.verbose:
print("Submission Command: \t", " ".join(submission_string))
if self._submission and not self._dummy:
try:
out_process = bash.execute(command=submission_string,
catch_STD=True,
env=self._environment)
std_out = "\n".join(map(str, out_process.stdout.readlines()))
# next sopt_job is queued with id:
id_start = std_out.find("<")
id_end = std_out.find(">")
job_id = int(str(std_out[id_start + 1:id_end]).strip())
if self.verbose:
print("process returned id: " + str(job_id))
if str(job_id) == "" and job_id.isalnum():
raise ValueError("Did not get at job ID!")
except Exception as e:
raise ChildProcessError("could not submit this command: \n" +
str(submission_string) + "\n\n" +
str(e))
else:
job_id = -1
os.chdir(orig_dir)
sub_job.jobID = job_id
return job_id
def get_queued_jobs(self) -> pd.DataFrame:
"""
This function updates the job-list of the queueing system in the class.
Returns
-------
pd.DataFrame
returns the job_queue as pandas dataFrame.
"""
# Do we need an update of the job list?
check_job_list = True
if hasattr(self, "_job_queue_time_stamp"):
last_update = datetime.now() - self._job_queue_time_stamp
check_job_list = last_update.seconds > self._refresh_job_queue_list_all_s
if not self._submission: # shortcut to reduce queue calls!
self._job_queue_list = pd.DataFrame(columns=[
"JOBID USER STAT QUEUE FROM_HOST EXEC_HOST JOB_NAME SUBMIT_TIME"
.split()
])
return self._job_queue_list
if check_job_list:
# try getting the lsf queue
if not self._dummy:
try:
# get all running and pending jobs
if self.bjobs_only_same_host:
out_process = bash.execute(
"bjobs -w | grep '$HOSTNAME|JOBID'",
catch_STD=True)
else:
out_process = bash.execute("bjobs -w", catch_STD=True)
job_list_str = list(
map(lambda x: x.decode("utf-8"),
out_process.stdout.readlines()))
# get all finished jobs
if self.bjobs_only_same_host:
out_process = bash.execute(
"bjobs -wd | grep '$HOSTNAME|JOBID'",
catch_STD=True)
else:
out_process = bash.execute("bjobs -wd", catch_STD=True)
job_list_finished_str = list(
map(lambda x: x.decode("utf-8"),
out_process.stdout.readlines()))
self._job_queue_time_stamp = datetime.now()
except Exception as err:
raise Exception("Could not get job_list!\nerr:\n" +
"\n".join(err.args))
else:
job_list_str = []
job_list_finished_str = []
# format information:
jlist = list(map(lambda x: x.strip().split(), job_list_str))
jlist_fin = list(
map(lambda x: x.strip().split(), job_list_finished_str))
if len(jlist) > 1:
header = jlist[0]
jobs = jlist[1:] + jlist_fin[1:]
jobs_dict = {}
for job in jobs:
jobID = int(job[0].split("[")[0])
user = job[1]
status = job[2]
queue = job[3]
from_host = job[4]
exec_host = job[5]
job_name = " ".join(job[6:-3])
submit_time = datetime.strptime(
str(datetime.now().year) + " " + " ".join(job[-3:]),
"%Y %b %d %H:%M")
values = [
jobID, user, status, queue, from_host, exec_host,
job_name, submit_time
]
jobs_dict.update({
jobID:
{key: value
for key, value in zip(header, values)}
})
self._job_queue_list = pd.DataFrame(jobs_dict, index=None).T
else:
self._job_queue_list = pd.DataFrame(columns=[
"JOBID USER STAT QUEUE FROM_HOST EXEC_HOST JOB_NAME SUBMIT_TIME"
.split()
])
else:
if self.verbose:
print("Skipping refresh of job list, as the last update is " +
str(last_update) + "s ago")
return self._job_queue_list
def submit_jobAarray_to_queue(self, sub_job: Submission_job) -> int:
"""
This functioncan be used for submission of a job array. The ammount of jobs is determined by the difference:
end_job-start_job
An array index variable is defined called ${JOBID} inside the command representing job x in the array.
Parameters
----------
sub_job: Submission_job
the job to be submitted
Returns
-------
int
return job ID
"""
# QUEUE checking to not double submit
if self._submission and self._block_double_submission:
if self.verbose:
print("check queue")
ids = self.search_queue_for_jobname(sub_job.jobName)
if len(ids) > 0:
if self.verbose:
print("\tSKIP - FOUND JOB: \t\t" +
"\n\t\t".join(map(str, ids)) +
"\n\t\t with jobname: " + sub_job.jobName)
return ids[0]
# generate submission_string:
submission_string = ""
if isinstance(sub_job.submit_from_dir, str) and os.path.isdir(
sub_job.submit_from_dir):
submission_string += "cd " + sub_job.submit_from_dir + " && "
if sub_job.jobLim is None:
jobLim = sub_job.end_job - sub_job.start_job
jobName = str(sub_job.jobName) + "[" + str(
sub_job.start_job) + "-" + str(
sub_job.end_job) + "]%" + str(jobLim)
submission_string += 'bsub -J " ' + jobName + ' " -W "' + str(
self._job_duration) + '" '
if isinstance(sub_job.jobGroup, str):
submission_string += " -g " + sub_job.jobGroup + " "
if not isinstance(sub_job.outLog, str) and not isinstance(
sub_job.errLog, str):
outLog = jobName + ".out"
submission_string += " -oo " + outLog
elif isinstance(sub_job.outLog, str):
submission_string += " -oo " + sub_job.outLog
if isinstance(sub_job.errLog, str):
submission_string += " -eo " + sub_job.errLog
nCPU = self._nmpi * self._nomp
submission_string += " -n " + str(nCPU) + " "
if isinstance(self.max_storage, int):
submission_string += ' -R "rusage[mem=' + str(
self._max_storage) + ']" '
if isinstance(sub_job.queue_after_jobID, (int, str)):
submission_string += " -w " + self._chain_prefix + "(" + str(
sub_job.queue_after_jobID) + ')" '
if self._begin_mail:
submission_string += " -B "
if self._end_mail:
submission_string += " -N "
if self._nomp > 1:
command = " export OMP_NUM_THREADS=" + str(
self._nomp) + " && " + sub_job.command + " "
else:
command = " " + sub_job.command + " "
# finalize string
submission_string = list(
map(lambda x: x.strip(), submission_string.split())) + [command]
if self.verbose:
print("Submission Command: \t", " ".join(submission_string))
if self._submission and not self._dummy:
try:
std_out_buff = bash.execute(command=submission_string,
env=self._environment)
std_out = "\n".join(std_out_buff.readlines())
# next sopt_job is queued with id:
id_start = std_out.find("<")
id_end = std_out.find(">")
job_id = str(std_out[id_start + 1:id_end]).strip()
if self.verbose:
print("process returned id: " + str(job_id))
if job_id == "" and job_id.isalnum():
raise ValueError("Did not get at job ID!")
except Exception as e:
raise ChildProcessError("could not submit this command: \n" +
" ".join(submission_string) + "\n\n" +
str(e))
else:
job_id = -1
sub_job.jobID = job_id
return int(job_id)