def main():
parser = argparse.ArgumentParser(
description='Automatically resubmit PBS jobs')
parser.add_argument('-d','--delay', type=str, default="15:00", \
help='How long to delay ("[[[DD:]HH:]MM:]SS") between executions. Default is "15:00".')
group = parser.add_mutually_exclusive_group()
group.add_argument('--hold',
action='store_true',
help='Place a hold on the currently running taskmaster')
group.add_argument('--release',
action='store_true',
help='Release the currently running taskmaster')
group.add_argument('--kill',
action='store_true',
help='Kill the currently running taskmaster')
args = parser.parse_args()
if args.hold:
jobid = pbs.job_id(name="taskmaster")
if len(jobid) != 0:
pbs.hold(jobid[-1])
elif args.release:
jobid = pbs.job_id(name="taskmaster")
if len(jobid) != 0:
pbs.release(jobid[-1])
elif args.kill:
jobid = pbs.job_id(name="taskmaster")
if len(jobid) != 0:
pbs.alter(jobid[-1], "-a " + pbs.exetime("10:00:00:00"))
pbs.delete(jobid[-1])
else:
# check if taskmaster already running (besides this one)
jobid = pbs.job_id(name="taskmaster")
tmaster_status = pbs.job_status(jobid)
for j in jobid:
if j != pbs.job_id():
if tmaster_status[j]["jobstatus"] != "C":
print "A taskmaster is already running. JobID:", j, " Status:", tmaster_status[
j]["jobstatus"]
sys.exit()
# continue jobs
db = pbs.JobDB()
db.update()
db.continue_all()
db.close()
# submit taskmaster
print "submit taskmaster"
j = pbs.PrismsDebugJob(nodes="1", ppn="1", name="taskmaster", \
exetime=pbs.exetime(args.delay), auto=False, message=None, \
command="taskmaster " + ' '.join(sys.argv[1:]))
j.submit(add=False)
def submit(self):
"""Submit a PBS job for this VASP relaxation"""
# first, check if the job has already been submitted and is not completed
db = pbs.JobDB()
print "rundir", self.calcdir
id = db.select_regex_id("rundir", self.calcdir)
print "id:", id
sys.stdout.flush()
if id != []:
for j in id:
job = db.select_job(j)
# taskstatus = ["Incomplete","Complete","Continued","Check","Error:.*","Aborted"]
# jobstatus = ["C","Q","R","E","W","H","M"]
if job["jobstatus"] != "C":
print "JobID:", job["jobid"], " Jobstatus:", job["jobstatus"], " Not submitting."
sys.stdout.flush()
return
#elif job["taskstatus"] in ["Complete", "Check"] or re.match( "Error:.*", job["taskstatus"]):
# print "JobID:", job["jobid"], " Taskstatus:", job["taskstatus"], " Not submitting."
# sys.stdout.flush()
# return
# second, only submit a job if relaxation status is "incomplete"
# construct the Relax object
relaxation = vasp.Relax(self.calcdir, self.run_settings())
# check the current status
(status, task) = relaxation.status()
if status == "complete":
print "Status:", status, " Not submitting."
sys.stdout.flush()
# ensure job marked as complete in db
if self.auto:
for j in id:
job = db.select_job(j)
if job["taskstatus"] == "Incomplete":
try:
pbs.complete_job(jobid=j)
except (pbs.PBSError, pbs.JobDBError, pbs.EligibilityError) as e:
print str(e)
sys.stdout.flush()
# ensure results report written
if not os.path.isfile(os.path.join(self.calcdir, "properties.calc.json")):
self.report()
return
elif status == "not_converging":
print "Status:", status, " Not submitting."
sys.stdout.flush()
return
elif status != "incomplete":
raise vaspwrapper.VaspWrapperError("unexpected relaxation status: '" + status + "' and task: '" + task + "'")
sys.stdout.flush()
return
print "Preparing to submit a VASP relaxation PBS job"
sys.stdout.flush()
# cd to configdir, submit jobs from configdir, then cd back to currdir
currdir = os.getcwd()
os.chdir(self.calcdir)
# determine the number of atoms in the configuration
print " Counting atoms in the POSCAR"
sys.stdout.flush()
pos = vasp.io.Poscar(os.path.join(self.configdir,"POS"))
N = len(pos.basis)
print " Constructing a PBS job"
sys.stdout.flush()
# construct a pbs.Job
job = pbs.Job(name=casm.jobname(self.configdir),\
account=self.settings["account"],\
nodes=int(math.ceil(float(N)/float(self.settings["atom_per_proc"])/float(self.settings["ppn"]))),\
ppn=int(self.settings["ppn"]),\
walltime=self.settings["walltime"],\
pmem=self.settings["pmem"],\
qos=self.settings["qos"],\
queue=self.settings["queue"],\
message=self.settings["message"],\
email=self.settings["email"],\
priority=self.settings["priority"],\
command="python -c \"import casm.vaspwrapper; casm.vaspwrapper.Relax('" + self.configdir + "').run()\"",\
auto=self.auto)
print " Submitting"
sys.stdout.flush()
# submit the job
job.submit()
# return to current directory
os.chdir(currdir)
print "CASM VASPWrapper relaxation PBS job submission complete\n"
sys.stdout.flush()
##DYNAMICALLY LOCATE PSEUDO_DIR##
if "QE_POTENTIAL_DIR" not in os.environ.keys():
print "NO SPECIFIED ENVIRONMENT VARIABLE QE_POTENTIAL_DIR\n"
print "USING DEFAULT OF ~/quantum_espresso/pseudo\n"
qe_pot = "~/quantum_espresso/pseudo"
else:
qe_pot = os.environ["QE_POTENTIAL_DIR"].encode(
'ascii', 'ignore')
for line in fileinput.input(os.path.join(relaxdir,
case["infilename"]),
inplace=1):
if "$!%&" in line:
line = line.replace("$!%&", qe_pot)
sys.stdout.write(line)
db = pbs.JobDB()
#initialize Relax object and begin relaxation
cmd = "python -c \"import quantumespresso; relaxation=quantumespresso.Relax('" + relaxdir + "');relaxation.settings['" + "infilename" + "']='" + case[
"infilename"] + "';relaxation.settings['" + "outfilename" + "']='" + case[
"outfilename"] + "';relaxation.run();\"\n"
job = pbs.Job(name="casm_unit_test",\
account=None,\
nodes=1,\
ppn=2,\
walltime="1:00:00",\
pmem=None,\
qos=None,\
queue="batch",\
message=None,\
def submit(self): #pylint: disable=too-many-statements
"""Submit a PBS job for this SeqQuest relaxation"""
print "Submitting..."
print "Configuration:", self.configname
# first, check if the job has already been submitted and is not completed
db = pbs.JobDB()
print "Calculation directory:", self.calcdir
id = db.select_regex_id("rundir", self.calcdir)
print "JobID:", id
sys.stdout.flush()
if id != []:
for j in id:
job = db.select_job(j)
if job["jobstatus"] != "C":
print "JobID:", job["jobid"], " Jobstatus:", job[
"jobstatus"], " Not submitting."
sys.stdout.flush()
return
# second, only submit a job if relaxation status is "incomplete"
# construct the Relax object
relaxation = seqquest.Relax(self.calcdir, self.run_settings())
# check the current status
(status, task) = relaxation.status()
if status == "complete":
print "Status:", status, " Not submitting."
sys.stdout.flush()
# ensure job marked as complete in db
if self.auto:
for j in id:
job = db.select_job(j)
if job["taskstatus"] == "Incomplete":
try:
pbs.complete_job(jobid=j)
except (pbs.PBSError, pbs.JobDBError,
pbs.EligibilityError) as e:
print str(e)
sys.stdout.flush()
# ensure results report written
if not os.path.isfile(
os.path.join(self.calcdir, "properties.calc.json")):
self.finalize()
return
elif status == "not_converging":
print "Status:", status, " Not submitting."
sys.stdout.flush()
return
elif status != "incomplete":
raise questwrapper.QuestWrapperError(
"unexpected relaxation status: '" + status + "' and task: '" +
task + "'")
# This code can never be reached...
# sys.stdout.flush()
# return
print "Preparing to submit a SeqQuest relaxation PBS job"
sys.stdout.flush()
# cd to configdir, submit jobs from configdir, then cd back to currdir
currdir = os.getcwd()
os.chdir(self.calcdir)
# determine the number of atoms in the configuration
print "Counting atoms in the POSCAR"
sys.stdout.flush()
geom = seqquest.seqquest_io.Geom.POS(
os.path.join(self.configdir, "POS"))
N = len(geom.basis)
# construct command to be run
cmd = ""
if self.settings["preamble"] is not None:
# Append any instructions given in the 'preamble' file, if given
preamble = self.casm_directories.settings_path_crawl(
self.settings["preamble"], self.configname, self.clex)
with open(preamble) as my_preamble:
cmd += "".join(my_preamble) + "\n"
if self.settings["prerun"] is not None:
cmd += self.settings["prerun"] + "\n"
cmd += "python -c \"import casm.vaspwrapper; casm.vaspwrapper.Relax('" + self.configdir + "').run()\"\n"
if self.settings["postrun"] is not None:
cmd += self.settings["postrun"] + "\n"
print "Constructing a PBS job"
sys.stdout.flush()
# construct a pbs.Job
job = pbs.Job(name=casm.jobname(self.configdir),\
account=self.settings["account"],\
nodes=int(math.ceil(float(N)/float(self.settings["atom_per_proc"])/float(self.settings["ppn"]))),\
ppn=int(self.settings["ppn"]),\
walltime=self.settings["walltime"],\
pmem=self.settings["pmem"],\
qos=self.settings["qos"],\
queue=self.settings["queue"],\
message=self.settings["message"],\
email=self.settings["email"],\
priority=self.settings["priority"],\
command=cmd,\
auto=self.auto)
print "Submitting"
sys.stdout.flush()
# submit the job
job.submit()
self.report_status("submitted")
# return to current directory
os.chdir(currdir)
print "CASM QuestWrapper relaxation PBS job submission complete\n"
sys.stdout.flush()
def main():
# input parser
desc = \
"""
Print or modify PBS job and task status.
By default, 'pstat' prints status for select jobs. Jobs are
selected by listing jobids or using --all, --range, or
--recent, optionally combined with --active. Running 'pstat'
with no selection is equivalent to selecting '--all --active'.
The default display style is a summary list. Other options are
--full or --series.
Using one of --complete, --continue, --error, --abort, or
--delete modifies status instead of printing. User
confirmation is required before a modification is applied,
unless the --force option is given.
Job status is as given by PBS for a single PBS job ('C', 'R',
'Q', etc.).
Task status is user-defined and defines the status of a single
PBS job within a possible series of jobs comprising some task.
'Auto' jobs may be re-submitted with the --continue option.
Please see:
https://github.com/prisms-center/pbs
for more information about 'auto' jobs.
Possible values for task status are:
"Complete": Job and task are complete.
"Incomplete": Job or task are incomplete.
"Continued": Job is complete, but task was not complete. In
this case, 'continuation_jobid' is set with
the jobid for the next job in the series of
jobs comprising some task.
"Check": Non-auto job is complete and requires user
input for status.
"Error:.*": Some kind of error was noted.
"Aborted": The job and task have been aborted.
"""
parser = argparse.ArgumentParser(
description=desc, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('job',
metavar='JOBID',
type=str,
nargs='*',
help='Job IDs to query or operate on')
style = parser.add_mutually_exclusive_group()
style.add_argument('-f',
'--full',
default=False,
action='store_true',
help='List all fields instead of summary')
style.add_argument('-s',
'--series',
default=False,
action='store_true',
help='List all fields grouped by continuation jobs')
group = parser.add_mutually_exclusive_group()
select = parser.add_mutually_exclusive_group()
select.add_argument('-a',
'--all',
default=False,
action='store_true',
help='Select all jobs in database')
select.add_argument(
'--range',
metavar=('MINID', 'MAXID'),
type=str,
nargs=2,
help='A range of Job IDs (inclusive) to query or operate on')
select.add_argument(
'--recent',
metavar='DD:HH:MM:SS',
type=str,
nargs=1,
help='Select jobs created or modified within given amout of time')
select.add_argument(
'--regex',
metavar=('KEY', 'REGEX'),
type=str,
nargs=2,
help=
'Select jobs where the value of column \'KEY\' matches the regular expression \'REGEX\'.'
)
parser.add_argument(
'--active',
default=False,
action='store_true',
help=
'Select active jobs only. May be combined with --range and --recent')
group.add_argument('--complete',
default=False,
action='store_true',
help='Mark jobs as \'Complete\'')
group.add_argument('--continue',
dest="cont",
default=False,
action='store_true',
help='Re-submit auto jobs')
group.add_argument('--reset',
default=False,
action='store_true',
help='Mark job as \'Incomplete\'')
group.add_argument('--abort',
default=False,
action='store_true',
help='Call qdel on job and mark as \'Aborted\'')
group.add_argument('--error',
metavar='ERRMSG',
type=str,
help='Add error message.')
group.add_argument(
'--delete',
default=False,
action='store_true',
help='Delete jobs from database. Aborts jobs that are still running.')
group.add_argument(
'--key',
type=str,
nargs=1,
help='Output data corresponding to \'key\' for selected jobs.')
parser.add_argument('--force',
default=False,
action='store_true',
help='Modify jobs without user confirmation')
args = parser.parse_args()
# open the Job database
db = pbs.JobDB()
db.update()
# perform an operation, or print jobs
if args.complete:
operate(args, \
db.eligible_to_complete, \
db.complete_job, \
"Jobs to be mark completed:", \
"Are you sure you want to mark the above jobs completed? (yes/no): ", \
"Marking job complete:",db)
elif args.cont:
operate(args, \
db.eligible_to_continue, \
db.continue_job, \
"Jobs to be continued:", \
"Are you sure you want to continue the above jobs? (yes/no): ", \
"Continuing job:",db)
elif args.reset:
operate(args, \
db.eligible_to_reset, \
db.reset_job, \
"Jobs to be reset:", \
"Are you sure you want to reset the above jobs? (yes/no): ", \
"Resetting job:",db)
elif args.abort:
operate(args, \
db.eligible_to_abort, \
db.abort_job, \
"Jobs to be aborted:", \
"Are you sure you want to abort the above jobs? (yes/no): ", \
"Aborting job:",db)
elif args.delete:
operate(args, \
db.eligible_to_delete, \
db.delete_job, \
"Jobs to be deleted:", \
"Are you sure you want to delete the above jobs? (yes/no): ", \
"Deleting job:",db)
elif args.error:
operate(args, \
db.eligible_to_error, \
db.error_job, \
"Jobs to be marked with an error:", \
"Are you sure you want to mark the above jobs with an error? (yes/no): ", \
"Marking job with an error:",db)
elif args.key:
print_data(args, db)
else:
print_jobs(args, db)
# close the database
db.close()
def test_run_single(self):
"""
Test vasp.Relax.run()
"""
curr = os.getcwd()
cases = self.cases["run_single"]
for case in cases:
# do tests
# load Relax directory
filesdir = join(fixtures.projects_dir, case["proj"])
if os.path.isdir(join(filesdir, "relaxdir")):
shutil.rmtree(join(filesdir, "relaxdir"))
shutil.copytree(filesdir, join(filesdir, "relaxdir"))
relaxdir = join(filesdir, "relaxdir")
##CREATE POTCAR##
if "VASP_POTENTIAL_DIR" not in os.environ.keys():
print "NO SPECIFIED ENVIRONMENT VARIABLE VASP_POTENTIAL_DIR\n"
print "USING DEFAULT OF $HOME/vasp_potentials\n"
vasp_pot = "$HOME/vasp_potentials"
else:
vasp_pot = os.environ["VASP_POTENTIAL_DIR"].encode(
'ascii', 'ignore')
with open(join(relaxdir, "POTCAR"), 'wb') as wfd:
for f in case["potcars"]:
with open(join(join(vasp_pot, f), "POTCAR"), 'rb') as fd:
shutil.copyfileobj(fd, wfd)
fd.close()
wfd.close()
print "Relax dir is", relaxdir
db = pbs.JobDB()
#initialize Relax object and begin relaxation
cmd = "python -c \"import vasp; relaxation=vasp.Relax('" + relaxdir + "');relaxation.run();\"\n"
job = pbs.Job(name="casm_unit_test",\
account=None,\
nodes=1,\
ppn=2,\
walltime="1:00:00",\
pmem=None,\
qos=None,\
queue="batch",\
message=None,\
email=None,\
priority=0,\
command=cmd,\
auto=True)
os.chdir(relaxdir)
job.submit()
os.chdir(curr)
db.update()
id = db.select_regex_id("rundir", relaxdir)
j = db.select_job(id[-1])
while j["jobstatus"] == "Q":
time.sleep(10)
db.update()
j = db.select_job(id[-1])
while j["jobstatus"] == "R":
time.sleep(10)
db.update()
j = db.select_job(id[-1])
self.assertTrue(exists(join(relaxdir, "run.final/OUTCAR")))
if relaxdir == join(filesdir, "relaxdir"):
shutil.rmtree(relaxdir)
print "done!"
def test_run_many(self):
"""
Test casm.vaspwrapper.run() using casm-calc --submit
"""
cases = self.cases["run_many"]
from casm.project import Project, Selection
import subprocess, shlex, sys, fileinput
curr_dir = os.getcwd()
for case in cases:
# do tests
# create casm project
filesdir = join(fixtures.projects_dir, case["proj"])
if os.path.isdir(join(filesdir, "casmproj_vasp")):
shutil.rmtree(join(filesdir, "casmproj_vasp"))
os.mkdir(join(filesdir, "casmproj_vasp"))
for file in os.listdir(filesdir):
if os.path.isfile(join(filesdir, file)):
shutil.copy(join(filesdir, file),
join(filesdir, "casmproj_vasp"))
casmproj = join(filesdir, "casmproj_vasp")
if "VASP_POTENTIAL_DIR" not in os.environ.keys():
print "NO SPECIFIED ENVIRONMENT VARIABLE VASP_POTENTIAL_DIR\n"
print "USING DEFAULT OF $HOME/vasp_potentials\n"
vasp_pot = "$HOME/vasp_potentials"
else:
vasp_pot = os.environ["VASP_POTENTIAL_DIR"].encode(
'ascii', 'ignore')
#proj = casm.project.Project(path=join(filesdir, "casmproj"))
print "casm project dir is", casmproj
os.chdir(casmproj)
pinit = subprocess.Popen(shlex.split("casm init"))
pinit.wait()
pcomp = subprocess.Popen(shlex.split("casm composition -c"))
pcomp.wait()
pcompsel = subprocess.Popen(shlex.split("casm composition -s 0"))
pcompsel.wait()
penums = subprocess.Popen(
shlex.split("casm enum -s --max " + str(case["max_vol"])))
penums.wait()
penumc = subprocess.Popen(shlex.split("casm enum -c -a"))
penumc.wait()
psel = subprocess.Popen(shlex.split("casm select --set-on"))
psel.wait()
settings_dir = join(casmproj,
"training_data/settings/calctype.default")
#Move files to setting directory
shutil.copyfile("INCAR", join(settings_dir, "INCAR"))
shutil.copyfile("KPOINTS", join(settings_dir, "KPOINTS"))
shutil.copyfile("POSCAR", join(settings_dir, "POSCAR"))
for line in fileinput.input("VSPECIES", inplace=1):
if "$!%&" in line:
line = line.replace("$!%&", vasp_pot)
sys.stdout.write(line)
shutil.copyfile("VSPECIES", join(settings_dir, "SPECIES"))
shutil.copyfile("vrelax.json", join(settings_dir, "relax.json"))
psubmit = subprocess.Popen(shlex.split("casm-calc --submit"))
psubmit.wait()
db = pbs.JobDB()
db.update()
train_data = join(casmproj, "training_data")
for dirName in os.listdir(train_data):
if "SCEL" in dirName and os.path.isdir(
join(train_data, dirName)):
print "Found:", dirName
for config in os.listdir(join(train_data, dirName)):
if os.path.isdir(
join(join(train_data, dirName), config)):
print "Checking", config
self.assertTrue(
exists(
join(
join(
join(join(train_data, dirName),
config)), "POS")))
id = db.select_regex_id(
"rundir",
join(
join(
join(
join(join(train_data, dirName),
config)),
"calctype.default")))
j = db.select_job(id[-1])
while (j["jobstatus"] == "Q"
or j["jobstatus"] == "R"):
time.sleep(10)
db.update()
j = db.select_job(id[-1])
filename = join(
join(
join(
join(
join(join(train_data, dirName),
config)),
"calctype.default")), "status.json")
file = open(filename)
status = json.load(file)
file.close()
if status["status"] == "complete":
self.assertTrue(
os.path.isdir(
join(
join(
join(
join(
join(
train_data,
dirName), config)),
"calctype.default"),
"run.final")))
self.assertTrue(
exists(
join(
join(
join(
join(
join(
train_data,
dirName), config)),
"calctype.default"),
"properties.calc.json")))
print "verified complete calculation!"
else:
self.assertFalse(
exists(
join(
join(
join(
join(
join(
train_data,
dirName), config)),
"calctype.default"),
"properties.calc.json")))
print dirName + config + " failed and did not produce properties.calc.json as expected"
os.chdir(curr_dir)
if os.path.isdir(join(filesdir, "casmproj_vasp")):
shutil.rmtree(join(filesdir, "casmproj_vasp"))
print "done!"