def connect():
try:
db = JobDB()
db.connect()
except Exception, e:
print(e)
raise Exception("connection error")
def connect():
try:
db = JobDB()
db.connect()
except Exception, e:
print(e)
raise Exception("connection error")
def do_job(self, args, arguments):
"""
::
Usage:
job server start
job server stop
job server clean
job server kill
job server deploy
job server ps
job stat
job list
job add JOBLIST [--host=HOST] [--options=OPTIONS] [--inputs=INPUTS] [--outputs=OUTPUTS]
job add --file=filename
job write --file=filename
job find --name=NAME
job find --attribute=ATTRIBUTE --value=VALUE
job delete JOBLIST
Arguments:
NAME the name of the job
HOST the host on which the job should run
OPTIONS options passed to the command
INPUTS input files
OUTPUTS output files
ATTRIBUTE an attribute
VALUE a value
JOBLIST the job list
Description:
manages a job catalog to submit them to a computation cloud
or Grid.
job server start
starts the job server
job server stop
stops the job server
job server clean
removes all data in the job server and does a graceful clean, e.g deletes all scheduled jobs
job server kill
kills just the job server, but does not delete the jobs from the schedulers.
this command should not be called in normal circumstances.
job set GROUP
sets the default job group
job add GROUP TODO
adds a job to a group
job server start
starts the server
job server stop
stops the server
job stat
prints a simple statistics of the jobs
job add NAMES [--host=HOST] [--option=OPTIONS] [--inputs=INPUTS] [--outputs=OUTPUTS]
adds a number of jobs
job add --file=filename
adds the job from the file. The postfix of the file deterimnes which
format it is. The formats supported are .csv, .yaml, .json
job write --file=filename
writes the jobs to a file. The postfix of the file deterimnes which
format it is. The formats supported are .csv, .yaml, .json
job list [--output=OUTPUT]
lists the jobs in the format specified
job insert NAME [HOST] [OPTIONS] [INPUT_FILES] [OUTPUT_FILES]
inserts the job with the name into the job database. Options,
input and output files could be specified
job find --name=NAME
find the job with the given name
job find --attribute=ATTRIBUTE --value=VALUE
find jobs that match the given attribute.
job delete JOBLIST
delete the job with the specified names in the joblist.
THE FOLLOWING IS NOT YET DEFINED OR MAY CHANGE
job add TODO
... not yet sure what in the rest of the command
adds a job to the job server and returns its id
job last
returns the last job added to the server
job delete ID
deletes the job from the job server and cancels it if it is scheduled for execution.
job info ID
give the info of a job
job submit ID HOST
submits the job with the given ID to the host
job list GROUP
lists the jobs in the group
job status [ID | GROUP]
list the status of a single job or the status of all jobs in the group
job status statistics
list the statistics of the jobs in the job server (e.g. for the states)
"""
# pprint(arguments)
if arguments.get("server"):
if arguments["start"]:
db = JobDB()
db.start()
Console.ok("job server start")
elif arguments["stop"]:
db = JobDB()
db.stop()
Console.ok("job server stop")
elif arguments["ps"]:
db = JobDB()
db.ps()
elif arguments["clean"]:
Console.ok("job server clean")
elif arguments["kill"]:
Console.ok("job server kill")
elif arguments["deploy"]:
Console.ok("job server deploy")
elif arguments["delete"] and arguments["JOBLIST"]:
joblist = hostlist.expand_hostlist(arguments["JOBLIST"])
# debug msg
print(joblist)
db = JobDB()
db.connect()
for job in joblist:
# if job exists:
Console.ok("delete job {:}".format(job))
db.delete_jobs(attribute="name", value=job)
elif arguments["add"]:
joblist = hostlist.expand_hostlist(arguments["JOBLIST"])
host = arguments["--host"]
inputs = [None]
outputs = [None]
options = [None]
if arguments["--inputs"]:
inputs = hostlist.expand_hostlist(arguments["--inputs"])
if arguments["--outputs"]:
outputs = hostlist.expand_hostlist(arguments["--outputs"])
if arguments["--options"]:
options = hostlist.expand_hostlist(arguments["--options"])
# check if inputs are either 0, 1 or the length of joblist
def expand_parameter(parameter, label):
"""
:param parameter:
:param label:
:return: list of strings
"""
_parameter = parameter
if len(_parameter) == 1:
_parameter = _parameter * len(joblist)
elif len(_parameter) == len(joblist):
pass
else:
Console.error("the number of input files do not match the hostlist")
print("joblist count:", len(joblist))
print(label, "count: ", len(_parameter))
return _parameter
options = expand_parameter(options, "options")
inputs = expand_parameter(inputs, "inputs")
outputs = expand_parameter(inputs, "outputs")
pprint(joblist)
pprint(inputs)
pprint(outputs)
# dependent on if 0, 1, or length of joblist handle that
# debug msg
print(joblist)
for i in range(len(joblist)):
banner(i)
Console.ok("add job : {:} ".format(joblist[i]))
Console.ok(" input : {:} ".format(inputs[i]))
Console.ok(" output: {:} ".format(outputs[i]))
elif arguments["stat"]:
Console.ok("job stat")
elif arguments["list"]:
output = arguments["--output"]
Console.ok("lists the jobs in the format specified")
elif arguments["insert"]:
name = arguments["NAME"]
host = arguments["HOST"]
options = arguments["OPTIONS"]
input_files = arguments["INPUT_FILES"]
output_file = arguments["OUTPUT_FILES"]
Console.ok("insert")
elif arguments["find"] and arguments["--name"]:
name = arguments["NAME"]
Console.ok("find the job with the given name")
elif arguments["find"] and arguments["--attribute"] and arguments["--value"]:
name = arguments["NAME"]
attribute = arguments["--attribute"]
value = arguments["--value"]
Console.ok("job find --attribute=ATTRIBUTE --value=VALUE")
pass
def main():
db = JobDB()
banner("info")
db.info()
db.start()
# db.ps()
db.connect()
db.clear()
# Insert two jobs - one with only a name and one with input and output files
job0_id = db.insert("job0")
job1_id = db.insert("job1", "input1", "output1")
# Print job IDs of both added jobs
print job0_id
print job1_id
# Get all jobs with given parameters
# Available key names at this time are:
# _id
# job_name
# input_filename
# output_filename
jobs = db.find_jobs("job_name", "job1")
# Print out all returned jobs
for job in jobs:
print job
# Query by job ID to return a single job
singleJob = list(db.find_jobs("_id", job0_id))
# Print out the first job in the list
print singleJob[0]
# Print out count of all jobs
print db.count()
# Print out count of jobs given query parameters
print db.count("job_name", "job1")
# Show updating a job attribute
print "\nORIGINAL JOB:"
singleJob = list(db.find_jobs("_id", job1_id))
print singleJob[0]
# Update the input filename
db.update_job_attribute(job1_id, "input_filename", "new_input_file")
# Print out the updated job
print "\nUPDATED JOB:"
singleJob = list(db.find_jobs("_id", job1_id))
print singleJob[0]
# Add a job using add()
job = {"job_name": "job25"}
db.add(job)
# Show modify() functionality
job = {"job_name": "job30", "input_filename": "file1"}
db.modify(job)
print "\nORIGINAL JOB"
for job in db.find_jobs():
print job
job = {"job_name": "job30", "input_filename": "file2"}
db.modify(job)
print "MODIFIED JOB"
for job in db.find_jobs():
print job
# Show job statuses functionality
print "\nJOB STATUSES:"
db.job_status_stats()
print "\nJOB STATUSES WITH JOBS PRINTED:"
db.job_status_stats(True)
# SHOW FIND_JOBS_WITH_FILE
job100_id = db.insert("job100")
job101_id = db.insert("job101", "file100", "file200")
job102_id = db.insert("job102", "file200", "file300")
inputs, outputs = db.find_jobs_with_file("file200")
print("\nJobs with matching file in input:")
for job in inputs:
print job
print("\nJobs with matching file in output:")
for job in outputs:
print job
# Delete all jobs
db.clear()
# SHOW ADD_FROM_YAML
db.add_from_yaml("job_example.yaml")
for job in db.find_jobs():
print job
# Delete all jobs
db.clear()
print "Database cleared."
print "Job count: " + str(db.count())
db.stop()
class TestJobDB:
def setup(self):
# HEADING()
self.db = JobDB()
def teardown(self):
# HEADING()
pass
def test_001_start(self):
"""
tests if the mongo db can be started
:return:
"""
HEADING()
self.db.start()
up = self.db.isup()
result = up
assert result
def test_002_up(self):
"""
tests if the mongo db can be started
:return:
"""
HEADING()
up = self.db.isup()
print(up)
self.db.stop()
down = not self.db.isup()
self.db.start()
assert up and down
def test_003_pid(self):
"""
tests if the mongo db can be started
:return:
"""
HEADING()
pid = self.db.pid()
print(pid)
assert True
def test_004_connect(self):
"""
tests if a mongo db can be connected to
:return:
"""
HEADING()
self.db.connect()
result = True
assert result
def test_005_clear(self):
"""
tests clearing all jobs from the db
:return:
"""
HEADING()
db = self.db
db.connect()
db.clear()
# assert not os.path.isfile(path_expand("~/.cloudmesh/pbs/pbs.db"))
assert (len(db) == 0)
def test_006_add(self):
"""
tests adding jobs to the db
:return:
"""
HEADING()
db = self.db
count = 5
db.connect()
db.delete_jobs()
for id in range(0, count):
job = db.insert("job" + str(id))
assert len(db) == count
def test_007_delete(self):
"""
tests deleting a single job from the db
:return:
"""
HEADING()
db = self.db
db.connect()
print("AAA")
before_count = len(db)
print("CCC", len(db))
job = db.insert("deleteme")
print("DDD", len(db))
job = db.delete_jobs("job_name", "deleteme")
print("EEE")
after_count = len(db)
print("FFF", len(db))
assert (before_count - after_count == 0)
def test_008_modify(self):
"""
tests modifying a single job in the db
:return:
"""
HEADING()
db = self.db
db.connect()
job = {"job_name": "modifyme", "input_filename": "file1"}
db.add(job)
originalFilename = self.db.find_jobs("job_name",
"modifyme")[0]["input_filename"]
job = {"job_name": "modifyme", "input_filename": "file2"}
db.modify(job)
newFilename = self.db.find_jobs("job_name",
"modifyme")[0]["input_filename"]
assert (originalFilename != newFilename)
def test_010_info(self):
"""
prints the info about the db
:return:
"""
HEADING()
db = self.db
db.connect()
db.info()
pass
def test_011_len(self):
"""
tests modifying a single job in the db
:return:
"""
HEADING()
db = self.db
db.connect()
count = len(db)
print(count)
assert count == 6
def test_012_yaml_load(self):
"""
tests adding jobs from a YAML file
:return:
"""
HEADING()
db = self.db
db.connect()
# Clear all jobs currently in the database to ensure a correct final assertion
db.clear()
# Add the jobs outlined in the YAML file
db.add_from_yaml("etc/jobs.yaml")
count_fgrep = len(Shell.fgrep("input:", "etc/jobs.yaml").split("\n"))
# Assert that the correct number jobs have been added
assert (db.count() == count_fgrep)
def test_013_find_files(self):
"""
tests searching for a file
the file being searched for exists in 3 files: twice as input and twice as output
:return:
"""
HEADING()
db = self.db
db.connect()
# Clear all jobs currently in the database to ensure a correct final assertion
db.clear()
# Add the jobs outlined in the YAML file
db.add_from_yaml("etc/jobs.yaml")
inputs, outputs = db.find_jobs_with_file("in1.txt")
# Assert that the lengths of the inputs and outputs arrays are correct
count_fgrep = len(
Shell.fgrep("in1.txt", "etc/jobs.yaml").strip().split("\n"))
assert (len(inputs) == count_fgrep)
def test_14_jobid(self):
HEADING()
db = self.db
db.connect()
db.set_jobid(11)
a = db.get_jobid()
db.incr_jobid()
b = db.get_jobid()
print(a, b)
assert b == a + 1
def test_15_jobscript(self):
HEADING()
db = self.db
db.connect()
name = "test1"
contents = "hallo"
db.add_script(name, contents)
db.add_script("test2", "other")
script = db.get_script(name)
print("Script:", script)
print("Content:", contents)
db.write_script(name, "/tmp/script.txt")
what = Shell.cat("/tmp/script.txt")
assert contents == what
assert contents == script
def test_999_stop(self):
"""
tests if the mongo db can be shutdown
:return:
"""
HEADING()
self.db.stop()
result = True
assert result
def main():
db = JobDB()
banner("info")
db.info()
db.start()
# db.ps()
db.connect()
db.clear()
# Insert two jobs - one with only a name and one with input and output files
job0_id = db.insert("job0")
job1_id = db.insert("job1", "input1", "output1")
# Print job IDs of both added jobs
print job0_id
print job1_id
# Get all jobs with given parameters
# Available key names at this time are:
# _id
# job_name
# input_filename
# output_filename
jobs = db.find_jobs("job_name", "job1")
# Print out all returned jobs
for job in jobs:
print job
# Query by job ID to return a single job
singleJob = list(db.find_jobs("_id", job0_id))
# Print out the first job in the list
print singleJob[0]
# Print out count of all jobs
print db.count()
# Print out count of jobs given query parameters
print db.count("job_name", "job1")
# Show updating a job attribute
print "\nORIGINAL JOB:"
singleJob = list(db.find_jobs("_id", job1_id))
print singleJob[0]
# Update the input filename
db.update_job_attribute(job1_id, "input_filename", "new_input_file")
# Print out the updated job
print "\nUPDATED JOB:"
singleJob = list(db.find_jobs("_id", job1_id))
print singleJob[0]
# Add a job using add()
job = {"job_name": "job25"}
db.add(job)
# Show modify() functionality
job = {"job_name": "job30", "input_filename": "file1"}
db.modify(job)
print "\nORIGINAL JOB"
for job in db.find_jobs():
print job
job = {"job_name": "job30", "input_filename": "file2"}
db.modify(job)
print "MODIFIED JOB"
for job in db.find_jobs():
print job
# Show job statuses functionality
print "\nJOB STATUSES:"
db.job_status_stats()
print "\nJOB STATUSES WITH JOBS PRINTED:"
db.job_status_stats(True)
# SHOW FIND_JOBS_WITH_FILE
job100_id = db.insert("job100")
job101_id = db.insert("job101", "file100", "file200")
job102_id = db.insert("job102", "file200", "file300")
inputs, outputs = db.find_jobs_with_file("file200")
print("\nJobs with matching file in input:")
for job in inputs:
print job
print("\nJobs with matching file in output:")
for job in outputs:
print job
# Delete all jobs
db.clear()
# SHOW ADD_FROM_YAML
db.add_from_yaml("job_example.yaml")
for job in db.find_jobs():
print job
# Delete all jobs
db.clear()
print "Database cleared."
print "Job count: " + str(db.count())
db.stop()
class TestJobDB:
def setup(self):
# HEADING()
self.db = JobDB()
def teardown(self):
# HEADING()
pass
def test_001_start(self):
"""
tests if the mongo db can be started
:return:
"""
HEADING()
self.db.start()
up = self.db.isup()
result = up
assert result
def test_002_up(self):
"""
tests if the mongo db can be started
:return:
"""
HEADING()
up = self.db.isup()
print (up)
self.db.stop()
down = not self.db.isup()
self.db.start()
assert up and down
def test_003_pid(self):
"""
tests if the mongo db can be started
:return:
"""
HEADING()
pid = self.db.pid()
print (pid)
assert True
def test_004_connect(self):
"""
tests if a mongo db can be connected to
:return:
"""
HEADING()
self.db.connect()
result = True
assert result
def test_005_clear(self):
"""
tests clearing all jobs from the db
:return:
"""
HEADING()
db = self.db
db.connect()
db.clear()
# assert not os.path.isfile(path_expand("~/.cloudmesh/pbs/pbs.db"))
assert(len(db) == 0)
def test_006_add(self):
"""
tests adding jobs to the db
:return:
"""
HEADING()
db = self.db
count = 5
db.connect()
db.delete_jobs()
for id in range(0,count):
job = db.insert("job" + str(id))
assert len(db) == count
def test_007_delete(self):
"""
tests deleting a single job from the db
:return:
"""
HEADING()
db = self.db
db.connect()
print ("AAA")
before_count = len(db)
print ("CCC", len(db))
job = db.insert("deleteme")
print ("DDD", len(db))
job = db.delete_jobs("job_name", "deleteme")
print ("EEE")
after_count = len(db)
print ("FFF", len(db))
assert(before_count - after_count == 0)
def test_008_modify(self):
"""
tests modifying a single job in the db
:return:
"""
HEADING()
db = self.db
db.connect()
job = {"job_name": "modifyme", "input_filename":"file1"}
db.add(job)
originalFilename = self.db.find_jobs("job_name", "modifyme")[0]["input_filename"]
job = {"job_name": "modifyme", "input_filename":"file2"}
db.modify(job)
newFilename = self.db.find_jobs("job_name", "modifyme")[0]["input_filename"]
assert(originalFilename != newFilename)
def test_010_info(self):
"""
prints the info about the db
:return:
"""
HEADING()
db = self.db
db.connect()
db.info()
pass
def test_011_len(self):
"""
tests modifying a single job in the db
:return:
"""
HEADING()
db = self.db
db.connect()
count = len(db)
print (count)
assert count == 6
def test_012_yaml_load(self):
"""
tests adding jobs from a YAML file
:return:
"""
HEADING()
db = self.db
db.connect()
# Clear all jobs currently in the database to ensure a correct final assertion
db.clear()
# Add the jobs outlined in the YAML file
db.add_from_yaml("etc/jobs.yaml")
count_fgrep = len(Shell.fgrep("input:", "etc/jobs.yaml").split("\n"))
# Assert that the correct number jobs have been added
assert(db.count() == count_fgrep)
def test_013_find_files(self):
"""
tests searching for a file
the file being searched for exists in 3 files: twice as input and twice as output
:return:
"""
HEADING()
db = self.db
db.connect()
# Clear all jobs currently in the database to ensure a correct final assertion
db.clear()
# Add the jobs outlined in the YAML file
db.add_from_yaml("etc/jobs.yaml")
inputs, outputs = db.find_jobs_with_file("in1.txt")
# Assert that the lengths of the inputs and outputs arrays are correct
count_fgrep = len(Shell.fgrep("in1.txt", "etc/jobs.yaml").strip().split("\n"))
assert(len(inputs) == count_fgrep)
def test_14_jobid(self):
HEADING()
db = self.db
db.connect()
db.set_jobid(11)
a = db.get_jobid()
db.incr_jobid()
b = db.get_jobid()
print (a, b)
assert b == a + 1
def test_15_jobscript(self):
HEADING()
db = self.db
db.connect()
name = "test1"
contents = "hallo"
db.add_script(name, contents)
db.add_script("test2", "other")
script = db.get_script(name)
print ("Script:", script)
print ("Content:", contents)
db.write_script(name, "/tmp/script.txt")
what = Shell.cat("/tmp/script.txt")
assert contents == what
assert contents == script
def test_999_stop(self):
"""
tests if the mongo db can be shutdown
:return:
"""
HEADING()
self.db.stop()
result = True
assert result