def main():
db = JobDB()
banner("info")
db.info()
db.start()
# db.ps()
db.connect()
db.clear()
# Insert two jobs - one with only a name and one with input and output files
job0_id = db.insert("job0")
job1_id = db.insert("job1", "input1", "output1")
# Print job IDs of both added jobs
print job0_id
print job1_id
# Get all jobs with given parameters
# Available key names at this time are:
# _id
# job_name
# input_filename
# output_filename
jobs = db.find_jobs("job_name", "job1")
# Print out all returned jobs
for job in jobs:
print job
# Query by job ID to return a single job
singleJob = list(db.find_jobs("_id", job0_id))
# Print out the first job in the list
print singleJob[0]
# Print out count of all jobs
print db.count()
# Print out count of jobs given query parameters
print db.count("job_name", "job1")
# Show updating a job attribute
print "\nORIGINAL JOB:"
singleJob = list(db.find_jobs("_id", job1_id))
print singleJob[0]
# Update the input filename
db.update_job_attribute(job1_id, "input_filename", "new_input_file")
# Print out the updated job
print "\nUPDATED JOB:"
singleJob = list(db.find_jobs("_id", job1_id))
print singleJob[0]
# Add a job using add()
job = {"job_name": "job25"}
db.add(job)
# Show modify() functionality
job = {"job_name": "job30", "input_filename": "file1"}
db.modify(job)
print "\nORIGINAL JOB"
for job in db.find_jobs():
print job
job = {"job_name": "job30", "input_filename": "file2"}
db.modify(job)
print "MODIFIED JOB"
for job in db.find_jobs():
print job
# Show job statuses functionality
print "\nJOB STATUSES:"
db.job_status_stats()
print "\nJOB STATUSES WITH JOBS PRINTED:"
db.job_status_stats(True)
# SHOW FIND_JOBS_WITH_FILE
job100_id = db.insert("job100")
job101_id = db.insert("job101", "file100", "file200")
job102_id = db.insert("job102", "file200", "file300")
inputs, outputs = db.find_jobs_with_file("file200")
print("\nJobs with matching file in input:")
for job in inputs:
print job
print("\nJobs with matching file in output:")
for job in outputs:
print job
# Delete all jobs
db.clear()
# SHOW ADD_FROM_YAML
db.add_from_yaml("job_example.yaml")
for job in db.find_jobs():
print job
# Delete all jobs
db.clear()
print "Database cleared."
print "Job count: " + str(db.count())
db.stop()
def main():
db = JobDB()
banner("info")
db.info()
db.start()
# db.ps()
db.connect()
db.clear()
# Insert two jobs - one with only a name and one with input and output files
job0_id = db.insert("job0")
job1_id = db.insert("job1", "input1", "output1")
# Print job IDs of both added jobs
print job0_id
print job1_id
# Get all jobs with given parameters
# Available key names at this time are:
# _id
# job_name
# input_filename
# output_filename
jobs = db.find_jobs("job_name", "job1")
# Print out all returned jobs
for job in jobs:
print job
# Query by job ID to return a single job
singleJob = list(db.find_jobs("_id", job0_id))
# Print out the first job in the list
print singleJob[0]
# Print out count of all jobs
print db.count()
# Print out count of jobs given query parameters
print db.count("job_name", "job1")
# Show updating a job attribute
print "\nORIGINAL JOB:"
singleJob = list(db.find_jobs("_id", job1_id))
print singleJob[0]
# Update the input filename
db.update_job_attribute(job1_id, "input_filename", "new_input_file")
# Print out the updated job
print "\nUPDATED JOB:"
singleJob = list(db.find_jobs("_id", job1_id))
print singleJob[0]
# Add a job using add()
job = {"job_name": "job25"}
db.add(job)
# Show modify() functionality
job = {"job_name": "job30", "input_filename": "file1"}
db.modify(job)
print "\nORIGINAL JOB"
for job in db.find_jobs():
print job
job = {"job_name": "job30", "input_filename": "file2"}
db.modify(job)
print "MODIFIED JOB"
for job in db.find_jobs():
print job
# Show job statuses functionality
print "\nJOB STATUSES:"
db.job_status_stats()
print "\nJOB STATUSES WITH JOBS PRINTED:"
db.job_status_stats(True)
# SHOW FIND_JOBS_WITH_FILE
job100_id = db.insert("job100")
job101_id = db.insert("job101", "file100", "file200")
job102_id = db.insert("job102", "file200", "file300")
inputs, outputs = db.find_jobs_with_file("file200")
print("\nJobs with matching file in input:")
for job in inputs:
print job
print("\nJobs with matching file in output:")
for job in outputs:
print job
# Delete all jobs
db.clear()
# SHOW ADD_FROM_YAML
db.add_from_yaml("job_example.yaml")
for job in db.find_jobs():
print job
# Delete all jobs
db.clear()
print "Database cleared."
print "Job count: " + str(db.count())
db.stop()
class TestJobDB:
def setup(self):
# HEADING()
self.db = JobDB()
def teardown(self):
# HEADING()
pass
def test_001_start(self):
"""
tests if the mongo db can be started
:return:
"""
HEADING()
self.db.start()
up = self.db.isup()
result = up
assert result
def test_002_up(self):
"""
tests if the mongo db can be started
:return:
"""
HEADING()
up = self.db.isup()
print(up)
self.db.stop()
down = not self.db.isup()
self.db.start()
assert up and down
def test_003_pid(self):
"""
tests if the mongo db can be started
:return:
"""
HEADING()
pid = self.db.pid()
print(pid)
assert True
def test_004_connect(self):
"""
tests if a mongo db can be connected to
:return:
"""
HEADING()
self.db.connect()
result = True
assert result
def test_005_clear(self):
"""
tests clearing all jobs from the db
:return:
"""
HEADING()
db = self.db
db.connect()
db.clear()
# assert not os.path.isfile(path_expand("~/.cloudmesh/pbs/pbs.db"))
assert (len(db) == 0)
def test_006_add(self):
"""
tests adding jobs to the db
:return:
"""
HEADING()
db = self.db
count = 5
db.connect()
db.delete_jobs()
for id in range(0, count):
job = db.insert("job" + str(id))
assert len(db) == count
def test_007_delete(self):
"""
tests deleting a single job from the db
:return:
"""
HEADING()
db = self.db
db.connect()
print("AAA")
before_count = len(db)
print("CCC", len(db))
job = db.insert("deleteme")
print("DDD", len(db))
job = db.delete_jobs("job_name", "deleteme")
print("EEE")
after_count = len(db)
print("FFF", len(db))
assert (before_count - after_count == 0)
def test_008_modify(self):
"""
tests modifying a single job in the db
:return:
"""
HEADING()
db = self.db
db.connect()
job = {"job_name": "modifyme", "input_filename": "file1"}
db.add(job)
originalFilename = self.db.find_jobs("job_name",
"modifyme")[0]["input_filename"]
job = {"job_name": "modifyme", "input_filename": "file2"}
db.modify(job)
newFilename = self.db.find_jobs("job_name",
"modifyme")[0]["input_filename"]
assert (originalFilename != newFilename)
def test_010_info(self):
"""
prints the info about the db
:return:
"""
HEADING()
db = self.db
db.connect()
db.info()
pass
def test_011_len(self):
"""
tests modifying a single job in the db
:return:
"""
HEADING()
db = self.db
db.connect()
count = len(db)
print(count)
assert count == 6
def test_012_yaml_load(self):
"""
tests adding jobs from a YAML file
:return:
"""
HEADING()
db = self.db
db.connect()
# Clear all jobs currently in the database to ensure a correct final assertion
db.clear()
# Add the jobs outlined in the YAML file
db.add_from_yaml("etc/jobs.yaml")
count_fgrep = len(Shell.fgrep("input:", "etc/jobs.yaml").split("\n"))
# Assert that the correct number jobs have been added
assert (db.count() == count_fgrep)
def test_013_find_files(self):
"""
tests searching for a file
the file being searched for exists in 3 files: twice as input and twice as output
:return:
"""
HEADING()
db = self.db
db.connect()
# Clear all jobs currently in the database to ensure a correct final assertion
db.clear()
# Add the jobs outlined in the YAML file
db.add_from_yaml("etc/jobs.yaml")
inputs, outputs = db.find_jobs_with_file("in1.txt")
# Assert that the lengths of the inputs and outputs arrays are correct
count_fgrep = len(
Shell.fgrep("in1.txt", "etc/jobs.yaml").strip().split("\n"))
assert (len(inputs) == count_fgrep)
def test_14_jobid(self):
HEADING()
db = self.db
db.connect()
db.set_jobid(11)
a = db.get_jobid()
db.incr_jobid()
b = db.get_jobid()
print(a, b)
assert b == a + 1
def test_15_jobscript(self):
HEADING()
db = self.db
db.connect()
name = "test1"
contents = "hallo"
db.add_script(name, contents)
db.add_script("test2", "other")
script = db.get_script(name)
print("Script:", script)
print("Content:", contents)
db.write_script(name, "/tmp/script.txt")
what = Shell.cat("/tmp/script.txt")
assert contents == what
assert contents == script
def test_999_stop(self):
"""
tests if the mongo db can be shutdown
:return:
"""
HEADING()
self.db.stop()
result = True
assert result
class TestJobDB:
def setup(self):
# HEADING()
self.db = JobDB()
def teardown(self):
# HEADING()
pass
def test_001_start(self):
"""
tests if the mongo db can be started
:return:
"""
HEADING()
self.db.start()
up = self.db.isup()
result = up
assert result
def test_002_up(self):
"""
tests if the mongo db can be started
:return:
"""
HEADING()
up = self.db.isup()
print (up)
self.db.stop()
down = not self.db.isup()
self.db.start()
assert up and down
def test_003_pid(self):
"""
tests if the mongo db can be started
:return:
"""
HEADING()
pid = self.db.pid()
print (pid)
assert True
def test_004_connect(self):
"""
tests if a mongo db can be connected to
:return:
"""
HEADING()
self.db.connect()
result = True
assert result
def test_005_clear(self):
"""
tests clearing all jobs from the db
:return:
"""
HEADING()
db = self.db
db.connect()
db.clear()
# assert not os.path.isfile(path_expand("~/.cloudmesh/pbs/pbs.db"))
assert(len(db) == 0)
def test_006_add(self):
"""
tests adding jobs to the db
:return:
"""
HEADING()
db = self.db
count = 5
db.connect()
db.delete_jobs()
for id in range(0,count):
job = db.insert("job" + str(id))
assert len(db) == count
def test_007_delete(self):
"""
tests deleting a single job from the db
:return:
"""
HEADING()
db = self.db
db.connect()
print ("AAA")
before_count = len(db)
print ("CCC", len(db))
job = db.insert("deleteme")
print ("DDD", len(db))
job = db.delete_jobs("job_name", "deleteme")
print ("EEE")
after_count = len(db)
print ("FFF", len(db))
assert(before_count - after_count == 0)
def test_008_modify(self):
"""
tests modifying a single job in the db
:return:
"""
HEADING()
db = self.db
db.connect()
job = {"job_name": "modifyme", "input_filename":"file1"}
db.add(job)
originalFilename = self.db.find_jobs("job_name", "modifyme")[0]["input_filename"]
job = {"job_name": "modifyme", "input_filename":"file2"}
db.modify(job)
newFilename = self.db.find_jobs("job_name", "modifyme")[0]["input_filename"]
assert(originalFilename != newFilename)
def test_010_info(self):
"""
prints the info about the db
:return:
"""
HEADING()
db = self.db
db.connect()
db.info()
pass
def test_011_len(self):
"""
tests modifying a single job in the db
:return:
"""
HEADING()
db = self.db
db.connect()
count = len(db)
print (count)
assert count == 6
def test_012_yaml_load(self):
"""
tests adding jobs from a YAML file
:return:
"""
HEADING()
db = self.db
db.connect()
# Clear all jobs currently in the database to ensure a correct final assertion
db.clear()
# Add the jobs outlined in the YAML file
db.add_from_yaml("etc/jobs.yaml")
count_fgrep = len(Shell.fgrep("input:", "etc/jobs.yaml").split("\n"))
# Assert that the correct number jobs have been added
assert(db.count() == count_fgrep)
def test_013_find_files(self):
"""
tests searching for a file
the file being searched for exists in 3 files: twice as input and twice as output
:return:
"""
HEADING()
db = self.db
db.connect()
# Clear all jobs currently in the database to ensure a correct final assertion
db.clear()
# Add the jobs outlined in the YAML file
db.add_from_yaml("etc/jobs.yaml")
inputs, outputs = db.find_jobs_with_file("in1.txt")
# Assert that the lengths of the inputs and outputs arrays are correct
count_fgrep = len(Shell.fgrep("in1.txt", "etc/jobs.yaml").strip().split("\n"))
assert(len(inputs) == count_fgrep)
def test_14_jobid(self):
HEADING()
db = self.db
db.connect()
db.set_jobid(11)
a = db.get_jobid()
db.incr_jobid()
b = db.get_jobid()
print (a, b)
assert b == a + 1
def test_15_jobscript(self):
HEADING()
db = self.db
db.connect()
name = "test1"
contents = "hallo"
db.add_script(name, contents)
db.add_script("test2", "other")
script = db.get_script(name)
print ("Script:", script)
print ("Content:", contents)
db.write_script(name, "/tmp/script.txt")
what = Shell.cat("/tmp/script.txt")
assert contents == what
assert contents == script
def test_999_stop(self):
"""
tests if the mongo db can be shutdown
:return:
"""
HEADING()
self.db.stop()
result = True
assert result
