def connect(): try: db = JobDB() db.connect() except Exception, e: print(e) raise Exception("connection error")
# # how to test # # python eve/run.py in one terminal # # curl -i http://localhost:5000/jobs # # from flask.ext.bootstrap import Bootstrap from eve_docs import eve_docs from cloudmesh_job.cm_jobdb import JobDB db = JobDB() print db.port my_settings = { 'MONGO_HOST': 'localhost', 'MONGO_PORT': db.port, 'MONGO_DBNAME': 'jobsdb', 'DOMAIN': { 'jobs': {} } } from eve import Eve app = Eve(settings=my_settings) Bootstrap(app) app.register_blueprint(eve_docs, url_prefix='/docs')
def setup(self): # HEADING() self.db = JobDB()
class TestJobDB: def setup(self): # HEADING() self.db = JobDB() def teardown(self): # HEADING() pass def test_001_start(self): """ tests if the mongo db can be started :return: """ self.db.start() # identify a test to see if mongod is started result = True assert result def test_100_stop(self): """ tests if the mongo db can be shutdown :return: """ self.db.stop() result = True assert result def test_002_clear(self): HEADING() # self.db.clear() # assert not os.path.isfile(path_expand("~/.cloudmesh/pbs/pbs.db")) assert True def test_003_init(self): HEADING() print (self.db) def test_004_add(self): HEADING() db = self.db count = 5 db.connect() for id in range(0,count-1): job = db.insert("job" + str(id)) print ("len", len(db)) assert len(db) == count def test_004_add(self): HEADING() db = self.db db.connect() job = db.insert("deleteme") before_count = len(db) job = db.delete("deleteme") after_count = len(db) assert(before_count - after_count == 1) def test_005_update(self): HEADING() # self.db.update(host="india") # self.db.list() assert True
def do_job(self, args, arguments): """ :: Usage: job server start job server stop job server clean job server kill job server deploy job server ps job stat job list job add JOBLIST [--host=HOST] [--options=OPTIONS] [--inputs=INPUTS] [--outputs=OUTPUTS] job add --file=filename job write --file=filename job find --name=NAME job find --attribute=ATTRIBUTE --value=VALUE job delete JOBLIST Arguments: NAME the name of the job HOST the host on which the job should run OPTIONS options passed to the command INPUTS input files OUTPUTS output files ATTRIBUTE an attribute VALUE a value JOBLIST the job list Description: manages a job catalog to submit them to a computation cloud or Grid. job server start starts the job server job server stop stops the job server job server clean removes all data in the job server and does a graceful clean, e.g deletes all scheduled jobs job server kill kills just the job server, but does not delete the jobs from the schedulers. this command should not be called in normal circumstances. job set GROUP sets the default job group job add GROUP TODO adds a job to a group job server start starts the server job server stop stops the server job stat prints a simple statistics of the jobs job add NAMES [--host=HOST] [--option=OPTIONS] [--inputs=INPUTS] [--outputs=OUTPUTS] adds a number of jobs job add --file=filename adds the job from the file. The postfix of the file deterimnes which format it is. The formats supported are .csv, .yaml, .json job write --file=filename writes the jobs to a file. The postfix of the file deterimnes which format it is. The formats supported are .csv, .yaml, .json job list [--output=OUTPUT] lists the jobs in the format specified job insert NAME [HOST] [OPTIONS] [INPUT_FILES] [OUTPUT_FILES] inserts the job with the name into the job database. Options, input and output files could be specified job find --name=NAME find the job with the given name job find --attribute=ATTRIBUTE --value=VALUE find jobs that match the given attribute. job delete JOBLIST delete the job with the specified names in the joblist. THE FOLLOWING IS NOT YET DEFINED OR MAY CHANGE job add TODO ... not yet sure what in the rest of the command adds a job to the job server and returns its id job last returns the last job added to the server job delete ID deletes the job from the job server and cancels it if it is scheduled for execution. job info ID give the info of a job job submit ID HOST submits the job with the given ID to the host job list GROUP lists the jobs in the group job status [ID | GROUP] list the status of a single job or the status of all jobs in the group job status statistics list the statistics of the jobs in the job server (e.g. for the states) """ # pprint(arguments) if arguments.get("server"): if arguments["start"]: db = JobDB() db.start() Console.ok("job server start") elif arguments["stop"]: db = JobDB() db.stop() Console.ok("job server stop") elif arguments["ps"]: db = JobDB() db.ps() elif arguments["clean"]: Console.ok("job server clean") elif arguments["kill"]: Console.ok("job server kill") elif arguments["deploy"]: Console.ok("job server deploy") elif arguments["delete"] and arguments["JOBLIST"]: joblist = hostlist.expand_hostlist(arguments["JOBLIST"]) # debug msg print(joblist) db = JobDB() db.connect() for job in joblist: # if job exists: Console.ok("delete job {:}".format(job)) db.delete_jobs(attribute="name", value=job) elif arguments["add"]: joblist = hostlist.expand_hostlist(arguments["JOBLIST"]) host = arguments["--host"] inputs = [None] outputs = [None] options = [None] if arguments["--inputs"]: inputs = hostlist.expand_hostlist(arguments["--inputs"]) if arguments["--outputs"]: outputs = hostlist.expand_hostlist(arguments["--outputs"]) if arguments["--options"]: options = hostlist.expand_hostlist(arguments["--options"]) # check if inputs are either 0, 1 or the length of joblist def expand_parameter(parameter, label): """ :param parameter: :param label: :return: list of strings """ _parameter = parameter if len(_parameter) == 1: _parameter = _parameter * len(joblist) elif len(_parameter) == len(joblist): pass else: Console.error("the number of input files do not match the hostlist") print("joblist count:", len(joblist)) print(label, "count: ", len(_parameter)) return _parameter options = expand_parameter(options, "options") inputs = expand_parameter(inputs, "inputs") outputs = expand_parameter(inputs, "outputs") pprint(joblist) pprint(inputs) pprint(outputs) # dependent on if 0, 1, or length of joblist handle that # debug msg print(joblist) for i in range(len(joblist)): banner(i) Console.ok("add job : {:} ".format(joblist[i])) Console.ok(" input : {:} ".format(inputs[i])) Console.ok(" output: {:} ".format(outputs[i])) elif arguments["stat"]: Console.ok("job stat") elif arguments["list"]: output = arguments["--output"] Console.ok("lists the jobs in the format specified") elif arguments["insert"]: name = arguments["NAME"] host = arguments["HOST"] options = arguments["OPTIONS"] input_files = arguments["INPUT_FILES"] output_file = arguments["OUTPUT_FILES"] Console.ok("insert") elif arguments["find"] and arguments["--name"]: name = arguments["NAME"] Console.ok("find the job with the given name") elif arguments["find"] and arguments["--attribute"] and arguments["--value"]: name = arguments["NAME"] attribute = arguments["--attribute"] value = arguments["--value"] Console.ok("job find --attribute=ATTRIBUTE --value=VALUE") pass
def start(cls): db = JobDB() db.start() Console.ok("job server start")
def stop(cls): db = JobDB() db.stop() Console.ok("job server stop")
def main(): db = JobDB() banner("info") db.info() db.start() # db.ps() db.connect() db.clear() # Insert two jobs - one with only a name and one with input and output files job0_id = db.insert("job0") job1_id = db.insert("job1", "input1", "output1") # Print job IDs of both added jobs print job0_id print job1_id # Get all jobs with given parameters # Available key names at this time are: # _id # job_name # input_filename # output_filename jobs = db.find_jobs("job_name", "job1") # Print out all returned jobs for job in jobs: print job # Query by job ID to return a single job singleJob = list(db.find_jobs("_id", job0_id)) # Print out the first job in the list print singleJob[0] # Print out count of all jobs print db.count() # Print out count of jobs given query parameters print db.count("job_name", "job1") # Show updating a job attribute print "\nORIGINAL JOB:" singleJob = list(db.find_jobs("_id", job1_id)) print singleJob[0] # Update the input filename db.update_job_attribute(job1_id, "input_filename", "new_input_file") # Print out the updated job print "\nUPDATED JOB:" singleJob = list(db.find_jobs("_id", job1_id)) print singleJob[0] # Add a job using add() job = {"job_name": "job25"} db.add(job) # Show modify() functionality job = {"job_name": "job30", "input_filename": "file1"} db.modify(job) print "\nORIGINAL JOB" for job in db.find_jobs(): print job job = {"job_name": "job30", "input_filename": "file2"} db.modify(job) print "MODIFIED JOB" for job in db.find_jobs(): print job # Show job statuses functionality print "\nJOB STATUSES:" db.job_status_stats() print "\nJOB STATUSES WITH JOBS PRINTED:" db.job_status_stats(True) # SHOW FIND_JOBS_WITH_FILE job100_id = db.insert("job100") job101_id = db.insert("job101", "file100", "file200") job102_id = db.insert("job102", "file200", "file300") inputs, outputs = db.find_jobs_with_file("file200") print("\nJobs with matching file in input:") for job in inputs: print job print("\nJobs with matching file in output:") for job in outputs: print job # Delete all jobs db.clear() # SHOW ADD_FROM_YAML db.add_from_yaml("job_example.yaml") for job in db.find_jobs(): print job # Delete all jobs db.clear() print "Database cleared." print "Job count: " + str(db.count()) db.stop()
class cm_shell_job: db = None def activate_cm_shell_job(self): self.register_command_topic('HPC', 'job') @command def do_job(self, args, arguments): """ :: Usage: job server start job server stop job server clean job server deploy job server ps job server info job server pid job script add FILENAME [--name=NAME] job script delete NAMES job script get NAME FILENAME job script cat NAME job script list job info job stat job list [--output=FORMAT] job add JOBLIST [--host=HOST] [--options=OPTIONS] [--inputs=INPUTS] [--outputs=OUTPUTS] job load FILE job write --file=filename job find --name=NAME job find --attribute=ATTRIBUTE --value=VALUE job delete JOBLIST Arguments: NAME the name of the job HOST the host on which the job should run OPTIONS options passed to the command INPUTS input files OUTPUTS output files ATTRIBUTE an attribute VALUE a value JOBLIST the job list Description: manages a job catalog to submit them to a computation cloud or Grid. Server Management job server start starts the job server job server stop stops the job server job server clean removes all data in the job server and does a graceful clean, e.g deletes all scheduled jobs job server kill kills just the job server, but does not delete the jobs from the schedulers. this command should not be called in normal circumstances. Job Management job set GROUP sets the default job group job add GROUP TODO adds a job to a group job server start starts the server job server stop stops the server job stat prints a simple statistics of the jobs job add NAMES [--host=HOST] [--option=OPTIONS] [--inputs=INPUTS] [--outputs=OUTPUTS] adds a number of jobs job add --file=filename adds the job from the file. The postfix of the file deterimnes which format it is. The formats supported are .csv, .yaml, .json job write --file=filename writes the jobs to a file. The postfix of the file deterimnes which format it is. Thfe formats supported are .csv, .yaml, .json job list [--output=OUTPUT] lists the jobs in the format specified job find --name=NAME find the job with the given name job find --attribute=ATTRIBUTE --value=VALUE find jobs that match the given attribute. job delete JOBLIST delete the job with the specified names in the joblist. THE FOLLOWING IS NOT YET DEFINED OR MAY CHANGE job add TODO ... not yet sure what in the rest of the command adds a job to the job server and returns its id job last returns the last job added to the server job delete ID deletes the job from the job server and cancels it if it is scheduled for execution. job info ID give the info of a job job submit ID HOST submits the job with the given ID to the host job list GROUP lists the jobs in the group job status [ID | GROUP] list the status of a single job or the status of all jobs in the group job status statistics list the statistics of the jobs in the job server (e.g. for the states) """ # pprint(arguments) def connect(): try: db = JobDB() db.connect() except Exception, e: print(e) raise Exception("connection error") return db if arguments["script"]: if arguments["add"]: # job script add FILENAME [--name=NAME] filename = arguments["FILENAME"] base = os.path.basename(filename) if "." in base: name = base.split(".")[0] else: name = base if arguments["--name"] is not None: name = arguments["--name"] print("Adding script {:} <- {:}".format(name, filename)) db = connect() db.add_script_from_file(name, filename) return elif arguments["delete"]: # job script delete NAMES if arguments["NAMES"]: names = hostlist.expand_hostlist(arguments["NAMES"]) db = connect() for name in names: print("Delete Script", name) db.delete_script(name) return elif arguments["get"]: # job script get NAME FILENAME Console.ok("get script") if arguments["NAME"]: name = arguments["NAME"] filename = arguments["FILENAME"] db = connect() db.write_script(name, filename) return elif arguments["cat"]: # job script cat NAME if arguments["NAME"]: name = arguments["NAME"] db = connect() script = db.get_script(name) print(script) return elif arguments["list"]: db = connect() scripts = db.list_scripts() if scripts is not None: print("\n".join(scripts)) return if arguments["server"]: if arguments["start"]: db = JobDB() db.start() return elif arguments["stop"]: db = JobDB() db.stop() return elif arguments["ps"]: db = connect() db.ps() return elif arguments["clean"]: db = connect() db.delete_jobs() return elif arguments["deploy"]: db = JobDB() db.deploy() return elif arguments["pid"]: try: db = connect() print(db.pid()) except: print("ERROR: connecting to server") return elif arguments["info"]: try: db = connect() db.info() except Exception, e: print("ERROR: connecting to server") print(e) return
class TestJobDB: def setup(self): # HEADING() self.db = JobDB() def teardown(self): # HEADING() pass def test_001_start(self): """ tests if the mongo db can be started :return: """ HEADING() self.db.start() up = self.db.isup() result = up assert result def test_002_up(self): """ tests if the mongo db can be started :return: """ HEADING() up = self.db.isup() print(up) self.db.stop() down = not self.db.isup() self.db.start() assert up and down def test_003_pid(self): """ tests if the mongo db can be started :return: """ HEADING() pid = self.db.pid() print(pid) assert True def test_004_connect(self): """ tests if a mongo db can be connected to :return: """ HEADING() self.db.connect() result = True assert result def test_005_clear(self): """ tests clearing all jobs from the db :return: """ HEADING() db = self.db db.connect() db.clear() # assert not os.path.isfile(path_expand("~/.cloudmesh/pbs/pbs.db")) assert (len(db) == 0) def test_006_add(self): """ tests adding jobs to the db :return: """ HEADING() db = self.db count = 5 db.connect() db.delete_jobs() for id in range(0, count): job = db.insert("job" + str(id)) assert len(db) == count def test_007_delete(self): """ tests deleting a single job from the db :return: """ HEADING() db = self.db db.connect() print("AAA") before_count = len(db) print("CCC", len(db)) job = db.insert("deleteme") print("DDD", len(db)) job = db.delete_jobs("job_name", "deleteme") print("EEE") after_count = len(db) print("FFF", len(db)) assert (before_count - after_count == 0) def test_008_modify(self): """ tests modifying a single job in the db :return: """ HEADING() db = self.db db.connect() job = {"job_name": "modifyme", "input_filename": "file1"} db.add(job) originalFilename = self.db.find_jobs("job_name", "modifyme")[0]["input_filename"] job = {"job_name": "modifyme", "input_filename": "file2"} db.modify(job) newFilename = self.db.find_jobs("job_name", "modifyme")[0]["input_filename"] assert (originalFilename != newFilename) def test_010_info(self): """ prints the info about the db :return: """ HEADING() db = self.db db.connect() db.info() pass def test_011_len(self): """ tests modifying a single job in the db :return: """ HEADING() db = self.db db.connect() count = len(db) print(count) assert count == 6 def test_012_yaml_load(self): """ tests adding jobs from a YAML file :return: """ HEADING() db = self.db db.connect() # Clear all jobs currently in the database to ensure a correct final assertion db.clear() # Add the jobs outlined in the YAML file db.add_from_yaml("etc/jobs.yaml") count_fgrep = len(Shell.fgrep("input:", "etc/jobs.yaml").split("\n")) # Assert that the correct number jobs have been added assert (db.count() == count_fgrep) def test_013_find_files(self): """ tests searching for a file the file being searched for exists in 3 files: twice as input and twice as output :return: """ HEADING() db = self.db db.connect() # Clear all jobs currently in the database to ensure a correct final assertion db.clear() # Add the jobs outlined in the YAML file db.add_from_yaml("etc/jobs.yaml") inputs, outputs = db.find_jobs_with_file("in1.txt") # Assert that the lengths of the inputs and outputs arrays are correct count_fgrep = len( Shell.fgrep("in1.txt", "etc/jobs.yaml").strip().split("\n")) assert (len(inputs) == count_fgrep) def test_14_jobid(self): HEADING() db = self.db db.connect() db.set_jobid(11) a = db.get_jobid() db.incr_jobid() b = db.get_jobid() print(a, b) assert b == a + 1 def test_15_jobscript(self): HEADING() db = self.db db.connect() name = "test1" contents = "hallo" db.add_script(name, contents) db.add_script("test2", "other") script = db.get_script(name) print("Script:", script) print("Content:", contents) db.write_script(name, "/tmp/script.txt") what = Shell.cat("/tmp/script.txt") assert contents == what assert contents == script def test_999_stop(self): """ tests if the mongo db can be shutdown :return: """ HEADING() self.db.stop() result = True assert result
class TestJobDB: def setup(self): # HEADING() self.db = JobDB() def teardown(self): # HEADING() pass def test_001_start(self): """ tests if the mongo db can be started :return: """ HEADING() self.db.start() up = self.db.isup() result = up assert result def test_002_up(self): """ tests if the mongo db can be started :return: """ HEADING() up = self.db.isup() print (up) self.db.stop() down = not self.db.isup() self.db.start() assert up and down def test_003_pid(self): """ tests if the mongo db can be started :return: """ HEADING() pid = self.db.pid() print (pid) assert True def test_004_connect(self): """ tests if a mongo db can be connected to :return: """ HEADING() self.db.connect() result = True assert result def test_005_clear(self): """ tests clearing all jobs from the db :return: """ HEADING() db = self.db db.connect() db.clear() # assert not os.path.isfile(path_expand("~/.cloudmesh/pbs/pbs.db")) assert(len(db) == 0) def test_006_add(self): """ tests adding jobs to the db :return: """ HEADING() db = self.db count = 5 db.connect() db.delete_jobs() for id in range(0,count): job = db.insert("job" + str(id)) assert len(db) == count def test_007_delete(self): """ tests deleting a single job from the db :return: """ HEADING() db = self.db db.connect() print ("AAA") before_count = len(db) print ("CCC", len(db)) job = db.insert("deleteme") print ("DDD", len(db)) job = db.delete_jobs("job_name", "deleteme") print ("EEE") after_count = len(db) print ("FFF", len(db)) assert(before_count - after_count == 0) def test_008_modify(self): """ tests modifying a single job in the db :return: """ HEADING() db = self.db db.connect() job = {"job_name": "modifyme", "input_filename":"file1"} db.add(job) originalFilename = self.db.find_jobs("job_name", "modifyme")[0]["input_filename"] job = {"job_name": "modifyme", "input_filename":"file2"} db.modify(job) newFilename = self.db.find_jobs("job_name", "modifyme")[0]["input_filename"] assert(originalFilename != newFilename) def test_010_info(self): """ prints the info about the db :return: """ HEADING() db = self.db db.connect() db.info() pass def test_011_len(self): """ tests modifying a single job in the db :return: """ HEADING() db = self.db db.connect() count = len(db) print (count) assert count == 6 def test_012_yaml_load(self): """ tests adding jobs from a YAML file :return: """ HEADING() db = self.db db.connect() # Clear all jobs currently in the database to ensure a correct final assertion db.clear() # Add the jobs outlined in the YAML file db.add_from_yaml("etc/jobs.yaml") count_fgrep = len(Shell.fgrep("input:", "etc/jobs.yaml").split("\n")) # Assert that the correct number jobs have been added assert(db.count() == count_fgrep) def test_013_find_files(self): """ tests searching for a file the file being searched for exists in 3 files: twice as input and twice as output :return: """ HEADING() db = self.db db.connect() # Clear all jobs currently in the database to ensure a correct final assertion db.clear() # Add the jobs outlined in the YAML file db.add_from_yaml("etc/jobs.yaml") inputs, outputs = db.find_jobs_with_file("in1.txt") # Assert that the lengths of the inputs and outputs arrays are correct count_fgrep = len(Shell.fgrep("in1.txt", "etc/jobs.yaml").strip().split("\n")) assert(len(inputs) == count_fgrep) def test_14_jobid(self): HEADING() db = self.db db.connect() db.set_jobid(11) a = db.get_jobid() db.incr_jobid() b = db.get_jobid() print (a, b) assert b == a + 1 def test_15_jobscript(self): HEADING() db = self.db db.connect() name = "test1" contents = "hallo" db.add_script(name, contents) db.add_script("test2", "other") script = db.get_script(name) print ("Script:", script) print ("Content:", contents) db.write_script(name, "/tmp/script.txt") what = Shell.cat("/tmp/script.txt") assert contents == what assert contents == script def test_999_stop(self): """ tests if the mongo db can be shutdown :return: """ HEADING() self.db.stop() result = True assert result