def connect(): try: db = JobDB() db.connect() except Exception, e: print(e) raise Exception("connection error")
# # how to test # # python eve/run.py in one terminal # # curl -i http://localhost:5000/jobs # # from flask.ext.bootstrap import Bootstrap from eve_docs import eve_docs from cloudmesh_job.cm_jobdb import JobDB db = JobDB() print db.port my_settings = { 'MONGO_HOST': 'localhost', 'MONGO_PORT': db.port, 'MONGO_DBNAME': 'jobsdb', 'DOMAIN': { 'jobs': {} } } from eve import Eve app = Eve(settings=my_settings) Bootstrap(app) app.register_blueprint(eve_docs, url_prefix='/docs')
def start(cls): db = JobDB() db.start() Console.ok("job server start")
def stop(cls): db = JobDB() db.stop() Console.ok("job server stop")
class cm_shell_job: db = None def activate_cm_shell_job(self): self.register_command_topic('HPC', 'job') @command def do_job(self, args, arguments): """ :: Usage: job server start job server stop job server clean job server deploy job server ps job server info job server pid job script add FILENAME [--name=NAME] job script delete NAMES job script get NAME FILENAME job script cat NAME job script list job info job stat job list [--output=FORMAT] job add JOBLIST [--host=HOST] [--options=OPTIONS] [--inputs=INPUTS] [--outputs=OUTPUTS] job load FILE job write --file=filename job find --name=NAME job find --attribute=ATTRIBUTE --value=VALUE job delete JOBLIST Arguments: NAME the name of the job HOST the host on which the job should run OPTIONS options passed to the command INPUTS input files OUTPUTS output files ATTRIBUTE an attribute VALUE a value JOBLIST the job list Description: manages a job catalog to submit them to a computation cloud or Grid. Server Management job server start starts the job server job server stop stops the job server job server clean removes all data in the job server and does a graceful clean, e.g deletes all scheduled jobs job server kill kills just the job server, but does not delete the jobs from the schedulers. this command should not be called in normal circumstances. Job Management job set GROUP sets the default job group job add GROUP TODO adds a job to a group job server start starts the server job server stop stops the server job stat prints a simple statistics of the jobs job add NAMES [--host=HOST] [--option=OPTIONS] [--inputs=INPUTS] [--outputs=OUTPUTS] adds a number of jobs job add --file=filename adds the job from the file. The postfix of the file deterimnes which format it is. The formats supported are .csv, .yaml, .json job write --file=filename writes the jobs to a file. The postfix of the file deterimnes which format it is. Thfe formats supported are .csv, .yaml, .json job list [--output=OUTPUT] lists the jobs in the format specified job find --name=NAME find the job with the given name job find --attribute=ATTRIBUTE --value=VALUE find jobs that match the given attribute. job delete JOBLIST delete the job with the specified names in the joblist. THE FOLLOWING IS NOT YET DEFINED OR MAY CHANGE job add TODO ... not yet sure what in the rest of the command adds a job to the job server and returns its id job last returns the last job added to the server job delete ID deletes the job from the job server and cancels it if it is scheduled for execution. job info ID give the info of a job job submit ID HOST submits the job with the given ID to the host job list GROUP lists the jobs in the group job status [ID | GROUP] list the status of a single job or the status of all jobs in the group job status statistics list the statistics of the jobs in the job server (e.g. for the states) """ # pprint(arguments) def connect(): try: db = JobDB() db.connect() except Exception, e: print(e) raise Exception("connection error") return db if arguments["script"]: if arguments["add"]: # job script add FILENAME [--name=NAME] filename = arguments["FILENAME"] base = os.path.basename(filename) if "." in base: name = base.split(".")[0] else: name = base if arguments["--name"] is not None: name = arguments["--name"] print("Adding script {:} <- {:}".format(name, filename)) db = connect() db.add_script_from_file(name, filename) return elif arguments["delete"]: # job script delete NAMES if arguments["NAMES"]: names = hostlist.expand_hostlist(arguments["NAMES"]) db = connect() for name in names: print("Delete Script", name) db.delete_script(name) return elif arguments["get"]: # job script get NAME FILENAME Console.ok("get script") if arguments["NAME"]: name = arguments["NAME"] filename = arguments["FILENAME"] db = connect() db.write_script(name, filename) return elif arguments["cat"]: # job script cat NAME if arguments["NAME"]: name = arguments["NAME"] db = connect() script = db.get_script(name) print(script) return elif arguments["list"]: db = connect() scripts = db.list_scripts() if scripts is not None: print("\n".join(scripts)) return if arguments["server"]: if arguments["start"]: db = JobDB() db.start() return elif arguments["stop"]: db = JobDB() db.stop() return elif arguments["ps"]: db = connect() db.ps() return elif arguments["clean"]: db = connect() db.delete_jobs() return elif arguments["deploy"]: db = JobDB() db.deploy() return elif arguments["pid"]: try: db = connect() print(db.pid()) except: print("ERROR: connecting to server") return elif arguments["info"]: try: db = connect() db.info() except Exception, e: print("ERROR: connecting to server") print(e) return
def setup(self): # HEADING() self.db = JobDB()
def main(): db = JobDB() banner("info") db.info() db.start() # db.ps() db.connect() db.clear() # Insert two jobs - one with only a name and one with input and output files job0_id = db.insert("job0") job1_id = db.insert("job1", "input1", "output1") # Print job IDs of both added jobs print job0_id print job1_id # Get all jobs with given parameters # Available key names at this time are: # _id # job_name # input_filename # output_filename jobs = db.find_jobs("job_name", "job1") # Print out all returned jobs for job in jobs: print job # Query by job ID to return a single job singleJob = list(db.find_jobs("_id", job0_id)) # Print out the first job in the list print singleJob[0] # Print out count of all jobs print db.count() # Print out count of jobs given query parameters print db.count("job_name", "job1") # Show updating a job attribute print "\nORIGINAL JOB:" singleJob = list(db.find_jobs("_id", job1_id)) print singleJob[0] # Update the input filename db.update_job_attribute(job1_id, "input_filename", "new_input_file") # Print out the updated job print "\nUPDATED JOB:" singleJob = list(db.find_jobs("_id", job1_id)) print singleJob[0] # Add a job using add() job = {"job_name": "job25"} db.add(job) # Show modify() functionality job = {"job_name": "job30", "input_filename": "file1"} db.modify(job) print "\nORIGINAL JOB" for job in db.find_jobs(): print job job = {"job_name": "job30", "input_filename": "file2"} db.modify(job) print "MODIFIED JOB" for job in db.find_jobs(): print job # Show job statuses functionality print "\nJOB STATUSES:" db.job_status_stats() print "\nJOB STATUSES WITH JOBS PRINTED:" db.job_status_stats(True) # SHOW FIND_JOBS_WITH_FILE job100_id = db.insert("job100") job101_id = db.insert("job101", "file100", "file200") job102_id = db.insert("job102", "file200", "file300") inputs, outputs = db.find_jobs_with_file("file200") print("\nJobs with matching file in input:") for job in inputs: print job print("\nJobs with matching file in output:") for job in outputs: print job # Delete all jobs db.clear() # SHOW ADD_FROM_YAML db.add_from_yaml("job_example.yaml") for job in db.find_jobs(): print job # Delete all jobs db.clear() print "Database cleared." print "Job count: " + str(db.count()) db.stop()