def connect():
try:
db = JobDB()
db.connect()
except Exception, e:
print(e)
raise Exception("connection error")
#
# how to test
#
# python eve/run.py in one terminal
#
# curl -i http://localhost:5000/jobs
#
#
from flask.ext.bootstrap import Bootstrap
from eve_docs import eve_docs
from cloudmesh_job.cm_jobdb import JobDB
db = JobDB()
print db.port
my_settings = {
'MONGO_HOST': 'localhost',
'MONGO_PORT': db.port,
'MONGO_DBNAME': 'jobsdb',
'DOMAIN': {
'jobs': {}
}
}
from eve import Eve
app = Eve(settings=my_settings)
Bootstrap(app)
app.register_blueprint(eve_docs, url_prefix='/docs')
def start(cls):
db = JobDB()
db.start()
Console.ok("job server start")
def stop(cls):
db = JobDB()
db.stop()
Console.ok("job server stop")
class cm_shell_job:
db = None
def activate_cm_shell_job(self):
self.register_command_topic('HPC', 'job')
@command
def do_job(self, args, arguments):
"""
::
Usage:
job server start
job server stop
job server clean
job server deploy
job server ps
job server info
job server pid
job script add FILENAME [--name=NAME]
job script delete NAMES
job script get NAME FILENAME
job script cat NAME
job script list
job info
job stat
job list [--output=FORMAT]
job add JOBLIST [--host=HOST] [--options=OPTIONS] [--inputs=INPUTS] [--outputs=OUTPUTS]
job load FILE
job write --file=filename
job find --name=NAME
job find --attribute=ATTRIBUTE --value=VALUE
job delete JOBLIST
Arguments:
NAME the name of the job
HOST the host on which the job should run
OPTIONS options passed to the command
INPUTS input files
OUTPUTS output files
ATTRIBUTE an attribute
VALUE a value
JOBLIST the job list
Description:
manages a job catalog to submit them to a computation cloud
or Grid.
Server Management
job server start
starts the job server
job server stop
stops the job server
job server clean
removes all data in the job server and does a graceful clean, e.g deletes all scheduled jobs
job server kill
kills just the job server, but does not delete the jobs from the schedulers.
this command should not be called in normal circumstances.
Job Management
job set GROUP
sets the default job group
job add GROUP TODO
adds a job to a group
job server start
starts the server
job server stop
stops the server
job stat
prints a simple statistics of the jobs
job add NAMES [--host=HOST] [--option=OPTIONS] [--inputs=INPUTS] [--outputs=OUTPUTS]
adds a number of jobs
job add --file=filename
adds the job from the file. The postfix of the file deterimnes which
format it is. The formats supported are .csv, .yaml, .json
job write --file=filename
writes the jobs to a file. The postfix of the file deterimnes which
format it is. Thfe formats supported are .csv, .yaml, .json
job list [--output=OUTPUT]
lists the jobs in the format specified
job find --name=NAME
find the job with the given name
job find --attribute=ATTRIBUTE --value=VALUE
find jobs that match the given attribute.
job delete JOBLIST
delete the job with the specified names in the joblist.
THE FOLLOWING IS NOT YET DEFINED OR MAY CHANGE
job add TODO
... not yet sure what in the rest of the command
adds a job to the job server and returns its id
job last
returns the last job added to the server
job delete ID
deletes the job from the job server and cancels it if it is scheduled for execution.
job info ID
give the info of a job
job submit ID HOST
submits the job with the given ID to the host
job list GROUP
lists the jobs in the group
job status [ID | GROUP]
list the status of a single job or the status of all jobs in the group
job status statistics
list the statistics of the jobs in the job server (e.g. for the states)
"""
# pprint(arguments)
def connect():
try:
db = JobDB()
db.connect()
except Exception, e:
print(e)
raise Exception("connection error")
return db
if arguments["script"]:
if arguments["add"]:
# job script add FILENAME [--name=NAME]
filename = arguments["FILENAME"]
base = os.path.basename(filename)
if "." in base:
name = base.split(".")[0]
else:
name = base
if arguments["--name"] is not None:
name = arguments["--name"]
print("Adding script {:} <- {:}".format(name, filename))
db = connect()
db.add_script_from_file(name, filename)
return
elif arguments["delete"]:
# job script delete NAMES
if arguments["NAMES"]:
names = hostlist.expand_hostlist(arguments["NAMES"])
db = connect()
for name in names:
print("Delete Script", name)
db.delete_script(name)
return
elif arguments["get"]:
# job script get NAME FILENAME
Console.ok("get script")
if arguments["NAME"]:
name = arguments["NAME"]
filename = arguments["FILENAME"]
db = connect()
db.write_script(name, filename)
return
elif arguments["cat"]:
# job script cat NAME
if arguments["NAME"]:
name = arguments["NAME"]
db = connect()
script = db.get_script(name)
print(script)
return
elif arguments["list"]:
db = connect()
scripts = db.list_scripts()
if scripts is not None:
print("\n".join(scripts))
return
if arguments["server"]:
if arguments["start"]:
db = JobDB()
db.start()
return
elif arguments["stop"]:
db = JobDB()
db.stop()
return
elif arguments["ps"]:
db = connect()
db.ps()
return
elif arguments["clean"]:
db = connect()
db.delete_jobs()
return
elif arguments["deploy"]:
db = JobDB()
db.deploy()
return
elif arguments["pid"]:
try:
db = connect()
print(db.pid())
except:
print("ERROR: connecting to server")
return
elif arguments["info"]:
try:
db = connect()
db.info()
except Exception, e:
print("ERROR: connecting to server")
print(e)
return
def setup(self):
# HEADING()
self.db = JobDB()
def main():
db = JobDB()
banner("info")
db.info()
db.start()
# db.ps()
db.connect()
db.clear()
# Insert two jobs - one with only a name and one with input and output files
job0_id = db.insert("job0")
job1_id = db.insert("job1", "input1", "output1")
# Print job IDs of both added jobs
print job0_id
print job1_id
# Get all jobs with given parameters
# Available key names at this time are:
# _id
# job_name
# input_filename
# output_filename
jobs = db.find_jobs("job_name", "job1")
# Print out all returned jobs
for job in jobs:
print job
# Query by job ID to return a single job
singleJob = list(db.find_jobs("_id", job0_id))
# Print out the first job in the list
print singleJob[0]
# Print out count of all jobs
print db.count()
# Print out count of jobs given query parameters
print db.count("job_name", "job1")
# Show updating a job attribute
print "\nORIGINAL JOB:"
singleJob = list(db.find_jobs("_id", job1_id))
print singleJob[0]
# Update the input filename
db.update_job_attribute(job1_id, "input_filename", "new_input_file")
# Print out the updated job
print "\nUPDATED JOB:"
singleJob = list(db.find_jobs("_id", job1_id))
print singleJob[0]
# Add a job using add()
job = {"job_name": "job25"}
db.add(job)
# Show modify() functionality
job = {"job_name": "job30", "input_filename": "file1"}
db.modify(job)
print "\nORIGINAL JOB"
for job in db.find_jobs():
print job
job = {"job_name": "job30", "input_filename": "file2"}
db.modify(job)
print "MODIFIED JOB"
for job in db.find_jobs():
print job
# Show job statuses functionality
print "\nJOB STATUSES:"
db.job_status_stats()
print "\nJOB STATUSES WITH JOBS PRINTED:"
db.job_status_stats(True)
# SHOW FIND_JOBS_WITH_FILE
job100_id = db.insert("job100")
job101_id = db.insert("job101", "file100", "file200")
job102_id = db.insert("job102", "file200", "file300")
inputs, outputs = db.find_jobs_with_file("file200")
print("\nJobs with matching file in input:")
for job in inputs:
print job
print("\nJobs with matching file in output:")
for job in outputs:
print job
# Delete all jobs
db.clear()
# SHOW ADD_FROM_YAML
db.add_from_yaml("job_example.yaml")
for job in db.find_jobs():
print job
# Delete all jobs
db.clear()
print "Database cleared."
print "Job count: " + str(db.count())
db.stop()
