def run(self):
# get logfile reader
self.log = LogFile.LogFile(self.log_file)
# get google compute engine driver to interact with the compute engine
self.myDriver = GCEManager(self.service_account_email_address,
self.pem_file,
self.auth_account,
project=self.project_id)
# start job manager, which has some useful functions for checking on the status and starting jobs
self.jobManager = JobManager(
self.job_csv_file,
self.disk_csv_file,
self.myDriver,
self.log,
self.storage_directory,
self.max_instances,
self.rootdir,
update=(not self.restart),
StackdriverAPIKey=self.StackdriverAPIKey,
activateStackDriver=self.activateStackDriver)
# hard restart if restart is true to remove previous instances and disks
if self.restart:
self.log.write(
"performing hard reset of all running instances and disks")
self.jobManager.shutDown(wait=False)
# need to reboot manager too so it knows the jobs are gone
self.jobManager = JobManager(
self.job_csv_file,
self.disk_csv_file,
self.myDriver,
self.log,
self.storage_directory,
self.max_instances,
self.rootdir,
StackdriverAPIKey=self.StackdriverAPIKey,
activateStackDriver=self.activateStackDriver)
# run jobs
while (self.jobManager.remainingJobs()):
print "updating job status for all jobs"
self.jobManager.updateJobStatus()
print "starting new jobs"
self.jobManager.startNewJobs()
print "waiting for jobs to complete"
time.sleep(
self.cycle_period) # cycle every min or whatever is specified
self.jobManager.writeInstanceSummary()
# clean up
self.jobManager.shutDown()
def prepareData(envLLs):
if not hasattr(envLLs, '__iter__'):
envLLs = (envLLs, )
manager = JobManager()
for envLL in envLLs:
tiles = NED.getTiles(envLL)
for tile in tiles:
manager.addJob("Preparing %s" % (tile[0]), NED.prepDataFile, tile)
manager.finish()
console.printMessage("Postprocessing contours...")
NED.removeSeaLevelContours()
NED.simplifyContours(1.0)
NED.convertContourElevationsToFt()
NED.clusterContoursOnGeoColumn()
NED.analyzeContoursTable()
class RESTfulHTTPRequestHandler(DefaultHTTPRequestHandler):
"""Default HTTP Request Handler Interface class."""
jobManager = JobManager()
def _handle_OPTIONS(self):
"""Handle OPTIONS function."""
try:
self.send_response(200, "OK")
self.send_header('Access-Control-Allow-Origin', '*')
self.send_header('Access-Control-Allow-Methods', 'POST, OPTIONS')
self.end_headers()
logger.debug("Sent response: \"200\"")
except Exception as ex:
logger.error(str(ex))
raise ex
def _handle_GET(self):
"""Handle GET function. Override this method."""
try:
parsed_path = urlparse(self.path)
if (parsed_path.path == "/api/jobs"):
if (parsed_path.query == "status=running"):
self._handle_get_jobs_running()
elif (parsed_path.query == "status=stopped"):
self._handle_get_jobs_stopped()
elif (parsed_path.query == "status=completed"):
self._handle_get_jobs_completed()
else:
self._handle_get_jobs_all()
m = re.match(r"/api/jobs/(\d+)", parsed_path.path)
if (m):
self._handle_get_job(int(m[1]))
else:
self.send_response(404)
self.end_headers()
logger.debug("Sent response: \"404\"")
except Exception as ex:
logger.error(ex)
raise ex
#self.send_response(501, "Not implemented")
def _handle_POST(self):
"""Handle POST function."""
try:
parsed_path = urlparse(self.path)
if (parsed_path.path == "/api/jobs"):
self._handle_post_job_start()
else:
self.send_response(404)
self.end_headers()
except Exception as ex:
logger.error(str(ex))
raise ex
def _handle_DELETE(self):
"""Handle DELETE function."""
try:
self.send_response(404)
self.end_headers()
except Exception as ex:
logger.error(str(ex))
raise ex
def _handle_PUT(self):
"""Handle PUT function."""
try:
parsed_path = urlparse(self.path)
m = re.match(r"/api/jobs/(\d+)/stop", parsed_path.path)
if (m):
self._handle_put_job_stop(int(m[1]))
else:
self.send_response(404)
self.end_headers()
except Exception as ex:
logger.error(str(ex))
raise ex
def _handle_post_job_start(self):
"""Handle POST to /api/jobs"""
try:
content_length = int(self.headers['Content-Length'])
body = self.rfile.read(content_length)
self.send_response(200)
self.end_headers()
jobData = json.loads(body)
# process json and gets args
cmd = jobData["command"]
jobMap = dict()
if cmd != "":
id = self.jobManager.create_job(cmd)
jobMap = {
"id": id,
"status": "running"
}
else:
jobMap = {
"id": -1,
"status": "stopped"
}
jsonString = json.dumps(jobMap)
self.wfile.write(bytes(jsonString, "utf8"))
except Exception as ex:
logger.error(str(ex))
raise ex
def _handle_put_job_stop(self, id):
"""Handle PUT to /api/jobs/<id>/stop"""
if self.jobManager.kill_job(id):
self.send_response(200)
self.end_headers()
else:
self.send_response(500)
def _send_json_response(self, code, map):
"""Sends HTTP response with JSON in the body"""
jsonString = json.dumps(map)
self.send_response(code)
self.send_header('Content-type', 'application/json;charset=utf-8')
self.send_header('Content-length', len(jsonString))
self.end_headers()
self.wfile.write(bytes(jsonString, "utf8"))
def _handle_get_job(self, id):
"""Handle GET to /api/jobs/<id>"""
job = self.jobManager.get_job(id)
jobMap = dict()
code = 404
if job:
jobstatus = jobStatusAsString(job.status)
jobout = job.stdout.decode("utf-8")
jobMap = {"id": job.id, "status": jobstatus, "command": job.command, "stdout": jobout}
code = 200
else:
jobMap = {"id": -1}
self._send_json_response(code, jobMap)
def _handle_get_jobs_all(self):
"""Handle GET to /api/jobs"""
joblist = self.jobManager.get_jobs()
jobMap = {
"jobs": joblist
}
self._send_json_response(200, jobMap)
def _handle_get_jobs_running(self):
"""Handle GET to /api/jobs?status=running"""
joblist = self.jobManager.get_jobs_running()
jobMap = {
"jobs": joblist
}
self._send_json_response(200, jobMap)
def _handle_get_jobs_stopped(self):
"""Handle GET to /api/jobs?status=stopped"""
joblist = self.jobManager.get_jobs_stopped()
jobMap = {
"jobs": joblist
}
self._send_json_response(200, jobMap)
def _handle_get_jobs_completed(self):
"""Handle GET to /api/jobs?status=completed"""
joblist = self.jobManager.get_jobs_completed()
jobMap = {
"jobs": joblist
}
self._send_json_response(200, jobMap)
def __init__(self) :
self.m_oEventSvr = LibEvent.LibEvent() # object for sync
self.m_oJobManagerSvr = JobManager()
from OutputModule import OutputModule
#Python modules
import string
import sys
########################## Begin ############################
if (len(sys.argv) == 2 and sys.argv[1] == "--help"):
print "Usage: ./COCOAss cocoaInputFile.txt baseCOCOADescriptionFile output.txt"
sys.exit()
if (len(sys.argv) != 4):
print "Bad usage. Type ./COCOAss --help"
sys.exit()
#Names
nameOfCocoa = sys.argv[2]
nameOfInput = sys.argv[1]
nameOfCVS = sys.argv[3]
#InputParser
myParser = InputParser(nameOfInput, nameOfCocoa)
#TagExpander
myExpander = TagExpander(myParser.read())
expanded = myExpander.expand()
#JobManager
manager = JobManager(nameOfCocoa, expanded)
output = manager.loop()
#WriteOutput
outputObject = OutputModule(nameOfCVS, output)
outputObject.write()
