def confirmationNumberCallback(self, response):
"""Asynchronous callback when model run is populated to confirmation queue."""
if response.error: raise tornado.web.HTTPError(500)
try:
json = tornado.escape.json_decode(response.body)
logging.info(json)
res = json["response"]
model = modelManager.getModel(res["task"]["modelName"], res["task"]["modelVersion"])
task = model.fromDict(res["task"])
code = res["code"]
except KeyError:
logging.info("Bad response from queue server")
raise tornado.web.HTTPError(500)
self.render(config.confirmTemplatePath, email=task.emailAddress, code=code)
def post(self, modelName):
"""Post handler to actually create a model task with given parameters."""
modelVersion = self.get_argument("modelVersion")
try:
model = modelManager.getModel(modelName, modelVersion)
except KeyError:
self.queueErrorRender(
" We are sorry. Your model request did not match any models available."
" This is probably caused by an upgrade to the model versions, " +
" and will be fixed if you resubmit your request.")
return
try:
task = self.setupModelTask(model)
except ValidationError, e:
logging.exception(e)
self.render(config.modelTemplatePath, model=model, errorText=str(e))
return
def processTask(self, taskDict):
"""Handle creation and running of a model and setup heartbeat thread.
This is the heart of a worker. When we find a model on the queue, this
method takes the request and decodes it into something that can be processed.
It will then spawn a heartbeat thread that continues to check into the server
while we actually enter the models "run" method. From there, it is all up
to the model to handle.
"""
taskId = None
if "taskId" in taskDict:
taskId = taskDict["taskId"]
try:
try:
model = modelManager.getModel(taskDict["modelName"], taskDict["modelVersion"])
logging.info("Creating a model task for '%s'", taskDict["modelName"])
taskObject = model.fromDict(taskDict)
except KeyError, e:
logging.warning("Was unable to deserialize model task (%s), model task: %s", e, taskDict)
if taskId:
self.notifyFailedTask(taskId)
return
keepAliveThread = TaskKeepAliveThread(self.taskKeepAliveRequest, taskObject.taskId)
keepAliveThread.start()
try:
resultsEmail = taskObject.run()
logging.info("Model finished running, sending email")
if self.serverHasTask(taskObject.taskId):
npsgd.email_manager.blockingEmailSend(resultsEmail)
logging.info("Email sent, model is 100% complete!")
self.notifySucceedTask(taskObject.taskId)
else:
logging.warning("Skipping task completion since the server forgot about our task")
except RuntimeError, e:
logging.error("Some kind of error during processing model task, notifying server of failure")
logging.exception(e)
self.notifyFailedTask(taskObject.taskId)
