def confirmationNumberCallback(self, response): """Asynchronous callback when model run is populated to confirmation queue.""" if response.error: raise tornado.web.HTTPError(500) try: json = tornado.escape.json_decode(response.body) logging.info(json) res = json["response"] model = modelManager.getModel(res["task"]["modelName"], res["task"]["modelVersion"]) task = model.fromDict(res["task"]) code = res["code"] except KeyError: logging.info("Bad response from queue server") raise tornado.web.HTTPError(500) self.render(config.confirmTemplatePath, email=task.emailAddress, code=code)
def post(self, modelName): """Post handler to actually create a model task with given parameters.""" modelVersion = self.get_argument("modelVersion") try: model = modelManager.getModel(modelName, modelVersion) except KeyError: self.queueErrorRender( " We are sorry. Your model request did not match any models available." " This is probably caused by an upgrade to the model versions, " + " and will be fixed if you resubmit your request.") return try: task = self.setupModelTask(model) except ValidationError, e: logging.exception(e) self.render(config.modelTemplatePath, model=model, errorText=str(e)) return
def processTask(self, taskDict): """Handle creation and running of a model and setup heartbeat thread. This is the heart of a worker. When we find a model on the queue, this method takes the request and decodes it into something that can be processed. It will then spawn a heartbeat thread that continues to check into the server while we actually enter the models "run" method. From there, it is all up to the model to handle. """ taskId = None if "taskId" in taskDict: taskId = taskDict["taskId"] try: try: model = modelManager.getModel(taskDict["modelName"], taskDict["modelVersion"]) logging.info("Creating a model task for '%s'", taskDict["modelName"]) taskObject = model.fromDict(taskDict) except KeyError, e: logging.warning("Was unable to deserialize model task (%s), model task: %s", e, taskDict) if taskId: self.notifyFailedTask(taskId) return keepAliveThread = TaskKeepAliveThread(self.taskKeepAliveRequest, taskObject.taskId) keepAliveThread.start() try: resultsEmail = taskObject.run() logging.info("Model finished running, sending email") if self.serverHasTask(taskObject.taskId): npsgd.email_manager.blockingEmailSend(resultsEmail) logging.info("Email sent, model is 100% complete!") self.notifySucceedTask(taskObject.taskId) else: logging.warning("Skipping task completion since the server forgot about our task") except RuntimeError, e: logging.error("Some kind of error during processing model task, notifying server of failure") logging.exception(e) self.notifyFailedTask(taskObject.taskId)