def processResponse(self, result, sendResponse = True): "Process job completion, timeout or failure event" self.status = jobStatusMap[self.job.status] self.error = self.job.error # Update server factory's jobInfos dictionary with current job info self.channel.factory.jobInfos[self.job.jobID] = {'status': self.job.status, 'error': self.job.error, 'input': self.job.input, 'output': self.job.output} if isinstance(result, failure.Failure): # Generate job failure message to be written to log msg = "Job #%d has failed: %s" % (self.job.jobID, self.error) # Write message to event log twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, msg) if sendResponse: # Send job failure response to the client self._sendError(http.INTERNAL_SERVER_ERROR, msg) else: # Generate job success message to be written to log msg = "Job #%d has successfully completed" % self.job.jobID # Write message to event log twisted_logger.writeLog(self.channel.logPrefix, self.channel.logName, msg) if sendResponse: # Send job results to the client self._sendResponse(result, http.OK) # Delete original job entity del self.job self.job = None if not sendResponse: self.finished = True self.finish()
def stopService(self): """Stop SCSServer service @return: deferred which will 'fire' when all of the service termination tasks have completed """ if self.running: twisted_logger.writeLog(self.logPrefix, self.logName, "Stopping '%s' service..." % self.name) actions = [] if not self.listeningPort or not self.listeningPort.connected: err_msg = "Unable to stop '%s' server: server not listening on localhost:%d port" % (self.name, self.serverInfo['port']) twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) else: # Stop running SCS server msg = "'%s' server service shutdown in progress..." % self.name twisted_logger.writeLog(self.logPrefix, self.logName, msg) actions.append(self.listeningPort.stopListening().addCallback(self._stopSuccessHandler).addErrback(self._stopFailureHandler)) for srv in self.services[:]: try: deferred = self.removeService(srv) if deferred: actions.append(deferred) except Exception, err: twisted_logger.writeErr(self.logPrefix, self.logName, "Failure stopping '%s' service: %s" % (srv.name, str(err))) service.Service.stopService(self)
def __jobStartHandler(self, result): "Report job start success or failure to the client that sent this job request" if isinstance(result, failure.Failure): # Job start failure self.status = jobStatusMap['FAILURE'] self.error = "Job start failure" err_msg = "%s: %s" % (self.error, result.getErrorMessage()) twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, err_msg) # Trap failure event result.trap(Exception) # Delete job entity (it is not going to be processed anyway) del self.job self.job = None # Send failure response to external client self._sendError(http.INTERNAL_SERVER_ERROR, self.error) self.finish() else: msg = "New job #%d has been created. Status: '%s'" % (self.job.jobID, self.job.status) twisted_logger.writeLog(self.channel.logPrefix, self.channel.logName, msg) # Add job info to server factory's jobInfos dictionary self.channel.factory.jobInfos[self.job.jobID] = {'status': self.job.status, 'error': self.job.error, 'input': self.job.input, 'output': self.job.output} return "ok"
def sendRequest(self, request, deferred=None): """Send request to external server @param request: request message (dictionary) to be sent to external server @param deferred: deferred to be associated with the given job (for job request only) @type request: dictionary containing {'scs_jobid': <jobid>, 'request': <request - text>} @type deferred: defer.Deferred instance """ # Reset <self.logPrefix> to include 'scs_jobid' self.logPrefix += " JOBID #%d" % request["scs_jobid"] try: # Create JobRequest instance that will be responsible for: # 1. processing job request # 2. scheduling regular job 'STATUS' requests # 3. Handling job response - when it comes from the server self.jobRequest = client_request.JobRequest(request, deferred, self) # Process job request self.jobRequest.processRequest() except RuntimeError, err: twisted_logger.writeErr(self.logPrefix, self.logName, str(err)) if isinstance(deferred.__class__, defer.Deferred): twisted_logger.writeLog( self.logPrefix, self.logName, "Executing external job step deferred's errback() to report job start failure", ) deferred.errback(failure.Failure(str(err)), RuntimeError)
def stopService(self): "Stop SCSClient service" twisted_logger.writeLog(self.logPrefix, self.logName, "Stopping '%s' service..." % self.name) actions = [] if self.running: for clientProtocol in self.clientProtocols[:]: if hasattr(clientProtocol, "jobRequest"): clientProtocol._stopJobRequest() if clientProtocol.connected == 1: msg = "Client is being disconnected from %s server..." % self.peer twisted_logger.writeLog(self.logPrefix, self.logName, msg) actions.append(self.stopClient(clientProtocol)) for srv in self.services[:]: try: deferred = self.removeService(srv) if deferred: actions.append(deferred) except Exception, err: twisted_logger.writeErr( self.logPrefix, self.logName, "Failure stopping '%s' service: %s" % (srv.name, str(err)) ) service.Service.stopService(self)
def _stop(self): """JobRequest termination: 1. fire callback or errback method associated with request's deferred 2. perform cleanup """ if not self.stopped: # Stop job status requests and remove such requests from factory's cachedRequests list (if any) self.statusRequest._stop() # Delete JobStatusRequest entity - to free up memory del self.statusRequest self.stopped = True # Execute deferred's callback() or errback() to notify JobStep (and Job) instance of job step completion if hasattr(self, 'deferred') and self.deferred.called == 0: if self.status == 'FAILURE' and self.error: msg = "Executing deferred's errback() to notify of job step failure" twisted_logger.writeErr(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg) # Execute deferred's errback() to notify of job's failure self.deferred.errback(failure.Failure(self.error, RuntimeError)) elif self.status == 'SUCCESS': msg = "Executing deferred's callback() to notify of job step success" twisted_logger.writeLog(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg) # Execute deferred's callback() to notify of job's success self.deferred.callback(self.response) else: self.deferred.called = 1 else: msg = "Deferred's callback/errback has been canceled" twisted_logger.writeLog(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)
def addOfflineDeferred(self, name, deferred, reset = True): "Add new external deferred to workflow's <extOfflineDeferreds> dictionary" if not self.namedServices.has_key(name): err_msg = "addOfflineDeferred() failure: '%s' workflow entity has not been started" % name twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) else: self.namedServices[name].workflow.addOfflineDeferred(deferred, reset)
def start(self, name): """Start SCS server service <name> @param name: SCS server name @return: deferred which will 'fire' when given server (service) has started """ failPrefix = "Failure starting '%s' SCS server service" % name if self.namedServices.has_key(name): if self.namedServices[name].running: err_msg = "%s: Server is already running on port #%d" % (failPrefix, self.serverInfo[name]['port']) twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg else: # Server service has stopped: delete that service before restarting it self.removeService(self.getServiceNamed(name)) if not self.serverInfo[name]['enabled']: err_msg = "%s: Server is disabled" % failPrefix twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg if not self.running: self.running = 1 if self.namedServices.has_key(name): serverService = self.namedServices[name] defered = serverService.startService() else: serverService = _SCSServerService(name, self.serverInfo[name], self.logName, self.logDir) defered = addService(self, serverService) return defered
def setWorkflow(self, name): "Set server's workflow attribute" try: listeningPort = self.getResource(name) listeningPort.factory._setWorkflow() except Exception, err: err_msg = "Unable to set server's workflow: %s" % str(err) twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg
def connectionLost(self, reason): "Client connection lost" msg = "Connection lost with %s client: %s" % (self.peer, reason.getErrorMessage()) twisted_logger.writeErr(self.logPrefix, self.logName, msg) http.HTTPChannel.connectionLost(self, reason) # Remove itself from factory's protocols list if self in self.factory.protocols: self.factory.protocols.remove(self)
def _connFailureHandler(self, fail): "Client connection's failure handler" err_msg = "Connection failure: %s" % fail.getErrorMessage() twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) fail.trap(Exception) # Cleanup client deferreds for d in self.deferreds: if d.called == 1: self.deferreds.remove(d)
def _failure(self, error): "Process job failure event. Executes self._stop() that fires self.deferred.errback()" if self.status is None: # Update job request's status and error attributes self.status = 'FAILURE' self.error = error err_msg = "Job step (job #%d) has failed: %s" % (self.scs_jobid, error) twisted_logger.writeErr(self.clientProtocol.logPrefix, self.clientProtocol.logName, err_msg) # Stop job request's processing self._stop()
def _loadInfoComplete(self, result): "Server data load success handler. Completes load of self.serverInfo dictionary" if result == (): err_msg = "scs.scs_server table contains no server information (empty)" twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg for (server_name, wf_name, port, protocol, max_connections, enabled) in result: self.serverInfo[server_name] = {'workflow': wf_name, 'port': port, 'protocol': protocol, 'max_connections': max_connections, 'enabled': enabled}
def _disconnFailureHandler(self, failure, protocol): "Client disconnect failure handler" err_msg = "Client disconnect failure: %s" % failure.getErrorMessage() twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) fail.trap(Exception) # Delete protocol's deferred nad protocol entity itself del protocol.deferred del protocol for d in self.deferreds: if d.called == 1: self.deferreds.remove(d)
def _sendResponse(self, response, code = None, code_message = None): "Send response to the client" if code: self.setResponseCode(code, code_message) msg = "Sending response to %s client: %s" % (self.channel.peer, response) twisted_logger.writeLog(self.channel.logPrefix, self.channel.logName, msg) try: self.write(str(response)) except Exception, err: err_msg = "Failure sending '%s' response to %s: %s" % (response['type'], self.channel.peer, str(err)) twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, err_msg)
def stop(self, workflowName): """Stop workflow service @param workflowName: workflow name """ if not workflowName in self.namedServices.keys(): err_msg = "Unable to stop '%s' workflow service: service has not been started" % workflowName twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg # Remove workflow from WorkflowManager's services collection self.removeService(self.getServiceNamed(workflowName))
def getResource(self, name): """Obtain given workflow instance @param name: workflow name @return: _Workflow instance """ if not self.namedServices.has_key(name): err_msg = "'%s' workflow has not been started" % name twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg return self.namedServices[name].workflow
def startService(self): """Start given SCSServer service @return: deferred which will 'fire' when all of the service initialization tasks have completed """ actions = [] failPrefix = "Failure starting '%s' SCS server service" % self.name try: factory = _SCSServerFactory(self.name, self.serverInfo, self.logDir) except RuntimeError, err: err_msg = "%s: %s" % (failPrefix, str(err)) twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
def stop(self, name): """Stop SCS server service <name> @param name: SCS server name @return: deferred which will 'fire' when given service has stopped """ if not self.namedServices.has_key(name): err_msg = "Unable to stop '%s' SCS server service: service has not been started" % name twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg serverService = self.getServiceNamed(name) return defer.maybeDeferred(self.removeService, serverService)
def processRequest(self): """ Process given job status request @attention: This method is scheduled (by the JobRequest entity that created this JobStatusRequest instance) to run in a loop (with the given time interval). It simply executes self.clientProtocol.sendLine() of an SCSClientProtocol instance that created this JobStatusRequest entity, passing in itself """ msg = "Sending job status request (job #%d) to '%s' server" % (self.jobRequest.scs_jobid, self.jobRequest.clientProtocol.peer) twisted_logger.writeLog(self.jobRequest.clientProtocol.logPrefix, self.jobRequest.clientProtocol.logName, msg) try: self.jobRequest.clientProtocol.sendLine(self.request) except Exception, err: err_msg = "Unable to send job status request corresponding to job #%d: %s" % (self.jobRequest.scs_jobid, str(err)) twisted_logger.writeErr(self.jobRequest.clientProtocol.logPrefix, self.jobRequest.clientProtocol.logName, err_msg)
def process(self): """Process job request - start job (job.Job instance) to execute a set of job steps associated with the server's workflow @note: This method simply creates Job instance and starts its execution @return: deferred which will 'fire' when Job instance is initialized and first JobStep has started execution """ try: self._checkRequest() except RuntimeError, err: error = str(err) err_msg = "Invalid request received from %s: %s" % (self.channel.peer, error) twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, err_msg) # Send failure response to the client self._sendError(http.BAD_REQUEST, 'Invalid request: %s' % error) self.finish()
def run(self, request, ext_deferred): """Start asynchronous job exection by executing first job step @return: deferred (twisted.internet.defer.Deferred instance) which will 'fire' when first JobStep has been started """ self.input = request # Set <self.deferred> to the supplied external (SCSServer entity's) deferred self.deferred = ext_deferred if self.steps == []: err_msg = "Job #%d does not include any job steps" % self.jobID twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) return Failure(err_msg, RuntimeError) else: # Start first job step's execution return self.runJobStep(None)
def connectionMade(self): "Client connection made" http.HTTPChannel.connectionMade(self) peer = self.transport.getPeer() self.peer = "%s:%d" % (peer.host, peer.port) msg = "Connection established from %s" % (self.peer) twisted_logger.writeLog(self.logPrefix, self.logName, msg) if self.factory.numPorts >= self.factory.maxConnections: err_msg = "Client connection (%s) rejected: maximum number of client connections reached" % self.peer twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) self.transport.loseConnection() else: msg = "Client connection from %s is accepted" % (self.peer) twisted_logger.writeLog(self.logPrefix, self.logName, msg) if self not in self.factory.protocols: # Add protocol (self) to the factory's <factory.protocols> list self.factory.protocols.append(self)
def lineReceived(self, response): """Process response received from the server @param response: data received from external server @type response: dictionary @attention: <response> message should be in the following form: {'type': <response type>, 'scs_jobid': <jobid>, ...} """ response = json.loads(response) twisted_logger.writeLog(self.logPrefix, self.logName, "Received message: %s" % response) try: # Verify request message response = self.__checkResponse(response) except RuntimeError, err: err_msg = "Invalid response received from %s: %s" % (self.peer, str(err)) twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) if hasattr(self, "jobRequest") and not self.jobRequest.stopped: self.jobRequest._stop()
def _loadComplete(self, result, workflowInfo): "Complete loading workflowInfo data (started by self._loadInfo() and self._loadStepInfo()" if result == (): err_msg = "scs.workflow_step table contains no workflow information (empty)" twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg for (wf_name, step_id, step_no, client_name, input, out_flag, enabled) in result: if not self.workflowInfo.has_key(wf_name): if not workflowInfo.has_key(wf_name): err_msg = "'%s' workflow is not defined within scs.workflow_lookup table" % wf_name twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg self.workflowInfo.setdefault(wf_name, {'enabled': workflowInfo[wf_name]['enabled'], 'timeout': workflowInfo[wf_name]['timeout'], 'steps': []}) stepRec = {'stepID': step_id, 'stepNo': step_no, 'clientName': client_name, 'inputSrc': input, 'outFlag': out_flag, 'enabled': enabled} self.workflowInfo[wf_name]['steps'].append(stepRec)
def start(self, name): """Start SCS client service <name> @param name: SCS client name @return: deferred which will 'fire' when given client (service) has started """ failPrefix = "Failure starting '%s' SCS client service" % name if not self.clientInfo.has_key(name): err_msg = "%s: Unknown client" % failPrefix twisted_logger.writeLog(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg if not self.clientInfo[name]["enabled"]: err_msg = "%s: Client is disabled" % failPrefix twisted_logger.writeLog(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg if self.namedServices.has_key(name): if self.namedServices[name].running: err_msg = "%s: Client service is already running" % failPrefix twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg else: # Client service has stopped: delete that service before restarting it self.removeService(self.getServiceNamed(name)) if not self.running: self.running = 1 if self.namedServices.has_key(name): clientService = self.namedServices[name] defered = clientService.startService() else: clientService = _SCSClientService(name, self.clientInfo[name], self.logName, self.logDir) defered = addService(self, clientService) return defered
def start(self, workflowName): """Create workflow instance <workflowName> @param workflowName: workflow name @type workflowName: str """ failPrefix = "Failure starting '%s' workflow service" % workflowName if self.namedServices.has_key(workflowName): if self.namedServices[workflowName].running: err_msg = "%s: Workflow service is already running" % failPrefix twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg else: # Client service has stopped: delete that service before restarting it self.removeService(self.getServiceNamed(workflowName)) if self.workflowInfo.has_key(workflowName): if not self.workflowInfo[workflowName]['enabled']: err_msg = "%s: workflow is disabled" % failPrefix twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg else: err_msg = "%s: unknown workflow" % failPrefix twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) raise RuntimeError, err_msg if not self.running: self.running = 1 if self.namedServices.has_key(workflowName): workflowService = self.namedServices[workflowName] workflowService.startService() else: workflowService = _WorkflowService(workflowName, self.workflowInfo[workflowName], self.logName) addService(self, workflowService)
def __failure(self, nothing, err_msg): "Last part of Job's failure handler" msg = "Job has failed: %s" % err_msg twisted_logger.writeErr(self.logPrefix, self.logName, msg) if self.deferred and self.deferred.called == 0: self.deferred.errback(Failure(err_msg, RuntimeError))
def __handleDBFailure(self, fail, stmt, type): "Record <scs.job_step> insert/update failure details" operationType = {'insert': 'inserting into', 'update': 'updating'} err_msg = "Failure %s <scs.job> table: %s" % (operationType[type], fail.getErrorMessage()) twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) twisted_logger.writeErr(self.logPrefix, self.logName, stmt)
def _loadInfoFailure(self, fail, query): "Server data load failure handler" err_msg = "Failure loading workflow info: %s" % fail.getErrorMessage() twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) twisted_logger.writeErr(self.logPrefix, self.logName, "SQL Query: '%s'" % query)