Exemplo n.º 1
0
    def processResponse(self, result, sendResponse = True):
        "Process job completion, timeout or failure event"
        self.status = jobStatusMap[self.job.status]
        self.error = self.job.error
        # Update server factory's jobInfos dictionary with current job info
        self.channel.factory.jobInfos[self.job.jobID] = {'status': self.job.status, 'error': self.job.error,
                                                          'input': self.job.input, 'output': self.job.output}

        if isinstance(result, failure.Failure):
            # Generate job failure message to be written to log
            msg = "Job #%d has failed: %s" % (self.job.jobID, self.error)
            # Write message to event log
            twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, msg)
            if sendResponse:
                # Send job failure response to the client
                self._sendError(http.INTERNAL_SERVER_ERROR, msg)
        else:
            # Generate job success message to be written to log
            msg = "Job #%d has successfully completed" % self.job.jobID
            # Write message to event log
            twisted_logger.writeLog(self.channel.logPrefix, self.channel.logName, msg)
            if sendResponse:
                # Send job results to the client
                self._sendResponse(result, http.OK)
        
        # Delete original job entity
        del self.job
        self.job = None

        if not sendResponse:
            self.finished = True
            
        self.finish()
Exemplo n.º 2
0
 def stopService(self):
     """Stop SCSServer service
     
     @return: deferred which will 'fire' when all of the service termination tasks have completed
     """
     if self.running:
         twisted_logger.writeLog(self.logPrefix, self.logName, "Stopping '%s' service..." % self.name)
         actions = []
         
         if not self.listeningPort or not self.listeningPort.connected:
             err_msg = "Unable to stop '%s' server: server not listening on localhost:%d port" % (self.name, self.serverInfo['port'])
             twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
         else:
             # Stop running SCS server
             msg = "'%s' server service shutdown in progress..." % self.name
             twisted_logger.writeLog(self.logPrefix, self.logName, msg)
             actions.append(self.listeningPort.stopListening().addCallback(self._stopSuccessHandler).addErrback(self._stopFailureHandler))
         
         for srv in self.services[:]:
             try:
                 deferred = self.removeService(srv)
                 if deferred:
                     actions.append(deferred)
             except Exception, err:
                 twisted_logger.writeErr(self.logPrefix, self.logName, "Failure stopping '%s' service: %s" % (srv.name, str(err)))
                 
         service.Service.stopService(self)    
Exemplo n.º 3
0
    def __jobStartHandler(self, result):
        "Report job start success or failure to the client that sent this job request"        

        if isinstance(result, failure.Failure):
            # Job start failure
            self.status = jobStatusMap['FAILURE']
            self.error = "Job start failure"
            err_msg = "%s: %s" % (self.error, result.getErrorMessage())
            twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, err_msg)
            # Trap failure event
            result.trap(Exception)
            # Delete job entity (it is not going to be processed anyway)
            del self.job
            self.job = None

            # Send failure response to external client
            self._sendError(http.INTERNAL_SERVER_ERROR, self.error)
            self.finish()
        else:
            msg = "New job #%d has been created. Status: '%s'" % (self.job.jobID, self.job.status)
            twisted_logger.writeLog(self.channel.logPrefix, self.channel.logName, msg)
            # Add job info to server factory's jobInfos dictionary
            self.channel.factory.jobInfos[self.job.jobID] = {'status': self.job.status, 'error': self.job.error,
                                                             'input': self.job.input, 'output': self.job.output}
            return "ok"
Exemplo n.º 4
0
 def sendRequest(self, request, deferred=None):
     """Send request to external server
     
     @param request: request message (dictionary) to be sent to external server
     @param deferred: deferred to be associated with the given job (for job request only)
     @type request: dictionary containing {'scs_jobid': <jobid>, 
                                           'request': <request - text>}
     @type deferred: defer.Deferred instance 
     """
     # Reset <self.logPrefix> to include 'scs_jobid'
     self.logPrefix += " JOBID #%d" % request["scs_jobid"]
     try:
         # Create JobRequest instance that will be responsible for:
         # 1. processing job request
         # 2. scheduling regular job 'STATUS' requests
         # 3. Handling job response - when it comes from the server
         self.jobRequest = client_request.JobRequest(request, deferred, self)
         # Process job request
         self.jobRequest.processRequest()
     except RuntimeError, err:
         twisted_logger.writeErr(self.logPrefix, self.logName, str(err))
         if isinstance(deferred.__class__, defer.Deferred):
             twisted_logger.writeLog(
                 self.logPrefix,
                 self.logName,
                 "Executing external job step deferred's errback() to report job start failure",
             )
             deferred.errback(failure.Failure(str(err)), RuntimeError)
Exemplo n.º 5
0
    def stopService(self):
        "Stop SCSClient service"
        twisted_logger.writeLog(self.logPrefix, self.logName, "Stopping '%s' service..." % self.name)
        actions = []

        if self.running:
            for clientProtocol in self.clientProtocols[:]:
                if hasattr(clientProtocol, "jobRequest"):
                    clientProtocol._stopJobRequest()
                if clientProtocol.connected == 1:
                    msg = "Client is being disconnected from %s server..." % self.peer
                    twisted_logger.writeLog(self.logPrefix, self.logName, msg)
                    actions.append(self.stopClient(clientProtocol))

            for srv in self.services[:]:
                try:
                    deferred = self.removeService(srv)
                    if deferred:
                        actions.append(deferred)
                except Exception, err:
                    twisted_logger.writeErr(
                        self.logPrefix, self.logName, "Failure stopping '%s' service: %s" % (srv.name, str(err))
                    )

            service.Service.stopService(self)
Exemplo n.º 6
0
    def _stop(self):
        """JobRequest termination: 
            1. fire callback or errback method associated with request's deferred        
            2. perform cleanup
        """
        if not self.stopped:
            # Stop job status requests and remove such requests from factory's cachedRequests list (if any)
            self.statusRequest._stop()
            # Delete JobStatusRequest entity - to free up memory
            del self.statusRequest            
            self.stopped = True

            # Execute deferred's callback() or errback() to notify JobStep (and Job) instance of job step completion
            if hasattr(self, 'deferred') and self.deferred.called == 0:
                if self.status == 'FAILURE' and self.error:
                    msg = "Executing deferred's errback() to notify of job step failure"
                    twisted_logger.writeErr(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)
                    # Execute deferred's errback() to notify of job's failure
                    self.deferred.errback(failure.Failure(self.error, RuntimeError))
                elif self.status == 'SUCCESS':
                    msg = "Executing deferred's callback() to notify of job step success"
                    twisted_logger.writeLog(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)
                    # Execute deferred's callback() to notify of job's success
                    self.deferred.callback(self.response)
                else:
                    self.deferred.called = 1
                    
            else:
                msg = "Deferred's callback/errback has been canceled"
                twisted_logger.writeLog(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)
Exemplo n.º 7
0
 def addOfflineDeferred(self, name, deferred, reset = True):
     "Add new external deferred to workflow's <extOfflineDeferreds> dictionary"
     if not self.namedServices.has_key(name):            
         err_msg = "addOfflineDeferred() failure: '%s' workflow entity has not been started" % name
         twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
     else:
         self.namedServices[name].workflow.addOfflineDeferred(deferred, reset)
Exemplo n.º 8
0
 def start(self, name):
     """Start SCS server service <name>
     
     @param name: SCS server name
     @return: deferred which will 'fire' when given server (service) has started
     """
     failPrefix = "Failure starting '%s' SCS server service" % name
     
     if self.namedServices.has_key(name):
         if self.namedServices[name].running:
             err_msg = "%s: Server is already running on port #%d" % (failPrefix, self.serverInfo[name]['port'])
             twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)                
             raise RuntimeError, err_msg
         else:
             # Server service has stopped: delete that service before restarting it
             self.removeService(self.getServiceNamed(name))
             
     if not self.serverInfo[name]['enabled']:
         err_msg = "%s: Server is disabled" % failPrefix
         twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
         raise RuntimeError, err_msg
         
     if not self.running:
         self.running = 1
         
     if self.namedServices.has_key(name):
         serverService = self.namedServices[name]
         defered = serverService.startService()
     else:
         serverService = _SCSServerService(name, self.serverInfo[name], self.logName, self.logDir)
         defered = addService(self, serverService)
         
     return defered
Exemplo n.º 9
0
 def setWorkflow(self, name):
     "Set server's workflow attribute"
     try:
         listeningPort = self.getResource(name)
         listeningPort.factory._setWorkflow()
     except Exception, err:
         err_msg = "Unable to set server's workflow: %s" % str(err)
         twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
         raise RuntimeError, err_msg
Exemplo n.º 10
0
 def connectionLost(self, reason):
     "Client connection lost"
     msg = "Connection lost with %s client: %s" % (self.peer, reason.getErrorMessage())
     twisted_logger.writeErr(self.logPrefix, self.logName, msg)
     http.HTTPChannel.connectionLost(self, reason)
     
     # Remove itself from factory's protocols list
     if self in self.factory.protocols:
         self.factory.protocols.remove(self)
Exemplo n.º 11
0
    def _connFailureHandler(self, fail):
        "Client connection's failure handler"
        err_msg = "Connection failure: %s" % fail.getErrorMessage()
        twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
        fail.trap(Exception)

        # Cleanup client deferreds
        for d in self.deferreds:
            if d.called == 1:
                self.deferreds.remove(d)
Exemplo n.º 12
0
 def _failure(self, error):
     "Process job failure event. Executes self._stop() that fires self.deferred.errback()"
     if self.status is None:
         # Update job request's status and error attributes
         self.status = 'FAILURE'
         self.error = error
         err_msg = "Job step (job #%d) has failed: %s" % (self.scs_jobid, error)
         twisted_logger.writeErr(self.clientProtocol.logPrefix, self.clientProtocol.logName, err_msg)
         # Stop job request's processing
         self._stop()
Exemplo n.º 13
0
 def _loadInfoComplete(self, result):
     "Server data load success handler. Completes load of self.serverInfo dictionary"
     if result == ():
         err_msg = "scs.scs_server table contains no server information (empty)"
         twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
         raise RuntimeError, err_msg
     
     for (server_name, wf_name, port, protocol, max_connections, enabled) in result:
         self.serverInfo[server_name] = {'workflow': wf_name, 'port': port, 'protocol': protocol, 
                                           'max_connections': max_connections, 'enabled': enabled}
Exemplo n.º 14
0
    def _disconnFailureHandler(self, failure, protocol):
        "Client disconnect failure handler"
        err_msg = "Client disconnect failure: %s" % failure.getErrorMessage()
        twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
        fail.trap(Exception)
        # Delete protocol's deferred nad protocol entity itself
        del protocol.deferred
        del protocol

        for d in self.deferreds:
            if d.called == 1:
                self.deferreds.remove(d)
Exemplo n.º 15
0
 def _sendResponse(self, response, code = None, code_message = None):
     "Send response to the client"
     if code:
         self.setResponseCode(code, code_message)
         
     msg = "Sending response to %s client: %s" % (self.channel.peer, response)
     twisted_logger.writeLog(self.channel.logPrefix, self.channel.logName, msg)        
     try:
         self.write(str(response))
     except Exception, err:
         err_msg = "Failure sending '%s' response to %s: %s" % (response['type'], self.channel.peer, str(err))
         twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, err_msg)            
Exemplo n.º 16
0
 def stop(self, workflowName):
     """Stop workflow service
     
     @param workflowName: workflow name
     """
     if not workflowName in self.namedServices.keys():
         err_msg = "Unable to stop '%s' workflow service: service has not been started" % workflowName
         twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
         raise RuntimeError, err_msg
     
     # Remove workflow from WorkflowManager's services collection
     self.removeService(self.getServiceNamed(workflowName))
Exemplo n.º 17
0
 def getResource(self, name):
     """Obtain given workflow instance
     
     @param name: workflow name
     @return: _Workflow instance
     """
     if not self.namedServices.has_key(name):
         err_msg = "'%s' workflow has not been started" % name
         twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
         raise RuntimeError, err_msg
             
     return self.namedServices[name].workflow
Exemplo n.º 18
0
    def startService(self):
        """Start given SCSServer service
        
        @return: deferred which will 'fire' when all of the service initialization tasks have completed
        """
        actions = []
        failPrefix = "Failure starting '%s' SCS server service" % self.name

        try:
            factory = _SCSServerFactory(self.name, self.serverInfo, self.logDir)
        except RuntimeError, err:
            err_msg = "%s: %s" % (failPrefix, str(err))
            twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
Exemplo n.º 19
0
 def stop(self, name):
     """Stop SCS server service <name>
     
     @param name: SCS server name
     @return: deferred which will 'fire' when given service has stopped
     """
     if not self.namedServices.has_key(name):
         err_msg = "Unable to stop '%s' SCS server service: service has not been started" % name
         twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
         raise RuntimeError, err_msg
     
     serverService = self.getServiceNamed(name)
     return defer.maybeDeferred(self.removeService, serverService)
Exemplo n.º 20
0
 def processRequest(self):
     """
     Process given job status request
     
     @attention: This method is scheduled (by the JobRequest entity that created this JobStatusRequest instance)
                 to run in a loop (with the given time interval). It simply executes self.clientProtocol.sendLine() 
                 of an SCSClientProtocol instance that created this JobStatusRequest entity, passing in itself
     """
     msg = "Sending job status request (job #%d) to '%s' server" % (self.jobRequest.scs_jobid, self.jobRequest.clientProtocol.peer)
     twisted_logger.writeLog(self.jobRequest.clientProtocol.logPrefix, self.jobRequest.clientProtocol.logName, msg)
     try:
         self.jobRequest.clientProtocol.sendLine(self.request)
     except Exception, err:
         err_msg = "Unable to send job status request corresponding to job #%d: %s" % (self.jobRequest.scs_jobid, str(err))
         twisted_logger.writeErr(self.jobRequest.clientProtocol.logPrefix, self.jobRequest.clientProtocol.logName, err_msg)
Exemplo n.º 21
0
    def process(self):
        """Process job request - start job (job.Job instance) to execute a set of job steps associated with the server's workflow

        @note: This method simply creates Job instance and starts its execution
        @return: deferred which will 'fire' when Job instance is initialized and 
                 first JobStep has started execution
        """
        try:
            self._checkRequest()
        except RuntimeError, err:
            error = str(err)
            err_msg = "Invalid request received from %s: %s" % (self.channel.peer, error)
            twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, err_msg)
            # Send failure response to the client
            self._sendError(http.BAD_REQUEST, 'Invalid request: %s' % error)            
            self.finish()
Exemplo n.º 22
0
    def run(self, request, ext_deferred):
        """Start asynchronous job exection by executing first job step
        
        @return: deferred (twisted.internet.defer.Deferred instance) which will 'fire' when first JobStep has been started
        """
        self.input = request
        # Set <self.deferred> to the supplied external (SCSServer entity's) deferred
        self.deferred = ext_deferred        

        if self.steps == []:
            err_msg = "Job #%d does not include any job steps" % self.jobID
            twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
            return Failure(err_msg, RuntimeError)
        else:
            # Start first job step's execution
            return self.runJobStep(None)
Exemplo n.º 23
0
 def connectionMade(self):
     "Client connection made"
     http.HTTPChannel.connectionMade(self)
     peer = self.transport.getPeer()
     self.peer = "%s:%d" % (peer.host, peer.port)
     msg = "Connection established from %s" % (self.peer)
     twisted_logger.writeLog(self.logPrefix, self.logName, msg)
     if self.factory.numPorts >= self.factory.maxConnections:
         err_msg = "Client connection (%s) rejected: maximum number of client connections reached" % self.peer
         twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
         self.transport.loseConnection()
     else:
         msg = "Client connection from %s is accepted" % (self.peer)
         twisted_logger.writeLog(self.logPrefix, self.logName, msg)
 
         if self not in self.factory.protocols:
             # Add protocol (self) to the factory's <factory.protocols> list
             self.factory.protocols.append(self)
Exemplo n.º 24
0
 def lineReceived(self, response):
     """Process response received from the server
     
     @param response: data received from external server
     @type response: dictionary
     
     @attention: <response> message should be in the following form:
                 {'type': <response type>, 'scs_jobid': <jobid>, ...}
     """
     response = json.loads(response)
     twisted_logger.writeLog(self.logPrefix, self.logName, "Received message: %s" % response)
     try:
         # Verify request message
         response = self.__checkResponse(response)
     except RuntimeError, err:
         err_msg = "Invalid response received from %s: %s" % (self.peer, str(err))
         twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
         if hasattr(self, "jobRequest") and not self.jobRequest.stopped:
             self.jobRequest._stop()
Exemplo n.º 25
0
 def _loadComplete(self, result, workflowInfo):
     "Complete loading workflowInfo data (started by self._loadInfo() and self._loadStepInfo()"
     if result == ():
         err_msg = "scs.workflow_step table contains no workflow information (empty)"
         twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
         raise RuntimeError, err_msg
     
     for (wf_name, step_id, step_no, client_name, input, out_flag, enabled) in result:
         if not self.workflowInfo.has_key(wf_name):
             if not workflowInfo.has_key(wf_name):
                 err_msg = "'%s' workflow is not defined within scs.workflow_lookup table" % wf_name
                 twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
                 raise RuntimeError, err_msg
             
             self.workflowInfo.setdefault(wf_name, {'enabled': workflowInfo[wf_name]['enabled'], 
                                                    'timeout': workflowInfo[wf_name]['timeout'], 'steps': []})
             
         stepRec = {'stepID': step_id, 'stepNo': step_no, 'clientName': client_name, 
                    'inputSrc': input, 'outFlag': out_flag, 'enabled': enabled}
         self.workflowInfo[wf_name]['steps'].append(stepRec)
Exemplo n.º 26
0
    def start(self, name):
        """Start SCS client service <name>
        
        @param name: SCS client name
        @return: deferred which will 'fire' when given client (service) has started
        """
        failPrefix = "Failure starting '%s' SCS client service" % name

        if not self.clientInfo.has_key(name):
            err_msg = "%s: Unknown client" % failPrefix
            twisted_logger.writeLog(self.logPrefix, self.logName, err_msg)
            raise RuntimeError, err_msg

        if not self.clientInfo[name]["enabled"]:
            err_msg = "%s: Client is disabled" % failPrefix
            twisted_logger.writeLog(self.logPrefix, self.logName, err_msg)
            raise RuntimeError, err_msg

        if self.namedServices.has_key(name):
            if self.namedServices[name].running:
                err_msg = "%s: Client service is already running" % failPrefix
                twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
                raise RuntimeError, err_msg
            else:
                # Client service has stopped: delete that service before restarting it
                self.removeService(self.getServiceNamed(name))

        if not self.running:
            self.running = 1

        if self.namedServices.has_key(name):
            clientService = self.namedServices[name]
            defered = clientService.startService()
        else:
            clientService = _SCSClientService(name, self.clientInfo[name], self.logName, self.logDir)
            defered = addService(self, clientService)

        return defered
Exemplo n.º 27
0
    def start(self, workflowName):
        """Create workflow instance <workflowName>
        
        @param workflowName: workflow name 
        @type workflowName: str
        """
        failPrefix = "Failure starting '%s' workflow service" % workflowName

        if self.namedServices.has_key(workflowName):
            if self.namedServices[workflowName].running:
                err_msg = "%s: Workflow service is already running" % failPrefix
                twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)                
                raise RuntimeError, err_msg
            else:
                # Client service has stopped: delete that service before restarting it
                self.removeService(self.getServiceNamed(workflowName))
                
        if self.workflowInfo.has_key(workflowName):
            if not self.workflowInfo[workflowName]['enabled']:
                err_msg = "%s: workflow is disabled" % failPrefix
                twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) 
                raise RuntimeError, err_msg
        else:
            err_msg = "%s: unknown workflow" % failPrefix
            twisted_logger.writeErr(self.logPrefix, self.logName, err_msg) 
            raise RuntimeError, err_msg
            
        if not self.running:
            self.running = 1
            
        if self.namedServices.has_key(workflowName):
            workflowService = self.namedServices[workflowName]
            workflowService.startService()
        else:
            workflowService = _WorkflowService(workflowName, self.workflowInfo[workflowName], self.logName)
            addService(self, workflowService)
Exemplo n.º 28
0
 def __failure(self, nothing, err_msg):
     "Last part of Job's failure handler"
     msg = "Job has failed: %s" % err_msg
     twisted_logger.writeErr(self.logPrefix, self.logName, msg)
     if self.deferred and self.deferred.called == 0:            
         self.deferred.errback(Failure(err_msg, RuntimeError))
Exemplo n.º 29
0
 def __handleDBFailure(self, fail, stmt, type):
     "Record <scs.job_step> insert/update failure details"
     operationType = {'insert': 'inserting into', 'update': 'updating'}
     err_msg = "Failure %s <scs.job> table: %s" % (operationType[type], fail.getErrorMessage())
     twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
     twisted_logger.writeErr(self.logPrefix, self.logName, stmt)
Exemplo n.º 30
0
 def _loadInfoFailure(self, fail, query):
     "Server data load failure handler"
     err_msg = "Failure loading workflow info: %s" % fail.getErrorMessage()
     twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
     twisted_logger.writeErr(self.logPrefix, self.logName, "SQL Query: '%s'" % query)