def processResponse(self, result, sendResponse = True):
"Process job completion, timeout or failure event"
self.status = jobStatusMap[self.job.status]
self.error = self.job.error
# Update server factory's jobInfos dictionary with current job info
self.channel.factory.jobInfos[self.job.jobID] = {'status': self.job.status, 'error': self.job.error,
'input': self.job.input, 'output': self.job.output}
if isinstance(result, failure.Failure):
# Generate job failure message to be written to log
msg = "Job #%d has failed: %s" % (self.job.jobID, self.error)
# Write message to event log
twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, msg)
if sendResponse:
# Send job failure response to the client
self._sendError(http.INTERNAL_SERVER_ERROR, msg)
else:
# Generate job success message to be written to log
msg = "Job #%d has successfully completed" % self.job.jobID
# Write message to event log
twisted_logger.writeLog(self.channel.logPrefix, self.channel.logName, msg)
if sendResponse:
# Send job results to the client
self._sendResponse(result, http.OK)
# Delete original job entity
del self.job
self.job = None
if not sendResponse:
self.finished = True
self.finish()
def stopService(self):
"""Stop SCSServer service
@return: deferred which will 'fire' when all of the service termination tasks have completed
"""
if self.running:
twisted_logger.writeLog(self.logPrefix, self.logName, "Stopping '%s' service..." % self.name)
actions = []
if not self.listeningPort or not self.listeningPort.connected:
err_msg = "Unable to stop '%s' server: server not listening on localhost:%d port" % (self.name, self.serverInfo['port'])
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
else:
# Stop running SCS server
msg = "'%s' server service shutdown in progress..." % self.name
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
actions.append(self.listeningPort.stopListening().addCallback(self._stopSuccessHandler).addErrback(self._stopFailureHandler))
for srv in self.services[:]:
try:
deferred = self.removeService(srv)
if deferred:
actions.append(deferred)
except Exception, err:
twisted_logger.writeErr(self.logPrefix, self.logName, "Failure stopping '%s' service: %s" % (srv.name, str(err)))
service.Service.stopService(self)
def __jobStartHandler(self, result):
"Report job start success or failure to the client that sent this job request"
if isinstance(result, failure.Failure):
# Job start failure
self.status = jobStatusMap['FAILURE']
self.error = "Job start failure"
err_msg = "%s: %s" % (self.error, result.getErrorMessage())
twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, err_msg)
# Trap failure event
result.trap(Exception)
# Delete job entity (it is not going to be processed anyway)
del self.job
self.job = None
# Send failure response to external client
self._sendError(http.INTERNAL_SERVER_ERROR, self.error)
self.finish()
else:
msg = "New job #%d has been created. Status: '%s'" % (self.job.jobID, self.job.status)
twisted_logger.writeLog(self.channel.logPrefix, self.channel.logName, msg)
# Add job info to server factory's jobInfos dictionary
self.channel.factory.jobInfos[self.job.jobID] = {'status': self.job.status, 'error': self.job.error,
'input': self.job.input, 'output': self.job.output}
return "ok"
def sendRequest(self, request, deferred=None):
"""Send request to external server
@param request: request message (dictionary) to be sent to external server
@param deferred: deferred to be associated with the given job (for job request only)
@type request: dictionary containing {'scs_jobid': <jobid>,
'request': <request - text>}
@type deferred: defer.Deferred instance
"""
# Reset <self.logPrefix> to include 'scs_jobid'
self.logPrefix += " JOBID #%d" % request["scs_jobid"]
try:
# Create JobRequest instance that will be responsible for:
# 1. processing job request
# 2. scheduling regular job 'STATUS' requests
# 3. Handling job response - when it comes from the server
self.jobRequest = client_request.JobRequest(request, deferred, self)
# Process job request
self.jobRequest.processRequest()
except RuntimeError, err:
twisted_logger.writeErr(self.logPrefix, self.logName, str(err))
if isinstance(deferred.__class__, defer.Deferred):
twisted_logger.writeLog(
self.logPrefix,
self.logName,
"Executing external job step deferred's errback() to report job start failure",
)
deferred.errback(failure.Failure(str(err)), RuntimeError)
def stopService(self):
"Stop SCSClient service"
twisted_logger.writeLog(self.logPrefix, self.logName, "Stopping '%s' service..." % self.name)
actions = []
if self.running:
for clientProtocol in self.clientProtocols[:]:
if hasattr(clientProtocol, "jobRequest"):
clientProtocol._stopJobRequest()
if clientProtocol.connected == 1:
msg = "Client is being disconnected from %s server..." % self.peer
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
actions.append(self.stopClient(clientProtocol))
for srv in self.services[:]:
try:
deferred = self.removeService(srv)
if deferred:
actions.append(deferred)
except Exception, err:
twisted_logger.writeErr(
self.logPrefix, self.logName, "Failure stopping '%s' service: %s" % (srv.name, str(err))
)
service.Service.stopService(self)
def _stop(self):
"""JobRequest termination:
1. fire callback or errback method associated with request's deferred
2. perform cleanup
"""
if not self.stopped:
# Stop job status requests and remove such requests from factory's cachedRequests list (if any)
self.statusRequest._stop()
# Delete JobStatusRequest entity - to free up memory
del self.statusRequest
self.stopped = True
# Execute deferred's callback() or errback() to notify JobStep (and Job) instance of job step completion
if hasattr(self, 'deferred') and self.deferred.called == 0:
if self.status == 'FAILURE' and self.error:
msg = "Executing deferred's errback() to notify of job step failure"
twisted_logger.writeErr(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)
# Execute deferred's errback() to notify of job's failure
self.deferred.errback(failure.Failure(self.error, RuntimeError))
elif self.status == 'SUCCESS':
msg = "Executing deferred's callback() to notify of job step success"
twisted_logger.writeLog(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)
# Execute deferred's callback() to notify of job's success
self.deferred.callback(self.response)
else:
self.deferred.called = 1
else:
msg = "Deferred's callback/errback has been canceled"
twisted_logger.writeLog(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)
def addOfflineDeferred(self, name, deferred, reset = True):
"Add new external deferred to workflow's <extOfflineDeferreds> dictionary"
if not self.namedServices.has_key(name):
err_msg = "addOfflineDeferred() failure: '%s' workflow entity has not been started" % name
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
else:
self.namedServices[name].workflow.addOfflineDeferred(deferred, reset)
def start(self, name):
"""Start SCS server service <name>
@param name: SCS server name
@return: deferred which will 'fire' when given server (service) has started
"""
failPrefix = "Failure starting '%s' SCS server service" % name
if self.namedServices.has_key(name):
if self.namedServices[name].running:
err_msg = "%s: Server is already running on port #%d" % (failPrefix, self.serverInfo[name]['port'])
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
else:
# Server service has stopped: delete that service before restarting it
self.removeService(self.getServiceNamed(name))
if not self.serverInfo[name]['enabled']:
err_msg = "%s: Server is disabled" % failPrefix
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
if not self.running:
self.running = 1
if self.namedServices.has_key(name):
serverService = self.namedServices[name]
defered = serverService.startService()
else:
serverService = _SCSServerService(name, self.serverInfo[name], self.logName, self.logDir)
defered = addService(self, serverService)
return defered
def setWorkflow(self, name):
"Set server's workflow attribute"
try:
listeningPort = self.getResource(name)
listeningPort.factory._setWorkflow()
except Exception, err:
err_msg = "Unable to set server's workflow: %s" % str(err)
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
def connectionLost(self, reason):
"Client connection lost"
msg = "Connection lost with %s client: %s" % (self.peer, reason.getErrorMessage())
twisted_logger.writeErr(self.logPrefix, self.logName, msg)
http.HTTPChannel.connectionLost(self, reason)
# Remove itself from factory's protocols list
if self in self.factory.protocols:
self.factory.protocols.remove(self)
def _connFailureHandler(self, fail):
"Client connection's failure handler"
err_msg = "Connection failure: %s" % fail.getErrorMessage()
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
fail.trap(Exception)
# Cleanup client deferreds
for d in self.deferreds:
if d.called == 1:
self.deferreds.remove(d)
def _failure(self, error):
"Process job failure event. Executes self._stop() that fires self.deferred.errback()"
if self.status is None:
# Update job request's status and error attributes
self.status = 'FAILURE'
self.error = error
err_msg = "Job step (job #%d) has failed: %s" % (self.scs_jobid, error)
twisted_logger.writeErr(self.clientProtocol.logPrefix, self.clientProtocol.logName, err_msg)
# Stop job request's processing
self._stop()
def _loadInfoComplete(self, result):
"Server data load success handler. Completes load of self.serverInfo dictionary"
if result == ():
err_msg = "scs.scs_server table contains no server information (empty)"
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
for (server_name, wf_name, port, protocol, max_connections, enabled) in result:
self.serverInfo[server_name] = {'workflow': wf_name, 'port': port, 'protocol': protocol,
'max_connections': max_connections, 'enabled': enabled}
def _disconnFailureHandler(self, failure, protocol):
"Client disconnect failure handler"
err_msg = "Client disconnect failure: %s" % failure.getErrorMessage()
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
fail.trap(Exception)
# Delete protocol's deferred nad protocol entity itself
del protocol.deferred
del protocol
for d in self.deferreds:
if d.called == 1:
self.deferreds.remove(d)
def _sendResponse(self, response, code = None, code_message = None):
"Send response to the client"
if code:
self.setResponseCode(code, code_message)
msg = "Sending response to %s client: %s" % (self.channel.peer, response)
twisted_logger.writeLog(self.channel.logPrefix, self.channel.logName, msg)
try:
self.write(str(response))
except Exception, err:
err_msg = "Failure sending '%s' response to %s: %s" % (response['type'], self.channel.peer, str(err))
twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, err_msg)
def stop(self, workflowName):
"""Stop workflow service
@param workflowName: workflow name
"""
if not workflowName in self.namedServices.keys():
err_msg = "Unable to stop '%s' workflow service: service has not been started" % workflowName
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
# Remove workflow from WorkflowManager's services collection
self.removeService(self.getServiceNamed(workflowName))
def getResource(self, name):
"""Obtain given workflow instance
@param name: workflow name
@return: _Workflow instance
"""
if not self.namedServices.has_key(name):
err_msg = "'%s' workflow has not been started" % name
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
return self.namedServices[name].workflow
def startService(self):
"""Start given SCSServer service
@return: deferred which will 'fire' when all of the service initialization tasks have completed
"""
actions = []
failPrefix = "Failure starting '%s' SCS server service" % self.name
try:
factory = _SCSServerFactory(self.name, self.serverInfo, self.logDir)
except RuntimeError, err:
err_msg = "%s: %s" % (failPrefix, str(err))
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
def stop(self, name):
"""Stop SCS server service <name>
@param name: SCS server name
@return: deferred which will 'fire' when given service has stopped
"""
if not self.namedServices.has_key(name):
err_msg = "Unable to stop '%s' SCS server service: service has not been started" % name
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
serverService = self.getServiceNamed(name)
return defer.maybeDeferred(self.removeService, serverService)
def processRequest(self):
"""
Process given job status request
@attention: This method is scheduled (by the JobRequest entity that created this JobStatusRequest instance)
to run in a loop (with the given time interval). It simply executes self.clientProtocol.sendLine()
of an SCSClientProtocol instance that created this JobStatusRequest entity, passing in itself
"""
msg = "Sending job status request (job #%d) to '%s' server" % (self.jobRequest.scs_jobid, self.jobRequest.clientProtocol.peer)
twisted_logger.writeLog(self.jobRequest.clientProtocol.logPrefix, self.jobRequest.clientProtocol.logName, msg)
try:
self.jobRequest.clientProtocol.sendLine(self.request)
except Exception, err:
err_msg = "Unable to send job status request corresponding to job #%d: %s" % (self.jobRequest.scs_jobid, str(err))
twisted_logger.writeErr(self.jobRequest.clientProtocol.logPrefix, self.jobRequest.clientProtocol.logName, err_msg)
def process(self):
"""Process job request - start job (job.Job instance) to execute a set of job steps associated with the server's workflow
@note: This method simply creates Job instance and starts its execution
@return: deferred which will 'fire' when Job instance is initialized and
first JobStep has started execution
"""
try:
self._checkRequest()
except RuntimeError, err:
error = str(err)
err_msg = "Invalid request received from %s: %s" % (self.channel.peer, error)
twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, err_msg)
# Send failure response to the client
self._sendError(http.BAD_REQUEST, 'Invalid request: %s' % error)
self.finish()
def run(self, request, ext_deferred):
"""Start asynchronous job exection by executing first job step
@return: deferred (twisted.internet.defer.Deferred instance) which will 'fire' when first JobStep has been started
"""
self.input = request
# Set <self.deferred> to the supplied external (SCSServer entity's) deferred
self.deferred = ext_deferred
if self.steps == []:
err_msg = "Job #%d does not include any job steps" % self.jobID
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
return Failure(err_msg, RuntimeError)
else:
# Start first job step's execution
return self.runJobStep(None)
def connectionMade(self):
"Client connection made"
http.HTTPChannel.connectionMade(self)
peer = self.transport.getPeer()
self.peer = "%s:%d" % (peer.host, peer.port)
msg = "Connection established from %s" % (self.peer)
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
if self.factory.numPorts >= self.factory.maxConnections:
err_msg = "Client connection (%s) rejected: maximum number of client connections reached" % self.peer
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
self.transport.loseConnection()
else:
msg = "Client connection from %s is accepted" % (self.peer)
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
if self not in self.factory.protocols:
# Add protocol (self) to the factory's <factory.protocols> list
self.factory.protocols.append(self)
def lineReceived(self, response):
"""Process response received from the server
@param response: data received from external server
@type response: dictionary
@attention: <response> message should be in the following form:
{'type': <response type>, 'scs_jobid': <jobid>, ...}
"""
response = json.loads(response)
twisted_logger.writeLog(self.logPrefix, self.logName, "Received message: %s" % response)
try:
# Verify request message
response = self.__checkResponse(response)
except RuntimeError, err:
err_msg = "Invalid response received from %s: %s" % (self.peer, str(err))
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
if hasattr(self, "jobRequest") and not self.jobRequest.stopped:
self.jobRequest._stop()
def _loadComplete(self, result, workflowInfo):
"Complete loading workflowInfo data (started by self._loadInfo() and self._loadStepInfo()"
if result == ():
err_msg = "scs.workflow_step table contains no workflow information (empty)"
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
for (wf_name, step_id, step_no, client_name, input, out_flag, enabled) in result:
if not self.workflowInfo.has_key(wf_name):
if not workflowInfo.has_key(wf_name):
err_msg = "'%s' workflow is not defined within scs.workflow_lookup table" % wf_name
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
self.workflowInfo.setdefault(wf_name, {'enabled': workflowInfo[wf_name]['enabled'],
'timeout': workflowInfo[wf_name]['timeout'], 'steps': []})
stepRec = {'stepID': step_id, 'stepNo': step_no, 'clientName': client_name,
'inputSrc': input, 'outFlag': out_flag, 'enabled': enabled}
self.workflowInfo[wf_name]['steps'].append(stepRec)
def start(self, name):
"""Start SCS client service <name>
@param name: SCS client name
@return: deferred which will 'fire' when given client (service) has started
"""
failPrefix = "Failure starting '%s' SCS client service" % name
if not self.clientInfo.has_key(name):
err_msg = "%s: Unknown client" % failPrefix
twisted_logger.writeLog(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
if not self.clientInfo[name]["enabled"]:
err_msg = "%s: Client is disabled" % failPrefix
twisted_logger.writeLog(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
if self.namedServices.has_key(name):
if self.namedServices[name].running:
err_msg = "%s: Client service is already running" % failPrefix
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
else:
# Client service has stopped: delete that service before restarting it
self.removeService(self.getServiceNamed(name))
if not self.running:
self.running = 1
if self.namedServices.has_key(name):
clientService = self.namedServices[name]
defered = clientService.startService()
else:
clientService = _SCSClientService(name, self.clientInfo[name], self.logName, self.logDir)
defered = addService(self, clientService)
return defered
def start(self, workflowName):
"""Create workflow instance <workflowName>
@param workflowName: workflow name
@type workflowName: str
"""
failPrefix = "Failure starting '%s' workflow service" % workflowName
if self.namedServices.has_key(workflowName):
if self.namedServices[workflowName].running:
err_msg = "%s: Workflow service is already running" % failPrefix
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
else:
# Client service has stopped: delete that service before restarting it
self.removeService(self.getServiceNamed(workflowName))
if self.workflowInfo.has_key(workflowName):
if not self.workflowInfo[workflowName]['enabled']:
err_msg = "%s: workflow is disabled" % failPrefix
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
else:
err_msg = "%s: unknown workflow" % failPrefix
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
if not self.running:
self.running = 1
if self.namedServices.has_key(workflowName):
workflowService = self.namedServices[workflowName]
workflowService.startService()
else:
workflowService = _WorkflowService(workflowName, self.workflowInfo[workflowName], self.logName)
addService(self, workflowService)
def __failure(self, nothing, err_msg):
"Last part of Job's failure handler"
msg = "Job has failed: %s" % err_msg
twisted_logger.writeErr(self.logPrefix, self.logName, msg)
if self.deferred and self.deferred.called == 0:
self.deferred.errback(Failure(err_msg, RuntimeError))
def __handleDBFailure(self, fail, stmt, type):
"Record <scs.job_step> insert/update failure details"
operationType = {'insert': 'inserting into', 'update': 'updating'}
err_msg = "Failure %s <scs.job> table: %s" % (operationType[type], fail.getErrorMessage())
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
twisted_logger.writeErr(self.logPrefix, self.logName, stmt)
def _loadInfoFailure(self, fail, query):
"Server data load failure handler"
err_msg = "Failure loading workflow info: %s" % fail.getErrorMessage()
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
twisted_logger.writeErr(self.logPrefix, self.logName, "SQL Query: '%s'" % query)
