def startService(self):
"Start monitoring service"
twisted_logger.writeLog(self.logPrefix, self.logName, "Starting '%s' service..." % self.name)
if scheduleChecks:
# Start server connection checks (monitoring)
self._startServerChecks()
return service.Service.startService(self)
def test_sendRequest(self):
"Test that _SCSClientService.sendRequest() successfully sends request to external server"
self.logPrefix = "SCSClientServiceTest.test_sendRequest"
testRequest = {'scs_jobid': 5, 'request': 'I want to run marathon'}
def requestReceivedHandler(data):
twisted_logger.writeLog(self.logPrefix, None, "Server received request: '%s'" % data)
def checkResults(protocol):
twisted_logger.writeLog(self.logPrefix, None, "Checking test results...")
self.assertEquals(isinstance(protocol.jobRequest, client.client_request.JobRequest), True)
self.assertEquals(protocol.jobRequest.request['type'], 'REQUEST')
self.assertEquals(protocol.jobRequest.scs_jobid, testRequest['scs_jobid'])
self.assertEquals(protocol.jobRequest.request['request'], testRequest['request'])
twisted_logger.writeLog(self.logPrefix, None, "Test succeeded!")
# Start test server
factory = protocol.ServerFactory()
factory.protocol = ServerProtocol
factory.protocol.dataRcvdDeferred = defer.Deferred().addCallback(requestReceivedHandler)
self.listeningPort = reactor.listenTCP(testPort, factory)
self.assertEquals(self.clientService.clientProtocols, [])
deferred = defer.Deferred()
twisted_logger.writeLog(self.logPrefix, None, "Sending test request to the server...")
return self.clientService.sendRequest(testRequest, deferred).addCallback(checkResults)
def checkResults(nothing):
twisted_logger.writeLog(self.logPrefix, None, "Checking test results...")
self.assertEquals(self.controller.namedServices.has_key('ServerManager'), True)
self.assertEquals(self.controller.namedServices.has_key('ClientManager'), True)
self.assertEquals(self.controller.namedServices.has_key('WorkflowManager'), True)
self.assertEquals(self.controller.namedServices['ServerManager'].running, True)
self.assertEquals(self.controller.namedServices['ClientManager'].running, True)
self.assertEquals(self.controller.namedServices['WorkflowManager'].running, True)
# Verify that when Workflow entities have any steps (clients) - corresponding ClientService
# entities have non-empty onlineDeferreds and offlineDeferreds dictionaries
for srv in self.controller.namedServices['WorkflowManager']:
if isinstance(srv, SCS_controller.workflow._WorkflowService):
for step in srv.workflow.steps:
clientService = step['client']
self.assertNotEquals(clientService.onlineDeferreds, {})
self.assertNotEquals(clientService.offlineDeferreds, {})
for srv1 in clientService.services:
if isinstance(srv1, SCS_controller.client._SCSServerMonitor):
monitorService = srv1
self.assertEquals(monitorService.onlineDeferreds, clientService.onlineDeferreds)
self.assertEquals(monitorService.offlineDeferreds, clientService.offlineDeferreds)
twisted_logger.writeLog(self.logPrefix, None, "Test succeeded!")
def runJobStep(self, result, step_no = 0):
"""Start asynchronous job step (<step_no>) execution
@return: deferred which will 'fire' when given JobStep instance has started execution
"""
def updateJobStatus(res):
"Update job status to 'RUNNING'"
self.status = 'RUNNING'
stmt = "update scs.job set status = '%s', text_input = '%s', start_time = now() " \
"where job_id = %d" % (self.status, self.input.replace("'", "''"), self.jobID)
self.db.execute(stmt).addErrback(self.__handleDBFailure, stmt, 'update')
# Schedule job timeout check using workflow's <timeout> attribute
if timeoutFlag and self.workflow.timeout:
self.callID = reactor.callLater(self.workflow.timeout, self._timeout)
step = self.steps[step_no]
if step_no == 0:
request = self.input
else:
# if isinstance(self.steps[step.inputSource - 1].output, dict):
# request = eval(self.steps[step.inputSource - 1].output)
# else:
request = self.steps[step.inputSource - 1].output
# Add <self.failureHandler()> errback method to given job step's deferred
step.deferred.addErrback(self.failureHandler)
# Execute job step's run() and add <self.failureHandler()> errback to it - to handle job failure event
twisted_logger.writeLog(self.logPrefix, self.logName, "Executing job step #%d... " % (step_no + 1))
if step_no == 0:
return step.run(request).addCallback(updateJobStatus).addErrback(self.failureHandler)
else:
return step.run(request).addErrback(self.failureHandler)
def stopService(self):
"""Stop SCSServer service
@return: deferred which will 'fire' when all of the service termination tasks have completed
"""
if self.running:
twisted_logger.writeLog(self.logPrefix, self.logName, "Stopping '%s' service..." % self.name)
actions = []
if not self.listeningPort or not self.listeningPort.connected:
err_msg = "Unable to stop '%s' server: server not listening on localhost:%d port" % (self.name, self.serverInfo['port'])
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
else:
# Stop running SCS server
msg = "'%s' server service shutdown in progress..." % self.name
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
actions.append(self.listeningPort.stopListening().addCallback(self._stopSuccessHandler).addErrback(self._stopFailureHandler))
for srv in self.services[:]:
try:
deferred = self.removeService(srv)
if deferred:
actions.append(deferred)
except Exception, err:
twisted_logger.writeErr(self.logPrefix, self.logName, "Failure stopping '%s' service: %s" % (srv.name, str(err)))
service.Service.stopService(self)
def processResponse(self, result, sendResponse = True):
"Process job completion, timeout or failure event"
self.status = jobStatusMap[self.job.status]
self.error = self.job.error
# Update server factory's jobInfos dictionary with current job info
self.channel.factory.jobInfos[self.job.jobID] = {'status': self.job.status, 'error': self.job.error,
'input': self.job.input, 'output': self.job.output}
if isinstance(result, failure.Failure):
# Generate job failure message to be written to log
msg = "Job #%d has failed: %s" % (self.job.jobID, self.error)
# Write message to event log
twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, msg)
if sendResponse:
# Send job failure response to the client
self._sendError(http.INTERNAL_SERVER_ERROR, msg)
else:
# Generate job success message to be written to log
msg = "Job #%d has successfully completed" % self.job.jobID
# Write message to event log
twisted_logger.writeLog(self.channel.logPrefix, self.channel.logName, msg)
if sendResponse:
# Send job results to the client
self._sendResponse(result, http.OK)
# Delete original job entity
del self.job
self.job = None
if not sendResponse:
self.finished = True
self.finish()
def checkResults(nothing):
twisted_logger.writeLog(self.logPrefix, None, "Checking test results...")
self.assertEquals(clientManager.running, 1)
self.assertEquals(clientManager.namedServices.has_key(testClientName), True)
clientService = clientManager.namedServices[testClientName]
self.assertEquals(clientService.running, 1)
twisted_logger.writeLog(self.logPrefix, None, "Test succeeded!")
def checkResults(protocol):
twisted_logger.writeLog(self.logPrefix, None, "Checking test results...")
self.assertEquals(isinstance(protocol.jobRequest, client.client_request.JobRequest), True)
self.assertEquals(protocol.jobRequest.request['type'], 'REQUEST')
self.assertEquals(protocol.jobRequest.scs_jobid, testRequest['scs_jobid'])
self.assertEquals(protocol.jobRequest.request['request'], testRequest['request'])
twisted_logger.writeLog(self.logPrefix, None, "Test succeeded!")
def checkResults(nothing):
twisted_logger.writeLog(self.logPrefix, None, "Checking test results...")
self.assertEquals(serverManager.running == 1, True)
self.assertEquals(serverManager.namedServices.has_key(testServerName), True)
self.assertEquals(serverManager.namedServices[testServerName].running, True)
self.assertEquals(serverManager.namedServices[testServerName].listeningPort.connected, True)
twisted_logger.writeLog(self.logPrefix, None, "Test succeeded!")
def _stop(self):
"""JobRequest termination:
1. fire callback or errback method associated with request's deferred
2. perform cleanup
"""
if not self.stopped:
# Stop job status requests and remove such requests from factory's cachedRequests list (if any)
self.statusRequest._stop()
# Delete JobStatusRequest entity - to free up memory
del self.statusRequest
self.stopped = True
# Execute deferred's callback() or errback() to notify JobStep (and Job) instance of job step completion
if hasattr(self, 'deferred') and self.deferred.called == 0:
if self.status == 'FAILURE' and self.error:
msg = "Executing deferred's errback() to notify of job step failure"
twisted_logger.writeErr(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)
# Execute deferred's errback() to notify of job's failure
self.deferred.errback(failure.Failure(self.error, RuntimeError))
elif self.status == 'SUCCESS':
msg = "Executing deferred's callback() to notify of job step success"
twisted_logger.writeLog(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)
# Execute deferred's callback() to notify of job's success
self.deferred.callback(self.response)
else:
self.deferred.called = 1
else:
msg = "Deferred's callback/errback has been canceled"
twisted_logger.writeLog(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)
def stopService(self):
"Stop SCSClient service"
twisted_logger.writeLog(self.logPrefix, self.logName, "Stopping '%s' service..." % self.name)
actions = []
if self.running:
for clientProtocol in self.clientProtocols[:]:
if hasattr(clientProtocol, "jobRequest"):
clientProtocol._stopJobRequest()
if clientProtocol.connected == 1:
msg = "Client is being disconnected from %s server..." % self.peer
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
actions.append(self.stopClient(clientProtocol))
for srv in self.services[:]:
try:
deferred = self.removeService(srv)
if deferred:
actions.append(deferred)
except Exception, err:
twisted_logger.writeErr(
self.logPrefix, self.logName, "Failure stopping '%s' service: %s" % (srv.name, str(err))
)
service.Service.stopService(self)
def __jobStartHandler(self, result):
"Report job start success or failure to the client that sent this job request"
if isinstance(result, failure.Failure):
# Job start failure
self.status = jobStatusMap['FAILURE']
self.error = "Job start failure"
err_msg = "%s: %s" % (self.error, result.getErrorMessage())
twisted_logger.writeErr(self.channel.logPrefix, self.channel.logName, err_msg)
# Trap failure event
result.trap(Exception)
# Delete job entity (it is not going to be processed anyway)
del self.job
self.job = None
# Send failure response to external client
self._sendError(http.INTERNAL_SERVER_ERROR, self.error)
self.finish()
else:
msg = "New job #%d has been created. Status: '%s'" % (self.job.jobID, self.job.status)
twisted_logger.writeLog(self.channel.logPrefix, self.channel.logName, msg)
# Add job info to server factory's jobInfos dictionary
self.channel.factory.jobInfos[self.job.jobID] = {'status': self.job.status, 'error': self.job.error,
'input': self.job.input, 'output': self.job.output}
return "ok"
def startService(self):
"Start SCSClient service"
twisted_logger.writeLog(self.logPrefix, self.logName, "Starting '%s' service..." % self.name)
actions = []
if self.logDir:
# Initialize and start log and error services
msg = "Opening '%s.log' and '%s.err' files in '%s' folder..." % (
self.logFileName,
self.logFileName,
self.logDir,
)
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
twisted_logger.initLogging(self.name, self.logFileName, self.logDir, self)
for loggerService in self.services:
if not loggerService.running:
actions.append(defer.maybeDeferred(loggerService.startService))
if scheduleChecks:
monitoringService = _SCSServerMonitor(self.name, self.clientInfo, self.logName, self.logDir)
self.addService(monitoringService)
monitoringService.parent = self
monitoringService.onlineDeferreds = self.onlineDeferreds
monitoringService.offlineDeferreds = self.offlineDeferreds
actions.append(defer.maybeDeferred(monitoringService.startService))
service.Service.startService(self)
return defer.DeferredList(actions)
def sendRequest(self, request, deferred=None):
"""Send request to external server
@param request: request message (dictionary) to be sent to external server
@param deferred: deferred to be associated with the given job (for job request only)
@type request: dictionary containing {'scs_jobid': <jobid>,
'request': <request - text>}
@type deferred: defer.Deferred instance
"""
# Reset <self.logPrefix> to include 'scs_jobid'
self.logPrefix += " JOBID #%d" % request["scs_jobid"]
try:
# Create JobRequest instance that will be responsible for:
# 1. processing job request
# 2. scheduling regular job 'STATUS' requests
# 3. Handling job response - when it comes from the server
self.jobRequest = client_request.JobRequest(request, deferred, self)
# Process job request
self.jobRequest.processRequest()
except RuntimeError, err:
twisted_logger.writeErr(self.logPrefix, self.logName, str(err))
if isinstance(deferred.__class__, defer.Deferred):
twisted_logger.writeLog(
self.logPrefix,
self.logName,
"Executing external job step deferred's errback() to report job start failure",
)
deferred.errback(failure.Failure(str(err)), RuntimeError)
def handleWorkflowReconnect(result, wf):
global reconnectHandled, reconnectCount
twisted_logger.writeLog(self.logPrefix, None, "Workflow.handleClientReconnect() has been called back ...")
reconnectCount += 1
reconnectHandled = True
err_msg = "Workflow.handleClientReconnect() resets workflow status to online when other client(s) are still offline"
raise self.failureException, err_msg
def enable(self, workflowName):
"""Enable workflow. SCSServer entity that 'owns' this workflow will start accepting external client's
job requests - provided that workflow is also 'online' (all SCSClient entities used by the workflow
are connected to their external servers)"""
if workflowName in self.namedServices.keys():
msg = "Enabling '%s' workflow..." % workflowName
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
self.namedServices[workflowName].workflow.enable()
def checkResults(data, response):
twisted_logger.writeLog(self.logPrefix, None, "Checking test results...")
self.assertEquals(data, self.request['request'])
self.assertEquals(self.jobRequest.status, 'SUCCESS')
self.assertEquals(self.jobRequest.request['request'], self.request['request'])
self.assertEquals(self.jobRequest.response, response['result_set'])
self.assertEquals(self.jobRequest.error, None)
twisted_logger.writeLog(self.logPrefix, None, "Test succeeded!")
def checkResults(data):
twisted_logger.writeLog(self.logPrefix, None, "Checking test results...")
data = eval(data)
self.assertEquals(data['type'], 'REQUEST')
# self.assertEquals(data['scs_jobid'], self.request['scs_jobid'])
self.assertEquals(data['request'], self.request['request'])
self.assertEquals(self.jobRequest.status, None)
twisted_logger.writeLog(self.logPrefix, None, "Test succeeded!")
def checkResults(result, wf):
"Verify that workflow is offline and that workflow reconnect event has not occured"
global reconnectHandled, reconnectCount
twisted_logger.writeLog(self.logPrefix, None, "Checking results...")
self.assertEquals(wf.online, False)
self.assertEquals(reconnectHandled, False)
self.assertEquals(reconnectCount, 0)
twisted_logger.writeLog(self.logPrefix, None, "Success...")
def checkResults(result, wf):
"Verify that workflow is ONLINE and that workflow reconnect is handled (only once)"
global reconnectHandled, reconnectCount
twisted_logger.writeLog(self.logPrefix, None, "Checking results...")
self.assertEquals(wf.online, True)
self.assertEquals(reconnectHandled, True)
self.assertEquals(reconnectCount, 1)
twisted_logger.writeLog(self.logPrefix, None, "Success...")
def checkAfterStartUp(nothing, origEnabled):
twisted_logger.writeLog(self.logPrefix, None, "Checking test results after startup...")
for clientname in clientManager.clientInfo.keys():
self.assertEquals(clientManager.namedServices.has_key(clientname), True)
self.assertEquals(clientManager.namedServices[clientname].running == 1, True)
clientManager.clientInfo[clientname]['enabled'] = origEnabled[clientname]
twisted_logger.writeLog(self.logPrefix, None, "All client services have been started")
return clientManager.shutDown().addCallback(checkAfterShutDown)
def test__init__(self):
"Test that JobRequest.__init__() correctly initializes JobRequest instance"
self.logPrefix = "test__init__"
twisted_logger.writeLog(self.logPrefix, None, "Checking test results...")
self.assertEquals(self.httpRequest.job, None)
self.assertEquals(self.httpRequest.status, None)
self.assertEquals(self.httpRequest.response, None)
twisted_logger.writeLog(self.logPrefix, None, "Test succeeded!")
def _success(self):
"Process job success event. Executes self._stop() that fires self.deferred.callback()"
if self.status is None:
self.status = 'SUCCESS'
msg = "Job step (job #%d) has successfully completed" % self.scs_jobid
twisted_logger.writeLog(self.clientProtocol.logPrefix, self.clientProtocol.logName, msg)
# Stop job request's processing
self._stop()
def restartService(nothing):
twisted_logger.writeLog(self.logPrefix, None, "Verifying that '%s' service has stopped..." % testClientName)
self.assertEquals(clientManager.running, True)
# Verify that server is deleted from ServerManager.services list and ServerManager.namedServices dictionary
self.assertEquals(clientManager.namedServices.has_key(testClientName), False)
twisted_logger.writeLog(self.logPrefix, None, "Service has stopped. Restarting the service...")
# Restart the server
return clientManager.start(testClientName).addCallback(checkResults)
def enable(self):
"""Enable workflow. SCSServer entity that 'owns' this workflow will start accepting external client's
job requests - provided that workflow is also 'online' (all SCSClient entities used by the workflow
are connected to their external servers)"""
# Schedule self._checkEnabled()
self._scheduleCheckEnabled()
self.enabled = True
msg = "'%s' workflow is enabled" % self.name
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
def _loadInfoComplete(self, result):
"Client info SQL load success handler. Completes load of self.clientInfo dictionary"
if result == ():
err_msg = "SCS client data is not setup properly"
twisted_logger.writeLog(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
for (client_name, host, port, enabled) in result:
self.clientInfo[client_name] = {"host": host, "port": port, "enabled": enabled}
def disable(self):
"Disable workflow. SCSServer entity that 'owns' this workflow will start rejecting external client's job requests"
# Cancel scheduled task(s)
if self.loop and self.loop.running:
self.loop.stop()
self.enabled = False
msg = "'%s' workflow is disabled" % self.name
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
def checkJobStepRecords(expect_status):
twisted_logger.writeLog(self.logPrefix, None, "Checking scs.job_step results...")
query = "select step_id, status, text_input, output, start_time, end_time from " \
"scs.job_step where job_id = %d" % self.job.jobID
res = synchdb.query(query)
for (step_id, status, text_input, output, start_time, end_time) in res:
self.assertEquals(status, expect_status)
self.assertNotEquals(start_time, None)
self.assertNotEquals(end_time, None)
def test_run_FAILURE(self):
"Test that JobStep.run() raises exception when SCSClientManager.sendRequest() fails sending request to the server"
self.logPrefix = "test_run_FAILURE"
self.assertEquals(self.js.status, 'NEW')
testRequest = {'scs_jobid': testJobID, 'request': "I want to run a marathon"}
self.js.deferred = None
twisted_logger.writeLog(self.logPrefix, None, "Test client has not started... Trying to execute JobStep.run()...")
# Test client (self.client) is not running... SCSJobStep.run() should fail
return self.js.run(str(testRequest)).addErrback(self.__checkJobStepResults, 'FAILURE')
def runJobStep(nothing):
global failFlag
twisted_logger.writeLog(self.logPrefix, None, "Test client is connected to server...")
testRequest = {'scs_jobid': testJobID, 'request': "I want to run a marathon"}
self.js.deferred.addErrback(self.__checkJobStepResults, 'SUCCESS')
# Set <failFlag> to True - to imitate job step failure on the server
failFlag = False
twisted_logger.writeLog(self.logPrefix, None, "Executing JobStep.run()...")
return self.js.run(testRequest).addCallback(self.__checkJobStepResults, 'RUNNING')
