def _loadInfo(self):
"""Load <self.clientInfo> dictionary with client configuration data (from scs.scs_client
and scs.ext_server tables) for all SCS clients"""
self.clientInfo = {}
query = (
"select c.client_name, s.host, s.port, c.enabled from scs.scs_client c, "
"scs.ext_server s where c.server_name = s.server_name"
)
db = TwistedMySQLdb()
return db.query(query).addCallback(self._loadInfoComplete).addErrback(self._loadInfoFailure, query)
def __init__(self, workflow, jobID, info, logName = None):
"""Class's contructor. Parameters:
@param workflow: Workflow instance
@type workflow: C{workflow.Workflow} instance
@param jobID: jobid
@type jobID: int
@param info: Dictionary containing job step's information
@type info: C{dict} containing following keys: {'stepID', 'stepNo', 'client', 'input_src', 'out_flag'}
@param logName: name used to identify log/error files being used by given JobStep instance
"""
self.input = None
self.output = None
self.status = "NEW"
self.jobID = jobID
self.logName = logName
self.workflow = workflow
self.db = TwistedMySQLdb()
self.stepID = info['stepID']
self.client = info['clientName']
self.stepNo = info['stepNo']
self.lastStep = info['outFlag']
self.inputSource = info['inputSrc']
# Set JobStep's deferred and add callback() and errback() methods to it
self.deferred = defer.Deferred()
self.deferred.addCallback(self.successHandler).addErrback(self.failureHandler)
# Set logger
self.logPrefix = 'JOB: #%d, STEP: #%d' % (self.jobID, self.stepNo)
def __init__(self, logDir = None):
"""Class's constructor
@param logDir: folder where log and error files will be created
"""
service.MultiService.__init__(self)
self.db = TwistedMySQLdb()
self.workflowInfo = {}
self.name = 'WorkflowManager'
self.logDir = logDir
self.logName = self.name
self.logPrefix = 'WORKFLOW_MANAGER'
def __init__(self, name, info, logName):
"""WorkFlow class's constructor.
1. Initializes essential instance attributes;
2. Sets SCS clients associated with the given workflow;
3. Verifies if all SCS clients are 'online' and sets workflow.online attribute
4. Verifies if workflow is enabled and sets workflow.enabled attribute
5. Schedules regular check of workflow's 'enabled' status
@param name: workflow name
@param info: <name> - workflow name
<info> - dictionary containing workflow step info
<logDir> - log directory
@param logName: unique log/error name (identifier) that will be used to write to these particular files
"""
self.name = name
self.steps = []
self.info = info
self.loop = None
self.timeout = None
self.online = False
self.enabled = False
self.db = TwistedMySQLdb()
self.clientManager = client.SCSClientManager()
self.logName = logName
self.logPrefix = self.name
self.extOnlineDeferreds = {}
self.extOfflineDeferreds = {}
try:
clients = self.__initialize()
except:
raise
if clients != []:
self._setClients(clients)
# Execute self._checkEnabled() to see if workflow is enabled (set self.enabled attribute)
self._checkEnabled()
# Schedule checking for anabled/disabled status (if <autoDiscovery is True)
self._scheduleCheckEnabled()
def _loadInfo(self):
"Load <self.resourceInfo> dictionary with server configuration data (from scs.scs_server table) for all SCS servers"
# Obtain SynchDB instance (connected to the database)
query = "select server_name, wf_name, port, protocol, max_connections, enabled from scs.scs_server"
db = TwistedMySQLdb()
return db.query(query).addCallback(self._loadInfoComplete).addErrback(self._loadInfoFailure, query)
class _Workflow(object):
"""WorkFlow class - implements general SCS workflow functionality"""
def __init__(self, name, info, logName):
"""WorkFlow class's constructor.
1. Initializes essential instance attributes;
2. Sets SCS clients associated with the given workflow;
3. Verifies if all SCS clients are 'online' and sets workflow.online attribute
4. Verifies if workflow is enabled and sets workflow.enabled attribute
5. Schedules regular check of workflow's 'enabled' status
@param name: workflow name
@param info: <name> - workflow name
<info> - dictionary containing workflow step info
<logDir> - log directory
@param logName: unique log/error name (identifier) that will be used to write to these particular files
"""
self.name = name
self.steps = []
self.info = info
self.loop = None
self.timeout = None
self.online = False
self.enabled = False
self.db = TwistedMySQLdb()
self.clientManager = client.SCSClientManager()
self.logName = logName
self.logPrefix = self.name
self.extOnlineDeferreds = {}
self.extOfflineDeferreds = {}
try:
clients = self.__initialize()
except:
raise
if clients != []:
self._setClients(clients)
# Execute self._checkEnabled() to see if workflow is enabled (set self.enabled attribute)
self._checkEnabled()
# Schedule checking for anabled/disabled status (if <autoDiscovery is True)
self._scheduleCheckEnabled()
def __initialize(self):
"""Initialize given SCS client instance
@return: clients - a list of started SCSClient instances that belong to given workflow
"""
if not self.info.has_key('enabled'):
err_msg = "Workflow information does not have 'enabled' data"
raise RuntimeError, err_msg
clients = []
self.enabled = self.info['enabled']
self.timeout = self.info['timeout']
for infoRec in self.info['steps']:
if infoRec['enabled']:
stepRec = {}
# IMPORTANT: 'client' data will be set by seld.__setClients() - after verifying that
# client hass actually been started, and adding workflow's online/offline
# deferreds to client factory's extOnlineDeferreds/extOfflineDeferreds dictionaries
stepRec = copy.copy(infoRec)
stepRec['client'] = None
# Try to get client's service instance - if such service has already been started
if self.clientManager.namedServices.has_key(stepRec['clientName']):
clients.append(self.clientManager.namedServices[stepRec['clientName']])
self.steps.append(stepRec)
return clients
def __callbackDeferred(self, deferred, reset, result):
"""Call deferred and reset its <called> attribute so it can be called again
@param deferred: deferred to be called back ('fired')
@param reset: True/False indicating if deferred's deferred.called attribute
should be reset back to False so that it can be called again
@param result: result to be supplied (passed) to deferred's callback method
"""
deferred.callback(result)
if reset:
deferred.called = False
def _setClients(self, clients):
"""Set client instances in the <clients> list into corresponding step's 'client' data, and add workflow's
online and offline deferreds to that client's <extOnlineDeferreds> and <extOfflineDeferreds> dictionaries
@param clients: SCSClient instances to be added to job steps's 'client' data
"""
for client in clients:
for stepRec in self.steps:
if client.name == stepRec['clientName']:
# Initialize client online/offline deferreds
onlineDeferred = defer.Deferred().addCallback(self.handleClientReconnect)
offlineDeferred = defer.Deferred().addCallback(self.handleClientDisconnect)
# Add these deferreds to corresponding client's dictionaries - to be called back when appropriate events occur
client.addOnlineDeferred(onlineDeferred, reset = True)
client.addOfflineDeferred(offlineDeferred, reset = True)
# Set current step record's 'client' data
stepRec['client'] = client
break
# All clients have been set: check 'online' status of all clients, and set workflow's status correspondingly
self._checkOnline()
def _setOnline(self, online):
"""Set <online> attribute
If workflow's <self.online> changes from False ("offline") to True ("online"), call back 'online'
external deferreds (deferreds in <self.extOnlineDeferreds> dictionary). If on the other hand online
status changes from True ("online" to False ("offline"), call back 'offline' deferreds (deferreds
in <self.extOfflineDeferreds> dictionary)
@param online: True/False flag indicating if workflow is online or offline
"""
if self.online != online:
self.online = online
twisted_logger.writeLog(self.logPrefix, self.logName, "Changing online status to %s" % online)
# Call back (trigger) external deferreds
if online:
[self.__callbackDeferred(deferred, self.extOnlineDeferreds[deferred], online) for deferred in self.extOnlineDeferreds.keys()]
else:
[self.__callbackDeferred(deferred, self.extOfflineDeferreds[deferred], online) for deferred in self.extOfflineDeferreds.keys()]
def _checkOnline(self):
"Check workflow's status (ONLINE/OFFLINE)"
online = True
# Re-evaluate workflow's status
for step in self.steps:
if step['client'] is None:
online = False
elif not step['client'].online:
online = False
if not online:
break
if online:
# Check that all enabled clients that belong to a given workflow have been started
if useDB:
# Perform SQL query to obtain a list of enabled steps
query = "select client_name, step_no from scs.workflow_step where wf_name = '%s' and enabled = 1" % self.name
self.db.query(query).addCallback(self._checkOnlineHandler).addErrback(self._dbFailureHandler, query)
return None
else:
if len(self.steps) < len(self.info['steps']):
online = False
else:
# Use initial <self.info> record that was used to initialize current workflow instance
for infoRec in self.info['steps']:
if infoRec['enabled']:
stepNo = infoRec['stepNo']
if not self.steps[stepNo - 1]['client']:
online = False
break
self._setOnline(online)
def _checkEnabled(self):
"""Check if workflow is enabled. This method should be used as a deferred's callback handler - to
regularly check if workflow is enabled. This is intended for allowing dynamicly enabling/disabling
workflow using scs.workflow_lookup table"""
if useDB:
# Perform SQL query to obtain a list of enabled steps
query = "select enabled from scs.workflow_lookup where wf_name = '%s'" % self.name
self.db.query(query).addCallback(self._checkEnabledHandler).addErrback(self._dbFailureHandler, query)
else:
# Use initial <self.info> to define the value of <self.enabled>
self.enabled = self.info['enabled']
def _dbFailureHandler(self, fail, stmt):
"SQL opertion failure handler"
err_msg = "SQL Failure: %s" % fail.getErrorMessage()
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
err_msg = "SQL statement: '%s'" % stmt
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
def _checkOnlineHandler(self, result):
"Set <self.online> attribute using SQL query result"
online = True
if len(self.steps) < len(result):
online = False
else:
if result == ():
online = False
else:
for (client_name, step_no) in result:
# Verify that given step has 'client' instance reference within <self.steps>
if not self.steps[step_no - 1]['client']:
online = False
break
self._setOnline(online)
def _checkEnabledHandler(self, result):
"Set <self.enabled> attribute using SQL query result"
self.enabled = (result[0][0] == True)
def _scheduleCheckEnabled(self):
"Schedule regular checks of workflow's 'enabled' status"
if autoDiscovery:
if not self.loop:
# Schedule execution of itself in a loop with <checkInterval> seconds interval
self.loop = task.LoopingCall(self._checkEnabled)
if not self.loop.running:
self.loop.start(checkInterval, now = False)
def enable(self):
"""Enable workflow. SCSServer entity that 'owns' this workflow will start accepting external client's
job requests - provided that workflow is also 'online' (all SCSClient entities used by the workflow
are connected to their external servers)"""
# Schedule self._checkEnabled()
self._scheduleCheckEnabled()
self.enabled = True
msg = "'%s' workflow is enabled" % self.name
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
def disable(self):
"Disable workflow. SCSServer entity that 'owns' this workflow will start rejecting external client's job requests"
# Cancel scheduled task(s)
if self.loop and self.loop.running:
self.loop.stop()
self.enabled = False
msg = "'%s' workflow is disabled" % self.name
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
def setClients(self, clientsList = None):
"Set <self.clients> dictionary with the _SCSClientService entities"
self.clients = {}
if clientsList:
for client in clientsList:
self.clients[client.name]
def addOnlineDeferred(self, deferred, reset = True):
"Add new external deferred to <extOnlineDeferreds> dictionary"
self.extOnlineDeferreds[deferred] = reset
def addOfflineDeferred(self, deferred, reset = True):
"Add new external deferred to <extOfflineDeferreds> dictionary"
self.extOfflineDeferreds[deferred] = reset
def handleClientReconnect(self, result):
"""Deferred event handler for client connect event. This method should be
called back by SCSClientProtocol entity in the event of connectionMade() event"""
self._checkOnline()
def handleClientDisconnect(self, result):
"""Deferred event handler for client disconnect event. This method should be called back
by SCSClientFactory entity in the event of clientConnectionLost() or clientConnectionFailed() event"""
self._setOnline(False)
class WorkflowManager(singleton.Singleton, service.MultiService):
"""Implements general SCS workflow management functionality. Implemented using Singleton design pattern
(extends <singleton.Singleton>) and ensures that class gets initialized only once - no matter how many
times constructor is called (uses @singleton.uniq() decorator for its __init() constructor
Implements IResourceManager interface
"""
implements(IResourceManager)
@singleton.uniq
def __init__(self, logDir = None):
"""Class's constructor
@param logDir: folder where log and error files will be created
"""
service.MultiService.__init__(self)
self.db = TwistedMySQLdb()
self.workflowInfo = {}
self.name = 'WorkflowManager'
self.logDir = logDir
self.logName = self.name
self.logPrefix = 'WORKFLOW_MANAGER'
def _loadInfo(self):
"""Select workflow data from scs.workflow_lookup table
@return: deferred which will 'fire' when SELECT query completes loading data
"""
# Obtain database connection object that can be used to execute SQL query
self.workflowInfo = {}
query = "select wf_name, enabled, job_timeout from scs.workflow_lookup"
return self.db.query(query).addCallback(self._loadStepInfo).addErrback(self._loadInfoFailure, query)
def _loadStepInfo(self, result):
"""Load workflow step info from scs.workflow_step table"
@return: deferred which will 'fire' when data is loaded
"""
workflowInfo = {}
for (wf_name, enabled, timeout) in result:
workflowInfo[wf_name] = {'enabled': enabled == True, 'timeout': timeout}
query = "select wf_name, step_id, step_no, client_name, input, out_flag, " \
"enabled from scs.workflow_step order by wf_name, step_no"
return self.db.query(query).addCallback(self._loadComplete, workflowInfo).addErrback(self._loadInfoFailure, query)
def _loadComplete(self, result, workflowInfo):
"Complete loading workflowInfo data (started by self._loadInfo() and self._loadStepInfo()"
if result == ():
err_msg = "scs.workflow_step table contains no workflow information (empty)"
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
for (wf_name, step_id, step_no, client_name, input, out_flag, enabled) in result:
if not self.workflowInfo.has_key(wf_name):
if not workflowInfo.has_key(wf_name):
err_msg = "'%s' workflow is not defined within scs.workflow_lookup table" % wf_name
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
self.workflowInfo.setdefault(wf_name, {'enabled': workflowInfo[wf_name]['enabled'],
'timeout': workflowInfo[wf_name]['timeout'], 'steps': []})
stepRec = {'stepID': step_id, 'stepNo': step_no, 'clientName': client_name,
'inputSrc': input, 'outFlag': out_flag, 'enabled': enabled}
self.workflowInfo[wf_name]['steps'].append(stepRec)
def _loadInfoFailure(self, fail, query):
"Workflow load failure handler"
err_msg = "Failure loading workflow info: %s" % fail.getErrorMessage()
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
twisted_logger.writeErr(self.logPrefix, self.logName, "SQL Query: '%s'" % query)
def start(self, workflowName):
"""Create workflow instance <workflowName>
@param workflowName: workflow name
@type workflowName: str
"""
failPrefix = "Failure starting '%s' workflow service" % workflowName
if self.namedServices.has_key(workflowName):
if self.namedServices[workflowName].running:
err_msg = "%s: Workflow service is already running" % failPrefix
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
else:
# Client service has stopped: delete that service before restarting it
self.removeService(self.getServiceNamed(workflowName))
if self.workflowInfo.has_key(workflowName):
if not self.workflowInfo[workflowName]['enabled']:
err_msg = "%s: workflow is disabled" % failPrefix
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
else:
err_msg = "%s: unknown workflow" % failPrefix
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
if not self.running:
self.running = 1
if self.namedServices.has_key(workflowName):
workflowService = self.namedServices[workflowName]
workflowService.startService()
else:
workflowService = _WorkflowService(workflowName, self.workflowInfo[workflowName], self.logName)
addService(self, workflowService)
def stop(self, workflowName):
"""Stop workflow service
@param workflowName: workflow name
"""
if not workflowName in self.namedServices.keys():
err_msg = "Unable to stop '%s' workflow service: service has not been started" % workflowName
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
# Remove workflow from WorkflowManager's services collection
self.removeService(self.getServiceNamed(workflowName))
def enable(self, workflowName):
"""Enable workflow. SCSServer entity that 'owns' this workflow will start accepting external client's
job requests - provided that workflow is also 'online' (all SCSClient entities used by the workflow
are connected to their external servers)"""
if workflowName in self.namedServices.keys():
msg = "Enabling '%s' workflow..." % workflowName
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
self.namedServices[workflowName].workflow.enable()
def disable(self, workflowName):
"Disable workflow. SCSServer entity that 'owns' this workflow will start rejecting external client's job requests"
if workflowName in self.namedServices.keys():
msg = "Disabling '%s' workflow..." % workflowName
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
self.namedServices[workflowName].workflow.disable()
def startUp(self):
"Start all enabled SCS workflows"
for workflowName in self.workflowInfo.keys():
if self.workflowInfo[workflowName]['enabled']:
try:
self.start(workflowName)
except:
pass
def shutDown(self):
"Stop all SCS workflows"
for workflowName in self.namedServices.keys():
try:
self.stop(workflowName)
except:
pass
def startService(self):
"Start WorkflowManager service"
actions = []
# Initialize log and error service
baseLogFileName = "workflow_manager"
msg = "Opening '%s.log' and '%s.err' files in '%s' folder..." % (baseLogFileName, baseLogFileName, self.logDir)
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
twisted_logger.initLogging(self.name, baseLogFileName, self.logDir, self)
for srv in self.services:
if not srv.running:
actions.append(defer.maybeDeferred(srv.startService))
twisted_logger.writeLog(self.logPrefix, self.logName, "Starting '%s' service..." % self.name)
actions.append(defer.maybeDeferred(self.startUp))
service.Service.startService(self)
return defer.DeferredList(actions)
def stopService(self):
"Stop WorkflowManager service"
twisted_logger.writeLog(self.logPrefix, self.logName, "Stopping '%s' service..." % self.name)
deferreds = []
for srv in self.services[:]:
deferred = self.removeService(srv)
if deferred:
deferreds.append(deferred)
service.Service.stopService(self)
return defer.DeferredList(deferreds)
def getResource(self, name):
"""Obtain given workflow instance
@param name: workflow name
@return: _Workflow instance
"""
if not self.namedServices.has_key(name):
err_msg = "'%s' workflow has not been started" % name
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
raise RuntimeError, err_msg
return self.namedServices[name].workflow
def addOnlineDeferred(self, name, deferred, reset = True):
"Add new external deferred to workflow's <extOnlineDeferreds> dictionary"
if not self.namedServices.has_key(name):
err_msg = "addOnlineDeferred() failure: '%s' workflow entity has not been started" % name
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
else:
self.namedServices[name].workflow.addOnlineDeferred(deferred, reset)
def addOfflineDeferred(self, name, deferred, reset = True):
"Add new external deferred to workflow's <extOfflineDeferreds> dictionary"
if not self.namedServices.has_key(name):
err_msg = "addOfflineDeferred() failure: '%s' workflow entity has not been started" % name
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
else:
self.namedServices[name].workflow.addOfflineDeferred(deferred, reset)
class JobStep(object):
"""JobStep class - implements individual job step's functionality. Multiple (potentially) instances of this class
belong to a single job. JobStep instances are initialized and orchestrated by a job.Job class.
"""
def __init__(self, workflow, jobID, info, logName = None):
"""Class's contructor. Parameters:
@param workflow: Workflow instance
@type workflow: C{workflow.Workflow} instance
@param jobID: jobid
@type jobID: int
@param info: Dictionary containing job step's information
@type info: C{dict} containing following keys: {'stepID', 'stepNo', 'client', 'input_src', 'out_flag'}
@param logName: name used to identify log/error files being used by given JobStep instance
"""
self.input = None
self.output = None
self.status = "NEW"
self.jobID = jobID
self.logName = logName
self.workflow = workflow
self.db = TwistedMySQLdb()
self.stepID = info['stepID']
self.client = info['clientName']
self.stepNo = info['stepNo']
self.lastStep = info['outFlag']
self.inputSource = info['inputSrc']
# Set JobStep's deferred and add callback() and errback() methods to it
self.deferred = defer.Deferred()
self.deferred.addCallback(self.successHandler).addErrback(self.failureHandler)
# Set logger
self.logPrefix = 'JOB: #%d, STEP: #%d' % (self.jobID, self.stepNo)
def __failure(self, nothing, err_msg):
"Last part of JobStep's failure handler"
msg = "Job step (%s) has failed: %s" % (self.client, err_msg)
twisted_logger.writeErr(self.logPrefix, self.logName, msg)
return Failure(err_msg, RuntimeError)
def __success(self, nothing):
"Last part of JobStep's success handler"
msg = "Job step (%s) has successfully completed" % self.client
twisted_logger.writeLog(self.logPrefix, self.logName, msg)
return None
def __handleDBFailure(self, fail, stmt, type):
"Record <scs.job_step> insert/update failure details"
operationType = {'insert': 'inserting into', 'update': 'updating'}
err_msg = "Failure %s <scs.job_step> table: %s" % (operationType[type], fail.getErrorMessage())
twisted_logger.writeErr(self.logPrefix, self.logName, err_msg)
def init(self):
"Initialization: Insert new record into <scs.job_step> table"
stmt = "insert scs.job_step (job_id, step_id, status) values " \
"(%d, %d, '%s')" % (self.jobID, self.stepID, self.status)
return self.db.execute(stmt).addErrback(self.__handleDBFailure, stmt, 'insert')
def run(self, request):
"""Start current job step's execution by sending request (<input>) to remote server
@param request: request to be sent to external server
@type request: dict {'scs_jobid': <jobid>, 'request': <request_data (text)>}
@return: deferred - a Deferred which will fire None or a Failure when database UPDATE statement is complete
"""
error = None
self.input = request
request = {'scs_jobid': self.jobID, 'request': request}
try:
client.SCSClientManager().sendRequest(self.client, request, self.deferred)
except Exception, err:
error = str(err)
self.status = 'FAILURE'
stmt = "update scs.job_step set status = '%s', text_input = '%s', error = '%s', start_time = now(), " \
"end_time = now() where job_id = %d and step_id = %d" % (self.status, str(self.input).replace("'", "''"),
error.replace("'", "''"), self.jobID, self.stepID)
else:
