class TaskBase:
def __init__(self):
self.transClient = TransformationClient()
def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''):
return S_ERROR("Not implemented")
def submitTransformationTasks(self,taskDict):
return S_ERROR("Not implemented")
def submitTasksToExternal(self,task):
return S_ERROR("Not implemented")
def updateDBAfterTaskSubmission(self,taskDict):
updated = 0
startTime = time.time()
for taskID in sortList(taskDict.keys()):
transID = taskDict[taskID]['TransformationID']
if taskDict[taskID]['Success']:
res = self.transClient.setTaskStatusAndWmsID(transID,taskID,'Submitted',str(taskDict[taskID]['ExternalID']))
if not res['OK']:
gLogger.warn("updateDBAfterSubmission: Failed to update task status after submission" , "%s %s" % (taskDict[taskID]['ExternalID'],res['Message']))
updated +=1
gLogger.info("updateDBAfterSubmission: Updated %d tasks in %.1f seconds" % (updated,time.time()-startTime))
return S_OK()
def updateTransformationReservedTasks(self,taskDicts):
return S_ERROR("Not implemented")
def getSubmittedTaskStatus(self,taskDicts):
return S_ERROR("Not implemented")
def getSubmittedFileStatus(self,fileDicts):
return S_ERROR("Not implemented")
class TaskBase:
def __init__(self):
self.transClient = TransformationClient()
def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''):
return S_ERROR("Not implemented")
def submitTransformationTasks(self,taskDict):
return S_ERROR("Not implemented")
def submitTasksToExternal(self,task):
return S_ERROR("Not implemented")
def updateDBAfterTaskSubmission(self,taskDict):
updated = 0
startTime = time.time()
for taskID in sortList(taskDict.keys()):
transID = taskDict[taskID]['TransformationID']
if taskDict[taskID]['Success']:
res = self.transClient.setTaskStatusAndWmsID(transID,taskID,'Submitted',str(taskDict[taskID]['ExternalID']))
if not res['OK']:
gLogger.warn("updateDBAfterSubmission: Failed to update task status after submission" , "%s %s" % (taskDict[taskID]['ExternalID'],res['Message']))
updated +=1
gLogger.info("updateDBAfterSubmission: Updated %d tasks in %.1f seconds" % (updated,time.time()-startTime))
return S_OK()
def updateTransformationReservedTasks(self,taskDicts):
return S_ERROR("Not implemented")
def getSubmittedTaskStatus(self,taskDicts):
return S_ERROR("Not implemented")
def getSubmittedFileStatus(self,fileDicts):
return S_ERROR("Not implemented")
class TaskBase(object):
''' The other classes inside here inherits from this one.
'''
def __init__(self, transClient=None, logger=None):
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
if not logger:
self.log = gLogger.getSubLogger('TaskBase')
else:
self.log = logger
def prepareTransformationTasks(self,
transBody,
taskDict,
owner='',
ownerGroup='',
ownerDN=''):
return S_ERROR("Not implemented")
def submitTransformationTasks(self, taskDict):
return S_ERROR("Not implemented")
def submitTasksToExternal(self, task):
return S_ERROR("Not implemented")
def updateDBAfterTaskSubmission(self, taskDict):
""" Sets tasks status after the submission to "Submitted", in case of success
"""
updated = 0
startTime = time.time()
for taskID in sorted(taskDict):
transID = taskDict[taskID]['TransformationID']
if taskDict[taskID]['Success']:
res = self.transClient.setTaskStatusAndWmsID(
transID, taskID, 'Submitted',
str(taskDict[taskID]['ExternalID']))
if not res['OK']:
self.log.warn(
"updateDBAfterSubmission: Failed to update task status after submission",
"%s %s" %
(taskDict[taskID]['ExternalID'], res['Message']))
updated += 1
self.log.info(
"updateDBAfterSubmission: Updated %d tasks in %.1f seconds" %
(updated, time.time() - startTime))
return S_OK()
def updateTransformationReservedTasks(self, taskDicts):
return S_ERROR("Not implemented")
def getSubmittedTaskStatus(self, taskDicts):
return S_ERROR("Not implemented")
def getSubmittedFileStatus(self, fileDicts):
return S_ERROR("Not implemented")
class TaskBase( TransformationAgentsUtilities ):
''' The other classes inside here inherits from this one.
'''
def __init__( self, transClient = None, logger = None ):
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
if not logger:
self.log = gLogger.getSubLogger( 'TaskBase' )
else:
self.log = logger
self.pluginLocation = 'DIRAC.TransformationSystem.Client.TaskManagerPlugin'
self.transInThread = {}
self.debug = False
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '',
bulkSubmissionFlag = False ):
return S_ERROR( "Not implemented" )
def submitTransformationTasks( self, taskDict ):
return S_ERROR( "Not implemented" )
def submitTasksToExternal( self, task ):
return S_ERROR( "Not implemented" )
def updateDBAfterTaskSubmission( self, taskDict ):
""" Sets tasks status after the submission to "Submitted", in case of success
"""
updated = 0
startTime = time.time()
for taskID, task in taskDict.iteritems():
transID = task['TransformationID']
if task['Success']:
res = self.transClient.setTaskStatusAndWmsID( transID, taskID, 'Submitted',
str( task['ExternalID'] ) )
if not res['OK']:
self._logWarn( "Failed to update task status after submission" ,
"%s %s" % ( task['ExternalID'], res['Message'] ),
transID = transID, method = 'updateDBAfterSubmission' )
updated += 1
if updated:
self._logInfo( "Updated %d tasks in %.1f seconds" % ( updated, time.time() - startTime ),
transID = transID, method = 'updateDBAfterSubmission' )
return S_OK()
def updateTransformationReservedTasks( self, taskDicts ):
return S_ERROR( "Not implemented" )
def getSubmittedTaskStatus( self, taskDicts ):
return S_ERROR( "Not implemented" )
def getSubmittedFileStatus( self, fileDicts ):
return S_ERROR( "Not implemented" )
class TaskBase( object ):
''' The other classes inside here inherits from this one.
'''
def __init__( self, transClient = None, logger = None ):
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
if not logger:
self.log = gLogger.getSubLogger( 'TaskBase' )
else:
self.log = logger
def prepareTransformationTasks( self, transBody, taskDict ):
return S_ERROR( "Not implemented" )
def submitTransformationTasks( self, taskDict ):
return S_ERROR( "Not implemented" )
def submitTasksToExternal( self, task ):
return S_ERROR( "Not implemented" )
def updateDBAfterTaskSubmission( self, taskDict ):
""" Sets tasks status after the submission to "Submitted", in case of success
"""
updated = 0
startTime = time.time()
for taskID in sorted( taskDict ):
transID = taskDict[taskID]['TransformationID']
if taskDict[taskID]['Success']:
res = self.transClient.setTaskStatusAndWmsID( transID, taskID, 'Submitted',
str( taskDict[taskID]['ExternalID'] ) )
if not res['OK']:
self.log.warn( "updateDBAfterSubmission: Failed to update task status after submission" ,
"%s %s" % ( taskDict[taskID]['ExternalID'], res['Message'] ) )
updated += 1
self.log.info( "updateDBAfterSubmission: Updated %d tasks in %.1f seconds" % ( updated, time.time() - startTime ) )
return S_OK()
def updateTransformationReservedTasks( self, taskDicts ):
return S_ERROR( "Not implemented" )
def getSubmittedTaskStatus( self, taskDicts ):
return S_ERROR( "Not implemented" )
def getSubmittedFileStatus( self, fileDicts ):
return S_ERROR( "Not implemented" )
class TaskBase(TransformationAgentsUtilities):
''' The other classes inside here inherits from this one.
'''
def __init__(self, transClient=None, logger=None):
""" c'tor """
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
if not logger:
self.log = gLogger.getSubLogger('TaskBase')
else:
self.log = logger
self.pluginLocation = 'DIRAC.TransformationSystem.Client.TaskManagerPlugin'
super(TaskBase, self).__init__()
def prepareTransformationTasks(
self,
_transBody,
_taskDict, # pylint: disable=no-self-use, unused-argument
owner='',
ownerGroup='',
ownerDN='',
bulkSubmissionFlag=False): # pylint: disable=unused-argument
""" To make sure the method is implemented in the derived class """
if owner or ownerGroup or ownerDN or bulkSubmissionFlag: # Makes pylint happy
pass
return S_ERROR("Not implemented")
def submitTransformationTasks(self, _taskDict): # pylint: disable=no-self-use
""" To make sure the method is implemented in the derived class """
return S_ERROR("Not implemented")
def submitTasksToExternal(self, _task): # pylint: disable=no-self-use
""" To make sure the method is implemented in the derived class """
return S_ERROR("Not implemented")
def updateDBAfterTaskSubmission(self, taskDict):
""" Sets tasks status after the submission to "Submitted", in case of success
"""
updated = 0
startTime = time.time()
for taskID, task in taskDict.iteritems():
transID = task['TransformationID']
if task['Success']:
res = self.transClient.setTaskStatusAndWmsID(
transID, taskID, 'Submitted', str(task['ExternalID']))
if not res['OK']:
self._logWarn(
"Failed to update task status after submission",
"%s %s" % (task['ExternalID'], res['Message']),
transID=transID,
method='updateDBAfterSubmission')
updated += 1
if updated:
self._logInfo("Updated %d tasks in %.1f seconds" %
(updated, time.time() - startTime),
transID=transID,
method='updateDBAfterSubmission')
return S_OK()
def updateTransformationReservedTasks(self, _taskDicts): # pylint: disable=no-self-use
""" To make sure the method is implemented in the derived class """
return S_ERROR("Not implemented")
def getSubmittedTaskStatus(self, _taskDicts): # pylint: disable=no-self-use
""" To make sure the method is implemented in the derived class """
return S_ERROR("Not implemented")
def getSubmittedFileStatus(self, _fileDicts): # pylint: disable=no-self-use
""" To make sure the method is implemented in the derived class """
return S_ERROR("Not implemented")
class TaskManagerAgentBase(AgentModule):
#############################################################################
def initialize(self):
self.section = self.am_getOption("section")
gMonitor.registerActivity("SubmittedTasks","Automatically submitted tasks","Transformation Monitoring","Tasks", gMonitor.OP_ACUM)
self.transClient = TransformationClient()
return S_OK()
#############################################################################
def execute(self):
""" The TaskManagerBase execution method. """
# Determine whether the task status is to be monitored and updated
enableTaskMonitor = self.am_getOption('MonitorTasks','')
if not enableTaskMonitor:
gLogger.info("execute: Monitoring of tasks is disabled.")
gLogger.info("execute: To enable create the 'MonitorTasks' option")
else:
res = self.updateTaskStatus()
if not res['OK']:
gLogger.warn('execute: Failed to update task states', res['Message'])
# Determine whether the task files status is to be monitored and updated
enableFileMonitor = self.am_getOption('MonitorFiles','')
if not enableFileMonitor:
gLogger.info("execute: Monitoring of files is disabled.")
gLogger.info("execute: To enable create the 'MonitorFiles' option")
else:
res = self.updateFileStatus()
if not res['OK']:
gLogger.warn('execute: Failed to update file states', res['Message'])
# Determine whether the checking of reserved tasks is to be performed
enableCheckReserved = self.am_getOption('CheckReserved','')
if not enableCheckReserved:
gLogger.info("execute: Checking of reserved tasks is disabled.")
gLogger.info("execute: To enable create the 'CheckReserved' option")
else:
res = self.checkReservedTasks()
if not res['OK']:
gLogger.warn('execute: Failed to checked reserved tasks',res['Message'])
# Determine whether the submission of tasks is to be executed
enableSubmission = self.am_getOption('SubmitTasks','')
if not enableSubmission:
gLogger.info("execute: Submission of tasks is disabled.")
gLogger.info("execute: To enable create the 'SubmitTasks' option")
else:
res = self.submitTasks()
if not res['OK']:
gLogger.warn('execute: Failed to submit created tasks', res['Message'])
return S_OK()
def _selectTransformations(self,transType=[],status=['Active','Completing'],agentType=['Automatic']):
selectCond = {}
if status:
selectCond['Status'] = status
if transType:
selectCond['Type'] = transType
if agentType:
selectCond['AgentType'] = agentType
res = self.transClient.getTransformations(condDict=selectCond)
if not res['OK']:
gLogger.error("_selectTransformations: Failed to get transformations for selection.",res['Message'])
elif not res['Value']:
gLogger.info("_selectTransformations: No transformations found for selection.")
else:
gLogger.info("_selectTransformations: Obtained %d transformations for selection" % len(res['Value']))
return res
def updateTaskStatus(self):
gLogger.info("updateTaskStatus: Updating the Status of tasks")
# Get the transformations to be updated
status = self.am_getOption('UpdateTasksStatus',['Active','Completing','Stopped'])
res = self._selectTransformations(transType=self.transType,status=status,agentType=[])
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
# Get the tasks which are in a UPDATE state
updateStatus = self.am_getOption('TaskUpdateStatus',['Checking','Deleted','Killed','Staging','Stalled','Matched','Rescheduled','Completed','Submitted','Received','Waiting','Running'])
condDict = {"TransformationID":transID,"ExternalStatus":updateStatus}
timeStamp = str(datetime.datetime.utcnow() - datetime.timedelta(minutes=10))
res = self.transClient.getTransformationTasks(condDict=condDict,older=timeStamp, timeStamp='LastUpdateTime')
if not res['OK']:
gLogger.error("updateTaskStatus: Failed to get tasks to update for transformation", "%s %s" % (transID,res['Message']))
continue
if not res['Value']:
gLogger.verbose("updateTaskStatus: No tasks found to update for transformation %s" % transID)
continue
res = self.getSubmittedTaskStatus(res['Value'])
if not res['OK']:
gLogger.error("updateTaskStatus: Failed to get updated task statuses for transformation", "%s %s" % (transID,res['Message']))
continue
statusDict = res['Value']
for status in sortList(statusDict.keys()):
taskIDs = statusDict[status]
gLogger.info("updateTaskStatus: Updating %d task(s) from transformation %d to %s" % (len(taskIDs),transID,status))
res = self.transClient.setTaskStatus(transID,taskIDs,status)
if not res['OK']:
gLogger.error("updateTaskStatus: Failed to update task status for transformation", "%s %s" % (transID,res['Message']))
gLogger.info("updateTaskStatus: Transformation task status update complete")
return S_OK()
def updateFileStatus(self):
gLogger.info("updateFileStatus: Updating Status of task files")
#Get the transformations to be updated
status = self.am_getOption('UpdateFilesStatus',['Active','Completing','Stopped'])
res = self._selectTransformations(transType=self.transType,status=status,agentType=[])
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
# Get the files which are in a UPDATE state
updateStatus = self.am_getOption('FileUpdateStatus',['Submitted','Received','Waiting','Running'])
timeStamp = str(datetime.datetime.utcnow() - datetime.timedelta(minutes=10))
condDict = {'TransformationID' : transID, 'Status' : ['Assigned']}
res = self.transClient.getTransformationFiles(condDict=condDict,older=timeStamp, timeStamp='LastUpdate')
if not res['OK']:
gLogger.error("updateFileStatus: Failed to get transformation files to update.",res['Message'])
continue
if not res['Value']:
gLogger.info("updateFileStatus: No files to be updated for transformation %s." % transID)
continue
res = self.getSubmittedFileStatus(res['Value'])
if not res['OK']:
gLogger.error("updateFileStatus: Failed to get updated file statuses for transformation","%s %s" % (transID,res['Message']))
continue
statusDict = res['Value']
if not statusDict:
gLogger.info("updateFileStatus: No file statuses to be updated for transformation %s." % transID)
continue
fileReport = FileReport(server=self.transClient.getServer())
for lfn,status in statusDict.items():
fileReport.setFileStatus(int(transID),lfn,status)
res = fileReport.commit()
if not res['OK']:
gLogger.error("updateFileStatus: Failed to update file status for transformation", "%s %s" % (transID, res['Message']))
else:
for status,update in res['Value'].items():
gLogger.info("updateFileStatus: Updated %s files for %s to %s." % (update, transID, status))
gLogger.info("updateFileStatus: Transformation file status update complete")
return S_OK()
def checkReservedTasks(self):
gLogger.info("checkReservedTasks: Checking Reserved tasks")
# Get the transformations which should be checked
status = self.am_getOption('CheckReservedStatus',['Active','Completing','Stopped'])
res = self._selectTransformations(transType=self.transType,status=status,agentType=[])
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
# Select the tasks which have been in Reserved status for more than 1 hour for selected transformations
condDict = {"TransformationID":transID,"ExternalStatus":'Reserved'}
time_stamp_older = str(datetime.datetime.utcnow() - datetime.timedelta(hours=1))
time_stamp_newer = str(datetime.datetime.utcnow() - datetime.timedelta(days=7))
res = self.transClient.getTransformationTasks(condDict=condDict,older=time_stamp_older,newer=time_stamp_newer, timeStamp='LastUpdateTime')
if not res['OK']:
gLogger.error("checkReservedTasks: Failed to get Reserved tasks for transformation", "%s %s" % (transID,res['Message']))
continue
if not res['Value']:
gLogger.verbose("checkReservedTasks: No Reserved tasks found for transformation %s" % transID)
continue
res = self.updateTransformationReservedTasks(res['Value'])
if not res['OK']:
gLogger.info("checkReservedTasks: No Reserved tasks found for transformation %s" % transID)
continue
noTasks = res['Value']['NoTasks']
taskNameIDs = res['Value']['TaskNameIDs']
# For the tasks with no associated request found re-set the status of the task in the transformationDB
for taskName in noTasks:
transID,taskID = taskName.split('_')
gLogger.info("checkReservedTasks: Resetting status of %s to Created as no associated task found" % (taskName))
res = self.transClient.setTaskStatus(int(transID),int(taskID),'Created')
if not res['OK']:
gLogger.warn("checkReservedTasks: Failed to update task status and ID after recovery", "%s %s" % (taskName,res['Message']))
# For the tasks for which an associated request was found update the task details in the transformationDB
for taskName,extTaskID in taskNameIDs.items():
transID,taskID = taskName.split('_')
gLogger.info("checkReservedTasks: Resetting status of %s to Created with ID %s" % (taskName,extTaskID))
res = self.transClient.setTaskStatusAndWmsID(int(transID),int(taskID),'Submitted',str(extTaskID))
if not res['OK']:
gLogger.warn("checkReservedTasks: Failed to update task status and ID after recovery", "%s %s" % (taskName,res['Message']))
gLogger.info("checkReservedTasks: Updating of reserved tasks complete")
return S_OK()
def submitTasks(self):
gLogger.info("submitTasks: Submitting tasks for transformations")
res = getProxyInfo(False,False)
if not res['OK']:
gLogger.error("submitTasks: Failed to determine credentials for submission",res['Message'])
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
gLogger.info("submitTasks: Tasks will be submitted with the credentials %s:%s" % (owner,ownerGroup))
# Get the transformations which should be submitted
tasksPerLoop = self.am_getOption('TasksPerLoop',50)
status = self.am_getOption('SubmitStatus',['Active','Completing'])
res = self._selectTransformations(transType=self.transType,status=status)
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
transBody = transformation['Body']
res = self.transClient.getTasksToSubmit(transID,tasksPerLoop)
if not res['OK']:
gLogger.error("submitTasks: Failed to obtain tasks for transformation", "%s %s" % (transID,res['Message']))
continue
tasks = res['Value']['JobDictionary']
if not tasks:
gLogger.verbose("submitTasks: No tasks found for submission for transformation %s" % transID)
continue
gLogger.info("submitTasks: Obtained %d tasks for submission for transformation %s" % (len(tasks),transID))
res = self.prepareTransformationTasks(transBody,tasks,owner,ownerGroup)
if not res['OK']:
gLogger.error("submitTasks: Failed to prepare tasks for transformation", "%s %s" % (transID,res['Message']))
continue
res = self.submitTransformationTasks(res['Value'])
if not res['OK']:
gLogger.error("submitTasks: Failed to submit prepared tasks for transformation", "%s %s" % (transID,res['Message']))
continue
res = self.updateDBAfterTaskSubmission(res['Value'])
if not res['OK']:
gLogger.error("submitTasks: Failed to update DB after task submission for transformation", "%s %s" % (transID,res['Message']))
continue
gLogger.info("submitTasks: Submission of transformation tasks complete")
return S_OK()
class TaskManagerAgentBase( AgentModule ):
''' To be extended. The extension needs to:
- provide a taskManager object as data member
- provide a shifterProxy (string) as data member
- provide a transType (list of strings) as data member
'''
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
self.taskManager = None
self.shifterProxy = ''
self.transClient = TransformationClient()
self.transType = []
#############################################################################
def initialize( self ):
''' agent initialization
'''
if not self.taskManager:
return S_ERROR( 'No task manager provided!' )
if not self.shifterProxy:
return S_ERROR( 'No shifter proxy provided!' )
self.am_setOption( 'shifterProxy', self.shifterProxy )
if not self.transType:
return S_ERROR( 'No transformation types to look for!' )
gLogger.info( "Looking for %s" % self.transType )
gMonitor.registerActivity( "SubmittedTasks", "Automatically submitted tasks", "Transformation Monitoring", "Tasks",
gMonitor.OP_ACUM )
return S_OK()
#############################################################################
def execute( self ):
''' The TaskManagerBase execution method.
'''
# Determine whether the task status is to be monitored and updated
enableTaskMonitor = self.am_getOption( 'MonitorTasks', '' )
if not enableTaskMonitor:
gLogger.info( "execute: Monitoring of tasks is disabled." )
gLogger.info( "execute: To enable create the 'MonitorTasks' option" )
else:
res = self.updateTaskStatus()
if not res['OK']:
gLogger.warn( 'execute: Failed to update task states', res['Message'] )
# Determine whether the task files status is to be monitored and updated
enableFileMonitor = self.am_getOption( 'MonitorFiles', '' )
if not enableFileMonitor:
gLogger.info( "execute: Monitoring of files is disabled." )
gLogger.info( "execute: To enable create the 'MonitorFiles' option" )
else:
res = self.updateFileStatus()
if not res['OK']:
gLogger.warn( 'execute: Failed to update file states', res['Message'] )
# Determine whether the checking of reserved tasks is to be performed
enableCheckReserved = self.am_getOption( 'CheckReserved', '' )
if not enableCheckReserved:
gLogger.info( "execute: Checking of reserved tasks is disabled." )
gLogger.info( "execute: To enable create the 'CheckReserved' option" )
else:
res = self.checkReservedTasks()
if not res['OK']:
gLogger.warn( 'execute: Failed to checked reserved tasks', res['Message'] )
# Determine whether the submission of tasks is to be executed
enableSubmission = self.am_getOption( 'SubmitTasks', '' )
if not enableSubmission:
gLogger.info( "execute: Submission of tasks is disabled." )
gLogger.info( "execute: To enable create the 'SubmitTasks' option" )
else:
res = self.submitTasks()
if not res['OK']:
gLogger.warn( 'execute: Failed to submit created tasks', res['Message'] )
return S_OK()
def _selectTransformations( self, transType = [], status = ['Active', 'Completing'], agentType = ['Automatic'] ):
''' get the transformations
'''
selectCond = {}
if status:
selectCond['Status'] = status
if transType:
selectCond['Type'] = transType
if agentType:
selectCond['AgentType'] = agentType
res = self.transClient.getTransformations( condDict = selectCond )
if not res['OK']:
gLogger.error( "_selectTransformations: Failed to get transformations for selection.", res['Message'] )
elif not res['Value']:
gLogger.info( "_selectTransformations: No transformations found for selection." )
else:
gLogger.info( "_selectTransformations: Obtained %d transformations for selection" % len( res['Value'] ) )
return res
def updateTaskStatus( self ):
''' Updates the task status
'''
gLogger.info( "updateTaskStatus: Updating the Status of tasks" )
# Get the transformations to be updated
status = self.am_getOption( 'UpdateTasksStatus', ['Active', 'Completing', 'Stopped'] )
res = self._selectTransformations( transType = self.transType, status = status, agentType = [] )
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
# Get the tasks which are in a UPDATE state
updateStatus = self.am_getOption( 'TaskUpdateStatus', ['Checking', 'Deleted', 'Killed', 'Staging', 'Stalled',
'Matched', 'Rescheduled', 'Completed', 'Submitted',
'Received', 'Waiting', 'Running'] )
condDict = {"TransformationID":transID, "ExternalStatus":updateStatus}
timeStamp = str( datetime.datetime.utcnow() - datetime.timedelta( minutes = 10 ) )
res = self.transClient.getTransformationTasks( condDict = condDict,
older = timeStamp,
timeStamp = 'LastUpdateTime' )
if not res['OK']:
gLogger.error( "updateTaskStatus: Failed to get tasks to update for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
if not res['Value']:
gLogger.verbose( "updateTaskStatus: No tasks found to update for transformation %s" % transID )
continue
res = self.taskManager.getSubmittedTaskStatus( res['Value'] )
if not res['OK']:
gLogger.error( "updateTaskStatus: Failed to get updated task statuses for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
statusDict = res['Value']
for status in sortList( statusDict.keys() ):
taskIDs = statusDict[status]
gLogger.info( "updateTaskStatus: Updating %d task(s) from transformation %d to %s" % ( len( taskIDs ),
transID, status ) )
res = self.transClient.setTaskStatus( transID, taskIDs, status )
if not res['OK']:
gLogger.error( "updateTaskStatus: Failed to update task status for transformation", "%s %s" % ( transID,
res['Message'] ) )
gLogger.info( "updateTaskStatus: Transformation task status update complete" )
return S_OK()
def updateFileStatus( self ):
''' Update the files status
'''
gLogger.info( "updateFileStatus: Updating Status of task files" )
#Get the transformations to be updated
status = self.am_getOption( 'UpdateFilesStatus', ['Active', 'Completing', 'Stopped'] )
res = self._selectTransformations( transType = self.transType, status = status, agentType = [] )
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
timeStamp = str( datetime.datetime.utcnow() - datetime.timedelta( minutes = 10 ) )
condDict = {'TransformationID' : transID, 'Status' : ['Assigned']}
res = self.transClient.getTransformationFiles( condDict = condDict, older = timeStamp, timeStamp = 'LastUpdate' )
if not res['OK']:
gLogger.error( "updateFileStatus: Failed to get transformation files to update.", res['Message'] )
continue
if not res['Value']:
gLogger.info( "updateFileStatus: No files to be updated for transformation %s." % transID )
continue
res = self.taskManager.getSubmittedFileStatus( res['Value'] )
if not res['OK']:
gLogger.error( "updateFileStatus: Failed to get updated file statuses for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
statusDict = res['Value']
if not statusDict:
gLogger.info( "updateFileStatus: No file statuses to be updated for transformation %s." % transID )
continue
fileReport = FileReport( server = self.transClient.getServer() )
for lfn, status in statusDict.items():
fileReport.setFileStatus( int( transID ), lfn, status )
res = fileReport.commit()
if not res['OK']:
gLogger.error( "updateFileStatus: Failed to update file status for transformation", "%s %s" % ( transID,
res['Message'] ) )
else:
for status, update in res['Value'].items():
gLogger.info( "updateFileStatus: Updated %s files for %s to %s." % ( update, transID, status ) )
gLogger.info( "updateFileStatus: Transformation file status update complete" )
return S_OK()
def checkReservedTasks( self ):
gLogger.info( "checkReservedTasks: Checking Reserved tasks" )
# Get the transformations which should be checked
status = self.am_getOption( 'CheckReservedStatus', ['Active', 'Completing', 'Stopped'] )
res = self._selectTransformations( transType = self.transType, status = status, agentType = [] )
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
# Select the tasks which have been in Reserved status for more than 1 hour for selected transformations
condDict = {"TransformationID":transID, "ExternalStatus":'Reserved'}
time_stamp_older = str( datetime.datetime.utcnow() - datetime.timedelta( hours = 1 ) )
time_stamp_newer = str( datetime.datetime.utcnow() - datetime.timedelta( days = 7 ) )
res = self.transClient.getTransformationTasks( condDict = condDict, older = time_stamp_older,
newer = time_stamp_newer )
if not res['OK']:
gLogger.error( "checkReservedTasks: Failed to get Reserved tasks for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
if not res['Value']:
gLogger.verbose( "checkReservedTasks: No Reserved tasks found for transformation %s" % transID )
continue
res = self.taskManager.updateTransformationReservedTasks( res['Value'] )
if not res['OK']:
gLogger.info( "checkReservedTasks: No Reserved tasks found for transformation %s" % transID )
continue
noTasks = res['Value']['NoTasks']
taskNameIDs = res['Value']['TaskNameIDs']
# For the tasks with no associated request found re-set the status of the task in the transformationDB
for taskName in noTasks:
transID, taskID = taskName.split( '_' )
gLogger.info( "checkReservedTasks: Resetting status of %s to Created as no associated task found" % ( taskName ) )
res = self.transClient.setTaskStatus( int( transID ), int( taskID ), 'Created' )
if not res['OK']:
gLogger.warn( "checkReservedTasks: Failed to update task status and ID after recovery", "%s %s" % ( taskName,
res['Message'] ) )
# For the tasks for which an associated request was found update the task details in the transformationDB
for taskName, extTaskID in taskNameIDs.items():
transID, taskID = taskName.split( '_' )
gLogger.info( "checkReservedTasks: Resetting status of %s to Created with ID %s" % ( taskName, extTaskID ) )
res = self.transClient.setTaskStatusAndWmsID( int( transID ), int( taskID ), 'Submitted', str( extTaskID ) )
if not res['OK']:
gLogger.warn( "checkReservedTasks: Failed to update task status and ID after recovery", "%s %s" % ( taskName,
res['Message'] ) )
gLogger.info( "checkReservedTasks: Updating of reserved tasks complete" )
return S_OK()
def submitTasks( self ):
""" Submit the tasks to an external system, using the taskManager provided
"""
gLogger.info( "submitTasks: Submitting tasks for transformations" )
res = getProxyInfo( False, False )
if not res['OK']:
gLogger.error( "submitTasks: Failed to determine credentials for submission", res['Message'] )
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
gLogger.info( "submitTasks: Tasks will be submitted with the credentials %s:%s" % ( owner, ownerGroup ) )
# Get the transformations which should be submitted
tasksPerLoop = self.am_getOption( 'TasksPerLoop', 50 )
status = self.am_getOption( 'SubmitStatus', ['Active', 'Completing'] )
res = self._selectTransformations( transType = self.transType, status = status )
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
transBody = transformation['Body']
res = self.transClient.getTasksToSubmit( transID, tasksPerLoop )
if not res['OK']:
gLogger.error( "submitTasks: Failed to obtain tasks for transformation", "%s %s" % ( transID, res['Message'] ) )
continue
tasks = res['Value']['JobDictionary']
if not tasks:
gLogger.verbose( "submitTasks: No tasks found for submission for transformation %s" % transID )
continue
gLogger.info( "submitTasks: Obtained %d tasks for submission for transformation %s" % ( len( tasks ), transID ) )
res = self.taskManager.prepareTransformationTasks( transBody, tasks, owner, ownerGroup )
if not res['OK']:
gLogger.error( "submitTasks: Failed to prepare tasks for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
res = self.taskManager.submitTransformationTasks( res['Value'] )
if not res['OK']:
gLogger.error( "submitTasks: Failed to submit prepared tasks for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
res = self.taskManager.updateDBAfterTaskSubmission( res['Value'] )
if not res['OK']:
gLogger.error( "submitTasks: Failed to update DB after task submission for transformation", "%s %s" % ( transID,
res['Message'] ) )
continue
gLogger.info( "submitTasks: Submission of transformation tasks complete" )
return S_OK()
