def initializeHandler(cls, serviceInfoDict):
csPath = serviceInfoDict['serviceSectionPath']
cls.bundleManager = BundleManager(csPath)
updateBundleTime = gConfig.getValue("%s/BundlesLifeTime" % csPath, 3600 * 6)
gLogger.info("Bundles will be updated each %s secs" % updateBundleTime)
gThreadScheduler.addPeriodicTask(updateBundleTime, cls.bundleManager.updateBundles)
return S_OK()
def __init__( self, monitor = None ):
self.__idMap = {}
self.__execTypes = {}
self.__executorsLock = threading.Lock()
self.__tasksLock = threading.Lock()
self.__freezerLock = threading.Lock()
self.__tasks = {}
self.__log = gLogger.getSubLogger( "ExecMind" )
self.__taskFreezer = []
self.__queues = ExecutorQueues( self.__log )
self.__states = ExecutorState( self.__log )
self.__cbHolder = ExecutorDispatcherCallbacks()
self.__monitor = monitor
gThreadScheduler.addPeriodicTask( 60, self.__doPeriodicStuff )
#If a task is frozen too many times, send error or forget task?
self.__failedOnTooFrozen = True
#If a task fails to properly dispatch, freeze or forget task?
self.__freezeOnFailedDispatch = True
#If a task needs to go to an executor that has not connected. Freeze or forget the task?
self.__freezeOnUnknownExecutor = True
if self.__monitor:
self.__monitor.registerActivity( "executors", "Executor reactors connected",
"Executors", "executors", self.__monitor.OP_MEAN, 300 )
self.__monitor.registerActivity( "tasks", "Tasks processed",
"Executors", "tasks", self.__monitor.OP_RATE, 300 )
self.__monitor.registerActivity( "taskTime", "Task processing time",
"Executors", "seconds", self.__monitor.OP_MEAN, 300 )
def __init__(self, monitor=None):
"""
:param monitor: good question.... what's meant to be used for monitoring.
Either a :py:class`DIRAC.FrameworkSystem.Client.MonitoringClient.MonitoringClient` or a
:py:class`DIRAC.MonitoringSystem.Client.MonitoringReporter.MonitoringReporter`
"""
self.__idMap = {}
self.__execTypes = {}
self.__executorsLock = threading.Lock()
self.__tasksLock = threading.Lock()
self.__freezerLock = threading.Lock()
self.__tasks = {}
self.__log = gLogger.getSubLogger(self.__class__.__name__)
self.__taskFreezer = []
self.__queues = ExecutorQueues(self.__log)
self.__states = ExecutorState(self.__log)
self.__cbHolder = ExecutorDispatcherCallbacks()
self.__monitor = None
if isinstance(monitor, MonitoringReporter):
self.__monitoringReporter = monitor
gThreadScheduler.addPeriodicTask(60, self.__doPeriodicStuff)
# If a task is frozen too many times, send error or forget task?
self.__failedOnTooFrozen = True
# If a task fails to properly dispatch, freeze or forget task?
self.__freezeOnFailedDispatch = True
# If a task needs to go to an executor that has not connected. Freeze or forget the task?
self.__freezeOnUnknownExecutor = True
def __init__(self, monitor=None):
self.__idMap = {}
self.__execTypes = {}
self.__executorsLock = threading.Lock()
self.__tasksLock = threading.Lock()
self.__freezerLock = threading.Lock()
self.__tasks = {}
self.__log = gLogger.getSubLogger("ExecMind")
self.__taskFreezer = []
self.__queues = ExecutorQueues(self.__log)
self.__states = ExecutorState(self.__log)
self.__cbHolder = ExecutorDispatcherCallbacks()
self.__monitor = monitor
gThreadScheduler.addPeriodicTask(60, self.__doPeriodicStuff)
# If a task is frozen too many times, send error or forget task?
self.__failedOnTooFrozen = True
# If a task fails to properly dispatch, freeze or forget task?
self.__freezeOnFailedDispatch = True
# If a task needs to go to an executor that has not connected. Freeze or forget the task?
self.__freezeOnUnknownExecutor = True
if self.__monitor:
self.__monitor.registerActivity("executors",
"Executor reactors connected",
"Executors", "executors",
self.__monitor.OP_MEAN, 300)
self.__monitor.registerActivity("tasks", "Tasks processed",
"Executors", "tasks",
self.__monitor.OP_RATE, 300)
self.__monitor.registerActivity("taskTime", "Task processing time",
"Executors", "seconds",
self.__monitor.OP_MEAN, 300)
def initialize( self ):
#Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR( "Could not build service URL for %s" % self._name )
gLogger.verbose( "Service URL is %s" % self._url )
#Load handler
result = self._loadHandlerInit()
if not result[ 'OK' ]:
return result
self._handler = result[ 'Value' ]
#Initialize lock manager
self._lockManager = LockManager( self._cfg.getMaxWaitingPetitions() )
self._initMonitoring()
self._threadPool = ThreadPool( max( 1, self._cfg.getMinThreads() ),
max( 0, self._cfg.getMaxThreads() ),
self._cfg.getMaxWaitingPetitions() )
self._threadPool.daemonize()
self._msgBroker = MessageBroker( "%sMSB" % self._name, threadPool = self._threadPool )
#Create static dict
self._serviceInfoDict = { 'serviceName' : self._name,
'serviceSectionPath' : PathFinder.getServiceSection( self._name ),
'URL' : self._cfg.getURL(),
'messageSender' : MessageSender( self._name, self._msgBroker ),
'validNames' : self._validNames,
'csPaths' : [ PathFinder.getServiceSection( svcName ) for svcName in self._validNames ]
}
#Call static initialization function
try:
self._handler[ 'class' ]._rh__initializeClass( dict( self._serviceInfoDict ),
self._lockManager,
self._msgBroker,
self._monitor )
if self._handler[ 'init' ]:
for initFunc in self._handler[ 'init' ]:
gLogger.verbose( "Executing initialization function" )
try:
result = initFunc( dict( self._serviceInfoDict ) )
except Exception as excp:
gLogger.exception( "Exception while calling initialization function", lException = excp )
return S_ERROR( "Exception while calling initialization function: %s" % str( excp ) )
if not isReturnStructure( result ):
return S_ERROR( "Service initialization function %s must return S_OK/S_ERROR" % initFunc )
if not result[ 'OK' ]:
return S_ERROR( "Error while initializing %s: %s" % ( self._name, result[ 'Message' ] ) )
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception( e )
gLogger.exception( errMsg )
return S_ERROR( errMsg )
#Load actions after the handler has initialized itself
result = self._loadActions()
if not result[ 'OK' ]:
return result
self._actions = result[ 'Value' ]
gThreadScheduler.addPeriodicTask( 30, self.__reportThreadPoolContents )
return S_OK()
class ProxyManagerHandler(RequestHandler):
__maxExtraLifeFactor = 1.5
__proxyDB = False
@classmethod
def initializeHandler(cls, serviceInfoDict):
requireVoms = cls.srv_getCSOption("RequireVOMS", False)
useMyProxy = cls.srv_getCSOption("UseMyProxy", False)
try:
cls.__proxyDB = ProxyDB(requireVoms=requireVoms,
useMyProxy=useMyProxy)
except RuntimeError, excp:
return S_ERROR("Can't connect to ProxyDB: %s" % excp)
gThreadScheduler.addPeriodicTask(900,
cls.__proxyDB.purgeExpiredTokens,
elapsedTime=900)
gThreadScheduler.addPeriodicTask(900,
cls.__proxyDB.purgeExpiredRequests,
elapsedTime=900)
gThreadScheduler.addPeriodicTask(3600, cls.__proxyDB.purgeLogs)
gThreadScheduler.addPeriodicTask(3600,
cls.__proxyDB.purgeExpiredProxies)
gLogger.info(
"VOMS: %s\nMyProxy: %s\n MyProxy Server: %s" %
(requireVoms, useMyProxy, cls.__proxyDB.getMyProxyServer()))
return S_OK()
def initializeMatcherHandler( serviceInfo ):
""" Matcher Service initialization
"""
global gJobDB
global gJobLoggingDB
global gTaskQueueDB
global gPilotAgentsDB
gJobDB = JobDB()
gJobLoggingDB = JobLoggingDB()
gTaskQueueDB = TaskQueueDB()
gPilotAgentsDB = PilotAgentsDB()
gMonitor.registerActivity( 'matchTime', "Job matching time",
'Matching', "secs" , gMonitor.OP_MEAN, 300 )
gMonitor.registerActivity( 'matchesDone', "Job Match Request",
'Matching', "matches" , gMonitor.OP_RATE, 300 )
gMonitor.registerActivity( 'matchesOK', "Matched jobs",
'Matching', "matches" , gMonitor.OP_RATE, 300 )
gMonitor.registerActivity( 'numTQs', "Number of Task Queues",
'Matching', "tqsk queues" , gMonitor.OP_MEAN, 300 )
gTaskQueueDB.recalculateTQSharesForAll()
gThreadScheduler.addPeriodicTask( 120, gTaskQueueDB.recalculateTQSharesForAll )
gThreadScheduler.addPeriodicTask( 60, sendNumTaskQueues )
sendNumTaskQueues()
return S_OK()
def initializeMatcherHandler(serviceInfo):
""" Matcher Service initialization
"""
global gJobDB
global gTaskQueueDB
global jlDB
global pilotAgentsDB
gJobDB = JobDB()
gTaskQueueDB = TaskQueueDB()
jlDB = JobLoggingDB()
pilotAgentsDB = PilotAgentsDB()
gMonitor.registerActivity('matchTime', "Job matching time", 'Matching',
"secs", gMonitor.OP_MEAN, 300)
gMonitor.registerActivity('matchesDone', "Job Match Request", 'Matching',
"matches", gMonitor.OP_RATE, 300)
gMonitor.registerActivity('matchesOK', "Matched jobs", 'Matching',
"matches", gMonitor.OP_RATE, 300)
gMonitor.registerActivity('numTQs', "Number of Task Queues", 'Matching',
"tqsk queues", gMonitor.OP_MEAN, 300)
gTaskQueueDB.recalculateTQSharesForAll()
gThreadScheduler.addPeriodicTask(120,
gTaskQueueDB.recalculateTQSharesForAll)
gThreadScheduler.addPeriodicTask(60, sendNumTaskQueues)
sendNumTaskQueues()
return S_OK()
def initializeMatcherHandler(serviceInfo):
""" Matcher Service initialization
"""
global jobDB
global jobLoggingDB
global taskQueueDB
jobDB = JobDB()
jobLoggingDB = JobLoggingDB()
taskQueueDB = TaskQueueDB()
gMonitor.registerActivity("matchTime", "Job matching time", "Matching", "secs", gMonitor.OP_MEAN, 300)
gMonitor.registerActivity(
"matchTaskQueues", "Task queues checked per job", "Matching", "task queues", gMonitor.OP_MEAN, 300
)
gMonitor.registerActivity("matchesDone", "Job Matches", "Matching", "matches", gMonitor.OP_MEAN, 300)
gMonitor.registerActivity("numTQs", "Number of Task Queues", "Matching", "tqsk queues", gMonitor.OP_MEAN, 300)
taskQueueDB.recalculateTQSharesForAll()
gThreadScheduler.addPeriodicTask(120, taskQueueDB.recalculateTQSharesForAll)
gThreadScheduler.addPeriodicTask(120, sendNumTaskQueues)
sendNumTaskQueues()
return S_OK()
def initialize(self):
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % self._name)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result['OK']:
return result
self._handler = result['Value']
# Initialize lock manager
self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
self._initMonitoring()
self._threadPool = ThreadPool(max(1, self._cfg.getMinThreads()),
max(0, self._cfg.getMaxThreads()),
self._cfg.getMaxWaitingPetitions())
self._threadPool.daemonize()
self._msgBroker = MessageBroker("%sMSB" % self._name, threadPool=self._threadPool)
# Create static dict
self._serviceInfoDict = {'serviceName': self._name,
'serviceSectionPath': PathFinder.getServiceSection(self._name),
'URL': self._cfg.getURL(),
'messageSender': MessageSender(self._name, self._msgBroker),
'validNames': self._validNames,
'csPaths': [PathFinder.getServiceSection(svcName) for svcName in self._validNames]
}
# Call static initialization function
try:
self._handler['class']._rh__initializeClass(dict(self._serviceInfoDict),
self._lockManager,
self._msgBroker,
self._monitor)
if self._handler['init']:
for initFunc in self._handler['init']:
gLogger.verbose("Executing initialization function")
try:
result = initFunc(dict(self._serviceInfoDict))
except Exception as excp:
gLogger.exception("Exception while calling initialization function", lException=excp)
return S_ERROR("Exception while calling initialization function: %s" % str(excp))
if not isReturnStructure(result):
return S_ERROR("Service initialization function %s must return S_OK/S_ERROR" % initFunc)
if not result['OK']:
return S_ERROR("Error while initializing %s: %s" % (self._name, result['Message']))
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception(e)
gLogger.exception(errMsg)
return S_ERROR(errMsg)
# Load actions after the handler has initialized itself
result = self._loadActions()
if not result['OK']:
return result
self._actions = result['Value']
gThreadScheduler.addPeriodicTask(30, self.__reportThreadPoolContents)
return S_OK()
class FTSManagerHandler(RequestHandler):
"""
.. class:: FTSManagerHandler
"""
@classmethod
def initializeHandler(self, serviceInfoDict):
""" initialize handler """
try:
self.ftsDB = FTSDB()
except RuntimeError, error:
gLogger.exception(error)
return S_ERROR(error)
self.ftsValidator = FTSValidator()
# # create tables for empty db
getTables = self.ftsDB.getTables()
if not getTables['OK']:
gLogger.error(getTables['Message'])
return getTables
getTables = getTables['Value']
toCreate = [
tab for tab in self.ftsDB.getTableMeta().keys()
if tab not in getTables
]
if toCreate:
createTables = self.ftsDB.createTables(toCreate)
if not createTables['OK']:
gLogger.error(createTables['Message'])
return createTables
# # always re-create views
createViews = self.ftsDB.createViews(True)
if not createViews['OK']:
return createViews
# # connect
connect = self.ftsDB._connect()
if not connect['OK']:
gLogger.error(connect['Message'])
return connect
# # get FTSStrategy
self.ftsStrategy = self.getFtsStrategy()
# # put DataManager proxy to env
dmProxy = self.refreshProxy()
if not dmProxy['OK']:
return dmProxy
# # every 10 minutes update RW access in FTSGraph
gThreadScheduler.addPeriodicTask(600, self.updateRWAccess)
# # every hour replace FTSGraph
gThreadScheduler.addPeriodicTask(FTSHistoryView.INTERVAL,
self.updateFTSStrategy)
# # every 6 hours refresh DataManager proxy
gThreadScheduler.addPeriodicTask(21600, self.refreshProxy)
return S_OK()
def initializeReqProxyHandler(serviceInfo):
"""init RequestProxy handler
:param serviceInfo: whatever
"""
gLogger.info("Initalizing ReqProxyHandler")
gThreadScheduler.addPeriodicTask(120, ReqProxyHandler.sweeper)
return S_OK()
def initializeNotificationHandler( cls, serviceInfo ): """ Handler initialization """ global gNotDB gNotDB = NotificationDB() gThreadScheduler.addPeriodicTask( 3600, gNotDB.purgeExpiredNotifications ) gThreadScheduler.addPeriodicTask( 3600, purgeDelayedEMails ) return S_OK()
def initializeNotificationHandler( cls, serviceInfo ): """ Handler initialization """ global gNotDB gNotDB = NotificationDB() gThreadScheduler.addPeriodicTask( 3600, gNotDB.purgeExpiredNotifications ) gThreadScheduler.addPeriodicTask( 3600, purgeDelayedEMails ) return S_OK()
def initializeHandler(cls, serviceInfo):
"""Handler initialization"""
cls.notDB = NotificationDB()
cls.mailCache = DictCache()
gThreadScheduler.addPeriodicTask(3600,
cls.notDB.purgeExpiredNotifications)
gThreadScheduler.addPeriodicTask(3600, cls.mailCache.purgeExpired())
return S_OK()
def initializeRequestProxyHandler( serviceInfo ):
""" init RequestProxy handler
:param serviceInfo: whatever
"""
gLogger.info("Initalizing RequestProxyHandler")
gThreadScheduler.addPeriodicTask( 120, RequestProxyHandler.sweeper )
return S_OK()
def initializeBundleDeliveryHandler( serviceInfoDict ): global gBundleManager csPath = serviceInfoDict[ 'serviceSectionPath' ] gBundleManager = BundleManager( csPath ) updateBundleTime = gConfig.getValue( "%s/BundlesLifeTime" % csPath, 3600 * 6 ) gLogger.info( "Bundles will be updated each %s secs" % updateBundleTime ) gThreadScheduler.addPeriodicTask( updateBundleTime, gBundleManager.updateBundles ) return S_OK()
def initializeVirtualMachineManagerHandler( serviceInfo ):
global gVirtualMachineDB
gVirtualMachineDB = VirtualMachineDB()
gVirtualMachineDB.declareStalledInstances()
if gVirtualMachineDB._connected:
gThreadScheduler.addPeriodicTask( 60 * 15, gVirtualMachineDB.declareStalledInstances )
return S_OK()
return S_ERROR()
def initializeDataStoreHandler(serviceInfo):
global gAccountingDB
gAccountingDB = AccountingDB()
gAccountingDB.autoCompactDB()
result = gAccountingDB.markAllPendingRecordsAsNotTaken()
if not result['OK']:
return result
gThreadScheduler.addPeriodicTask(60, gAccountingDB.loadPendingRecords)
return S_OK()
def initializeHandler( cls, svcInfoDict ):
multiPath = PathFinder.getDatabaseSection( "Accounting/MultiDB" )
cls.__acDB = MultiAccountingDB( multiPath )
cls.__acDB.autoCompactDB()
result = cls.__acDB.markAllPendingRecordsAsNotTaken()
if not result[ 'OK' ]:
return result
gThreadScheduler.addPeriodicTask( 60, cls.__acDB.loadPendingRecords )
return S_OK()
def initializeDataStoreHandler( serviceInfo ):
global gAccountingDB
gAccountingDB = AccountingDB()
gAccountingDB.autoCompactDB()
result = gAccountingDB.markAllPendingRecordsAsNotTaken()
if not result[ 'OK' ]:
return result
gThreadScheduler.addPeriodicTask( 60, gAccountingDB.loadPendingRecords )
return S_OK()
def initializeHandler(cls, serviceInfo):
cls.virtualMachineDB = VirtualMachineDB()
cls.haltStalledInstances()
cls.checkStalledInstances()
if cls.virtualMachineDB._connected:
gThreadScheduler.addPeriodicTask(60 * 15, cls.checkStalledInstances)
return S_OK()
return S_ERROR()
def initializeVirtualMachineManagerHandler(serviceInfo):
global gVirtualMachineDB
gVirtualMachineDB = VirtualMachineDB()
gVirtualMachineDB.declareStalledInstances()
if gVirtualMachineDB._connected:
gThreadScheduler.addPeriodicTask(
60 * 15, gVirtualMachineDB.declareStalledInstances)
return S_OK()
return S_ERROR()
def submitJob(self,executableFile,proxy,dummy=None):
""" Method to submit job, should be overridden in sub-class.
"""
self.log.verbose('Setting up proxy for payload')
result = self.writeProxyToFile(proxy)
if not result['OK']:
return result
payloadProxy = result['Value']
if not os.environ.has_key('X509_USER_PROXY'):
self.log.error('X509_USER_PROXY variable for pilot proxy not found in local environment')
return S_ERROR('X509_USER_PROXY not found')
pilotProxy = os.environ['X509_USER_PROXY']
self.log.info('Pilot proxy X509_USER_PROXY=%s' %pilotProxy)
os.environ[ 'GLEXEC_CLIENT_CERT' ] = payloadProxy
os.environ[ 'GLEXEC_SOURCE_PROXY' ] = payloadProxy
self.log.info('Set payload proxy variables:\nGLEXEC_CLIENT_CERT=%s\nGLEXEC_SOURCE_PROXY=%s' %(payloadProxy,payloadProxy))
#Determine glexec location (default to standard InProcess behaviour if not found)
glexecLocation = None
result = self.glexecLocate()
if result['OK']:
glexecLocation = result['Value']
self.log.info('glexec found for local site at %s' %glexecLocation)
if glexecLocation:
result = self.recursivelyChangePermissions()
if not result['OK']:
self.log.error('Permissions change failed, continuing regardless...')
else:
self.log.info('glexec not found, no permissions to change')
#Test glexec with payload proxy prior to submitting the job
result = self.glexecTest(glexecLocation)
if not result['OK']:
self.analyseExitCode(result['Value']) #take no action as we currently default to InProcess
self.log.info('glexec test failed, will submit payload regardless...')
glexecLocation=None
#Revert to InProcess behaviour
if not glexecLocation:
self.log.info('glexec is not found, setting X509_USER_PROXY for payload proxy')
os.environ[ 'X509_USER_PROXY' ] = payloadProxy
self.log.verbose('Starting process for monitoring payload proxy')
gThreadScheduler.addPeriodicTask(self.proxyCheckPeriod,self.monitorProxy,taskArgs=(glexecLocation,pilotProxy,payloadProxy),executions=0,elapsedTime=0)
#Submit job
self.log.info('Changing permissions of executable to 0755')
try:
os.chmod(os.path.abspath(executableFile),0755)
except Exception,x:
self.log.error('Failed to change permissions of executable to 0755 with exception:\n%s' %(x))
def initializeHandler(cls, svcInfoDict):
multiPath = PathFinder.getDatabaseSection("Accounting/MultiDB")
cls.__acDB = MultiAccountingDB(multiPath)
# we can run multiple services in read only mode. In that case we do not bucket
cls.runBucketing = getServiceOption(svcInfoDict, "RunBucketing", True)
if cls.runBucketing:
cls.__acDB.autoCompactDB() # pylint: disable=no-member
result = cls.__acDB.markAllPendingRecordsAsNotTaken() # pylint: disable=no-member
if not result["OK"]:
return result
gThreadScheduler.addPeriodicTask(60, cls.__acDB.loadPendingRecords) # pylint: disable=no-member
return S_OK()
def initializeBigDataJobManagerHandler( _serviceInfo ):
global gBigDataDB
gBigDataDB = BigDataDB()
gBigDataDB.declareStalledJobs()
if gBigDataDB._connected:
gThreadScheduler.addPeriodicTask( 60 * 30, gBigDataDB.declareStalledJobs )
return S_OK()
return S_ERROR()
def initializeHandler( cls, svcInfoDict ):
multiPath = PathFinder.getDatabaseSection( "Accounting/MultiDB" )
cls.__acDB = MultiAccountingDB( multiPath )
#we can run multiple services in read only mode. In that case we do not bucket
cls.runBucketing = getServiceOption( svcInfoDict, 'RunBucketing', True )
if cls.runBucketing:
cls.__acDB.autoCompactDB() #pylint: disable=no-member
result = cls.__acDB.markAllPendingRecordsAsNotTaken() #pylint: disable=no-member
if not result[ 'OK' ]:
return result
gThreadScheduler.addPeriodicTask( 60, cls.__acDB.loadPendingRecords ) #pylint: disable=no-member
return S_OK()
def submitJob(self,executableFile,proxy, dummy=None):
""" Method to submit job, should be overridden in sub-class.
"""
# FIXME: when not running on a WN this will not work
ret = getProxyInfo()
if not ret['OK']:
pilotProxy = None
else:
pilotProxy = ret['Value']['path']
print 'pilotProxy', pilotProxy
payloadEnv = dict( os.environ )
payloadProxy = ''
if proxy:
self.log.verbose('Setting up proxy for payload')
result = self.writeProxyToFile(proxy)
if not result['OK']:
return result
payloadProxy = result['Value']
# pilotProxy = os.environ['X509_USER_PROXY']
payloadEnv[ 'X509_USER_PROXY' ] = payloadProxy
self.log.verbose('Starting process for monitoring payload proxy')
gThreadScheduler.addPeriodicTask(self.proxyCheckPeriod,self.monitorProxy,taskArgs=(pilotProxy,payloadProxy),executions=0,elapsedTime=0)
if not os.access(executableFile, 5):
os.chmod(executableFile,0755)
cmd = os.path.abspath(executableFile)
self.log.verbose('CE submission command: %s' %(cmd))
result = systemCall(0,cmd,callbackFunction = self.sendOutput,env=payloadEnv)
if payloadProxy:
os.unlink(payloadProxy)
ret = S_OK()
if not result['OK']:
self.log.error('Fail to run InProcess',result['Message'])
elif result['Value'][0] < 0:
self.log.error('InProcess Job Execution Failed')
self.log.info('Exit status:',result['Value'][0])
return S_ERROR('InProcess Job Execution Failed')
elif result['Value'][0] > 0:
self.log.error('Fail in payload execution')
self.log.info('Exit status:',result['Value'][0])
ret['PayloadFailed'] = result['Value'][0]
else:
self.log.debug('InProcess CE result OK')
self.submittedJobs += 1
return ret
def initializeHandler( cls, serviceInfo ):
"""
Handler class initialization
"""
# Check the flag for monitoring of the state of the host
hostMonitoring = cls.srv_getCSOption( 'HostMonitoring', True )
if hostMonitoring:
client = SystemAdministratorClient( 'localhost' )
gThreadScheduler.addPeriodicTask( 60, client.storeHostInfo )
return S_OK( 'Initialization went well' )
def initializeHandler( cls, serviceInfo ):
"""
Handler class initialization
"""
# Check the flag for monitoring of the state of the host
hostMonitoring = cls.srv_getCSOption( 'HostMonitoring', True )
if hostMonitoring:
gThreadScheduler.addPeriodicTask( 60, cls.__storeHostInfo )
#the SystemAdministrator service does not has to use the client to report data about the host.
return S_OK( 'Initialization went well' )
def initializeHandler(cls, serviceInfo):
"""
Handler class initialization
"""
# Check the flag for monitoring of the state of the host
hostMonitoring = cls.srv_getCSOption('HostMonitoring', True)
if hostMonitoring:
gThreadScheduler.addPeriodicTask(60, cls.__storeHostInfo)
#the SystemAdministrator service does not has to use the client to report data about the host.
return S_OK('Initialization went well')
def initializeHandler(cls, serviceInfoDict):
gLogger.notice("Initializing Executor dispatcher")
cls.__eDispatch = ExecutorDispatcher(cls.srv_getMonitor())
cls.__callbacks = ExecutorMindHandler.MindCallbacks(
cls.__sendTask, cls.exec_dispatch, cls.__execDisconnected,
cls.exec_taskProcessed, cls.exec_taskFreeze, cls.exec_taskError)
cls.__eDispatch.setCallbacks(cls.__callbacks)
cls.__allowedClients = []
if cls.log.shown("VERBOSE"):
gThreadScheduler.setMinValidPeriod(1)
gThreadScheduler.addPeriodicTask(
10, lambda: cls.log.verbose(
"== Internal state ==\n%s\n===========" % pprint.pformat(
cls.__eDispatch._internals())))
def initializeHandler( cls, serviceInfoDict ):
gLogger.notice( "Initializing Executor dispatcher" )
cls.__eDispatch = ExecutorDispatcher( cls.srv_getMonitor() )
cls.__callbacks = ExecutorMindHandler.MindCallbacks( cls.__sendTask,
cls.exec_dispatch,
cls.__execDisconnected,
cls.exec_taskProcessed,
cls.exec_taskFreeze,
cls.exec_taskError )
cls.__eDispatch.setCallbacks( cls.__callbacks )
cls.__allowedClients = []
if cls.log.shown( "VERBOSE" ):
gThreadScheduler.setMinValidPeriod( 1 )
gThreadScheduler.addPeriodicTask( 10, lambda: cls.log.verbose( "== Internal state ==\n%s\n===========" % pprint.pformat( cls.__eDispatch._internals() ) ) )
def initializeHandler(cls, serviceInfoDict):
useMyProxy = cls.srv_getCSOption("UseMyProxy", False)
try:
result = ObjectLoader().loadObject('FrameworkSystem.DB.ProxyDB',
'ProxyDB')
if not result['OK']:
gLogger.error('Failed to load ProxyDB class: %s' %
result['Message'])
return result
dbClass = result['Value']
cls.__proxyDB = dbClass(useMyProxy=useMyProxy)
except RuntimeError as excp:
return S_ERROR("Can't connect to ProxyDB: %s" % excp)
gThreadScheduler.addPeriodicTask(900,
cls.__proxyDB.purgeExpiredTokens,
elapsedTime=900)
gThreadScheduler.addPeriodicTask(900,
cls.__proxyDB.purgeExpiredRequests,
elapsedTime=900)
gThreadScheduler.addPeriodicTask(21600, cls.__proxyDB.purgeLogs)
gThreadScheduler.addPeriodicTask(3600,
cls.__proxyDB.purgeExpiredProxies)
gLogger.info("MyProxy: %s\n MyProxy Server: %s" %
(useMyProxy, cls.__proxyDB.getMyProxyServer()))
return S_OK()
def __init__(self, basePath, daysToLog=100):
self.__basePath = basePath
self.__messagesQueue = Queue.Queue()
self.__requiredFields = ('timestamp', 'success', 'sourceIP',
'sourcePort', 'sourceIdentity',
'destinationIP', 'destinationPort',
'destinationService', 'action')
threading.Thread.__init__(self)
self.__secsToLog = daysToLog * 86400
gThreadScheduler.addPeriodicTask(86400,
self.__launchCleaningOldLogFiles,
elapsedTime=(time.time() % 86400) +
3600)
self.setDaemon(True)
self.start()
def initializeProxyManagerHandler( serviceInfo ):
global gProxyDB
serviceCS = serviceInfo [ 'serviceSectionPath' ]
requireVoms = gConfig.getValue( "%s/RequireVOMS" % serviceCS, "no" ).lower() in ( "yes", "y", "1" )
useMyProxy = gConfig.getValue( "%s/UseMyProxy" % serviceCS, "no" ).lower() in ( "yes", "y", "1" )
try:
gProxyDB = ProxyDB( requireVoms = requireVoms,
useMyProxy = useMyProxy )
except:
return S_ERROR( "Can't initialize ProxyDB" )
gThreadScheduler.addPeriodicTask( 900, gProxyDB.purgeExpiredTokens, elapsedTime = 900 )
gThreadScheduler.addPeriodicTask( 900, gProxyDB.purgeExpiredRequests, elapsedTime = 900 )
gLogger.info( "VOMS: %s\nMyProxy: %s\n MyProxy Server: %s" % ( requireVoms, useMyProxy, gProxyDB.getMyProxyServer() ) )
return S_OK()
def initializeMatcherHandler(serviceInfo):
""" Matcher Service initialization
"""
global gJobDB
global gJobLoggingDB
global gTaskQueueDB
global gPilotAgentsDB
# Create JobDB object and initialize its tables.
gJobDB = JobDB()
res = gJobDB._checkTable()
if not res['OK']:
return res
# Create JobLoggingDB object and initialize its tables.
gJobLoggingDB = JobLoggingDB()
res = gJobLoggingDB._checkTable()
if not res['OK']:
return res
gTaskQueueDB = TaskQueueDB()
# Create PilotAgentsDB object and initialize its tables.
gPilotAgentsDB = PilotAgentsDB()
res = gPilotAgentsDB._checkTable()
if not res['OK']:
return res
gMonitor.registerActivity('matchTime', "Job matching time", 'Matching',
"secs", gMonitor.OP_MEAN, 300)
gMonitor.registerActivity('matchesDone', "Job Match Request", 'Matching',
"matches", gMonitor.OP_RATE, 300)
gMonitor.registerActivity('matchesOK', "Matched jobs", 'Matching',
"matches", gMonitor.OP_RATE, 300)
gMonitor.registerActivity('numTQs', "Number of Task Queues", 'Matching',
"tqsk queues", gMonitor.OP_MEAN, 300)
gTaskQueueDB.recalculateTQSharesForAll()
gThreadScheduler.addPeriodicTask(120,
gTaskQueueDB.recalculateTQSharesForAll)
gThreadScheduler.addPeriodicTask(60, sendNumTaskQueues)
sendNumTaskQueues()
return S_OK()
def __initializeMonitor(self):
"""
Initialize the system monitoring.
"""
# This flag is used to activate ES based monitoring
if self.activityMonitoring:
self.log.debug("Monitoring of the agent is enabled.")
# The import needs to be here because of the CS must be initialized before importing
# this class (see https://github.com/DIRACGrid/DIRAC/issues/4793)
from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter
self.activityMonitoringReporter = MonitoringReporter(
monitoringType="AgentMonitoring")
# With the help of this periodic task we commit the data to ES at an interval of 100 seconds.
gThreadScheduler.addPeriodicTask(
100, self.__activityMonitoringReporting)
self.__monitorLastStatsUpdate = time.time()
def initializeHandler(cls, serviceInfoDict):
""" Initialization of DB objects and OptimizationMind
"""
cls.jobDB = JobDB()
cls.jobLoggingDB = JobLoggingDB()
cls.taskQueueDB = TaskQueueDB()
cls.pilotAgentsDB = PilotAgentsDB()
cls.pilotsLoggingDB = None
enablePilotsLogging = Operations().getValue(
'/Services/JobMonitoring/usePilotsLoggingFlag', False)
if enablePilotsLogging:
cls.pilotsLoggingDB = PilotsLoggingDB()
cls.msgClient = MessageClient("WorkloadManagement/OptimizationMind")
cls.__connectToOptMind()
gThreadScheduler.addPeriodicTask(60, cls.__connectToOptMind)
return S_OK()
def __init__( self, basePath, daysToLog = 100 ):
self.__basePath = basePath
self.__messagesQueue = Queue.Queue()
self.__requiredFields = ( 'timestamp',
'success',
'sourceIP',
'sourcePort',
'sourceIdentity',
'destinationIP',
'destinationPort',
'destinationService',
'action' )
threading.Thread.__init__( self )
self.__secsToLog = daysToLog * 86400
gThreadScheduler.addPeriodicTask( 86400,
self.__launchCleaningOldLogFiles,
elapsedTime = ( time.time() % 86400 ) + 3600 )
self.setDaemon( True )
self.start()
def initializeProxyManagerHandler(serviceInfo):
global gProxyDB
serviceCS = serviceInfo['serviceSectionPath']
requireVoms = gConfig.getValue("%s/RequireVOMS" % serviceCS,
"no").lower() in ("yes", "y", "1")
useMyProxy = gConfig.getValue("%s/UseMyProxy" % serviceCS,
"no").lower() in ("yes", "y", "1")
try:
gProxyDB = ProxyDB(requireVoms=requireVoms, useMyProxy=useMyProxy)
except:
return S_ERROR("Can't initialize ProxyDB")
gThreadScheduler.addPeriodicTask(900,
gProxyDB.purgeExpiredTokens,
elapsedTime=900)
gThreadScheduler.addPeriodicTask(900,
gProxyDB.purgeExpiredRequests,
elapsedTime=900)
gLogger.info("VOMS: %s\nMyProxy: %s\n MyProxy Server: %s" %
(requireVoms, useMyProxy, gProxyDB.getMyProxyServer()))
return S_OK()
def initializeHandler(cls, serviceInfo):
"""
Handler class initialization
"""
# Check the flag for monitoring of the state of the host
hostMonitoring = cls.srv_getCSOption('HostMonitoring', True)
if hostMonitoring:
gThreadScheduler.addPeriodicTask(60, cls.__storeHostInfo)
# the SystemAdministrator service does not has to use the client to report data about the host.
keepSoftwareVersions = cls.srv_getCSOption('KeepSoftwareVersions', 0)
if keepSoftwareVersions > 0:
gLogger.info("The last %s software version will be kept and the rest will be deleted!" % keepSoftwareVersions)
gThreadScheduler.addPeriodicTask(600,
cls.__deleteOldSoftware,
(keepSoftwareVersions, ),
executions=2) # it is enough to try 2 times
return S_OK('Initialization went well')
def initializeHandler( cls, serviceInfo ):
"""
Handler class initialization
"""
# Check the flag for monitoring of the state of the host
hostMonitoring = cls.srv_getCSOption( 'HostMonitoring', True )
if hostMonitoring:
gThreadScheduler.addPeriodicTask( 60, cls.__storeHostInfo )
#the SystemAdministrator service does not has to use the client to report data about the host.
# Check the flag for dynamic monitoring
dynamicMonitoring = cls.srv_getCSOption( 'DynamicMonitoring', False )
if dynamicMonitoring:
global gMonitoringReporter
gMonitoringReporter = MonitoringReporter( monitoringType = "ComponentMonitoring" )
gThreadScheduler.addPeriodicTask( 120, cls.__storeProfiling )
return S_OK( 'Initialization went well' )
def __initializeMonitor(self):
"""
Initialize the system monitoring.
"""
# This flag is used to activate ES based monitoring
# if the "EnableActivityMonitoring" flag in "yes" or "true" in the cfg file.
self.activityMonitoring = (
Operations().getValue("EnableActivityMonitoring", False)
or self.am_getOption("EnableActivityMonitoring", False))
if self.activityMonitoring:
# The import needs to be here because of the CS must be initialized before importing
# this class (see https://github.com/DIRACGrid/DIRAC/issues/4793)
from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter
self.activityMonitoringReporter = MonitoringReporter(
monitoringType="ComponentMonitoring")
# With the help of this periodic task we commit the data to ES at an interval of 100 seconds.
gThreadScheduler.addPeriodicTask(
100, self.__activityMonitoringReporting)
else:
if self.__moduleProperties['standalone']:
self.monitor = gMonitor
else:
self.monitor = MonitoringClient()
self.monitor.setComponentType(self.monitor.COMPONENT_AGENT)
self.monitor.setComponentName(self.__moduleProperties['fullName'])
self.monitor.initialize()
self.monitor.registerActivity('CPU', "CPU Usage", 'Framework',
"CPU,%", self.monitor.OP_MEAN, 600)
self.monitor.registerActivity('MEM', "Memory Usage", 'Framework',
'Memory,MB', self.monitor.OP_MEAN,
600)
# Component monitor
for field in ('version', 'DIRACVersion', 'description',
'platform'):
self.monitor.setComponentExtraParam(
field, self.__codeProperties[field])
self.monitor.setComponentExtraParam('startTime', Time.dateTime())
self.monitor.setComponentExtraParam('cycles', 0)
self.monitor.disable()
self.__monitorLastStatsUpdate = time.time()
def initializeHandler(cls, serviceInfo):
"""
Handler class initialization
"""
# Check the flag for monitoring of the state of the host
hostMonitoring = cls.srv_getCSOption('HostMonitoring', True)
if hostMonitoring:
gThreadScheduler.addPeriodicTask(60, cls.__storeHostInfo)
# the SystemAdministrator service does not has to use the client to report data about the host.
# Check the flag for dynamic monitoring
dynamicMonitoring = cls.srv_getCSOption('DynamicMonitoring', False)
messageQueue = cls.srv_getCSOption('MessageQueue', 'dirac.componentmonitoring')
if dynamicMonitoring:
global gMonitoringReporter
gMonitoringReporter = MonitoringReporter(
monitoringType="ComponentMonitoring", failoverQueueName=messageQueue)
gThreadScheduler.addPeriodicTask(120, cls.__storeProfiling)
keepSoftwareVersions = cls.srv_getCSOption('KeepSoftwareVersions', 0)
if keepSoftwareVersions > 0:
gLogger.info("The last %s software version will be kept and the rest will be deleted!" % keepSoftwareVersions)
gThreadScheduler.addPeriodicTask(600,
cls.__deleteOldSoftware,
(keepSoftwareVersions, ),
executions=2) # it is enough to try 2 times
return S_OK('Initialization went well')
def initializeHandler(cls, serviceInfo):
"""
Handler class initialization
"""
# Check the flag for monitoring of the state of the host
hostMonitoring = cls.srv_getCSOption('HostMonitoring', True)
if hostMonitoring:
gThreadScheduler.addPeriodicTask(60, cls.__storeHostInfo)
# the SystemAdministrator service does not has to use the client to report data about the host.
# Check the flag for dynamic monitoring
dynamicMonitoring = cls.srv_getCSOption('DynamicMonitoring', False)
messageQueue = cls.srv_getCSOption('MessageQueue',
'dirac.componentmonitoring')
if dynamicMonitoring:
global gMonitoringReporter
gMonitoringReporter = MonitoringReporter(
monitoringType="ComponentMonitoring",
failoverQueueName=messageQueue)
gThreadScheduler.addPeriodicTask(120, cls.__storeProfiling)
keepSoftwareVersions = cls.srv_getCSOption('KeepSoftwareVersions', 0)
if keepSoftwareVersions > 0:
gLogger.info(
"The last %s software version will be kept and the rest will be deleted!"
% keepSoftwareVersions)
gThreadScheduler.addPeriodicTask(
600,
cls.__deleteOldSoftware, (keepSoftwareVersions, ),
executions=2) # it is enough to try 2 times
return S_OK('Initialization went well')
def __init__( self, logger = False ):
if logger:
self.log = logger
else:
self.log = gLogger
self.__modLock = threading.Lock()
self.__transports = {}
self.__listenPersistConn = False
self.__msgCounter = 0
result = gThreadScheduler.addPeriodicTask( 5, self.__sendKeepAlives )
if not result[ 'OK' ]:
self.log.fatal( "Cannot add task to thread scheduler", result[ 'Message' ] )
self.__keepAlivesTask = result[ 'Value' ]
def __init__( self, logger = False ):
if logger:
self.log = logger
else:
self.log = gLogger
self.__modLock = threading.Lock()
self.__transports = {}
self.__listenPersistConn = False
self.__msgCounter = 0
result = gThreadScheduler.addPeriodicTask( 5, self.__sendKeepAlives )
if not result[ 'OK' ]:
self.log.fatal( "Cannot add task to thread scheduler", result[ 'Message' ] )
self.__keepAlivesTask = result[ 'Value' ]
def __init__(self): self.__messagesList = [] self.__maxMessagesInBundle = 1000 self.__maxMessagesWaiting = 10000 self.__taskId = gThreadScheduler.addPeriodicTask(30, self.__sendData)
def setLogStore(self, logStore):
while self.__securityLogStore:
self.__securityLogStore.pop()
self.__securityLogStore.append(logStore)
gThreadScheduler.addPeriodicTask(10, self.__sendData, executions=1)
def initializeNotificationHandler( serviceInfo ): global gNotDB gNotDB = NotificationDB() gThreadScheduler.addPeriodicTask( 3600, gNotDB.purgeExpiredNotifications() ) return S_OK()
def submitJob( self, executableFile, proxy, **kwargs ):
""" Method to submit job (overriding base method).
:param executableFile: file to execute via systemCall. Normally the JobWrapperTemplate when invoked by the JobAgent.
:type executableFile: string
:param proxy: the proxy used for running the job (the payload). It will be dumped to a file.
:type proxy: string
"""
ret = getProxyInfo()
if not ret['OK']:
pilotProxy = None
else:
pilotProxy = ret['Value']['path']
self.log.notice( 'Pilot Proxy:', pilotProxy )
payloadEnv = dict( os.environ )
payloadProxy = ''
renewTask = None
if proxy:
self.log.verbose( 'Setting up proxy for payload' )
result = self.writeProxyToFile( proxy )
if not result['OK']:
return result
payloadProxy = result['Value'] # proxy file location
# pilotProxy = os.environ['X509_USER_PROXY']
payloadEnv[ 'X509_USER_PROXY' ] = payloadProxy
self.log.verbose( 'Starting process for monitoring payload proxy' )
result = gThreadScheduler.addPeriodicTask( self.proxyCheckPeriod, self.monitorProxy,
taskArgs = ( pilotProxy, payloadProxy ),
executions = 0, elapsedTime = 0 )
if result[ 'OK' ]:
renewTask = result[ 'Value' ]
if not os.access( executableFile, 5 ):
os.chmod( executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH )
cmd = os.path.abspath( executableFile )
self.log.verbose( 'CE submission command: %s' % ( cmd ) )
result = systemCall( 0, cmd, callbackFunction = self.sendOutput, env = payloadEnv )
if payloadProxy:
os.unlink( payloadProxy )
if renewTask:
gThreadScheduler.removeTask( renewTask )
ret = S_OK()
if not result['OK']:
self.log.error( 'Fail to run InProcess', result['Message'] )
elif result['Value'][0] > 128:
# negative exit values are returned as 256 - exit
self.log.warn( 'InProcess Job Execution Failed' )
self.log.info( 'Exit status:', result['Value'][0] - 256 )
if result['Value'][0] - 256 == -2:
error = 'JobWrapper initialization error'
elif result['Value'][0] - 256 == -1:
error = 'JobWrapper execution error'
else:
error = 'InProcess Job Execution Failed'
res = S_ERROR( error )
res['Value'] = result['Value'][0] - 256
return res
elif result['Value'][0] > 0:
self.log.warn( 'Fail in payload execution' )
self.log.info( 'Exit status:', result['Value'][0] )
ret['PayloadFailed'] = result['Value'][0]
else:
self.log.debug( 'InProcess CE result OK' )
self.submittedJobs += 1
return ret
def submitJob( self, executableFile, proxyObj, jobData ):
""" Method to submit job
"""
self.log.info( "Executable file is %s" % executableFile )
self.__proxyObj = proxyObj
self.__execFile = executableFile
self.__jobData = jobData
self.log.verbose( 'Setting up proxy for payload' )
result = self.writeProxyToFile( self.__proxyObj )
if not result['OK']:
return result
self.__payloadProxyLocation = result['Value']
glEnabled = True
glOK = True
if gConfig.getValue( "/DIRAC/Security/UseServerCertificate", False ):
self.log.info( "Running with a certificate. Avoid using glexec" )
glEnabled = False
else:
result = ProxyInfo.getProxyInfo( self.__pilotProxyLocation, disableVOMS = True )
if result[ 'OK' ]:
if not Properties.GENERIC_PILOT in result[ 'Value' ].get( 'groupProperties', [] ):
self.log.info( "Pilot is NOT running with a generic pilot. Skipping glexec" )
glEnabled = False
else:
self.log.info( "Pilot is generic. Trying glexec" )
if not glEnabled:
self.log.notice( "glexec is not enabled ")
else:
self.log.info( "Trying glexec..." )
for step in ( self.__check_credentials, self.__locate_glexec,
self.__prepare_glenv, self.__prepare_tmpdir,
self.__test, self.__construct_payload ):
self.log.info( "Running step %s" % step.__name__ )
result = step()
if not result[ 'OK' ]:
self.log.error( "Step %s failed: %s" % ( step.__name__, result[ 'Message' ] ) )
if self.ceParameters.get( "RescheduleOnError", False ):
result = S_ERROR( 'glexec CE failed on step %s : %s' % ( step.__name__, result[ 'Message' ] ) )
result['ReschedulePayload'] = True
return result
glOK = False
break
if not glOK:
self.log.notice( "glexec failed miserably... Running without it." )
self.log.verbose( 'Starting process for monitoring payload proxy' )
result = gThreadScheduler.addPeriodicTask( self.proxyCheckPeriod, self.monitorProxy,
taskArgs = ( self.__pilotProxyLocation, self.__payloadProxyLocation ),
executions = 0, elapsedTime = 0 )
if not result[ 'OK' ]:
return S_ERROR( "Could not schedule monitor proxy task: %s" % result[ 'Message' ] )
pTask = result[ 'Value' ]
if glEnabled and glOK:
result = self.__execute( [ self.__glCommand ] )
else:
result = self.__executeInProcess( executableFile )
gThreadScheduler.removeTask( pTask )
self.__cleanup()
return result
def submitJob( self, executableFile, proxy, **kwargs ):
""" Method to submit job, overridden from super-class.
"""
self.log.verbose( 'Setting up proxy for payload' )
result = self.writeProxyToFile( proxy )
if not result['OK']:
return result
payloadProxy = result['Value']
if not 'X509_USER_PROXY' in os.environ:
self.log.error( 'X509_USER_PROXY variable for pilot proxy not found in local environment' )
return S_ERROR( DErrno.EPROXYFIND, "X509_USER_PROXY not found")
pilotProxy = os.environ['X509_USER_PROXY']
self.log.info( 'Pilot proxy X509_USER_PROXY=%s' % pilotProxy )
# See if a fixed value has been given
payloadUsername = self.ceParameters.get( 'PayloadUser' )
if payloadUsername:
self.log.info( 'Payload username %s from PayloadUser in ceParameters' % payloadUsername )
else:
# First username in the sequence to use when running payload job
# If first is pltXXp00 then have pltXXp01, pltXXp02, ...
try:
baseUsername = self.ceParameters.get('BaseUsername')
baseCounter = int( baseUsername[-2:] )
self.log.info( "Base username from BaseUsername in ceParameters : %s" % baseUsername )
except:
baseUsername = os.environ['USER'] + '00p00'
baseCounter = 0
self.log.info( 'Base username from $USER + 00p00 : %s' % baseUsername )
# Next one in the sequence
payloadUsername = baseUsername[:-2] + ( '%02d' % (baseCounter + self.submittedJobs) )
self.log.info( 'Payload username set to %s using jobs counter' % payloadUsername )
try:
payloadUID = pwd.getpwnam(payloadUsername).pw_uid
payloadGID = pwd.getpwnam(payloadUsername).pw_gid
except KeyError:
error = S_ERROR( 'User "' + str(payloadUsername) + '" does not exist!' )
return error
self.log.verbose( 'Starting process for monitoring payload proxy' )
gThreadScheduler.addPeriodicTask( self.proxyCheckPeriod, self.monitorProxy,
taskArgs = ( pilotProxy, payloadProxy, payloadUsername, payloadUID, payloadGID ),
executions = 0, elapsedTime = 0 )
# Submit job
self.log.info( 'Changing permissions of executable (%s) to 0755' % executableFile )
try:
os.chmod( os.path.abspath( executableFile ), stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH )
except OSError as x:
self.log.error( 'Failed to change permissions of executable to 0755 with exception',
'\n%s' % ( x ) )
result = self.sudoExecute( os.path.abspath( executableFile ), payloadProxy, payloadUsername, payloadUID, payloadGID )
if not result['OK']:
self.log.error( 'Failed sudoExecute', result )
return result
self.log.debug( 'Sudo CE result OK' )
self.submittedJobs += 1
return S_OK()
def submitJob( self, executableFile, proxy, dummy = None ):
""" Method to submit job, should be overridden in sub-class.
"""
ret = getProxyInfo()
if not ret['OK']:
pilotProxy = None
else:
pilotProxy = ret['Value']['path']
self.log.notice( 'Pilot Proxy:', pilotProxy )
payloadEnv = dict( os.environ )
payloadProxy = ''
if proxy:
self.log.verbose( 'Setting up proxy for payload' )
result = self.writeProxyToFile( proxy )
if not result['OK']:
return result
payloadProxy = result['Value']
# pilotProxy = os.environ['X509_USER_PROXY']
payloadEnv[ 'X509_USER_PROXY' ] = payloadProxy
self.log.verbose( 'Starting process for monitoring payload proxy' )
renewTask = None
result = gThreadScheduler.addPeriodicTask( self.proxyCheckPeriod, self.monitorProxy, taskArgs = ( pilotProxy, payloadProxy ), executions = 0, elapsedTime = 0 )
if result[ 'OK' ]:
renewTask = result[ 'Value' ]
if not os.access( executableFile, 5 ):
os.chmod( executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH )
cmd = os.path.abspath( executableFile )
self.log.verbose( 'CE submission command: %s' % ( cmd ) )
result = systemCall( 0, cmd, callbackFunction = self.sendOutput, env = payloadEnv )
if payloadProxy:
os.unlink( payloadProxy )
if renewTask:
gThreadScheduler.removeTask( renewTask )
ret = S_OK()
if not result['OK']:
self.log.error( 'Fail to run InProcess', result['Message'] )
elif result['Value'][0] > 128:
# negative exit values are returned as 256 - exit
self.log.warn( 'InProcess Job Execution Failed' )
self.log.info( 'Exit status:', result['Value'][0] - 256 )
if result['Value'][0] - 256 == -2:
error = 'Error in the initialization of the DIRAC JobWrapper'
elif result['Value'][0] - 256 == -1:
error = 'Error in the execution of the DIRAC JobWrapper'
else:
error = 'InProcess Job Execution Failed'
res = S_ERROR( error )
res['Value'] = result['Value'][0] - 256
return res
elif result['Value'][0] > 0:
self.log.warn( 'Fail in payload execution' )
self.log.info( 'Exit status:', result['Value'][0] )
ret['PayloadFailed'] = result['Value'][0]
else:
self.log.debug( 'InProcess CE result OK' )
self.submittedJobs += 1
return ret
def initializeHandler( cls, serviceInfoDict ): cls.msgClient = MessageClient( "WorkloadManagement/OptimizationMind" ) cls.__connectToOptMind() gThreadScheduler.addPeriodicTask( 60, cls.__connectToOptMind ) return S_OK()