def initializeHandler(cls, serviceInfoDict): csPath = serviceInfoDict['serviceSectionPath'] cls.bundleManager = BundleManager(csPath) updateBundleTime = gConfig.getValue("%s/BundlesLifeTime" % csPath, 3600 * 6) gLogger.info("Bundles will be updated each %s secs" % updateBundleTime) gThreadScheduler.addPeriodicTask(updateBundleTime, cls.bundleManager.updateBundles) return S_OK()
def __init__( self, monitor = None ): self.__idMap = {} self.__execTypes = {} self.__executorsLock = threading.Lock() self.__tasksLock = threading.Lock() self.__freezerLock = threading.Lock() self.__tasks = {} self.__log = gLogger.getSubLogger( "ExecMind" ) self.__taskFreezer = [] self.__queues = ExecutorQueues( self.__log ) self.__states = ExecutorState( self.__log ) self.__cbHolder = ExecutorDispatcherCallbacks() self.__monitor = monitor gThreadScheduler.addPeriodicTask( 60, self.__doPeriodicStuff ) #If a task is frozen too many times, send error or forget task? self.__failedOnTooFrozen = True #If a task fails to properly dispatch, freeze or forget task? self.__freezeOnFailedDispatch = True #If a task needs to go to an executor that has not connected. Freeze or forget the task? self.__freezeOnUnknownExecutor = True if self.__monitor: self.__monitor.registerActivity( "executors", "Executor reactors connected", "Executors", "executors", self.__monitor.OP_MEAN, 300 ) self.__monitor.registerActivity( "tasks", "Tasks processed", "Executors", "tasks", self.__monitor.OP_RATE, 300 ) self.__monitor.registerActivity( "taskTime", "Task processing time", "Executors", "seconds", self.__monitor.OP_MEAN, 300 )
def __init__(self, monitor=None): """ :param monitor: good question.... what's meant to be used for monitoring. Either a :py:class`DIRAC.FrameworkSystem.Client.MonitoringClient.MonitoringClient` or a :py:class`DIRAC.MonitoringSystem.Client.MonitoringReporter.MonitoringReporter` """ self.__idMap = {} self.__execTypes = {} self.__executorsLock = threading.Lock() self.__tasksLock = threading.Lock() self.__freezerLock = threading.Lock() self.__tasks = {} self.__log = gLogger.getSubLogger(self.__class__.__name__) self.__taskFreezer = [] self.__queues = ExecutorQueues(self.__log) self.__states = ExecutorState(self.__log) self.__cbHolder = ExecutorDispatcherCallbacks() self.__monitor = None if isinstance(monitor, MonitoringReporter): self.__monitoringReporter = monitor gThreadScheduler.addPeriodicTask(60, self.__doPeriodicStuff) # If a task is frozen too many times, send error or forget task? self.__failedOnTooFrozen = True # If a task fails to properly dispatch, freeze or forget task? self.__freezeOnFailedDispatch = True # If a task needs to go to an executor that has not connected. Freeze or forget the task? self.__freezeOnUnknownExecutor = True
def __init__(self, monitor=None): self.__idMap = {} self.__execTypes = {} self.__executorsLock = threading.Lock() self.__tasksLock = threading.Lock() self.__freezerLock = threading.Lock() self.__tasks = {} self.__log = gLogger.getSubLogger("ExecMind") self.__taskFreezer = [] self.__queues = ExecutorQueues(self.__log) self.__states = ExecutorState(self.__log) self.__cbHolder = ExecutorDispatcherCallbacks() self.__monitor = monitor gThreadScheduler.addPeriodicTask(60, self.__doPeriodicStuff) # If a task is frozen too many times, send error or forget task? self.__failedOnTooFrozen = True # If a task fails to properly dispatch, freeze or forget task? self.__freezeOnFailedDispatch = True # If a task needs to go to an executor that has not connected. Freeze or forget the task? self.__freezeOnUnknownExecutor = True if self.__monitor: self.__monitor.registerActivity("executors", "Executor reactors connected", "Executors", "executors", self.__monitor.OP_MEAN, 300) self.__monitor.registerActivity("tasks", "Tasks processed", "Executors", "tasks", self.__monitor.OP_RATE, 300) self.__monitor.registerActivity("taskTime", "Task processing time", "Executors", "seconds", self.__monitor.OP_MEAN, 300)
def initialize( self ): #Build the URLs self._url = self._cfg.getURL() if not self._url: return S_ERROR( "Could not build service URL for %s" % self._name ) gLogger.verbose( "Service URL is %s" % self._url ) #Load handler result = self._loadHandlerInit() if not result[ 'OK' ]: return result self._handler = result[ 'Value' ] #Initialize lock manager self._lockManager = LockManager( self._cfg.getMaxWaitingPetitions() ) self._initMonitoring() self._threadPool = ThreadPool( max( 1, self._cfg.getMinThreads() ), max( 0, self._cfg.getMaxThreads() ), self._cfg.getMaxWaitingPetitions() ) self._threadPool.daemonize() self._msgBroker = MessageBroker( "%sMSB" % self._name, threadPool = self._threadPool ) #Create static dict self._serviceInfoDict = { 'serviceName' : self._name, 'serviceSectionPath' : PathFinder.getServiceSection( self._name ), 'URL' : self._cfg.getURL(), 'messageSender' : MessageSender( self._name, self._msgBroker ), 'validNames' : self._validNames, 'csPaths' : [ PathFinder.getServiceSection( svcName ) for svcName in self._validNames ] } #Call static initialization function try: self._handler[ 'class' ]._rh__initializeClass( dict( self._serviceInfoDict ), self._lockManager, self._msgBroker, self._monitor ) if self._handler[ 'init' ]: for initFunc in self._handler[ 'init' ]: gLogger.verbose( "Executing initialization function" ) try: result = initFunc( dict( self._serviceInfoDict ) ) except Exception as excp: gLogger.exception( "Exception while calling initialization function", lException = excp ) return S_ERROR( "Exception while calling initialization function: %s" % str( excp ) ) if not isReturnStructure( result ): return S_ERROR( "Service initialization function %s must return S_OK/S_ERROR" % initFunc ) if not result[ 'OK' ]: return S_ERROR( "Error while initializing %s: %s" % ( self._name, result[ 'Message' ] ) ) except Exception as e: errMsg = "Exception while initializing %s" % self._name gLogger.exception( e ) gLogger.exception( errMsg ) return S_ERROR( errMsg ) #Load actions after the handler has initialized itself result = self._loadActions() if not result[ 'OK' ]: return result self._actions = result[ 'Value' ] gThreadScheduler.addPeriodicTask( 30, self.__reportThreadPoolContents ) return S_OK()
class ProxyManagerHandler(RequestHandler): __maxExtraLifeFactor = 1.5 __proxyDB = False @classmethod def initializeHandler(cls, serviceInfoDict): requireVoms = cls.srv_getCSOption("RequireVOMS", False) useMyProxy = cls.srv_getCSOption("UseMyProxy", False) try: cls.__proxyDB = ProxyDB(requireVoms=requireVoms, useMyProxy=useMyProxy) except RuntimeError, excp: return S_ERROR("Can't connect to ProxyDB: %s" % excp) gThreadScheduler.addPeriodicTask(900, cls.__proxyDB.purgeExpiredTokens, elapsedTime=900) gThreadScheduler.addPeriodicTask(900, cls.__proxyDB.purgeExpiredRequests, elapsedTime=900) gThreadScheduler.addPeriodicTask(3600, cls.__proxyDB.purgeLogs) gThreadScheduler.addPeriodicTask(3600, cls.__proxyDB.purgeExpiredProxies) gLogger.info( "VOMS: %s\nMyProxy: %s\n MyProxy Server: %s" % (requireVoms, useMyProxy, cls.__proxyDB.getMyProxyServer())) return S_OK()
def initializeMatcherHandler( serviceInfo ): """ Matcher Service initialization """ global gJobDB global gJobLoggingDB global gTaskQueueDB global gPilotAgentsDB gJobDB = JobDB() gJobLoggingDB = JobLoggingDB() gTaskQueueDB = TaskQueueDB() gPilotAgentsDB = PilotAgentsDB() gMonitor.registerActivity( 'matchTime', "Job matching time", 'Matching', "secs" , gMonitor.OP_MEAN, 300 ) gMonitor.registerActivity( 'matchesDone', "Job Match Request", 'Matching', "matches" , gMonitor.OP_RATE, 300 ) gMonitor.registerActivity( 'matchesOK', "Matched jobs", 'Matching', "matches" , gMonitor.OP_RATE, 300 ) gMonitor.registerActivity( 'numTQs', "Number of Task Queues", 'Matching', "tqsk queues" , gMonitor.OP_MEAN, 300 ) gTaskQueueDB.recalculateTQSharesForAll() gThreadScheduler.addPeriodicTask( 120, gTaskQueueDB.recalculateTQSharesForAll ) gThreadScheduler.addPeriodicTask( 60, sendNumTaskQueues ) sendNumTaskQueues() return S_OK()
def initializeMatcherHandler(serviceInfo): """ Matcher Service initialization """ global gJobDB global gTaskQueueDB global jlDB global pilotAgentsDB gJobDB = JobDB() gTaskQueueDB = TaskQueueDB() jlDB = JobLoggingDB() pilotAgentsDB = PilotAgentsDB() gMonitor.registerActivity('matchTime', "Job matching time", 'Matching', "secs", gMonitor.OP_MEAN, 300) gMonitor.registerActivity('matchesDone', "Job Match Request", 'Matching', "matches", gMonitor.OP_RATE, 300) gMonitor.registerActivity('matchesOK', "Matched jobs", 'Matching', "matches", gMonitor.OP_RATE, 300) gMonitor.registerActivity('numTQs', "Number of Task Queues", 'Matching', "tqsk queues", gMonitor.OP_MEAN, 300) gTaskQueueDB.recalculateTQSharesForAll() gThreadScheduler.addPeriodicTask(120, gTaskQueueDB.recalculateTQSharesForAll) gThreadScheduler.addPeriodicTask(60, sendNumTaskQueues) sendNumTaskQueues() return S_OK()
def initializeMatcherHandler(serviceInfo): """ Matcher Service initialization """ global jobDB global jobLoggingDB global taskQueueDB jobDB = JobDB() jobLoggingDB = JobLoggingDB() taskQueueDB = TaskQueueDB() gMonitor.registerActivity("matchTime", "Job matching time", "Matching", "secs", gMonitor.OP_MEAN, 300) gMonitor.registerActivity( "matchTaskQueues", "Task queues checked per job", "Matching", "task queues", gMonitor.OP_MEAN, 300 ) gMonitor.registerActivity("matchesDone", "Job Matches", "Matching", "matches", gMonitor.OP_MEAN, 300) gMonitor.registerActivity("numTQs", "Number of Task Queues", "Matching", "tqsk queues", gMonitor.OP_MEAN, 300) taskQueueDB.recalculateTQSharesForAll() gThreadScheduler.addPeriodicTask(120, taskQueueDB.recalculateTQSharesForAll) gThreadScheduler.addPeriodicTask(120, sendNumTaskQueues) sendNumTaskQueues() return S_OK()
def initialize(self): # Build the URLs self._url = self._cfg.getURL() if not self._url: return S_ERROR("Could not build service URL for %s" % self._name) gLogger.verbose("Service URL is %s" % self._url) # Load handler result = self._loadHandlerInit() if not result['OK']: return result self._handler = result['Value'] # Initialize lock manager self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions()) self._initMonitoring() self._threadPool = ThreadPool(max(1, self._cfg.getMinThreads()), max(0, self._cfg.getMaxThreads()), self._cfg.getMaxWaitingPetitions()) self._threadPool.daemonize() self._msgBroker = MessageBroker("%sMSB" % self._name, threadPool=self._threadPool) # Create static dict self._serviceInfoDict = {'serviceName': self._name, 'serviceSectionPath': PathFinder.getServiceSection(self._name), 'URL': self._cfg.getURL(), 'messageSender': MessageSender(self._name, self._msgBroker), 'validNames': self._validNames, 'csPaths': [PathFinder.getServiceSection(svcName) for svcName in self._validNames] } # Call static initialization function try: self._handler['class']._rh__initializeClass(dict(self._serviceInfoDict), self._lockManager, self._msgBroker, self._monitor) if self._handler['init']: for initFunc in self._handler['init']: gLogger.verbose("Executing initialization function") try: result = initFunc(dict(self._serviceInfoDict)) except Exception as excp: gLogger.exception("Exception while calling initialization function", lException=excp) return S_ERROR("Exception while calling initialization function: %s" % str(excp)) if not isReturnStructure(result): return S_ERROR("Service initialization function %s must return S_OK/S_ERROR" % initFunc) if not result['OK']: return S_ERROR("Error while initializing %s: %s" % (self._name, result['Message'])) except Exception as e: errMsg = "Exception while initializing %s" % self._name gLogger.exception(e) gLogger.exception(errMsg) return S_ERROR(errMsg) # Load actions after the handler has initialized itself result = self._loadActions() if not result['OK']: return result self._actions = result['Value'] gThreadScheduler.addPeriodicTask(30, self.__reportThreadPoolContents) return S_OK()
class FTSManagerHandler(RequestHandler): """ .. class:: FTSManagerHandler """ @classmethod def initializeHandler(self, serviceInfoDict): """ initialize handler """ try: self.ftsDB = FTSDB() except RuntimeError, error: gLogger.exception(error) return S_ERROR(error) self.ftsValidator = FTSValidator() # # create tables for empty db getTables = self.ftsDB.getTables() if not getTables['OK']: gLogger.error(getTables['Message']) return getTables getTables = getTables['Value'] toCreate = [ tab for tab in self.ftsDB.getTableMeta().keys() if tab not in getTables ] if toCreate: createTables = self.ftsDB.createTables(toCreate) if not createTables['OK']: gLogger.error(createTables['Message']) return createTables # # always re-create views createViews = self.ftsDB.createViews(True) if not createViews['OK']: return createViews # # connect connect = self.ftsDB._connect() if not connect['OK']: gLogger.error(connect['Message']) return connect # # get FTSStrategy self.ftsStrategy = self.getFtsStrategy() # # put DataManager proxy to env dmProxy = self.refreshProxy() if not dmProxy['OK']: return dmProxy # # every 10 minutes update RW access in FTSGraph gThreadScheduler.addPeriodicTask(600, self.updateRWAccess) # # every hour replace FTSGraph gThreadScheduler.addPeriodicTask(FTSHistoryView.INTERVAL, self.updateFTSStrategy) # # every 6 hours refresh DataManager proxy gThreadScheduler.addPeriodicTask(21600, self.refreshProxy) return S_OK()
def initializeReqProxyHandler(serviceInfo): """init RequestProxy handler :param serviceInfo: whatever """ gLogger.info("Initalizing ReqProxyHandler") gThreadScheduler.addPeriodicTask(120, ReqProxyHandler.sweeper) return S_OK()
def initializeNotificationHandler( cls, serviceInfo ): """ Handler initialization """ global gNotDB gNotDB = NotificationDB() gThreadScheduler.addPeriodicTask( 3600, gNotDB.purgeExpiredNotifications ) gThreadScheduler.addPeriodicTask( 3600, purgeDelayedEMails ) return S_OK()
def initializeNotificationHandler( cls, serviceInfo ): """ Handler initialization """ global gNotDB gNotDB = NotificationDB() gThreadScheduler.addPeriodicTask( 3600, gNotDB.purgeExpiredNotifications ) gThreadScheduler.addPeriodicTask( 3600, purgeDelayedEMails ) return S_OK()
def initializeHandler(cls, serviceInfo): """Handler initialization""" cls.notDB = NotificationDB() cls.mailCache = DictCache() gThreadScheduler.addPeriodicTask(3600, cls.notDB.purgeExpiredNotifications) gThreadScheduler.addPeriodicTask(3600, cls.mailCache.purgeExpired()) return S_OK()
def initializeRequestProxyHandler( serviceInfo ): """ init RequestProxy handler :param serviceInfo: whatever """ gLogger.info("Initalizing RequestProxyHandler") gThreadScheduler.addPeriodicTask( 120, RequestProxyHandler.sweeper ) return S_OK()
def initializeBundleDeliveryHandler( serviceInfoDict ): global gBundleManager csPath = serviceInfoDict[ 'serviceSectionPath' ] gBundleManager = BundleManager( csPath ) updateBundleTime = gConfig.getValue( "%s/BundlesLifeTime" % csPath, 3600 * 6 ) gLogger.info( "Bundles will be updated each %s secs" % updateBundleTime ) gThreadScheduler.addPeriodicTask( updateBundleTime, gBundleManager.updateBundles ) return S_OK()
def initializeVirtualMachineManagerHandler( serviceInfo ): global gVirtualMachineDB gVirtualMachineDB = VirtualMachineDB() gVirtualMachineDB.declareStalledInstances() if gVirtualMachineDB._connected: gThreadScheduler.addPeriodicTask( 60 * 15, gVirtualMachineDB.declareStalledInstances ) return S_OK() return S_ERROR()
def initializeDataStoreHandler(serviceInfo): global gAccountingDB gAccountingDB = AccountingDB() gAccountingDB.autoCompactDB() result = gAccountingDB.markAllPendingRecordsAsNotTaken() if not result['OK']: return result gThreadScheduler.addPeriodicTask(60, gAccountingDB.loadPendingRecords) return S_OK()
def initializeHandler( cls, svcInfoDict ): multiPath = PathFinder.getDatabaseSection( "Accounting/MultiDB" ) cls.__acDB = MultiAccountingDB( multiPath ) cls.__acDB.autoCompactDB() result = cls.__acDB.markAllPendingRecordsAsNotTaken() if not result[ 'OK' ]: return result gThreadScheduler.addPeriodicTask( 60, cls.__acDB.loadPendingRecords ) return S_OK()
def initializeDataStoreHandler( serviceInfo ): global gAccountingDB gAccountingDB = AccountingDB() gAccountingDB.autoCompactDB() result = gAccountingDB.markAllPendingRecordsAsNotTaken() if not result[ 'OK' ]: return result gThreadScheduler.addPeriodicTask( 60, gAccountingDB.loadPendingRecords ) return S_OK()
def initializeHandler(cls, serviceInfo): cls.virtualMachineDB = VirtualMachineDB() cls.haltStalledInstances() cls.checkStalledInstances() if cls.virtualMachineDB._connected: gThreadScheduler.addPeriodicTask(60 * 15, cls.checkStalledInstances) return S_OK() return S_ERROR()
def initializeVirtualMachineManagerHandler(serviceInfo): global gVirtualMachineDB gVirtualMachineDB = VirtualMachineDB() gVirtualMachineDB.declareStalledInstances() if gVirtualMachineDB._connected: gThreadScheduler.addPeriodicTask( 60 * 15, gVirtualMachineDB.declareStalledInstances) return S_OK() return S_ERROR()
def submitJob(self,executableFile,proxy,dummy=None): """ Method to submit job, should be overridden in sub-class. """ self.log.verbose('Setting up proxy for payload') result = self.writeProxyToFile(proxy) if not result['OK']: return result payloadProxy = result['Value'] if not os.environ.has_key('X509_USER_PROXY'): self.log.error('X509_USER_PROXY variable for pilot proxy not found in local environment') return S_ERROR('X509_USER_PROXY not found') pilotProxy = os.environ['X509_USER_PROXY'] self.log.info('Pilot proxy X509_USER_PROXY=%s' %pilotProxy) os.environ[ 'GLEXEC_CLIENT_CERT' ] = payloadProxy os.environ[ 'GLEXEC_SOURCE_PROXY' ] = payloadProxy self.log.info('Set payload proxy variables:\nGLEXEC_CLIENT_CERT=%s\nGLEXEC_SOURCE_PROXY=%s' %(payloadProxy,payloadProxy)) #Determine glexec location (default to standard InProcess behaviour if not found) glexecLocation = None result = self.glexecLocate() if result['OK']: glexecLocation = result['Value'] self.log.info('glexec found for local site at %s' %glexecLocation) if glexecLocation: result = self.recursivelyChangePermissions() if not result['OK']: self.log.error('Permissions change failed, continuing regardless...') else: self.log.info('glexec not found, no permissions to change') #Test glexec with payload proxy prior to submitting the job result = self.glexecTest(glexecLocation) if not result['OK']: self.analyseExitCode(result['Value']) #take no action as we currently default to InProcess self.log.info('glexec test failed, will submit payload regardless...') glexecLocation=None #Revert to InProcess behaviour if not glexecLocation: self.log.info('glexec is not found, setting X509_USER_PROXY for payload proxy') os.environ[ 'X509_USER_PROXY' ] = payloadProxy self.log.verbose('Starting process for monitoring payload proxy') gThreadScheduler.addPeriodicTask(self.proxyCheckPeriod,self.monitorProxy,taskArgs=(glexecLocation,pilotProxy,payloadProxy),executions=0,elapsedTime=0) #Submit job self.log.info('Changing permissions of executable to 0755') try: os.chmod(os.path.abspath(executableFile),0755) except Exception,x: self.log.error('Failed to change permissions of executable to 0755 with exception:\n%s' %(x))
def initializeHandler(cls, svcInfoDict): multiPath = PathFinder.getDatabaseSection("Accounting/MultiDB") cls.__acDB = MultiAccountingDB(multiPath) # we can run multiple services in read only mode. In that case we do not bucket cls.runBucketing = getServiceOption(svcInfoDict, "RunBucketing", True) if cls.runBucketing: cls.__acDB.autoCompactDB() # pylint: disable=no-member result = cls.__acDB.markAllPendingRecordsAsNotTaken() # pylint: disable=no-member if not result["OK"]: return result gThreadScheduler.addPeriodicTask(60, cls.__acDB.loadPendingRecords) # pylint: disable=no-member return S_OK()
def initializeBigDataJobManagerHandler( _serviceInfo ): global gBigDataDB gBigDataDB = BigDataDB() gBigDataDB.declareStalledJobs() if gBigDataDB._connected: gThreadScheduler.addPeriodicTask( 60 * 30, gBigDataDB.declareStalledJobs ) return S_OK() return S_ERROR()
def initializeHandler( cls, svcInfoDict ): multiPath = PathFinder.getDatabaseSection( "Accounting/MultiDB" ) cls.__acDB = MultiAccountingDB( multiPath ) #we can run multiple services in read only mode. In that case we do not bucket cls.runBucketing = getServiceOption( svcInfoDict, 'RunBucketing', True ) if cls.runBucketing: cls.__acDB.autoCompactDB() #pylint: disable=no-member result = cls.__acDB.markAllPendingRecordsAsNotTaken() #pylint: disable=no-member if not result[ 'OK' ]: return result gThreadScheduler.addPeriodicTask( 60, cls.__acDB.loadPendingRecords ) #pylint: disable=no-member return S_OK()
def submitJob(self,executableFile,proxy, dummy=None): """ Method to submit job, should be overridden in sub-class. """ # FIXME: when not running on a WN this will not work ret = getProxyInfo() if not ret['OK']: pilotProxy = None else: pilotProxy = ret['Value']['path'] print 'pilotProxy', pilotProxy payloadEnv = dict( os.environ ) payloadProxy = '' if proxy: self.log.verbose('Setting up proxy for payload') result = self.writeProxyToFile(proxy) if not result['OK']: return result payloadProxy = result['Value'] # pilotProxy = os.environ['X509_USER_PROXY'] payloadEnv[ 'X509_USER_PROXY' ] = payloadProxy self.log.verbose('Starting process for monitoring payload proxy') gThreadScheduler.addPeriodicTask(self.proxyCheckPeriod,self.monitorProxy,taskArgs=(pilotProxy,payloadProxy),executions=0,elapsedTime=0) if not os.access(executableFile, 5): os.chmod(executableFile,0755) cmd = os.path.abspath(executableFile) self.log.verbose('CE submission command: %s' %(cmd)) result = systemCall(0,cmd,callbackFunction = self.sendOutput,env=payloadEnv) if payloadProxy: os.unlink(payloadProxy) ret = S_OK() if not result['OK']: self.log.error('Fail to run InProcess',result['Message']) elif result['Value'][0] < 0: self.log.error('InProcess Job Execution Failed') self.log.info('Exit status:',result['Value'][0]) return S_ERROR('InProcess Job Execution Failed') elif result['Value'][0] > 0: self.log.error('Fail in payload execution') self.log.info('Exit status:',result['Value'][0]) ret['PayloadFailed'] = result['Value'][0] else: self.log.debug('InProcess CE result OK') self.submittedJobs += 1 return ret
def initializeHandler( cls, serviceInfo ): """ Handler class initialization """ # Check the flag for monitoring of the state of the host hostMonitoring = cls.srv_getCSOption( 'HostMonitoring', True ) if hostMonitoring: client = SystemAdministratorClient( 'localhost' ) gThreadScheduler.addPeriodicTask( 60, client.storeHostInfo ) return S_OK( 'Initialization went well' )
def initializeHandler( cls, serviceInfo ): """ Handler class initialization """ # Check the flag for monitoring of the state of the host hostMonitoring = cls.srv_getCSOption( 'HostMonitoring', True ) if hostMonitoring: gThreadScheduler.addPeriodicTask( 60, cls.__storeHostInfo ) #the SystemAdministrator service does not has to use the client to report data about the host. return S_OK( 'Initialization went well' )
def initializeHandler(cls, serviceInfo): """ Handler class initialization """ # Check the flag for monitoring of the state of the host hostMonitoring = cls.srv_getCSOption('HostMonitoring', True) if hostMonitoring: gThreadScheduler.addPeriodicTask(60, cls.__storeHostInfo) #the SystemAdministrator service does not has to use the client to report data about the host. return S_OK('Initialization went well')
def initializeHandler(cls, serviceInfoDict): gLogger.notice("Initializing Executor dispatcher") cls.__eDispatch = ExecutorDispatcher(cls.srv_getMonitor()) cls.__callbacks = ExecutorMindHandler.MindCallbacks( cls.__sendTask, cls.exec_dispatch, cls.__execDisconnected, cls.exec_taskProcessed, cls.exec_taskFreeze, cls.exec_taskError) cls.__eDispatch.setCallbacks(cls.__callbacks) cls.__allowedClients = [] if cls.log.shown("VERBOSE"): gThreadScheduler.setMinValidPeriod(1) gThreadScheduler.addPeriodicTask( 10, lambda: cls.log.verbose( "== Internal state ==\n%s\n===========" % pprint.pformat( cls.__eDispatch._internals())))
def initializeHandler( cls, serviceInfoDict ): gLogger.notice( "Initializing Executor dispatcher" ) cls.__eDispatch = ExecutorDispatcher( cls.srv_getMonitor() ) cls.__callbacks = ExecutorMindHandler.MindCallbacks( cls.__sendTask, cls.exec_dispatch, cls.__execDisconnected, cls.exec_taskProcessed, cls.exec_taskFreeze, cls.exec_taskError ) cls.__eDispatch.setCallbacks( cls.__callbacks ) cls.__allowedClients = [] if cls.log.shown( "VERBOSE" ): gThreadScheduler.setMinValidPeriod( 1 ) gThreadScheduler.addPeriodicTask( 10, lambda: cls.log.verbose( "== Internal state ==\n%s\n===========" % pprint.pformat( cls.__eDispatch._internals() ) ) )
def initializeHandler(cls, serviceInfoDict): useMyProxy = cls.srv_getCSOption("UseMyProxy", False) try: result = ObjectLoader().loadObject('FrameworkSystem.DB.ProxyDB', 'ProxyDB') if not result['OK']: gLogger.error('Failed to load ProxyDB class: %s' % result['Message']) return result dbClass = result['Value'] cls.__proxyDB = dbClass(useMyProxy=useMyProxy) except RuntimeError as excp: return S_ERROR("Can't connect to ProxyDB: %s" % excp) gThreadScheduler.addPeriodicTask(900, cls.__proxyDB.purgeExpiredTokens, elapsedTime=900) gThreadScheduler.addPeriodicTask(900, cls.__proxyDB.purgeExpiredRequests, elapsedTime=900) gThreadScheduler.addPeriodicTask(21600, cls.__proxyDB.purgeLogs) gThreadScheduler.addPeriodicTask(3600, cls.__proxyDB.purgeExpiredProxies) gLogger.info("MyProxy: %s\n MyProxy Server: %s" % (useMyProxy, cls.__proxyDB.getMyProxyServer())) return S_OK()
def __init__(self, basePath, daysToLog=100): self.__basePath = basePath self.__messagesQueue = Queue.Queue() self.__requiredFields = ('timestamp', 'success', 'sourceIP', 'sourcePort', 'sourceIdentity', 'destinationIP', 'destinationPort', 'destinationService', 'action') threading.Thread.__init__(self) self.__secsToLog = daysToLog * 86400 gThreadScheduler.addPeriodicTask(86400, self.__launchCleaningOldLogFiles, elapsedTime=(time.time() % 86400) + 3600) self.setDaemon(True) self.start()
def initializeProxyManagerHandler( serviceInfo ): global gProxyDB serviceCS = serviceInfo [ 'serviceSectionPath' ] requireVoms = gConfig.getValue( "%s/RequireVOMS" % serviceCS, "no" ).lower() in ( "yes", "y", "1" ) useMyProxy = gConfig.getValue( "%s/UseMyProxy" % serviceCS, "no" ).lower() in ( "yes", "y", "1" ) try: gProxyDB = ProxyDB( requireVoms = requireVoms, useMyProxy = useMyProxy ) except: return S_ERROR( "Can't initialize ProxyDB" ) gThreadScheduler.addPeriodicTask( 900, gProxyDB.purgeExpiredTokens, elapsedTime = 900 ) gThreadScheduler.addPeriodicTask( 900, gProxyDB.purgeExpiredRequests, elapsedTime = 900 ) gLogger.info( "VOMS: %s\nMyProxy: %s\n MyProxy Server: %s" % ( requireVoms, useMyProxy, gProxyDB.getMyProxyServer() ) ) return S_OK()
def initializeMatcherHandler(serviceInfo): """ Matcher Service initialization """ global gJobDB global gJobLoggingDB global gTaskQueueDB global gPilotAgentsDB # Create JobDB object and initialize its tables. gJobDB = JobDB() res = gJobDB._checkTable() if not res['OK']: return res # Create JobLoggingDB object and initialize its tables. gJobLoggingDB = JobLoggingDB() res = gJobLoggingDB._checkTable() if not res['OK']: return res gTaskQueueDB = TaskQueueDB() # Create PilotAgentsDB object and initialize its tables. gPilotAgentsDB = PilotAgentsDB() res = gPilotAgentsDB._checkTable() if not res['OK']: return res gMonitor.registerActivity('matchTime', "Job matching time", 'Matching', "secs", gMonitor.OP_MEAN, 300) gMonitor.registerActivity('matchesDone', "Job Match Request", 'Matching', "matches", gMonitor.OP_RATE, 300) gMonitor.registerActivity('matchesOK', "Matched jobs", 'Matching', "matches", gMonitor.OP_RATE, 300) gMonitor.registerActivity('numTQs', "Number of Task Queues", 'Matching', "tqsk queues", gMonitor.OP_MEAN, 300) gTaskQueueDB.recalculateTQSharesForAll() gThreadScheduler.addPeriodicTask(120, gTaskQueueDB.recalculateTQSharesForAll) gThreadScheduler.addPeriodicTask(60, sendNumTaskQueues) sendNumTaskQueues() return S_OK()
def __initializeMonitor(self): """ Initialize the system monitoring. """ # This flag is used to activate ES based monitoring if self.activityMonitoring: self.log.debug("Monitoring of the agent is enabled.") # The import needs to be here because of the CS must be initialized before importing # this class (see https://github.com/DIRACGrid/DIRAC/issues/4793) from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter self.activityMonitoringReporter = MonitoringReporter( monitoringType="AgentMonitoring") # With the help of this periodic task we commit the data to ES at an interval of 100 seconds. gThreadScheduler.addPeriodicTask( 100, self.__activityMonitoringReporting) self.__monitorLastStatsUpdate = time.time()
def initializeHandler(cls, serviceInfoDict): """ Initialization of DB objects and OptimizationMind """ cls.jobDB = JobDB() cls.jobLoggingDB = JobLoggingDB() cls.taskQueueDB = TaskQueueDB() cls.pilotAgentsDB = PilotAgentsDB() cls.pilotsLoggingDB = None enablePilotsLogging = Operations().getValue( '/Services/JobMonitoring/usePilotsLoggingFlag', False) if enablePilotsLogging: cls.pilotsLoggingDB = PilotsLoggingDB() cls.msgClient = MessageClient("WorkloadManagement/OptimizationMind") cls.__connectToOptMind() gThreadScheduler.addPeriodicTask(60, cls.__connectToOptMind) return S_OK()
def __init__( self, basePath, daysToLog = 100 ): self.__basePath = basePath self.__messagesQueue = Queue.Queue() self.__requiredFields = ( 'timestamp', 'success', 'sourceIP', 'sourcePort', 'sourceIdentity', 'destinationIP', 'destinationPort', 'destinationService', 'action' ) threading.Thread.__init__( self ) self.__secsToLog = daysToLog * 86400 gThreadScheduler.addPeriodicTask( 86400, self.__launchCleaningOldLogFiles, elapsedTime = ( time.time() % 86400 ) + 3600 ) self.setDaemon( True ) self.start()
def initializeProxyManagerHandler(serviceInfo): global gProxyDB serviceCS = serviceInfo['serviceSectionPath'] requireVoms = gConfig.getValue("%s/RequireVOMS" % serviceCS, "no").lower() in ("yes", "y", "1") useMyProxy = gConfig.getValue("%s/UseMyProxy" % serviceCS, "no").lower() in ("yes", "y", "1") try: gProxyDB = ProxyDB(requireVoms=requireVoms, useMyProxy=useMyProxy) except: return S_ERROR("Can't initialize ProxyDB") gThreadScheduler.addPeriodicTask(900, gProxyDB.purgeExpiredTokens, elapsedTime=900) gThreadScheduler.addPeriodicTask(900, gProxyDB.purgeExpiredRequests, elapsedTime=900) gLogger.info("VOMS: %s\nMyProxy: %s\n MyProxy Server: %s" % (requireVoms, useMyProxy, gProxyDB.getMyProxyServer())) return S_OK()
def initializeHandler(cls, serviceInfo): """ Handler class initialization """ # Check the flag for monitoring of the state of the host hostMonitoring = cls.srv_getCSOption('HostMonitoring', True) if hostMonitoring: gThreadScheduler.addPeriodicTask(60, cls.__storeHostInfo) # the SystemAdministrator service does not has to use the client to report data about the host. keepSoftwareVersions = cls.srv_getCSOption('KeepSoftwareVersions', 0) if keepSoftwareVersions > 0: gLogger.info("The last %s software version will be kept and the rest will be deleted!" % keepSoftwareVersions) gThreadScheduler.addPeriodicTask(600, cls.__deleteOldSoftware, (keepSoftwareVersions, ), executions=2) # it is enough to try 2 times return S_OK('Initialization went well')
def initializeHandler( cls, serviceInfo ): """ Handler class initialization """ # Check the flag for monitoring of the state of the host hostMonitoring = cls.srv_getCSOption( 'HostMonitoring', True ) if hostMonitoring: gThreadScheduler.addPeriodicTask( 60, cls.__storeHostInfo ) #the SystemAdministrator service does not has to use the client to report data about the host. # Check the flag for dynamic monitoring dynamicMonitoring = cls.srv_getCSOption( 'DynamicMonitoring', False ) if dynamicMonitoring: global gMonitoringReporter gMonitoringReporter = MonitoringReporter( monitoringType = "ComponentMonitoring" ) gThreadScheduler.addPeriodicTask( 120, cls.__storeProfiling ) return S_OK( 'Initialization went well' )
def __initializeMonitor(self): """ Initialize the system monitoring. """ # This flag is used to activate ES based monitoring # if the "EnableActivityMonitoring" flag in "yes" or "true" in the cfg file. self.activityMonitoring = ( Operations().getValue("EnableActivityMonitoring", False) or self.am_getOption("EnableActivityMonitoring", False)) if self.activityMonitoring: # The import needs to be here because of the CS must be initialized before importing # this class (see https://github.com/DIRACGrid/DIRAC/issues/4793) from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter self.activityMonitoringReporter = MonitoringReporter( monitoringType="ComponentMonitoring") # With the help of this periodic task we commit the data to ES at an interval of 100 seconds. gThreadScheduler.addPeriodicTask( 100, self.__activityMonitoringReporting) else: if self.__moduleProperties['standalone']: self.monitor = gMonitor else: self.monitor = MonitoringClient() self.monitor.setComponentType(self.monitor.COMPONENT_AGENT) self.monitor.setComponentName(self.__moduleProperties['fullName']) self.monitor.initialize() self.monitor.registerActivity('CPU', "CPU Usage", 'Framework', "CPU,%", self.monitor.OP_MEAN, 600) self.monitor.registerActivity('MEM', "Memory Usage", 'Framework', 'Memory,MB', self.monitor.OP_MEAN, 600) # Component monitor for field in ('version', 'DIRACVersion', 'description', 'platform'): self.monitor.setComponentExtraParam( field, self.__codeProperties[field]) self.monitor.setComponentExtraParam('startTime', Time.dateTime()) self.monitor.setComponentExtraParam('cycles', 0) self.monitor.disable() self.__monitorLastStatsUpdate = time.time()
def initializeHandler(cls, serviceInfo): """ Handler class initialization """ # Check the flag for monitoring of the state of the host hostMonitoring = cls.srv_getCSOption('HostMonitoring', True) if hostMonitoring: gThreadScheduler.addPeriodicTask(60, cls.__storeHostInfo) # the SystemAdministrator service does not has to use the client to report data about the host. # Check the flag for dynamic monitoring dynamicMonitoring = cls.srv_getCSOption('DynamicMonitoring', False) messageQueue = cls.srv_getCSOption('MessageQueue', 'dirac.componentmonitoring') if dynamicMonitoring: global gMonitoringReporter gMonitoringReporter = MonitoringReporter( monitoringType="ComponentMonitoring", failoverQueueName=messageQueue) gThreadScheduler.addPeriodicTask(120, cls.__storeProfiling) keepSoftwareVersions = cls.srv_getCSOption('KeepSoftwareVersions', 0) if keepSoftwareVersions > 0: gLogger.info("The last %s software version will be kept and the rest will be deleted!" % keepSoftwareVersions) gThreadScheduler.addPeriodicTask(600, cls.__deleteOldSoftware, (keepSoftwareVersions, ), executions=2) # it is enough to try 2 times return S_OK('Initialization went well')
def initializeHandler(cls, serviceInfo): """ Handler class initialization """ # Check the flag for monitoring of the state of the host hostMonitoring = cls.srv_getCSOption('HostMonitoring', True) if hostMonitoring: gThreadScheduler.addPeriodicTask(60, cls.__storeHostInfo) # the SystemAdministrator service does not has to use the client to report data about the host. # Check the flag for dynamic monitoring dynamicMonitoring = cls.srv_getCSOption('DynamicMonitoring', False) messageQueue = cls.srv_getCSOption('MessageQueue', 'dirac.componentmonitoring') if dynamicMonitoring: global gMonitoringReporter gMonitoringReporter = MonitoringReporter( monitoringType="ComponentMonitoring", failoverQueueName=messageQueue) gThreadScheduler.addPeriodicTask(120, cls.__storeProfiling) keepSoftwareVersions = cls.srv_getCSOption('KeepSoftwareVersions', 0) if keepSoftwareVersions > 0: gLogger.info( "The last %s software version will be kept and the rest will be deleted!" % keepSoftwareVersions) gThreadScheduler.addPeriodicTask( 600, cls.__deleteOldSoftware, (keepSoftwareVersions, ), executions=2) # it is enough to try 2 times return S_OK('Initialization went well')
def __init__( self, logger = False ): if logger: self.log = logger else: self.log = gLogger self.__modLock = threading.Lock() self.__transports = {} self.__listenPersistConn = False self.__msgCounter = 0 result = gThreadScheduler.addPeriodicTask( 5, self.__sendKeepAlives ) if not result[ 'OK' ]: self.log.fatal( "Cannot add task to thread scheduler", result[ 'Message' ] ) self.__keepAlivesTask = result[ 'Value' ]
def __init__( self, logger = False ): if logger: self.log = logger else: self.log = gLogger self.__modLock = threading.Lock() self.__transports = {} self.__listenPersistConn = False self.__msgCounter = 0 result = gThreadScheduler.addPeriodicTask( 5, self.__sendKeepAlives ) if not result[ 'OK' ]: self.log.fatal( "Cannot add task to thread scheduler", result[ 'Message' ] ) self.__keepAlivesTask = result[ 'Value' ]
def __init__(self): self.__messagesList = [] self.__maxMessagesInBundle = 1000 self.__maxMessagesWaiting = 10000 self.__taskId = gThreadScheduler.addPeriodicTask(30, self.__sendData)
def setLogStore(self, logStore): while self.__securityLogStore: self.__securityLogStore.pop() self.__securityLogStore.append(logStore) gThreadScheduler.addPeriodicTask(10, self.__sendData, executions=1)
def initializeNotificationHandler( serviceInfo ): global gNotDB gNotDB = NotificationDB() gThreadScheduler.addPeriodicTask( 3600, gNotDB.purgeExpiredNotifications() ) return S_OK()
def submitJob( self, executableFile, proxy, **kwargs ): """ Method to submit job (overriding base method). :param executableFile: file to execute via systemCall. Normally the JobWrapperTemplate when invoked by the JobAgent. :type executableFile: string :param proxy: the proxy used for running the job (the payload). It will be dumped to a file. :type proxy: string """ ret = getProxyInfo() if not ret['OK']: pilotProxy = None else: pilotProxy = ret['Value']['path'] self.log.notice( 'Pilot Proxy:', pilotProxy ) payloadEnv = dict( os.environ ) payloadProxy = '' renewTask = None if proxy: self.log.verbose( 'Setting up proxy for payload' ) result = self.writeProxyToFile( proxy ) if not result['OK']: return result payloadProxy = result['Value'] # proxy file location # pilotProxy = os.environ['X509_USER_PROXY'] payloadEnv[ 'X509_USER_PROXY' ] = payloadProxy self.log.verbose( 'Starting process for monitoring payload proxy' ) result = gThreadScheduler.addPeriodicTask( self.proxyCheckPeriod, self.monitorProxy, taskArgs = ( pilotProxy, payloadProxy ), executions = 0, elapsedTime = 0 ) if result[ 'OK' ]: renewTask = result[ 'Value' ] if not os.access( executableFile, 5 ): os.chmod( executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH ) cmd = os.path.abspath( executableFile ) self.log.verbose( 'CE submission command: %s' % ( cmd ) ) result = systemCall( 0, cmd, callbackFunction = self.sendOutput, env = payloadEnv ) if payloadProxy: os.unlink( payloadProxy ) if renewTask: gThreadScheduler.removeTask( renewTask ) ret = S_OK() if not result['OK']: self.log.error( 'Fail to run InProcess', result['Message'] ) elif result['Value'][0] > 128: # negative exit values are returned as 256 - exit self.log.warn( 'InProcess Job Execution Failed' ) self.log.info( 'Exit status:', result['Value'][0] - 256 ) if result['Value'][0] - 256 == -2: error = 'JobWrapper initialization error' elif result['Value'][0] - 256 == -1: error = 'JobWrapper execution error' else: error = 'InProcess Job Execution Failed' res = S_ERROR( error ) res['Value'] = result['Value'][0] - 256 return res elif result['Value'][0] > 0: self.log.warn( 'Fail in payload execution' ) self.log.info( 'Exit status:', result['Value'][0] ) ret['PayloadFailed'] = result['Value'][0] else: self.log.debug( 'InProcess CE result OK' ) self.submittedJobs += 1 return ret
def submitJob( self, executableFile, proxyObj, jobData ): """ Method to submit job """ self.log.info( "Executable file is %s" % executableFile ) self.__proxyObj = proxyObj self.__execFile = executableFile self.__jobData = jobData self.log.verbose( 'Setting up proxy for payload' ) result = self.writeProxyToFile( self.__proxyObj ) if not result['OK']: return result self.__payloadProxyLocation = result['Value'] glEnabled = True glOK = True if gConfig.getValue( "/DIRAC/Security/UseServerCertificate", False ): self.log.info( "Running with a certificate. Avoid using glexec" ) glEnabled = False else: result = ProxyInfo.getProxyInfo( self.__pilotProxyLocation, disableVOMS = True ) if result[ 'OK' ]: if not Properties.GENERIC_PILOT in result[ 'Value' ].get( 'groupProperties', [] ): self.log.info( "Pilot is NOT running with a generic pilot. Skipping glexec" ) glEnabled = False else: self.log.info( "Pilot is generic. Trying glexec" ) if not glEnabled: self.log.notice( "glexec is not enabled ") else: self.log.info( "Trying glexec..." ) for step in ( self.__check_credentials, self.__locate_glexec, self.__prepare_glenv, self.__prepare_tmpdir, self.__test, self.__construct_payload ): self.log.info( "Running step %s" % step.__name__ ) result = step() if not result[ 'OK' ]: self.log.error( "Step %s failed: %s" % ( step.__name__, result[ 'Message' ] ) ) if self.ceParameters.get( "RescheduleOnError", False ): result = S_ERROR( 'glexec CE failed on step %s : %s' % ( step.__name__, result[ 'Message' ] ) ) result['ReschedulePayload'] = True return result glOK = False break if not glOK: self.log.notice( "glexec failed miserably... Running without it." ) self.log.verbose( 'Starting process for monitoring payload proxy' ) result = gThreadScheduler.addPeriodicTask( self.proxyCheckPeriod, self.monitorProxy, taskArgs = ( self.__pilotProxyLocation, self.__payloadProxyLocation ), executions = 0, elapsedTime = 0 ) if not result[ 'OK' ]: return S_ERROR( "Could not schedule monitor proxy task: %s" % result[ 'Message' ] ) pTask = result[ 'Value' ] if glEnabled and glOK: result = self.__execute( [ self.__glCommand ] ) else: result = self.__executeInProcess( executableFile ) gThreadScheduler.removeTask( pTask ) self.__cleanup() return result
def submitJob( self, executableFile, proxy, **kwargs ): """ Method to submit job, overridden from super-class. """ self.log.verbose( 'Setting up proxy for payload' ) result = self.writeProxyToFile( proxy ) if not result['OK']: return result payloadProxy = result['Value'] if not 'X509_USER_PROXY' in os.environ: self.log.error( 'X509_USER_PROXY variable for pilot proxy not found in local environment' ) return S_ERROR( DErrno.EPROXYFIND, "X509_USER_PROXY not found") pilotProxy = os.environ['X509_USER_PROXY'] self.log.info( 'Pilot proxy X509_USER_PROXY=%s' % pilotProxy ) # See if a fixed value has been given payloadUsername = self.ceParameters.get( 'PayloadUser' ) if payloadUsername: self.log.info( 'Payload username %s from PayloadUser in ceParameters' % payloadUsername ) else: # First username in the sequence to use when running payload job # If first is pltXXp00 then have pltXXp01, pltXXp02, ... try: baseUsername = self.ceParameters.get('BaseUsername') baseCounter = int( baseUsername[-2:] ) self.log.info( "Base username from BaseUsername in ceParameters : %s" % baseUsername ) except: baseUsername = os.environ['USER'] + '00p00' baseCounter = 0 self.log.info( 'Base username from $USER + 00p00 : %s' % baseUsername ) # Next one in the sequence payloadUsername = baseUsername[:-2] + ( '%02d' % (baseCounter + self.submittedJobs) ) self.log.info( 'Payload username set to %s using jobs counter' % payloadUsername ) try: payloadUID = pwd.getpwnam(payloadUsername).pw_uid payloadGID = pwd.getpwnam(payloadUsername).pw_gid except KeyError: error = S_ERROR( 'User "' + str(payloadUsername) + '" does not exist!' ) return error self.log.verbose( 'Starting process for monitoring payload proxy' ) gThreadScheduler.addPeriodicTask( self.proxyCheckPeriod, self.monitorProxy, taskArgs = ( pilotProxy, payloadProxy, payloadUsername, payloadUID, payloadGID ), executions = 0, elapsedTime = 0 ) # Submit job self.log.info( 'Changing permissions of executable (%s) to 0755' % executableFile ) try: os.chmod( os.path.abspath( executableFile ), stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH ) except OSError as x: self.log.error( 'Failed to change permissions of executable to 0755 with exception', '\n%s' % ( x ) ) result = self.sudoExecute( os.path.abspath( executableFile ), payloadProxy, payloadUsername, payloadUID, payloadGID ) if not result['OK']: self.log.error( 'Failed sudoExecute', result ) return result self.log.debug( 'Sudo CE result OK' ) self.submittedJobs += 1 return S_OK()
def submitJob( self, executableFile, proxy, dummy = None ): """ Method to submit job, should be overridden in sub-class. """ ret = getProxyInfo() if not ret['OK']: pilotProxy = None else: pilotProxy = ret['Value']['path'] self.log.notice( 'Pilot Proxy:', pilotProxy ) payloadEnv = dict( os.environ ) payloadProxy = '' if proxy: self.log.verbose( 'Setting up proxy for payload' ) result = self.writeProxyToFile( proxy ) if not result['OK']: return result payloadProxy = result['Value'] # pilotProxy = os.environ['X509_USER_PROXY'] payloadEnv[ 'X509_USER_PROXY' ] = payloadProxy self.log.verbose( 'Starting process for monitoring payload proxy' ) renewTask = None result = gThreadScheduler.addPeriodicTask( self.proxyCheckPeriod, self.monitorProxy, taskArgs = ( pilotProxy, payloadProxy ), executions = 0, elapsedTime = 0 ) if result[ 'OK' ]: renewTask = result[ 'Value' ] if not os.access( executableFile, 5 ): os.chmod( executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH ) cmd = os.path.abspath( executableFile ) self.log.verbose( 'CE submission command: %s' % ( cmd ) ) result = systemCall( 0, cmd, callbackFunction = self.sendOutput, env = payloadEnv ) if payloadProxy: os.unlink( payloadProxy ) if renewTask: gThreadScheduler.removeTask( renewTask ) ret = S_OK() if not result['OK']: self.log.error( 'Fail to run InProcess', result['Message'] ) elif result['Value'][0] > 128: # negative exit values are returned as 256 - exit self.log.warn( 'InProcess Job Execution Failed' ) self.log.info( 'Exit status:', result['Value'][0] - 256 ) if result['Value'][0] - 256 == -2: error = 'Error in the initialization of the DIRAC JobWrapper' elif result['Value'][0] - 256 == -1: error = 'Error in the execution of the DIRAC JobWrapper' else: error = 'InProcess Job Execution Failed' res = S_ERROR( error ) res['Value'] = result['Value'][0] - 256 return res elif result['Value'][0] > 0: self.log.warn( 'Fail in payload execution' ) self.log.info( 'Exit status:', result['Value'][0] ) ret['PayloadFailed'] = result['Value'][0] else: self.log.debug( 'InProcess CE result OK' ) self.submittedJobs += 1 return ret
def initializeHandler( cls, serviceInfoDict ): cls.msgClient = MessageClient( "WorkloadManagement/OptimizationMind" ) cls.__connectToOptMind() gThreadScheduler.addPeriodicTask( 60, cls.__connectToOptMind ) return S_OK()