def initialize( self ): #Build the URLs self._url = self._cfg.getURL() if not self._url: return S_ERROR( "Could not build service URL for %s" % self._name ) gLogger.verbose( "Service URL is %s" % self._url ) #Load handler result = self._loadHandlerInit() if not result[ 'OK' ]: return result self._handler = result[ 'Value' ] #Initialize lock manager self._lockManager = LockManager( self._cfg.getMaxWaitingPetitions() ) self._initMonitoring() self._threadPool = ThreadPool( max( 1, self._cfg.getMinThreads() ), max( 0, self._cfg.getMaxThreads() ), self._cfg.getMaxWaitingPetitions() ) self._threadPool.daemonize() self._msgBroker = MessageBroker( "%sMSB" % self._name, threadPool = self._threadPool ) #Create static dict self._serviceInfoDict = { 'serviceName' : self._name, 'serviceSectionPath' : PathFinder.getServiceSection( self._name ), 'URL' : self._cfg.getURL(), 'messageSender' : MessageSender( self._name, self._msgBroker ), 'validNames' : self._validNames, 'csPaths' : [ PathFinder.getServiceSection( svcName ) for svcName in self._validNames ] } #Call static initialization function try: self._handler[ 'class' ]._rh__initializeClass( dict( self._serviceInfoDict ), self._lockManager, self._msgBroker, self._monitor ) if self._handler[ 'init' ]: for initFunc in self._handler[ 'init' ]: gLogger.verbose( "Executing initialization function" ) try: result = initFunc( dict( self._serviceInfoDict ) ) except Exception as excp: gLogger.exception( "Exception while calling initialization function", lException = excp ) return S_ERROR( "Exception while calling initialization function: %s" % str( excp ) ) if not isReturnStructure( result ): return S_ERROR( "Service initialization function %s must return S_OK/S_ERROR" % initFunc ) if not result[ 'OK' ]: return S_ERROR( "Error while initializing %s: %s" % ( self._name, result[ 'Message' ] ) ) except Exception as e: errMsg = "Exception while initializing %s" % self._name gLogger.exception( e ) gLogger.exception( errMsg ) return S_ERROR( errMsg ) #Load actions after the handler has initialized itself result = self._loadActions() if not result[ 'OK' ]: return result self._actions = result[ 'Value' ] gThreadScheduler.addPeriodicTask( 30, self.__reportThreadPoolContents ) return S_OK()
def initialize(self): # Build the URLs self._url = self._cfg.getURL() if not self._url: return S_ERROR("Could not build service URL for %s" % self._name) gLogger.verbose("Service URL is %s" % self._url) # Load handler result = self._loadHandlerInit() if not result['OK']: return result self._handler = result['Value'] # Initialize lock manager self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions()) self._initMonitoring() self._threadPool = ThreadPool(max(1, self._cfg.getMinThreads()), max(0, self._cfg.getMaxThreads()), self._cfg.getMaxWaitingPetitions()) self._threadPool.daemonize() self._msgBroker = MessageBroker("%sMSB" % self._name, threadPool=self._threadPool) # Create static dict self._serviceInfoDict = {'serviceName': self._name, 'serviceSectionPath': PathFinder.getServiceSection(self._name), 'URL': self._cfg.getURL(), 'messageSender': MessageSender(self._name, self._msgBroker), 'validNames': self._validNames, 'csPaths': [PathFinder.getServiceSection(svcName) for svcName in self._validNames] } # Call static initialization function try: self._handler['class']._rh__initializeClass(dict(self._serviceInfoDict), self._lockManager, self._msgBroker, self._monitor) if self._handler['init']: for initFunc in self._handler['init']: gLogger.verbose("Executing initialization function") try: result = initFunc(dict(self._serviceInfoDict)) except Exception as excp: gLogger.exception("Exception while calling initialization function", lException=excp) return S_ERROR("Exception while calling initialization function: %s" % str(excp)) if not isReturnStructure(result): return S_ERROR("Service initialization function %s must return S_OK/S_ERROR" % initFunc) if not result['OK']: return S_ERROR("Error while initializing %s: %s" % (self._name, result['Message'])) except Exception as e: errMsg = "Exception while initializing %s" % self._name gLogger.exception(e) gLogger.exception(errMsg) return S_ERROR(errMsg) # Load actions after the handler has initialized itself result = self._loadActions() if not result['OK']: return result self._actions = result['Value'] gThreadScheduler.addPeriodicTask(30, self.__reportThreadPoolContents) return S_OK()
def __initializeService(cls, relativeUrl, absoluteUrl): """ Initialize a service. The work is only perform once at the first request. :param relativeUrl: relative URL, e.g. ``/<System>/<Component>`` :param absoluteUrl: full URL e.g. ``https://<host>:<port>/<System>/<Component>`` :returns: S_OK """ # If the initialization was already done successfuly, # we can just return if cls.__init_done: return S_OK() # Otherwise, do the work but with a lock with cls.__init_lock: # Check again that the initialization was not done by another thread # while we were waiting for the lock if cls.__init_done: return S_OK() # Url starts with a "/", we just remove it serviceName = relativeUrl[1:] cls._startTime = datetime.utcnow() sLog.info("First use, initializing service...", "%s" % relativeUrl) cls._authManager = AuthManager( "%s/Authorization" % PathFinder.getServiceSection(serviceName)) cls._initMonitoring(serviceName, absoluteUrl) cls._serviceName = serviceName cls._validNames = [serviceName] serviceInfo = { 'serviceName': serviceName, 'serviceSectionPath': PathFinder.getServiceSection(serviceName), 'csPaths': [PathFinder.getServiceSection(serviceName)], 'URL': absoluteUrl } cls._serviceInfoDict = serviceInfo cls.__monitorLastStatsUpdate = time.time() cls.initializeHandler(serviceInfo) cls.__init_done = True return S_OK()
def __init__( self, nameList ): self.serviceName = nameList[0] self.serviceURL = False self.nameList = nameList self.pathList = [] for svcName in nameList: self.pathList.append( PathFinder.getServiceSection( svcName ) )
def initializePlottingHandler(serviceInfo): #Get data location plottingSection = PathFinder.getServiceSection("Framework/Plotting") dataPath = gConfig.getValue("%s/DataLocation" % plottingSection, "data/graphs") dataPath = dataPath.strip() if "/" != dataPath[0]: dataPath = os.path.realpath( "%s/%s" % (gConfig.getValue('/LocalSite/InstancePath', rootPath), dataPath)) gLogger.info("Data will be written into %s" % dataPath) try: os.makedirs(dataPath) except: pass try: testFile = "%s/plot__.test" % dataPath fd = file(testFile, "w") fd.close() os.unlink(testFile) except IOError: gLogger.fatal("Can't write to %s" % dataPath) return S_ERROR("Data location is not writable") gPlotCache.setPlotsLocation(dataPath) gMonitor.registerActivity("plotsDrawn", "Drawn plot images", "Plotting requests", "plots", gMonitor.OP_SUM) return S_OK()
def __init__(self, nameList): self.serviceName = nameList[0] self.serviceURL = None self.nameList = nameList self.pathList = [] for svcName in nameList: self.pathList.append(PathFinder.getServiceSection(svcName))
def export_sendMail( self, address, subject, body, fromAddress, avoidSpam = False ): """ Send an email with supplied body to the specified address using the Mail utility. if avoidSpam is True, then emails are first added to a set so that duplicates are removed, and sent every hour. """ gLogger.verbose( 'Received signal to send the following mail to %s:\nSubject = %s\n%s' % ( address, subject, body ) ) eMail = Mail() notificationSection = PathFinder.getServiceSection( "Framework/Notification" ) csSection = notificationSection + '/SMTP' eMail._smtpHost = gConfig.getValue( '%s/Host' % csSection ) eMail._smtpPort = gConfig.getValue( '%s/Port' % csSection ) eMail._smtpLogin = gConfig.getValue( '%s/Login' % csSection ) eMail._smtpPasswd = gConfig.getValue( '%s/Password' % csSection ) eMail._smtpPtcl = gConfig.getValue( '%s/Protocol' % csSection ) eMail._subject = subject eMail._message = body eMail._mailAddress = address if not fromAddress == 'None': eMail._fromAddress = fromAddress if gConfig.getValue( '%s/FromAddress' % csSection ): eMail._fromAddress = gConfig.getValue( '%s/FromAddress' % csSection ) if avoidSpam: gMailSet.add(eMail) return S_OK("Mail added to gMailSet") else: result = eMail._send() if not result['OK']: gLogger.warn( 'Could not send mail with the following message:\n%s' % result['Message'] ) else: gLogger.info( 'Mail sent successfully to %s with subject %s' % ( address, subject ) ) gLogger.debug( result['Value'] ) return result
def export_sendMail( self, address, subject, body, fromAddress, avoidSpam = False ): """ Send an email with supplied body to the specified address using the Mail utility. if avoidSpam is True, then emails are first added to a set so that duplicates are removed, and sent every hour. """ gLogger.verbose( 'Received signal to send the following mail to %s:\nSubject = %s\n%s' % ( address, subject, body ) ) eMail = Mail() notificationSection = PathFinder.getServiceSection( "Framework/Notification" ) csSection = notificationSection + '/SMTPServer' eMail._smtpHost = gConfig.getValue( '%s/Host' % csSection ) eMail._smtpPort = gConfig.getValue( '%s/Port' % csSection ) eMail._smtpLogin = gConfig.getValue( '%s/Login' % csSection ) eMail._smtpPasswd = gConfig.getValue( '%s/Password' % csSection ) eMail._smtpPtcl = gConfig.getValue( '%s/Protocol' % csSection ) eMail._subject = subject eMail._message = body eMail._mailAddress = address if not fromAddress == 'None': eMail._fromAddress = fromAddress if gConfig.getValue( '%s/FromAddress' % csSection ): eMail._fromAddress = gConfig.getValue( '%s/FromAddress' % csSection ) if avoidSpam: gMailSet.add(eMail) return S_OK("Mail added to gMailSet") else: result = eMail._send() if not result['OK']: gLogger.warn( 'Could not send mail with the following message:\n%s' % result['Message'] ) else: gLogger.info( 'Mail sent successfully to %s with subject %s' % ( address, subject ) ) gLogger.debug( result['Value'] ) return result
def initializePlottingHandler( serviceInfo ): #Get data location plottingSection = PathFinder.getServiceSection( "Framework/Plotting" ) dataPath = gConfig.getValue( "%s/DataLocation" % plottingSection, "data/graphs" ) dataPath = dataPath.strip() if "/" != dataPath[0]: dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) ) gLogger.info( "Data will be written into %s" % dataPath ) try: os.makedirs( dataPath ) except: pass try: testFile = "%s/plot__.test" % dataPath fd = file( testFile, "w" ) fd.close() os.unlink( testFile ) except IOError: gLogger.fatal( "Can't write to %s" % dataPath ) return S_ERROR( "Data location is not writable" ) gPlotCache.setPlotsLocation( dataPath ) gMonitor.registerActivity( "plotsDrawn", "Drawn plot images", "Plotting requests", "plots", gMonitor.OP_SUM ) return S_OK()
def initializeDataLoggingHandler(serviceInfo): global dataPath global logDB logDB = DataLoggingDB() monitoringSection = PathFinder.getServiceSection( "DataManagement/DataLogging") #Get data location retDict = gConfig.getOption("%s/DataLocation" % monitoringSection, "dataLoggingPlots") if not retDict['OK']: return retDict dataPath = retDict['Value'].strip() if "/" != dataPath[0]: dataPath = os.path.realpath( "%s/%s" % (gConfig.getValue('/LocalSite/InstancePath', rootPath), dataPath)) gLogger.info("Data will be written into %s" % dataPath) try: os.makedirs(dataPath) except: pass try: testFile = "%s/mon.jarl.test" % dataPath fd = file(testFile, "w") fd.close() os.unlink(testFile) except IOError: gLogger.fatal("Can't write to %s" % dataPath) return S_ERROR("Data location is not writable") return S_OK()
def initializeReportGeneratorHandler( serviceInfo ): global gAccountingDB gAccountingDB = AccountingDB( readOnly = True ) #Get data location reportSection = PathFinder.getServiceSection( "Accounting/ReportGenerator" ) dataPath = gConfig.getValue( "%s/DataLocation" % reportSection, "data/accountingGraphs" ) dataPath = dataPath.strip() if "/" != dataPath[0]: dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) ) gLogger.info( "Data will be written into %s" % dataPath ) try: os.makedirs( dataPath ) except: pass try: testFile = "%s/acc.jarl.test" % dataPath fd = file( testFile, "w" ) fd.close() os.unlink( testFile ) except IOError: gLogger.fatal( "Can't write to %s" % dataPath ) return S_ERROR( "Data location is not writable" ) gDataCache.setGraphsLocation( dataPath ) gMonitor.registerActivity( "plotsDrawn", "Drawn plot images", "Accounting reports", "plots", gMonitor.OP_SUM ) gMonitor.registerActivity( "reportsRequested", "Generated reports", "Accounting reports", "reports", gMonitor.OP_SUM ) return S_OK()
def initializeReportGeneratorHandler(serviceInfo): global gAccountingDB gAccountingDB = AccountingDB(readOnly=True) #Get data location reportSection = PathFinder.getServiceSection("Accounting/ReportGenerator") dataPath = gConfig.getValue("%s/DataLocation" % reportSection, "data/accountingGraphs") dataPath = dataPath.strip() if "/" != dataPath[0]: dataPath = os.path.realpath( "%s/%s" % (gConfig.getValue('/LocalSite/InstancePath', rootPath), dataPath)) gLogger.info("Data will be written into %s" % dataPath) try: os.makedirs(dataPath) except: pass try: testFile = "%s/acc.jarl.test" % dataPath fd = file(testFile, "w") fd.close() os.unlink(testFile) except IOError: gLogger.fatal("Can't write to %s" % dataPath) return S_ERROR("Data location is not writable") gDataCache.setGraphsLocation(dataPath) gMonitor.registerActivity("plotsDrawn", "Drawn plot images", "Accounting reports", "plots", gMonitor.OP_SUM) gMonitor.registerActivity("reportsRequested", "Generated reports", "Accounting reports", "reports", gMonitor.OP_SUM) return S_OK()
def initialize(self): ''' NotifyAgent initialization ''' try: with sqlite3.connect(self.cacheFile) as conn: conn.execute( '''CREATE TABLE IF NOT EXISTS ProductionManagementCache( reqId VARCHAR(64) NOT NULL DEFAULT "", reqType VARCHAR(64) NOT NULL DEFAULT "", reqWG VARCHAR(64) NOT NULL DEFAULT "", reqName VARCHAR(64) NOT NULL DEFAULT "", SimCondition VARCHAR(64) NOT NULL DEFAULT "", ProPath VARCHAR(64) NOT NULL DEFAULT "", thegroup VARCHAR(64) NOT NULL DEFAULT "", reqInform VARCHAR(64) NOT NULL DEFAULT "" );''') except sqlite3.OperationalError: self.log.error('Email cache database is locked') self.diracAdmin = DiracAdmin() self.csS = PathFinder.getServiceSection( 'ProductionManagement/ProductionRequest') self.fromAddress = gConfig.getValue('%s/fromAddress' % self.csS, '') if not self.fromAddress: self.log.info( 'No fromAddress is defined, a default value will be used instead' ) self.fromAddress = '*****@*****.**' return S_OK()
def initializeMonitoringHandler( serviceInfo ): #Check that the path is writable monitoringSection = PathFinder.getServiceSection( "Framework/Monitoring" ) #Get data location dataPath = gConfig.getValue( "%s/DataLocation" % monitoringSection, "data/monitoring" ) dataPath = dataPath.strip() if "/" != dataPath[0]: dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) ) gLogger.info( "Data will be written into %s" % dataPath ) try: os.makedirs( dataPath ) except: pass try: testFile = "%s/mon.jarl.test" % dataPath fd = file( testFile, "w" ) fd.close() os.unlink( testFile ) except IOError: gLogger.fatal( "Can't write to %s" % dataPath ) return S_ERROR( "Data location is not writable" ) #Define globals gServiceInterface.initialize( dataPath ) if not gServiceInterface.initializeDB(): return S_ERROR( "Can't start db engine" ) gMonitor.registerActivity( "cachedplots", "Cached plot images", "Monitoring plots", "plots", gMonitor.OP_SUM ) gMonitor.registerActivity( "drawnplots", "Drawn plot images", "Monitoring plots", "plots", gMonitor.OP_SUM ) return S_OK()
def __init__(self, serviceData): """ Init the variables for the service :param serviceData: dict with modName, standalone, loadName, moduleObj, classObj. e.g.: {'modName': 'Framework/serviceName', 'standalone': True, 'loadName': 'Framework/serviceName', 'moduleObj': <module 'serviceNameHandler' from '/home/DIRAC/FrameworkSystem/Service/serviceNameHandler.pyo'>, 'classObj': <class 'serviceNameHandler.serviceHandler'>} """ self._svcData = serviceData self._name = serviceData["modName"] self._startTime = datetime.datetime.utcnow() self._validNames = [serviceData["modName"]] if serviceData["loadName"] not in self._validNames: self._validNames.append(serviceData["loadName"]) self._cfg = ServiceConfiguration(list(self._validNames)) self._standalone = serviceData["standalone"] self.__monitorLastStatsUpdate = time.time() self._stats = {"queries": 0, "connections": 0} self._authMgr = AuthManager( "%s/Authorization" % PathFinder.getServiceSection(serviceData["loadName"])) self._transportPool = getGlobalTransportPool() self.__cloneId = 0 self.__maxFD = 0 self.activityMonitoring = False # Check if monitoring is enabled if "Monitoring" in Operations().getMonitoringBackends( monitoringType="ServiceMonitoring"): self.activityMonitoring = True
def export_sendSMS( self, userName, body, fromAddress ): """ Send an SMS with supplied body to the specified DIRAC user using the Mail utility via an SMS switch. """ gLogger.verbose( 'Received signal to send the following SMS to %s:\n%s' % ( userName, body ) ) mobile = gConfig.getValue( '/Registry/Users/%s/Mobile' % userName, '' ) if not mobile: return S_ERROR( 'No registered mobile number for %s' % userName ) csSection = PathFinder.getServiceSection( 'Framework/Notification' ) smsSwitch = gConfig.getValue( '%s/SMSSwitch' % csSection, '' ) if not smsSwitch: return S_ERROR( 'No SMS switch is defined in CS path %s/SMSSwitch' % csSection ) address = '%s@%s' % ( mobile, smsSwitch ) subject = 'DIRAC SMS' m = Mail() m._subject = subject m._message = body m._mailAddress = address if not fromAddress == 'None': m._fromAddress = fromAddress result = m._send() if not result['OK']: gLogger.warn( 'Could not send SMS to %s with the following message:\n%s' % ( userName, result['Message'] ) ) else: gLogger.info( 'SMS sent successfully to %s ' % ( userName ) ) gLogger.debug( result['Value'] ) return result
def initializeDataLoggingHandler( serviceInfo ): global dataPath global logDB logDB = DataLoggingDB() monitoringSection = PathFinder.getServiceSection( "DataManagement/DataLogging" ) #Get data location retDict = gConfig.getOption( "%s/DataLocation" % monitoringSection, "dataLoggingPlots" ) if not retDict[ 'OK' ]: return retDict dataPath = retDict[ 'Value' ].strip() if "/" != dataPath[0]: dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) ) gLogger.info( "Data will be written into %s" % dataPath ) try: os.makedirs( dataPath ) except: pass try: testFile = "%s/mon.jarl.test" % dataPath fd = file( testFile, "w" ) fd.close() os.unlink( testFile ) except IOError: gLogger.fatal( "Can't write to %s" % dataPath ) return S_ERROR( "Data location is not writable" ) return S_OK()
def __init__(self, serviceData): """ Init the variables for the service :param serviceData: dict with modName, standalone, loadName, moduleObj, classObj. e.g.: {'modName': 'Framework/serviceName', 'standalone': True, 'loadName': 'Framework/serviceName', 'moduleObj': <module 'serviceNameHandler' from '/home/DIRAC/FrameworkSystem/Service/serviceNameHandler.pyo'>, 'classObj': <class 'serviceNameHandler.serviceHandler'>} Standalone is true if there is only one service started If it's false, every service is linked to a different MonitoringClient """ self._svcData = serviceData self._name = serviceData['modName'] self._startTime = Time.dateTime() self._validNames = [serviceData['modName']] if serviceData['loadName'] not in self._validNames: self._validNames.append(serviceData['loadName']) self._cfg = ServiceConfiguration(list(self._validNames)) if serviceData['standalone']: self._monitor = gMonitor else: self._monitor = MonitoringClient() self.__monitorLastStatsUpdate = time.time() self._stats = {'queries': 0, 'connections': 0} self._authMgr = AuthManager( "%s/Authorization" % PathFinder.getServiceSection(serviceData['loadName'])) self._transportPool = getGlobalTransportPool() self.__cloneId = 0 self.__maxFD = 0
def export_sendSMS(self, userName, body, fromAddress): """ Send an SMS with supplied body to the specified DIRAC user using the Mail utility via an SMS switch. """ gLogger.verbose( 'Received signal to send the following SMS to %s:\n%s' % (userName, body)) mobile = gConfig.getValue('/Registry/Users/%s/Mobile' % userName, '') if not mobile: return S_ERROR('No registered mobile number for %s' % userName) csSection = PathFinder.getServiceSection('Framework/Notification') smsSwitch = gConfig.getValue('%s/SMSSwitch' % csSection, '') if not smsSwitch: return S_ERROR('No SMS switch is defined in CS path %s/SMSSwitch' % csSection) address = '%s@%s' % (mobile, smsSwitch) subject = 'DIRAC SMS' m = Mail() m._subject = subject m._message = body m._mailAddress = address if not fromAddress == 'None': m._fromAddress = fromAddress result = m._send() if not result['OK']: gLogger.warn( 'Could not send SMS to %s with the following message:\n%s' % (userName, result['Message'])) else: gLogger.info('SMS sent successfully to %s ' % (userName)) gLogger.debug(result['Value']) return result
def initialize(self): # Build the URLs self._url = self._cfg.getURL() if not self._url: return S_ERROR("Could not build service URL for %s" % self._name) gLogger.verbose("Service URL is %s" % self._url) # Load handler result = self._loadHandlerInit() if not result["OK"]: return result self._handler = result["Value"] # Initialize lock manager self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions()) self._initMonitoring() self._threadPool = ThreadPool(1, max(0, self._cfg.getMaxThreads()), self._cfg.getMaxWaitingPetitions()) self._threadPool.daemonize() self._msgBroker = MessageBroker("%sMSB" % self._name, threadPool=self._threadPool) # Create static dict self._serviceInfoDict = { "serviceName": self._name, "serviceSectionPath": PathFinder.getServiceSection(self._name), "URL": self._cfg.getURL(), "messageSender": MessageSender(self._name, self._msgBroker), "validNames": self._validNames, "csPaths": [PathFinder.getServiceSection(svcName) for svcName in self._validNames], } # Call static initialization function try: self._handler["class"]._rh__initializeClass( dict(self._serviceInfoDict), self._lockManager, self._msgBroker, self._monitor ) if self._handler["init"]: for initFunc in self._handler["init"]: gLogger.verbose("Executing initialization function") try: result = initFunc(dict(self._serviceInfoDict)) except Exception, excp: gLogger.exception("Exception while calling initialization function") return S_ERROR("Exception while calling initialization function: %s" % str(excp)) if not isReturnStructure(result): return S_ERROR("Service initialization function %s must return S_OK/S_ERROR" % initFunc) if not result["OK"]: return S_ERROR("Error while initializing %s: %s" % (self._name, result["Message"])) except Exception, e: errMsg = "Exception while initializing %s" % self._name gLogger.exception(errMsg) return S_ERROR(errMsg)
def _getCSAuthorizarionSection(cls, serviceName): """Search service auth section. :param str serviceName: service name, see :py:meth:`_getFullComponentName` :return: str """ return "%s/Authorization" % PathFinder.getServiceSection(serviceName)
def export_sendMail(self, address, subject, body, fromAddress, avoidSpam=False): """ Send an email with supplied body to the specified address using the Mail utility. :param basestring address: recipient addresses :param basestring subject: subject of letter :param basestring body: body of letter :param basestring fromAddress: sender address, if None, will be used default from CS :param bool avoidSpam: if True, then emails are first added to a set so that duplicates are removed, and sent every hour. :return: S_OK(basestring)/S_ERROR() -- basestring is status message """ gLogger.verbose( 'Received signal to send the following mail to %s:\nSubject = %s\n%s' % (address, subject, body)) if gMailCache.exists(hash(address + subject + body)) and not avoidSpam: return S_OK( 'Email with the same content already sent today to current addresses, come back tomorrow' ) eMail = Mail() notificationSection = PathFinder.getServiceSection( "Framework/Notification") csSection = notificationSection + '/SMTP' eMail._smtpHost = gConfig.getValue('%s/Host' % csSection) eMail._smtpPort = gConfig.getValue('%s/Port' % csSection) eMail._smtpLogin = gConfig.getValue('%s/Login' % csSection) eMail._smtpPasswd = gConfig.getValue('%s/Password' % csSection) eMail._smtpPtcl = gConfig.getValue('%s/Protocol' % csSection) eMail._subject = subject eMail._message = body eMail._mailAddress = address if not fromAddress == 'None': eMail._fromAddress = fromAddress eMail._fromAddress = gConfig.getValue( '%s/FromAddress' % csSection) or eMail._fromAddress if avoidSpam and eMail in gMailSet: return S_OK("Mail already sent") else: result = eMail._send() if not result['OK']: gLogger.warn( 'Could not send mail with the following message:\n%s' % result['Message']) else: gMailCache.add(hash(address + subject + body), 3600 * 24) gLogger.info('Mail sent successfully to %s with subject %s' % (address, subject)) gLogger.debug(result['Value']) if avoidSpam: gMailSet.add(eMail) return result
def __getTplFolder(tt): csS = PathFinder.getServiceSection('ProductionManagement/ProductionRequest') if not csS: return S_ERROR("No ProductionRequest parameters in CS") tplFolder = gConfig.getValue('%s/templateFolder' % csS, '') if not tplFolder: return S_ERROR("No templateFolder in ProductionRequest parameters in CS") if not os.path.exists(tplFolder) or not os.path.isdir(tplFolder): return S_ERROR("Template Folder %s doesn't exist" % tplFolder) return S_OK(tplFolder)
def __getRequestDBPath( self ): """ Obtain the root of the requestDB from the configuration """ csSection = csSection = PathFinder.getServiceSection( "RequestManagement/RequestManager" ) root = gConfig.getValue( '%s/Path' % csSection ) if not root: diracRoot = gConfig.getValue( '/LocalSite/InstancePath', rootPath ) root = diracRoot + '/requestdb' if not os.path.exists( root ): os.makedirs( root ) return root
def __init__( self, csSection = False ): threading.Thread.__init__( self ) if csSection: self.csSection = csSection else: self.csSection = PathFinder.getServiceSection( "Framework/SiteMap" ) self.refreshPeriod = self._getCSValue( "RefreshPeriod", 300 ) self.gridsToMap = [] self.history = [] self.sitesData = {} self.refresh() self.setDaemon( 1 )
def __init__(self, csSection=False): threading.Thread.__init__(self) if csSection: self.csSection = csSection else: self.csSection = PathFinder.getServiceSection("Framework/SiteMap") self.refreshPeriod = self._getCSValue("RefreshPeriod", 300) self.gridsToMap = [] self.history = [] self.sitesData = {} self.refresh() self.setDaemon(1)
def export_sendMail(self, address, subject, body, fromAddress, avoidSpam=False): """ Send an email with supplied body to the specified address using the Mail utility. :param six.string_types address: recipient addresses :param six.string_types subject: subject of letter :param six.string_types body: body of letter :param six.string_types fromAddress: sender address, if None, will be used default from CS :param bool avoidSpam: Deprecated :return: S_OK(six.string_types)/S_ERROR() -- six.string_types is status message """ self.log.verbose( 'Received signal to send the following mail to %s:\nSubject = %s\n%s' % (address, subject, body)) if self.mailCache.exists(hash(address + subject + body)): return S_OK( 'Email with the same content already sent today to current addresses, come back tomorrow' ) eMail = Mail() notificationSection = PathFinder.getServiceSection( "Framework/Notification") csSection = notificationSection + '/SMTP' eMail._smtpHost = gConfig.getValue('%s/Host' % csSection) eMail._smtpPort = gConfig.getValue('%s/Port' % csSection) eMail._smtpLogin = gConfig.getValue('%s/Login' % csSection) eMail._smtpPasswd = gConfig.getValue('%s/Password' % csSection) eMail._smtpPtcl = gConfig.getValue('%s/Protocol' % csSection) eMail._subject = subject eMail._message = body eMail._mailAddress = address if not fromAddress == 'None': eMail._fromAddress = fromAddress eMail._fromAddress = gConfig.getValue( '%s/FromAddress' % csSection) or eMail._fromAddress result = eMail._send() if not result['OK']: self.log.warn( 'Could not send mail with the following message:\n%s' % result['Message']) else: self.mailCache.add(hash(address + subject + body), 3600 * 24) self.log.info('Mail sent successfully to %s with subject %s' % (address, subject)) self.log.debug(result['Value']) return result
def __getRequestDBPath(): """ get the fs root path of the requestDB from the DIRAC configuration :warn: if root path doesn't exist, it will be created """ csSection = csSection = PathFinder.getServiceSection( "RequestManagement/RequestManager" ) root = gConfig.getValue( '%s/Path' % csSection, 'requestdb' ) diracRoot = gConfig.getValue( '/LocalSite/InstancePath', rootPath ) # if the path return by the gConfig is absolute the following line does not change it, # otherwise it makes it relative to diracRoot root = os.path.join( diracRoot, root ) if not os.path.exists( root ): os.makedirs( root ) return root
def __getRequestDBPath(): """ get the fs root path of the requestDB from the DIRAC configuration :warn: if root path doesn't exist, it will be created """ csSection = csSection = PathFinder.getServiceSection("RequestManagement/RequestManager") root = gConfig.getValue("%s/Path" % csSection, "requestdb") diracRoot = gConfig.getValue("/LocalSite/InstancePath", rootPath) # if the path return by the gConfig is absolute the following line does not change it, # otherwise it makes it relative to diracRoot root = os.path.join(diracRoot, root) if not os.path.exists(root): os.makedirs(root) return root
def _getComponentInfoDict(cls, serviceName, fullURL): """Fill service information. :param str serviceName: service name, see :py:meth:`_getFullComponentName` :param str fullURL: incoming request path :return: dict """ path = PathFinder.getServiceSection(serviceName) cls._serviceInfoDict = { "serviceName": serviceName, "serviceSectionPath": path, "csPaths": [path], "URL": fullURL, } return cls._serviceInfoDict
def export_sendMail(self, address, subject, body, fromAddress): """Send an email with supplied body to the specified address using the Mail utility. :param str address: recipient addresses :param str subject: subject of letter :param str body: body of letter :param str fromAddress: sender address, if "", will be used default from CS :return: S_OK(str)/S_ERROR() -- str is status message """ self.log.verbose( "Received signal to send the following mail to %s:\nSubject = %s\n%s" % (address, subject, body)) if self.mailCache.exists(hash(address + subject + body)): return S_OK( "Email with the same content already sent today to current addresses, come back tomorrow" ) eMail = Mail() notificationSection = PathFinder.getServiceSection( "Framework/Notification") csSection = notificationSection + "/SMTP" eMail._smtpHost = gConfig.getValue("%s/Host" % csSection) eMail._smtpPort = gConfig.getValue("%s/Port" % csSection) eMail._smtpLogin = gConfig.getValue("%s/Login" % csSection) eMail._smtpPasswd = gConfig.getValue("%s/Password" % csSection) eMail._smtpPtcl = gConfig.getValue("%s/Protocol" % csSection) eMail._subject = subject eMail._message = body eMail._mailAddress = address if fromAddress: eMail._fromAddress = fromAddress eMail._fromAddress = gConfig.getValue( "%s/FromAddress" % csSection) or eMail._fromAddress result = eMail._send() if not result["OK"]: self.log.warn( "Could not send mail with the following message:\n%s" % result["Message"]) else: self.mailCache.add(hash(address + subject + body), 3600 * 24) self.log.info("Mail sent successfully to %s with subject %s" % (address, subject)) self.log.debug(result["Value"]) return result
def __init__( self, serviceData ): self._svcData = serviceData self._name = serviceData[ 'loadName' ] self._startTime = Time.dateTime() self._validNames = [ serviceData[ 'modName' ] ] if serviceData[ 'loadName' ] not in self._validNames: self._validNames.append( serviceData[ 'loadName' ] ) self._cfg = ServiceConfiguration( list( self._validNames ) ) if serviceData[ 'standalone' ]: self._monitor = gMonitor else: self._monitor = MonitoringClient() self.__monitorLastStatsUpdate = time.time() self._stats = { 'queries' : 0, 'connections' : 0 } self._authMgr = AuthManager( "%s/Authorization" % PathFinder.getServiceSection( serviceData[ 'loadName' ] ) ) self._transportPool = getGlobalTransportPool() self.__cloneId = 0 self.__maxFD = 0
def __init__( self, serviceData ): self._svcData = serviceData self._name = serviceData[ 'modName' ] self._startTime = Time.dateTime() self._validNames = [ serviceData[ 'modName' ] ] if serviceData[ 'loadName' ] not in self._validNames: self._validNames.append( serviceData[ 'loadName' ] ) self._cfg = ServiceConfiguration( list( self._validNames ) ) if serviceData[ 'standalone' ]: self._monitor = gMonitor else: self._monitor = MonitoringClient() self.__monitorLastStatsUpdate = time.time() self._stats = { 'queries' : 0, 'connections' : 0 } self._authMgr = AuthManager( "%s/Authorization" % PathFinder.getServiceSection( serviceData[ 'loadName' ] ) ) self._transportPool = getGlobalTransportPool() self.__cloneId = 0 self.__maxFD = 0
def initializeRequestManagerHandler(serviceInfo): global requestDB csSection = PathFinder.getServiceSection( "RequestManagement/RequestManager" ) backend = gConfig.getValue('%s/Backend' % csSection) if not backend: fatStr = "RequestManager.initializeRequestManagerHandler: Failed to get backed for RequestDB from CS." gLogger.fatal(fatStr) return S_ERROR(fatStr) gLogger.info("RequestManager.initializeRequestManagerHandler: Initialising with backend",backend) if backend == 'file': from DIRAC.RequestManagementSystem.DB.RequestDBFile import RequestDBFile requestDB = RequestDBFile() elif backend == 'mysql': from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL requestDB = RequestDBMySQL() else: fatStr = "RequestManager.initializeRequestManagerHandler: Supplied backend is not supported." gLogger.fatal(fatStr,backend) return S_ERROR(fatStr) return S_OK()
def export_sendSMS(self, userName, body, fromAddress): """Send an SMS with supplied body to the specified DIRAC user using the Mail utility via an SMS switch. :param str userName: user name :param str body: message :param str fromAddress: sender address :return: S_OK()/S_ERROR() """ self.log.verbose( "Received signal to send the following SMS to %s:\n%s" % (userName, body)) mobile = gConfig.getValue("/Registry/Users/%s/Mobile" % userName, "") if not mobile: return S_ERROR("No registered mobile number for %s" % userName) csSection = PathFinder.getServiceSection("Framework/Notification") smsSwitch = gConfig.getValue("%s/SMSSwitch" % csSection, "") if not smsSwitch: return S_ERROR("No SMS switch is defined in CS path %s/SMSSwitch" % csSection) address = "%s@%s" % (mobile, smsSwitch) subject = "DIRAC SMS" eMail = Mail() eMail._subject = subject eMail._message = body eMail._mailAddress = address if fromAddress: eMail._fromAddress = fromAddress result = eMail._send() if not result["OK"]: self.log.warn( "Could not send SMS to %s with the following message:\n%s" % (userName, result["Message"])) else: self.log.info("SMS sent successfully to %s " % (userName)) self.log.debug(result["Value"]) return result
def initializeTransferDBMonitoringHandler( serviceInfo ): """ handler initialization :param tuple serviceInfo: service info """ global transferDB transferDB = TransferDB() monitoringSection = PathFinder.getServiceSection( "DataManagement/TransferDBMonitoring" ) #Get data location retDict = gConfig.getOption( "%s/DataLocation" % monitoringSection ) if not retDict["OK"]: return retDict dataPath = retDict["Value"].strip() if "/" != dataPath[0]: dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) ) gLogger.info( "Data will be written into %s path" % dataPath ) ## check data path try: ## exists?? if os.path.exists( dataPath ): ## and it's a dir?? if os.path.isdir( dataPath): ## and writable?? if os.access( dataPath, os.W_OK ): return S_OK() else: return S_ERROR( "Data path %s exists, but it is not writable!" % dataPath ) else: return S_ERROR( "Data path %s exists, but points to a file!" % dataPath ) else: ## create os.makedirs( dataPath ) except ( OSError, IOError ) , anError: return S_ERROR( str(anError) )
def initialize(self): # Build the URLs self._url = self._cfg.getURL() if not self._url: return S_ERROR("Could not build service URL for %s" % self._name) gLogger.verbose("Service URL is %s" % self._url) # Load handler result = self._loadHandlerInit() if not result['OK']: return result self._handler = result['Value'] # Initialize lock manager self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions()) # TODO: remove ThreadPool if useThreadPoolExecutor: self._threadPool = ThreadPoolExecutor( max(0, self._cfg.getMaxThreads())) else: self._threadPool = ThreadPool(max(1, self._cfg.getMinThreads()), max(0, self._cfg.getMaxThreads()), self._cfg.getMaxWaitingPetitions()) self._threadPool.daemonize() self._msgBroker = MessageBroker("%sMSB" % self._name, threadPool=self._threadPool) # Create static dict self._serviceInfoDict = { 'serviceName': self._name, 'serviceSectionPath': PathFinder.getServiceSection(self._name), 'URL': self._cfg.getURL(), 'messageSender': MessageSender(self._name, self._msgBroker), 'validNames': self._validNames, 'csPaths': [ PathFinder.getServiceSection(svcName) for svcName in self._validNames ] } # Initialize Monitoring # This is a flag used to check whether "EnableActivityMonitoring" is enabled or not from the config file. self.activityMonitoring = (Operations().getValue( "EnableActivityMonitoring", False) or getServiceOption( self._serviceInfoDict, "EnableActivityMonitoring", False)) if self.activityMonitoring: # The import needs to be here because of the CS must be initialized before importing # this class (see https://github.com/DIRACGrid/DIRAC/issues/4793) from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter self.activityMonitoringReporter = MonitoringReporter( monitoringType="ComponentMonitoring") gThreadScheduler.addPeriodicTask( 100, self.__activityMonitoringReporting) elif self._standalone: self._monitor = gMonitor else: self._monitor = MonitoringClient() self._initMonitoring() # Call static initialization function try: if self.activityMonitoring: self._handler['class']._rh__initializeClass( dict(self._serviceInfoDict), self._lockManager, self._msgBroker, self.activityMonitoringReporter) else: self._handler['class']._rh__initializeClass( dict(self._serviceInfoDict), self._lockManager, self._msgBroker, self._monitor) if self._handler['init']: for initFunc in self._handler['init']: gLogger.verbose("Executing initialization function") try: result = initFunc(dict(self._serviceInfoDict)) except Exception as excp: gLogger.exception( "Exception while calling initialization function", lException=excp) return S_ERROR( "Exception while calling initialization function: %s" % str(excp)) if not isReturnStructure(result): return S_ERROR( "Service initialization function %s must return S_OK/S_ERROR" % initFunc) if not result['OK']: return S_ERROR("Error while initializing %s: %s" % (self._name, result['Message'])) except Exception as e: errMsg = "Exception while initializing %s" % self._name gLogger.exception(e) gLogger.exception(errMsg) return S_ERROR(errMsg) # Load actions after the handler has initialized itself result = self._loadActions() if not result['OK']: return result self._actions = result['Value'] if not self.activityMonitoring: gThreadScheduler.addPeriodicTask(30, self.__reportThreadPoolContents) return S_OK()
def initialize(self): # Build the URLs self._url = self._cfg.getURL() if not self._url: return S_ERROR("Could not build service URL for %s" % self._name) gLogger.verbose("Service URL is %s" % self._url) # Load handler result = self._loadHandlerInit() if not result["OK"]: return result self._handler = result["Value"] # Initialize lock manager self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions()) self._threadPool = ThreadPoolExecutor(max(0, self._cfg.getMaxThreads())) self._msgBroker = MessageBroker("%sMSB" % self._name, threadPool=self._threadPool) # Create static dict self._serviceInfoDict = { "serviceName": self._name, "serviceSectionPath": PathFinder.getServiceSection(self._name), "URL": self._cfg.getURL(), "messageSender": MessageSender(self._name, self._msgBroker), "validNames": self._validNames, "csPaths": [ PathFinder.getServiceSection(svcName) for svcName in self._validNames ], } self.securityLogging = Operations().getValue( "EnableSecurityLogging", True) and getServiceOption( self._serviceInfoDict, "EnableSecurityLogging", True) # Initialize Monitoring # The import needs to be here because of the CS must be initialized before importing # this class (see https://github.com/DIRACGrid/DIRAC/issues/4793) from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter self.activityMonitoringReporter = MonitoringReporter( monitoringType="ServiceMonitoring") self._initMonitoring() # Call static initialization function try: self._handler["class"]._rh__initializeClass( dict(self._serviceInfoDict), self._lockManager, self._msgBroker, self.activityMonitoringReporter) if self._handler["init"]: for initFunc in self._handler["init"]: gLogger.verbose("Executing initialization function") try: result = initFunc(dict(self._serviceInfoDict)) except Exception as excp: gLogger.exception( "Exception while calling initialization function", lException=excp) return S_ERROR( "Exception while calling initialization function: %s" % str(excp)) if not isReturnStructure(result): return S_ERROR( "Service initialization function %s must return S_OK/S_ERROR" % initFunc) if not result["OK"]: return S_ERROR("Error while initializing %s: %s" % (self._name, result["Message"])) except Exception as e: errMsg = "Exception while initializing %s" % self._name gLogger.exception(e) gLogger.exception(errMsg) return S_ERROR(errMsg) if self.activityMonitoring: gThreadScheduler.addPeriodicTask(30, self.__reportActivity) gThreadScheduler.addPeriodicTask( 100, self.__activityMonitoringReporting) # Load actions after the handler has initialized itself result = self._loadActions() if not result["OK"]: return result self._actions = result["Value"] return S_OK()
def informPeople(rec, oldstate, state, author, inform): """ inform utility """ if not state or state == 'New': return # was no state change or resurrect reqId = rec['RequestID'] csS = PathFinder.getServiceSection( 'ProductionManagement/ProductionRequest') if not csS: gLogger.error('No ProductionRequest section in configuration') return fromAddress = gConfig.getValue('%s/fromAddress' % csS, '') if not fromAddress: gLogger.error('No fromAddress is defined in CS path %s/fromAddress' % csS) return sendNotifications = gConfig.getValue('%s/sendNotifications' % csS, 'Yes') if sendNotifications != 'Yes': gLogger.info('No notifications will be send') return footer = "\n\nNOTE: it is an automated notification." footer += " Don't reply please.\n" footer += "DIRAC Web portal: https://lhcb-portal-dirac.cern.ch/DIRAC/s:%s/g:" % \ PathFinder.getDIRACSetup() ppath = "/?view=tabs&theme=Grey&url_state=1|*LHCbDIRAC.ProductionRequestManager.classes.ProductionRequestManager:,\n\n" ppath += 'The request details:\n' ppath += ' Type: %s' % str(rec['RequestType']) ppath += ' Name: %s\n' % str(rec['RequestName']) ppath += ' Conditions: %s\n' % str(rec['SimCondition']) ppath += ' Processing pass: %s\n' % str(rec['ProPath']) gLogger.info(".... %s ...." % ppath) authorMail = getUserOption(author, 'Email') if authorMail: if not state in ['BK Check', 'Submitted']: if state == 'BK OK': subj = 'DIRAC: please resign your Production Request %s' % reqId body = '\n'.join([ 'Customized Simulation Conditions in your request was registered.', 'Since Bookkeeping expert could make changes in your request,', 'you are asked to confirm it.' ]) else: subj = "DIRAC: the state of Production Request %s is changed to '%s'; %s;%s" % ( reqId, state, rec.get('RequestWG', ''), rec.get('RequestName', '')) body = '\n'.join([ 'The state of your request is changed.', 'This mail is for information only.' ]) notification = NotificationClient() res = notification.sendMail(authorMail, subj, body + footer + 'lhcb_user' + ppath, fromAddress, True) if not res['OK']: gLogger.error("_inform_people: can't send email: %s" % res['Message']) if inform: subj = "DIRAC: the state of %s Production Request %s is changed to '%s'; %s;%s" % ( rec['RequestType'], reqId, state, rec.get( 'RequestWG', ''), rec.get('RequestName', '')) body = '\n'.join([ 'You have received this mail because you are' 'in the subscription list for this request' ]) for x in inform.replace(" ", ",").split(","): if x: if x.find("@") > 0: eMail = x else: eMail = getUserOption(x, 'Email') if eMail: notification = NotificationClient() res = notification.sendMail( eMail, subj, body + footer + 'lhcb_user' + ppath, fromAddress, True) if not res['OK']: gLogger.error("_inform_people: can't send email: %s" % res['Message']) if state == 'Accepted': subj = "DIRAC: the Production Request %s is accepted; %s;%s" % ( reqId, rec.get('RequestWG', ''), rec.get('RequestName', '')) body = '\n'.join([ "The Production Request is signed and ready to process", "You are informed as member of %s group" ]) groups = ['lhcb_prmgr'] for group in groups: for man in _getMemberMails(group): notification = NotificationClient() res = notification.sendMail( man, subj, body % group + footer + group + ppath, fromAddress, True) if not res['OK']: gLogger.error("_inform_people: can't send email: %s" % res['Message']) elif state == 'PPG OK' and oldstate == 'Accepted': subj = "DIRAC: returned Production Request %s; %s;%s" % ( reqId, rec.get('RequestWG', ''), rec.get('RequestName', '')) body = '\n'.join([ "Production Request is returned by Production Manager.", "As member of %s group, your are asked to correct and sign", "or to reject it.", "", "In case some other member of the group has already", "done that, please ignore this mail." ]) groups = ['lhcb_tech'] for group in groups: for man in _getMemberMails(group): notification = NotificationClient() res = notification.sendMail( man, subj, body % group + footer + group + ppath, fromAddress, True) if not res['OK']: gLogger.error("_inform_people: can't send email: %s" % res['Message']) elif state == 'BK Check': groups = ['lhcb_bk'] _aggregate(reqId, rec.get('RequestType', ''), rec.get('RequestWG', ''), rec.get('RequestName', ''), rec['SimCondition'], rec['ProPath'], groups, rec.get('reqInform', inform)) elif state == 'Submitted': groups = ['lhcb_ppg', 'lhcb_tech'] _aggregate(reqId, rec.get('RequestType', ''), rec.get('RequestWG', ''), rec.get('RequestName', ''), rec['SimCondition'], rec['ProPath'], groups, rec.get('reqInform', inform)) else: return