def getCAsLocation():
""" Retrieve the CA's files location
"""
#Grid-Security
retVal = gConfig.getOption( '%s/Grid-Security' % g_SecurityConfPath )
if retVal[ 'OK' ]:
casPath = "%s/certificates" % retVal[ 'Value' ]
if os.path.isdir( casPath ):
return casPath
#CAPath
retVal = gConfig.getOption( '%s/CALocation' % g_SecurityConfPath )
if retVal[ 'OK' ]:
casPath = retVal[ 'Value' ]
if os.path.isdir( casPath ):
return casPath
# Look up the X509_CERT_DIR environment variable
if os.environ.has_key( 'X509_CERT_DIR' ):
casPath = os.environ[ 'X509_CERT_DIR' ]
return casPath
#rootPath./etc/grid-security/certificates
casPath = "%s/etc/grid-security/certificates" % DIRAC.rootPath
if os.path.isdir( casPath ):
return casPath
#/etc/grid-security/certificates
casPath = "/etc/grid-security/certificates"
if os.path.isdir( casPath ):
return casPath
#No CA's location found
return False
def getHostCertificateAndKeyLocation(specificLocation=None):
""" Retrieve the host certificate files location.
Lookup order:
* ``specificLocation`` (probably broken, don't use it)
* Environment variables (``DIRAC_X509_HOST_CERT`` and ``DIRAC_X509_HOST_KEY``)
* CS (``/DIRAC/Security/CertFile`` and ``/DIRAC/Security/CertKey``)
* Alternative exotic options, with ``prefix`` in ``server``, ``host``, ``dirac``, ``service``:
* in `<DIRAC rootpath>/etc/grid-security/` for ``<prefix>cert.pem`` and ``<prefix>key.pem``
* in the path defined in the CS in ``/DIRAC/Security/Grid-Security``
:param specificLocation: CS path to look for a the path to cert and key, which then should be the same.
Probably does not work, don't use it
:returns: tuple ``(<cert location>, <key location>)`` or ``False``
"""
fileDict = {}
# First, check the environment variables
for fileType, envVar in (('cert', 'DIRAC_X509_HOST_CERT'), ('key', 'DIRAC_X509_HOST_KEY')):
if envVar in os.environ and os.path.exists(os.environ[envVar]):
fileDict[fileType] = os.environ[envVar]
for fileType in ("cert", "key"):
# Check if we already have the info
if fileType in fileDict:
continue
# Direct file in config
retVal = gConfig.getOption('%s/%sFile' % (g_SecurityConfPath, fileType.capitalize()))
if retVal['OK']:
fileDict[fileType] = retVal['Value']
continue
fileFound = False
for filePrefix in ("server", "host", "dirac", "service"):
# Possible grid-security's
paths = []
retVal = gConfig.getOption('%s/Grid-Security' % g_SecurityConfPath)
if retVal['OK']:
paths.append(retVal['Value'])
paths.append("%s/etc/grid-security/" % DIRAC.rootPath)
for path in paths:
filePath = os.path.realpath("%s/%s%s.pem" % (path, filePrefix, fileType))
if os.path.isfile(filePath):
fileDict[fileType] = filePath
fileFound = True
break
if fileFound:
break
if "cert" not in fileDict or "key" not in fileDict:
return False
# we can specify a location outside /opt/dirac/etc/grid-security directory
if specificLocation:
fileDict["cert"] = gConfig.getValue(specificLocation, fileDict["cert"])
fileDict["key"] = gConfig.getValue(specificLocation, fileDict["key"])
return (fileDict["cert"], fileDict["key"])
def getCAsLocation():
"""Retrieve the CA's files location"""
# Grid-Security
retVal = gConfig.getOption("%s/Grid-Security" % g_SecurityConfPath)
if retVal["OK"]:
casPath = "%s/certificates" % retVal["Value"]
if os.path.isdir(casPath):
return casPath
# CAPath
retVal = gConfig.getOption("%s/CALocation" % g_SecurityConfPath)
if retVal["OK"]:
casPath = retVal["Value"]
if os.path.isdir(casPath):
return casPath
# Look up the X509_CERT_DIR environment variable
if "X509_CERT_DIR" in os.environ:
casPath = os.environ["X509_CERT_DIR"]
return casPath
# rootPath./etc/grid-security/certificates
casPath = "%s/etc/grid-security/certificates" % DIRAC.rootPath
if os.path.isdir(casPath):
return casPath
# /etc/grid-security/certificates
casPath = "/etc/grid-security/certificates"
if os.path.isdir(casPath):
return casPath
# No CA's location found
return False
def getHostCertificateAndKeyLocation():
""" Retrieve the host certificate files location
"""
fileDict = {}
for fileType in ( "cert", "key" ):
#Direct file in config
retVal = gConfig.getOption( '%s/%sFile' % ( g_SecurityConfPath, fileType.capitalize() ) )
if retVal[ 'OK' ]:
fileDict[ fileType ] = retVal[ 'Value' ]
continue
fileFound = False
for filePrefix in ( "server", "host", "dirac", "service" ):
#Possible grid-security's
paths = []
retVal = gConfig.getOption( '%s/Grid-Security' % g_SecurityConfPath )
if retVal[ 'OK' ]:
paths.append( retVal[ 'Value' ] )
paths.append( "%s/etc/grid-security/" % DIRAC.rootPath )
#paths.append( os.path.expanduser( "~/.globus" ) )
for path in paths:
filePath = os.path.realpath( "%s/%s%s.pem" % ( path, filePrefix, fileType ) )
if os.path.isfile( filePath ):
fileDict[ fileType ] = filePath
fileFound = True
break
if fileFound:
break
if "cert" not in fileDict.keys() or "key" not in fileDict.keys():
return False
return ( fileDict[ "cert" ], fileDict[ "key" ] )
def setupDB():
"""Get configuration from a cfg file and instantiate a DB"""
gLogger.setLevel("DEBUG")
result = gConfig.getOption("/Systems/Databases/Host")
if not result["OK"]:
result["Value"] = "mysql"
host = result["Value"]
result = gConfig.getOption("/Systems/Databases/User")
if not result["OK"]:
result["Value"] = "Dirac"
user = result["Value"]
result = gConfig.getOption("/Systems/Databases/Password")
if not result["OK"]:
result["Value"] = "Dirac"
password = result["Value"]
result = gConfig.getOption("/Systems/Databases/Port")
if not result["OK"]:
result["Value"] = 3306
port = int(result["Value"])
return getDB(host, user, password, "AccountingDB", port)
def setupDB():
""" Get configuration from a cfg file and instantiate a DB
"""
gLogger.setLevel('DEBUG')
result = gConfig.getOption('/Systems/Databases/Host')
if not result['OK']:
result['Value'] = 'mysql'
host = result['Value']
result = gConfig.getOption('/Systems/Databases/User')
if not result['OK']:
result['Value'] = 'Dirac'
user = result['Value']
result = gConfig.getOption('/Systems/Databases/Password')
if not result['OK']:
result['Value'] = 'Dirac'
password = result['Value']
result = gConfig.getOption('/Systems/Databases/Port')
if not result['OK']:
result['Value'] = 3306
port = int(result['Value'])
return getDB(host, user, password, 'AccountingDB', port)
def getCAsLocation():
""" Retrieve the CA's files location
"""
#Grid-Security
retVal = gConfig.getOption('%s/Grid-Security' % g_SecurityConfPath)
if retVal['OK']:
casPath = "%s/certificates" % retVal['Value']
if os.path.isdir(casPath):
return casPath
#CAPath
retVal = gConfig.getOption('%s/CALocation' % g_SecurityConfPath)
if retVal['OK']:
casPath = retVal['Value']
if os.path.isdir(casPath):
return casPath
# Look up the X509_CERT_DIR environment variable
if os.environ.has_key('X509_CERT_DIR'):
casPath = os.environ['X509_CERT_DIR']
return casPath
#rootPath./etc/grid-security/certificates
casPath = "%s/etc/grid-security/certificates" % DIRAC.rootPath
if os.path.isdir(casPath):
return casPath
#/etc/grid-security/certificates
casPath = "/etc/grid-security/certificates"
if os.path.isdir(casPath):
return casPath
#No CA's location found
return False
def getHostCertificateAndKeyLocation():
""" Retrieve the host certificate files location
"""
fileDict = {}
for fileType in ("cert", "key"):
#Direct file in config
retVal = gConfig.getOption('%s/%sFile' %
(g_SecurityConfPath, fileType.capitalize()))
if retVal['OK']:
fileDict[fileType] = retVal['Value']
continue
fileFound = False
for filePrefix in ("server", "host", "dirac", "service"):
#Possible grid-security's
paths = []
retVal = gConfig.getOption('%s/Grid-Security' % g_SecurityConfPath)
if retVal['OK']:
paths.append(retVal['Value'])
paths.append("%s/etc/grid-security/" % DIRAC.rootPath)
#paths.append( os.path.expanduser( "~/.globus" ) )
for path in paths:
filePath = os.path.realpath("%s/%s%s.pem" %
(path, filePrefix, fileType))
if os.path.isfile(filePath):
fileDict[fileType] = filePath
fileFound = True
break
if fileFound:
break
if "cert" not in fileDict.keys() or "key" not in fileDict.keys():
return False
return (fileDict["cert"], fileDict["key"])
def test_verifyCatalogConfiguration(self):
fileMetadataOption = gConfig.getOption(
"Systems/DataManagement/Production/Services/MultiVOFileCatalog/FileMetadata"
)
dirMetadataOption = gConfig.getOption(
"Systems/DataManagement/Production/Services/MultiVOFileCatalog/DirectoryMetadata"
)
self.assertTrue(fileMetadataOption["OK"])
self.assertEqual(fileMetadataOption["Value"], "MultiVOFileMetadata")
self.assertTrue(dirMetadataOption["OK"])
self.assertEqual(dirMetadataOption["Value"], "MultiVODirectoryMetadata")
def test_verifyCatalogConfiguration(self):
fileMetadataOption = gConfig.getOption(
'Systems/DataManagement/Production/Services/MultiVOFileCatalog/FileMetadata'
)
dirMetadataOption = gConfig.getOption(
'Systems/DataManagement/Production/Services/MultiVOFileCatalog/DirectoryMetadata'
)
self.assertTrue(fileMetadataOption['OK'])
self.assertEqual(fileMetadataOption['Value'], 'MultiVOFileMetadata')
self.assertTrue(dirMetadataOption['OK'])
self.assertEqual(dirMetadataOption['Value'],
'MultiVODirectoryMetadata')
def getOption(self, optionPath, optionVOPath):
value = None
if optionVOPath:
result = gConfig.getOption(optionVOPath)
if not result['OK']:
if not 'does not exist' in result['Message']:
return result
else:
value = result['Value']
if value is None:
return gConfig.getOption(optionPath)
else:
return S_OK(value)
def getOption( self, optionPath, optionVOPath ):
value = None
if optionVOPath:
result = gConfig.getOption( optionVOPath )
if not result['OK']:
if not 'does not exist' in result['Message']:
return result
else:
value = result['Value']
if value is None:
return gConfig.getOption( optionPath )
else:
return S_OK(value)
def __request(self):
req = {}
global pageNumber
global globalSort
global numberOfJobs
globalSort = []
numberOfJobs = 500
pageNumber = 0
if request.params.has_key("getSiteHistory") and len(
request.params["getSiteHistory"]) > 0:
req["ExpandSiteHistory"] = str(request.params["getSiteHistory"])
elif request.params.has_key("getServiceHistory") and len(
request.params["getServiceHistory"]) > 0:
req["ExpandServiceHistory"] = str(
request.params["getServiceHistory"])
elif request.params.has_key("getResourceHistory") and len(
request.params["getResourceHistory"]) > 0:
req["ExpandResourceHistory"] = str(
request.params["getResourceHistory"])
else:
result = gConfig.getOption("/Website/ListSeparator")
if result["OK"]:
separator = result["Value"]
else:
separator = ":::"
if request.params.has_key("siteName") and len(
request.params["siteName"]) > 0:
if str(request.params["siteName"]) != "All":
req["SiteName"] = str(
request.params["siteName"]).split(separator)
gLogger.info("REQUEST:", req)
return req
def initialize(self):
"""API initialization
:return: S_OK()/S_ERROR()
"""
if self.__initialized["OK"]:
return self.__initialized
res = self.__getCertificateID() if gConfig.useServerCertificate(
) else self.__getProxyID()
if not res["OK"]:
gLogger.error(res["Message"])
self.__initialized = S_ERROR("Cannot locate client credentials")
return self.__initialized
retVal = gConfig.getOption("/DIRAC/Configuration/MasterServer")
if not retVal["OK"]:
self.__initialized = S_ERROR(
"Master server is not known. Is everything initialized?")
return self.__initialized
self.__rpcClient = ConfigurationClient(
url=gConfig.getValue("/DIRAC/Configuration/MasterServer", ""))
self.__csMod = Modificator(
self.__rpcClient,
"%s - %s - %s" %
(self.__userGroup, self.__userDN,
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")),
)
retVal = self.downloadCSData()
if not retVal["OK"]:
self.__initialized = S_ERROR(
"Can not download the remote cfg. Is everything initialized?")
return self.__initialized
self.__initialized = S_OK()
return self.__initialized
def __getComponentDefinitionFromCS(self, system, setup, instance, cType,
component):
componentName = "%s/%s" % (system, component)
compDict = {
'ComponentName': componentName,
'Type': cType,
'Setup': setup
}
componentSection = "/Systems/%s/%s/%s/%s" % (
system, instance, "%ss" % cType.capitalize(), component)
compStatus = gConfig.getValue("%s/Status" % componentSection, 'Active')
if compStatus.lower() in ("inactive", ):
compDict['Status'] = compStatus.lower().capitalize()
if cType == 'service':
result = gConfig.getOption("%s/Port" % componentSection)
if not result['OK']:
compDict['Status'] = 'Error'
compDict[
'Message'] = "Component seems to be defined wrong in CS: %s" % result[
'Message']
return compDict
try:
compDict['Port'] = int(result['Value'])
except:
compDict['Status'] = 'Error'
compDict[
'Message'] = "Port for component doesn't seem to be a number"
return compDict
return compDict
def _showSoftware():
"""Show available software"""
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine()
from DIRAC import gConfig, gLogger
base = '/Operations/Defaults/AvailableTarBalls'
platforms = gConfig.getSections(base)
for platform in platforms['Value']:
gLogger.notice("For platform %s, here are the available software" % platform)
apps = gConfig.getSections(base + "/" + platform)
for app in apps['Value']:
if clip.software and app.lower() != clip.software.lower():
continue
gLogger.notice(" - %s" % app)
versions = gConfig.getSections(base + "/" + platform + "/" + app)
if clip.appsOnly:
continue
for vers in versions['Value']:
gLogger.notice(" * %s" % vers)
depsb = gConfig.getSections(base + "/" + platform + "/" + app + "/" + vers)
if 'Dependencies' in depsb['Value']:
gLogger.notice(" Depends on")
deps = gConfig.getSections( os.path.join( base, platform, app, vers , "Dependencies") )
for dep in deps['Value']:
depversions = gConfig.getOption(base + "/" + platform + "/" + app + "/" + vers + "/Dependencies/" + dep + "/version")
gLogger.notice(" %s %s" % (dep, depversions['Value']))
if not len(versions['Value']):
gLogger.notice(" No version available")
def initialize(self):
if self.__initialized['OK']:
return self.__initialized
if not gConfig.useServerCertificate():
res = self.__getProxyID()
else:
res = self.__getCertificateID()
if not res:
self.__initialized = S_ERROR("Cannot locate client credentials")
return self.__initialized
retVal = gConfig.getOption("/DIRAC/Configuration/MasterServer")
if not retVal['OK']:
self.__initialized = S_ERROR(
"Master server is not known. Is everything initialized?")
return self.__initialized
self.__rpcClient = RPCClient(
gConfig.getValue("/DIRAC/Configuration/MasterServer", ""))
self.__csMod = Modificator(
self.__rpcClient,
"%s - %s - %s" % (self.__userGroup, self.__userDN,
Time.dateTime().strftime("%Y-%m-%d %H:%M:%S")))
retVal = self.downloadCSData()
if not retVal['OK']:
self.__initialized = S_ERROR(
"Can not download the remote cfg. Is everything initialized?")
return self.__initialized
self.__initialized = S_OK()
return self.__initialized
def __getActiveSEs( self, selist, access = "Read" ):
activeSE = []
for se in selist:
res = gConfig.getOption( '/Resources/StorageElements/%s/%sAccess' % ( se, access ), 'Unknown' )
if res['OK'] and res['Value'] == 'Active':
activeSE.append( se )
return activeSE
def initializeDataLoggingHandler( serviceInfo ):
global dataPath
global logDB
logDB = DataLoggingDB()
monitoringSection = PathFinder.getServiceSection( "DataManagement/DataLogging" )
#Get data location
retDict = gConfig.getOption( "%s/DataLocation" % monitoringSection, "dataLoggingPlots" )
if not retDict[ 'OK' ]:
return retDict
dataPath = retDict[ 'Value' ].strip()
if "/" != dataPath[0]:
dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) )
gLogger.info( "Data will be written into %s" % dataPath )
try:
os.makedirs( dataPath )
except:
pass
try:
testFile = "%s/mon.jarl.test" % dataPath
fd = file( testFile, "w" )
fd.close()
os.unlink( testFile )
except IOError:
gLogger.fatal( "Can't write to %s" % dataPath )
return S_ERROR( "Data location is not writable" )
return S_OK()
def initializeDataLoggingHandler(serviceInfo):
global dataPath
global logDB
logDB = DataLoggingDB()
monitoringSection = PathFinder.getServiceSection(
"DataManagement/DataLogging")
#Get data location
retDict = gConfig.getOption("%s/DataLocation" % monitoringSection,
"dataLoggingPlots")
if not retDict['OK']:
return retDict
dataPath = retDict['Value'].strip()
if "/" != dataPath[0]:
dataPath = os.path.realpath(
"%s/%s" %
(gConfig.getValue('/LocalSite/InstancePath', rootPath), dataPath))
gLogger.info("Data will be written into %s" % dataPath)
try:
os.makedirs(dataPath)
except:
pass
try:
testFile = "%s/mon.jarl.test" % dataPath
fd = file(testFile, "w")
fd.close()
os.unlink(testFile)
except IOError:
gLogger.fatal("Can't write to %s" % dataPath)
return S_ERROR("Data location is not writable")
return S_OK()
def initialize(self):
self.SystemLoggingDB = SystemLoggingDB()
self.notification=NotificationClient()
userString = self.am_getOption( "Reviewer", 'mseco' )
self.log.debug( "Users to be notified", ": " + userString )
userList = List.fromChar( userString, ",")
mailList = []
for user in userList:
retval = gConfig.getOption( "/Registry/Users/" + user + "/Email" )
if not retval['OK']:
self.log.warn( "Could not get user's mail", retval['Message'] )
else:
mailList.append( retval['Value'] )
if not mailList:
mailList = gConfig.getValue( '/Operations/EMail/Logging', [] )
if not len(mailList):
errString = "There are no valid users in the list"
varString = "[" + ','.join( userList ) + "]"
self.log.error( errString, varString )
return S_ERROR( errString + varString )
self.log.info( "List of mails to be notified", ','.join( mailList ) )
self._mailAddress = mailList
self._subject = 'New error messages were entered in the SystemLoggingDB'
return S_OK()
def __getComponentDefinitionFromCS(self, system, setup, instance, cType, component):
"""
Gets the basic component details from the configuration file.
:type system: string
:param system: The system name.
:type setup: string
:param setup: The setup site.
:type instance: string
:param instance: The instance.
:type cType: string
:param cType: The component type.
:type component: string
:param component: The component name.
:return: a component dictionary.
"""
componentName = "%s/%s" % (system, component)
compDict = {"ComponentName": componentName, "Type": cType, "Setup": setup}
componentSection = "/Systems/%s/%s/%s/%s" % (system, instance, "%ss" % cType.capitalize(), component)
compStatus = gConfig.getValue("%s/Status" % componentSection, "Active")
if compStatus.lower() in ("inactive",):
compDict["Status"] = compStatus.lower().capitalize()
if cType == "service":
result = gConfig.getOption("%s/Port" % componentSection)
if not result["OK"]:
compDict["Status"] = "Error"
compDict["Message"] = "Component seems to be defined wrong in CS: %s" % result["Message"]
return compDict
try:
compDict["Port"] = int(result["Value"])
except Exception:
compDict["Status"] = "Error"
compDict["Message"] = "Port for component doesn't seem to be a number"
return compDict
return compDict
def uploadProcessListToFileCatalog(self, path_to_process_list, appVersion):
"""Upload the new processList to the FileCatalog
"""
from ILCDIRAC.Core.Utilities.FileUtils import upload
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gConfig, exit as dexit
datMan = DataManager()
gLogger.notice("Removing process list from file catalog" + path_to_process_list)
res = datMan.removeFile(path_to_process_list)
if not res['OK']:
gLogger.error("Could not remove process list from file catalog, do it by hand")
dexit(2)
gLogger.notice("Done removing process list from file catalog")
res = upload(os.path.dirname(path_to_process_list) + "/", self.location )
if not res['OK']:
gLogger.error("something went wrong in the copy")
dexit(2)
gLogger.notice("Putting process list to local processlist directory")
localprocesslistpath = gConfig.getOption("/LocalSite/ProcessListPath", "")
if localprocesslistpath['Value']:
try:
localSvnRepo = "/afs/cern.ch/eng/clic/software/whizard/whizard_195/"
shutil.copy(self.location, localSvnRepo) ## because it does not make a difference if we hardcode it here or in ${DIRAC}/etc/dirac.cfg, yours truly APS, JFS
except OSError, err:
gLogger.error("Copy of process list to %s failed with error %s!" % (localSvnRepo, str(err)))
try:
subprocess.call( ["svn","ci", os.path.join( localSvnRepo, os.path.basename(localprocesslistpath['Value'] )), "-m'Process list for whizard version %s'" % appVersion ], shell=False )
except OSError, err:
gLogger.error("Commit failed! Error: %s" % str(err))
def __request(self):
req = {}
global pageNumber
global globalSort
global numberOfJobs
globalSort = []
numberOfJobs = 500
pageNumber = 0
if request.params.has_key("getSiteHistory") and len(request.params["getSiteHistory"]) > 0:
req["ExpandSiteHistory"] = str(request.params["getSiteHistory"])
elif request.params.has_key("getServiceHistory") and len(request.params["getServiceHistory"]) > 0:
req["ExpandServiceHistory"] = str(request.params["getServiceHistory"])
elif request.params.has_key("getResourceHistory") and len(request.params["getResourceHistory"]) > 0:
req["ExpandResourceHistory"] = str(request.params["getResourceHistory"])
else:
result = gConfig.getOption("/Website/ListSeparator")
if result["OK"]:
separator = result["Value"]
else:
separator = ":::"
if request.params.has_key("siteName") and len(request.params["siteName"]) > 0:
if str(request.params["siteName"]) != "All":
req["SiteName"] = str(request.params["siteName"]).split(separator)
gLogger.info("REQUEST:",req)
return req
def _getTestsList():
result = {}
for test in gConfig.getSections('/Systems/Framework/Production/Agents/SAMLauncherAgent')['Value']:
result[test] = {}
for option in gConfig.getOptions('/Systems/Framework/Production/Agents/SAMLauncherAgent/'+test)['Value']:
result[test][option] = gConfig.getOption('/Systems/Framework/Production/Agents/SAMLauncherAgent/'+test+'/'+option)['Value']
return result
def __request(self):
req = {}
global R_NUMBER
global P_NUMBER
R_NUMBER = 25
if request.params.has_key("limit") and len(request.params["limit"]) > 0:
R_NUMBER = int(request.params["limit"])
P_NUMBER = 0
if request.params.has_key("start") and len(request.params["start"]) > 0:
P_NUMBER = int(request.params["start"])
result = gConfig.getOption("/Website/ListSeparator")
if result["OK"]:
separator = result["Value"]
else:
separator = ":::"
RPC = getRPCClient("DataManagement/FileCatalog")
result = RPC.getMetadataFields()
gLogger.always(" +++ ",result)
if not result["OK"]:
return {}
result = result["Value"]
meta = []
for key,value in result.items():
for j in value:
meta.append(j)
gLogger.always(" * * * ",meta)
for i in request.params:
if i in meta:
meta_list = str(request.params[i]).split(separator)
if len(meta_list) == 1:
meta_list = meta_list[0]
req[i] = meta_list
gLogger.always(" * * * ",req)
return req
def uploadProcessListToFileCatalog(self, path_to_process_list, appVersion):
"""Upload the new processList to the FileCatalog
"""
from ILCDIRAC.Core.Utilities.FileUtils import upload
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gConfig, exit as dexit
datMan = DataManager()
LOG.notice("Removing process list from file catalog" + path_to_process_list)
res = datMan.removeFile(path_to_process_list)
if not res['OK']:
LOG.error("Could not remove process list from file catalog, do it by hand")
dexit(2)
LOG.notice("Done removing process list from file catalog")
res = upload(os.path.dirname(path_to_process_list) + "/", self.location )
if not res['OK']:
LOG.error("something went wrong in the copy")
dexit(2)
LOG.notice("Putting process list to local processlist directory")
localprocesslistpath = gConfig.getOption("/LocalSite/ProcessListPath", "")
if localprocesslistpath['Value']:
try:
localSvnRepo = "/afs/cern.ch/eng/clic/software/whizard/whizard_195/"
shutil.copy(self.location, localSvnRepo) ## because it does not make a difference if we hardcode it here or in ${DIRAC}/etc/dirac.cfg, yours truly APS, JFS
except OSError, err:
LOG.error("Copy of process list to %s failed with error %s!" % (localSvnRepo, str(err)))
try:
subprocess.call( ["svn","ci", os.path.join( localSvnRepo, os.path.basename(localprocesslistpath['Value'] )), "-m'Process list for whizard version %s'" % appVersion ], shell=False )
except OSError, err:
LOG.error("Commit failed! Error: %s" % str(err))
def getElasticDBParameters(fullname):
"""
Retrieve Database parameters from CS
fullname should be of the form <System>/<DBname>
"""
cs_path = getDatabaseSection(fullname)
parameters = {}
result = gConfig.getOption(cs_path + '/Host')
if not result['OK']:
# No host name found, try at the common place
result = gConfig.getOption('/Systems/NoSQLDatabases/Host')
if not result['OK']:
return S_ERROR('Failed to get the configuration parameter: Host')
dbHost = result['Value']
# Check if the host is the local one and then set it to 'localhost' to use
# a socket connection
if dbHost != 'localhost':
localHostName = socket.getfqdn()
if localHostName == dbHost:
dbHost = 'localhost'
parameters['Host'] = dbHost
# Elasticsearch standard port
dbPort = 9200
result = gConfig.getOption(cs_path + '/Port')
if not result['OK']:
# No individual port number found, try at the common place
result = gConfig.getOption('/Systems/NoSQLDatabases/Port')
if result['OK']:
dbPort = int(result['Value'])
else:
dbPort = int(result['Value'])
parameters['Port'] = dbPort
result = gConfig.getOption(cs_path + '/User')
if not result['OK']:
# No individual user name found, try at the common place
result = gConfig.getOption('/Systems/NoSQLDatabases/User')
if not result['OK']:
return S_ERROR('Failed to get the configuration parameter: User')
dbUser = result['Value']
parameters['User'] = dbUser
result = gConfig.getOption(cs_path + '/Password')
if not result['OK']:
# No individual password found, try at the common place
result = gConfig.getOption('/Systems/NoSQLDatabases/Password')
if not result['OK']:
return S_ERROR(
'Failed to get the configuration parameter: Password')
dbPass = result['Value']
parameters['Password'] = dbPass
return S_OK(parameters)
def __init__( self ):
self.rootConfigPath = '/Resources/StorageElements'
self.valid = True
self.proxy = False
res = gConfig.getOption( "%s/UseProxy" % self.rootConfigPath )
if res['OK'] and ( res['Value'] == 'True' ):
self.proxy = True
def __request(self):
gLogger.info("!!! PARAMS: ",str(request.params))
req = {}
try:
start = int( request.params[ 'start' ] )
except:
start = 0
try:
limit = int( request.params[ 'limit' ] )
except:
limit = 25
try:
sortDirection = str( request.params[ 'sortDirection' ] ).strip()
except:
sortDirection = "ASC"
try:
sortField = str( request.params[ 'sortField' ] ).strip()
except:
sortField = "UserName"
sort = [[sortField, sortDirection]]
gLogger.info("!!! S O R T : ",sort)
result = gConfig.getOption("/Website/ListSeparator")
if result["OK"]:
separator = result["Value"]
else:
separator = ":::"
if request.params.has_key("username") and len(request.params["username"]) > 0:
if str(request.params["username"]) != "All":
req["UserName"] = str(request.params["username"]).split(separator)
if request.params.has_key("usergroup") and len(request.params["usergroup"]) > 0:
if str(request.params["usergroup"]) != "All":
req["UserGroup"] = str(request.params["usergroup"]).split(separator)
if request.params.has_key("persistent") and len(request.params["persistent"]) > 0:
if str(request.params["persistent"]) in ["True","False"]:
req["PersistentFlag"] = str(request.params["persistent"])
before = False
after = False
if request.params.has_key("expiredBefore") and len(request.params["expiredBefore"]) > 0:
try:
before = int(request.params["expiredBefore"])
except:
pass
if request.params.has_key("expiredAfter") and len(request.params["expiredAfter"]) > 0:
try:
after = int(request.params["expiredAfter"])
except:
pass
if before and after:
if before > after:
req["beforeDate"] = before
req["afterDate"] = after
else:
if before:
req["beforeDate"] = before
if after:
req["afterDate"] = after
gLogger.always("REQUEST:",req)
return (start, limit, sort, req)
def __request(self):
req = {}
lhcbGroup = credentials.getSelectedGroup()
lhcbUser = str(credentials.getUsername())
global pageNumber
if request.params.has_key("id") and len(request.params["id"]) > 0:
pageNumber = 0
req["JobID"] = str(request.params["id"])
elif request.params.has_key("expand") and len(request.params["expand"]) > 0:
globalSort = [["GridSite","ASC"]]
numberOfJobs = 500
pageNumber = 0
req["ExpandSite"] = str(request.params["expand"])
else:
result = gConfig.getOption("/Website/ListSeparator")
if result["OK"]:
separator = result["Value"]
else:
separator = ":::"
pageNumber = 0
numberOfJobs = 500
if request.params.has_key("prod") and len(request.params["prod"]) > 0:
if str(request.params["prod"]) != "All":
req["JobGroup"] = str(request.params["prod"]).split(separator)
if request.params.has_key("site") and len(request.params["site"]) > 0:
if str(request.params["site"]) != "All":
tmp = str(request.params["site"]).split(separator)
if len(tmp) == 1:
req["ExpandSite"] = tmp[0]
else:
req["GridSite"] = tmp
if request.params.has_key("stat") and len(request.params["stat"]) > 0:
if str(request.params["stat"]) != "All":
req["Status"] = str(request.params["stat"]).split(separator)
if request.params.has_key("minorstat") and len(request.params["minorstat"]) > 0:
if str(request.params["minorstat"]) != "All":
req["MinorStatus"] = str(request.params["minorstat"]).split(separator)
if request.params.has_key("app") and len(request.params["app"]) > 0:
if str(request.params["app"]) != "All":
req["ApplicationStatus"] = str(request.params["app"]).split(separator)
# if lhcbGroup == "lhcb" or lhcbGroup == "lhcb_user":
# req["Owner"] = str(lhcbUser)
else:
if request.params.has_key("owner") and len(request.params["owner"]) > 0:
if str(request.params["owner"]) != "All":
req["Owner"] = str(request.params["owner"]).split(separator)
if request.params.has_key("date") and len(request.params["date"]) > 0:
if str(request.params["date"]) != "YYYY-mm-dd":
req["LastUpdate"] = str(request.params["date"])
if request.params.has_key("sort") and len(request.params["sort"]) > 0:
globalSort = str(request.params["sort"])
key,value = globalSort.split(" ")
globalSort = [[str(key),str(value)]]
else:
# globalSort = [["JobID","DESC"]]
globalSort = [["GridSite","ASC"]]
gLogger.info("REQUEST:",req)
return req
def __init__(self):
# create client instance using GGUS wsdl:
self.gclient = Client("https://prod-ars.ggus.eu/arsys/WSDL/public/prod-ars/GGUS")
authInfo = self.gclient.factory.create("AuthenticationInfo")
authInfo.userName = "******"
authInfo.password = "******"
self.vo = gConfig.getOption("DIRAC/VirtualOrganization")
self.gclient.set_options(soapheaders=authInfo)
def __request(self):
req = {}
global R_NUMBER
global P_NUMBER
global globalSort
globalSort = []
R_NUMBER = 25
if request.params.has_key("limit") and len(request.params["limit"]) > 0:
R_NUMBER = int(request.params["limit"])
P_NUMBER = 0
if request.params.has_key("start") and len(request.params["start"]) > 0:
P_NUMBER = int(request.params["start"])
if request.params.has_key("getSiteHistory") and len(request.params["getSiteHistory"]) > 0:
req["ExpandSiteHistory"] = str(request.params["getSiteHistory"])
elif request.params.has_key("getServiceHistory") and len(request.params["getServiceHistory"]) > 0:
req["ExpandServiceHistory"] = str(request.params["getServiceHistory"])
elif request.params.has_key("getResourceHistory") and len(request.params["getResourceHistory"]) > 0:
req["ExpandResourceHistory"] = str(request.params["getResourceHistory"])
elif request.params.has_key("getStorageHistory") and len(request.params["getStorageHistory"]) > 0:
req["ExpandStorageElementHistory"] = str(request.params["getStorageHistory"])
else:
result = gConfig.getOption("/Website/ListSeparator")
if result["OK"]:
separator = result["Value"]
else:
separator = ":::"
if not request.params.has_key("mode") or not len(request.params["mode"]) > 0:
return req
mode = request.params["mode"]
if not mode in MODELIST:
return req
selectors = [
"SiteName",
"SiteType",
"Status",
"ResourceType",
"ResourceName",
"ServiceType",
"ServiceName",
"StorageSiteName",
"StorageElementName"
]
gLogger.info("params: ",request.params)
for i in selectors:
if request.params.has_key(i) and len(request.params[i]) > 0:
if str(request.params[i]) != "All":
req[i] = request.params[i].split(separator)
if "All" in req[i]:
req[i].remove("All")
if i == "StorageSiteName":
req["SiteName"] = req[i]
del req[i]
if mode in STORELIST:
status = mode[7:]
req['StatusType'] = status
gLogger.info("Request:",req)
return req
def setup( self, channels, bandwithds, failedFiles ):
""" prepare fts graph
:param dict channels: { channelID : { "Files" : long , Size = long, "ChannelName" : str,
"Source" : str, "Destination" : str , "ChannelName" : str, "Status" : str } }
:param dict bandwidths: { channelID { "Throughput" : float, "Fileput" : float, "SucessfulFiles" : long, "FailedFiles" : long } }
:param dict failedFiles: { channelID : int }
channelInfo { channelName : { "ChannelID" : int, "TimeToStart" : float} }
"""
graph = FTSGraph( "sites" )
sites = gConfig.getSections( "/Resources/Sites/LCG" )
if not sites["OK"]:
return sites
sites = sites["Value"]
sitesDict = {}
for site in sites:
ses = gConfig.getOption( "/Resources/Sites/LCG/%s/SE" % site, [] )
siteName = site.split(".")[1] if "." in site else site
if ses["OK"] and ses["Value"]:
sitesDict[siteName] = ses["Value"]
## create nodes
for site, ses in sitesDict.items():
rwDict = self.__getRWAccessForSE( ses )
if not rwDict["OK"]:
return rwDict
graph.addNode( LCGSite( site, { "SEs" : rwDict["Value"] } ) )
## channels { channelID : { "Files" : long , Size = long, "ChannelName" : str,
## "Source" : str, "Destination" : str ,
## "ChannelName" : str, "Status" : str } }
## bandwidths { channelID { "Throughput" : float, "Fileput" : float,
## "SucessfulFiles" : long, "FailedFiles" : long } }
## channelInfo { channelName : { "ChannelID" : int, "TimeToStart" : float} }
for channelID, channelDict in channels.items():
sourceName = channelDict["Source"]
destName = channelDict["Destination"]
fromNode = graph.getNode( sourceName )
toNode = graph.getNode( destName )
if fromNode and toNode:
rwAttrs = { "status" : channels[channelID]["Status"],
"files" : channelDict["Files"],
"size" : channelDict["Size"],
"successfulAttempts" : bandwithds[channelID]["SuccessfulFiles"],
"failedAttempts" : bandwithds[channelID]["FailedFiles"],
"distinctFailedFiles" : failedFiles.get( channelID, 0 ),
"fileput" : bandwithds[channelID]["Fileput"],
"throughput" : bandwithds[channelID]["Throughput"] }
roAttrs = { "channelID" : channelID,
"channelName" : channelDict["ChannelName"],
"acceptableFailureRate" : self.acceptableFailureRate,
"acceptableFailedFiles" : self.acceptableFailedFiles,
"schedulingType" : self.schedulingType }
ftsChannel = FTSChannel( fromNode, toNode, rwAttrs, roAttrs )
graph.addEdge( ftsChannel )
self.ftsGraph = graph
self.lastRssUpdate = datetime.datetime.now()
return S_OK()
def __getActiveSEs(self, selist, access="Read"):
activeSE = []
for se in selist:
res = gConfig.getOption(
'/Resources/StorageElements/%s/%sAccess' % (se, access),
'Unknown')
if res['OK'] and res['Value'] == 'Active':
activeSE.append(se)
return activeSE
def initialize(self):
""" Standard constructor
"""
self.jobDB = JobDB()
self.am_setOption("PollingTime", 120)
self.__jobDBFields = []
for field in self.__summaryKeyFieldsMapping:
if field == 'User':
field = 'Owner'
elif field == 'UserGroup':
field = 'OwnerGroup'
self.__jobDBFields.append(field)
result = gConfig.getOption("/Systems/RabbitMQ/User")
if not result['OK']:
raise RuntimeError(
'Failed to get the configuration parameters: User')
user = result['Value']
result = gConfig.getOption("/Systems/RabbitMQ/Password")
if not result['OK']:
raise RuntimeError(
'Failed to get the configuration parameters: Password')
password = result['Value']
result = gConfig.getOption("/Systems/RabbitMQ/Host")
if not result['OK']:
raise RuntimeError(
'Failed to get the configuration parameters: Host')
self.host = result['Value']
self.credentials = pika.PlainCredentials(user, password)
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(host=self.host,
credentials=self.credentials))
self.channel = self.connection.channel()
self.channel.exchange_declare(exchange='mdatabase',
exchange_type='fanout')
return S_OK()
def getServiceOption( serviceInfo, optionName, defaultValue ):
""" Get service option resolving default values from the master service
"""
if optionName[0] == "/":
return gConfig.getValue( optionName, defaultValue )
for csPath in serviceInfo[ 'csPaths' ]:
result = gConfig.getOption( "%s/%s" % ( csPath, optionName, ), defaultValue )
if result[ 'OK' ]:
return result[ 'Value' ]
return defaultValue
def web_getSelectionData(self):
sData = self.getSessionData()
callback = {}
user = sData["user"]["username"]
if user == "Anonymous":
self.finish({
"success": "false",
"error": "You are not authorize to access these data"
})
if len(self.request.arguments) > 0:
tmp = {}
for i in self.request.arguments:
tmp[i] = str(self.request.arguments[i])
callback["extra"] = tmp
rpcClient = RPCClient("Framework/ProxyManager")
retVal = yield self.threadTask(rpcClient.getContents, {}, [], 0, 0)
if not retVal["OK"]:
self.finish({"success": "false", "error": retVal["Message"]})
data = retVal["Value"]
users = []
groups = []
for record in data["Records"]:
users.append(str(record[0]))
groups.append(str(record[2]))
users = uniqueElements(users)
groups = uniqueElements(groups)
users.sort()
groups.sort()
users = map(lambda x: [x], users)
groups = map(lambda x: [x], groups)
callback["username"] = users
callback["usergroup"] = groups
result = gConfig.getOption(
"/WebApp/ProxyManagementMonitoring/TimeSpan",
"86400,432000,604800,2592000")
if result["OK"]:
tmp = result["Value"]
tmp = tmp.split(", ")
if len(tmp) > 0:
timespan = []
for i in tmp:
human_readable = self.__humanize_time(i)
timespan.append([i, human_readable])
else:
timespan = [["Nothing to display"]]
else:
timespan = [["Error during RPC call"]]
callback["expiredBefore"] = timespan
callback["expiredAfter"] = timespan
self.finish(callback)
def __getCSStorageElementStatus( self, elementName, statusType, default ):
'''
Gets from the CS the StorageElements status
'''
cs_path = "/Resources/StorageElements"
if not isinstance( elementName, list ):
elementName = [ elementName ]
statuses = self.rssConfig.getConfigStatusType( 'StorageElement' )
#statuses = self.__opHelper.getOptionsDict( 'RSSConfiguration/GeneralConfig/Resources/StorageElement' )
#statuses = gConfig.getOptionsDict( '/Operations/RSSConfiguration/GeneralConfig/Resources/StorageElement' )
result = {}
for element in elementName:
if statusType is not None:
# Added Active by default
#res = gConfig.getOption( "%s/%s/%s" % ( cs_path, element, statusType ), 'Allowed' )
res = gConfig.getOption( "%s/%s/%s" % ( cs_path, element, statusType ), 'Active' )
if res[ 'OK' ] and res[ 'Value' ]:
result[ element ] = { statusType : res[ 'Value' ] }
else:
res = gConfig.getOptionsDict( "%s/%s" % ( cs_path, element ) )
if res[ 'OK' ] and res[ 'Value' ]:
elementStatuses = {}
for elementStatusType, value in res[ 'Value' ].items():
#k = k.replace( 'Access', '' )
if elementStatusType in statuses:
elementStatuses[ elementStatusType ] = value
# If there is no status defined in the CS, we add by default Read and
# Write as Active.
if elementStatuses == {}:
#elementStatuses = { 'ReadAccess' : 'Allowed', 'WriteAccess' : 'Allowed' }
elementStatuses = { 'ReadAccess' : 'Active', 'WriteAccess' : 'Active' }
result[ element ] = elementStatuses
if result:
return S_OK( result )
if default is not None:
# sec check
if statusType is None:
statusType = 'none'
defList = [ [ el, statusType, default ] for el in elementName ]
return S_OK( getDictFromList( defList ) )
_msg = "StorageElement '%s', with statusType '%s' is unknown for CS."
return S_ERROR( _msg % ( elementName, statusType ) )
def ex_getOption(cls, optName, defaultValue=None):
if defaultValue is None:
if optName in cls.__defaults:
defaultValue = cls.__defaults[optName]
if optName and optName[0] == "/":
return gConfig.getValue(optName, defaultValue)
for section in (cls.__properties["section"], cls.__properties["loadSection"]):
result = gConfig.getOption("%s/%s" % (section, optName), defaultValue)
if result["OK"]:
return result["Value"]
return defaultValue
def ex_getOption( cls, optName, defaultValue = None ):
if defaultValue == None:
if optName in cls.__defaults:
defaultValue = cls.__defaults[ optName ]
if optName and optName[0] == "/":
return gConfig.getValue( optName, defaultValue )
for section in ( cls.__properties[ 'section' ], cls.__properties[ 'loadSection' ] ):
result = gConfig.getOption( "%s/%s" % ( section, optName ), defaultValue )
if result[ 'OK' ]:
return result[ 'Value' ]
return defaultValue
def __getCSStorageElementStatus(self, elementName, statusType, default):
"""
Gets from the CS the StorageElements status
"""
cs_path = "/Resources/StorageElements"
if not isinstance(elementName, list):
elementName = [elementName]
statuses = self.rssConfig.getConfigStatusType('StorageElement')
result = {}
for element in elementName:
if statusType is not None:
# Added Active by default
res = gConfig.getOption(
"%s/%s/%s" % (cs_path, element, statusType), 'Active')
if res['OK'] and res['Value']:
result[element] = {statusType: res['Value']}
else:
res = gConfig.getOptionsDict("%s/%s" % (cs_path, element))
if res['OK'] and res['Value']:
elementStatuses = {}
for elementStatusType, value in res['Value'].items():
if elementStatusType in statuses:
elementStatuses[elementStatusType] = value
# If there is no status defined in the CS, we add by default Read and
# Write as Active.
if elementStatuses == {}:
elementStatuses = {
'ReadAccess': 'Active',
'WriteAccess': 'Active'
}
result[element] = elementStatuses
if result:
return S_OK(result)
if default is not None:
# sec check
if statusType is None:
statusType = 'none'
defList = [[el, statusType, default] for el in elementName]
return S_OK(getDictFromList(defList))
_msg = "StorageElement '%s', with statusType '%s' is unknown for CS."
return S_ERROR(_msg % (elementName, statusType))
def am_getOption( self, optionName, defaultValue = None ):
if defaultValue == None:
if optionName in self.__configDefaults:
defaultValue = self.__configDefaults[ optionName ]
if optionName and optionName[0] == "/":
return gConfig.getValue( optionName, defaultValue )
for section in ( self.__moduleProperties[ 'section' ], self.__moduleProperties[ 'loadSection' ] ):
result = gConfig.getOption( "%s/%s" % ( section, optionName ), defaultValue )
if result[ 'OK' ]:
return result[ 'Value' ]
return defaultValue
def test_verifyCatalogConfiguration(self):
fileMetadataOption = gConfig.getOption(
'Systems/DataManagement/Production/Services/MultiVOFileCatalog/FileMetadata'
)
dirMetadataOption = gConfig.getOption(
'Systems/DataManagement/Production/Services/MultiVOFileCatalog/DirectoryMetadata'
)
masterOption = gConfig.getOption(
'Resources/FileCatalogs/MultiVOFileCatalog/Master')
self.assertTrue(fileMetadataOption['OK'])
self.assertEqual(fileMetadataOption['Value'], 'MultiVOFileMetadata')
self.assertTrue(dirMetadataOption['OK'])
self.assertEqual(dirMetadataOption['Value'],
'MultiVODirectoryMetadata')
self.assertTrue(masterOption['OK'])
self.assertEqual(masterOption['Value'], 'True')
# the default catalog is not a master anymore...
dmasterOption = gConfig.getOption(
'Resources/FileCatalogs/FileCatalog/Master')
self.assertTrue(dmasterOption['OK'])
self.assertEqual(dmasterOption['Value'], 'False')
def __request(self):
req = {}
global pageNumber
global numberOfJobs
global globalSort
if request.params.has_key("id") and len(request.params["id"]) > 0:
pageNumber = 0
req["JobID"] = str(request.params["id"])
elif request.params.has_key("expand") and len(request.params["expand"]) > 0:
globalSort = [["GridSite","ASC"]]
numberOfJobs = 500
pageNumber = 0
req["ExpandSite"] = str(request.params["expand"])
else:
result = gConfig.getOption("/Website/ListSeparator")
if result["OK"]:
separator = result["Value"]
else:
separator = ":::"
pageNumber = 0
numberOfJobs = 500
if request.params.has_key("country") and len(request.params["country"]) > 0:
if str(request.params["country"]) != "All":
code = self.__reverseCountry()
tmpValue = str(request.params["country"]).split(separator)
newValue = []
for i in tmpValue:
if code.has_key(i):
newValue.append(code[i])
req["Country"] = newValue
# req["Country"] = str(request.params["country"]).split(separator)
if request.params.has_key("site") and len(request.params["site"]) > 0:
if str(request.params["site"]) != "All":
req["Site"] = str(request.params["site"]).split(separator)
if request.params.has_key("status") and len(request.params["status"]) > 0:
if str(request.params["status"]) != "All":
req["Status"] = str(request.params["status"]).split(separator)
if request.params.has_key("maskstatus") and len(request.params["maskstatus"]) > 0:
if str(request.params["maskstatus"]) != "All":
req["MaskStatus"] = str(request.params["maskstatus"]).split(separator)
if request.params.has_key("gridtype") and len(request.params["gridtype"]) > 0:
if str(request.params["gridtype"]) != "All":
req["GridType"] = str(request.params["gridtype"]).split(separator)
else:
if request.params.has_key("owner") and len(request.params["owner"]) > 0:
if str(request.params["owner"]) != "All":
req["Owner"] = str(request.params["owner"]).split(separator)
if request.params.has_key("date") and len(request.params["date"]) > 0:
if str(request.params["date"]) != "YYYY-mm-dd":
req["LastUpdate"] = str(request.params["date"])
globalSort = []
gLogger.info("REQUEST:",req)
return req
def __getLaunchpadOpts(self):
options = gConfig.getOptions("/Website/Launchpad/Options", False)
if options and options["OK"]:
options = self.__getDataFromCS()
else:
options = "false"
override = gConfig.getOption("/Website/Launchpad/OptionsOverride")
if override["OK"]:
override = str(override["Value"]).lower()
if override == "true":
pass
else:
override = "false"
else:
override = "false"
separator = gConfig.getOption("/Website/Launchpad/ListSeparator")
if separator["OK"]:
separator = separator["Value"]
else:
separator = "false"
return {"success":"true","result":options,"override":override,"separator":separator}
def uploadTarBall(self):
"""get the tarballURL from the CS and upload the tarball there. Exits when error is encountered"""
gLogger.notice("Uploading TarBall to the Grid")
from ILCDIRAC.Core.Utilities.FileUtils import upload
tarballurl = gConfig.getOption("%(softSec)s/%(platform)s/%(appname)s/TarBallURL" % self.parameter, "")
if not tarballurl['OK'] or not tarballurl['Value']:
gLogger.error('TarBallURL for application %(appname)s not defined' % self.parameter)
dexit(255)
res = upload(tarballurl['Value'], self.appTar)
if not res['OK']:
gLogger.error("Upload to %s failed" % tarballurl['Value'], res['Message'])
dexit(255)
def initialize( self ):
""" Standard constructor
"""
self.jobDB = JobDB()
self.am_setOption( "PollingTime", 120 )
self.__jobDBFields = []
for field in self.__summaryKeyFieldsMapping:
if field == 'User':
field = 'Owner'
elif field == 'UserGroup':
field = 'OwnerGroup'
self.__jobDBFields.append( field )
result = gConfig.getOption( "/Systems/RabbitMQ/User" )
if not result['OK']:
raise RuntimeError( 'Failed to get the configuration parameters: User' )
user = result['Value']
result = gConfig.getOption( "/Systems/RabbitMQ/Password" )
if not result['OK']:
raise RuntimeError( 'Failed to get the configuration parameters: Password' )
password = result['Value']
result = gConfig.getOption( "/Systems/RabbitMQ/Host" )
if not result['OK']:
raise RuntimeError( 'Failed to get the configuration parameters: Host' )
self.host = result['Value']
self.credentials = pika.PlainCredentials( user, password )
self.connection = pika.BlockingConnection( pika.ConnectionParameters( host = self.host, credentials = self.credentials ) )
self.channel = self.connection.channel()
self.channel.exchange_declare( exchange = 'mdatabase',
exchange_type = 'fanout' )
return S_OK()
def ex_getOption(cls, optName, defaultValue=None):
if defaultValue == None:
if optName in cls.__defaults:
defaultValue = cls.__defaults[optName]
if optName and optName[0] == "/":
return gConfig.getValue(optName, defaultValue)
for section in (cls.__properties['section'],
cls.__properties['loadSection']):
result = gConfig.getOption("%s/%s" % (section, optName),
defaultValue)
if result['OK']:
return result['Value']
return defaultValue
def getCAsLocation():
""" Retrieve the CA's files location
"""
#Grid-Security
retVal = gConfig.getOption( '%s/Grid-Security' % g_SecurityConfPath )
if retVal[ 'OK' ]:
casPath = "%s/certificates" % retVal[ 'Value' ]
gLogger.debug( "Trying %s for CAs" % casPath )
if os.path.isdir( casPath ):
gLogger.debug( "Using %s/Grid-Security + /certificates as location for CA's" % g_SecurityConfPath )
return casPath
#CAPath
retVal = gConfig.getOption( '%s/CALocation' % g_SecurityConfPath )
if retVal[ 'OK' ]:
casPath = retVal[ 'Value' ]
gLogger.debug( "Trying %s for CAs" % casPath )
if os.path.isdir( casPath ):
gLogger.debug( "Using %s/CALocation as location for CA's" % g_SecurityConfPath )
return casPath
# Look up the X509_CERT_DIR environment variable
if os.environ.has_key( 'X509_CERT_DIR' ):
gLogger.debug( "Using X509_CERT_DIR env var as location for CA's" )
casPath = os.environ[ 'X509_CERT_DIR' ]
return casPath
#rootPath./etc/grid-security/certificates
casPath = "%s/etc/grid-security/certificates" % DIRAC.rootPath
gLogger.debug( "Trying %s for CAs" % casPath )
if os.path.isdir( casPath ):
gLogger.debug( "Using <DIRACRoot>/etc/grid-security/certificates as location for CA's" )
return casPath
#/etc/grid-security/certificates
casPath = "/etc/grid-security/certificates"
gLogger.debug( "Trying %s for CAs" % casPath )
if os.path.isdir( casPath ):
gLogger.debug( "Using autodiscovered %s location for CA's" % casPath )
return casPath
#No CA's location found
return False
def web_getSelectionData( self ):
sData = self.getSessionData()
callback = {}
user = sData["user"]["username"]
if user == "Anonymous":
self.finish( {"success":"false", "error":"You are not authorize to access these data"} )
if len( self.request.arguments ) > 0:
tmp = {}
for i in self.request.arguments:
tmp[i] = str( self.request.arguments[i] )
callback["extra"] = tmp
rpcClient = RPCClient( "Framework/ProxyManager" )
retVal = yield self.threadTask( rpcClient.getContents, {}, [], 0, 0 )
if not retVal[ "OK" ]:
self.finish( {"success":"false", "error":retVal["Message"]} )
data = retVal[ "Value" ]
users = []
groups = []
for record in data[ "Records" ]:
users.append( str( record[0] ) )
groups.append( str( record[2] ) )
users = uniqueElements( users )
groups = uniqueElements( groups )
users.sort()
groups.sort()
users = map( lambda x: [x], users )
groups = map( lambda x: [x], groups )
callback["username"] = users
callback["usergroup"] = groups
result = gConfig.getOption( "/WebApp/ProxyManagementMonitoring/TimeSpan", "86400,432000,604800,2592000" )
if result["OK"]:
tmp = result["Value"]
tmp = tmp.split( ", " )
if len( tmp ) > 0:
timespan = []
for i in tmp:
human_readable = self.__humanize_time( i )
timespan.append( [i, human_readable] )
else:
timespan = [["Nothing to display"]]
else:
timespan = [["Error during RPC call"]]
callback["expiredBefore"] = timespan
callback["expiredAfter"] = timespan
self.finish( callback )
def am_getOption(self, optionName, defaultValue=None):
""" Gets an option from the agent's configuration section.
The section will be a subsection of the /Systems section in the CS.
"""
if defaultValue is None:
if optionName in self.__configDefaults:
defaultValue = self.__configDefaults[optionName]
if optionName and optionName[0] == "/":
return gConfig.getValue(optionName, defaultValue)
for section in (self.__moduleProperties['section'], self.__moduleProperties['loadSection']):
result = gConfig.getOption("%s/%s" % (section, optionName), defaultValue)
if result['OK']:
return result['Value']
return defaultValue
def __getSelectionData(self):
callback = {}
if not authorizeAction():
return {"success":"false","error":"You are not authorize to access these data"}
if len(request.params) > 0:
tmp = {}
for i in request.params:
tmp[i] = str(request.params[i])
callback["extra"] = tmp
rpcClient = getRPCClient( "Framework/ProxyManager" )
retVal = rpcClient.getContents( {}, [], 0, 0 )
if not retVal[ "OK" ]:
return {"success":"false","error":retVal["Message"]}
data = retVal[ "Value" ]
users = []
groups = []
for record in data[ "Records" ]:
users.append( str(record[0]) )
groups.append( str(record[2]) )
users = uniqueElements(users)
groups = uniqueElements(groups)
users.sort()
groups.sort()
users = map(lambda x: [x], users)
groups = map(lambda x: [x], groups)
if len(users) > 1:
users.insert(0,["All"])
if len(groups) > 1:
groups.insert(0,["All"])
callback["username"] = users
callback["usergroup"] = groups
result = gConfig.getOption("/Website/ProxyManagementMonitoring/TimeSpan")
if result["OK"]:
tmp = result["Value"]
tmp = tmp.split(", ")
if len(tmp)>0:
timespan = []
for i in tmp:
human_readable = self.__humanize_time(i)
timespan.append([i, human_readable])
else:
timespan = [["Nothing to display"]]
else:
timespan = [["Error during RPC call"]]
callback["expiredBefore"] = timespan
callback["expiredAfter"] = timespan
return callback
def initialize( self ):
if not gConfig.useServerCertificate():
res = self.__getProxyID()
else:
res = self.__getCertificateID()
if not res:
return False
retVal = gConfig.getOption( "/DIRAC/Configuration/MasterServer" )
if not retVal[ 'OK' ]:
gLogger.warn( "Master server is not known. Is everything initialized?" )
return False
self.__rpcClient = RPCClient( gConfig.getValue( "/DIRAC/Configuration/MasterServer", "" ) )
self.__csMod = Modificator( self.__rpcClient, "%s - %s" % ( self.__userGroup, self.__userDN ) )
retVal = self.downloadCSData()
if not retVal[ 'OK' ]:
gLogger.error( "Can not download the remote cfg. Is everything initialized?" )
return False
return True
def initialize( self ):
self.SystemLoggingDB = SystemLoggingDB()
self.agentName = self.am_getModuleParam( 'fullName' )
self.notification = NotificationClient()
mailList = self.am_getOption( "MailList", [] )
userString = self.am_getOption( "Reviewer", 'mseco' )
userList = List.fromChar( userString, "," )
self.log.debug( "Users to be notified", ": " + userString )
for user in userList:
retval = gConfig.getOption( "/Registry/Users/" + user + "/email" )
if not retval['OK']:
self.log.warn( "Could not get user's mail", retval['Message'] )
else:
mailList.append( retval['Value'] )
if not mailList:
mailList = gConfig.getValue( '/Operations/EMail/Logging', [] )
if not len( mailList ):
errString = "There are no valid users in the list"
varString = "[" + ','.join( userList ) + "]"
self.log.error( errString, varString )
return S_ERROR( errString + varString )
self.log.info("List of mails to be notified", ','.join(mailList))
self._mailAddress = mailList
self._threshold = int( self.am_getOption( 'Threshold', 10 ) )
self.__days = self.am_getOption( 'QueryPeriod',7 )
self._period=int( self.__days ) * day
self._limit = int ( self.am_getOption( 'NumberOfErrors', 10 ) )
string = "The %i most common errors in the SystemLoggingDB" % self._limit
self._subject = string + " for the last %s days" % self.__days
return S_OK()