def main():
Script.addDefaultOptionValue("/DIRAC/Security/SkipCAChecks", "yes")
Script.parseCommandLine(ignoreErrors=True)
bdc = BundleDeliveryClient()
result = bdc.syncCAs()
if not result["OK"]:
DIRAC.gLogger.error("Error while updating CAs", result["Message"])
DIRAC.exit(1)
elif result["Value"]:
DIRAC.gLogger.notice("CAs got updated")
else:
DIRAC.gLogger.notice("CAs are already synchronized")
result = bdc.syncCRLs()
if not result["OK"]:
DIRAC.gLogger.error("Error while updating CRLs", result["Message"])
DIRAC.exit(1)
elif result["Value"]:
DIRAC.gLogger.notice("CRLs got updated")
else:
DIRAC.gLogger.notice("CRLs are already synchronized")
DIRAC.exit(0)
def checkCAs( self ):
if not "X509_CERT_DIR" in os.environ:
gLogger.warn( "X509_CERT_DIR is unset. Abort check of CAs" )
return
caDir = os.environ[ "X509_CERT_DIR" ]
# In globus standards .r0 files are CRLs. They have the same names of the CAs but diffent file extension
searchExp = os.path.join( caDir, "*.r0" )
crlList = glob.glob( searchExp )
if not crlList:
gLogger.warn( "No CRL files found for %s. Abort check of CAs" % searchExp )
return
newestFPath = max( crlList, key=os.path.getmtime )
newestFTime = os.path.getmtime( newestFPath )
if newestFTime > ( time.time() - ( 2 * 24 * 3600 ) ):
# At least one of the files has been updated in the last 2 days
return S_OK()
if not os.access(caDir, os.W_OK):
gLogger.error("Your CRLs appear to be outdated, but you have no access to update them.")
# Try to continue anyway...
return S_OK()
# Update the CAs & CRLs
gLogger.notice( "Your CRLs appear to be outdated; attempting to update them..." )
bdc = BundleDeliveryClient()
res = bdc.syncCAs()
if not res[ 'OK' ]:
gLogger.error( "Failed to update CAs", res[ 'Message' ] )
res = bdc.syncCRLs()
if not res[ 'OK' ]:
gLogger.error( "Failed to update CRLs", res[ 'Message' ] )
# Continue even if the update failed...
return S_OK()
def checkCAs(self):
if not "X509_CERT_DIR" in os.environ:
gLogger.warn("X509_CERT_DIR is unset. Abort check of CAs")
return
caDir = os.environ["X509_CERT_DIR"]
# In globus standards .r0 files are CRLs. They have the same names of the CAs but diffent file extension
searchExp = os.path.join(caDir, "*.r0")
crlList = glob.glob(searchExp)
if not crlList:
gLogger.warn("No CRL files found for %s. Abort check of CAs" %
searchExp)
return
newestFPath = max(crlList, key=os.path.getmtime)
newestFTime = os.path.getmtime(newestFPath)
if newestFTime > (time.time() - (2 * 24 * 3600)):
# At least one of the files has been updated in the last 2 days
return S_OK()
if not os.access(caDir, os.W_OK):
gLogger.error(
"Your CRLs appear to be outdated, but you have no access to update them."
)
# Try to continue anyway...
return S_OK()
# Update the CAs & CRLs
gLogger.notice(
"Your CRLs appear to be outdated; attempting to update them...")
bdc = BundleDeliveryClient()
res = bdc.syncCAs()
if not res['OK']:
gLogger.error("Failed to update CAs", res['Message'])
res = bdc.syncCRLs()
if not res['OK']:
gLogger.error("Failed to update CRLs", res['Message'])
# Continue even if the update failed...
return S_OK()
def __init__(self,
host,
port,
user=None,
password=None,
indexPrefix='',
useSSL=True):
""" c'tor
:param self: self reference
:param str host: name of the database for example: MonitoringDB
:param str port: The full name of the database for example: 'Monitoring/MonitoringDB'
:param str user: user name to access the db
:param str password: if the db is password protected we need to provide a password
:param str indexPrefix: it is the indexPrefix used to get all indexes
:param bool useSSL: We can disable using secure connection. By default we use secure connection.
"""
self.__indexPrefix = indexPrefix
self._connected = False
if user and password:
sLog.debug("Specified username and password")
if port:
self.__url = "https://%s:%s@%s:%d" % (user, password, host,
port)
else:
self.__url = "https://%s:%s@%s" % (user, password, host)
else:
sLog.debug("Username and password not specified")
if port:
self.__url = "http://%s:%d" % (host, port)
else:
self.__url = "http://%s" % host
if port:
sLog.verbose("Connecting to %s:%s, useSSL = %s" %
(host, port, useSSL))
else:
sLog.verbose("Connecting to %s, useSSL = %s" % (host, useSSL))
if useSSL:
bd = BundleDeliveryClient()
retVal = bd.getCAs()
casFile = None
if not retVal['OK']:
sLog.error("CAs file does not exists:", retVal['Message'])
casFile = certifi.where()
else:
casFile = retVal['Value']
self.__client = Elasticsearch(self.__url,
timeout=self.__timeout,
use_ssl=True,
verify_certs=True,
ca_certs=casFile)
else:
self.__client = Elasticsearch(self.__url, timeout=self.__timeout)
self.__tryToConnect()
def execute(self):
"""The main agent execution method"""
bdc = BundleDeliveryClient()
result = bdc.syncCAs()
if not result["OK"]:
self.log.error("Error while updating CAs", result["Message"])
elif result["Value"]:
self.log.info("CAs got updated")
else:
self.log.info("CAs are already synchronized")
result = bdc.syncCRLs()
if not result["OK"]:
self.log.error("Error while updating CRLs", result["Message"])
elif result["Value"]:
self.log.info("CRLs got updated")
else:
self.log.info("CRLs are already synchronized")
return S_OK()
def __init__(self, host, port, user=None, password=None, indexPrefix='', useSSL=True):
""" c'tor
:param self: self reference
:param str host: name of the database for example: MonitoringDB
:param str port: The full name of the database for example: 'Monitoring/MonitoringDB'
:param str user: user name to access the db
:param str password: if the db is password protected we need to provide a password
:param str indexPrefix: it is the indexPrefix used to get all indexes
:param bool useSSL: We can disable using secure connection. By default we use secure connection.
"""
self.__indexPrefix = indexPrefix
self._connected = False
if user and password:
gLogger.debug("Specified username and password")
self.__url = "https://%s:%s@%s:%d" % (user, password, host, port)
else:
gLogger.debug("Username and password not specified")
self.__url = "http://%s:%d" % (host, port)
gLogger.verbose("Connecting to %s:%s, useSSL = %s" % (host, port, useSSL))
if useSSL:
bd = BundleDeliveryClient()
retVal = bd.getCAs()
casFile = None
if not retVal['OK']:
gLogger.error("CAs file does not exists:", retVal['Message'])
casFile = certifi.where()
else:
casFile = retVal['Value']
self.__client = Elasticsearch(self.__url,
timeout=self.__timeout,
use_ssl=True,
verify_certs=True,
ca_certs=casFile)
else:
self.__client = Elasticsearch(self.__url, timeout=self.__timeout)
gLogger.verbose("ElasticSearchDB URL: %s" % self.__url)
self.__tryToConnect()
def execute(self):
""" The main agent execution method
"""
bdc = BundleDeliveryClient()
result = bdc.syncCAs()
if not result["OK"]:
self.log.error("Error while updating CAs", result["Message"])
elif result["Value"]:
self.log.info("CAs got updated")
else:
self.log.info("CAs are already synchronized")
result = bdc.syncCRLs()
if not result["OK"]:
self.log.error("Error while updating CRLs", result["Message"])
elif result["Value"]:
self.log.info("CRLs got updated")
else:
self.log.info("CRLs are already synchronized")
return S_OK()
if not skipCADownload:
DIRAC.gConfig.setOptionValue('/DIRAC/Security/SkipCAChecks', 'yes')
if not skipCADownload:
Script.enableCS()
try:
dirName = os.path.join(DIRAC.rootPath, 'etc', 'grid-security',
'certificates')
if not os.path.exists(dirName):
os.makedirs(dirName)
except:
DIRAC.gLogger.exception()
DIRAC.gLogger.fatal('Fail to create directory:', dirName)
DIRAC.exit(-1)
try:
from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient
bdc = BundleDeliveryClient()
result = bdc.syncCAs()
if result['OK']:
result = bdc.syncCRLs()
except:
DIRAC.gLogger.exception('Could not import BundleDeliveryClient')
pass
if not skipCAChecks:
Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks')
if ceName or siteName:
# This is used in the pilot context, we should have a proxy, or a certificate, and access to CS
if useServerCert:
# Being sure it was not there before
Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate')
Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate',
def runDiracConfigure(params):
Script.registerSwitch("S:", "Setup=", "Set <setup> as DIRAC setup", params.setSetup)
Script.registerSwitch("e:", "Extensions=", "Set <extensions> as DIRAC extensions", params.setExtensions)
Script.registerSwitch("C:", "ConfigurationServer=", "Set <server> as DIRAC configuration server", params.setServer)
Script.registerSwitch("I", "IncludeAllServers", "include all Configuration Servers", params.setAllServers)
Script.registerSwitch("n:", "SiteName=", "Set <sitename> as DIRAC Site Name", params.setSiteName)
Script.registerSwitch("N:", "CEName=", "Determiner <sitename> from <cename>", params.setCEName)
Script.registerSwitch("V:", "VO=", "Set the VO name", params.setVO)
Script.registerSwitch("W:", "gateway=", "Configure <gateway> as DIRAC Gateway for the site", params.setGateway)
Script.registerSwitch("U", "UseServerCertificate", "Configure to use Server Certificate", params.setServerCert)
Script.registerSwitch("H", "SkipCAChecks", "Configure to skip check of CAs", params.setSkipCAChecks)
Script.registerSwitch("D", "SkipCADownload", "Configure to skip download of CAs", params.setSkipCADownload)
Script.registerSwitch(
"M", "SkipVOMSDownload", "Configure to skip download of VOMS info", params.setSkipVOMSDownload
)
Script.registerSwitch("v", "UseVersionsDir", "Use versions directory", params.setUseVersionsDir)
Script.registerSwitch("A:", "Architecture=", "Configure /Architecture=<architecture>", params.setArchitecture)
Script.registerSwitch("L:", "LocalSE=", "Configure LocalSite/LocalSE=<localse>", params.setLocalSE)
Script.registerSwitch(
"F",
"ForceUpdate",
"Force Update of cfg file (i.e. dirac.cfg) (otherwise nothing happens if dirac.cfg already exists)",
params.forceUpdate,
)
Script.registerSwitch("O:", "output=", "output configuration file", params.setOutput)
Script.parseCommandLine(ignoreErrors=True)
if not params.logLevel:
params.logLevel = DIRAC.gConfig.getValue(cfgInstallPath("LogLevel"), "")
if params.logLevel:
DIRAC.gLogger.setLevel(params.logLevel)
else:
DIRAC.gConfig.setOptionValue(cfgInstallPath("LogLevel"), params.logLevel)
if not params.gatewayServer:
newGatewayServer = DIRAC.gConfig.getValue(cfgInstallPath("Gateway"), "")
if newGatewayServer:
params.setGateway(newGatewayServer)
if not params.configurationServer:
newConfigurationServer = DIRAC.gConfig.getValue(cfgInstallPath("ConfigurationServer"), "")
if newConfigurationServer:
params.setServer(newConfigurationServer)
if not params.includeAllServers:
newIncludeAllServer = DIRAC.gConfig.getValue(cfgInstallPath("IncludeAllServers"), False)
if newIncludeAllServer:
params.setAllServers(True)
if not params.setup:
newSetup = DIRAC.gConfig.getValue(cfgInstallPath("Setup"), "")
if newSetup:
params.setSetup(newSetup)
if not params.siteName:
newSiteName = DIRAC.gConfig.getValue(cfgInstallPath("SiteName"), "")
if newSiteName:
params.setSiteName(newSiteName)
if not params.ceName:
newCEName = DIRAC.gConfig.getValue(cfgInstallPath("CEName"), "")
if newCEName:
params.setCEName(newCEName)
if not params.useServerCert:
newUserServerCert = DIRAC.gConfig.getValue(cfgInstallPath("UseServerCertificate"), False)
if newUserServerCert:
params.setServerCert(newUserServerCert)
if not params.skipCAChecks:
newSkipCAChecks = DIRAC.gConfig.getValue(cfgInstallPath("SkipCAChecks"), False)
if newSkipCAChecks:
params.setSkipCAChecks(newSkipCAChecks)
if not params.skipCADownload:
newSkipCADownload = DIRAC.gConfig.getValue(cfgInstallPath("SkipCADownload"), False)
if newSkipCADownload:
params.setSkipCADownload(newSkipCADownload)
if not params.useVersionsDir:
newUseVersionsDir = DIRAC.gConfig.getValue(cfgInstallPath("UseVersionsDir"), False)
if newUseVersionsDir:
params.setUseVersionsDir(newUseVersionsDir)
# Set proper Defaults in configuration (even if they will be properly overwrite by gComponentInstaller
instancePath = os.path.dirname(os.path.dirname(DIRAC.rootPath))
rootPath = os.path.join(instancePath, "pro")
DIRAC.gConfig.setOptionValue(cfgInstallPath("InstancePath"), instancePath)
DIRAC.gConfig.setOptionValue(cfgInstallPath("RootPath"), rootPath)
if not params.architecture:
newArchitecture = DIRAC.gConfig.getValue(cfgInstallPath("Architecture"), "")
if newArchitecture:
params.setArchitecture(newArchitecture)
if not params.vo:
newVO = DIRAC.gConfig.getValue(cfgInstallPath("VirtualOrganization"), "")
if newVO:
params.setVO(newVO)
if not params.extensions:
newExtensions = DIRAC.gConfig.getValue(cfgInstallPath("Extensions"), "")
if newExtensions:
params.setExtensions(newExtensions)
DIRAC.gLogger.notice("Executing: %s " % (" ".join(sys.argv)))
DIRAC.gLogger.notice('Checking DIRAC installation at "%s"' % DIRAC.rootPath)
if params.update:
if params.outputFile:
DIRAC.gLogger.notice("Will update the output file %s" % params.outputFile)
else:
DIRAC.gLogger.notice("Will update %s" % DIRAC.gConfig.diracConfigFilePath)
if params.setup:
DIRAC.gLogger.verbose("/DIRAC/Setup =", params.setup)
if params.vo:
DIRAC.gLogger.verbose("/DIRAC/VirtualOrganization =", params.vo)
if params.configurationServer:
DIRAC.gLogger.verbose("/DIRAC/Configuration/Servers =", params.configurationServer)
if params.siteName:
DIRAC.gLogger.verbose("/LocalSite/Site =", params.siteName)
if params.architecture:
DIRAC.gLogger.verbose("/LocalSite/Architecture =", params.architecture)
if params.localSE:
DIRAC.gLogger.verbose("/LocalSite/localSE =", params.localSE)
if not params.useServerCert:
DIRAC.gLogger.verbose("/DIRAC/Security/UseServerCertificate =", "no")
# Being sure it was not there before
Script.localCfg.deleteOption("/DIRAC/Security/UseServerCertificate")
Script.localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "no")
else:
DIRAC.gLogger.verbose("/DIRAC/Security/UseServerCertificate =", "yes")
# Being sure it was not there before
Script.localCfg.deleteOption("/DIRAC/Security/UseServerCertificate")
Script.localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
host = DIRAC.gConfig.getValue(cfgInstallPath("Host"), "")
if host:
DIRAC.gConfig.setOptionValue(cfgPath("DIRAC", "Hostname"), host)
if params.skipCAChecks:
DIRAC.gLogger.verbose("/DIRAC/Security/SkipCAChecks =", "yes")
# Being sure it was not there before
Script.localCfg.deleteOption("/DIRAC/Security/SkipCAChecks")
Script.localCfg.addDefaultEntry("/DIRAC/Security/SkipCAChecks", "yes")
else:
# Necessary to allow initial download of CA's
if not params.skipCADownload:
DIRAC.gConfig.setOptionValue("/DIRAC/Security/SkipCAChecks", "yes")
if not params.skipCADownload:
Script.enableCS()
try:
dirName = os.path.join(DIRAC.rootPath, "etc", "grid-security", "certificates")
mkDir(dirName)
except Exception:
DIRAC.gLogger.exception()
DIRAC.gLogger.fatal("Fail to create directory:", dirName)
DIRAC.exit(-1)
try:
bdc = BundleDeliveryClient()
result = bdc.syncCAs()
if result["OK"]:
result = bdc.syncCRLs()
except Exception as e:
DIRAC.gLogger.error("Failed to sync CAs and CRLs: %s" % str(e))
Script.localCfg.deleteOption("/DIRAC/Security/SkipCAChecks")
if params.ceName or params.siteName:
# This is used in the pilot context, we should have a proxy, or a certificate, and access to CS
if params.useServerCert:
# Being sure it was not there before
Script.localCfg.deleteOption("/DIRAC/Security/UseServerCertificate")
Script.localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
Script.enableCS()
# Get the site resource section
gridSections = DIRAC.gConfig.getSections("/Resources/Sites/")
if not gridSections["OK"]:
DIRAC.gLogger.warn("Could not get grid sections list")
grids = []
else:
grids = gridSections["Value"]
# try to get siteName from ceName or Local SE from siteName using Remote Configuration
for grid in grids:
siteSections = DIRAC.gConfig.getSections("/Resources/Sites/%s/" % grid)
if not siteSections["OK"]:
DIRAC.gLogger.warn("Could not get %s site list" % grid)
sites = []
else:
sites = siteSections["Value"]
if not params.siteName:
if params.ceName:
for site in sites:
res = DIRAC.gConfig.getSections("/Resources/Sites/%s/%s/CEs/" % (grid, site), [])
if not res["OK"]:
DIRAC.gLogger.warn("Could not get %s CEs list" % site)
if params.ceName in res["Value"]:
params.siteName = site
break
if params.siteName:
DIRAC.gLogger.notice("Setting /LocalSite/Site = %s" % params.siteName)
Script.localCfg.addDefaultEntry("/LocalSite/Site", params.siteName)
DIRAC.__siteName = False
if params.ceName:
DIRAC.gLogger.notice("Setting /LocalSite/GridCE = %s" % params.ceName)
Script.localCfg.addDefaultEntry("/LocalSite/GridCE", params.ceName)
if not params.localSE and params.siteName in sites:
params.localSE = getSEsForSite(params.siteName)
if params.localSE["OK"] and params.localSE["Value"]:
params.localSE = ",".join(params.localSE["Value"])
DIRAC.gLogger.notice("Setting /LocalSite/LocalSE =", params.localSE)
Script.localCfg.addDefaultEntry("/LocalSite/LocalSE", params.localSE)
break
if params.gatewayServer:
DIRAC.gLogger.verbose("/DIRAC/Gateways/%s =" % DIRAC.siteName(), params.gatewayServer)
Script.localCfg.addDefaultEntry("/DIRAC/Gateways/%s" % DIRAC.siteName(), params.gatewayServer)
# Create the local cfg if it is not yet there
if not params.outputFile:
params.outputFile = DIRAC.gConfig.diracConfigFilePath
params.outputFile = os.path.abspath(params.outputFile)
if not os.path.exists(params.outputFile):
configDir = os.path.dirname(params.outputFile)
mkDir(configDir)
params.update = True
DIRAC.gConfig.dumpLocalCFGToFile(params.outputFile)
if params.includeAllServers:
# We need user proxy or server certificate to continue in order to get all the CS URLs
if not params.useServerCert:
Script.enableCS()
result = getProxyInfo()
if not result["OK"]:
DIRAC.gLogger.notice("Configuration is not completed because no user proxy is available")
DIRAC.gLogger.notice("Create one using dirac-proxy-init and execute again with -F option")
return 1
else:
Script.localCfg.deleteOption("/DIRAC/Security/UseServerCertificate")
# When using Server Certs CA's will be checked, the flag only disables initial download
# this will be replaced by the use of SkipCADownload
Script.localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
Script.enableCS()
DIRAC.gConfig.setOptionValue("/DIRAC/Configuration/Servers", ",".join(DIRAC.gConfig.getServersList()))
DIRAC.gLogger.verbose("/DIRAC/Configuration/Servers =", ",".join(DIRAC.gConfig.getServersList()))
if params.useServerCert:
# always removing before dumping
Script.localCfg.deleteOption("/DIRAC/Security/UseServerCertificate")
Script.localCfg.deleteOption("/DIRAC/Security/SkipCAChecks")
Script.localCfg.deleteOption("/DIRAC/Security/SkipVOMSDownload")
if params.update:
DIRAC.gConfig.dumpLocalCFGToFile(params.outputFile)
# ## LAST PART: do the vomsdir/vomses magic
# This has to be done for all VOs in the installation
if params.skipVOMSDownload:
return 0
result = Registry.getVOMSServerInfo()
if not result["OK"]:
return 1
error = ""
vomsDict = result["Value"]
for vo in vomsDict:
voName = vomsDict[vo]["VOMSName"]
vomsDirPath = os.path.join(DIRAC.rootPath, "etc", "grid-security", "vomsdir", voName)
vomsesDirPath = os.path.join(DIRAC.rootPath, "etc", "grid-security", "vomses")
for path in (vomsDirPath, vomsesDirPath):
mkDir(path)
vomsesLines = []
for vomsHost in vomsDict[vo].get("Servers", {}):
hostFilePath = os.path.join(vomsDirPath, "%s.lsc" % vomsHost)
try:
DN = vomsDict[vo]["Servers"][vomsHost]["DN"]
CA = vomsDict[vo]["Servers"][vomsHost]["CA"]
port = vomsDict[vo]["Servers"][vomsHost]["Port"]
if not DN or not CA or not port:
DIRAC.gLogger.error("DN = %s" % DN)
DIRAC.gLogger.error("CA = %s" % CA)
DIRAC.gLogger.error("Port = %s" % port)
DIRAC.gLogger.error("Missing Parameter for %s" % vomsHost)
continue
with open(hostFilePath, "wt") as fd:
fd.write("%s\n%s\n" % (DN, CA))
vomsesLines.append('"%s" "%s" "%s" "%s" "%s" "24"' % (voName, vomsHost, port, DN, voName))
DIRAC.gLogger.notice("Created vomsdir file %s" % hostFilePath)
except Exception:
DIRAC.gLogger.exception("Could not generate vomsdir file for host", vomsHost)
error = "Could not generate vomsdir file for VO %s, host %s" % (voName, vomsHost)
try:
vomsesFilePath = os.path.join(vomsesDirPath, voName)
with open(vomsesFilePath, "wt") as fd:
fd.write("%s\n" % "\n".join(vomsesLines))
DIRAC.gLogger.notice("Created vomses file %s" % vomsesFilePath)
except Exception:
DIRAC.gLogger.exception("Could not generate vomses file")
error = "Could not generate vomses file for VO %s" % voName
if params.useServerCert:
Script.localCfg.deleteOption("/DIRAC/Security/UseServerCertificate")
# When using Server Certs CA's will be checked, the flag only disables initial download
# this will be replaced by the use of SkipCADownload
Script.localCfg.deleteOption("/DIRAC/Security/SkipCAChecks")
if error:
return 1
return 0
else:
# Necessary to allow initial download of CA's
if not skipCADownload:
DIRAC.gConfig.setOptionValue( '/DIRAC/Security/SkipCAChecks', 'yes' )
if not skipCADownload:
Script.enableCS()
try:
dirName = os.path.join( DIRAC.rootPath, 'etc', 'grid-security', 'certificates' )
mkDir(dirName)
except:
DIRAC.gLogger.exception()
DIRAC.gLogger.fatal( 'Fail to create directory:', dirName )
DIRAC.exit( -1 )
try:
from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient
bdc = BundleDeliveryClient()
result = bdc.syncCAs()
if result[ 'OK' ]:
result = bdc.syncCRLs()
except:
DIRAC.gLogger.exception( 'Could not import BundleDeliveryClient' )
pass
if not skipCAChecks:
Script.localCfg.deleteOption( '/DIRAC/Security/SkipCAChecks' )
if ceName or siteName:
# This is used in the pilot context, we should have a proxy, or a certificate, and access to CS
if useServerCert:
# Being sure it was not there before
Script.localCfg.deleteOption( '/DIRAC/Security/UseServerCertificate' )
Script.localCfg.addDefaultEntry( '/DIRAC/Security/UseServerCertificate', 'yes' )
def __init__(self,
host,
port,
user=None,
password=None,
indexPrefix='',
useSSL=True):
""" c'tor
:param self: self reference
:param str host: name of the database for example: MonitoringDB
:param str port: The full name of the database for example: 'Monitoring/MonitoringDB'
:param str user: user name to access the db
:param str password: if the db is password protected we need to provide a password
:param str indexPrefix: it is the indexPrefix used to get all indexes
:param bool useSSL: We can disable using secure connection. By default we use secure connection.
"""
self.__indexPrefix = indexPrefix
self._connected = False
if user and password:
sLog.debug("Specified username and password")
if port:
self.__url = "https://%s:%s@%s:%d" % (user, password, host,
port)
else:
self.__url = "https://%s:%s@%s" % (user, password, host)
else:
sLog.debug("Username and password not specified")
if port:
self.__url = "http://%s:%d" % (host, port)
else:
self.__url = "http://%s" % host
if port:
sLog.verbose("Connecting to %s:%s, useSSL = %s" %
(host, port, useSSL))
else:
sLog.verbose("Connecting to %s, useSSL = %s" % (host, useSSL))
if useSSL:
bd = BundleDeliveryClient()
retVal = bd.getCAs()
casFile = None
if not retVal['OK']:
sLog.error("CAs file does not exists:", retVal['Message'])
casFile = certifi.where()
else:
casFile = retVal['Value']
self.client = Elasticsearch(self.__url,
timeout=self.__timeout,
use_ssl=True,
verify_certs=True,
ca_certs=casFile)
else:
self.client = Elasticsearch(self.__url, timeout=self.__timeout)
# Before we use the database we try to connect
# and retrieve the cluster name
try:
if self.client.ping():
# Returns True if the cluster is running, False otherwise
result = self.client.info()
self.clusterName = result.get("cluster_name", " ") # pylint: disable=no-member
sLog.info("Database info\n", json.dumps(result, indent=4))
self._connected = True
else:
sLog.error("Cannot ping ElasticsearchDB!")
except ConnectionError as e:
sLog.error(repr(e))
def main():
global logLevel
global setup
global configurationServer
global includeAllServers
global gatewayServer
global siteName
global useServerCert
global skipCAChecks
global skipCADownload
global useVersionsDir
global architecture
global localSE
global ceName
global vo
global update
global outputFile
global skipVOMSDownload
global extensions
Script.disableCS()
Script.registerSwitch("S:", "Setup=", "Set <setup> as DIRAC setup",
setSetup)
Script.registerSwitch("e:", "Extensions=",
"Set <extensions> as DIRAC extensions",
setExtensions)
Script.registerSwitch("C:", "ConfigurationServer=",
"Set <server> as DIRAC configuration server",
setServer)
Script.registerSwitch("I", "IncludeAllServers",
"include all Configuration Servers", setAllServers)
Script.registerSwitch("n:", "SiteName=",
"Set <sitename> as DIRAC Site Name", setSiteName)
Script.registerSwitch("N:", "CEName=",
"Determiner <sitename> from <cename>", setCEName)
Script.registerSwitch("V:", "VO=", "Set the VO name", setVO)
Script.registerSwitch("W:", "gateway=",
"Configure <gateway> as DIRAC Gateway for the site",
setGateway)
Script.registerSwitch("U", "UseServerCertificate",
"Configure to use Server Certificate", setServerCert)
Script.registerSwitch("H", "SkipCAChecks",
"Configure to skip check of CAs", setSkipCAChecks)
Script.registerSwitch("D", "SkipCADownload",
"Configure to skip download of CAs",
setSkipCADownload)
Script.registerSwitch("M", "SkipVOMSDownload",
"Configure to skip download of VOMS info",
setSkipVOMSDownload)
Script.registerSwitch("v", "UseVersionsDir", "Use versions directory",
setUseVersionsDir)
Script.registerSwitch("A:", "Architecture=",
"Configure /Architecture=<architecture>",
setArchitecture)
Script.registerSwitch("L:", "LocalSE=",
"Configure LocalSite/LocalSE=<localse>", setLocalSE)
Script.registerSwitch(
"F", "ForceUpdate",
"Force Update of cfg file (i.e. dirac.cfg) (otherwise nothing happens if dirac.cfg already exists)",
forceUpdate)
Script.registerSwitch("O:", "output=", "output configuration file",
setOutput)
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], '\nUsage:',
' %s [options] ...\n' % Script.scriptName
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getExtraCLICFGFiles()
if not logLevel:
logLevel = DIRAC.gConfig.getValue(cfgInstallPath('LogLevel'), '')
if logLevel:
DIRAC.gLogger.setLevel(logLevel)
else:
DIRAC.gConfig.setOptionValue(cfgInstallPath('LogLevel'), logLevel)
if not gatewayServer:
newGatewayServer = DIRAC.gConfig.getValue(cfgInstallPath('Gateway'),
'')
if newGatewayServer:
setGateway(newGatewayServer)
if not configurationServer:
newConfigurationServer = DIRAC.gConfig.getValue(
cfgInstallPath('ConfigurationServer'), '')
if newConfigurationServer:
setServer(newConfigurationServer)
if not includeAllServers:
newIncludeAllServer = DIRAC.gConfig.getValue(
cfgInstallPath('IncludeAllServers'), False)
if newIncludeAllServer:
setAllServers(True)
if not setup:
newSetup = DIRAC.gConfig.getValue(cfgInstallPath('Setup'), '')
if newSetup:
setSetup(newSetup)
if not siteName:
newSiteName = DIRAC.gConfig.getValue(cfgInstallPath('SiteName'), '')
if newSiteName:
setSiteName(newSiteName)
if not ceName:
newCEName = DIRAC.gConfig.getValue(cfgInstallPath('CEName'), '')
if newCEName:
setCEName(newCEName)
if not useServerCert:
newUserServerCert = DIRAC.gConfig.getValue(
cfgInstallPath('UseServerCertificate'), False)
if newUserServerCert:
setServerCert(newUserServerCert)
if not skipCAChecks:
newSkipCAChecks = DIRAC.gConfig.getValue(
cfgInstallPath('SkipCAChecks'), False)
if newSkipCAChecks:
setSkipCAChecks(newSkipCAChecks)
if not skipCADownload:
newSkipCADownload = DIRAC.gConfig.getValue(
cfgInstallPath('SkipCADownload'), False)
if newSkipCADownload:
setSkipCADownload(newSkipCADownload)
if not useVersionsDir:
newUseVersionsDir = DIRAC.gConfig.getValue(
cfgInstallPath('UseVersionsDir'), False)
if newUseVersionsDir:
setUseVersionsDir(newUseVersionsDir)
# Set proper Defaults in configuration (even if they will be properly overwrite by gComponentInstaller
instancePath = os.path.dirname(os.path.dirname(DIRAC.rootPath))
rootPath = os.path.join(instancePath, 'pro')
DIRAC.gConfig.setOptionValue(cfgInstallPath('InstancePath'),
instancePath)
DIRAC.gConfig.setOptionValue(cfgInstallPath('RootPath'), rootPath)
if not architecture:
newArchitecture = DIRAC.gConfig.getValue(
cfgInstallPath('Architecture'), '')
if newArchitecture:
setArchitecture(newArchitecture)
if not vo:
newVO = DIRAC.gConfig.getValue(cfgInstallPath('VirtualOrganization'),
'')
if newVO:
setVO(newVO)
if not extensions:
newExtensions = DIRAC.gConfig.getValue(cfgInstallPath('Extensions'),
'')
if newExtensions:
setExtensions(newExtensions)
DIRAC.gLogger.notice('Executing: %s ' % (' '.join(sys.argv)))
DIRAC.gLogger.notice('Checking DIRAC installation at "%s"' %
DIRAC.rootPath)
if update:
if outputFile:
DIRAC.gLogger.notice('Will update the output file %s' % outputFile)
else:
DIRAC.gLogger.notice('Will update %s' %
DIRAC.gConfig.diracConfigFilePath)
if setup:
DIRAC.gLogger.verbose('/DIRAC/Setup =', setup)
if vo:
DIRAC.gLogger.verbose('/DIRAC/VirtualOrganization =', vo)
if configurationServer:
DIRAC.gLogger.verbose('/DIRAC/Configuration/Servers =',
configurationServer)
if siteName:
DIRAC.gLogger.verbose('/LocalSite/Site =', siteName)
if architecture:
DIRAC.gLogger.verbose('/LocalSite/Architecture =', architecture)
if localSE:
DIRAC.gLogger.verbose('/LocalSite/localSE =', localSE)
if not useServerCert:
DIRAC.gLogger.verbose('/DIRAC/Security/UseServerCertificate =', 'no')
# Being sure it was not there before
Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate')
Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate',
'no')
else:
DIRAC.gLogger.verbose('/DIRAC/Security/UseServerCertificate =', 'yes')
# Being sure it was not there before
Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate')
Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate',
'yes')
host = DIRAC.gConfig.getValue(cfgInstallPath("Host"), "")
if host:
DIRAC.gConfig.setOptionValue(cfgPath("DIRAC", "Hostname"), host)
if skipCAChecks:
DIRAC.gLogger.verbose('/DIRAC/Security/SkipCAChecks =', 'yes')
# Being sure it was not there before
Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks')
Script.localCfg.addDefaultEntry('/DIRAC/Security/SkipCAChecks', 'yes')
else:
# Necessary to allow initial download of CA's
if not skipCADownload:
DIRAC.gConfig.setOptionValue('/DIRAC/Security/SkipCAChecks', 'yes')
if not skipCADownload:
Script.enableCS()
try:
dirName = os.path.join(DIRAC.rootPath, 'etc', 'grid-security',
'certificates')
mkDir(dirName)
except BaseException:
DIRAC.gLogger.exception()
DIRAC.gLogger.fatal('Fail to create directory:', dirName)
DIRAC.exit(-1)
try:
bdc = BundleDeliveryClient()
result = bdc.syncCAs()
if result['OK']:
result = bdc.syncCRLs()
except Exception as e:
DIRAC.gLogger.error('Failed to sync CAs and CRLs: %s' % str(e))
if not skipCAChecks:
Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks')
if ceName or siteName:
# This is used in the pilot context, we should have a proxy, or a certificate, and access to CS
if useServerCert:
# Being sure it was not there before
Script.localCfg.deleteOption(
'/DIRAC/Security/UseServerCertificate')
Script.localCfg.addDefaultEntry(
'/DIRAC/Security/UseServerCertificate', 'yes')
Script.enableCS()
# Get the site resource section
gridSections = DIRAC.gConfig.getSections('/Resources/Sites/')
if not gridSections['OK']:
DIRAC.gLogger.warn('Could not get grid sections list')
grids = []
else:
grids = gridSections['Value']
# try to get siteName from ceName or Local SE from siteName using Remote Configuration
for grid in grids:
siteSections = DIRAC.gConfig.getSections('/Resources/Sites/%s/' %
grid)
if not siteSections['OK']:
DIRAC.gLogger.warn('Could not get %s site list' % grid)
sites = []
else:
sites = siteSections['Value']
if not siteName:
if ceName:
for site in sites:
res = DIRAC.gConfig.getSections(
'/Resources/Sites/%s/%s/CEs/' % (grid, site), [])
if not res['OK']:
DIRAC.gLogger.warn('Could not get %s CEs list' %
site)
if ceName in res['Value']:
siteName = site
break
if siteName:
DIRAC.gLogger.notice('Setting /LocalSite/Site = %s' % siteName)
Script.localCfg.addDefaultEntry('/LocalSite/Site', siteName)
DIRAC.__siteName = False
if ceName:
DIRAC.gLogger.notice('Setting /LocalSite/GridCE = %s' %
ceName)
Script.localCfg.addDefaultEntry('/LocalSite/GridCE',
ceName)
if not localSE and siteName in sites:
localSE = getSEsForSite(siteName)
if localSE['OK'] and localSE['Value']:
localSE = ','.join(localSE['Value'])
DIRAC.gLogger.notice('Setting /LocalSite/LocalSE =',
localSE)
Script.localCfg.addDefaultEntry(
'/LocalSite/LocalSE', localSE)
break
if gatewayServer:
DIRAC.gLogger.verbose('/DIRAC/Gateways/%s =' % DIRAC.siteName(),
gatewayServer)
Script.localCfg.addDefaultEntry(
'/DIRAC/Gateways/%s' % DIRAC.siteName(), gatewayServer)
# Create the local cfg if it is not yet there
if not outputFile:
outputFile = DIRAC.gConfig.diracConfigFilePath
outputFile = os.path.abspath(outputFile)
if not os.path.exists(outputFile):
configDir = os.path.dirname(outputFile)
mkDir(configDir)
update = True
DIRAC.gConfig.dumpLocalCFGToFile(outputFile)
if includeAllServers:
# We need user proxy or server certificate to continue in order to get all the CS URLs
if not useServerCert:
Script.enableCS()
result = getProxyInfo()
if not result['OK']:
DIRAC.gLogger.notice(
'Configuration is not completed because no user proxy is available'
)
DIRAC.gLogger.notice(
'Create one using dirac-proxy-init and execute again with -F option'
)
sys.exit(1)
else:
Script.localCfg.deleteOption(
'/DIRAC/Security/UseServerCertificate')
# When using Server Certs CA's will be checked, the flag only disables initial download
# this will be replaced by the use of SkipCADownload
Script.localCfg.addDefaultEntry(
'/DIRAC/Security/UseServerCertificate', 'yes')
Script.enableCS()
DIRAC.gConfig.setOptionValue('/DIRAC/Configuration/Servers',
','.join(DIRAC.gConfig.getServersList()))
DIRAC.gLogger.verbose('/DIRAC/Configuration/Servers =',
','.join(DIRAC.gConfig.getServersList()))
if useServerCert:
# always removing before dumping
Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate')
Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks')
Script.localCfg.deleteOption('/DIRAC/Security/SkipVOMSDownload')
if update:
DIRAC.gConfig.dumpLocalCFGToFile(outputFile)
# ## LAST PART: do the vomsdir/vomses magic
# This has to be done for all VOs in the installation
if skipVOMSDownload:
# We stop here
sys.exit(0)
result = Registry.getVOMSServerInfo()
if not result['OK']:
sys.exit(1)
error = ''
vomsDict = result['Value']
for vo in vomsDict:
voName = vomsDict[vo]['VOMSName']
vomsDirPath = os.path.join(DIRAC.rootPath, 'etc', 'grid-security',
'vomsdir', voName)
vomsesDirPath = os.path.join(DIRAC.rootPath, 'etc', 'grid-security',
'vomses')
for path in (vomsDirPath, vomsesDirPath):
mkDir(path)
vomsesLines = []
for vomsHost in vomsDict[vo].get('Servers', {}):
hostFilePath = os.path.join(vomsDirPath, "%s.lsc" % vomsHost)
try:
DN = vomsDict[vo]['Servers'][vomsHost]['DN']
CA = vomsDict[vo]['Servers'][vomsHost]['CA']
port = vomsDict[vo]['Servers'][vomsHost]['Port']
if not DN or not CA or not port:
DIRAC.gLogger.error('DN = %s' % DN)
DIRAC.gLogger.error('CA = %s' % CA)
DIRAC.gLogger.error('Port = %s' % port)
DIRAC.gLogger.error('Missing Parameter for %s' % vomsHost)
continue
with open(hostFilePath, "wt") as fd:
fd.write("%s\n%s\n" % (DN, CA))
vomsesLines.append('"%s" "%s" "%s" "%s" "%s" "24"' %
(voName, vomsHost, port, DN, voName))
DIRAC.gLogger.notice("Created vomsdir file %s" % hostFilePath)
except Exception:
DIRAC.gLogger.exception(
"Could not generate vomsdir file for host", vomsHost)
error = "Could not generate vomsdir file for VO %s, host %s" % (
voName, vomsHost)
try:
vomsesFilePath = os.path.join(vomsesDirPath, voName)
with open(vomsesFilePath, "wt") as fd:
fd.write("%s\n" % "\n".join(vomsesLines))
DIRAC.gLogger.notice("Created vomses file %s" % vomsesFilePath)
except Exception:
DIRAC.gLogger.exception("Could not generate vomses file")
error = "Could not generate vomses file for VO %s" % voName
if useServerCert:
Script.localCfg.deleteOption('/DIRAC/Security/UseServerCertificate')
# When using Server Certs CA's will be checked, the flag only disables initial download
# this will be replaced by the use of SkipCADownload
Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks')
if error:
sys.exit(1)
sys.exit(0)
