def __init__(self, adminUrls=None):
'''initialise'''
adminUrls = adminUrls or {}
self.__soapClients = {}
locs = getHostCertificateAndKeyLocation()
if not locs:
raise RuntimeError("Cannot find the host cert and key files")
hostCert, hostKey = locs
gLogger.info("using host cert: %s" % hostCert)
gLogger.info("using host key: %s" % hostKey)
result = gConfig.getSections('/Registry/VOMS/URLs')
if not result['OK']:
raise Exception(result['Message'])
self.__vos = result['Value']
for vo in self.__vos:
result = gConfig.getOptionsDict('/Registry/VOMS/URLs/%s' % vo)
if not result['OK']:
gLogger.error(result['Message'])
continue
url_dict = result['Value']
if 'VOMSAdmin' not in url_dict:
gLogger.error("Skipping setting up VOMSService for VO: %s "
"as no VOMSAdmin option in config" % vo)
continue
retries = 3
while retries:
try:
admin = adminUrls.get(vo, url_dict['VOMSAdmin'])
httpstransport = HTTPSClientCertTransport(hostCert,
hostKey,
getCAsLocation())
adminClient = Client(admin + '?wsdl',
transport=httpstransport)
adminClient.set_options(headers={"X-VOMS-CSRF-GUARD": "1"})
compatClient = Client(os.path.join(os.path.dirname(admin),
'VOMSCompatibility?wsdl'),
transport=HTTPSClientCertTransport(hostCert,
hostKey,
getCAsLocation()))
compatClient.set_options(headers={"X-VOMS-CSRF-GUARD": "1"})
self.__soapClients[vo] = {'Admin': adminClient, 'Compat': compatClient}
break
except Exception:
gLogger.warn("Failed to connect suds client to VOMSAdmin or VOMSCompatibility URL, retrying...")
retries -= 1
else:
gLogger.error("Maximum number of retries reached. Skipping "
"setting up VOMSService for VO: %s" % vo)
gLogger.error(traceback.format_exc())
if not self.__soapClients:
raise RuntimeError("Couldn't setup ANY SOAP clients")
def __init__(self):
""" c'tor
Just setting defaults
"""
self.workDir = '' # Working directory where the files are going to be stored
self.jsonFile = 'pilot.json' # default filename of the pilot json file
# domain name of the web server(s) used to upload the pilot json file and the pilot scripts
self.pilotFileServer = ''
# pilot sync parameters
self.pilotRepo = 'https://github.com/DIRACGrid/Pilot.git' # repository of the pilot
self.pilotVORepo = '' # repository of the VO that can contain a pilot extension
# 'pilotLocalRepo' = 'pilotLocalRepo' # local repository to be created
# 'pilotVOLocalRepo' = 'pilotVOLocalRepo' # local VO repository to be created
self.pilotSetup = gConfig.getValue('/DIRAC/Setup', '')
self.projectDir = ''
# where the find the pilot scripts in the VO pilot repository
self.pilotScriptPath = 'Pilot' # where the find the pilot scripts in the pilot repository
self.pilotVOScriptPath = ''
self.pilotVersion = ''
self.pilotVOVersion = ''
self.pilotRepoBranch = 'master'
self.pilotVORepoBranch = 'master'
self.certAndKeyLocation = getHostCertificateAndKeyLocation()
self.casLocation = getCAsLocation()
self._checksumDict = {}
# If this is set to True, we will attempt to upload the files
# to all the servers in the list. Obviously, it will only work
# for the DIRAC web servers (see _upload)
self.uploadToWebApp = True
self.log = gLogger.getSubLogger(__name__)
def checkCAs(self):
caDir = getCAsLocation()
if not caDir:
gLogger.warn("No valid CA dir found.")
return
# In globus standards .r0 files are CRLs. They have the same names of the CAs but diffent file extension
searchExp = os.path.join(caDir, "*.r0")
crlList = glob.glob(searchExp)
if not crlList:
gLogger.warn("No CRL files found for %s. Abort check of CAs" % searchExp)
return
newestFPath = max(crlList, key=os.path.getmtime)
newestFTime = os.path.getmtime(newestFPath)
if newestFTime > (time.time() - (2 * 24 * 3600)):
# At least one of the files has been updated in the last 2 days
return S_OK()
if not os.access(caDir, os.W_OK):
gLogger.error("Your CRLs appear to be outdated, but you have no access to update them.")
# Try to continue anyway...
return S_OK()
# Update the CAs & CRLs
gLogger.notice("Your CRLs appear to be outdated; attempting to update them...")
bdc = BundleDeliveryClient()
res = bdc.syncCAs()
if not res["OK"]:
gLogger.error("Failed to update CAs", res["Message"])
res = bdc.syncCRLs()
if not res["OK"]:
gLogger.error("Failed to update CRLs", res["Message"])
# Continue even if the update failed...
return S_OK()
def _downTimeCurlDownload(self, entity=None, startDate=None):
""" Download ongoing downtimes for entity using the GOC DB programmatic interface
"""
# GOCDB-PI url and method settings
#
# Set the GOCDB URL
gocdbpi_url = "https://goc.egi.eu/gocdbpi/public/?method=get_downtime"
# Set the desidered start date
if startDate is None:
when = "&ongoing_only=yes"
gocdbpi_startDate = ""
else:
when = "&startdate="
gocdbpi_startDate = startDate
# GOCDB-PI to query
gocdb_ep = gocdbpi_url
if entity is not None:
if isinstance(entity, basestring):
gocdb_ep = gocdb_ep + "&topentity=" + entity
gocdb_ep = gocdb_ep + when + gocdbpi_startDate + "&scope="
caPath = getCAsLocation()
dtPage = requests.get(gocdb_ep, verify=caPath)
dt = dtPage.text
return dt
def getHostnameDowntime(self, hostname, startDate=None, ongoing=False):
params = hostname
if startDate and ongoing:
return S_ERROR(
"Invalid parameter combination - do not specify startDate with ongoing"
)
if startDate:
params += '&startdate=' + startDate
if ongoing:
params += '&ongoing_only=yes'
caPath = getCAsLocation()
try:
response = requests.get(
'https://goc.egi.eu/gocdbpi/public/?method=get_downtime&topentity='
+ params,
verify=caPath)
response.raise_for_status()
except requests.exceptions.RequestException as e:
return S_ERROR("Error %s" % e)
return S_OK(response.text)
def __init__(self):
""" c'tor
Just setting defaults
"""
self.jsonFile = 'pilot.json' # default filename of the pilot json file
# domain name of the web server used to upload the pilot json file and the pilot scripts
self.pilotFileServer = ''
# pilot sync parameters
self.pilotRepo = 'https://github.com/DIRACGrid/Pilot.git' # repository of the pilot
self.pilotVORepo = '' # repository of the VO that can contain a pilot extension
# 'pilotLocalRepo' = 'pilotLocalRepo' # local repository to be created
# 'pilotVOLocalRepo' = 'pilotVOLocalRepo' # local VO repository to be created
self.pilotSetup = gConfig.getValue('/DIRAC/Setup', '')
self.projectDir = ''
# where the find the pilot scripts in the VO pilot repository
self.pilotScriptPath = 'Pilot' # where the find the pilot scripts in the pilot repository
self.pilotVOScriptPath = ''
self.pilotVersion = ''
self.pilotVOVersion = ''
self.certAndKeyLocation = getHostCertificateAndKeyLocation()
self.casLocation = getCAsLocation()
self.log = gLogger.getSubLogger(__name__)
def _downTimeCurlDownload(self, entity=None, startDate=None):
""" Download ongoing downtimes for entity using the GOC DB programmatic interface
"""
# GOCDB-PI url and method settings
#
# Set the GOCDB URL
gocdbpi_url = "https://goc.egi.eu/gocdbpi/public/?method=get_downtime"
# Set the desidered start date
if startDate is None:
when = "&ongoing_only=yes"
gocdbpi_startDate = ""
else:
when = "&startdate="
gocdbpi_startDate = startDate
# GOCDB-PI to query
gocdb_ep = gocdbpi_url
if entity is not None:
if isinstance(entity, basestring):
gocdb_ep = gocdb_ep + "&topentity=" + entity
gocdb_ep = gocdb_ep + when + gocdbpi_startDate + "&scope="
caPath = getCAsLocation()
dtPage = requests.get(gocdb_ep, verify=caPath)
dt = dtPage.text
return dt
def initialize(self):
"""Initial settings"""
self.workingDirectory = self.am_getOption("WorkDirectory")
self.saveDir = self.am_getOption("SaveDirectory", self.saveDir)
self.uploadLocations = self.am_getOption("UploadLocations", self.uploadLocations)
includeMasterCS = self.am_getOption("IncludeMasterCS", self.includeMasterCS)
if isinstance(includeMasterCS, str) and includeMasterCS.lower() in ["n", "no", "false"]:
self.includeMasterCS = False
self.certAndKeyLocation = getHostCertificateAndKeyLocation()
self.casLocation = getCAsLocation()
return S_OK()
def _downloadJsonFile(self, occupancyLFN, filePath):
""" Download the json file at the location using requests
:param occupancyLFN: this is actually a full https URL
:param filePath: destination path for the file
"""
try:
with open(filePath, 'wt') as fd:
caPath = getCAsLocation()
res = requests.get(occupancyLFN, verify=caPath)
res.raise_for_status()
fd.write(res.content)
except Exception as e:
self.log.debug("Exception while copying", repr(e))
def initialize(self):
""" Initial settings
"""
self.workingDirectory = self.am_getOption('WorkDirectory')
self.saveDir = self.am_getOption('SaveDirectory', self.saveDir)
self.uploadLocations = self.am_getOption('UploadLocations',
self.uploadLocations)
includeMasterCS = self.am_getOption('IncludeMasterCS',
self.includeMasterCS)
if isinstance(includeMasterCS,
six.string_types) and includeMasterCS.lower() in [
'n', 'no', 'false'
]:
self.includeMasterCS = False
self.certAndKeyLocation = getHostCertificateAndKeyLocation()
self.casLocation = getCAsLocation()
return S_OK()
def _getServiceEndpointCurlDownload(self, granularity, entity):
"""
Calls method `get_service_endpoint` from the GOC DB programmatic interface.
:params:
:attr:`granularity` : a string. Could be in ('hostname', 'sitename', 'roc',
'country', 'service_type', 'monitored')
:attr:`entity` : a string. Actual name of the entity.
"""
if not isinstance(granularity, str) or not isinstance(entity, str):
raise ValueError("Arguments must be strings.")
# GOCDB-PI query
gocdb_ep = "https://goc.egi.eu/gocdbpi/public/?method=get_service_endpoint&" + granularity + "=" + entity
caPath = getCAsLocation()
service_endpoint_page = requests.get(gocdb_ep, verify=caPath)
return service_endpoint_page.text
def _getServiceEndpointCurlDownload(self, granularity, entity):
"""
Calls method `get_service_endpoint` from the GOC DB programmatic interface.
:params:
:attr:`granularity` : a string. Could be in ('hostname', 'sitename', 'roc',
'country', 'service_type', 'monitored')
:attr:`entity` : a string. Actual name of the entity.
"""
if not isinstance(granularity, basestring) or not isinstance(entity, basestring):
raise ValueError("Arguments must be strings.")
# GOCDB-PI query
gocdb_ep = "https://goc.egi.eu/gocdbpi/public/?method=get_service_endpoint&" \
+ granularity + '=' + entity
caPath = getCAsLocation()
service_endpoint_page = requests.get(gocdb_ep, verify=caPath)
return service_endpoint_page.text
def getHostnameDowntime(self, hostname, startDate=None, ongoing=False):
params = hostname
if startDate and ongoing:
return S_ERROR("Invalid parameter combination - do not specify startDate with ongoing")
if startDate:
params += '&startdate=' + startDate
if ongoing:
params += '&ongoing_only=yes'
caPath = getCAsLocation()
try:
response = requests.get(
'https://goc.egi.eu/gocdbpi/public/?method=get_downtime&topentity=' + params, verify=caPath)
response.raise_for_status()
except requests.exceptions.RequestException as e:
return S_ERROR("Error %s" % e)
return S_OK(response.text)
def getUsers(self):
"""Get all the users of the VOMS VO with their detailed information
:return: user dictionary keyed by the user DN
"""
if not self.urls:
return S_ERROR(DErrno.ENOAUTH, "No VOMS server defined")
userProxy = getProxyLocation()
caPath = getCAsLocation()
rawUserList = []
result = None
for url in self.urls:
rawUserList = []
startIndex = 0
result = None
error = None
urlDone = False
while not urlDone:
try:
result = requests.get(
url,
headers={"X-VOMS-CSRF-GUARD": "y"},
cert=userProxy,
verify=caPath,
params={"startIndex": str(startIndex), "pageSize": "100"},
)
except requests.ConnectionError as exc:
error = "%s:%s" % (url, repr(exc))
urlDone = True
continue
if result.status_code != 200:
error = "Failed to contact the VOMS server: %s" % result.text
urlDone = True
continue
userList = result.json()["result"]
rawUserList.extend(userList)
if len(userList) < 100:
urlDone = True
startIndex += 100
# This URL did not work, try another one
if error:
continue
else:
break
if error:
return S_ERROR(DErrno.ENOAUTH, "Failed to contact the VOMS server: %s" % error)
# We have got the user info, reformat it
resultDict = {}
for user in rawUserList:
for cert in user["certificates"]:
dn = cert["subjectString"]
resultDict[dn] = user
resultDict[dn]["CA"] = cert["issuerString"]
resultDict[dn]["certSuspended"] = cert.get("suspended")
resultDict[dn]["suspended"] = user.get("suspended")
resultDict[dn]["mail"] = user.get("emailAddress")
resultDict[dn]["Roles"] = user.get("fqans")
attributes = user.get("attributes")
if attributes:
for attribute in user.get("attributes", []):
if attribute.get("name") == "nickname":
resultDict[dn]["nickname"] = attribute.get("value")
self.userDict = dict(resultDict)
return S_OK(resultDict)
def getUsers(self):
""" Get all the users of the VOMS VO with their detailed information
:return: user dictionary keyed by the user DN
"""
if not self.urls:
return S_ERROR(DErrno.ENOAUTH, "No VOMS server defined")
userProxy = getProxyLocation()
caPath = getCAsLocation()
rawUserList = []
result = None
for url in self.urls:
rawUserList = []
startIndex = 0
result = None
error = None
urlDone = False
while not urlDone:
try:
result = requests.get(url,
headers={"X-VOMS-CSRF-GUARD": "y"},
cert=userProxy,
verify=caPath,
params={"startIndex": str(startIndex),
"pageSize": "100"})
except requests.ConnectionError as exc:
error = "%s:%s" % (url, repr(exc))
urlDone = True
continue
if result.status_code != 200:
error = "Failed to contact the VOMS server: %s" % result.text
urlDone = True
continue
userList = result.json()['result']
rawUserList.extend(userList)
if len(userList) < 100:
urlDone = True
startIndex += 100
# This URL did not work, try another one
if error:
continue
else:
break
if error:
return S_ERROR(DErrno.ENOAUTH, "Failed to contact the VOMS server: %s" % error)
# We have got the user info, reformat it
resultDict = {}
for user in rawUserList:
for cert in user['certificates']:
dn = cert['subjectString']
resultDict[dn] = user
resultDict[dn]['CA'] = cert['issuerString']
resultDict[dn]['certSuspended'] = cert['suspended']
resultDict[dn]['certSuspensionReason'] = cert['suspensionReason']
resultDict[dn]['mail'] = user['emailAddress']
resultDict[dn]['Roles'] = user['fqans']
attributes = user.get('attributes')
if attributes:
for attribute in user.get('attributes', []):
if attribute.get('name') == 'nickname':
resultDict[dn]['nickname'] = attribute.get('value')
self.userDict = dict(resultDict)
return S_OK(resultDict)
