def __init__(self, adminUrls=None): '''initialise''' adminUrls = adminUrls or {} self.__soapClients = {} locs = getHostCertificateAndKeyLocation() if not locs: raise RuntimeError("Cannot find the host cert and key files") hostCert, hostKey = locs gLogger.info("using host cert: %s" % hostCert) gLogger.info("using host key: %s" % hostKey) result = gConfig.getSections('/Registry/VOMS/URLs') if not result['OK']: raise Exception(result['Message']) self.__vos = result['Value'] for vo in self.__vos: result = gConfig.getOptionsDict('/Registry/VOMS/URLs/%s' % vo) if not result['OK']: gLogger.error(result['Message']) continue url_dict = result['Value'] if 'VOMSAdmin' not in url_dict: gLogger.error("Skipping setting up VOMSService for VO: %s " "as no VOMSAdmin option in config" % vo) continue retries = 3 while retries: try: admin = adminUrls.get(vo, url_dict['VOMSAdmin']) httpstransport = HTTPSClientCertTransport(hostCert, hostKey, getCAsLocation()) adminClient = Client(admin + '?wsdl', transport=httpstransport) adminClient.set_options(headers={"X-VOMS-CSRF-GUARD": "1"}) compatClient = Client(os.path.join(os.path.dirname(admin), 'VOMSCompatibility?wsdl'), transport=HTTPSClientCertTransport(hostCert, hostKey, getCAsLocation())) compatClient.set_options(headers={"X-VOMS-CSRF-GUARD": "1"}) self.__soapClients[vo] = {'Admin': adminClient, 'Compat': compatClient} break except Exception: gLogger.warn("Failed to connect suds client to VOMSAdmin or VOMSCompatibility URL, retrying...") retries -= 1 else: gLogger.error("Maximum number of retries reached. Skipping " "setting up VOMSService for VO: %s" % vo) gLogger.error(traceback.format_exc()) if not self.__soapClients: raise RuntimeError("Couldn't setup ANY SOAP clients")
def __init__(self): """ c'tor Just setting defaults """ self.workDir = '' # Working directory where the files are going to be stored self.jsonFile = 'pilot.json' # default filename of the pilot json file # domain name of the web server(s) used to upload the pilot json file and the pilot scripts self.pilotFileServer = '' # pilot sync parameters self.pilotRepo = 'https://github.com/DIRACGrid/Pilot.git' # repository of the pilot self.pilotVORepo = '' # repository of the VO that can contain a pilot extension # 'pilotLocalRepo' = 'pilotLocalRepo' # local repository to be created # 'pilotVOLocalRepo' = 'pilotVOLocalRepo' # local VO repository to be created self.pilotSetup = gConfig.getValue('/DIRAC/Setup', '') self.projectDir = '' # where the find the pilot scripts in the VO pilot repository self.pilotScriptPath = 'Pilot' # where the find the pilot scripts in the pilot repository self.pilotVOScriptPath = '' self.pilotVersion = '' self.pilotVOVersion = '' self.pilotRepoBranch = 'master' self.pilotVORepoBranch = 'master' self.certAndKeyLocation = getHostCertificateAndKeyLocation() self.casLocation = getCAsLocation() self._checksumDict = {} # If this is set to True, we will attempt to upload the files # to all the servers in the list. Obviously, it will only work # for the DIRAC web servers (see _upload) self.uploadToWebApp = True self.log = gLogger.getSubLogger(__name__)
def checkCAs(self): caDir = getCAsLocation() if not caDir: gLogger.warn("No valid CA dir found.") return # In globus standards .r0 files are CRLs. They have the same names of the CAs but diffent file extension searchExp = os.path.join(caDir, "*.r0") crlList = glob.glob(searchExp) if not crlList: gLogger.warn("No CRL files found for %s. Abort check of CAs" % searchExp) return newestFPath = max(crlList, key=os.path.getmtime) newestFTime = os.path.getmtime(newestFPath) if newestFTime > (time.time() - (2 * 24 * 3600)): # At least one of the files has been updated in the last 2 days return S_OK() if not os.access(caDir, os.W_OK): gLogger.error("Your CRLs appear to be outdated, but you have no access to update them.") # Try to continue anyway... return S_OK() # Update the CAs & CRLs gLogger.notice("Your CRLs appear to be outdated; attempting to update them...") bdc = BundleDeliveryClient() res = bdc.syncCAs() if not res["OK"]: gLogger.error("Failed to update CAs", res["Message"]) res = bdc.syncCRLs() if not res["OK"]: gLogger.error("Failed to update CRLs", res["Message"]) # Continue even if the update failed... return S_OK()
def _downTimeCurlDownload(self, entity=None, startDate=None): """ Download ongoing downtimes for entity using the GOC DB programmatic interface """ # GOCDB-PI url and method settings # # Set the GOCDB URL gocdbpi_url = "https://goc.egi.eu/gocdbpi/public/?method=get_downtime" # Set the desidered start date if startDate is None: when = "&ongoing_only=yes" gocdbpi_startDate = "" else: when = "&startdate=" gocdbpi_startDate = startDate # GOCDB-PI to query gocdb_ep = gocdbpi_url if entity is not None: if isinstance(entity, basestring): gocdb_ep = gocdb_ep + "&topentity=" + entity gocdb_ep = gocdb_ep + when + gocdbpi_startDate + "&scope=" caPath = getCAsLocation() dtPage = requests.get(gocdb_ep, verify=caPath) dt = dtPage.text return dt
def getHostnameDowntime(self, hostname, startDate=None, ongoing=False): params = hostname if startDate and ongoing: return S_ERROR( "Invalid parameter combination - do not specify startDate with ongoing" ) if startDate: params += '&startdate=' + startDate if ongoing: params += '&ongoing_only=yes' caPath = getCAsLocation() try: response = requests.get( 'https://goc.egi.eu/gocdbpi/public/?method=get_downtime&topentity=' + params, verify=caPath) response.raise_for_status() except requests.exceptions.RequestException as e: return S_ERROR("Error %s" % e) return S_OK(response.text)
def __init__(self): """ c'tor Just setting defaults """ self.jsonFile = 'pilot.json' # default filename of the pilot json file # domain name of the web server used to upload the pilot json file and the pilot scripts self.pilotFileServer = '' # pilot sync parameters self.pilotRepo = 'https://github.com/DIRACGrid/Pilot.git' # repository of the pilot self.pilotVORepo = '' # repository of the VO that can contain a pilot extension # 'pilotLocalRepo' = 'pilotLocalRepo' # local repository to be created # 'pilotVOLocalRepo' = 'pilotVOLocalRepo' # local VO repository to be created self.pilotSetup = gConfig.getValue('/DIRAC/Setup', '') self.projectDir = '' # where the find the pilot scripts in the VO pilot repository self.pilotScriptPath = 'Pilot' # where the find the pilot scripts in the pilot repository self.pilotVOScriptPath = '' self.pilotVersion = '' self.pilotVOVersion = '' self.certAndKeyLocation = getHostCertificateAndKeyLocation() self.casLocation = getCAsLocation() self.log = gLogger.getSubLogger(__name__)
def _downTimeCurlDownload(self, entity=None, startDate=None): """ Download ongoing downtimes for entity using the GOC DB programmatic interface """ # GOCDB-PI url and method settings # # Set the GOCDB URL gocdbpi_url = "https://goc.egi.eu/gocdbpi/public/?method=get_downtime" # Set the desidered start date if startDate is None: when = "&ongoing_only=yes" gocdbpi_startDate = "" else: when = "&startdate=" gocdbpi_startDate = startDate # GOCDB-PI to query gocdb_ep = gocdbpi_url if entity is not None: if isinstance(entity, basestring): gocdb_ep = gocdb_ep + "&topentity=" + entity gocdb_ep = gocdb_ep + when + gocdbpi_startDate + "&scope=" caPath = getCAsLocation() dtPage = requests.get(gocdb_ep, verify=caPath) dt = dtPage.text return dt
def initialize(self): """Initial settings""" self.workingDirectory = self.am_getOption("WorkDirectory") self.saveDir = self.am_getOption("SaveDirectory", self.saveDir) self.uploadLocations = self.am_getOption("UploadLocations", self.uploadLocations) includeMasterCS = self.am_getOption("IncludeMasterCS", self.includeMasterCS) if isinstance(includeMasterCS, str) and includeMasterCS.lower() in ["n", "no", "false"]: self.includeMasterCS = False self.certAndKeyLocation = getHostCertificateAndKeyLocation() self.casLocation = getCAsLocation() return S_OK()
def _downloadJsonFile(self, occupancyLFN, filePath): """ Download the json file at the location using requests :param occupancyLFN: this is actually a full https URL :param filePath: destination path for the file """ try: with open(filePath, 'wt') as fd: caPath = getCAsLocation() res = requests.get(occupancyLFN, verify=caPath) res.raise_for_status() fd.write(res.content) except Exception as e: self.log.debug("Exception while copying", repr(e))
def initialize(self): """ Initial settings """ self.workingDirectory = self.am_getOption('WorkDirectory') self.saveDir = self.am_getOption('SaveDirectory', self.saveDir) self.uploadLocations = self.am_getOption('UploadLocations', self.uploadLocations) includeMasterCS = self.am_getOption('IncludeMasterCS', self.includeMasterCS) if isinstance(includeMasterCS, six.string_types) and includeMasterCS.lower() in [ 'n', 'no', 'false' ]: self.includeMasterCS = False self.certAndKeyLocation = getHostCertificateAndKeyLocation() self.casLocation = getCAsLocation() return S_OK()
def _getServiceEndpointCurlDownload(self, granularity, entity): """ Calls method `get_service_endpoint` from the GOC DB programmatic interface. :params: :attr:`granularity` : a string. Could be in ('hostname', 'sitename', 'roc', 'country', 'service_type', 'monitored') :attr:`entity` : a string. Actual name of the entity. """ if not isinstance(granularity, str) or not isinstance(entity, str): raise ValueError("Arguments must be strings.") # GOCDB-PI query gocdb_ep = "https://goc.egi.eu/gocdbpi/public/?method=get_service_endpoint&" + granularity + "=" + entity caPath = getCAsLocation() service_endpoint_page = requests.get(gocdb_ep, verify=caPath) return service_endpoint_page.text
def _getServiceEndpointCurlDownload(self, granularity, entity): """ Calls method `get_service_endpoint` from the GOC DB programmatic interface. :params: :attr:`granularity` : a string. Could be in ('hostname', 'sitename', 'roc', 'country', 'service_type', 'monitored') :attr:`entity` : a string. Actual name of the entity. """ if not isinstance(granularity, basestring) or not isinstance(entity, basestring): raise ValueError("Arguments must be strings.") # GOCDB-PI query gocdb_ep = "https://goc.egi.eu/gocdbpi/public/?method=get_service_endpoint&" \ + granularity + '=' + entity caPath = getCAsLocation() service_endpoint_page = requests.get(gocdb_ep, verify=caPath) return service_endpoint_page.text
def getHostnameDowntime(self, hostname, startDate=None, ongoing=False): params = hostname if startDate and ongoing: return S_ERROR("Invalid parameter combination - do not specify startDate with ongoing") if startDate: params += '&startdate=' + startDate if ongoing: params += '&ongoing_only=yes' caPath = getCAsLocation() try: response = requests.get( 'https://goc.egi.eu/gocdbpi/public/?method=get_downtime&topentity=' + params, verify=caPath) response.raise_for_status() except requests.exceptions.RequestException as e: return S_ERROR("Error %s" % e) return S_OK(response.text)
def getUsers(self): """Get all the users of the VOMS VO with their detailed information :return: user dictionary keyed by the user DN """ if not self.urls: return S_ERROR(DErrno.ENOAUTH, "No VOMS server defined") userProxy = getProxyLocation() caPath = getCAsLocation() rawUserList = [] result = None for url in self.urls: rawUserList = [] startIndex = 0 result = None error = None urlDone = False while not urlDone: try: result = requests.get( url, headers={"X-VOMS-CSRF-GUARD": "y"}, cert=userProxy, verify=caPath, params={"startIndex": str(startIndex), "pageSize": "100"}, ) except requests.ConnectionError as exc: error = "%s:%s" % (url, repr(exc)) urlDone = True continue if result.status_code != 200: error = "Failed to contact the VOMS server: %s" % result.text urlDone = True continue userList = result.json()["result"] rawUserList.extend(userList) if len(userList) < 100: urlDone = True startIndex += 100 # This URL did not work, try another one if error: continue else: break if error: return S_ERROR(DErrno.ENOAUTH, "Failed to contact the VOMS server: %s" % error) # We have got the user info, reformat it resultDict = {} for user in rawUserList: for cert in user["certificates"]: dn = cert["subjectString"] resultDict[dn] = user resultDict[dn]["CA"] = cert["issuerString"] resultDict[dn]["certSuspended"] = cert.get("suspended") resultDict[dn]["suspended"] = user.get("suspended") resultDict[dn]["mail"] = user.get("emailAddress") resultDict[dn]["Roles"] = user.get("fqans") attributes = user.get("attributes") if attributes: for attribute in user.get("attributes", []): if attribute.get("name") == "nickname": resultDict[dn]["nickname"] = attribute.get("value") self.userDict = dict(resultDict) return S_OK(resultDict)
def getUsers(self): """ Get all the users of the VOMS VO with their detailed information :return: user dictionary keyed by the user DN """ if not self.urls: return S_ERROR(DErrno.ENOAUTH, "No VOMS server defined") userProxy = getProxyLocation() caPath = getCAsLocation() rawUserList = [] result = None for url in self.urls: rawUserList = [] startIndex = 0 result = None error = None urlDone = False while not urlDone: try: result = requests.get(url, headers={"X-VOMS-CSRF-GUARD": "y"}, cert=userProxy, verify=caPath, params={"startIndex": str(startIndex), "pageSize": "100"}) except requests.ConnectionError as exc: error = "%s:%s" % (url, repr(exc)) urlDone = True continue if result.status_code != 200: error = "Failed to contact the VOMS server: %s" % result.text urlDone = True continue userList = result.json()['result'] rawUserList.extend(userList) if len(userList) < 100: urlDone = True startIndex += 100 # This URL did not work, try another one if error: continue else: break if error: return S_ERROR(DErrno.ENOAUTH, "Failed to contact the VOMS server: %s" % error) # We have got the user info, reformat it resultDict = {} for user in rawUserList: for cert in user['certificates']: dn = cert['subjectString'] resultDict[dn] = user resultDict[dn]['CA'] = cert['issuerString'] resultDict[dn]['certSuspended'] = cert['suspended'] resultDict[dn]['certSuspensionReason'] = cert['suspensionReason'] resultDict[dn]['mail'] = user['emailAddress'] resultDict[dn]['Roles'] = user['fqans'] attributes = user.get('attributes') if attributes: for attribute in user.get('attributes', []): if attribute.get('name') == 'nickname': resultDict[dn]['nickname'] = attribute.get('value') self.userDict = dict(resultDict) return S_OK(resultDict)