def get_srmv2_sites(cloud=None, token=None, debug=False): ''' Gets a list of SRMV2 enabled DDM sites in a given cloud. @param cloud is the ATLAS cloud name @param token restricts the output to only certain srmv2 tokens @param debug indicates if debugging messages are printed @return a list of ATLAS srmv2-enabled site names if token is given, only the site with the specific token type will be selected. ''' srmv2_sites = [] ## a better way of getting all sites within a cloud ## however, it seems there is a bug in DQ2 API so it ## always returns an empty site list. # all_sites = TiersOfATLAS.getSitesInCloud(cloud) ## a bit of hack with non-public DQ2 API interface cache = TiersOfATLAS.ToACache all_sites = [] if not cloud: all_sites = TiersOfATLAS.getAllDestinationSites() else: if cloud == 'T0': return ['CERNPROD'] if cloud not in cache.dbcloud: return [] all_sites = TiersOfATLAS.getSites(cache.dbcloud[cloud]) for site in all_sites: srm = TiersOfATLAS.getSiteProperty(site,'srm') # presuming the srm endpoint looks like: # token:ATLASDATADISK:srm://grid-cert-03.roma1.infn.it ... if srm is not None and srm.find('token') != -1: if token: if srm.split(':')[1] == token: srmv2_sites.append(site) else: srmv2_sites.append(site) return srmv2_sites
def get_srmv2_sites(cloud=None, token=None, debug=False): ''' Gets a list of SRMV2 enabled DDM sites in a given cloud. @param cloud is the ATLAS cloud name @param token restricts the output to only certain srmv2 tokens @param debug indicates if debugging messages are printed @return a list of ATLAS srmv2-enabled site names if token is given, only the site with the specific token type will be selected. ''' srmv2_sites = [] ## a better way of getting all sites within a cloud ## however, it seems there is a bug in DQ2 API so it ## always returns an empty site list. # all_sites = TiersOfATLAS.getSitesInCloud(cloud) ## a bit of hack with non-public DQ2 API interface cache = TiersOfATLAS.ToACache all_sites = [] if not cloud: all_sites = TiersOfATLAS.getAllDestinationSites() else: if cloud == 'T0': return ['CERNPROD'] if cloud not in cache.dbcloud: return [] all_sites = TiersOfATLAS.getSites(cache.dbcloud[cloud]) for site in all_sites: srm = TiersOfATLAS.getSiteProperty(site, 'srm') # presuming the srm endpoint looks like: # token:ATLASDATADISK:srm://grid-cert-03.roma1.infn.it ... if srm is not None and srm.find('token') != -1: if token: if srm.split(':')[1] == token: srmv2_sites.append(site) else: srmv2_sites.append(site) return srmv2_sites
def main(): usage = "usage: %prog [options]" parser = OptionParser(usage=usage) parser.add_option("-q", action="store_true", default=False, help="quiet mode", dest="quiet") parser.add_option("-d", action="store_true", default=False, help="debug mode", dest="debug") (options, args) = parser.parse_args() if len(args) != 0: parser.error("incorrect number of arguments") return 1 loglevel = 'INFO' if options.quiet: loglevel = 'WARNING' if options.debug: loglevel = 'DEBUG' logger = logging.getLogger() logger.setLevel(logging._levelNames[loglevel]) fmt = '[UKDATA:%(levelname)s %(asctime)s] %(message)s' formatter = logging.Formatter(fmt, '%d %b %T') handler = logging.StreamHandler(sys.stdout) handler.setFormatter(formatter) logger.handlers = [] logger.addHandler(handler) proxy = 'X509_USER_PROXY=/home/d0/love/certs/prod.out' labels = TiersOfATLAS.getSites('UKSITES') #labels = ['UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK', 'UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK'] #labels = ['UKI-SCOTGRID-GLASGOW_MCDISK'] #labels= ['UKI-NORTHGRID-LIV-HEP_MCDISK'] #labels= ['UKI-SOUTHGRID-BHAM-HEP_MCDISK'] #labels = [ # 'UKI-LT2-IC-HEP_PRODDISK', # 'UKI-LT2-IC-HEP_MCDISK', # 'UKI-LT2-IC-HEP_DATADISK', # 'UKI-LT2-IC-HEP_HOTDISK', # ] #labels = ['UKI-SOUTHGRID-SUSX_PRODDISK'] try: labels.remove('UKI-SCOTGRID-GLASGOW_PPSDATADISK') except: pass try: labels.remove('RAL-LCG2_PPSDATADISK') except: pass for label in labels: # if 'SUSX' not in label: continue srm = TiersOfATLAS.getSiteProperty(label, 'srm') domain = TiersOfATLAS.getSiteProperty(label, 'domain') srmmatch = _SRMMATCH.match(srm) srmtok = srmmatch.group(1) srmpath = srmmatch.group(2) srmhost = srmmatch.group(3) cmd = '%s srm-get-space-tokens -retry_num=2 %s -space_desc=%s' % (proxy, srmpath, srmtok) logging.debug(cmd) status, output = commands.getstatusoutput(cmd) if status == 0: tokmatch = _TOKENMATCH.match(output) if tokmatch: tokenid = tokmatch.group(1) msg = "Found token ID %s: %s" % (srmtok, tokenid) logging.debug(msg) else: msg = "Token ID not found for %s at %s" % (srmtok, label) logging.warn(msg) continue else: msg = "Cannot get token ID using cmd %s" % cmd logging.warn(msg) continue # replace this with lcg-stmd cmd = '%s srm-get-space-metadata -retry_num=1 -space_tokens=%s %s' % (proxy, tokenid, srmpath) logging.debug(cmd) status, output = commands.getstatusoutput(cmd) if status == 0: spacematch = _SPACEMATCH.match(output) if spacematch: totsize = spacematch.group(1) freesize = spacematch.group(3) else: msg = "Cannot parse output: %s" % cmd logging.warn(msg) continue else: msg = "Cannot get srm-get-space-metadata for tokenid: %s" % tokenid logging.warn(msg) continue msg = "%s totalsize: %s freesize: %s" % (label, totsize, freesize) logging.info(msg) vals = (label, totsize, freesize) fields = "label=%s&tsize=%s&usize=%s" % vals _curl.setopt(pycurl.URL, _UPDATEURL) _curl.setopt(pycurl.POST, 1) _curl.setopt(pycurl.POSTFIELDS, fields) # _curl.setopt(pycurl.VERBOSE, True) try: _curl.perform() _buffer.seek(0) if _curl.getinfo(pycurl.HTTP_CODE) != 200: msg = "failed: %s%s" % (_UPDATEURL, fields) logging.debug(msg) # read from start of buffer _buffer.seek(0) logging.debug(_buffer.read()) for row in _buffer: print row sys.exit(1) except pycurl.error: msg = "Problem contacting server: %s" % _UPDATEURL logging.error(msg) raise # hack for RAL FARM area label= 'RAL-LCG2_RALFARM' totsize = '102000000000000.0' freesize = '0.0' vals = (label, totsize, freesize) fields = "label=%s&tsize=%s&usize=%s" % vals _curl.setopt(pycurl.URL, _UPDATEURL) _curl.setopt(pycurl.POST, 1) _curl.setopt(pycurl.POSTFIELDS, fields) try: _curl.perform() _buffer.seek(0) code = _curl.getinfo(pycurl.HTTP_CODE) if code == 500: for row in _buffer: print row sys.exit(1) except pycurl.error: msg = "Problem contacting server: %s" % _UPDATEURL logging.error(msg) raise
def main(): usage = "usage: %prog [options]" parser = OptionParser(usage=usage) parser.add_option("-q", action="store_true", default=False, help="quiet mode", dest="quiet") parser.add_option("-d", action="store_true", default=False, help="debug mode", dest="debug") (options, args) = parser.parse_args() if len(args) != 0: parser.error("incorrect number of arguments") return 1 loglevel = 'INFO' if options.quiet: loglevel = 'WARNING' if options.debug: loglevel = 'DEBUG' logger = logging.getLogger() logger.setLevel(logging._levelNames[loglevel]) fmt = '[UKDATA:%(levelname)s %(asctime)s] %(message)s' formatter = logging.Formatter(fmt, '%d %b %T') handler = logging.StreamHandler(sys.stdout) handler.setFormatter(formatter) logger.handlers = [] logger.addHandler(handler) proxy = 'X509_USER_PROXY=/home/d0/love/certs/prod.out' labels = TiersOfATLAS.getSites('UKSITES') #labels = ['UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK', 'UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK'] #labels = ['UKI-SCOTGRID-GLASGOW_MCDISK'] #labels= ['UKI-NORTHGRID-LIV-HEP_MCDISK'] #labels= ['UKI-SOUTHGRID-BHAM-HEP_MCDISK'] #labels = [ # 'UKI-LT2-IC-HEP_PRODDISK', # 'UKI-LT2-IC-HEP_MCDISK', # 'UKI-LT2-IC-HEP_DATADISK', # 'UKI-LT2-IC-HEP_HOTDISK', # ] #labels = ['UKI-SOUTHGRID-SUSX_PRODDISK'] try: labels.remove('UKI-SCOTGRID-GLASGOW_PPSDATADISK') except: pass try: labels.remove('RAL-LCG2_PPSDATADISK') except: pass for label in labels: # if 'SUSX' not in label: continue srm = TiersOfATLAS.getSiteProperty(label, 'srm') domain = TiersOfATLAS.getSiteProperty(label, 'domain') srmmatch = _SRMMATCH.match(srm) srmtok = srmmatch.group(1) srmpath = srmmatch.group(2) srmhost = srmmatch.group(3) cmd = '%s srm-get-space-tokens -retry_num=2 %s -space_desc=%s' % ( proxy, srmpath, srmtok) logging.debug(cmd) status, output = commands.getstatusoutput(cmd) if status == 0: tokmatch = _TOKENMATCH.match(output) if tokmatch: tokenid = tokmatch.group(1) msg = "Found token ID %s: %s" % (srmtok, tokenid) logging.debug(msg) else: msg = "Token ID not found for %s at %s" % (srmtok, label) logging.warn(msg) continue else: msg = "Cannot get token ID using cmd %s" % cmd logging.warn(msg) continue # replace this with lcg-stmd cmd = '%s srm-get-space-metadata -retry_num=1 -space_tokens=%s %s' % ( proxy, tokenid, srmpath) logging.debug(cmd) status, output = commands.getstatusoutput(cmd) if status == 0: spacematch = _SPACEMATCH.match(output) if spacematch: totsize = spacematch.group(1) freesize = spacematch.group(3) else: msg = "Cannot parse output: %s" % cmd logging.warn(msg) continue else: msg = "Cannot get srm-get-space-metadata for tokenid: %s" % tokenid logging.warn(msg) continue msg = "%s totalsize: %s freesize: %s" % (label, totsize, freesize) logging.info(msg) vals = (label, totsize, freesize) fields = "label=%s&tsize=%s&usize=%s" % vals _curl.setopt(pycurl.URL, _UPDATEURL) _curl.setopt(pycurl.POST, 1) _curl.setopt(pycurl.POSTFIELDS, fields) # _curl.setopt(pycurl.VERBOSE, True) try: _curl.perform() _buffer.seek(0) if _curl.getinfo(pycurl.HTTP_CODE) != 200: msg = "failed: %s%s" % (_UPDATEURL, fields) logging.debug(msg) # read from start of buffer _buffer.seek(0) logging.debug(_buffer.read()) for row in _buffer: print row sys.exit(1) except pycurl.error: msg = "Problem contacting server: %s" % _UPDATEURL logging.error(msg) raise # hack for RAL FARM area label = 'RAL-LCG2_RALFARM' totsize = '102000000000000.0' freesize = '0.0' vals = (label, totsize, freesize) fields = "label=%s&tsize=%s&usize=%s" % vals _curl.setopt(pycurl.URL, _UPDATEURL) _curl.setopt(pycurl.POST, 1) _curl.setopt(pycurl.POSTFIELDS, fields) try: _curl.perform() _buffer.seek(0) code = _curl.getinfo(pycurl.HTTP_CODE) if code == 500: for row in _buffer: print row sys.exit(1) except pycurl.error: msg = "Problem contacting server: %s" % _UPDATEURL logging.error(msg) raise