def get_srmv2_sites(cloud=None, token=None, debug=False):
'''
Gets a list of SRMV2 enabled DDM sites in a given cloud.
@param cloud is the ATLAS cloud name
@param token restricts the output to only certain srmv2 tokens
@param debug indicates if debugging messages are printed
@return a list of ATLAS srmv2-enabled site names
if token is given, only the site with the specific token type
will be selected.
'''
srmv2_sites = []
## a better way of getting all sites within a cloud
## however, it seems there is a bug in DQ2 API so it
## always returns an empty site list.
# all_sites = TiersOfATLAS.getSitesInCloud(cloud)
## a bit of hack with non-public DQ2 API interface
cache = TiersOfATLAS.ToACache
all_sites = []
if not cloud:
all_sites = TiersOfATLAS.getAllDestinationSites()
else:
if cloud == 'T0':
return ['CERNPROD']
if cloud not in cache.dbcloud:
return []
all_sites = TiersOfATLAS.getSites(cache.dbcloud[cloud])
for site in all_sites:
srm = TiersOfATLAS.getSiteProperty(site,'srm')
# presuming the srm endpoint looks like:
# token:ATLASDATADISK:srm://grid-cert-03.roma1.infn.it ...
if srm is not None and srm.find('token') != -1:
if token:
if srm.split(':')[1] == token:
srmv2_sites.append(site)
else:
srmv2_sites.append(site)
return srmv2_sites
def get_srmv2_sites(cloud=None, token=None, debug=False):
'''
Gets a list of SRMV2 enabled DDM sites in a given cloud.
@param cloud is the ATLAS cloud name
@param token restricts the output to only certain srmv2 tokens
@param debug indicates if debugging messages are printed
@return a list of ATLAS srmv2-enabled site names
if token is given, only the site with the specific token type
will be selected.
'''
srmv2_sites = []
## a better way of getting all sites within a cloud
## however, it seems there is a bug in DQ2 API so it
## always returns an empty site list.
# all_sites = TiersOfATLAS.getSitesInCloud(cloud)
## a bit of hack with non-public DQ2 API interface
cache = TiersOfATLAS.ToACache
all_sites = []
if not cloud:
all_sites = TiersOfATLAS.getAllDestinationSites()
else:
if cloud == 'T0':
return ['CERNPROD']
if cloud not in cache.dbcloud:
return []
all_sites = TiersOfATLAS.getSites(cache.dbcloud[cloud])
for site in all_sites:
srm = TiersOfATLAS.getSiteProperty(site, 'srm')
# presuming the srm endpoint looks like:
# token:ATLASDATADISK:srm://grid-cert-03.roma1.infn.it ...
if srm is not None and srm.find('token') != -1:
if token:
if srm.split(':')[1] == token:
srmv2_sites.append(site)
else:
srmv2_sites.append(site)
return srmv2_sites
def main():
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option("-q", action="store_true", default=False,
help="quiet mode", dest="quiet")
parser.add_option("-d", action="store_true", default=False,
help="debug mode", dest="debug")
(options, args) = parser.parse_args()
if len(args) != 0:
parser.error("incorrect number of arguments")
return 1
loglevel = 'INFO'
if options.quiet:
loglevel = 'WARNING'
if options.debug:
loglevel = 'DEBUG'
logger = logging.getLogger()
logger.setLevel(logging._levelNames[loglevel])
fmt = '[UKDATA:%(levelname)s %(asctime)s] %(message)s'
formatter = logging.Formatter(fmt, '%d %b %T')
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)
logger.handlers = []
logger.addHandler(handler)
proxy = 'X509_USER_PROXY=/home/d0/love/certs/prod.out'
labels = TiersOfATLAS.getSites('UKSITES')
#labels = ['UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK', 'UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK']
#labels = ['UKI-SCOTGRID-GLASGOW_MCDISK']
#labels= ['UKI-NORTHGRID-LIV-HEP_MCDISK']
#labels= ['UKI-SOUTHGRID-BHAM-HEP_MCDISK']
#labels = [
# 'UKI-LT2-IC-HEP_PRODDISK',
# 'UKI-LT2-IC-HEP_MCDISK',
# 'UKI-LT2-IC-HEP_DATADISK',
# 'UKI-LT2-IC-HEP_HOTDISK',
# ]
#labels = ['UKI-SOUTHGRID-SUSX_PRODDISK']
try:
labels.remove('UKI-SCOTGRID-GLASGOW_PPSDATADISK')
except:
pass
try:
labels.remove('RAL-LCG2_PPSDATADISK')
except:
pass
for label in labels:
# if 'SUSX' not in label: continue
srm = TiersOfATLAS.getSiteProperty(label, 'srm')
domain = TiersOfATLAS.getSiteProperty(label, 'domain')
srmmatch = _SRMMATCH.match(srm)
srmtok = srmmatch.group(1)
srmpath = srmmatch.group(2)
srmhost = srmmatch.group(3)
cmd = '%s srm-get-space-tokens -retry_num=2 %s -space_desc=%s' % (proxy, srmpath, srmtok)
logging.debug(cmd)
status, output = commands.getstatusoutput(cmd)
if status == 0:
tokmatch = _TOKENMATCH.match(output)
if tokmatch:
tokenid = tokmatch.group(1)
msg = "Found token ID %s: %s" % (srmtok, tokenid)
logging.debug(msg)
else:
msg = "Token ID not found for %s at %s" % (srmtok, label)
logging.warn(msg)
continue
else:
msg = "Cannot get token ID using cmd %s" % cmd
logging.warn(msg)
continue
# replace this with lcg-stmd
cmd = '%s srm-get-space-metadata -retry_num=1 -space_tokens=%s %s' % (proxy, tokenid, srmpath)
logging.debug(cmd)
status, output = commands.getstatusoutput(cmd)
if status == 0:
spacematch = _SPACEMATCH.match(output)
if spacematch:
totsize = spacematch.group(1)
freesize = spacematch.group(3)
else:
msg = "Cannot parse output: %s" % cmd
logging.warn(msg)
continue
else:
msg = "Cannot get srm-get-space-metadata for tokenid: %s" % tokenid
logging.warn(msg)
continue
msg = "%s totalsize: %s freesize: %s" % (label, totsize, freesize)
logging.info(msg)
vals = (label, totsize, freesize)
fields = "label=%s&tsize=%s&usize=%s" % vals
_curl.setopt(pycurl.URL, _UPDATEURL)
_curl.setopt(pycurl.POST, 1)
_curl.setopt(pycurl.POSTFIELDS, fields)
# _curl.setopt(pycurl.VERBOSE, True)
try:
_curl.perform()
_buffer.seek(0)
if _curl.getinfo(pycurl.HTTP_CODE) != 200:
msg = "failed: %s%s" % (_UPDATEURL, fields)
logging.debug(msg)
# read from start of buffer
_buffer.seek(0)
logging.debug(_buffer.read())
for row in _buffer:
print row
sys.exit(1)
except pycurl.error:
msg = "Problem contacting server: %s" % _UPDATEURL
logging.error(msg)
raise
# hack for RAL FARM area
label= 'RAL-LCG2_RALFARM'
totsize = '102000000000000.0'
freesize = '0.0'
vals = (label, totsize, freesize)
fields = "label=%s&tsize=%s&usize=%s" % vals
_curl.setopt(pycurl.URL, _UPDATEURL)
_curl.setopt(pycurl.POST, 1)
_curl.setopt(pycurl.POSTFIELDS, fields)
try:
_curl.perform()
_buffer.seek(0)
code = _curl.getinfo(pycurl.HTTP_CODE)
if code == 500:
for row in _buffer:
print row
sys.exit(1)
except pycurl.error:
msg = "Problem contacting server: %s" % _UPDATEURL
logging.error(msg)
raise
def main():
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option("-q",
action="store_true",
default=False,
help="quiet mode",
dest="quiet")
parser.add_option("-d",
action="store_true",
default=False,
help="debug mode",
dest="debug")
(options, args) = parser.parse_args()
if len(args) != 0:
parser.error("incorrect number of arguments")
return 1
loglevel = 'INFO'
if options.quiet:
loglevel = 'WARNING'
if options.debug:
loglevel = 'DEBUG'
logger = logging.getLogger()
logger.setLevel(logging._levelNames[loglevel])
fmt = '[UKDATA:%(levelname)s %(asctime)s] %(message)s'
formatter = logging.Formatter(fmt, '%d %b %T')
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)
logger.handlers = []
logger.addHandler(handler)
proxy = 'X509_USER_PROXY=/home/d0/love/certs/prod.out'
labels = TiersOfATLAS.getSites('UKSITES')
#labels = ['UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK', 'UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK']
#labels = ['UKI-SCOTGRID-GLASGOW_MCDISK']
#labels= ['UKI-NORTHGRID-LIV-HEP_MCDISK']
#labels= ['UKI-SOUTHGRID-BHAM-HEP_MCDISK']
#labels = [
# 'UKI-LT2-IC-HEP_PRODDISK',
# 'UKI-LT2-IC-HEP_MCDISK',
# 'UKI-LT2-IC-HEP_DATADISK',
# 'UKI-LT2-IC-HEP_HOTDISK',
# ]
#labels = ['UKI-SOUTHGRID-SUSX_PRODDISK']
try:
labels.remove('UKI-SCOTGRID-GLASGOW_PPSDATADISK')
except:
pass
try:
labels.remove('RAL-LCG2_PPSDATADISK')
except:
pass
for label in labels:
# if 'SUSX' not in label: continue
srm = TiersOfATLAS.getSiteProperty(label, 'srm')
domain = TiersOfATLAS.getSiteProperty(label, 'domain')
srmmatch = _SRMMATCH.match(srm)
srmtok = srmmatch.group(1)
srmpath = srmmatch.group(2)
srmhost = srmmatch.group(3)
cmd = '%s srm-get-space-tokens -retry_num=2 %s -space_desc=%s' % (
proxy, srmpath, srmtok)
logging.debug(cmd)
status, output = commands.getstatusoutput(cmd)
if status == 0:
tokmatch = _TOKENMATCH.match(output)
if tokmatch:
tokenid = tokmatch.group(1)
msg = "Found token ID %s: %s" % (srmtok, tokenid)
logging.debug(msg)
else:
msg = "Token ID not found for %s at %s" % (srmtok, label)
logging.warn(msg)
continue
else:
msg = "Cannot get token ID using cmd %s" % cmd
logging.warn(msg)
continue
# replace this with lcg-stmd
cmd = '%s srm-get-space-metadata -retry_num=1 -space_tokens=%s %s' % (
proxy, tokenid, srmpath)
logging.debug(cmd)
status, output = commands.getstatusoutput(cmd)
if status == 0:
spacematch = _SPACEMATCH.match(output)
if spacematch:
totsize = spacematch.group(1)
freesize = spacematch.group(3)
else:
msg = "Cannot parse output: %s" % cmd
logging.warn(msg)
continue
else:
msg = "Cannot get srm-get-space-metadata for tokenid: %s" % tokenid
logging.warn(msg)
continue
msg = "%s totalsize: %s freesize: %s" % (label, totsize, freesize)
logging.info(msg)
vals = (label, totsize, freesize)
fields = "label=%s&tsize=%s&usize=%s" % vals
_curl.setopt(pycurl.URL, _UPDATEURL)
_curl.setopt(pycurl.POST, 1)
_curl.setopt(pycurl.POSTFIELDS, fields)
# _curl.setopt(pycurl.VERBOSE, True)
try:
_curl.perform()
_buffer.seek(0)
if _curl.getinfo(pycurl.HTTP_CODE) != 200:
msg = "failed: %s%s" % (_UPDATEURL, fields)
logging.debug(msg)
# read from start of buffer
_buffer.seek(0)
logging.debug(_buffer.read())
for row in _buffer:
print row
sys.exit(1)
except pycurl.error:
msg = "Problem contacting server: %s" % _UPDATEURL
logging.error(msg)
raise
# hack for RAL FARM area
label = 'RAL-LCG2_RALFARM'
totsize = '102000000000000.0'
freesize = '0.0'
vals = (label, totsize, freesize)
fields = "label=%s&tsize=%s&usize=%s" % vals
_curl.setopt(pycurl.URL, _UPDATEURL)
_curl.setopt(pycurl.POST, 1)
_curl.setopt(pycurl.POSTFIELDS, fields)
try:
_curl.perform()
_buffer.seek(0)
code = _curl.getinfo(pycurl.HTTP_CODE)
if code == 500:
for row in _buffer:
print row
sys.exit(1)
except pycurl.error:
msg = "Problem contacting server: %s" % _UPDATEURL
logging.error(msg)
raise
