def get_srm_endpoint(dq2_site_id):
'''
Gets the SRM endpoint of a site registered in TiersOfATLAS.
@param dq2_site_id is a DQ2 site id
@return a dictionary containing the srm endpoint information
'''
srm_endpoint_info = {'token':None, 'endpt':None, 'se_host':None, 'se_path':None}
re_srm2 = re.compile('^token:(.*):(srm:\/\/.*)\s*$')
srm_endpt = TiersOfATLAS.getSiteProperty(dq2_site_id,'srm')
if srm_endpt:
mat = re_srm2.match(srm_endpt)
if mat:
# this is a SRMv2 endpoint specification
srm_endpoint_info['token'] = mat.group(1)
srm_endpoint_info['endpt'] = mat.group(2)
endpt_data = urisplit(srm_endpoint_info['endpt'])
srm_endpoint_info['se_host'] = endpt_data[1].split(':')[0]
srm_endpoint_info['se_path'] = endpt_data[3].replace('SFN=','')
else:
# this is a SRMv1 endpoint specification
srm_endpoint_info['token'] = None
srm_endpoint_info['endpt'] = srm_endpt
endpt_data = urisplit(srm_endpoint_info['endpt'])
srm_endpoint_info['se_host'] = endpt_data[1].split(':')[0]
srm_endpoint_info['se_path'] = endpt_data[2]
return srm_endpoint_info
def get_srmv2_sites(cloud=None, token=None, debug=False):
'''
Gets a list of SRMV2 enabled DDM sites in a given cloud.
@param cloud is the ATLAS cloud name
@param token restricts the output to only certain srmv2 tokens
@param debug indicates if debugging messages are printed
@return a list of ATLAS srmv2-enabled site names
if token is given, only the site with the specific token type
will be selected.
'''
srmv2_sites = []
## a better way of getting all sites within a cloud
## however, it seems there is a bug in DQ2 API so it
## always returns an empty site list.
# all_sites = TiersOfATLAS.getSitesInCloud(cloud)
## a bit of hack with non-public DQ2 API interface
cache = TiersOfATLAS.ToACache
all_sites = []
if not cloud:
all_sites = TiersOfATLAS.getAllDestinationSites()
else:
if cloud == 'T0':
return ['CERNPROD']
if cloud not in cache.dbcloud:
return []
all_sites = TiersOfATLAS.getSites(cache.dbcloud[cloud])
for site in all_sites:
srm = TiersOfATLAS.getSiteProperty(site,'srm')
# presuming the srm endpoint looks like:
# token:ATLASDATADISK:srm://grid-cert-03.roma1.infn.it ...
if srm is not None and srm.find('token') != -1:
if token:
if srm.split(':')[1] == token:
srmv2_sites.append(site)
else:
srmv2_sites.append(site)
return srmv2_sites
def get_srmv2_sites(cloud=None, token=None, debug=False):
'''
Gets a list of SRMV2 enabled DDM sites in a given cloud.
@param cloud is the ATLAS cloud name
@param token restricts the output to only certain srmv2 tokens
@param debug indicates if debugging messages are printed
@return a list of ATLAS srmv2-enabled site names
if token is given, only the site with the specific token type
will be selected.
'''
srmv2_sites = []
## a better way of getting all sites within a cloud
## however, it seems there is a bug in DQ2 API so it
## always returns an empty site list.
# all_sites = TiersOfATLAS.getSitesInCloud(cloud)
## a bit of hack with non-public DQ2 API interface
cache = TiersOfATLAS.ToACache
all_sites = []
if not cloud:
all_sites = TiersOfATLAS.getAllDestinationSites()
else:
if cloud == 'T0':
return ['CERNPROD']
if cloud not in cache.dbcloud:
return []
all_sites = TiersOfATLAS.getSites(cache.dbcloud[cloud])
for site in all_sites:
srm = TiersOfATLAS.getSiteProperty(site, 'srm')
# presuming the srm endpoint looks like:
# token:ATLASDATADISK:srm://grid-cert-03.roma1.infn.it ...
if srm is not None and srm.find('token') != -1:
if token:
if srm.split(':')[1] == token:
srmv2_sites.append(site)
else:
srmv2_sites.append(site)
return srmv2_sites
def getSiteProperty(self,seName,attribute):
methodName = 'getSiteProperty'
try:
return self.SC_SUCCEEDED,TiersOfATLAS.getSiteProperty(seName,attribute)
except:
errtype,errvalue = sys.exc_info()[:2]
errCode = self.checkError(errtype)
return errCode,'%s : %s %s' % (methodName,errtype.__name__,errvalue)
def get_lfc_host(dq2_site_id):
'''
Gets the LFC host of a site registered in TiersOfATLAS.
'''
lfc_url = TiersOfATLAS.getLocalCatalog(dq2_site_id)
if lfc_url:
return lfc_url.split('/')[2][:-1]
else:
return None
def get_lfc_host(dq2_site_id):
'''
Gets the LFC host of a site registered in TiersOfATLAS.
'''
lfc_url = TiersOfATLAS.getLocalCatalog(dq2_site_id)
if lfc_url:
return lfc_url.split('/')[2][:-1]
else:
return None
def get_srm_endpoint(dq2_site_id):
'''
Gets the SRM endpoint of a site registered in TiersOfATLAS.
@param dq2_site_id is a DQ2 site id
@return a dictionary containing the srm endpoint information
'''
srm_endpoint_info = {
'token': None,
'endpt': None,
'se_host': None,
'se_path': None
}
re_srm2 = re.compile('^token:(.*):(srm:\/\/.*)\s*$')
srm_endpt = TiersOfATLAS.getSiteProperty(dq2_site_id, 'srm')
if srm_endpt:
mat = re_srm2.match(srm_endpt)
if mat:
# this is a SRMv2 endpoint specification
srm_endpoint_info['token'] = mat.group(1)
srm_endpoint_info['endpt'] = mat.group(2)
endpt_data = urisplit(srm_endpoint_info['endpt'])
srm_endpoint_info['se_host'] = endpt_data[1].split(':')[0]
srm_endpoint_info['se_path'] = endpt_data[3].replace('SFN=', '')
else:
# this is a SRMv1 endpoint specification
srm_endpoint_info['token'] = None
srm_endpoint_info['endpt'] = srm_endpt
endpt_data = urisplit(srm_endpoint_info['endpt'])
srm_endpoint_info['se_host'] = endpt_data[1].split(':')[0]
srm_endpoint_info['se_path'] = endpt_data[2]
return srm_endpoint_info
locnum = 0
for i in locrange:
locnum += len(locations[i])
if locnum == 0:
print "ERROR no location"
sys.exit(0)
lfn_guid = {}
for guid, info in contents.iteritems():
lfn_guid[info["lfn"]] = guid
allLFCs = []
for i in locrange:
for location in locations[i]:
l = TiersOfATLAS.getLocalCatalog(location)
if l and l not in allLFCs and l.startswith("lfc") and l not in removefromlfclist:
allLFCs.append(l)
status, guidReplicas, guidSizes, guidmd5sum = getinputreplicas(lfn_guid, allLFCs)
print guidReplicas
locations_srm = {}
for i in locrange:
for location in locations[i]:
try:
if "srm" in TiersOfATLAS.ToACache.sites[location]:
tempsrm = TiersOfATLAS.ToACache.sites[location]["srm"]
tempsrm = re.sub("token:*\w*:", "", tempsrm)
tempsrm = re.sub(":*\d*/srm/managerv2\?SFN=", "", tempsrm)
print tempsrm
def stripSite(site):
dq2alternatename = TiersOfATLAS.getSiteProperty(site,'alternateName')
if not dq2alternatename:
return site
else:
return dq2alternatename[0]
def main():
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option("-q",
action="store_true",
default=False,
help="quiet mode",
dest="quiet")
parser.add_option("-d",
action="store_true",
default=False,
help="debug mode",
dest="debug")
(options, args) = parser.parse_args()
if len(args) != 0:
parser.error("incorrect number of arguments")
return 1
loglevel = 'INFO'
if options.quiet:
loglevel = 'WARNING'
if options.debug:
loglevel = 'DEBUG'
logger = logging.getLogger()
logger.setLevel(logging._levelNames[loglevel])
fmt = '[UKDATA:%(levelname)s %(asctime)s] %(message)s'
formatter = logging.Formatter(fmt, '%d %b %T')
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)
logger.handlers = []
logger.addHandler(handler)
proxy = 'X509_USER_PROXY=/home/d0/love/certs/prod.out'
labels = TiersOfATLAS.getSites('UKSITES')
#labels = ['UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK', 'UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK']
#labels = ['UKI-SCOTGRID-GLASGOW_MCDISK']
#labels= ['UKI-NORTHGRID-LIV-HEP_MCDISK']
#labels= ['UKI-SOUTHGRID-BHAM-HEP_MCDISK']
#labels = [
# 'UKI-LT2-IC-HEP_PRODDISK',
# 'UKI-LT2-IC-HEP_MCDISK',
# 'UKI-LT2-IC-HEP_DATADISK',
# 'UKI-LT2-IC-HEP_HOTDISK',
# ]
#labels = ['UKI-SOUTHGRID-SUSX_PRODDISK']
try:
labels.remove('UKI-SCOTGRID-GLASGOW_PPSDATADISK')
except:
pass
try:
labels.remove('RAL-LCG2_PPSDATADISK')
except:
pass
for label in labels:
# if 'SUSX' not in label: continue
srm = TiersOfATLAS.getSiteProperty(label, 'srm')
domain = TiersOfATLAS.getSiteProperty(label, 'domain')
srmmatch = _SRMMATCH.match(srm)
srmtok = srmmatch.group(1)
srmpath = srmmatch.group(2)
srmhost = srmmatch.group(3)
cmd = '%s srm-get-space-tokens -retry_num=2 %s -space_desc=%s' % (
proxy, srmpath, srmtok)
logging.debug(cmd)
status, output = commands.getstatusoutput(cmd)
if status == 0:
tokmatch = _TOKENMATCH.match(output)
if tokmatch:
tokenid = tokmatch.group(1)
msg = "Found token ID %s: %s" % (srmtok, tokenid)
logging.debug(msg)
else:
msg = "Token ID not found for %s at %s" % (srmtok, label)
logging.warn(msg)
continue
else:
msg = "Cannot get token ID using cmd %s" % cmd
logging.warn(msg)
continue
# replace this with lcg-stmd
cmd = '%s srm-get-space-metadata -retry_num=1 -space_tokens=%s %s' % (
proxy, tokenid, srmpath)
logging.debug(cmd)
status, output = commands.getstatusoutput(cmd)
if status == 0:
spacematch = _SPACEMATCH.match(output)
if spacematch:
totsize = spacematch.group(1)
freesize = spacematch.group(3)
else:
msg = "Cannot parse output: %s" % cmd
logging.warn(msg)
continue
else:
msg = "Cannot get srm-get-space-metadata for tokenid: %s" % tokenid
logging.warn(msg)
continue
msg = "%s totalsize: %s freesize: %s" % (label, totsize, freesize)
logging.info(msg)
vals = (label, totsize, freesize)
fields = "label=%s&tsize=%s&usize=%s" % vals
_curl.setopt(pycurl.URL, _UPDATEURL)
_curl.setopt(pycurl.POST, 1)
_curl.setopt(pycurl.POSTFIELDS, fields)
# _curl.setopt(pycurl.VERBOSE, True)
try:
_curl.perform()
_buffer.seek(0)
if _curl.getinfo(pycurl.HTTP_CODE) != 200:
msg = "failed: %s%s" % (_UPDATEURL, fields)
logging.debug(msg)
# read from start of buffer
_buffer.seek(0)
logging.debug(_buffer.read())
for row in _buffer:
print row
sys.exit(1)
except pycurl.error:
msg = "Problem contacting server: %s" % _UPDATEURL
logging.error(msg)
raise
# hack for RAL FARM area
label = 'RAL-LCG2_RALFARM'
totsize = '102000000000000.0'
freesize = '0.0'
vals = (label, totsize, freesize)
fields = "label=%s&tsize=%s&usize=%s" % vals
_curl.setopt(pycurl.URL, _UPDATEURL)
_curl.setopt(pycurl.POST, 1)
_curl.setopt(pycurl.POSTFIELDS, fields)
try:
_curl.perform()
_buffer.seek(0)
code = _curl.getinfo(pycurl.HTTP_CODE)
if code == 500:
for row in _buffer:
print row
sys.exit(1)
except pycurl.error:
msg = "Problem contacting server: %s" % _UPDATEURL
logging.error(msg)
raise
#!/usr/bin/env python
import os
from dq2.clientapi.DQ2 import DQ2
from dq2.info import TiersOfATLAS
from dq2.info.TiersOfATLAS import _refreshToACache, ToACache
_refreshToACache()
dq2localid = os.environ['DQ2_LOCAL_SITE_ID']
dq2localid_alternatename = TiersOfATLAS.getSiteProperty(dq2localid,'alternateName')
if not dq2localid_alternatename:
try:
dq2localid_alternatename = dq2localid.split("_")[0]
except:
dq2localid_alternatename = dq2localid
db_dataset = os.environ['ATLAS_DBRELEASE']
dq2=DQ2()
db_locations = dq2.listDatasetReplicas(db_dataset).values()[0][1]
print "db_locations = ", db_locations
db_site = dq2localid
for sitename in TiersOfATLAS.getAllSources():
if TiersOfATLAS.getSiteProperty(sitename,'alternateName')==dq2localid_alternatename and sitename in db_locations:
db_site = sitename
outFile = open('db_dq2localid.txt','w')
outFile.write('%s\n' % db_site)
outFile.close()
import commands
import sys
from dq2.info import TiersOfATLAS
outputFile = open('infoSites.txt','a')
site=sys.argv[1]
if site=="SWT2_CPB_DATADISK":
outputFile.write(site)
outputFile.write('\n')
outputFile.write('unknown\n')
outputFile.write('unknown\n')
exit()
srm = TiersOfATLAS.getSiteProperty(site, 'srm')
endpoint = "/".join(":".join(srm.split(':')[2:5]).split('/')[:3])
s, o = commands.getstatusoutput('srmping -debug=True -2 -retry_num=1 %s' % endpoint)
outputFile.write(site)
outputFile.write('\n')
foundType=0
foundversion=0
if s == 0:
for line in o.split('\n'):
if line.find('backend_type') > -1:
outputFile.write(line.replace('backend_type:','',1))
outputFile.write('\n')
foundType=1
if line.find('backend_version') > -1:
outputFile.write(line.strip('backend_version:'))
outputFile.write('\n')
foundversion=1
def stripSite(site):
dq2alternatename = TiersOfATLAS.getSiteProperty(site, 'alternateName')
if not dq2alternatename:
return site
else:
return dq2alternatename[0]
def resolve_dq2_local_site_id(ds_locations, site_domain, se_hostname, force_siteid_domain={}, force_siteid_se={}):
'''
resolves the DQ2_LOCAL_SITE_ID based on 3 factors:
- the dataset location list
- the site domain
- the default SE host
- the dictionaries containing the local_site_id enforcement
The "force_siteid_domain" gives a list in which the matched site_domain (regx match)
will be limited to the given list of DQ2 site ids.
The "force_siteid_se" gives a list in which the matched se_hostname (regx match)
will be limited to the given list of DQ2 site ids.
it returns a final DQ2 site id which is local and containing the dataset replica
or an empty string if there is no matched DQ2 site id.
'''
dq2_local_site_id = ''
toa_check = True
# get dq2 site ids matching the site_domain and se_hostname
dq2_local_ids = []
# checking if the given site_domain in the enforcement dictionary
if force_siteid_domain:
for mydomain, mysites in force_siteid_domain.items():
re_domain = re.compile(mydomain)
if re_domain.match(site_domain):
dq2_local_ids = mysites
toa_check = False
break
# checking if the given se_hostname in the enforcement dictionary
if force_siteid_se:
for myse, mysites in force_siteid_se.items():
re_se = re.compile(myse)
if re_se.match(se_hostname):
dq2_local_ids = mysites
toa_check = False
break
# if enforcement is never applied, do the detection for dq2_local_ids via ToA
if toa_check:
alternateNameDict = {}
for sitename in TiersOfATLAS.getAllSources():
# compose the altname:siteid dictionary
altnames = TiersOfATLAS.getSiteProperty(sitename,'alternateName')
if altnames:
for altname in altnames:
if altname not in alternateNameDict:
alternateNameDict[altname] = []
alternateNameDict[altname].append(sitename)
# First search for srm
dq2srm = TiersOfATLAS.getSiteProperty(sitename,'srm')
if dq2srm and dq2srm.find(se_hostname)>=0:
dq2_local_ids.append(sitename)
# Second search for domainname
dq2domain = TiersOfATLAS.getSiteProperty(sitename,'domain')
if dq2domain and dq2domain.find(site_domain)>=0:
dq2_local_ids.append(sitename)
# Thirdly search for sites with the same alternateName
more_dq2_local_ids = []
for sitename in dq2_local_ids:
altnames = TiersOfATLAS.getSiteProperty(sitename,'alternateName')
if altnames:
for altname in altnames:
if altname in alternateNameDict:
more_dq2_local_ids += alternateNameDict[altname]
dq2_local_ids += more_dq2_local_ids
# resolving the best location according to the dataset locations
# - get the common part between dq2_local_ids and ds_locations
# - pick the first one in the common part
candidates = list(set(dq2_local_ids) & set(ds_locations))
print >> sys.stdout, str(candidates)
if candidates:
## pick up the site whose corresponding se is matching the se_hostname
for c in candidates:
srm_info = get_srm_endpoint(c)
if srm_info['se_host'] == se_hostname:
dq2_local_site_id = c
break
## otherwise, take the first candidate in the list
if not dq2_local_site_id:
dq2_local_site_id = candidates[0]
return dq2_local_site_id
def resolve_dq2_local_site_id(ds_locations,
site_domain,
se_hostname,
force_siteid_domain={},
force_siteid_se={}):
'''
resolves the DQ2_LOCAL_SITE_ID based on 3 factors:
- the dataset location list
- the site domain
- the default SE host
- the dictionaries containing the local_site_id enforcement
The "force_siteid_domain" gives a list in which the matched site_domain (regx match)
will be limited to the given list of DQ2 site ids.
The "force_siteid_se" gives a list in which the matched se_hostname (regx match)
will be limited to the given list of DQ2 site ids.
it returns a final DQ2 site id which is local and containing the dataset replica
or an empty string if there is no matched DQ2 site id.
'''
dq2_local_site_id = ''
toa_check = True
# get dq2 site ids matching the site_domain and se_hostname
dq2_local_ids = []
# checking if the given site_domain in the enforcement dictionary
if force_siteid_domain:
for mydomain, mysites in force_siteid_domain.items():
re_domain = re.compile(mydomain)
if re_domain.match(site_domain):
dq2_local_ids = mysites
toa_check = False
break
# checking if the given se_hostname in the enforcement dictionary
if force_siteid_se:
for myse, mysites in force_siteid_se.items():
re_se = re.compile(myse)
if re_se.match(se_hostname):
dq2_local_ids = mysites
toa_check = False
break
# if enforcement is never applied, do the detection for dq2_local_ids via ToA
if toa_check:
alternateNameDict = {}
for sitename in TiersOfATLAS.getAllSources():
# compose the altname:siteid dictionary
altnames = TiersOfATLAS.getSiteProperty(sitename, 'alternateName')
if altnames:
for altname in altnames:
if altname not in alternateNameDict:
alternateNameDict[altname] = []
alternateNameDict[altname].append(sitename)
# First search for srm
dq2srm = TiersOfATLAS.getSiteProperty(sitename, 'srm')
if dq2srm and dq2srm.find(se_hostname) >= 0:
dq2_local_ids.append(sitename)
# Second search for domainname
dq2domain = TiersOfATLAS.getSiteProperty(sitename, 'domain')
if dq2domain and dq2domain.find(site_domain) >= 0:
dq2_local_ids.append(sitename)
# Thirdly search for sites with the same alternateName
more_dq2_local_ids = []
for sitename in dq2_local_ids:
altnames = TiersOfATLAS.getSiteProperty(sitename, 'alternateName')
if altnames:
for altname in altnames:
if altname in alternateNameDict:
more_dq2_local_ids += alternateNameDict[altname]
dq2_local_ids += more_dq2_local_ids
# resolving the best location according to the dataset locations
# - get the common part between dq2_local_ids and ds_locations
# - pick the first one in the common part
candidates = list(set(dq2_local_ids) & set(ds_locations))
print >> sys.stdout, str(candidates)
if candidates:
## pick up the site whose corresponding se is matching the se_hostname
for c in candidates:
srm_info = get_srm_endpoint(c)
if srm_info['se_host'] == se_hostname:
dq2_local_site_id = c
break
## otherwise, take the first candidate in the list
if not dq2_local_site_id:
dq2_local_site_id = candidates[0]
return dq2_local_site_id
if job.computingSite in ['', None, 'NULL']:
print 'computingSite is not yet defined'
sys.exit(0)
siteSpec = siteMapper.getSite(job.computingSite)
for file in job.Files:
if file.type in ['output', 'log']:
file.GUID = commands.getoutput('uuidgen')
if job.computingSite == file.destinationSE and \
siteSpec.setokens_output.has_key(file.destinationDBlockToken):
tmpSrcDDM = siteSpec.setokens_output[file.destinationDBlockToken]
else:
tmpSrcDDM = siteMapper.getSite(job.computingSite).ddm_output
srm = TiersOfATLAS.getSiteProperty(tmpSrcDDM, 'srm')
srm = re.sub('^token:[^:]+:', '', srm)
xml += """
<File ID="%s">
<logical>
<lfn name="%s"/>
</logical>
<metadata att_name="surl" att_value="%s/user.elmsheus/user.elmsheus.hc10006029.ANALY_LONG_BNL_ATLAS.1312433204.e0b.8181.ANALY_LONG_BNL_ATLAS/%s"/>
<metadata att_name="fsize" att_value="127340"/>
<metadata att_name="md5sum" att_value="03cea4013bdb9f2e44050449b6ebf079"/>
</File>""" % (file.GUID, file.lfn, srm, file.lfn)
xml += """
</POOLFILECATALOG>
"""
def main():
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option("-q", action="store_true", default=False,
help="quiet mode", dest="quiet")
parser.add_option("-d", action="store_true", default=False,
help="debug mode", dest="debug")
(options, args) = parser.parse_args()
if len(args) != 0:
parser.error("incorrect number of arguments")
return 1
loglevel = 'INFO'
if options.quiet:
loglevel = 'WARNING'
if options.debug:
loglevel = 'DEBUG'
logger = logging.getLogger()
logger.setLevel(logging._levelNames[loglevel])
fmt = '[UKDATA:%(levelname)s %(asctime)s] %(message)s'
formatter = logging.Formatter(fmt, '%d %b %T')
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)
logger.handlers = []
logger.addHandler(handler)
proxy = 'X509_USER_PROXY=/home/d0/love/certs/prod.out'
labels = TiersOfATLAS.getSites('UKSITES')
#labels = ['UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK', 'UKI-NORTHGRID-MAN-HEP1_LOCALGROUPDISK']
#labels = ['UKI-SCOTGRID-GLASGOW_MCDISK']
#labels= ['UKI-NORTHGRID-LIV-HEP_MCDISK']
#labels= ['UKI-SOUTHGRID-BHAM-HEP_MCDISK']
#labels = [
# 'UKI-LT2-IC-HEP_PRODDISK',
# 'UKI-LT2-IC-HEP_MCDISK',
# 'UKI-LT2-IC-HEP_DATADISK',
# 'UKI-LT2-IC-HEP_HOTDISK',
# ]
#labels = ['UKI-SOUTHGRID-SUSX_PRODDISK']
try:
labels.remove('UKI-SCOTGRID-GLASGOW_PPSDATADISK')
except:
pass
try:
labels.remove('RAL-LCG2_PPSDATADISK')
except:
pass
for label in labels:
# if 'SUSX' not in label: continue
srm = TiersOfATLAS.getSiteProperty(label, 'srm')
domain = TiersOfATLAS.getSiteProperty(label, 'domain')
srmmatch = _SRMMATCH.match(srm)
srmtok = srmmatch.group(1)
srmpath = srmmatch.group(2)
srmhost = srmmatch.group(3)
cmd = '%s srm-get-space-tokens -retry_num=2 %s -space_desc=%s' % (proxy, srmpath, srmtok)
logging.debug(cmd)
status, output = commands.getstatusoutput(cmd)
if status == 0:
tokmatch = _TOKENMATCH.match(output)
if tokmatch:
tokenid = tokmatch.group(1)
msg = "Found token ID %s: %s" % (srmtok, tokenid)
logging.debug(msg)
else:
msg = "Token ID not found for %s at %s" % (srmtok, label)
logging.warn(msg)
continue
else:
msg = "Cannot get token ID using cmd %s" % cmd
logging.warn(msg)
continue
# replace this with lcg-stmd
cmd = '%s srm-get-space-metadata -retry_num=1 -space_tokens=%s %s' % (proxy, tokenid, srmpath)
logging.debug(cmd)
status, output = commands.getstatusoutput(cmd)
if status == 0:
spacematch = _SPACEMATCH.match(output)
if spacematch:
totsize = spacematch.group(1)
freesize = spacematch.group(3)
else:
msg = "Cannot parse output: %s" % cmd
logging.warn(msg)
continue
else:
msg = "Cannot get srm-get-space-metadata for tokenid: %s" % tokenid
logging.warn(msg)
continue
msg = "%s totalsize: %s freesize: %s" % (label, totsize, freesize)
logging.info(msg)
vals = (label, totsize, freesize)
fields = "label=%s&tsize=%s&usize=%s" % vals
_curl.setopt(pycurl.URL, _UPDATEURL)
_curl.setopt(pycurl.POST, 1)
_curl.setopt(pycurl.POSTFIELDS, fields)
# _curl.setopt(pycurl.VERBOSE, True)
try:
_curl.perform()
_buffer.seek(0)
if _curl.getinfo(pycurl.HTTP_CODE) != 200:
msg = "failed: %s%s" % (_UPDATEURL, fields)
logging.debug(msg)
# read from start of buffer
_buffer.seek(0)
logging.debug(_buffer.read())
for row in _buffer:
print row
sys.exit(1)
except pycurl.error:
msg = "Problem contacting server: %s" % _UPDATEURL
logging.error(msg)
raise
# hack for RAL FARM area
label= 'RAL-LCG2_RALFARM'
totsize = '102000000000000.0'
freesize = '0.0'
vals = (label, totsize, freesize)
fields = "label=%s&tsize=%s&usize=%s" % vals
_curl.setopt(pycurl.URL, _UPDATEURL)
_curl.setopt(pycurl.POST, 1)
_curl.setopt(pycurl.POSTFIELDS, fields)
try:
_curl.perform()
_buffer.seek(0)
code = _curl.getinfo(pycurl.HTTP_CODE)
if code == 500:
for row in _buffer:
print row
sys.exit(1)
except pycurl.error:
msg = "Problem contacting server: %s" % _UPDATEURL
logging.error(msg)
raise
def getAvailableFiles(self,datasetSpec,siteEndPointMap,siteMapper,ngGroup=[],checkLFC=False):
# make logger
methodName = 'getAvailableFiles'
methodName += ' <datasetID={0}>'.format(datasetSpec.datasetID)
tmpLog = MsgWrapper(logger,methodName)
tmpLog.info('start datasetName={0}'.format(datasetSpec.datasetName))
try:
# list of NG endpoints
ngEndPoints = []
if 1 in ngGroup:
ngEndPoints += ['_SCRATCHDISK$','_LOCALGROUPDISK$','_LOCALGROUPTAPE$','_USERDISK$',
'_DAQ$','_TMPDISK$','_TZERO$','_GRIDFTP$','MOCKTEST$']
if 2 in ngGroup:
ngEndPoints += ['_LOCALGROUPTAPE$',
'_DAQ$','_TMPDISK$','_TZERO$','_GRIDFTP$','MOCKTEST$']
# get all associated endpoints
siteAllEndPointsMap = {}
for siteName,endPointPattList in siteEndPointMap.iteritems():
# get all endpoints matching with patterns
allEndPointList = []
for endPointPatt in endPointPattList:
if '*' in endPointPatt:
# wildcard
endPointPatt = endPointPatt.replace('*','.*')
for endPointToA in TiersOfATLAS.getAllDestinationSites():
if re.search('^'+endPointPatt+'$',endPointToA) != None:
if not endPointToA in allEndPointList:
allEndPointList.append(endPointToA)
else:
# normal endpoint
if endPointPatt in TiersOfATLAS.getAllDestinationSites() and \
not endPointPatt in allEndPointList:
allEndPointList.append(endPointPatt)
# get associated endpoints
siteAllEndPointsMap[siteName] = []
for endPoint in allEndPointList:
# append
if not self.checkNGEndPoint(endPoint,ngEndPoints) and \
not endPoint in siteAllEndPointsMap[siteName]:
siteAllEndPointsMap[siteName].append(endPoint)
else:
# already checked
continue
# get alternate name
altName = TiersOfATLAS.getSiteProperty(endPoint,'alternateName')
if altName != None and altName != ['']:
for assEndPoint in TiersOfATLAS.resolveGOC({altName[0]:None})[altName[0]]:
if not assEndPoint in siteAllEndPointsMap[siteName] and \
not self.checkNGEndPoint(assEndPoint,ngEndPoints):
siteAllEndPointsMap[siteName].append(assEndPoint)
# get replica map
tmpStat,tmpOut = self.listDatasetReplicas(datasetSpec.datasetName)
if tmpStat != self.SC_SUCCEEDED:
tmpLog.error('faild to get dataset replicas with {0}'.format(tmpOut))
raise tmpStat,tmpOut
datasetReplicaMap = tmpOut
# collect SE, LFC hosts, storage path, storage type
lfcSeMap = {}
storagePathMap = {}
completeReplicaMap = {}
siteHasCompleteReplica = False
for siteName,allEndPointList in siteAllEndPointsMap.iteritems():
tmpLfcSeMap = {}
tmpStoragePathMap = {}
tmpSiteSpec = siteMapper.getSite(siteName)
for tmpEndPoint in allEndPointList:
# storage type
if TiersOfATLAS.isTapeSite(tmpEndPoint):
storageType = 'localtape'
else:
storageType = 'localdisk'
# no scan when site has complete replicas
if datasetReplicaMap.has_key(tmpEndPoint) and datasetReplicaMap[tmpEndPoint][-1]['found'] != None \
and datasetReplicaMap[tmpEndPoint][-1]['total'] == datasetReplicaMap[tmpEndPoint][-1]['found']:
completeReplicaMap[tmpEndPoint] = storageType
siteHasCompleteReplica = True
# no LFC scan for many-time datasets
if datasetSpec.isManyTime():
continue
# get LFC
lfc = TiersOfATLAS.getLocalCatalog(tmpEndPoint)
# add map
if not tmpLfcSeMap.has_key(lfc):
tmpLfcSeMap[lfc] = []
# get SE
seStr = TiersOfATLAS.getSiteProperty(tmpEndPoint, 'srm')
tmpMatch = re.search('://([^:/]+):*\d*/',seStr)
if tmpMatch != None:
se = tmpMatch.group(1)
if not se in tmpLfcSeMap[lfc]:
tmpLfcSeMap[lfc].append(se)
else:
tmpLog.error('faild to extract SE from %s for %s:%s' % \
(seStr,siteName,tmpEndPoint))
# get SE + path
seStr = TiersOfATLAS.getSiteProperty(tmpEndPoint, 'srm')
tmpMatch = re.search('(srm://.+)$',seStr)
if tmpMatch == None:
tmpLog.error('faild to extract SE+PATH from %s for %s:%s' % \
(seStr,siteName,tmpEndPoint))
continue
# add full path to storage map
tmpSePath = tmpMatch.group(1)
tmpStoragePathMap[tmpSePath] = {'siteName':siteName,'storageType':storageType}
# add compact path
tmpSePath = re.sub('(:\d+)*/srm/[^\?]+\?SFN=','',tmpSePath)
tmpStoragePathMap[tmpSePath] = {'siteName':siteName,'storageType':storageType}
# add to map to trigger LFC scan if complete replica is missing at the site
if DataServiceUtils.isCachedFile(datasetSpec.datasetName,tmpSiteSpec):
pass
elif not siteHasCompleteReplica or checkLFC:
for tmpKey,tmpVal in tmpLfcSeMap.iteritems():
if not lfcSeMap.has_key(tmpKey):
lfcSeMap[tmpKey] = []
lfcSeMap[tmpKey] += tmpVal
for tmpKey,tmpVal in tmpStoragePathMap.iteritems():
storagePathMap[tmpKey] = tmpVal
# collect GUIDs and LFNs
fileMap = {}
lfnMap = {}
lfnFileSepcMap = {}
scopeMap = {}
for tmpFile in datasetSpec.Files:
fileMap[tmpFile.GUID] = tmpFile.lfn
lfnMap[tmpFile.lfn] = tmpFile
lfnFileSepcMap[tmpFile.lfn] = tmpFile
scopeMap[tmpFile.lfn] = tmpFile.scope
# get SURLs
surlMap = {}
for lfcHost,seList in lfcSeMap.iteritems():
tmpLog.debug('lookup in LFC:{0} for {1}'.format(lfcHost,str(seList)))
tmpStat,tmpRetMap = self.getSURLsFromLFC(fileMap,lfcHost,seList,scopes=scopeMap)
tmpLog.debug(str(tmpStat))
if tmpStat != self.SC_SUCCEEDED:
raise RuntimeError,tmpRetMap
for lfn,surls in tmpRetMap.iteritems():
if not surlMap.has_key(lfn):
surlMap[lfn] = surls
else:
surlMap[lfn] += surls
# make return
returnMap = {}
for siteName,allEndPointList in siteAllEndPointsMap.iteritems():
# set default return values
if not returnMap.has_key(siteName):
returnMap[siteName] = {'localdisk':[],'localtape':[],'cache':[],'remote':[]}
# loop over all files
tmpSiteSpec = siteMapper.getSite(siteName)
# check if the file is cached
if DataServiceUtils.isCachedFile(datasetSpec.datasetName,tmpSiteSpec):
for tmpFileSpec in datasetSpec.Files:
# add to cached file list
returnMap[siteName]['cache'].append(tmpFileSpec)
# complete replicas
if not checkLFC:
for tmpEndPoint in allEndPointList:
if completeReplicaMap.has_key(tmpEndPoint):
storageType = completeReplicaMap[tmpEndPoint]
returnMap[siteName][storageType] += datasetSpec.Files
# loop over all available LFNs
avaLFNs = surlMap.keys()
avaLFNs.sort()
for tmpLFN in avaLFNs:
tmpFileSpec = lfnFileSepcMap[tmpLFN]
# loop over all SURLs
for tmpSURL in surlMap[tmpLFN]:
for tmpSePath in storagePathMap.keys():
# check SURL
if tmpSURL.startswith(tmpSePath):
# add
siteName = storagePathMap[tmpSePath]['siteName']
storageType = storagePathMap[tmpSePath]['storageType']
if not tmpFileSpec in returnMap[siteName][storageType]:
returnMap[siteName][storageType].append(tmpFileSpec)
break
# dump
dumpStr = ''
for siteName,storageTypeFile in returnMap.iteritems():
dumpStr += '{0}:('.format(siteName)
for storageType,fileList in storageTypeFile.iteritems():
dumpStr += '{0}:{1},'.format(storageType,len(fileList))
dumpStr = dumpStr[:-1]
dumpStr += ') '
dumpStr= dumpStr[:-1]
tmpLog.debug(dumpStr)
# return
tmpLog.info('done')
return self.SC_SUCCEEDED,returnMap
except:
errtype,errvalue = sys.exc_info()[:2]
errMsg = 'failed with {0} {1}'.format(errtype.__name__,errvalue)
tmpLog.error(errMsg)
return self.SC_FAILED,'{0}.{1} {2}'.format(self.__class__.__name__,methodName,errMsg)
locnum = 0
for i in locrange:
locnum += len(locations[i])
if locnum == 0:
print 'ERROR no location'
sys.exit(0)
lfn_guid = {}
for guid, info in contents.iteritems():
lfn_guid[info['lfn']] = guid
allLFCs = []
for i in locrange:
for location in locations[i]:
l = TiersOfATLAS.getLocalCatalog(location)
if l and l not in allLFCs and l.startswith(
'lfc') and l not in removefromlfclist:
allLFCs.append(l)
status, guidReplicas, guidSizes, guidmd5sum = getinputreplicas(
lfn_guid, allLFCs)
print guidReplicas
locations_srm = {}
for i in locrange:
for location in locations[i]:
try:
if 'srm' in TiersOfATLAS.ToACache.sites[location]:
tempsrm = TiersOfATLAS.ToACache.sites[location]['srm']
tempsrm = re.sub('token:*\w*:', '', tempsrm)
def findLostFiles(self,datasetName,fileMap):
methodName = 'findLostFiles'
methodName += ' <datasetName={0}>'.format(datasetName)
tmpLog = MsgWrapper(logger,methodName)
tmpLog.info('start')
try:
# get replicas
tmpStat,tmpOut = self.listDatasetReplicas(datasetName)
if tmpStat != self.SC_SUCCEEDED:
tmpLog.error('faild to get dataset replicas with {0}'.format(tmpOut))
raise tmpStat,tmpOut
# check if complete replica is available
hasCompReplica = False
datasetReplicaMap = tmpOut
for tmpEndPoint in datasetReplicaMap.keys():
if datasetReplicaMap[tmpEndPoint][-1]['found'] != None and \
datasetReplicaMap[tmpEndPoint][-1]['total'] == datasetReplicaMap[tmpEndPoint][-1]['found']:
hasCompReplica = True
break
# no lost files
if hasCompReplica:
tmpLog.info('done with no lost files')
self.SC_SUCCEEDED,{}
# get LFNs and scopes
lfnMap = {}
scopeMap = {}
for tmpGUID in fileMap.keys():
tmpLFN = fileMap[tmpGUID]['lfn']
lfnMap[tmpGUID] = tmpLFN
scopeMap[tmpLFN] = fileMap[tmpGUID]['scope']
# get LFC and SE
lfcSeMap = {}
for tmpEndPoint in datasetReplicaMap.keys():
# get LFC
lfc = TiersOfATLAS.getLocalCatalog(tmpEndPoint)
# add map
if not lfcSeMap.has_key(lfc):
lfcSeMap[lfc] = []
# get SE
seStr = TiersOfATLAS.getSiteProperty(tmpEndPoint, 'srm')
tmpMatch = re.search('://([^:/]+):*\d*/',seStr)
if tmpMatch != None:
se = tmpMatch.group(1)
if not se in lfcSeMap[lfc]:
lfcSeMap[lfc].append(se)
# get SURLs
for lfcHost,seList in lfcSeMap.iteritems():
tmpStat,tmpRetMap = self.getSURLsFromLFC(lfnMap,lfcHost,seList,scopes=scopeMap)
if tmpStat != self.SC_SUCCEEDED:
tmpLog.error('faild to get SURLs with {0}'.format(tmpRetMap))
raise tmpStat,tmpRetMap
# look for missing files
newLfnMap = {}
for tmpGUID,tmpLFN in lfnMap.iteritems():
if not tmpLFN in tmpRetMap:
newLfnMap[tmpGUID] = tmpLFN
lfnMap = newLfnMap
tmpLog.info('done with lost '+','.join(str(tmpLFN) for tmpLFN in lfnMap.values()))
return self.SC_SUCCEEDED,lfnMap
except:
errtype,errvalue = sys.exc_info()[:2]
errCode = self.checkError(errtype)
errMsg = '{0} {1}'.format(errtype.__name__,errvalue)
tmpLog.error(errMsg)
return errCode,'{0} : {1}'.format(methodName,errMsg)
if job.computingSite in ['',None,'NULL']:
print 'computingSite is not yet defined'
sys.exit(0)
siteSpec = siteMapper.getSite(job.computingSite)
for file in job.Files:
if file.type in ['output','log']:
file.GUID = commands.getoutput('uuidgen')
if job.computingSite == file.destinationSE and \
siteSpec.setokens.has_key(file.destinationDBlockToken):
tmpSrcDDM = siteSpec.setokens[file.destinationDBlockToken]
else:
tmpSrcDDM = siteMapper.getSite(job.computingSite).ddm
srm = TiersOfATLAS.getSiteProperty(tmpSrcDDM,'srm')
srm = re.sub('^token:[^:]+:','',srm)
xml += """
<File ID="%s">
<logical>
<lfn name="%s"/>
</logical>
<metadata att_name="surl" att_value="%s/user.elmsheus/user.elmsheus.hc10006029.ANALY_LONG_BNL_ATLAS.1312433204.e0b.8181.ANALY_LONG_BNL_ATLAS/%s"/>
<metadata att_name="fsize" att_value="127340"/>
<metadata att_name="md5sum" att_value="03cea4013bdb9f2e44050449b6ebf079"/>
</File>""" % (file.GUID,file.lfn,srm,file.lfn)
xml += """
</POOLFILECATALOG>
"""
if (debug): sys.stderr.write("Using DQ2 clients\n")
if (location):
fe=getConfig(conffile,location,'FRONTIER_SERVER')
if (not fe): frc,fe=getFrontierEnvDQ2(location)
if (not fe): frc,fe=getFrontierEnvDQ2(location.upper())
if (frc == 0 and fe): print fe
rc += frc
else:
for gn in fsquidmap.keys():
frc,fe=getFrontierEnvDQ2(gn)
print gn,fe
rc += frc
elif (mode == 'listproto'):
if (ToA):
if (location):
seinfo = ToA.getSiteProperty(location.upper(), 'seinfo')
elif (AGIS):
seinfo = a.get_seinfo_entry()[location.upper()]
else:
sys.stderr.write("No AGIS or DQ2 clients available\n")
if (seinfo and seinfo.has_key('protocols')):
for p in seinfo['protocols']:
print p[0]
elif (mode == 'regexp'):
if (ToA):
if (location):
seinfo = ToA.getSiteProperty(location.upper(), 'seinfo')
elif (AGIS):
seinfo = a.get_seinfo_entry()[location.upper()]
else:
sys.stderr.write("No AGIS or DQ2 clients available\n")
