def resolve_dq2_local_site_id(ds_locations, site_domain, se_hostname, force_siteid_domain={}, force_siteid_se={}):
'''
resolves the DQ2_LOCAL_SITE_ID based on 3 factors:
- the dataset location list
- the site domain
- the default SE host
- the dictionaries containing the local_site_id enforcement
The "force_siteid_domain" gives a list in which the matched site_domain (regx match)
will be limited to the given list of DQ2 site ids.
The "force_siteid_se" gives a list in which the matched se_hostname (regx match)
will be limited to the given list of DQ2 site ids.
it returns a final DQ2 site id which is local and containing the dataset replica
or an empty string if there is no matched DQ2 site id.
'''
dq2_local_site_id = ''
toa_check = True
# get dq2 site ids matching the site_domain and se_hostname
dq2_local_ids = []
# checking if the given site_domain in the enforcement dictionary
if force_siteid_domain:
for mydomain, mysites in force_siteid_domain.items():
re_domain = re.compile(mydomain)
if re_domain.match(site_domain):
dq2_local_ids = mysites
toa_check = False
break
# checking if the given se_hostname in the enforcement dictionary
if force_siteid_se:
for myse, mysites in force_siteid_se.items():
re_se = re.compile(myse)
if re_se.match(se_hostname):
dq2_local_ids = mysites
toa_check = False
break
# if enforcement is never applied, do the detection for dq2_local_ids via ToA
if toa_check:
alternateNameDict = {}
for sitename in TiersOfATLAS.getAllSources():
# compose the altname:siteid dictionary
altnames = TiersOfATLAS.getSiteProperty(sitename,'alternateName')
if altnames:
for altname in altnames:
if altname not in alternateNameDict:
alternateNameDict[altname] = []
alternateNameDict[altname].append(sitename)
# First search for srm
dq2srm = TiersOfATLAS.getSiteProperty(sitename,'srm')
if dq2srm and dq2srm.find(se_hostname)>=0:
dq2_local_ids.append(sitename)
# Second search for domainname
dq2domain = TiersOfATLAS.getSiteProperty(sitename,'domain')
if dq2domain and dq2domain.find(site_domain)>=0:
dq2_local_ids.append(sitename)
# Thirdly search for sites with the same alternateName
more_dq2_local_ids = []
for sitename in dq2_local_ids:
altnames = TiersOfATLAS.getSiteProperty(sitename,'alternateName')
if altnames:
for altname in altnames:
if altname in alternateNameDict:
more_dq2_local_ids += alternateNameDict[altname]
dq2_local_ids += more_dq2_local_ids
# resolving the best location according to the dataset locations
# - get the common part between dq2_local_ids and ds_locations
# - pick the first one in the common part
candidates = list(set(dq2_local_ids) & set(ds_locations))
print >> sys.stdout, str(candidates)
if candidates:
## pick up the site whose corresponding se is matching the se_hostname
for c in candidates:
srm_info = get_srm_endpoint(c)
if srm_info['se_host'] == se_hostname:
dq2_local_site_id = c
break
## otherwise, take the first candidate in the list
if not dq2_local_site_id:
dq2_local_site_id = candidates[0]
return dq2_local_site_id
import os
from dq2.clientapi.DQ2 import DQ2
from dq2.info import TiersOfATLAS
from dq2.info.TiersOfATLAS import _refreshToACache, ToACache
_refreshToACache()
dq2localid = os.environ['DQ2_LOCAL_SITE_ID']
dq2localid_alternatename = TiersOfATLAS.getSiteProperty(dq2localid,'alternateName')
if not dq2localid_alternatename:
try:
dq2localid_alternatename = dq2localid.split("_")[0]
except:
dq2localid_alternatename = dq2localid
db_dataset = os.environ['ATLAS_DBRELEASE']
dq2=DQ2()
db_locations = dq2.listDatasetReplicas(db_dataset).values()[0][1]
print "db_locations = ", db_locations
db_site = dq2localid
for sitename in TiersOfATLAS.getAllSources():
if TiersOfATLAS.getSiteProperty(sitename,'alternateName')==dq2localid_alternatename and sitename in db_locations:
db_site = sitename
outFile = open('db_dq2localid.txt','w')
outFile.write('%s\n' % db_site)
outFile.close()
def resolve_dq2_local_site_id(ds_locations,
site_domain,
se_hostname,
force_siteid_domain={},
force_siteid_se={}):
'''
resolves the DQ2_LOCAL_SITE_ID based on 3 factors:
- the dataset location list
- the site domain
- the default SE host
- the dictionaries containing the local_site_id enforcement
The "force_siteid_domain" gives a list in which the matched site_domain (regx match)
will be limited to the given list of DQ2 site ids.
The "force_siteid_se" gives a list in which the matched se_hostname (regx match)
will be limited to the given list of DQ2 site ids.
it returns a final DQ2 site id which is local and containing the dataset replica
or an empty string if there is no matched DQ2 site id.
'''
dq2_local_site_id = ''
toa_check = True
# get dq2 site ids matching the site_domain and se_hostname
dq2_local_ids = []
# checking if the given site_domain in the enforcement dictionary
if force_siteid_domain:
for mydomain, mysites in force_siteid_domain.items():
re_domain = re.compile(mydomain)
if re_domain.match(site_domain):
dq2_local_ids = mysites
toa_check = False
break
# checking if the given se_hostname in the enforcement dictionary
if force_siteid_se:
for myse, mysites in force_siteid_se.items():
re_se = re.compile(myse)
if re_se.match(se_hostname):
dq2_local_ids = mysites
toa_check = False
break
# if enforcement is never applied, do the detection for dq2_local_ids via ToA
if toa_check:
alternateNameDict = {}
for sitename in TiersOfATLAS.getAllSources():
# compose the altname:siteid dictionary
altnames = TiersOfATLAS.getSiteProperty(sitename, 'alternateName')
if altnames:
for altname in altnames:
if altname not in alternateNameDict:
alternateNameDict[altname] = []
alternateNameDict[altname].append(sitename)
# First search for srm
dq2srm = TiersOfATLAS.getSiteProperty(sitename, 'srm')
if dq2srm and dq2srm.find(se_hostname) >= 0:
dq2_local_ids.append(sitename)
# Second search for domainname
dq2domain = TiersOfATLAS.getSiteProperty(sitename, 'domain')
if dq2domain and dq2domain.find(site_domain) >= 0:
dq2_local_ids.append(sitename)
# Thirdly search for sites with the same alternateName
more_dq2_local_ids = []
for sitename in dq2_local_ids:
altnames = TiersOfATLAS.getSiteProperty(sitename, 'alternateName')
if altnames:
for altname in altnames:
if altname in alternateNameDict:
more_dq2_local_ids += alternateNameDict[altname]
dq2_local_ids += more_dq2_local_ids
# resolving the best location according to the dataset locations
# - get the common part between dq2_local_ids and ds_locations
# - pick the first one in the common part
candidates = list(set(dq2_local_ids) & set(ds_locations))
print >> sys.stdout, str(candidates)
if candidates:
## pick up the site whose corresponding se is matching the se_hostname
for c in candidates:
srm_info = get_srm_endpoint(c)
if srm_info['se_host'] == se_hostname:
dq2_local_site_id = c
break
## otherwise, take the first candidate in the list
if not dq2_local_site_id:
dq2_local_site_id = candidates[0]
return dq2_local_site_id
