def resolve_dq2_local_site_id(ds_locations, site_domain, se_hostname, force_siteid_domain={}, force_siteid_se={}): ''' resolves the DQ2_LOCAL_SITE_ID based on 3 factors: - the dataset location list - the site domain - the default SE host - the dictionaries containing the local_site_id enforcement The "force_siteid_domain" gives a list in which the matched site_domain (regx match) will be limited to the given list of DQ2 site ids. The "force_siteid_se" gives a list in which the matched se_hostname (regx match) will be limited to the given list of DQ2 site ids. it returns a final DQ2 site id which is local and containing the dataset replica or an empty string if there is no matched DQ2 site id. ''' dq2_local_site_id = '' toa_check = True # get dq2 site ids matching the site_domain and se_hostname dq2_local_ids = [] # checking if the given site_domain in the enforcement dictionary if force_siteid_domain: for mydomain, mysites in force_siteid_domain.items(): re_domain = re.compile(mydomain) if re_domain.match(site_domain): dq2_local_ids = mysites toa_check = False break # checking if the given se_hostname in the enforcement dictionary if force_siteid_se: for myse, mysites in force_siteid_se.items(): re_se = re.compile(myse) if re_se.match(se_hostname): dq2_local_ids = mysites toa_check = False break # if enforcement is never applied, do the detection for dq2_local_ids via ToA if toa_check: alternateNameDict = {} for sitename in TiersOfATLAS.getAllSources(): # compose the altname:siteid dictionary altnames = TiersOfATLAS.getSiteProperty(sitename,'alternateName') if altnames: for altname in altnames: if altname not in alternateNameDict: alternateNameDict[altname] = [] alternateNameDict[altname].append(sitename) # First search for srm dq2srm = TiersOfATLAS.getSiteProperty(sitename,'srm') if dq2srm and dq2srm.find(se_hostname)>=0: dq2_local_ids.append(sitename) # Second search for domainname dq2domain = TiersOfATLAS.getSiteProperty(sitename,'domain') if dq2domain and dq2domain.find(site_domain)>=0: dq2_local_ids.append(sitename) # Thirdly search for sites with the same alternateName more_dq2_local_ids = [] for sitename in dq2_local_ids: altnames = TiersOfATLAS.getSiteProperty(sitename,'alternateName') if altnames: for altname in altnames: if altname in alternateNameDict: more_dq2_local_ids += alternateNameDict[altname] dq2_local_ids += more_dq2_local_ids # resolving the best location according to the dataset locations # - get the common part between dq2_local_ids and ds_locations # - pick the first one in the common part candidates = list(set(dq2_local_ids) & set(ds_locations)) print >> sys.stdout, str(candidates) if candidates: ## pick up the site whose corresponding se is matching the se_hostname for c in candidates: srm_info = get_srm_endpoint(c) if srm_info['se_host'] == se_hostname: dq2_local_site_id = c break ## otherwise, take the first candidate in the list if not dq2_local_site_id: dq2_local_site_id = candidates[0] return dq2_local_site_id
import os from dq2.clientapi.DQ2 import DQ2 from dq2.info import TiersOfATLAS from dq2.info.TiersOfATLAS import _refreshToACache, ToACache _refreshToACache() dq2localid = os.environ['DQ2_LOCAL_SITE_ID'] dq2localid_alternatename = TiersOfATLAS.getSiteProperty(dq2localid,'alternateName') if not dq2localid_alternatename: try: dq2localid_alternatename = dq2localid.split("_")[0] except: dq2localid_alternatename = dq2localid db_dataset = os.environ['ATLAS_DBRELEASE'] dq2=DQ2() db_locations = dq2.listDatasetReplicas(db_dataset).values()[0][1] print "db_locations = ", db_locations db_site = dq2localid for sitename in TiersOfATLAS.getAllSources(): if TiersOfATLAS.getSiteProperty(sitename,'alternateName')==dq2localid_alternatename and sitename in db_locations: db_site = sitename outFile = open('db_dq2localid.txt','w') outFile.write('%s\n' % db_site) outFile.close()
def resolve_dq2_local_site_id(ds_locations, site_domain, se_hostname, force_siteid_domain={}, force_siteid_se={}): ''' resolves the DQ2_LOCAL_SITE_ID based on 3 factors: - the dataset location list - the site domain - the default SE host - the dictionaries containing the local_site_id enforcement The "force_siteid_domain" gives a list in which the matched site_domain (regx match) will be limited to the given list of DQ2 site ids. The "force_siteid_se" gives a list in which the matched se_hostname (regx match) will be limited to the given list of DQ2 site ids. it returns a final DQ2 site id which is local and containing the dataset replica or an empty string if there is no matched DQ2 site id. ''' dq2_local_site_id = '' toa_check = True # get dq2 site ids matching the site_domain and se_hostname dq2_local_ids = [] # checking if the given site_domain in the enforcement dictionary if force_siteid_domain: for mydomain, mysites in force_siteid_domain.items(): re_domain = re.compile(mydomain) if re_domain.match(site_domain): dq2_local_ids = mysites toa_check = False break # checking if the given se_hostname in the enforcement dictionary if force_siteid_se: for myse, mysites in force_siteid_se.items(): re_se = re.compile(myse) if re_se.match(se_hostname): dq2_local_ids = mysites toa_check = False break # if enforcement is never applied, do the detection for dq2_local_ids via ToA if toa_check: alternateNameDict = {} for sitename in TiersOfATLAS.getAllSources(): # compose the altname:siteid dictionary altnames = TiersOfATLAS.getSiteProperty(sitename, 'alternateName') if altnames: for altname in altnames: if altname not in alternateNameDict: alternateNameDict[altname] = [] alternateNameDict[altname].append(sitename) # First search for srm dq2srm = TiersOfATLAS.getSiteProperty(sitename, 'srm') if dq2srm and dq2srm.find(se_hostname) >= 0: dq2_local_ids.append(sitename) # Second search for domainname dq2domain = TiersOfATLAS.getSiteProperty(sitename, 'domain') if dq2domain and dq2domain.find(site_domain) >= 0: dq2_local_ids.append(sitename) # Thirdly search for sites with the same alternateName more_dq2_local_ids = [] for sitename in dq2_local_ids: altnames = TiersOfATLAS.getSiteProperty(sitename, 'alternateName') if altnames: for altname in altnames: if altname in alternateNameDict: more_dq2_local_ids += alternateNameDict[altname] dq2_local_ids += more_dq2_local_ids # resolving the best location according to the dataset locations # - get the common part between dq2_local_ids and ds_locations # - pick the first one in the common part candidates = list(set(dq2_local_ids) & set(ds_locations)) print >> sys.stdout, str(candidates) if candidates: ## pick up the site whose corresponding se is matching the se_hostname for c in candidates: srm_info = get_srm_endpoint(c) if srm_info['se_host'] == se_hostname: dq2_local_site_id = c break ## otherwise, take the first candidate in the list if not dq2_local_site_id: dq2_local_site_id = candidates[0] return dq2_local_site_id