def __init__(self, cp):
super(FileLookup, self).__init__(cp)
self.cp = cp
self.section = "file_lookup"
self.priorities = self._parse_priority_rules()
dbsurl = cp.get('dbs', 'url')
dbsinst = cp.get('dbs', 'instance')
dbsparams = cp.get('dbs', 'params')
self.phedex_url = cp.get('phedex', 'url')
dbsconfig = {'dbs':dbsurl, 'dbsinst':dbsinst, 'dbsparams':dbsparams,
'phedex':self.phedex_url}
self._dbs = DBS(dbsconfig)
self.sitedb_url = cp.get('sitedb', 'url')
self.sitedb = SiteDBManager(self.sitedb_url)
self._downSites = []
self._lastSiteQuery = 0
self._lock = threading.Lock()
self._lfns = {}
self._lfns_cache = {}
self.acquireTURL = self.acquireValue
self.releaseTURL = self.releaseKey
class FileLookup(MappingManager):
"""Main class which perform LFN/PFN/Site/SE operations"""
def __init__(self, cp):
super(FileLookup, self).__init__(cp)
self.cp = cp
self.section = "file_lookup"
self.priorities = self._parse_priority_rules()
dbsurl = cp.get('dbs', 'url')
dbsinst = cp.get('dbs', 'instance')
dbsparams = cp.get('dbs', 'params')
self.phedex_url = cp.get('phedex', 'url')
dbsconfig = {'dbs':dbsurl, 'dbsinst':dbsinst, 'dbsparams':dbsparams,
'phedex':self.phedex_url}
self._dbs = DBS(dbsconfig)
self.sitedb_url = cp.get('sitedb', 'url')
self.sitedb = SiteDBManager(self.sitedb_url)
self._downSites = []
self._lastSiteQuery = 0
self._lock = threading.Lock()
self._lfns = {}
self._lfns_cache = {}
self.acquireTURL = self.acquireValue
self.releaseTURL = self.releaseKey
def acquireTURL(self, SURL):
"""Get TURL for provided SURL"""
self.log.info("Looking up TURL for SURL %s." % SURL)
TURL = self.acquireValue(SURL)
self.log.info("Found TURL %s for SURL %s." % (TURL, SURL))
return TURL
def releaseTURL(self, SURL):
"""Release TURL for provided SURL"""
self.log.info("Releasing SURL %s." % SURL)
self.releaseTURL(SURL)
def replicas(self, lfn, token=None, user=None):
"""Find LFN replicas in PhEDEx data-service"""
self.log.info("Looking for the block of LFN %s." % lfn)
block = self._dbs.blockLookup(lfn)
query = {'block':block}
self.log.info("Looking for replicas of %s" % block)
results = phedex_datasvc('fileReplicas', self.phedex_url, block=block)
blocks = [i for i in results['phedex']['block'] \
if i.get('name', None) == block]
if not blocks:
raise Exception("Requested LFN does not exist in any block known " \
"to PhEDEx.")
block = blocks[0]
files = [i for i in block.get('file', []) \
if i.get('name', None) == lfn]
if not files:
raise Exception("Internal error: PhEDEx does not think LFN is in " \
"the same block as DBS does.")
file = files[0]
replicas = [i['node'] for i in file.get('replica', []) if 'node' in i]
self.log.info("There are the following replicas of %s: %s." % \
(lfn, ', '.join(replicas)))
return replicas
def _parse_priority_rules(self):
"""Parse priority rules"""
priority_dict = {}
name_regexp = re.compile('priority_([0-9]+)')
try:
rules = self.cp.items(self.section)
except:
rules = {}
if not rules:
raise Exception("FileMover configured withot priority rules")
for name, value in rules:
value = value.strip()
value = re.compile(value)
m = name_regexp.match(name)
if not m:
continue
priority = long(m.groups()[0])
priority_dict[priority] = value
return priority_dict
def _getSiteStatus(self):
"""
Update the list of down/bad sites
"""
if time.time() - self._lastSiteQuery > 600:
# Update the list of bad sites.
self._lastSiteQuery = time.time()
def removeBadSites(self, sites):
"""
Given a list of sites, remove any which we do not want to transfer
with for some reason.
"""
self._getSiteStatus()
filtered_list = []
for site in sites:
if site not in self._downSites:
filtered_list.append(site)
return filtered_list
def pickSite(self, replicas, exclude_list=None):
"""Pick up site from provided replicases and exclude site list"""
priorities = self.priorities.keys()
priorities.sort()
source = None
for priority in priorities:
for site in replicas:
if exclude_list and exclude_list.count(site):
continue
m = self.priorities[priority].search(site)
if m:
source = site
break
if source != None:
break
if source == None:
raise ValueError("Could not match site to any priority. " \
"Possible sources: %s" % str(replicas))
return source
def mapLFN(self, site, lfn, protocol=None):
"""Map LFN to given site"""
if not protocol:
protocol = 'srmv2'
data = {'lfn': lfn, 'node': site}
if protocol:
data['protocol'] = protocol
self.log.info("Mapping LFN %s for site %s using PhEDEx datasvc." % \
(lfn, site))
data = phedex_datasvc('lfn2pfn', self.phedex_url, **data)
try:
pfn = data['phedex']['mapping'][0]['pfn']
except:
raise Exception("PhEDEx data service did not return a PFN!")
self.log.info("PhEDEx data service returned PFN %s for LFN %s." % \
(pfn, lfn))
return pfn
def getPFN(self, lfn, protocol=None, exclude_sites=None):
"""Get PFN for given LFN"""
# replicas = self.replicas(lfn)
# if len(replicas) == 0:
# raise Exception("The LFN %s has no known replicas in PhEDEx.")
# site = self.pickSite(replicas)
# pfn = self.mapLFN(site, lfn)
# return pfn
self._lock.acquire()
site = ''
try:
key = (lfn, protocol)
if key in self._lfns and time.time() - \
self._lfns_cache.get(key, 0) < 10:
return self._lfns[key], site
finally:
self._lock.release()
try:
replicas = self.replicas(lfn)
if len(replicas) == 0:
raise \
Exception("The LFN %s has no known replicas in PhEDEx."%lfn)
site = self.pickSite(replicas, exclude_sites)
except:
bsList = self._dbs.blockSiteLookup(lfn)
seList = [s for b, s in bsList]
msg = "Fail to look-up T[1-3] CMS site for\n"
msg += "LFN=%s\nSE's list %s\n" % (lfn, seList)
if not seList:
raise Exception(msg)
site = self.getSiteFromSDB(seList, exclude_sites)
if not site:
raise Exception(msg)
pfn = self.mapLFN(site, lfn, protocol=protocol)
self._lock.acquire()
try:
key = (lfn, protocol)
self._lfns[key] = pfn
self._lfns_cache[key] = time.time()
finally:
self._lock.release()
return pfn, site
def getSiteFromSDB(self, seList, exclude_sites):
"""
Get SE names for give cms names
"""
sites = []
for sename in seList:
site = self.sitedb.get_name(sename)
if site:
sites.append(site)
site = self.pickSite(sites, exclude_sites)
return site
def _lookup(self, SURL):
"""
Return the corresponding gsiftp TURL for a given SURL.
"""
# SURL (Storage URL, aka PFN) should be in a form
# <sfn|srm>://<SE_hostname>/<some_string>.root
pat = re.compile("(sfn|srm)://[a-zA-Z0-9].*.*root$")
if pat.match(SURL):
raise Exception("Bad SURL: %s" % SURL)
options = "-T srmv2 -b -p gsiftp"
cmd = "lcg-getturls %s %s" % (options, SURL)
self.log.info("Looking up TURL for %s." % SURL)
print cmd
fd = os.popen(cmd)
turl = fd.read()
print turl
if fd.close():
if not turl.startswith("gsiftp://"): # Sometimes lcg-* segfaults
self.log.error("Unable to get TURL for SURL %s." % SURL)
self.log.error("Error message: %s" % turl)
raise ValueError("Unable to get TURL for SURL %s." % SURL)
turl = turl.strip()
self.log.info("Found TURL %s for %s." % (turl, SURL))
return turl
def _release(self, SURL):
"""
def srmcp(lfn, verbose=None):
"""Invoke srmcp command over provided LFN"""
# query DBS for SE's
query = "find site where file=%s" % lfn
dbs_instances = [
"cms_dbs_prod_global",
"cms_dbs_caf_analysis_01",
"cms_dbs_ph_analysis_01",
"cms_dbs_ph_analysis_02",
"cms_dbs_prod_local_01",
"cms_dbs_prod_local_02",
"cms_dbs_prod_local_03",
"cms_dbs_prod_local_04",
"cms_dbs_prod_local_05",
"cms_dbs_prod_local_06",
"cms_dbs_prod_local_07",
"cms_dbs_prod_local_08",
"cms_dbs_prod_local_09",
]
sites = []
for dbs in dbs_instances:
url = "http://cmsdbsprod.cern.ch/%s/servlet/DBSServlet" % dbs
query_file = "find file.size where file=%s" % lfn
params = {"api": "executeQuery", "apiversion": "DBS_2_0_9", "query": query_file}
data = urllib2.urlopen(url, urllib.urlencode(params))
for item in [i for i in parser(data.read())]:
file_size = item["file.size"]
print "file: %s" % lfn
print "size: %s Bytes, %s MB" % (file_size, long(file_size) / 1024.0 / 1024.0)
print "Lookup LFN in %s" % dbs
params = {"api": "executeQuery", "apiversion": "DBS_2_0_9", "query": query}
data = urllib2.urlopen(url, urllib.urlencode(params))
sites = [i for i in parser(data.read())]
if not sites:
continue
if verbose:
print "DBS instance: %s" % dbs
print "------------"
print "LFN", lfn
print "located at the following sites"
print sites
print
break
if not sites:
msg = "No site found for given lfn"
print msg
sys.exit(1)
# query SiteDB for CMS names
sitedict = {}
sitedb_url = "https://cmsweb.cern.ch/sitedb/data/prod"
sitedbmgr = SiteDBManager(sitedb_url)
for item in sites:
site = item["site"]
if site:
sitedict[site] = sitedbmgr.get_name(site)
if verbose:
print "SiteDB reports:"
print "---------------"
print sitedict
print
# query Phedex for PFNs
pfnlist = []
for cmsname in sitedict.values():
url = "https://cmsweb.cern.ch/phedex/datasvc/json/prod/lfn2pfn"
params = {"protocol": "srmv2", "lfn": lfn, "node": cmsname}
if verbose:
print "Request %s?%s" % (url, urllib.urlencode(params, doseq=True))
data = urllib2.urlopen(url, urllib.urlencode(params, doseq=True))
result = json.loads(data.read())
try:
for item in result["phedex"]["mapping"]:
pfn = item["pfn"]
if pfn not in pfnlist:
pfnlist.append(pfn)
except:
msg = "Fail to look-up PFNs in Phedex\n" + str(result)
# raise Exception(msg)
print msg
continue
if verbose:
print "Phedex reports:"
print "--------------"
print pfnlist
print
print "----- END OF VERBOSE OUTPUT -----"
# finally let's create srmcp commands for each found pfn
for item in pfnlist:
filename = item.split("/")[-1]
# srmcp command
cmd = "srmcp -debug=true -srm_protocol_version=2"
cmd += " -retry_num=1 -streams_num=1 %s file:////tmp/%s" % (item, filename)
yield cmd
# lcg-cp command
cmd = "lcg-cp --verbose --vo cms %s file:////tmp/%s" % (item, filename)
yield cmd
