class FileLookup(MappingManager): """Main class which perform LFN/PFN/Site/SE operations""" def __init__(self, cp): super(FileLookup, self).__init__(cp) self.cp = cp self.section = "file_lookup" self.priorities = self._parse_priority_rules() dbsurl = cp.get('dbs', 'url') dbsinst = cp.get('dbs', 'instance') dbsparams = cp.get('dbs', 'params') self.phedex_url = cp.get('phedex', 'url') dbsconfig = {'dbs':dbsurl, 'dbsinst':dbsinst, 'dbsparams':dbsparams, 'phedex':self.phedex_url} self._dbs = DBS(dbsconfig) self.sitedb_url = cp.get('sitedb', 'url') self.sitedb = SiteDBManager(self.sitedb_url) self._downSites = [] self._lastSiteQuery = 0 self._lock = threading.Lock() self._lfns = {} self._lfns_cache = {} self.acquireTURL = self.acquireValue self.releaseTURL = self.releaseKey def acquireTURL(self, SURL): """Get TURL for provided SURL""" self.log.info("Looking up TURL for SURL %s." % SURL) TURL = self.acquireValue(SURL) self.log.info("Found TURL %s for SURL %s." % (TURL, SURL)) return TURL def releaseTURL(self, SURL): """Release TURL for provided SURL""" self.log.info("Releasing SURL %s." % SURL) self.releaseTURL(SURL) def replicas(self, lfn, token=None, user=None): """Find LFN replicas in PhEDEx data-service""" self.log.info("Looking for the block of LFN %s." % lfn) block = self._dbs.blockLookup(lfn) query = {'block':block} self.log.info("Looking for replicas of %s" % block) results = phedex_datasvc('fileReplicas', self.phedex_url, block=block) blocks = [i for i in results['phedex']['block'] \ if i.get('name', None) == block] if not blocks: raise Exception("Requested LFN does not exist in any block known " \ "to PhEDEx.") block = blocks[0] files = [i for i in block.get('file', []) \ if i.get('name', None) == lfn] if not files: raise Exception("Internal error: PhEDEx does not think LFN is in " \ "the same block as DBS does.") file = files[0] replicas = [i['node'] for i in file.get('replica', []) if 'node' in i] self.log.info("There are the following replicas of %s: %s." % \ (lfn, ', '.join(replicas))) return replicas def _parse_priority_rules(self): """Parse priority rules""" priority_dict = {} name_regexp = re.compile('priority_([0-9]+)') try: rules = self.cp.items(self.section) except: rules = {} if not rules: raise Exception("FileMover configured withot priority rules") for name, value in rules: value = value.strip() value = re.compile(value) m = name_regexp.match(name) if not m: continue priority = long(m.groups()[0]) priority_dict[priority] = value return priority_dict def _getSiteStatus(self): """ Update the list of down/bad sites """ if time.time() - self._lastSiteQuery > 600: # Update the list of bad sites. self._lastSiteQuery = time.time() def removeBadSites(self, sites): """ Given a list of sites, remove any which we do not want to transfer with for some reason. """ self._getSiteStatus() filtered_list = [] for site in sites: if site not in self._downSites: filtered_list.append(site) return filtered_list def pickSite(self, replicas, exclude_list=None): """Pick up site from provided replicases and exclude site list""" priorities = self.priorities.keys() priorities.sort() source = None for priority in priorities: for site in replicas: if exclude_list and exclude_list.count(site): continue m = self.priorities[priority].search(site) if m: source = site break if source != None: break if source == None: raise ValueError("Could not match site to any priority. " \ "Possible sources: %s" % str(replicas)) return source def mapLFN(self, site, lfn, protocol=None): """Map LFN to given site""" if not protocol: protocol = 'srmv2' data = {'lfn': lfn, 'node': site} if protocol: data['protocol'] = protocol self.log.info("Mapping LFN %s for site %s using PhEDEx datasvc." % \ (lfn, site)) data = phedex_datasvc('lfn2pfn', self.phedex_url, **data) try: pfn = data['phedex']['mapping'][0]['pfn'] except: raise Exception("PhEDEx data service did not return a PFN!") self.log.info("PhEDEx data service returned PFN %s for LFN %s." % \ (pfn, lfn)) return pfn def getPFN(self, lfn, protocol=None, exclude_sites=None): """Get PFN for given LFN""" # replicas = self.replicas(lfn) # if len(replicas) == 0: # raise Exception("The LFN %s has no known replicas in PhEDEx.") # site = self.pickSite(replicas) # pfn = self.mapLFN(site, lfn) # return pfn self._lock.acquire() site = '' try: key = (lfn, protocol) if key in self._lfns and time.time() - \ self._lfns_cache.get(key, 0) < 10: return self._lfns[key], site finally: self._lock.release() try: replicas = self.replicas(lfn) if len(replicas) == 0: raise \ Exception("The LFN %s has no known replicas in PhEDEx."%lfn) site = self.pickSite(replicas, exclude_sites) except: bsList = self._dbs.blockSiteLookup(lfn) seList = [s for b, s in bsList] msg = "Fail to look-up T[1-3] CMS site for\n" msg += "LFN=%s\nSE's list %s\n" % (lfn, seList) if not seList: raise Exception(msg) site = self.getSiteFromSDB(seList, exclude_sites) if not site: raise Exception(msg) pfn = self.mapLFN(site, lfn, protocol=protocol) self._lock.acquire() try: key = (lfn, protocol) self._lfns[key] = pfn self._lfns_cache[key] = time.time() finally: self._lock.release() return pfn, site def getSiteFromSDB(self, seList, exclude_sites): """ Get SE names for give cms names """ sites = [] for sename in seList: site = self.sitedb.get_name(sename) if site: sites.append(site) site = self.pickSite(sites, exclude_sites) return site def _lookup(self, SURL): """ Return the corresponding gsiftp TURL for a given SURL. """ # SURL (Storage URL, aka PFN) should be in a form # <sfn|srm>://<SE_hostname>/<some_string>.root pat = re.compile("(sfn|srm)://[a-zA-Z0-9].*.*root$") if pat.match(SURL): raise Exception("Bad SURL: %s" % SURL) options = "-T srmv2 -b -p gsiftp" cmd = "lcg-getturls %s %s" % (options, SURL) self.log.info("Looking up TURL for %s." % SURL) print cmd fd = os.popen(cmd) turl = fd.read() print turl if fd.close(): if not turl.startswith("gsiftp://"): # Sometimes lcg-* segfaults self.log.error("Unable to get TURL for SURL %s." % SURL) self.log.error("Error message: %s" % turl) raise ValueError("Unable to get TURL for SURL %s." % SURL) turl = turl.strip() self.log.info("Found TURL %s for %s." % (turl, SURL)) return turl def _release(self, SURL): """
def srmcp(lfn, verbose=None): """Invoke srmcp command over provided LFN""" # query DBS for SE's query = "find site where file=%s" % lfn dbs_instances = [ "cms_dbs_prod_global", "cms_dbs_caf_analysis_01", "cms_dbs_ph_analysis_01", "cms_dbs_ph_analysis_02", "cms_dbs_prod_local_01", "cms_dbs_prod_local_02", "cms_dbs_prod_local_03", "cms_dbs_prod_local_04", "cms_dbs_prod_local_05", "cms_dbs_prod_local_06", "cms_dbs_prod_local_07", "cms_dbs_prod_local_08", "cms_dbs_prod_local_09", ] sites = [] for dbs in dbs_instances: url = "http://cmsdbsprod.cern.ch/%s/servlet/DBSServlet" % dbs query_file = "find file.size where file=%s" % lfn params = {"api": "executeQuery", "apiversion": "DBS_2_0_9", "query": query_file} data = urllib2.urlopen(url, urllib.urlencode(params)) for item in [i for i in parser(data.read())]: file_size = item["file.size"] print "file: %s" % lfn print "size: %s Bytes, %s MB" % (file_size, long(file_size) / 1024.0 / 1024.0) print "Lookup LFN in %s" % dbs params = {"api": "executeQuery", "apiversion": "DBS_2_0_9", "query": query} data = urllib2.urlopen(url, urllib.urlencode(params)) sites = [i for i in parser(data.read())] if not sites: continue if verbose: print "DBS instance: %s" % dbs print "------------" print "LFN", lfn print "located at the following sites" print sites print break if not sites: msg = "No site found for given lfn" print msg sys.exit(1) # query SiteDB for CMS names sitedict = {} sitedb_url = "https://cmsweb.cern.ch/sitedb/data/prod" sitedbmgr = SiteDBManager(sitedb_url) for item in sites: site = item["site"] if site: sitedict[site] = sitedbmgr.get_name(site) if verbose: print "SiteDB reports:" print "---------------" print sitedict print # query Phedex for PFNs pfnlist = [] for cmsname in sitedict.values(): url = "https://cmsweb.cern.ch/phedex/datasvc/json/prod/lfn2pfn" params = {"protocol": "srmv2", "lfn": lfn, "node": cmsname} if verbose: print "Request %s?%s" % (url, urllib.urlencode(params, doseq=True)) data = urllib2.urlopen(url, urllib.urlencode(params, doseq=True)) result = json.loads(data.read()) try: for item in result["phedex"]["mapping"]: pfn = item["pfn"] if pfn not in pfnlist: pfnlist.append(pfn) except: msg = "Fail to look-up PFNs in Phedex\n" + str(result) # raise Exception(msg) print msg continue if verbose: print "Phedex reports:" print "--------------" print pfnlist print print "----- END OF VERBOSE OUTPUT -----" # finally let's create srmcp commands for each found pfn for item in pfnlist: filename = item.split("/")[-1] # srmcp command cmd = "srmcp -debug=true -srm_protocol_version=2" cmd += " -retry_num=1 -streams_num=1 %s file:////tmp/%s" % (item, filename) yield cmd # lcg-cp command cmd = "lcg-cp --verbose --vo cms %s file:////tmp/%s" % (item, filename) yield cmd