def get_pfns(lfn, verbose=None): """ Look-up LFN in Phedex and get corresponding list of PFNs """ pfnlist = [] selist = [] params = {'se':'*', 'lfn':lfn} json_dict = get_data(phedex_url('fileReplicas'), params) ddict = DotDict(json_dict) if not json_dict['phedex']['block']: return pfnlist, selist for fname in ddict.get('phedex.block.file'): for replica in fname['replica']: cmsname = replica['node'] se = replica['se'] if se not in selist: selist.append(se) # query Phedex for PFN params = {'protocol':'srmv2', 'lfn':lfn, 'node':cmsname} result = get_data(phedex_url('lfn2pfn'), params) try: for item in result['phedex']['mapping']: pfn = item['pfn'] if pfn not in pfnlist: pfnlist.append(pfn) except: msg = "Fail to look-up PFNs in Phedex\n" + str(result) print msg continue return pfnlist, selist
def get_pfns(lfn, verbose=None): """ Look-up LFN in Phedex and get corresponding list of PFNs """ pfnlist = [] selist = [] params = {'se': '*', 'lfn': lfn} json_dict = get_data(phedex_url('fileReplicas'), params) ddict = DotDict(json_dict) if not json_dict['phedex']['block']: return pfnlist, selist for fname in ddict.get('phedex.block.file'): for replica in fname['replica']: cmsname = replica['node'] se = replica['se'] if se not in selist: selist.append(se) # query Phedex for PFN params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname} result = get_data(phedex_url('lfn2pfn'), params) try: for item in result['phedex']['mapping']: pfn = item['pfn'] if pfn not in pfnlist: pfnlist.append(pfn) except: msg = "Fail to look-up PFNs in Phedex\n" + str(result) print msg continue return pfnlist, selist
def list_sites4file(self, **kwargs): """ Controller to get sites for given file """ url = phedex_url('fileReplicas') params = {'lfn': kwargs['filename']} return find_sites(url, params)
def list_sites4dataset(self, **kwargs): """ Controller to get sites for given dataset """ url = phedex_url('fileReplicas') params = {'dataset': kwargs['dataset']} return find_sites(url, params)
def list_sites4dataset(self, **kwargs): """ Controller to get sites for given dataset """ url = phedex_url("fileReplicas") params = {"dataset": kwargs["dataset"]} return find_sites(url, params)
def list_sites4file(self, **kwargs): """ Controller to get sites for given file """ url = phedex_url("fileReplicas") params = {"lfn": kwargs["filename"]} return find_sites(url, params)
def list_block4site(self, **kwargs): """ Controller to get site info """ url = phedex_url("blockreplicasummary") params = {"node": kwargs["sitename"]} data = get_data(url, params) plist = [Block(b) for b in data["phedex"]["block"]] return plist
def list_block4site(self, **kwargs): """ Controller to get site info """ url = phedex_url('blockreplicasummary') params = {'node': kwargs['sitename']} data = get_data(url, params) plist = [Block(b) for b in data['phedex']['block']] return plist
def list_sites(self, **kwargs): """ Controller to get site info """ url = phedex_url('nodeusage') params = {'node': kwargs['sitename']} data = get_data(url, params) plist = [Site(s) for s in data['phedex']['node']] return plist
def resolve_srm_path(node, verbose=None): """ Use TFC phedex API to resolve srm path for given node """ params = {'node': node} result = get_data(phedex_url('tfc'), params) for row in result['phedex']['storage-mapping']['array']: if row['protocol'] == 'srmv2' and row['element_name'] == 'lfn-to-pfn': yield (row['result'], row['path-match'])
def list_sites(self, **kwargs): """ Controller to get site info """ url = phedex_url("nodeusage") params = {"node": kwargs["sitename"]} data = get_data(url, params) plist = [Site(s) for s in data["phedex"]["node"]] return plist
def resolve_srm_path(node, verbose=None): """ Use TFC phedex API to resolve srm path for given node """ params = {'node':node} result = get_data(phedex_url('tfc'), params) for row in result['phedex']['storage-mapping']['array']: if row['protocol'] == 'srmv2' and row['element_name'] == 'lfn-to-pfn': yield (row['result'], row['path-match'])
def resolve_user_srm_path(node, ldir='/store/user', verbose=None): """ Use TFC phedex API to resolve srm path for given node """ # change ldir if user supplied full path, e.g. /xrootdfs/cms/store/... ldir = '/store/' + ldir.split('/store/')[-1] params = {'node':node, 'lfn':ldir, 'protocol': 'srmv2'} result = get_data(phedex_url('lfn2pfn'), params) for row in result['phedex']['mapping']: yield row['pfn']
def resolve_user_srm_path(node, ldir='/store/user', verbose=None): """ Use TFC phedex API to resolve srm path for given node """ # change ldir if user supplied full path, e.g. /xrootdfs/cms/store/... ldir = '/store/' + ldir.split('/store/')[-1] params = {'node': node, 'lfn': ldir, 'protocol': 'srmv2'} result = get_data(phedex_url('lfn2pfn'), params) for row in result['phedex']['mapping']: yield row['pfn']
def site_info(dst, verbose=None): """list files at given destination""" url = phedex_url('nodeusage') params = {'node': dst} data = get_data(url, params) res = [Site(s) for s in data['phedex']['node']] paths = [r for r in resolve_user_srm_path(dst)] pdirs = [p.split('=')[-1] for p in paths] for site in res: site.assign('pfn_path', paths) site.assign('default_path', pdirs) return res
def site_info(dst, verbose=None): """list files at given destination""" url = phedex_url("nodeusage") params = {"node": dst} data = get_data(url, params) res = [Site(s) for s in data["phedex"]["node"]] paths = [r for r in resolve_user_srm_path(dst)] pdirs = [p.split("=")[-1] for p in paths] for site in res: site.assign("pfn_path", paths) site.assign("default_path", pdirs) return res
def nodes(select=True): """ Yield list of Phedex nodes, I only select T2 and below """ result = get_data(phedex_url('nodes'), {}) pat = re.compile('^T[0-1]_[A-Z]+(_)[A-Z]+') lnodes = [] for row in result['phedex']['node']: if select and pat.match(row['name']): continue msg = "%s, SE: %s, description %s/%s" \ % (row['name'], row['se'], row['technology'], row['kind']) lnodes.append(msg) lnodes.sort() for row in lnodes: print row
def lfn2pfn(lfn, sename, mgr=None): "Find PFN for given LFN and SE" pfnlist = [] if not mgr: mgr = SiteDBManager() cmsname = mgr.get_name(sename) if cmsname: params = {'protocol':'srmv2', 'lfn':lfn, 'node':cmsname} result = get_data(phedex_url('lfn2pfn'), params) try: for item in result['phedex']['mapping']: pfn = item['pfn'] if pfn not in pfnlist: pfnlist.append(pfn) except: msg = "Fail to look-up PFNs in Phedex\n" + str(result) print msg return pfnlist
def lfn2pfn(lfn, sename, mgr=None): "Find PFN for given LFN and SE" pfnlist = [] if not mgr: mgr = SiteDBManager() cmsname = mgr.get_name(sename) if cmsname: params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname} result = get_data(phedex_url('lfn2pfn'), params) try: for item in result['phedex']['mapping']: pfn = item['pfn'] if pfn not in pfnlist: pfnlist.append(pfn) except: msg = "Fail to look-up PFNs in Phedex\n" + str(result) print msg return pfnlist
def list_du4site(self, **kwargs): """ Controller to get site info """ url = phedex_url('blockReplicas') site = kwargs['sitename'] params = {'node': site} data = get_data(url, params) nfiles = 0 nblocks = 0 size = 0 for row in data['phedex']['block']: nblocks += 1 nfiles += int(row['files']) for rep in row['replica']: if rep['node'] == site: size += long(rep['bytes']) return dict(nblocks=nblocks, nfiles=nfiles, totalsize=size)
def list_du4site(self, **kwargs): """ Controller to get site info """ url = phedex_url("blockReplicas") site = kwargs["sitename"] params = {"node": site} data = get_data(url, params) nfiles = 0 nblocks = 0 size = 0 for row in data["phedex"]["block"]: nblocks += 1 nfiles += int(row["files"]) for rep in row["replica"]: if rep["node"] == site: size += long(rep["bytes"]) return dict(nblocks=nblocks, nfiles=nfiles, totalsize=size)
def pfn_dst(lfn, dst, verbose=None): """ Look-up LFN in Phedex and return pfn dst for further processing """ dstfname = None pat = re.compile('^T[0-9]_[A-Z]+(_)[A-Z]+') if pat.match(dst): dst_split = dst.split(':') dst = dst_split[0] if len(dst_split) == 1: # copy to the node local_path = dst_split[1] for srm_path, lfn_match in resolve_srm_path(dst, verbose): lfn_pat = re.compile(lfn_match) if lfn_pat.match(lfn): srm_path = srm_path.replace('\?', '?').replace('$1', local_path) if verbose: print "Resolve %s into %s" % (dst, srm_path) dst = srm_path else: paths = [p for p in resolve_user_srm_path(dst, verbose=verbose)] dst = '%s/%s' % (paths[0], get_username()) check_permission(dst, verbose) else: if dst.find('file:///') == -1: dstfname = dst.split('/')[-1] if dstfname == '.': dstfname = None if dst[0] == '/': # absolute path if os.path.isdir(dst): ddir = dst dstfname = None else: ddir = '/'.join(dst.split('/')[:-1]) if not os.path.isdir(ddir): msg = 'Provided destination directory %s does not exists' % ddir raise Exception(msg) dst = 'file:///%s' % ddir else: ddir = '/'.join(dst.split('/')[:-1]).replace( '$PWD', os.getcwd()) if os.path.isdir(ddir): dst = 'file:///%s' % os.path.join(os.getcwd(), ddir) else: dst = 'file:///%s' % os.getcwd() pfnlist = [] if os.path.isfile(lfn) or lfn.find('file:///') != -1: # local file pfn = lfn.replace('file:///', '') if pfn[0] != '/': pfn = 'file:///%s' % os.path.join(os.getcwd(), pfn) else: pfn = 'file:///%s' % pfn pfnlist = [pfn] else: if lfn.find(':') != -1: node, lfn = lfn.split(':') params = {'node': node, 'lfn': lfn, 'protocol': 'srmv2'} method = 'lfn2pfn' else: params = {'se': '*', 'lfn': lfn} method = 'fileReplicas' json_dict = get_data(phedex_url(method), params) ddict = DotDict(json_dict) if verbose: print "Look-up LFN:" print lfn phedex = json_dict['phedex'] if phedex.has_key('mapping'): if not phedex['mapping']: msg = "LFN: %s\n" % lfn msg += 'No replicas found\n' msg += str(json_dict) raise Exception(msg) filelist = ddict.get('phedex.mapping.pfn') if not filelist: filelist = [] if isinstance(filelist, basestring): filelist = [filelist] for fname in filelist: pfnlist.append(fname) elif phedex.has_key('block') and not phedex['block']: msg = 'No replicas found in PhEDEx, will try to get original SE from DBS' print_warning(msg) sename = get_dbs_se(lfn) msg = 'Orignal LFN site %s' % sename print_info(msg) mgr = SiteDBManager() pfnlist = lfn2pfn(lfn, sename, mgr) filelist = ddict.get('phedex.block.file') if not filelist: filelist = [] for fname in filelist: for replica in fname['replica']: cmsname = replica['node'] se = replica['se'] if verbose: print "found LFN on node=%s, se=%s" % (cmsname, se) if cmsname.count('T0', 0, 2) == 1: continue # skip T0's # query Phedex for PFN params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname} result = get_data(phedex_url('lfn2pfn'), params) try: for item in result['phedex']['mapping']: pfn = item['pfn'] if pfn not in pfnlist: pfnlist.append(pfn) except: msg = "Fail to look-up PFNs in Phedex\n" + str(result) print msg continue if verbose > 1: print "PFN list:" for pfn in pfnlist: print pfn # finally return pfn and dst paths w/ file for further processing for item in pfnlist: ifile = item.split("/")[-1] if not dstfname else dstfname yield item, '%s/%s' % (dst, ifile)
def pfn_dst(lfn, dst, verbose=None): """ Look-up LFN in Phedex and return pfn dst for further processing """ dstfname = None pat = re.compile('^T[0-9]_[A-Z]+(_)[A-Z]+') if pat.match(dst): dst_split = dst.split(':') dst = dst_split[0] if len(dst_split) == 1: # copy to the node local_path = dst_split[1] for srm_path, lfn_match in resolve_srm_path(dst, verbose): lfn_pat = re.compile(lfn_match) if lfn_pat.match(lfn): srm_path = srm_path.replace('\?', '?').replace('$1', local_path) if verbose: print "Resolve %s into %s" % (dst, srm_path) dst = srm_path else: paths = [p for p in resolve_user_srm_path(dst, verbose=verbose)] dst = '%s/%s' % (paths[0], get_username()) check_permission(dst, verbose) else: if dst.find('file:///') == -1: dstfname = dst.split('/')[-1] if dstfname == '.': dstfname = None if dst[0] == '/': # absolute path if os.path.isdir(dst): ddir = dst dstfname = None else: ddir = '/'.join(dst.split('/')[:-1]) if not os.path.isdir(ddir): msg = 'Provided destination directory %s does not exists' % ddir raise Exception(msg) dst = 'file:///%s' % ddir else: ddir = '/'.join(dst.split('/')[:-1]).replace('$PWD', os.getcwd()) if os.path.isdir(ddir): dst = 'file:///%s' % os.path.join(os.getcwd(), ddir) else: dst = 'file:///%s' % os.getcwd() pfnlist = [] if os.path.isfile(lfn) or lfn.find('file:///') != -1: # local file pfn = lfn.replace('file:///', '') if pfn[0] != '/': pfn = 'file:///%s' % os.path.join(os.getcwd(), pfn) else: pfn = 'file:///%s' % pfn pfnlist = [pfn] else: if lfn.find(':') != -1: node, lfn = lfn.split(':') params = {'node':node, 'lfn':lfn, 'protocol':'srmv2'} method = 'lfn2pfn' else: params = {'se':'*', 'lfn':lfn} method = 'fileReplicas' json_dict = get_data(phedex_url(method), params) ddict = DotDict(json_dict) if verbose: print "Look-up LFN:" print lfn phedex = json_dict['phedex'] if phedex.has_key('mapping'): if not phedex['mapping']: msg = "LFN: %s\n" % lfn msg += 'No replicas found\n' msg += str(json_dict) raise Exception(msg) filelist = ddict.get('phedex.mapping.pfn') if not filelist: filelist = [] if isinstance(filelist, basestring): filelist = [filelist] for fname in filelist: pfnlist.append(fname) elif phedex.has_key('block') and not phedex['block']: msg = 'No replicas found in PhEDEx, will try to get original SE from DBS' print_warning(msg) sename = get_dbs_se(lfn) msg = 'Orignal LFN site %s' % sename print_info(msg) mgr = SiteDBManager() pfnlist = lfn2pfn(lfn, sename, mgr) filelist = ddict.get('phedex.block.file') if not filelist: filelist = [] for fname in filelist: for replica in fname['replica']: cmsname = replica['node'] se = replica['se'] if verbose: print "found LFN on node=%s, se=%s" % (cmsname, se) if cmsname.count('T0', 0, 2) == 1: continue # skip T0's # query Phedex for PFN params = {'protocol':'srmv2', 'lfn':lfn, 'node':cmsname} result = get_data(phedex_url('lfn2pfn'), params) try: for item in result['phedex']['mapping']: pfn = item['pfn'] if pfn not in pfnlist: pfnlist.append(pfn) except: msg = "Fail to look-up PFNs in Phedex\n" + str(result) print msg continue if verbose > 1: print "PFN list:" for pfn in pfnlist: print pfn # finally return pfn and dst paths w/ file for further processing for item in pfnlist: ifile = item.split("/")[-1] if not dstfname else dstfname yield item, '%s/%s' % (dst, ifile)