def get_pfns(lfn, verbose=None):
"""
Look-up LFN in Phedex and get corresponding list of PFNs
"""
pfnlist = []
selist = []
params = {'se':'*', 'lfn':lfn}
json_dict = get_data(phedex_url('fileReplicas'), params)
ddict = DotDict(json_dict)
if not json_dict['phedex']['block']:
return pfnlist, selist
for fname in ddict.get('phedex.block.file'):
for replica in fname['replica']:
cmsname = replica['node']
se = replica['se']
if se not in selist:
selist.append(se)
# query Phedex for PFN
params = {'protocol':'srmv2', 'lfn':lfn, 'node':cmsname}
result = get_data(phedex_url('lfn2pfn'), params)
try:
for item in result['phedex']['mapping']:
pfn = item['pfn']
if pfn not in pfnlist:
pfnlist.append(pfn)
except:
msg = "Fail to look-up PFNs in Phedex\n" + str(result)
print msg
continue
return pfnlist, selist
def get_pfns(lfn, verbose=None):
"""
Look-up LFN in Phedex and get corresponding list of PFNs
"""
pfnlist = []
selist = []
params = {'se': '*', 'lfn': lfn}
json_dict = get_data(phedex_url('fileReplicas'), params)
ddict = DotDict(json_dict)
if not json_dict['phedex']['block']:
return pfnlist, selist
for fname in ddict.get('phedex.block.file'):
for replica in fname['replica']:
cmsname = replica['node']
se = replica['se']
if se not in selist:
selist.append(se)
# query Phedex for PFN
params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname}
result = get_data(phedex_url('lfn2pfn'), params)
try:
for item in result['phedex']['mapping']:
pfn = item['pfn']
if pfn not in pfnlist:
pfnlist.append(pfn)
except:
msg = "Fail to look-up PFNs in Phedex\n" + str(result)
print msg
continue
return pfnlist, selist
def list_sites4file(self, **kwargs):
"""
Controller to get sites for given file
"""
url = phedex_url('fileReplicas')
params = {'lfn': kwargs['filename']}
return find_sites(url, params)
def list_sites4file(self, **kwargs):
"""
Controller to get sites for given file
"""
url = phedex_url('fileReplicas')
params = {'lfn': kwargs['filename']}
return find_sites(url, params)
def list_sites4dataset(self, **kwargs):
"""
Controller to get sites for given dataset
"""
url = phedex_url('fileReplicas')
params = {'dataset': kwargs['dataset']}
return find_sites(url, params)
def list_sites4dataset(self, **kwargs):
"""
Controller to get sites for given dataset
"""
url = phedex_url("fileReplicas")
params = {"dataset": kwargs["dataset"]}
return find_sites(url, params)
def list_sites4file(self, **kwargs):
"""
Controller to get sites for given file
"""
url = phedex_url("fileReplicas")
params = {"lfn": kwargs["filename"]}
return find_sites(url, params)
def list_sites4dataset(self, **kwargs):
"""
Controller to get sites for given dataset
"""
url = phedex_url('fileReplicas')
params = {'dataset': kwargs['dataset']}
return find_sites(url, params)
def list_block4site(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url("blockreplicasummary")
params = {"node": kwargs["sitename"]}
data = get_data(url, params)
plist = [Block(b) for b in data["phedex"]["block"]]
return plist
def list_block4site(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url('blockreplicasummary')
params = {'node': kwargs['sitename']}
data = get_data(url, params)
plist = [Block(b) for b in data['phedex']['block']]
return plist
def list_sites(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url('nodeusage')
params = {'node': kwargs['sitename']}
data = get_data(url, params)
plist = [Site(s) for s in data['phedex']['node']]
return plist
def resolve_srm_path(node, verbose=None):
"""
Use TFC phedex API to resolve srm path for given node
"""
params = {'node': node}
result = get_data(phedex_url('tfc'), params)
for row in result['phedex']['storage-mapping']['array']:
if row['protocol'] == 'srmv2' and row['element_name'] == 'lfn-to-pfn':
yield (row['result'], row['path-match'])
def list_sites(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url("nodeusage")
params = {"node": kwargs["sitename"]}
data = get_data(url, params)
plist = [Site(s) for s in data["phedex"]["node"]]
return plist
def list_block4site(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url('blockreplicasummary')
params = {'node': kwargs['sitename']}
data = get_data(url, params)
plist = [Block(b) for b in data['phedex']['block']]
return plist
def resolve_srm_path(node, verbose=None):
"""
Use TFC phedex API to resolve srm path for given node
"""
params = {'node':node}
result = get_data(phedex_url('tfc'), params)
for row in result['phedex']['storage-mapping']['array']:
if row['protocol'] == 'srmv2' and row['element_name'] == 'lfn-to-pfn':
yield (row['result'], row['path-match'])
def list_sites(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url('nodeusage')
params = {'node': kwargs['sitename']}
data = get_data(url, params)
plist = [Site(s) for s in data['phedex']['node']]
return plist
def resolve_user_srm_path(node, ldir='/store/user', verbose=None):
"""
Use TFC phedex API to resolve srm path for given node
"""
# change ldir if user supplied full path, e.g. /xrootdfs/cms/store/...
ldir = '/store/' + ldir.split('/store/')[-1]
params = {'node':node, 'lfn':ldir, 'protocol': 'srmv2'}
result = get_data(phedex_url('lfn2pfn'), params)
for row in result['phedex']['mapping']:
yield row['pfn']
def resolve_user_srm_path(node, ldir='/store/user', verbose=None):
"""
Use TFC phedex API to resolve srm path for given node
"""
# change ldir if user supplied full path, e.g. /xrootdfs/cms/store/...
ldir = '/store/' + ldir.split('/store/')[-1]
params = {'node': node, 'lfn': ldir, 'protocol': 'srmv2'}
result = get_data(phedex_url('lfn2pfn'), params)
for row in result['phedex']['mapping']:
yield row['pfn']
def site_info(dst, verbose=None):
"""list files at given destination"""
url = phedex_url('nodeusage')
params = {'node': dst}
data = get_data(url, params)
res = [Site(s) for s in data['phedex']['node']]
paths = [r for r in resolve_user_srm_path(dst)]
pdirs = [p.split('=')[-1] for p in paths]
for site in res:
site.assign('pfn_path', paths)
site.assign('default_path', pdirs)
return res
def site_info(dst, verbose=None):
"""list files at given destination"""
url = phedex_url("nodeusage")
params = {"node": dst}
data = get_data(url, params)
res = [Site(s) for s in data["phedex"]["node"]]
paths = [r for r in resolve_user_srm_path(dst)]
pdirs = [p.split("=")[-1] for p in paths]
for site in res:
site.assign("pfn_path", paths)
site.assign("default_path", pdirs)
return res
def site_info(dst, verbose=None):
"""list files at given destination"""
url = phedex_url('nodeusage')
params = {'node': dst}
data = get_data(url, params)
res = [Site(s) for s in data['phedex']['node']]
paths = [r for r in resolve_user_srm_path(dst)]
pdirs = [p.split('=')[-1] for p in paths]
for site in res:
site.assign('pfn_path', paths)
site.assign('default_path', pdirs)
return res
def nodes(select=True):
"""
Yield list of Phedex nodes, I only select T2 and below
"""
result = get_data(phedex_url('nodes'), {})
pat = re.compile('^T[0-1]_[A-Z]+(_)[A-Z]+')
lnodes = []
for row in result['phedex']['node']:
if select and pat.match(row['name']):
continue
msg = "%s, SE: %s, description %s/%s" \
% (row['name'], row['se'], row['technology'], row['kind'])
lnodes.append(msg)
lnodes.sort()
for row in lnodes:
print row
def nodes(select=True):
"""
Yield list of Phedex nodes, I only select T2 and below
"""
result = get_data(phedex_url('nodes'), {})
pat = re.compile('^T[0-1]_[A-Z]+(_)[A-Z]+')
lnodes = []
for row in result['phedex']['node']:
if select and pat.match(row['name']):
continue
msg = "%s, SE: %s, description %s/%s" \
% (row['name'], row['se'], row['technology'], row['kind'])
lnodes.append(msg)
lnodes.sort()
for row in lnodes:
print row
def lfn2pfn(lfn, sename, mgr=None):
"Find PFN for given LFN and SE"
pfnlist = []
if not mgr:
mgr = SiteDBManager()
cmsname = mgr.get_name(sename)
if cmsname:
params = {'protocol':'srmv2', 'lfn':lfn, 'node':cmsname}
result = get_data(phedex_url('lfn2pfn'), params)
try:
for item in result['phedex']['mapping']:
pfn = item['pfn']
if pfn not in pfnlist:
pfnlist.append(pfn)
except:
msg = "Fail to look-up PFNs in Phedex\n" + str(result)
print msg
return pfnlist
def lfn2pfn(lfn, sename, mgr=None):
"Find PFN for given LFN and SE"
pfnlist = []
if not mgr:
mgr = SiteDBManager()
cmsname = mgr.get_name(sename)
if cmsname:
params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname}
result = get_data(phedex_url('lfn2pfn'), params)
try:
for item in result['phedex']['mapping']:
pfn = item['pfn']
if pfn not in pfnlist:
pfnlist.append(pfn)
except:
msg = "Fail to look-up PFNs in Phedex\n" + str(result)
print msg
return pfnlist
def list_du4site(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url('blockReplicas')
site = kwargs['sitename']
params = {'node': site}
data = get_data(url, params)
nfiles = 0
nblocks = 0
size = 0
for row in data['phedex']['block']:
nblocks += 1
nfiles += int(row['files'])
for rep in row['replica']:
if rep['node'] == site:
size += long(rep['bytes'])
return dict(nblocks=nblocks, nfiles=nfiles, totalsize=size)
def list_du4site(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url('blockReplicas')
site = kwargs['sitename']
params = {'node': site}
data = get_data(url, params)
nfiles = 0
nblocks = 0
size = 0
for row in data['phedex']['block']:
nblocks += 1
nfiles += int(row['files'])
for rep in row['replica']:
if rep['node'] == site:
size += long(rep['bytes'])
return dict(nblocks=nblocks, nfiles=nfiles, totalsize=size)
def list_du4site(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url("blockReplicas")
site = kwargs["sitename"]
params = {"node": site}
data = get_data(url, params)
nfiles = 0
nblocks = 0
size = 0
for row in data["phedex"]["block"]:
nblocks += 1
nfiles += int(row["files"])
for rep in row["replica"]:
if rep["node"] == site:
size += long(rep["bytes"])
return dict(nblocks=nblocks, nfiles=nfiles, totalsize=size)
def pfn_dst(lfn, dst, verbose=None):
"""
Look-up LFN in Phedex and return pfn dst for further processing
"""
dstfname = None
pat = re.compile('^T[0-9]_[A-Z]+(_)[A-Z]+')
if pat.match(dst):
dst_split = dst.split(':')
dst = dst_split[0]
if len(dst_split) == 1: # copy to the node
local_path = dst_split[1]
for srm_path, lfn_match in resolve_srm_path(dst, verbose):
lfn_pat = re.compile(lfn_match)
if lfn_pat.match(lfn):
srm_path = srm_path.replace('\?',
'?').replace('$1', local_path)
if verbose:
print "Resolve %s into %s" % (dst, srm_path)
dst = srm_path
else:
paths = [p for p in resolve_user_srm_path(dst, verbose=verbose)]
dst = '%s/%s' % (paths[0], get_username())
check_permission(dst, verbose)
else:
if dst.find('file:///') == -1:
dstfname = dst.split('/')[-1]
if dstfname == '.':
dstfname = None
if dst[0] == '/': # absolute path
if os.path.isdir(dst):
ddir = dst
dstfname = None
else:
ddir = '/'.join(dst.split('/')[:-1])
if not os.path.isdir(ddir):
msg = 'Provided destination directory %s does not exists' % ddir
raise Exception(msg)
dst = 'file:///%s' % ddir
else:
ddir = '/'.join(dst.split('/')[:-1]).replace(
'$PWD', os.getcwd())
if os.path.isdir(ddir):
dst = 'file:///%s' % os.path.join(os.getcwd(), ddir)
else:
dst = 'file:///%s' % os.getcwd()
pfnlist = []
if os.path.isfile(lfn) or lfn.find('file:///') != -1: # local file
pfn = lfn.replace('file:///', '')
if pfn[0] != '/':
pfn = 'file:///%s' % os.path.join(os.getcwd(), pfn)
else:
pfn = 'file:///%s' % pfn
pfnlist = [pfn]
else:
if lfn.find(':') != -1:
node, lfn = lfn.split(':')
params = {'node': node, 'lfn': lfn, 'protocol': 'srmv2'}
method = 'lfn2pfn'
else:
params = {'se': '*', 'lfn': lfn}
method = 'fileReplicas'
json_dict = get_data(phedex_url(method), params)
ddict = DotDict(json_dict)
if verbose:
print "Look-up LFN:"
print lfn
phedex = json_dict['phedex']
if phedex.has_key('mapping'):
if not phedex['mapping']:
msg = "LFN: %s\n" % lfn
msg += 'No replicas found\n'
msg += str(json_dict)
raise Exception(msg)
filelist = ddict.get('phedex.mapping.pfn')
if not filelist:
filelist = []
if isinstance(filelist, basestring):
filelist = [filelist]
for fname in filelist:
pfnlist.append(fname)
elif phedex.has_key('block') and not phedex['block']:
msg = 'No replicas found in PhEDEx, will try to get original SE from DBS'
print_warning(msg)
sename = get_dbs_se(lfn)
msg = 'Orignal LFN site %s' % sename
print_info(msg)
mgr = SiteDBManager()
pfnlist = lfn2pfn(lfn, sename, mgr)
filelist = ddict.get('phedex.block.file')
if not filelist:
filelist = []
for fname in filelist:
for replica in fname['replica']:
cmsname = replica['node']
se = replica['se']
if verbose:
print "found LFN on node=%s, se=%s" % (cmsname, se)
if cmsname.count('T0', 0, 2) == 1:
continue # skip T0's
# query Phedex for PFN
params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname}
result = get_data(phedex_url('lfn2pfn'), params)
try:
for item in result['phedex']['mapping']:
pfn = item['pfn']
if pfn not in pfnlist:
pfnlist.append(pfn)
except:
msg = "Fail to look-up PFNs in Phedex\n" + str(result)
print msg
continue
if verbose > 1:
print "PFN list:"
for pfn in pfnlist:
print pfn
# finally return pfn and dst paths w/ file for further processing
for item in pfnlist:
ifile = item.split("/")[-1] if not dstfname else dstfname
yield item, '%s/%s' % (dst, ifile)
def pfn_dst(lfn, dst, verbose=None):
"""
Look-up LFN in Phedex and return pfn dst for further processing
"""
dstfname = None
pat = re.compile('^T[0-9]_[A-Z]+(_)[A-Z]+')
if pat.match(dst):
dst_split = dst.split(':')
dst = dst_split[0]
if len(dst_split) == 1: # copy to the node
local_path = dst_split[1]
for srm_path, lfn_match in resolve_srm_path(dst, verbose):
lfn_pat = re.compile(lfn_match)
if lfn_pat.match(lfn):
srm_path = srm_path.replace('\?', '?').replace('$1', local_path)
if verbose:
print "Resolve %s into %s" % (dst, srm_path)
dst = srm_path
else:
paths = [p for p in resolve_user_srm_path(dst, verbose=verbose)]
dst = '%s/%s' % (paths[0], get_username())
check_permission(dst, verbose)
else:
if dst.find('file:///') == -1:
dstfname = dst.split('/')[-1]
if dstfname == '.':
dstfname = None
if dst[0] == '/': # absolute path
if os.path.isdir(dst):
ddir = dst
dstfname = None
else:
ddir = '/'.join(dst.split('/')[:-1])
if not os.path.isdir(ddir):
msg = 'Provided destination directory %s does not exists' % ddir
raise Exception(msg)
dst = 'file:///%s' % ddir
else:
ddir = '/'.join(dst.split('/')[:-1]).replace('$PWD', os.getcwd())
if os.path.isdir(ddir):
dst = 'file:///%s' % os.path.join(os.getcwd(), ddir)
else:
dst = 'file:///%s' % os.getcwd()
pfnlist = []
if os.path.isfile(lfn) or lfn.find('file:///') != -1: # local file
pfn = lfn.replace('file:///', '')
if pfn[0] != '/':
pfn = 'file:///%s' % os.path.join(os.getcwd(), pfn)
else:
pfn = 'file:///%s' % pfn
pfnlist = [pfn]
else:
if lfn.find(':') != -1:
node, lfn = lfn.split(':')
params = {'node':node, 'lfn':lfn, 'protocol':'srmv2'}
method = 'lfn2pfn'
else:
params = {'se':'*', 'lfn':lfn}
method = 'fileReplicas'
json_dict = get_data(phedex_url(method), params)
ddict = DotDict(json_dict)
if verbose:
print "Look-up LFN:"
print lfn
phedex = json_dict['phedex']
if phedex.has_key('mapping'):
if not phedex['mapping']:
msg = "LFN: %s\n" % lfn
msg += 'No replicas found\n'
msg += str(json_dict)
raise Exception(msg)
filelist = ddict.get('phedex.mapping.pfn')
if not filelist:
filelist = []
if isinstance(filelist, basestring):
filelist = [filelist]
for fname in filelist:
pfnlist.append(fname)
elif phedex.has_key('block') and not phedex['block']:
msg = 'No replicas found in PhEDEx, will try to get original SE from DBS'
print_warning(msg)
sename = get_dbs_se(lfn)
msg = 'Orignal LFN site %s' % sename
print_info(msg)
mgr = SiteDBManager()
pfnlist = lfn2pfn(lfn, sename, mgr)
filelist = ddict.get('phedex.block.file')
if not filelist:
filelist = []
for fname in filelist:
for replica in fname['replica']:
cmsname = replica['node']
se = replica['se']
if verbose:
print "found LFN on node=%s, se=%s" % (cmsname, se)
if cmsname.count('T0', 0, 2) == 1:
continue # skip T0's
# query Phedex for PFN
params = {'protocol':'srmv2', 'lfn':lfn, 'node':cmsname}
result = get_data(phedex_url('lfn2pfn'), params)
try:
for item in result['phedex']['mapping']:
pfn = item['pfn']
if pfn not in pfnlist:
pfnlist.append(pfn)
except:
msg = "Fail to look-up PFNs in Phedex\n" + str(result)
print msg
continue
if verbose > 1:
print "PFN list:"
for pfn in pfnlist:
print pfn
# finally return pfn and dst paths w/ file for further processing
for item in pfnlist:
ifile = item.split("/")[-1] if not dstfname else dstfname
yield item, '%s/%s' % (dst, ifile)