def get_pfns(lfn, verbose=None):
"""
Look-up LFN in Phedex and get corresponding list of PFNs
"""
pfnlist = []
selist = []
params = {'se':'*', 'lfn':lfn}
json_dict = get_data(phedex_url('fileReplicas'), params)
ddict = DotDict(json_dict)
if not json_dict['phedex']['block']:
return pfnlist, selist
for fname in ddict.get('phedex.block.file'):
for replica in fname['replica']:
cmsname = replica['node']
se = replica['se']
if se not in selist:
selist.append(se)
# query Phedex for PFN
params = {'protocol':'srmv2', 'lfn':lfn, 'node':cmsname}
result = get_data(phedex_url('lfn2pfn'), params)
try:
for item in result['phedex']['mapping']:
pfn = item['pfn']
if pfn not in pfnlist:
pfnlist.append(pfn)
except:
msg = "Fail to look-up PFNs in Phedex\n" + str(result)
print msg
continue
return pfnlist, selist
def run_lumi_dict(arg, verbose=None):
"Return run-lumi info for given argument (dataset, block, file, run)"
try:
data = json.loads(arg)
except:
if isinstance(arg, basestring) and arg.find("{") != -1:
data = eval(arg, {"__builtins__": None}, {})
else:
data = arg # assume it is dataset/file/block/run
url = dbs_url()
run_lumi = {}
if isinstance(data, dict): # we got run-lumi dict
for run, lumis in data.items():
run_lumi[int(run)] = lumis
else:
if url.find("cmsdbsprod") != -1: # DBS2
run_lumi = dbs2.run_lumi(str(data), verbose)
else:
if pat_dataset.match(data):
params = {"dataset": data}
result = get_data(dbs_url("files"), params, verbose)
for row in result:
params = {"logical_file_name": row["logical_file_name"]}
run_lumi = parse_runlumis(get_data(dbs_url("filelumis"), params, verbose))
elif pat_block.match(data):
params = {"block_name": data}
run_lumi = parse_runlumis(get_data(dbs_url("filelumis"), params, verbose))
elif pat_lfn.match(data):
params = {"logical_file_name": data}
run_lumi = parse_runlumis(get_data(dbs_url("filelumis"), params, verbose))
elif pat_run.match(data):
params = {"run_num": data}
run_lumi = parse_runlumis(get_data(dbs_url("filelumis"), params, verbose))
return run_lumi
def get_pfns(lfn, verbose=None):
"""
Look-up LFN in Phedex and get corresponding list of PFNs
"""
pfnlist = []
selist = []
params = {'se': '*', 'lfn': lfn}
json_dict = get_data(phedex_url('fileReplicas'), params)
ddict = DotDict(json_dict)
if not json_dict['phedex']['block']:
return pfnlist, selist
for fname in ddict.get('phedex.block.file'):
for replica in fname['replica']:
cmsname = replica['node']
se = replica['se']
if se not in selist:
selist.append(se)
# query Phedex for PFN
params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname}
result = get_data(phedex_url('lfn2pfn'), params)
try:
for item in result['phedex']['mapping']:
pfn = item['pfn']
if pfn not in pfnlist:
pfnlist.append(pfn)
except:
msg = "Fail to look-up PFNs in Phedex\n" + str(result)
print msg
continue
return pfnlist, selist
def releases(rel_name=None, rfilter=None):
"Return information about CMS releases"
if rel_name:
if not pat_release.match(rel_name):
msg = 'Wrong CMSSW release name'
raise ValueError(msg)
args = {'release_name': rel_name}
else:
args = {}
url = tc_url('getReleasesInformation')
rel_info = get_data(url, args)
columns = rel_info['columns']
pat = re.compile('CMSSW_[1-9]_[0-9]_X\.*')
for key, val in rel_info['data'].iteritems():
if rfilter == 'list':
if pat.match(key) or not key.find('CMSSW') != -1 or \
key.find('EXPERIMENTAL') != -1 or \
key.find('CLANG') != -1 or \
key.find('_X_') != -1 or \
key.find('FORTIFIED') != -1:
continue
row = {}
pairs = zip(columns['release_name'], val)
for kkk, vvv in pairs:
if isinstance(kkk, basestring):
row[kkk] = vvv
elif isinstance(kkk, list):
for item in vvv:
row.setdefault('architectures',
[]).append(dict(zip(kkk, item)))
row['release_name'] = key
for item in row['architectures']:
if match_platform(item['architecture_name']):
yield row
def list_user(self, **kwargs):
"""
Controller to get site info
"""
url = sitedb_url("people")
params = {}
cert = os.path.join(os.environ["HOME"], ".globus/usercert.pem")
ckey = os.path.join(os.environ["HOME"], ".globus/userkey.pem")
data = get_data(url, params)
qname = kwargs["username"].lower()
users = []
for row in sitedb_parser(data):
username = row.get("username", "").lower()
forename = row.get("forename", "").lower()
surname = row.get("surname", "").lower()
email = row.get("email", "").lower()
if (
username
and qname.find(username) != -1
or forename
and qname.find(forename) != -1
or surname
and qname.find(surname) != -1
or email
and qname.find(email) != -1
):
users.append(row)
return [User(u) for u in users]
def releases(rel_name=None, rfilter=None):
"Return information about CMS releases"
if rel_name:
if not pat_release.match(rel_name):
msg = 'Wrong CMSSW release name'
raise ValueError(msg)
args = {'release_name': rel_name}
else:
args = {}
url = tc_url('getReleasesInformation')
rel_info = get_data(url, args)
columns = rel_info['columns']
pat = re.compile('CMSSW_[1-9]_[0-9]_X\.*')
for key, val in rel_info['data'].iteritems():
if rfilter == 'list':
if pat.match(key) or not key.find('CMSSW') != -1 or \
key.find('EXPERIMENTAL') != -1 or \
key.find('CLANG') != -1 or \
key.find('_X_') != -1 or \
key.find('FORTIFIED') != -1:
continue
row = {}
pairs = zip(columns['release_name'], val)
for kkk, vvv in pairs:
if isinstance(kkk, basestring):
row[kkk] = vvv
elif isinstance(kkk, list):
for item in vvv:
row.setdefault('architectures', []).append(dict(zip(kkk, item)))
row['release_name'] = key
for item in row['architectures']:
if match_platform(item['architecture_name']):
yield row
def resolve_srm_path(node, verbose=None):
"""
Use TFC phedex API to resolve srm path for given node
"""
params = {'node':node}
result = get_data(phedex_url('tfc'), params)
for row in result['phedex']['storage-mapping']['array']:
if row['protocol'] == 'srmv2' and row['element_name'] == 'lfn-to-pfn':
yield (row['result'], row['path-match'])
def list_block4site(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url('blockreplicasummary')
params = {'node': kwargs['sitename']}
data = get_data(url, params)
plist = [Block(b) for b in data['phedex']['block']]
return plist
def resolve_srm_path(node, verbose=None):
"""
Use TFC phedex API to resolve srm path for given node
"""
params = {'node': node}
result = get_data(phedex_url('tfc'), params)
for row in result['phedex']['storage-mapping']['array']:
if row['protocol'] == 'srmv2' and row['element_name'] == 'lfn-to-pfn':
yield (row['result'], row['path-match'])
def list_block4site(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url('blockreplicasummary')
params = {'node': kwargs['sitename']}
data = get_data(url, params)
plist = [Block(b) for b in data['phedex']['block']]
return plist
def list_sites(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url("nodeusage")
params = {"node": kwargs["sitename"]}
data = get_data(url, params)
plist = [Site(s) for s in data["phedex"]["node"]]
return plist
def list_sites(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url('nodeusage')
params = {'node': kwargs['sitename']}
data = get_data(url, params)
plist = [Site(s) for s in data['phedex']['node']]
return plist
def list_block4site(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url("blockreplicasummary")
params = {"node": kwargs["sitename"]}
data = get_data(url, params)
plist = [Block(b) for b in data["phedex"]["block"]]
return plist
def list_sites(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url('nodeusage')
params = {'node': kwargs['sitename']}
data = get_data(url, params)
plist = [Site(s) for s in data['phedex']['node']]
return plist
def list_runs(self, **kwargs):
"""
Controller to get runs
"""
url = conddb_url('getLumi')
run = kwargs.get('run')
params = {'Runs': run, 'lumiType': 'delivered'}
data = get_data(url, params)
plist = [Run(d) for d in data]
return plist
def resolve_user_srm_path(node, ldir='/store/user', verbose=None):
"""
Use TFC phedex API to resolve srm path for given node
"""
# change ldir if user supplied full path, e.g. /xrootdfs/cms/store/...
ldir = '/store/' + ldir.split('/store/')[-1]
params = {'node': node, 'lfn': ldir, 'protocol': 'srmv2'}
result = get_data(phedex_url('lfn2pfn'), params)
for row in result['phedex']['mapping']:
yield row['pfn']
def list_runs(self, **kwargs):
"""
Controller to get runs
"""
url = conddb_url("getLumi")
run = kwargs.get("run")
params = {"Runs": run, "lumiType": "delivered"}
data = get_data(url, params)
plist = [Run(d) for d in data]
return plist
def list_runs(self, **kwargs):
"""
Controller to get runs
"""
url = conddb_url('getLumi')
run = kwargs.get('run')
params = {'Runs':run, 'lumiType':'delivered'}
data = get_data(url, params)
plist = [Run(d) for d in data]
return plist
def resolve_user_srm_path(node, ldir='/store/user', verbose=None):
"""
Use TFC phedex API to resolve srm path for given node
"""
# change ldir if user supplied full path, e.g. /xrootdfs/cms/store/...
ldir = '/store/' + ldir.split('/store/')[-1]
params = {'node':node, 'lfn':ldir, 'protocol': 'srmv2'}
result = get_data(phedex_url('lfn2pfn'), params)
for row in result['phedex']['mapping']:
yield row['pfn']
def get_release_arch(rel):
"Return architecture for given CMSSW release"
args = {'release': rel}
releases = get_data(tc_url('py_getReleaseArchitectures'), args)
output = []
for item in releases:
rel_arch = item[0]
status = item[1]
if check_os(rel_arch):
output.append((rel_arch, status))
return output
def get_release_arch(rel):
"Return architecture for given CMSSW release"
args = {'release': rel}
releases = get_data(tc_url('py_getReleaseArchitectures'), args)
output = []
for item in releases:
rel_arch = item[0]
status = item[1]
if check_os(rel_arch):
output.append((rel_arch, status))
return output
def site_info(dst, verbose=None):
"""list files at given destination"""
url = phedex_url('nodeusage')
params = {'node': dst}
data = get_data(url, params)
res = [Site(s) for s in data['phedex']['node']]
paths = [r for r in resolve_user_srm_path(dst)]
pdirs = [p.split('=')[-1] for p in paths]
for site in res:
site.assign('pfn_path', paths)
site.assign('default_path', pdirs)
return res
def block_info(block, verbose=None):
"""Return block info"""
url = dbs_url()
if url.find('cmsdbsprod') != -1: # DBS2
return dbs2.block_info(block, verbose)
params = {'block_name': block, 'detail':'True'}
result = get_data(dbs_url('blocks'), params, verbose)
res = [Block(r) for r in result][0]
if len(res) != 1:
msg = 'The %s block yield %s results' % (block, len(res))
raise Exception(msg)
return res[0]
def dataset_info(dataset, verbose=None):
"""Return dataset info"""
url = dbs_url()
if url.find('cmsdbsprod') != -1: # DBS2
return dbs2.dataset_info(dataset, verbose)
params = {'dataset': dataset, 'detail':'True'}
result = get_data(dbs_url('datasets'), params, verbose)
res = [Dataset(r) for r in result]
if len(res) != 1:
msg = 'The %s dataset yield %s results' % (dataset, len(res))
raise Exception(msg)
return res[0]
def dataset_info(dataset, verbose=None):
"""Return dataset info"""
url = dbs_url()
if url.find('cmsdbsprod') != -1: # DBS2
return dbs2.dataset_info(dataset, verbose)
params = {'dataset': dataset, 'detail': 'True'}
result = get_data(dbs_url('datasets'), params, verbose)
res = [Dataset(r) for r in result]
if len(res) != 1:
msg = 'The %s dataset yield %s results' % (dataset, len(res))
raise Exception(msg)
return res[0]
def dataset_info(dataset, verbose=None):
"""Return dataset info"""
url = dbs_url()
if url.find("cmsdbsprod") != -1: # DBS2
return dbs2.dataset_info(dataset, verbose)
params = {"dataset": dataset, "detail": "True"}
result = get_data(dbs_url("datasets"), params, verbose)
res = [Dataset(r) for r in result]
if len(res) != 1:
msg = "The %s dataset yield %s results" % (dataset, len(res))
raise Exception(msg)
return res[0]
def site_info(dst, verbose=None):
"""list files at given destination"""
url = phedex_url('nodeusage')
params = {'node': dst}
data = get_data(url, params)
res = [Site(s) for s in data['phedex']['node']]
paths = [r for r in resolve_user_srm_path(dst)]
pdirs = [p.split('=')[-1] for p in paths]
for site in res:
site.assign('pfn_path', paths)
site.assign('default_path', pdirs)
return res
def block_info(block, verbose=None):
"""Return block info"""
url = dbs_url()
if url.find('cmsdbsprod') != -1: # DBS2
return dbs2.block_info(block, verbose)
params = {'block_name': block, 'detail': 'True'}
result = get_data(dbs_url('blocks'), params, verbose)
res = [Block(r) for r in result][0]
if len(res) != 1:
msg = 'The %s block yield %s results' % (block, len(res))
raise Exception(msg)
return res[0]
def block_info(block, verbose=None):
"""Return block info"""
url = dbs_url()
if url.find("cmsdbsprod") != -1: # DBS2
return dbs2.block_info(block, verbose)
params = {"block_name": block, "detail": "True"}
result = get_data(dbs_url("blocks"), params, verbose)
res = [Block(r) for r in result][0]
if len(res) != 1:
msg = "The %s block yield %s results" % (block, len(res))
raise Exception(msg)
return res[0]
def site_info(dst, verbose=None):
"""list files at given destination"""
url = phedex_url("nodeusage")
params = {"node": dst}
data = get_data(url, params)
res = [Site(s) for s in data["phedex"]["node"]]
paths = [r for r in resolve_user_srm_path(dst)]
pdirs = [p.split("=")[-1] for p in paths]
for site in res:
site.assign("pfn_path", paths)
site.assign("default_path", pdirs)
return res
def run_lumi_dict(arg, verbose=None):
"Return run-lumi info for given argument (dataset, block, file, run)"
try:
data = json.loads(arg)
except:
if isinstance(arg, basestring) and arg.find("{") != -1:
data = eval(arg, {"__builtins__": None}, {})
else:
data = arg # assume it is dataset/file/block/run
url = dbs_url()
run_lumi = {}
if isinstance(data, dict): # we got run-lumi dict
for run, lumis in data.items():
run_lumi[int(run)] = lumis
else:
if url.find('cmsdbsprod') != -1: # DBS2
run_lumi = dbs2.run_lumi(str(data), verbose)
else:
if pat_dataset.match(data):
params = {'dataset': data}
result = get_data(dbs_url('files'), params, verbose)
for row in result:
params = {'logical_file_name': row['logical_file_name']}
run_lumi = parse_runlumis(
get_data(dbs_url('filelumis'), params, verbose))
elif pat_block.match(data):
params = {'block_name': data}
run_lumi = parse_runlumis(
get_data(dbs_url('filelumis'), params, verbose))
elif pat_lfn.match(data):
params = {'logical_file_name': data}
run_lumi = parse_runlumis(
get_data(dbs_url('filelumis'), params, verbose))
elif pat_run.match(data):
params = {'run_num': data}
run_lumi = parse_runlumis(
get_data(dbs_url('filelumis'), params, verbose))
return run_lumi
def run_info(run, verbose=None):
"""Return run info"""
url = conddb_url('getLumi')
params = {'Runs': run, 'lumiType': 'delivered'}
data = get_data(url, params)
runinfo = [r for r in runsum(run)]
runlumi = [r for r in data]
plist = []
for run in runinfo:
for rec in runlumi:
if run['run'] == rec['Run']:
run.update(rec)
plist.append(Run(run))
return plist
def list_datasets(self, **kwargs):
"""
Controller to get DBS datasets
"""
url = dbs_url()
if url.find('cmsdbsprod') != -1: # DBS2
return dbs2.list_datasets(kwargs)
url = dbs_url('datasets')
params = {'dataset': kwargs['dataset']}
if kwargs['dataset'][0] == '*':
kwargs['dataset'] = '/' + kwargs['dataset']
data = get_data(url, kwargs)
plist = [Dataset(d) for d in data]
return plist
def run_info(run, verbose=None):
"""Return run info"""
url = conddb_url("getLumi")
params = {"Runs": run, "lumiType": "delivered"}
data = get_data(url, params)
runinfo = [r for r in runsum(run)]
runlumi = [r for r in data]
plist = []
for run in runinfo:
for rec in runlumi:
if run["run"] == rec["Run"]:
run.update(rec)
plist.append(Run(run))
return plist
def list_datasets(self, **kwargs):
"""
Controller to get DBS datasets
"""
url = dbs_url()
if url.find("cmsdbsprod") != -1: # DBS2
return dbs2.list_datasets(kwargs)
url = dbs_url("datasets")
params = {"dataset": kwargs["dataset"]}
if kwargs["dataset"][0] == "*":
kwargs["dataset"] = "/" + kwargs["dataset"]
data = get_data(url, kwargs)
plist = [Dataset(d) for d in data]
return plist
def run_info(run, verbose=None):
"""Return run info"""
url = conddb_url('getLumi')
params = {'Runs':run, 'lumiType':'delivered'}
data = get_data(url, params)
runinfo = [r for r in runsum(run)]
runlumi = [r for r in data]
plist = []
for run in runinfo:
for rec in runlumi:
if run['run'] == rec['Run']:
run.update(rec)
plist.append(Run(run))
return plist
def lfns(run=None, dataset=None):
"""
Get lfns list for provided run/dataset
"""
url = dbs_url('files') # DBS3
params = {'detail':'True'}
if run:
args['minrun'] = run
args['maxrun'] = run
if dataset:
args['dataset'] = dataset
params.update(args)
json_dict = get_data(url, params)
for row in json_dict:
yield row['logical_file_name']
def get_tickets(ticket=None, **kwds):
"""
Retrieve information about cmssh tickets
"""
res = []
url = 'https://api.github.com'
path = 'repos/vkuznet/cmssh/issues'
if ticket:
path += '/%s' % ticket
url += path
res = get_data(url, kwds)
if isinstance(res, dict):
res = [res]
res = [Ticket(t) for t in res]
return res
def lfns(run=None, dataset=None):
"""
Get lfns list for provided run/dataset
"""
url = dbs_url('files') # DBS3
params = {'detail': 'True'}
if run:
args['minrun'] = run
args['maxrun'] = run
if dataset:
args['dataset'] = dataset
params.update(args)
json_dict = get_data(url, params)
for row in json_dict:
yield row['logical_file_name']
def get_tickets(ticket=None, **kwds):
"""
Retrieve information about cmssh tickets
"""
res = []
url = 'https://api.github.com'
path = 'repos/vkuznet/cmssh/issues'
if ticket:
path += '/%s' % ticket
url += path
res = get_data(url, kwds)
if isinstance(res, dict):
res = [res]
res = [Ticket(t) for t in res]
return res
def list_files(self, **kwargs):
"""
Controller to get files
"""
url = dbs_url()
run = kwargs.get('run', None)
dataset = kwargs.get('dataset')
if url.find('cmsdbsprod') != -1: # DBS2
return dbs2.list_files(dataset, run)
url = dbs_url('files')
params = {'dataset': dataset, 'detail': 'True'}
if run:
params.update({'run_num': run})
data = get_data(url, params)
plist = [File(f) for f in data]
return plist
def nodes(select=True):
"""
Yield list of Phedex nodes, I only select T2 and below
"""
result = get_data(phedex_url('nodes'), {})
pat = re.compile('^T[0-1]_[A-Z]+(_)[A-Z]+')
lnodes = []
for row in result['phedex']['node']:
if select and pat.match(row['name']):
continue
msg = "%s, SE: %s, description %s/%s" \
% (row['name'], row['se'], row['technology'], row['kind'])
lnodes.append(msg)
lnodes.sort()
for row in lnodes:
print row
def nodes(select=True):
"""
Yield list of Phedex nodes, I only select T2 and below
"""
result = get_data(phedex_url('nodes'), {})
pat = re.compile('^T[0-1]_[A-Z]+(_)[A-Z]+')
lnodes = []
for row in result['phedex']['node']:
if select and pat.match(row['name']):
continue
msg = "%s, SE: %s, description %s/%s" \
% (row['name'], row['se'], row['technology'], row['kind'])
lnodes.append(msg)
lnodes.sort()
for row in lnodes:
print row
def list_files(self, **kwargs):
"""
Controller to get files
"""
url = dbs_url()
run = kwargs.get('run', None)
dataset = kwargs.get('dataset')
if url.find('cmsdbsprod') != -1: # DBS2
return dbs2.list_files(dataset, run)
url = dbs_url('files')
params = {'dataset': dataset, 'detail': 'True'}
if run:
params.update({'run_num': run})
data = get_data(url, params)
plist = [File(f) for f in data]
return plist
def list_files(self, **kwargs):
"""
Controller to get files
"""
url = dbs_url()
run = kwargs.get("run", None)
dataset = kwargs.get("dataset")
if url.find("cmsdbsprod") != -1: # DBS2
return dbs2.list_files(dataset, run)
url = dbs_url("files")
params = {"dataset": dataset, "detail": "True"}
if run:
params.update({"run_num": run})
data = get_data(url, params)
plist = [File(f) for f in data]
return plist
def find_sites(url, params):
"""Find sites"""
data = get_data(url, params)
sites = {}
for files in data["phedex"]["block"]:
for fdict in files["file"]:
replicas = fdict["replica"]
for replica in replicas:
node = replica.get("node", None)
se = replica.get("se", None)
if sites.has_key(node):
if se not in sites[node]:
sites[node] += [se]
else:
sites[node] = [se]
for key, val in sites.iteritems():
yield Site({"node": key, "se": val})
def find_sites(url, params):
"""Find sites"""
data = get_data(url, params)
sites = {}
for files in data['phedex']['block']:
for fdict in files['file']:
replicas = fdict['replica']
for replica in replicas:
node = replica.get('node', None)
se = replica.get('se', None)
if sites.has_key(node):
if se not in sites[node]:
sites[node] += [se]
else:
sites[node] = [se]
for key, val in sites.iteritems():
yield Site({'node': key, 'se': val})
def list_datasets(self, **kwargs):
"""
Controller to get DBS datasets
"""
url = dbs_url()
if url.find('cmsdbsprod') != -1: # DBS2
return dbs2.list_datasets(kwargs)
url = dbs_url('datasets')
params = {'dataset':kwargs.get('dataset', '*')}
if kwargs['dataset'][0] == '*':
kwargs['dataset'] = '/' + kwargs['dataset']
if kwargs.has_key('file'):
kwargs['logical_file_name'] = kwargs['file']
del kwargs['file']
data = get_data(url, kwargs)
plist = [Dataset(d) for d in data]
return plist
def find_sites(url, params):
"""Find sites"""
data = get_data(url, params)
sites = {}
for files in data['phedex']['block']:
for fdict in files['file']:
replicas = fdict['replica']
for replica in replicas:
node = replica.get('node', None)
se = replica.get('se', None)
if sites.has_key(node):
if se not in sites[node]:
sites[node] += [se]
else:
sites[node] = [se]
for key, val in sites.iteritems():
yield Site({'node': key, 'se': val})
def list_du4site(self, **kwargs):
"""
Controller to get site info
"""
url = phedex_url('blockReplicas')
site = kwargs['sitename']
params = {'node': site}
data = get_data(url, params)
nfiles = 0
nblocks = 0
size = 0
for row in data['phedex']['block']:
nblocks += 1
nfiles += int(row['files'])
for rep in row['replica']:
if rep['node'] == site:
size += long(rep['bytes'])
return dict(nblocks=nblocks, nfiles=nfiles, totalsize=size)
def lfn2pfn(lfn, sename, mgr=None):
"Find PFN for given LFN and SE"
pfnlist = []
if not mgr:
mgr = SiteDBManager()
cmsname = mgr.get_name(sename)
if cmsname:
params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname}
result = get_data(phedex_url('lfn2pfn'), params)
try:
for item in result['phedex']['mapping']:
pfn = item['pfn']
if pfn not in pfnlist:
pfnlist.append(pfn)
except:
msg = "Fail to look-up PFNs in Phedex\n" + str(result)
print msg
return pfnlist
def file_info(lfn, verbose=None):
"""Return file info"""
url = dbs_url()
if url.find('cmsdbsprod') != -1: # DBS2
return dbs2.file_info(lfn, verbose)
params = {'logical_file_name': lfn, 'detail': 'True'}
result = get_data(dbs_url('files'), params, verbose)
res = [File(r) for r in result]
if len(res) != 1:
msg = 'The %s LFN yield %s results' % (lfn, len(res))
raise Exception(msg)
lfnobj = res[0]
try:
pfnlist, selist = get_pfns(lfn, verbose)
lfnobj.assign('pfn', pfnlist)
lfnobj.assign('se', selist)
except:
lfnobj.assign('pfn', [])
lfnobj.assign('se', [])
return lfnobj
def list_user(self, **kwargs):
"""
Controller to get site info
"""
url = sitedb_url('people')
params = {}
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
ckey = os.path.join(os.environ['HOME'], '.globus/userkey.pem')
data = get_data(url, params)
qname = kwargs['username'].lower()
users = []
for row in sitedb_parser(data):
username = row.get('username', '').lower()
forename = row.get('forename', '').lower()
surname = row.get('surname', '').lower()
email = row.get('email', '').lower()
if username and qname.find(username) != -1 or \
forename and qname.find(forename) != -1 or \
surname and qname.find(surname) != -1 or \
email and qname.find(email) != -1:
users.append(row)
return [User(u) for u in users]
def list_jobs(self, **kwargs):
"Controller for jobs info"
url = dashboard_url(
'jobsummary-plot-or-table2') # JSON API (number 2 :)
params = {
'user': kwargs.get('user', ''),
'site': kwargs.get('site', ''),
'ce': '',
'submissiontool': '',
'dataset': kwargs.get('dataset', ''),
'application': '',
'rb': '',
'activity': '',
'grid': '',
'date1': '',
'date2': '',
'jobtype': '',
'tier': '',
'check': 'submitted',
}
data = get_data(url, params)
plist = [Job(r) for r in data['summaries']]
return plist
def pfn_dst(lfn, dst, verbose=None):
"""
Look-up LFN in Phedex and return pfn dst for further processing
"""
dstfname = None
pat = re.compile('^T[0-9]_[A-Z]+(_)[A-Z]+')
if pat.match(dst):
dst_split = dst.split(':')
dst = dst_split[0]
if len(dst_split) == 1: # copy to the node
local_path = dst_split[1]
for srm_path, lfn_match in resolve_srm_path(dst, verbose):
lfn_pat = re.compile(lfn_match)
if lfn_pat.match(lfn):
srm_path = srm_path.replace('\?',
'?').replace('$1', local_path)
if verbose:
print "Resolve %s into %s" % (dst, srm_path)
dst = srm_path
else:
paths = [p for p in resolve_user_srm_path(dst, verbose=verbose)]
dst = '%s/%s' % (paths[0], get_username())
check_permission(dst, verbose)
else:
if dst.find('file:///') == -1:
dstfname = dst.split('/')[-1]
if dstfname == '.':
dstfname = None
if dst[0] == '/': # absolute path
if os.path.isdir(dst):
ddir = dst
dstfname = None
else:
ddir = '/'.join(dst.split('/')[:-1])
if not os.path.isdir(ddir):
msg = 'Provided destination directory %s does not exists' % ddir
raise Exception(msg)
dst = 'file:///%s' % ddir
else:
ddir = '/'.join(dst.split('/')[:-1]).replace(
'$PWD', os.getcwd())
if os.path.isdir(ddir):
dst = 'file:///%s' % os.path.join(os.getcwd(), ddir)
else:
dst = 'file:///%s' % os.getcwd()
pfnlist = []
if os.path.isfile(lfn) or lfn.find('file:///') != -1: # local file
pfn = lfn.replace('file:///', '')
if pfn[0] != '/':
pfn = 'file:///%s' % os.path.join(os.getcwd(), pfn)
else:
pfn = 'file:///%s' % pfn
pfnlist = [pfn]
else:
if lfn.find(':') != -1:
node, lfn = lfn.split(':')
params = {'node': node, 'lfn': lfn, 'protocol': 'srmv2'}
method = 'lfn2pfn'
else:
params = {'se': '*', 'lfn': lfn}
method = 'fileReplicas'
json_dict = get_data(phedex_url(method), params)
ddict = DotDict(json_dict)
if verbose:
print "Look-up LFN:"
print lfn
phedex = json_dict['phedex']
if phedex.has_key('mapping'):
if not phedex['mapping']:
msg = "LFN: %s\n" % lfn
msg += 'No replicas found\n'
msg += str(json_dict)
raise Exception(msg)
filelist = ddict.get('phedex.mapping.pfn')
if not filelist:
filelist = []
if isinstance(filelist, basestring):
filelist = [filelist]
for fname in filelist:
pfnlist.append(fname)
elif phedex.has_key('block') and not phedex['block']:
msg = 'No replicas found in PhEDEx, will try to get original SE from DBS'
print_warning(msg)
sename = get_dbs_se(lfn)
msg = 'Orignal LFN site %s' % sename
print_info(msg)
mgr = SiteDBManager()
pfnlist = lfn2pfn(lfn, sename, mgr)
filelist = ddict.get('phedex.block.file')
if not filelist:
filelist = []
for fname in filelist:
for replica in fname['replica']:
cmsname = replica['node']
se = replica['se']
if verbose:
print "found LFN on node=%s, se=%s" % (cmsname, se)
if cmsname.count('T0', 0, 2) == 1:
continue # skip T0's
# query Phedex for PFN
params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname}
result = get_data(phedex_url('lfn2pfn'), params)
try:
for item in result['phedex']['mapping']:
pfn = item['pfn']
if pfn not in pfnlist:
pfnlist.append(pfn)
except:
msg = "Fail to look-up PFNs in Phedex\n" + str(result)
print msg
continue
if verbose > 1:
print "PFN list:"
for pfn in pfnlist:
print pfn
# finally return pfn and dst paths w/ file for further processing
for item in pfnlist:
ifile = item.split("/")[-1] if not dstfname else dstfname
yield item, '%s/%s' % (dst, ifile)
def read(url, output=None, debug=0):
"Get run information from RunSummary data-service"
encoding = 'utf-8'
key = None
cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem')
if os.path.isfile(url):
with open(url, 'r') as stream:
context = stream.read()
try:
pydoc.pager(context)
except:
print context
return
elif url.find('cmsweb.cern.ch') != -1:
data = get_data(url, decoder=None)
html = data
encoding = None
elif url.find('mcdb.cern.ch') != -1:
data = urllib.urlopen(url)
html = data.read().replace(' _place_holder;', '')
encoding = enc(data.headers, html)[0]
elif url.find('cern.ch') == -1:
data = urllib.urlopen(url)
html = data.read()
encoding = enc(data.headers, html)[0]
else:
with working_pem(PEMMGR.pem) as key:
data = get_data_sso(url, key, cert, debug)
html = data.read()
encoding = enc(data.headers, html)[0]
if encoding == 'us-ascii':
encoding = 'utf-8'
pager = os.environ.get('CMSSH_PAGER', None)
if html:
if int(os.environ.get('HTTPDEBUG', 0)):
print_info('read data')
print html
if encoding:
text = html.decode(encoding)
res = html2text(text, '')
if output:
with open(output, 'w') as stream:
stream.write(html)
else:
try:
if pager:
pydoc.pager(res.encode('utf-8'))
else:
wrapwrite(html2text(text, ''))
except:
wrapwrite(html2text(text, ''))
else:
if output:
with open(output, 'w') as stream:
stream.write(html)
else:
try:
if pager:
pydoc.pager(html)
else:
print html
except:
print html