def get_pfns(lfn, verbose=None): """ Look-up LFN in Phedex and get corresponding list of PFNs """ pfnlist = [] selist = [] params = {'se':'*', 'lfn':lfn} json_dict = get_data(phedex_url('fileReplicas'), params) ddict = DotDict(json_dict) if not json_dict['phedex']['block']: return pfnlist, selist for fname in ddict.get('phedex.block.file'): for replica in fname['replica']: cmsname = replica['node'] se = replica['se'] if se not in selist: selist.append(se) # query Phedex for PFN params = {'protocol':'srmv2', 'lfn':lfn, 'node':cmsname} result = get_data(phedex_url('lfn2pfn'), params) try: for item in result['phedex']['mapping']: pfn = item['pfn'] if pfn not in pfnlist: pfnlist.append(pfn) except: msg = "Fail to look-up PFNs in Phedex\n" + str(result) print msg continue return pfnlist, selist
def run_lumi_dict(arg, verbose=None): "Return run-lumi info for given argument (dataset, block, file, run)" try: data = json.loads(arg) except: if isinstance(arg, basestring) and arg.find("{") != -1: data = eval(arg, {"__builtins__": None}, {}) else: data = arg # assume it is dataset/file/block/run url = dbs_url() run_lumi = {} if isinstance(data, dict): # we got run-lumi dict for run, lumis in data.items(): run_lumi[int(run)] = lumis else: if url.find("cmsdbsprod") != -1: # DBS2 run_lumi = dbs2.run_lumi(str(data), verbose) else: if pat_dataset.match(data): params = {"dataset": data} result = get_data(dbs_url("files"), params, verbose) for row in result: params = {"logical_file_name": row["logical_file_name"]} run_lumi = parse_runlumis(get_data(dbs_url("filelumis"), params, verbose)) elif pat_block.match(data): params = {"block_name": data} run_lumi = parse_runlumis(get_data(dbs_url("filelumis"), params, verbose)) elif pat_lfn.match(data): params = {"logical_file_name": data} run_lumi = parse_runlumis(get_data(dbs_url("filelumis"), params, verbose)) elif pat_run.match(data): params = {"run_num": data} run_lumi = parse_runlumis(get_data(dbs_url("filelumis"), params, verbose)) return run_lumi
def get_pfns(lfn, verbose=None): """ Look-up LFN in Phedex and get corresponding list of PFNs """ pfnlist = [] selist = [] params = {'se': '*', 'lfn': lfn} json_dict = get_data(phedex_url('fileReplicas'), params) ddict = DotDict(json_dict) if not json_dict['phedex']['block']: return pfnlist, selist for fname in ddict.get('phedex.block.file'): for replica in fname['replica']: cmsname = replica['node'] se = replica['se'] if se not in selist: selist.append(se) # query Phedex for PFN params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname} result = get_data(phedex_url('lfn2pfn'), params) try: for item in result['phedex']['mapping']: pfn = item['pfn'] if pfn not in pfnlist: pfnlist.append(pfn) except: msg = "Fail to look-up PFNs in Phedex\n" + str(result) print msg continue return pfnlist, selist
def releases(rel_name=None, rfilter=None): "Return information about CMS releases" if rel_name: if not pat_release.match(rel_name): msg = 'Wrong CMSSW release name' raise ValueError(msg) args = {'release_name': rel_name} else: args = {} url = tc_url('getReleasesInformation') rel_info = get_data(url, args) columns = rel_info['columns'] pat = re.compile('CMSSW_[1-9]_[0-9]_X\.*') for key, val in rel_info['data'].iteritems(): if rfilter == 'list': if pat.match(key) or not key.find('CMSSW') != -1 or \ key.find('EXPERIMENTAL') != -1 or \ key.find('CLANG') != -1 or \ key.find('_X_') != -1 or \ key.find('FORTIFIED') != -1: continue row = {} pairs = zip(columns['release_name'], val) for kkk, vvv in pairs: if isinstance(kkk, basestring): row[kkk] = vvv elif isinstance(kkk, list): for item in vvv: row.setdefault('architectures', []).append(dict(zip(kkk, item))) row['release_name'] = key for item in row['architectures']: if match_platform(item['architecture_name']): yield row
def list_user(self, **kwargs): """ Controller to get site info """ url = sitedb_url("people") params = {} cert = os.path.join(os.environ["HOME"], ".globus/usercert.pem") ckey = os.path.join(os.environ["HOME"], ".globus/userkey.pem") data = get_data(url, params) qname = kwargs["username"].lower() users = [] for row in sitedb_parser(data): username = row.get("username", "").lower() forename = row.get("forename", "").lower() surname = row.get("surname", "").lower() email = row.get("email", "").lower() if ( username and qname.find(username) != -1 or forename and qname.find(forename) != -1 or surname and qname.find(surname) != -1 or email and qname.find(email) != -1 ): users.append(row) return [User(u) for u in users]
def resolve_srm_path(node, verbose=None): """ Use TFC phedex API to resolve srm path for given node """ params = {'node':node} result = get_data(phedex_url('tfc'), params) for row in result['phedex']['storage-mapping']['array']: if row['protocol'] == 'srmv2' and row['element_name'] == 'lfn-to-pfn': yield (row['result'], row['path-match'])
def list_block4site(self, **kwargs): """ Controller to get site info """ url = phedex_url('blockreplicasummary') params = {'node': kwargs['sitename']} data = get_data(url, params) plist = [Block(b) for b in data['phedex']['block']] return plist
def resolve_srm_path(node, verbose=None): """ Use TFC phedex API to resolve srm path for given node """ params = {'node': node} result = get_data(phedex_url('tfc'), params) for row in result['phedex']['storage-mapping']['array']: if row['protocol'] == 'srmv2' and row['element_name'] == 'lfn-to-pfn': yield (row['result'], row['path-match'])
def list_sites(self, **kwargs): """ Controller to get site info """ url = phedex_url("nodeusage") params = {"node": kwargs["sitename"]} data = get_data(url, params) plist = [Site(s) for s in data["phedex"]["node"]] return plist
def list_sites(self, **kwargs): """ Controller to get site info """ url = phedex_url('nodeusage') params = {'node': kwargs['sitename']} data = get_data(url, params) plist = [Site(s) for s in data['phedex']['node']] return plist
def list_block4site(self, **kwargs): """ Controller to get site info """ url = phedex_url("blockreplicasummary") params = {"node": kwargs["sitename"]} data = get_data(url, params) plist = [Block(b) for b in data["phedex"]["block"]] return plist
def list_runs(self, **kwargs): """ Controller to get runs """ url = conddb_url('getLumi') run = kwargs.get('run') params = {'Runs': run, 'lumiType': 'delivered'} data = get_data(url, params) plist = [Run(d) for d in data] return plist
def resolve_user_srm_path(node, ldir='/store/user', verbose=None): """ Use TFC phedex API to resolve srm path for given node """ # change ldir if user supplied full path, e.g. /xrootdfs/cms/store/... ldir = '/store/' + ldir.split('/store/')[-1] params = {'node': node, 'lfn': ldir, 'protocol': 'srmv2'} result = get_data(phedex_url('lfn2pfn'), params) for row in result['phedex']['mapping']: yield row['pfn']
def list_runs(self, **kwargs): """ Controller to get runs """ url = conddb_url("getLumi") run = kwargs.get("run") params = {"Runs": run, "lumiType": "delivered"} data = get_data(url, params) plist = [Run(d) for d in data] return plist
def list_runs(self, **kwargs): """ Controller to get runs """ url = conddb_url('getLumi') run = kwargs.get('run') params = {'Runs':run, 'lumiType':'delivered'} data = get_data(url, params) plist = [Run(d) for d in data] return plist
def resolve_user_srm_path(node, ldir='/store/user', verbose=None): """ Use TFC phedex API to resolve srm path for given node """ # change ldir if user supplied full path, e.g. /xrootdfs/cms/store/... ldir = '/store/' + ldir.split('/store/')[-1] params = {'node':node, 'lfn':ldir, 'protocol': 'srmv2'} result = get_data(phedex_url('lfn2pfn'), params) for row in result['phedex']['mapping']: yield row['pfn']
def get_release_arch(rel): "Return architecture for given CMSSW release" args = {'release': rel} releases = get_data(tc_url('py_getReleaseArchitectures'), args) output = [] for item in releases: rel_arch = item[0] status = item[1] if check_os(rel_arch): output.append((rel_arch, status)) return output
def site_info(dst, verbose=None): """list files at given destination""" url = phedex_url('nodeusage') params = {'node': dst} data = get_data(url, params) res = [Site(s) for s in data['phedex']['node']] paths = [r for r in resolve_user_srm_path(dst)] pdirs = [p.split('=')[-1] for p in paths] for site in res: site.assign('pfn_path', paths) site.assign('default_path', pdirs) return res
def block_info(block, verbose=None): """Return block info""" url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.block_info(block, verbose) params = {'block_name': block, 'detail':'True'} result = get_data(dbs_url('blocks'), params, verbose) res = [Block(r) for r in result][0] if len(res) != 1: msg = 'The %s block yield %s results' % (block, len(res)) raise Exception(msg) return res[0]
def dataset_info(dataset, verbose=None): """Return dataset info""" url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.dataset_info(dataset, verbose) params = {'dataset': dataset, 'detail':'True'} result = get_data(dbs_url('datasets'), params, verbose) res = [Dataset(r) for r in result] if len(res) != 1: msg = 'The %s dataset yield %s results' % (dataset, len(res)) raise Exception(msg) return res[0]
def dataset_info(dataset, verbose=None): """Return dataset info""" url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.dataset_info(dataset, verbose) params = {'dataset': dataset, 'detail': 'True'} result = get_data(dbs_url('datasets'), params, verbose) res = [Dataset(r) for r in result] if len(res) != 1: msg = 'The %s dataset yield %s results' % (dataset, len(res)) raise Exception(msg) return res[0]
def dataset_info(dataset, verbose=None): """Return dataset info""" url = dbs_url() if url.find("cmsdbsprod") != -1: # DBS2 return dbs2.dataset_info(dataset, verbose) params = {"dataset": dataset, "detail": "True"} result = get_data(dbs_url("datasets"), params, verbose) res = [Dataset(r) for r in result] if len(res) != 1: msg = "The %s dataset yield %s results" % (dataset, len(res)) raise Exception(msg) return res[0]
def block_info(block, verbose=None): """Return block info""" url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.block_info(block, verbose) params = {'block_name': block, 'detail': 'True'} result = get_data(dbs_url('blocks'), params, verbose) res = [Block(r) for r in result][0] if len(res) != 1: msg = 'The %s block yield %s results' % (block, len(res)) raise Exception(msg) return res[0]
def block_info(block, verbose=None): """Return block info""" url = dbs_url() if url.find("cmsdbsprod") != -1: # DBS2 return dbs2.block_info(block, verbose) params = {"block_name": block, "detail": "True"} result = get_data(dbs_url("blocks"), params, verbose) res = [Block(r) for r in result][0] if len(res) != 1: msg = "The %s block yield %s results" % (block, len(res)) raise Exception(msg) return res[0]
def site_info(dst, verbose=None): """list files at given destination""" url = phedex_url("nodeusage") params = {"node": dst} data = get_data(url, params) res = [Site(s) for s in data["phedex"]["node"]] paths = [r for r in resolve_user_srm_path(dst)] pdirs = [p.split("=")[-1] for p in paths] for site in res: site.assign("pfn_path", paths) site.assign("default_path", pdirs) return res
def run_lumi_dict(arg, verbose=None): "Return run-lumi info for given argument (dataset, block, file, run)" try: data = json.loads(arg) except: if isinstance(arg, basestring) and arg.find("{") != -1: data = eval(arg, {"__builtins__": None}, {}) else: data = arg # assume it is dataset/file/block/run url = dbs_url() run_lumi = {} if isinstance(data, dict): # we got run-lumi dict for run, lumis in data.items(): run_lumi[int(run)] = lumis else: if url.find('cmsdbsprod') != -1: # DBS2 run_lumi = dbs2.run_lumi(str(data), verbose) else: if pat_dataset.match(data): params = {'dataset': data} result = get_data(dbs_url('files'), params, verbose) for row in result: params = {'logical_file_name': row['logical_file_name']} run_lumi = parse_runlumis( get_data(dbs_url('filelumis'), params, verbose)) elif pat_block.match(data): params = {'block_name': data} run_lumi = parse_runlumis( get_data(dbs_url('filelumis'), params, verbose)) elif pat_lfn.match(data): params = {'logical_file_name': data} run_lumi = parse_runlumis( get_data(dbs_url('filelumis'), params, verbose)) elif pat_run.match(data): params = {'run_num': data} run_lumi = parse_runlumis( get_data(dbs_url('filelumis'), params, verbose)) return run_lumi
def run_info(run, verbose=None): """Return run info""" url = conddb_url('getLumi') params = {'Runs': run, 'lumiType': 'delivered'} data = get_data(url, params) runinfo = [r for r in runsum(run)] runlumi = [r for r in data] plist = [] for run in runinfo: for rec in runlumi: if run['run'] == rec['Run']: run.update(rec) plist.append(Run(run)) return plist
def list_datasets(self, **kwargs): """ Controller to get DBS datasets """ url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.list_datasets(kwargs) url = dbs_url('datasets') params = {'dataset': kwargs['dataset']} if kwargs['dataset'][0] == '*': kwargs['dataset'] = '/' + kwargs['dataset'] data = get_data(url, kwargs) plist = [Dataset(d) for d in data] return plist
def run_info(run, verbose=None): """Return run info""" url = conddb_url("getLumi") params = {"Runs": run, "lumiType": "delivered"} data = get_data(url, params) runinfo = [r for r in runsum(run)] runlumi = [r for r in data] plist = [] for run in runinfo: for rec in runlumi: if run["run"] == rec["Run"]: run.update(rec) plist.append(Run(run)) return plist
def list_datasets(self, **kwargs): """ Controller to get DBS datasets """ url = dbs_url() if url.find("cmsdbsprod") != -1: # DBS2 return dbs2.list_datasets(kwargs) url = dbs_url("datasets") params = {"dataset": kwargs["dataset"]} if kwargs["dataset"][0] == "*": kwargs["dataset"] = "/" + kwargs["dataset"] data = get_data(url, kwargs) plist = [Dataset(d) for d in data] return plist
def run_info(run, verbose=None): """Return run info""" url = conddb_url('getLumi') params = {'Runs':run, 'lumiType':'delivered'} data = get_data(url, params) runinfo = [r for r in runsum(run)] runlumi = [r for r in data] plist = [] for run in runinfo: for rec in runlumi: if run['run'] == rec['Run']: run.update(rec) plist.append(Run(run)) return plist
def lfns(run=None, dataset=None): """ Get lfns list for provided run/dataset """ url = dbs_url('files') # DBS3 params = {'detail':'True'} if run: args['minrun'] = run args['maxrun'] = run if dataset: args['dataset'] = dataset params.update(args) json_dict = get_data(url, params) for row in json_dict: yield row['logical_file_name']
def get_tickets(ticket=None, **kwds): """ Retrieve information about cmssh tickets """ res = [] url = 'https://api.github.com' path = 'repos/vkuznet/cmssh/issues' if ticket: path += '/%s' % ticket url += path res = get_data(url, kwds) if isinstance(res, dict): res = [res] res = [Ticket(t) for t in res] return res
def lfns(run=None, dataset=None): """ Get lfns list for provided run/dataset """ url = dbs_url('files') # DBS3 params = {'detail': 'True'} if run: args['minrun'] = run args['maxrun'] = run if dataset: args['dataset'] = dataset params.update(args) json_dict = get_data(url, params) for row in json_dict: yield row['logical_file_name']
def list_files(self, **kwargs): """ Controller to get files """ url = dbs_url() run = kwargs.get('run', None) dataset = kwargs.get('dataset') if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.list_files(dataset, run) url = dbs_url('files') params = {'dataset': dataset, 'detail': 'True'} if run: params.update({'run_num': run}) data = get_data(url, params) plist = [File(f) for f in data] return plist
def nodes(select=True): """ Yield list of Phedex nodes, I only select T2 and below """ result = get_data(phedex_url('nodes'), {}) pat = re.compile('^T[0-1]_[A-Z]+(_)[A-Z]+') lnodes = [] for row in result['phedex']['node']: if select and pat.match(row['name']): continue msg = "%s, SE: %s, description %s/%s" \ % (row['name'], row['se'], row['technology'], row['kind']) lnodes.append(msg) lnodes.sort() for row in lnodes: print row
def list_files(self, **kwargs): """ Controller to get files """ url = dbs_url() run = kwargs.get("run", None) dataset = kwargs.get("dataset") if url.find("cmsdbsprod") != -1: # DBS2 return dbs2.list_files(dataset, run) url = dbs_url("files") params = {"dataset": dataset, "detail": "True"} if run: params.update({"run_num": run}) data = get_data(url, params) plist = [File(f) for f in data] return plist
def find_sites(url, params): """Find sites""" data = get_data(url, params) sites = {} for files in data["phedex"]["block"]: for fdict in files["file"]: replicas = fdict["replica"] for replica in replicas: node = replica.get("node", None) se = replica.get("se", None) if sites.has_key(node): if se not in sites[node]: sites[node] += [se] else: sites[node] = [se] for key, val in sites.iteritems(): yield Site({"node": key, "se": val})
def find_sites(url, params): """Find sites""" data = get_data(url, params) sites = {} for files in data['phedex']['block']: for fdict in files['file']: replicas = fdict['replica'] for replica in replicas: node = replica.get('node', None) se = replica.get('se', None) if sites.has_key(node): if se not in sites[node]: sites[node] += [se] else: sites[node] = [se] for key, val in sites.iteritems(): yield Site({'node': key, 'se': val})
def list_datasets(self, **kwargs): """ Controller to get DBS datasets """ url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.list_datasets(kwargs) url = dbs_url('datasets') params = {'dataset':kwargs.get('dataset', '*')} if kwargs['dataset'][0] == '*': kwargs['dataset'] = '/' + kwargs['dataset'] if kwargs.has_key('file'): kwargs['logical_file_name'] = kwargs['file'] del kwargs['file'] data = get_data(url, kwargs) plist = [Dataset(d) for d in data] return plist
def list_du4site(self, **kwargs): """ Controller to get site info """ url = phedex_url('blockReplicas') site = kwargs['sitename'] params = {'node': site} data = get_data(url, params) nfiles = 0 nblocks = 0 size = 0 for row in data['phedex']['block']: nblocks += 1 nfiles += int(row['files']) for rep in row['replica']: if rep['node'] == site: size += long(rep['bytes']) return dict(nblocks=nblocks, nfiles=nfiles, totalsize=size)
def lfn2pfn(lfn, sename, mgr=None): "Find PFN for given LFN and SE" pfnlist = [] if not mgr: mgr = SiteDBManager() cmsname = mgr.get_name(sename) if cmsname: params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname} result = get_data(phedex_url('lfn2pfn'), params) try: for item in result['phedex']['mapping']: pfn = item['pfn'] if pfn not in pfnlist: pfnlist.append(pfn) except: msg = "Fail to look-up PFNs in Phedex\n" + str(result) print msg return pfnlist
def file_info(lfn, verbose=None): """Return file info""" url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.file_info(lfn, verbose) params = {'logical_file_name': lfn, 'detail': 'True'} result = get_data(dbs_url('files'), params, verbose) res = [File(r) for r in result] if len(res) != 1: msg = 'The %s LFN yield %s results' % (lfn, len(res)) raise Exception(msg) lfnobj = res[0] try: pfnlist, selist = get_pfns(lfn, verbose) lfnobj.assign('pfn', pfnlist) lfnobj.assign('se', selist) except: lfnobj.assign('pfn', []) lfnobj.assign('se', []) return lfnobj
def list_user(self, **kwargs): """ Controller to get site info """ url = sitedb_url('people') params = {} cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') ckey = os.path.join(os.environ['HOME'], '.globus/userkey.pem') data = get_data(url, params) qname = kwargs['username'].lower() users = [] for row in sitedb_parser(data): username = row.get('username', '').lower() forename = row.get('forename', '').lower() surname = row.get('surname', '').lower() email = row.get('email', '').lower() if username and qname.find(username) != -1 or \ forename and qname.find(forename) != -1 or \ surname and qname.find(surname) != -1 or \ email and qname.find(email) != -1: users.append(row) return [User(u) for u in users]
def list_jobs(self, **kwargs): "Controller for jobs info" url = dashboard_url( 'jobsummary-plot-or-table2') # JSON API (number 2 :) params = { 'user': kwargs.get('user', ''), 'site': kwargs.get('site', ''), 'ce': '', 'submissiontool': '', 'dataset': kwargs.get('dataset', ''), 'application': '', 'rb': '', 'activity': '', 'grid': '', 'date1': '', 'date2': '', 'jobtype': '', 'tier': '', 'check': 'submitted', } data = get_data(url, params) plist = [Job(r) for r in data['summaries']] return plist
def pfn_dst(lfn, dst, verbose=None): """ Look-up LFN in Phedex and return pfn dst for further processing """ dstfname = None pat = re.compile('^T[0-9]_[A-Z]+(_)[A-Z]+') if pat.match(dst): dst_split = dst.split(':') dst = dst_split[0] if len(dst_split) == 1: # copy to the node local_path = dst_split[1] for srm_path, lfn_match in resolve_srm_path(dst, verbose): lfn_pat = re.compile(lfn_match) if lfn_pat.match(lfn): srm_path = srm_path.replace('\?', '?').replace('$1', local_path) if verbose: print "Resolve %s into %s" % (dst, srm_path) dst = srm_path else: paths = [p for p in resolve_user_srm_path(dst, verbose=verbose)] dst = '%s/%s' % (paths[0], get_username()) check_permission(dst, verbose) else: if dst.find('file:///') == -1: dstfname = dst.split('/')[-1] if dstfname == '.': dstfname = None if dst[0] == '/': # absolute path if os.path.isdir(dst): ddir = dst dstfname = None else: ddir = '/'.join(dst.split('/')[:-1]) if not os.path.isdir(ddir): msg = 'Provided destination directory %s does not exists' % ddir raise Exception(msg) dst = 'file:///%s' % ddir else: ddir = '/'.join(dst.split('/')[:-1]).replace( '$PWD', os.getcwd()) if os.path.isdir(ddir): dst = 'file:///%s' % os.path.join(os.getcwd(), ddir) else: dst = 'file:///%s' % os.getcwd() pfnlist = [] if os.path.isfile(lfn) or lfn.find('file:///') != -1: # local file pfn = lfn.replace('file:///', '') if pfn[0] != '/': pfn = 'file:///%s' % os.path.join(os.getcwd(), pfn) else: pfn = 'file:///%s' % pfn pfnlist = [pfn] else: if lfn.find(':') != -1: node, lfn = lfn.split(':') params = {'node': node, 'lfn': lfn, 'protocol': 'srmv2'} method = 'lfn2pfn' else: params = {'se': '*', 'lfn': lfn} method = 'fileReplicas' json_dict = get_data(phedex_url(method), params) ddict = DotDict(json_dict) if verbose: print "Look-up LFN:" print lfn phedex = json_dict['phedex'] if phedex.has_key('mapping'): if not phedex['mapping']: msg = "LFN: %s\n" % lfn msg += 'No replicas found\n' msg += str(json_dict) raise Exception(msg) filelist = ddict.get('phedex.mapping.pfn') if not filelist: filelist = [] if isinstance(filelist, basestring): filelist = [filelist] for fname in filelist: pfnlist.append(fname) elif phedex.has_key('block') and not phedex['block']: msg = 'No replicas found in PhEDEx, will try to get original SE from DBS' print_warning(msg) sename = get_dbs_se(lfn) msg = 'Orignal LFN site %s' % sename print_info(msg) mgr = SiteDBManager() pfnlist = lfn2pfn(lfn, sename, mgr) filelist = ddict.get('phedex.block.file') if not filelist: filelist = [] for fname in filelist: for replica in fname['replica']: cmsname = replica['node'] se = replica['se'] if verbose: print "found LFN on node=%s, se=%s" % (cmsname, se) if cmsname.count('T0', 0, 2) == 1: continue # skip T0's # query Phedex for PFN params = {'protocol': 'srmv2', 'lfn': lfn, 'node': cmsname} result = get_data(phedex_url('lfn2pfn'), params) try: for item in result['phedex']['mapping']: pfn = item['pfn'] if pfn not in pfnlist: pfnlist.append(pfn) except: msg = "Fail to look-up PFNs in Phedex\n" + str(result) print msg continue if verbose > 1: print "PFN list:" for pfn in pfnlist: print pfn # finally return pfn and dst paths w/ file for further processing for item in pfnlist: ifile = item.split("/")[-1] if not dstfname else dstfname yield item, '%s/%s' % (dst, ifile)
def read(url, output=None, debug=0): "Get run information from RunSummary data-service" encoding = 'utf-8' key = None cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') if os.path.isfile(url): with open(url, 'r') as stream: context = stream.read() try: pydoc.pager(context) except: print context return elif url.find('cmsweb.cern.ch') != -1: data = get_data(url, decoder=None) html = data encoding = None elif url.find('mcdb.cern.ch') != -1: data = urllib.urlopen(url) html = data.read().replace(' _place_holder;', '') encoding = enc(data.headers, html)[0] elif url.find('cern.ch') == -1: data = urllib.urlopen(url) html = data.read() encoding = enc(data.headers, html)[0] else: with working_pem(PEMMGR.pem) as key: data = get_data_sso(url, key, cert, debug) html = data.read() encoding = enc(data.headers, html)[0] if encoding == 'us-ascii': encoding = 'utf-8' pager = os.environ.get('CMSSH_PAGER', None) if html: if int(os.environ.get('HTTPDEBUG', 0)): print_info('read data') print html if encoding: text = html.decode(encoding) res = html2text(text, '') if output: with open(output, 'w') as stream: stream.write(html) else: try: if pager: pydoc.pager(res.encode('utf-8')) else: wrapwrite(html2text(text, '')) except: wrapwrite(html2text(text, '')) else: if output: with open(output, 'w') as stream: stream.write(html) else: try: if pager: pydoc.pager(html) else: print html except: print html