def block_info(block, verbose=None): """Return block info""" url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.block_info(block, verbose) params = {'block_name': block, 'detail':'True'} result = get_data(dbs_url('blocks'), params, verbose) res = [Block(r) for r in result][0] if len(res) != 1: msg = 'The %s block yield %s results' % (block, len(res)) raise Exception(msg) return res[0]
def dataset_info(dataset, verbose=None): """Return dataset info""" url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.dataset_info(dataset, verbose) params = {'dataset': dataset, 'detail': 'True'} result = get_data(dbs_url('datasets'), params, verbose) res = [Dataset(r) for r in result] if len(res) != 1: msg = 'The %s dataset yield %s results' % (dataset, len(res)) raise Exception(msg) return res[0]
def dataset_info(dataset, verbose=None): """Return dataset info""" url = dbs_url() if url.find("cmsdbsprod") != -1: # DBS2 return dbs2.dataset_info(dataset, verbose) params = {"dataset": dataset, "detail": "True"} result = get_data(dbs_url("datasets"), params, verbose) res = [Dataset(r) for r in result] if len(res) != 1: msg = "The %s dataset yield %s results" % (dataset, len(res)) raise Exception(msg) return res[0]
def block_info(block, verbose=None): """Return block info""" url = dbs_url() if url.find("cmsdbsprod") != -1: # DBS2 return dbs2.block_info(block, verbose) params = {"block_name": block, "detail": "True"} result = get_data(dbs_url("blocks"), params, verbose) res = [Block(r) for r in result][0] if len(res) != 1: msg = "The %s block yield %s results" % (block, len(res)) raise Exception(msg) return res[0]
def dataset_info(dataset, verbose=None): """Return dataset info""" url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.dataset_info(dataset, verbose) params = {'dataset': dataset, 'detail':'True'} result = get_data(dbs_url('datasets'), params, verbose) res = [Dataset(r) for r in result] if len(res) != 1: msg = 'The %s dataset yield %s results' % (dataset, len(res)) raise Exception(msg) return res[0]
def block_info(block, verbose=None): """Return block info""" url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.block_info(block, verbose) params = {'block_name': block, 'detail': 'True'} result = get_data(dbs_url('blocks'), params, verbose) res = [Block(r) for r in result][0] if len(res) != 1: msg = 'The %s block yield %s results' % (block, len(res)) raise Exception(msg) return res[0]
def list_datasets(self, **kwargs): """ Controller to get DBS datasets """ url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.list_datasets(kwargs) url = dbs_url('datasets') params = {'dataset': kwargs['dataset']} if kwargs['dataset'][0] == '*': kwargs['dataset'] = '/' + kwargs['dataset'] data = get_data(url, kwargs) plist = [Dataset(d) for d in data] return plist
def list_datasets(self, **kwargs): """ Controller to get DBS datasets """ url = dbs_url() if url.find("cmsdbsprod") != -1: # DBS2 return dbs2.list_datasets(kwargs) url = dbs_url("datasets") params = {"dataset": kwargs["dataset"]} if kwargs["dataset"][0] == "*": kwargs["dataset"] = "/" + kwargs["dataset"] data = get_data(url, kwargs) plist = [Dataset(d) for d in data] return plist
def file_info(lfn, verbose=None): query = 'find file.name, file.numevents, file.size, file.createdate, file.createby, file.moddate, file.modby where file=%s' % lfn params = {"api": "executeQuery", "apiversion": "DBS_2_0_9", "query": query} default_instance = os.environ.get('DBS_INSTANCE') for inst in dbs_instances(): os.environ['DBS_INSTANCE'] = inst data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) try: rec = [f for f in qlxml_parser(data, 'file')][0] except: continue rec['logical_file_name'] = rec['file']['file.name'] rec['size'] = rec['file']['file.size'] rec['nevents'] = rec['file']['file.numevents'] rec['created'] = time.strftime( "%Y-%m-%d %H:%M:%S GMT", time.gmtime(rec['file']['file.createdate'])) rec['createdby'] = rec['file']['file.createby'] rec['modified'] = time.strftime( "%Y-%m-%d %H:%M:%S GMT", time.gmtime(rec['file']['file.moddate'])) rec['modifiedby'] = rec['file']['file.modby'] del rec['file'] lfnobj = File(rec) try: pfnlist, selist = get_pfns(lfn, verbose) if not selist: query = 'find site where file=%s' % lfn params.update({"query": query}) data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) try: rec = [f for f in qlxml_parser(data, 'site')][0] sename = rec['site']['site'] selist = [sename] pfnlist = lfn2pfn(lfn, sename) except: pass lfnobj.assign('pfn', pfnlist) lfnobj.assign('se', selist) except: traceback.print_exc() lfnobj.assign('pfn', []) lfnobj.assign('se', []) os.environ['DBS_INSTANCE'] = default_instance lfnobj.assign('dbs_instance', inst) return lfnobj os.environ['DBS_INSTANCE'] = default_instance msg = 'Fail to look-up LFN in %s DBS instances' % dbs_instances() print_error(msg)
def list_files(self, **kwargs): """ Controller to get files """ url = dbs_url() run = kwargs.get("run", None) dataset = kwargs.get("dataset") if url.find("cmsdbsprod") != -1: # DBS2 return dbs2.list_files(dataset, run) url = dbs_url("files") params = {"dataset": dataset, "detail": "True"} if run: params.update({"run_num": run}) data = get_data(url, params) plist = [File(f) for f in data] return plist
def list_files(self, **kwargs): """ Controller to get files """ url = dbs_url() run = kwargs.get('run', None) dataset = kwargs.get('dataset') if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.list_files(dataset, run) url = dbs_url('files') params = {'dataset': dataset, 'detail': 'True'} if run: params.update({'run_num': run}) data = get_data(url, params) plist = [File(f) for f in data] return plist
def list_datasets(self, **kwargs): """ Controller to get DBS datasets """ url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.list_datasets(kwargs) url = dbs_url('datasets') params = {'dataset':kwargs.get('dataset', '*')} if kwargs['dataset'][0] == '*': kwargs['dataset'] = '/' + kwargs['dataset'] if kwargs.has_key('file'): kwargs['logical_file_name'] = kwargs['file'] del kwargs['file'] data = get_data(url, kwargs) plist = [Dataset(d) for d in data] return plist
def file_info(lfn, verbose=None): query = 'find file.name, file.numevents, file.size, file.createdate, file.createby, file.moddate, file.modby where file=%s' % lfn params = {"api":"executeQuery", "apiversion": "DBS_2_0_9", "query":query} default_instance = os.environ.get('DBS_INSTANCE') for inst in dbs_instances(): os.environ['DBS_INSTANCE'] = inst data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) try: rec = [f for f in qlxml_parser(data, 'file')][0] except: continue rec['logical_file_name'] = rec['file']['file.name'] rec['size'] = rec['file']['file.size'] rec['nevents'] = rec['file']['file.numevents'] rec['created'] = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime(rec['file']['file.createdate'])) rec['createdby'] = rec['file']['file.createby'] rec['modified'] = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime(rec['file']['file.moddate'])) rec['modifiedby'] = rec['file']['file.modby'] del rec['file'] lfnobj = File(rec) try: pfnlist, selist = get_pfns(lfn, verbose) if not selist: query = 'find site where file=%s' % lfn params.update({"query":query}) data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) try: rec = [f for f in qlxml_parser(data, 'site')][0] sename = rec['site']['site'] selist = [sename] pfnlist = lfn2pfn(lfn, sename) except: pass lfnobj.assign('pfn', pfnlist) lfnobj.assign('se', selist) except: traceback.print_exc() lfnobj.assign('pfn', []) lfnobj.assign('se', []) os.environ['DBS_INSTANCE'] = default_instance lfnobj.assign('dbs_instance', inst) return lfnobj os.environ['DBS_INSTANCE'] = default_instance msg = 'Fail to look-up LFN in %s DBS instances' % dbs_instances() print_error(msg)
def file_info(lfn, verbose=None): """Return file info""" url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.file_info(lfn, verbose) params = {'logical_file_name': lfn, 'detail': 'True'} result = get_data(dbs_url('files'), params, verbose) res = [File(r) for r in result] if len(res) != 1: msg = 'The %s LFN yield %s results' % (lfn, len(res)) raise Exception(msg) lfnobj = res[0] try: pfnlist, selist = get_pfns(lfn, verbose) lfnobj.assign('pfn', pfnlist) lfnobj.assign('se', selist) except: lfnobj.assign('pfn', []) lfnobj.assign('se', []) return lfnobj
def file_info(lfn, verbose=None): """Return file info""" url = dbs_url() if url.find('cmsdbsprod') != -1: # DBS2 return dbs2.file_info(lfn, verbose) params = {'logical_file_name': lfn, 'detail':'True'} result = get_data(dbs_url('files'), params, verbose) res = [File(r) for r in result] if len(res) != 1: msg = 'The %s LFN yield %s results' % (lfn, len(res)) raise Exception(msg) lfnobj = res[0] try: pfnlist, selist = get_pfns(lfn, verbose) lfnobj.assign('pfn', pfnlist) lfnobj.assign('se', selist) except: lfnobj.assign('pfn', []) lfnobj.assign('se', []) return lfnobj
def file_info(lfn, verbose=None): """Return file info""" url = dbs_url() if url.find("cmsdbsprod") != -1: # DBS2 return dbs2.file_info(lfn, verbose) params = {"logical_file_name": lfn, "detail": "True"} result = get_data(dbs_url("files"), params, verbose) res = [File(r) for r in result] if len(res) != 1: msg = "The %s LFN yield %s results" % (lfn, len(res)) raise Exception(msg) lfnobj = res[0] try: pfnlist, selist = get_pfns(lfn, verbose) lfnobj.assign("pfn", pfnlist) lfnobj.assign("se", selist) except: lfnobj.assign("pfn", []) lfnobj.assign("se", []) return lfnobj
def run_lumi_dict(arg, verbose=None): "Return run-lumi info for given argument (dataset, block, file, run)" try: data = json.loads(arg) except: if isinstance(arg, basestring) and arg.find("{") != -1: data = eval(arg, {"__builtins__": None}, {}) else: data = arg # assume it is dataset/file/block/run url = dbs_url() run_lumi = {} if isinstance(data, dict): # we got run-lumi dict for run, lumis in data.items(): run_lumi[int(run)] = lumis else: if url.find("cmsdbsprod") != -1: # DBS2 run_lumi = dbs2.run_lumi(str(data), verbose) else: if pat_dataset.match(data): params = {"dataset": data} result = get_data(dbs_url("files"), params, verbose) for row in result: params = {"logical_file_name": row["logical_file_name"]} run_lumi = parse_runlumis(get_data(dbs_url("filelumis"), params, verbose)) elif pat_block.match(data): params = {"block_name": data} run_lumi = parse_runlumis(get_data(dbs_url("filelumis"), params, verbose)) elif pat_lfn.match(data): params = {"logical_file_name": data} run_lumi = parse_runlumis(get_data(dbs_url("filelumis"), params, verbose)) elif pat_run.match(data): params = {"run_num": data} run_lumi = parse_runlumis(get_data(dbs_url("filelumis"), params, verbose)) return run_lumi
def block_info(block, verbose=None): query = 'find block.name, block.sizei, block.createdate, block.createby, block.moddate, block.modby where block=%s' % block params = {"api":"executeQuery", "apiversion": "DBS_2_0_9", "query":query} data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) blk = [b for b in qlxml_parser(data, 'block')][0] blk['block_name'] = blk['block']['block.name'] blk['size'] = blk['block']['block.size'] blk['created'] = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime(blk['block']['block.createdate'])) blk['createdby'] = blk['block']['block.createby'] blk['modified'] = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime(blk['block']['block.moddate'])) blk['modifiedby'] = blk['block']['block.modby'] del blk['block'] return Block(blk)
def list_files(dataset, run=None): query = 'find file where dataset=%s' % dataset if run: query += ' and run=%s' % run params = {"api": "executeQuery", "apiversion": "DBS_2_0_9", "query": query} data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) gen = qlxml_parser(data, 'file') files = [] for rec in gen: rec['logical_file_name'] = rec['file']['file'] del rec['file'] files.append(rec) plist = [File(f) for f in files] return plist
def list_files(dataset, run=None): query = 'find file where dataset=%s' % dataset if run: query += ' and run=%s' % run params = {"api":"executeQuery", "apiversion": "DBS_2_0_9", "query":query} data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) gen = qlxml_parser(data, 'file') files = [] for rec in gen: rec['logical_file_name'] = rec['file']['file'] del rec['file'] files.append(rec) plist = [File(f) for f in files] return plist
def lfns(run=None, dataset=None): """ Get lfns list for provided run/dataset """ url = dbs_url('files') # DBS3 params = {'detail':'True'} if run: args['minrun'] = run args['maxrun'] = run if dataset: args['dataset'] = dataset params.update(args) json_dict = get_data(url, params) for row in json_dict: yield row['logical_file_name']
def block_info(block, verbose=None): query = 'find block.name, block.sizei, block.createdate, block.createby, block.moddate, block.modby where block=%s' % block params = {"api": "executeQuery", "apiversion": "DBS_2_0_9", "query": query} data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) blk = [b for b in qlxml_parser(data, 'block')][0] blk['block_name'] = blk['block']['block.name'] blk['size'] = blk['block']['block.size'] blk['created'] = time.strftime( "%Y-%m-%d %H:%M:%S GMT", time.gmtime(blk['block']['block.createdate'])) blk['createdby'] = blk['block']['block.createby'] blk['modified'] = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime(blk['block']['block.moddate'])) blk['modifiedby'] = blk['block']['block.modby'] del blk['block'] return Block(blk)
def lfns(run=None, dataset=None): """ Get lfns list for provided run/dataset """ url = dbs_url('files') # DBS3 params = {'detail': 'True'} if run: args['minrun'] = run args['maxrun'] = run if dataset: args['dataset'] = dataset params.update(args) json_dict = get_data(url, params) for row in json_dict: yield row['logical_file_name']
def dataset_info(dataset, verbose=None): query = 'find dataset.name, datatype, dataset.status, dataset.createdate, dataset.createby, dataset.moddate, dataset.modby, sum(block.size), count(block), sum(block.numfiles), sum(block.numevents) where dataset=%s' % dataset params = {"api":"executeQuery", "apiversion": "DBS_2_0_9", "query":query} data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) rec = [d for d in qlxml_parser(data, 'dataset')][0] rec['size'] = rec['dataset']['sum_block.size'] rec['nblocks'] = rec['dataset']['count_block'] rec['nfiles'] = rec['dataset']['sum_block.numfiles'] rec['nevents'] = rec['dataset']['sum_block.numevents'] rec['created'] = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime(rec['dataset']['dataset.createdate'])) rec['createdby'] = rec['dataset']['dataset.createby'] rec['modified'] = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime(rec['dataset']['dataset.moddate'])) rec['modifiedby'] = rec['dataset']['dataset.modby'] rec['status'] = rec['dataset']['dataset.status'] rec['datatype'] = rec['dataset']['datatype'] rec['dataset'] = rec['dataset']['dataset.name'] return Dataset(rec)
def get_dbs_se(lfn): "Get original SE from DBS for given LFN" # TODO: should have transparent access to DBS2/DBS3 query = 'find site where file=%s' % lfn params = {"api": "executeQuery", "apiversion": "DBS_2_0_9", "query": query} default_instance = os.environ.get('DBS_INSTANCE') for inst in dbs_instances(): params.update({"query": query}) os.environ['DBS_INSTANCE'] = inst data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) try: rec = [f for f in qlxml_parser(data, 'site')][0] sename = rec['site']['site'] except: continue os.environ['DBS_INSTANCE'] = default_instance return sename os.environ['DBS_INSTANCE'] = default_instance
def get_dbs_se(lfn): "Get original SE from DBS for given LFN" # TODO: should have transparent access to DBS2/DBS3 query = 'find site where file=%s' % lfn params = {"api":"executeQuery", "apiversion": "DBS_2_0_9", "query":query} default_instance = os.environ.get('DBS_INSTANCE') for inst in dbs_instances(): params.update({"query":query}) os.environ['DBS_INSTANCE'] = inst data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) try: rec = [f for f in qlxml_parser(data, 'site')][0] sename = rec['site']['site'] except: continue os.environ['DBS_INSTANCE'] = default_instance return sename os.environ['DBS_INSTANCE'] = default_instance
def list_datasets(kwargs): """Find sites""" dataset = kwargs.pop('dataset') # query = 'find dataset, dataset.createdate, dataset.createby, dataset.moddate, dataset.modby, datatype, dataset.status where dataset=%s' % dataset query = 'find dataset where dataset=%s' % dataset if kwargs.has_key('status'): kwargs['dataset.status'] = kwargs['status'] del kwargs['status'] else: kwargs['dataset.status'] = 'VALID' cond = '' for key, val in kwargs.items(): cond += ' and %s=%s' % (key, val) query += cond params = {"api":"executeQuery", "apiversion": "DBS_2_0_9", "query":query} data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) gen = qlxml_parser(data, 'dataset') plist = [Dataset(d['dataset']) for d in gen] return plist
def list_datasets(kwargs): """Find sites""" dataset = kwargs.pop('dataset') # query = 'find dataset, dataset.createdate, dataset.createby, dataset.moddate, dataset.modby, datatype, dataset.status where dataset=%s' % dataset query = 'find dataset where dataset=%s' % dataset if kwargs.has_key('status'): kwargs['dataset.status'] = kwargs['status'] del kwargs['status'] else: kwargs['dataset.status'] = 'VALID' cond = '' for key, val in kwargs.items(): cond += ' and %s=%s' % (key, val) query += cond params = {"api": "executeQuery", "apiversion": "DBS_2_0_9", "query": query} data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) gen = qlxml_parser(data, 'dataset') plist = [Dataset(d['dataset']) for d in gen] return plist
def dataset_info(dataset, verbose=None): query = 'find dataset.name, datatype, dataset.status, dataset.createdate, dataset.createby, dataset.moddate, dataset.modby, sum(block.size), count(block), sum(block.numfiles), sum(block.numevents) where dataset=%s' % dataset params = {"api": "executeQuery", "apiversion": "DBS_2_0_9", "query": query} data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) rec = [d for d in qlxml_parser(data, 'dataset')][0] rec['size'] = rec['dataset']['sum_block.size'] rec['nblocks'] = rec['dataset']['count_block'] rec['nfiles'] = rec['dataset']['sum_block.numfiles'] rec['nevents'] = rec['dataset']['sum_block.numevents'] rec['created'] = time.strftime( "%Y-%m-%d %H:%M:%S GMT", time.gmtime(rec['dataset']['dataset.createdate'])) rec['createdby'] = rec['dataset']['dataset.createby'] rec['modified'] = time.strftime( "%Y-%m-%d %H:%M:%S GMT", time.gmtime(rec['dataset']['dataset.moddate'])) rec['modifiedby'] = rec['dataset']['dataset.modby'] rec['status'] = rec['dataset']['dataset.status'] rec['datatype'] = rec['dataset']['datatype'] rec['dataset'] = rec['dataset']['dataset.name'] return Dataset(rec)
def run_lumi(arg, verbose=None): if pat_block.match(arg): query = 'find run,lumi where block=%s' % arg elif pat_lfn.match(arg): query = 'find run,lumi where file=%s' % arg elif pat_dataset.match(arg): query = 'find run,lumi where dataset=%s' % arg elif pat_run.match(arg): query = 'find run,lumi where run=%s' % arg params = {"api": "executeQuery", "apiversion": "DBS_2_0_9", "query": query} data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) run_lumi = {} for row in qlxml_parser(data, 'run'): rec = row['run'] run = rec['run'] lumi = rec['lumi'] if run_lumi.has_key(run): run_lumi[run].append(lumi) else: run_lumi[run] = [lumi] for key, val in run_lumi.items(): val.sort() run_lumi[key] = val return run_lumi
def run_lumi(arg, verbose=None): if pat_block.match(arg): query = 'find run,lumi where block=%s' % arg elif pat_lfn.match(arg): query = 'find run,lumi where file=%s' % arg elif pat_dataset.match(arg): query = 'find run,lumi where dataset=%s' % arg elif pat_run.match(arg): query = 'find run,lumi where run=%s' % arg params = {"api":"executeQuery", "apiversion": "DBS_2_0_9", "query":query} data = urllib2.urlopen(dbs_url(), urllib.urlencode(params)) run_lumi = {} for row in qlxml_parser(data, 'run'): rec = row['run'] run = rec['run'] lumi = rec['lumi'] if run_lumi.has_key(run): run_lumi[run].append(lumi) else: run_lumi[run] = [lumi] for key, val in run_lumi.items(): val.sort() run_lumi[key] = val return run_lumi
def run_lumi_dict(arg, verbose=None): "Return run-lumi info for given argument (dataset, block, file, run)" try: data = json.loads(arg) except: if isinstance(arg, basestring) and arg.find("{") != -1: data = eval(arg, {"__builtins__": None}, {}) else: data = arg # assume it is dataset/file/block/run url = dbs_url() run_lumi = {} if isinstance(data, dict): # we got run-lumi dict for run, lumis in data.items(): run_lumi[int(run)] = lumis else: if url.find('cmsdbsprod') != -1: # DBS2 run_lumi = dbs2.run_lumi(str(data), verbose) else: if pat_dataset.match(data): params = {'dataset': data} result = get_data(dbs_url('files'), params, verbose) for row in result: params = {'logical_file_name': row['logical_file_name']} run_lumi = parse_runlumis( get_data(dbs_url('filelumis'), params, verbose)) elif pat_block.match(data): params = {'block_name': data} run_lumi = parse_runlumis( get_data(dbs_url('filelumis'), params, verbose)) elif pat_lfn.match(data): params = {'logical_file_name': data} run_lumi = parse_runlumis( get_data(dbs_url('filelumis'), params, verbose)) elif pat_run.match(data): params = {'run_num': data} run_lumi = parse_runlumis( get_data(dbs_url('filelumis'), params, verbose)) return run_lumi