class DBS3LiteClient(object):
def __init__(self, url):
self._reader_url = '%s/%s' % (url, 'DBSReader')
self._writer_url = '%s/%s' % (url, 'DBSWriter')
self._migrate_url = '%s/%s' % (url, 'DBSMigrate')
self._gjrc = GridJSONRestClient(
get_cms_cert(create_config()),
cert_error_msg='VOMS proxy needed to query DBS3!',
cert_error_cls=UserError)
def get_dbs_block_list(self, **kwargs):
return self._gjrc.get(url=self._reader_url,
api='blocks',
params=kwargs)
def get_dbs_file_list(self, **kwargs):
return self._gjrc.get(url=self._reader_url, api='files', params=kwargs)
def insert_dbs_block_dump(self, data):
return self._gjrc.post(url=self._writer_url,
api='bulkblocks',
data=data)
def migration_request_status(self, **kwargs):
return self._gjrc.get(url=self._migrate_url,
api='status',
params=kwargs)
def migration_request_submit(self, data):
return self._gjrc.post(url=self._migrate_url, api='submit', data=data)
class DBS3LiteClient(object):
def __init__(self, url):
self._reader_url = '%s/%s' % (url, 'DBSReader')
self._writer_url = '%s/%s' % (url, 'DBSWriter')
self._migrate_url = '%s/%s' % (url, 'DBSMigrate')
self._gjrc = GridJSONRestClient(
cert_errror_msg='VOMS proxy needed to query DBS3!',
cert_errror_cls=UserError)
def listBlocks(self, **kwargs):
return self._gjrc.get(url=self._reader_url,
api='blocks',
params=kwargs)
def listFiles(self, **kwargs):
return self._gjrc.get(url=self._reader_url, api='files', params=kwargs)
def listFileParents(self, **kwargs):
return self._gjrc.get(url=self._reader_url,
api='fileparents',
params=kwargs)
def insertBulkBlock(self, data):
return self._gjrc.post(url=self._writer_url,
api='bulkblocks',
data=data)
def migrateSubmit(self, data):
return self._gjrc.post(url=self._migrate_url, api='submit', data=data)
def migrateStatus(self, **kwargs):
return self._gjrc.get(url=self._migrate_url,
api='status',
params=kwargs)
class SiteDB(object):
def __init__(self, url = None):
self._gjrc = GridJSONRestClient(url or 'https://cmsweb.cern.ch/sitedb/data/prod', 'VOMS proxy needed to query siteDB!')
def _query(self, api, **kwargs):
data = self._gjrc.get(api = api, params = kwargs or None)
columns = data['desc']['columns']
for row in data['result']:
yield dict(izip(columns, row))
def cms_name_to_se(self, cms_name):
cms_name_regex = re.compile(cms_name.replace('*', '.*').replace('%', '.*'))
psn_site_names = ifilter(lambda site: site['type'] == 'psn' and cms_name_regex.match(site['alias']), self._query('site-names'))
site_names = set(imap(lambda x: x['site_name'], psn_site_names))
return lmap(lambda x: x['fqdn'], ifilter(lambda x: (x['type'] == 'SE') and (x['site_name'] in site_names), self._query('site-resources')))
def se_to_cms_name(self, se):
site_names = []
for site_resource in ifilter(lambda resources: resources['fqdn'] == se, self._query('site-resources')):
site_names.extend(self._query('site-names', match = site_resource['site_name']))
return lmap(lambda x: x['alias'], ifilter(lambda site: site['type'] == 'cms', site_names))
def dn_to_username(self, dn):
for user in ifilter(lambda this_user: this_user['dn'] == dn, self._query('people')):
return user['username']
def username_to_dn(self, username):
for user in self._query('people', match = username):
return user['dn']
class DBS3Provider(CMSProvider):
alias = ['dbs3', 'dbs']
def __init__(self, config, datasetExpr, datasetNick=None, datasetID=0):
CMSProvider.__init__(self, config, datasetExpr, datasetNick, datasetID)
url_global_inst = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
if self._url == '':
self._url = url_global_inst
elif '/' not in self._url: # assume prod instance
self._url = 'https://cmsweb.cern.ch/dbs/prod/%s/DBSReader' % self._url
elif not self._url.startswith('http'): # eg. prod/phys03
self._url = 'https://cmsweb.cern.ch/dbs/%s/DBSReader' % self._url
self.usePhedex = (self._url == url_global_inst
) # Use DBS locality for private samples
self._gjrc = GridJSONRestClient(self._url,
'VOMS proxy needed to query DBS3!',
UserError)
def queryDBSv3(self, api, **kwargs):
return self._gjrc.get(api=api, params=kwargs)
def getCMSDatasets(self, datasetPath):
pd, sd, dt = (datasetPath.lstrip('/') + '/*/*/*').split('/')[:3]
tmp = self.queryDBSv3('datasets',
primary_ds_name=pd,
processed_ds_name=sd,
data_tier_name=dt)
return lmap(lambda x: x['dataset'], tmp)
def getCMSBlocksImpl(self, datasetPath, getSites):
def getNameSEList(blockinfo):
if getSites:
return (blockinfo['block_name'],
[blockinfo['origin_site_name']])
return (blockinfo['block_name'], None)
return lmap(
getNameSEList,
self.queryDBSv3('blocks', dataset=datasetPath, detail=getSites))
def getCMSFilesImpl(self, blockPath, onlyValid, queryLumi):
for fi in self.queryDBSv3('files', block_name=blockPath, detail=True):
if onlyValid and (fi['is_file_valid'] != 1):
continue
yield ({
DataProvider.URL: fi['logical_file_name'],
DataProvider.NEntries: fi['event_count']
}, None)
def getCMSLumisImpl(self, blockPath):
result = {}
for lumiInfo in self.queryDBSv3('filelumis', block_name=blockPath):
tmp = (int(lumiInfo['run_num']),
lmap(int, lumiInfo['lumi_section_num']))
result.setdefault(lumiInfo['logical_file_name'], []).append(tmp)
return result
def getBlocksInternal(self):
return self.getGCBlocks(usePhedex=self.usePhedex)
class SiteDB(object):
query_cache = {}
def __init__(self, url=None):
self._url = url or 'https://cmsweb.cern.ch/sitedb/data/prod'
self._gjrc = GridJSONRestClient(get_cms_cert(), self._url,
'VOMS proxy needed to query siteDB!')
def cms_name_to_se(self, cms_name):
cms_name_regex = re.compile(
cms_name.replace('*', '.*').replace('%', '.*'))
def _select_psn_site(site):
return site['type'] == 'psn' and cms_name_regex.match(
site['alias'])
psn_site_names = ifilter(_select_psn_site, self._query('site-names'))
site_names = set(imap(lambda x: x['site_name'], psn_site_names))
def _select_se(resource):
return (resource['type'] == 'SE') and (resource['site_name']
in site_names)
return lmap(lambda x: x['fqdn'],
ifilter(_select_se, self._query('site-resources')))
def dn_to_username(self, dn):
for user in ifilter(lambda this_user: this_user['dn'] == dn,
self._query('people')):
return user['username']
def se_to_cms_name(self, se_name):
site_names = []
resource_iter = self._query('site-resources')
for site_resource in ifilter(
lambda resource: resource['fqdn'] == se_name, resource_iter):
site_names.extend(
self._query('site-names', match=site_resource['site_name']))
return lmap(lambda x: x['alias'],
ifilter(lambda site: site['type'] == 'cms', site_names))
def username_to_dn(self, username):
for user in self._query('people', match=username):
return user['dn']
def _query(self, api, **kwargs):
key = (self._url, api, tuple(kwargs.items()))
if key not in SiteDB.query_cache:
SiteDB.query_cache[key] = self._gjrc.get(api=api,
params=kwargs or None)
data = SiteDB.query_cache[key]
columns = data['desc']['columns']
for row in data['result']:
yield dict(izip(columns, row))
class DBS3LiteClient(object): def __init__(self, url): self._reader_url = '%s/%s' % (url, 'DBSReader') self._writer_url = '%s/%s' % (url, 'DBSWriter') self._migrate_url = '%s/%s' % (url, 'DBSMigrate') self._gjrc = GridJSONRestClient(get_cms_cert(create_config()), cert_error_msg='VOMS proxy needed to query DBS3!', cert_error_cls=UserError) def get_dbs_block_list(self, **kwargs): return self._gjrc.get(url=self._reader_url, api='blocks', params=kwargs) def get_dbs_file_list(self, **kwargs): return self._gjrc.get(url=self._reader_url, api='files', params=kwargs) def insert_dbs_block_dump(self, data): return self._gjrc.post(url=self._writer_url, api='bulkblocks', data=data) def migration_request_status(self, **kwargs): return self._gjrc.get(url=self._migrate_url, api='status', params=kwargs) def migration_request_submit(self, data): return self._gjrc.post(url=self._migrate_url, api='submit', data=data)
class DBS3Provider(CMSBaseProvider):
alias = ['dbs3', 'dbs']
def __init__(self, config, datasetExpr, datasetNick = None, datasetID = 0):
CMSBaseProvider.__init__(self, config, datasetExpr, datasetNick, datasetID)
url_global_inst = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
if self._url == '':
self._url = url_global_inst
elif '/' not in self._url: # assume prod instance
self._url = 'https://cmsweb.cern.ch/dbs/prod/%s/DBSReader' % self._url
elif not self._url.startswith('http'): # eg. prod/phys03
self._url = 'https://cmsweb.cern.ch/dbs/%s/DBSReader' % self._url
self._usePhedex = (self._url == url_global_inst) # Use DBS locality for private samples
self._gjrc = GridJSONRestClient(self._url, 'VOMS proxy needed to query DBS3!', UserError)
def queryDBSv3(self, api, **kwargs):
return self._gjrc.get(api = api, params = kwargs)
def getCMSDatasets(self, datasetPath):
pd, sd, dt = (datasetPath.lstrip('/') + '/*/*/*').split('/')[:3]
tmp = self.queryDBSv3('datasets', primary_ds_name = pd, processed_ds_name = sd, data_tier_name = dt)
return lmap(lambda x: x['dataset'], tmp)
def getCMSBlocksImpl(self, datasetPath, getSites):
def getNameSEList(blockinfo):
if getSites:
return (blockinfo['block_name'], [blockinfo['origin_site_name']])
return (blockinfo['block_name'], None)
return lmap(getNameSEList, self.queryDBSv3('blocks', dataset = datasetPath, detail = getSites))
def getCMSFilesImpl(self, blockPath, onlyValid, queryLumi):
for fi in self.queryDBSv3('files', block_name = blockPath, detail = True):
if onlyValid and (fi['is_file_valid'] != 1):
continue
yield ({DataProvider.URL: fi['logical_file_name'], DataProvider.NEntries: fi['event_count']}, None)
def getCMSLumisImpl(self, blockPath):
result = {}
for lumiInfo in self.queryDBSv3('filelumis', block_name = blockPath):
tmp = (int(lumiInfo['run_num']), lmap(int, lumiInfo['lumi_section_num']))
result.setdefault(lumiInfo['logical_file_name'], []).append(tmp)
return result
def getBlocksInternal(self):
return self.getGCBlocks(usePhedex = self._usePhedex)
class DBS3Provider(CMSBaseProvider):
# required format: <dataset path>[@<instance>][#<block>]
alias_list = ['dbs3', 'dbs']
def __init__(self, config, datasource_name, dataset_expr, dataset_nick=None, dataset_proc=None):
CMSBaseProvider.__init__(self, config, datasource_name, dataset_expr, dataset_nick, dataset_proc)
if self._dataset_instance.startswith('http'):
self._url = self._dataset_instance
else:
self._url = 'https://cmsweb.cern.ch/dbs/%s/DBSReader' % self._dataset_instance
# Use DBS locality for private samples
self._use_phedex = (self._url == 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader')
self._gjrc = GridJSONRestClient(get_cms_cert(config), self._url,
'VOMS proxy needed to query DBS3!', UserError)
def _get_cms_dataset_list(self, dataset_path):
dataset_path_parts = (dataset_path.lstrip('/') + '/*/*/*').split('/')[:3]
primary_ds_name, processed_ds_name, data_tier_name = dataset_path_parts
tmp = self._query_dbsv3('datasets', primary_ds_name=primary_ds_name,
processed_ds_name=processed_ds_name, data_tier_name=data_tier_name)
return lmap(lambda x: x['dataset'], tmp)
def _get_cms_lumi_dict(self, block_path):
result = {}
for lumi_info_dict in self._query_dbsv3('filelumis', block_name=block_path):
tmp = (int(lumi_info_dict['run_num']), lmap(int, lumi_info_dict['lumi_section_num']))
result.setdefault(lumi_info_dict['logical_file_name'], []).append(tmp)
return result
def _iter_blocks_raw(self):
return self._get_gc_block_list(use_phedex=self._use_phedex)
def _iter_cms_blocks(self, dataset_path, do_query_sites):
def _get_name_locationinfo_list(blockinfo):
if do_query_sites:
return (blockinfo['block_name'], [(blockinfo['origin_site_name'], None, True)])
return (blockinfo['block_name'], None)
return lmap(_get_name_locationinfo_list,
self._query_dbsv3('blocks', dataset=dataset_path, detail=do_query_sites))
def _iter_cms_files(self, block_path, query_only_valid, query_lumi):
for cms_fi in self._query_dbsv3('files', block_name=block_path, detail=True):
if (cms_fi['is_file_valid'] == 1) or not query_only_valid:
fi = {DataProvider.URL: cms_fi['logical_file_name'],
DataProvider.NEntries: cms_fi['event_count']}
yield (fi, None)
def _query_dbsv3(self, api, **kwargs):
return self._gjrc.get(api=api, params=kwargs)
class DBS3LiteClient(object): def __init__(self, url): self._reader_url = '%s/%s' % (url, 'DBSReader') self._writer_url = '%s/%s' % (url, 'DBSWriter') self._migrate_url = '%s/%s' % (url, 'DBSMigrate') self._gjrc = GridJSONRestClient(cert_errror_msg = 'VOMS proxy needed to query DBS3!', cert_errror_cls = UserError) def listBlocks(self, **kwargs): return self._gjrc.get(url=self._reader_url, api='blocks', params=kwargs) def listFiles(self, **kwargs): return self._gjrc.get(url=self._reader_url, api='files', params=kwargs) def listFileParents(self, **kwargs): return self._gjrc.get(url=self._reader_url, api='fileparents', params=kwargs) def insertBulkBlock(self, data): return self._gjrc.post(url=self._writer_url, api='bulkblocks', data=data) def migrateSubmit(self, data): return self._gjrc.post(url=self._migrate_url, api='submit', data=data) def migrateStatus(self, **kwargs): return self._gjrc.get(url=self._migrate_url, api='status', params=kwargs)
class SiteDB(object):
def __init__(self, url=None):
self._gjrc = GridJSONRestClient(
url or 'https://cmsweb.cern.ch/sitedb/data/prod',
'VOMS proxy needed to query siteDB!')
def _query(self, api, **kwargs):
data = self._gjrc.get(api=api, params=kwargs or None)
columns = data['desc']['columns']
for row in data['result']:
yield dict(izip(columns, row))
def cms_name_to_se(self, cms_name):
cms_name_regex = re.compile(
cms_name.replace('*', '.*').replace('%', '.*'))
psn_site_names = ifilter(
lambda site: site['type'] == 'psn' and cms_name_regex.match(site[
'alias']), self._query('site-names'))
site_names = set(imap(lambda x: x['site_name'], psn_site_names))
return lmap(
lambda x: x['fqdn'],
ifilter(
lambda x:
(x['type'] == 'SE') and (x['site_name'] in site_names),
self._query('site-resources')))
def se_to_cms_name(self, se):
site_names = []
for site_resource in ifilter(lambda resources: resources['fqdn'] == se,
self._query('site-resources')):
site_names.extend(
self._query('site-names', match=site_resource['site_name']))
return lmap(lambda x: x['alias'],
ifilter(lambda site: site['type'] == 'cms', site_names))
def dn_to_username(self, dn):
for user in ifilter(lambda this_user: this_user['dn'] == dn,
self._query('people')):
return user['username']
def username_to_dn(self, username):
for user in self._query('people', match=username):
return user['dn']
class CRIC(object):
query_cache = {}
def __init__(self, url = None):
if url:
raise Exception('Custom url currently not supportet, please contact the grid-control developers!')
self._url = url or 'http://cms-cric.cern.ch/api'
self._gjrc = GridJSONRestClient(get_cms_cert(), self._url, 'VOMS proxy needed to query siteDB!')
self._url_people = 'http://cms-cric.cern.ch/api/accounts/user/query/?json&preset=people'
self._gjrc_people = GridJSONRestClient(get_cms_cert(), self._url_people, 'VOMS proxy needed to query CRIC!')
self._url_names = 'http://cms-cric.cern.ch/api/cms/site/query/?json&preset=site-names'
self._gjrc_names = GridJSONRestClient(get_cms_cert(), self._url_names, 'VOMS proxy needed to query CRIC!')
self._url_recources = 'http://wlcg-cric.cern.ch/api/core/service/query/?json&groupby=rcsite'
self._gjrc_recources = GridJSONRestClient(get_cms_cert(), self._url_recources, 'VOMS proxy needed to query siteDB!')
def cms_name_to_se(self, cms_name):
cms_name_regex = re.compile(cms_name.replace('*', '.*').replace('%', '.*'))
def _select_psn_site(site):
return site['type'] == 'psn' and cms_name_regex.match(site['alias'])
psn_site_names = ifilter(_select_psn_site, self._query('site-names'))
site_aliases = set(imap(lambda x: x['alias'], psn_site_names))
def _select_se(resource):
return (resource['type'] == 'SE') and (resource['alias'] in site_aliases)
return lmap(lambda x: x['fqdn'], ifilter(_select_se, self._query('site-resources')))
def dn_to_username(self, dn):
for user in ifilter(lambda this_user: this_user['dn'] == dn, self._query('people')):
return user['username']
def se_to_cms_name(self, se_name):
site_names = []
resource_iter = self._query('site-resources')
for site_resource in ifilter(lambda resource: resource['fqdn'] == se_name, resource_iter):
site_names.extend(self._query('site-names', name=site_resource['alias']))
return lmap(lambda x: x['alias'], ifilter(lambda site: site['type'] == 'cms', site_names))
def username_to_dn(self, username):
for user in self._query('people', username=username):
return user['dn']
def _query(self, api, **kwargs):
key = (self._url, api, tuple(kwargs.items()))
if key not in CRIC.query_cache:
if api == 'people':
CRIC.query_cache[key] = self._gjrc_people.get(api=None, params=kwargs or None)
elif api == 'site-names':
CRIC.query_cache[key] = self._gjrc_names.get(api=None, params=kwargs or None)
elif api == 'site-resources':
CRIC.query_cache[key] = self._gjrc_recources.get(api=None, params=kwargs or None)
else:
CRIC.query_cache[key] = self._gjrc.get(api=api, params=kwargs or None)
data = CRIC.query_cache[key]
# workaround for site-resources query
name = ''
fqdn = ''
thetype = ''
flavour = ''
alias = ''
if api == 'site-resources':
for d in data.keys():
name = d
for e in data[d]:
if 'cms' in e['usage'].keys() and len(e['usage']['cms'])>0:
alias = e['usage']['cms'][0]
thetype = e['type']
if thetype == 'CE':
fqdn = e['endpoint'].split(':')[0]
flavour = e['flavour']
yield {'name':name, 'type':thetype,'fqdn':fqdn,'flavour':flavour,'alias': alias}
elif thetype == 'SE':
for pk in e['protocols'].keys():
p = e['protocols'][pk]
fqdn = p['endpoint'].split(':')[0]
flavour = p['flavour']
yield {'name':name, 'type':thetype,'fqdn':fqdn,'flavour':flavour,'alias': alias}
else:
columns = data['desc']['columns']
for row in data['result']:
yield dict(izip(columns, row))