def create_dbs3_json_blocks(opts, dataset_blocks):
for (block, block_dump, block_size, dataset_type) in create_dbs3_proto_blocks(opts, dataset_blocks):
dataset = block[DataProvider.Dataset]
primary_dataset, processed_dataset, data_tier = dataset[1:].split('/')
# add primary dataset information
block_dump['primds'] = {'primary_ds_type': dataset_type, 'primary_ds_name': primary_dataset}
# add dataset information
block_dump['dataset'] = {
'dataset': dataset, 'processed_ds_name': processed_dataset, 'data_tier_name': data_tier,
'physics_group_name': None, 'dataset_access_type': 'VALID',
'xtcrosssection': None, # TODO: Add to meta data from FrameWorkJobReport, if possible!
}
# add block information
site_db = SiteDB()
try:
origin_site_name = site_db.se_to_cms_name(block[DataProvider.Locations][0])[0]
except IndexError:
origin_site_name = 'UNKNOWN'
block_dump['block'] = {'block_name': '%s#%s' % (dataset, block[DataProvider.BlockName]),
'file_count': len(block[DataProvider.FileList]),
'block_size': block_size, 'origin_site_name': origin_site_name}
if opts.do_close_blocks:
block_dump['block']['open_for_writing'] = 0
else:
block_dump['block']['open_for_writing'] = 1
# add acquisition_era, CRAB is important because of checks within DBS 3
block_dump['acquisition_era'] = {'acquisition_era_name': 'CRAB', 'start_date': 0}
# add processing_era
block_dump['processing_era'] = {'processing_version': 1, 'description': 'grid-control'}
yield validate_dbs3_json('blockBulk', block_dump)
from grid_control.gc_exceptions import RuntimeError
from grid_control.utils.webservice import readJSON
from grid_control_cms.provider_sitedb import SiteDB
def lfn2pfn(node, lfn):
return readJSON('https://cmsweb.cern.ch/phedex/datasvc/json/prod/lfn2pfn',
{'node': node, 'protocol': 'srmv2', 'lfn': lfn})['phedex']['mapping'][0]['pfn']
parser = optparse.OptionParser()
parser.add_option('-s', '--SE', dest='SE', default=None, help='Resolve LFN on CMS SE into PFN')
parser.add_option('', '--lfn', dest='lfn', default='/store/user/<hypernews name>', help='Name of default LFN')
parser.add_option('', '--se-prot', dest='seprot', default='srmv2', help='Name of default SE protocol')
(opts, args) = parseOptions(parser)
if opts.SE:
if '<hypernews name>' in opts.lfn:
token = AccessToken.getInstance('VomsProxy', getConfig(), None)
site_db = SiteDB()
hnName = site_db.dn_to_username(dn=token.getFQUsername())
if not hnName:
raise RuntimeError('Unable to map grid certificate to hypernews name!')
opts.lfn = opts.lfn.replace('<hypernews name>', hnName)
tmp = readJSON('https://cmsweb.cern.ch/phedex/datasvc/json/prod/lfn2pfn',
{'node': opts.SE, 'protocol': opts.seprot, 'lfn': opts.lfn})['phedex']['mapping']
for entry in tmp:
if len(tmp) > 1:
print entry['node'],
print entry['pfn']
def generateDBS3BlockDumps(opts, blocks):
for block_info in blocks:
block_dump = dict(dataset_conf_list=[], files=[], file_conf_list=[], file_parent_list=[])
locations = block_info[DataProvider.Locations]
dataset = block_info[DataProvider.Dataset]
primary_dataset, processed_dataset, data_tier = dataset[1:].split('/')
block_name = block_info[DataProvider.BlockName]
file_list = block_info[DataProvider.FileList]
block_size = 0
dataset_configurations = block_dump[u'dataset_conf_list']
for file_info in file_list:
metadata_info = dict(zip(block_info[DataProvider.Metadata], file_info[DataProvider.Metadata]))
parent_lfns = metadata_info['CMSSW_PARENT_LFN']
dataset_type = metadata_info['CMSSW_DATATYPE']
file_size = metadata_info['SE_OUTPUT_SIZE']
lfn = file_info[DataProvider.URL]
file_lumi_list = [{u'lumi_section_num': lumi_num, u'run_num': run_num} for run_num, lumi_num in
metadata_info['CMSSW_LUMIS']]
###add file information
file_dict = {u'check_sum': metadata_info['SE_OUTPUT_HASH_CRC32'],
u'file_lumi_list': file_lumi_list,
u'adler32': 'NOTSET',
u'event_count': metadata_info['CMSSW_EVENTS_WRITE'],
u'file_type': 'EDM',
u'logical_file_name': lfn,
u'file_size': file_size,
u'md5': metadata_info['SE_OUTPUT_HASH_MD5'],
u'auto_cross_section': 0.0
}
block_dump[u'files'].append(file_dict)
###add file configurations
file_conf_dict = {u'release_version': metadata_info['CMSSW_VERSION'],
u'pset_hash': metadata_info['CMSSW_CONFIG_HASH'],
u'lfn': lfn,
u'app_name': 'cmsRun',
u'output_module_label': 'crab2_mod_label',
u'global_tag': metadata_info['CMSSW_GLOBALTAG']#,#default=opts.globaltag)
}
block_dump[u'file_conf_list'].append(file_conf_dict)
###add file parentage information
file_parentage = [{'logical_file_name': lfn,
'parent_logical_file_name': parent_lfn} for parent_lfn in parent_lfns]
block_dump[u'file_parent_list'].extend(file_parentage)
###fill dataset configurations, same as file configurations with removed lfn
dataset_conf_dict = dict(file_conf_dict)
del dataset_conf_dict[u'lfn']
if dataset_conf_dict not in dataset_configurations:
dataset_configurations.append(dataset_conf_dict)
###update block size for block summary information
block_size += file_size
###add primary dataset information
block_dump[u'primds'] = {u'primary_ds_type': dataset_type,
u'primary_ds_name': primary_dataset}
###add dataset information
block_dump[u'dataset'] = {u'physics_group_name': None,
u'dataset_access_type': 'VALID',
u'data_tier_name': data_tier,
u'processed_ds_name': processed_dataset,
u'xtcrosssection': None,###Add to meta data from FrameWorkJobReport, if possible!
u'dataset': dataset
}
###add block information
site_db = SiteDB()
try:
origin_site_name = site_db.se_to_cms_name(locations[0])[0]
except IndexError:
origin_site_name = 'UNKNOWN'
block_dump[u'block'] = {u'open_for_writing': 0,
u'block_name': '%s#%s' % (dataset, block_name),
u'file_count': len(file_list),
u'block_size': block_size,
u'origin_site_name': origin_site_name}
###add acquisition_era, CRAB is important because of checks within DBS 3
block_dump[u'acquisition_era'] = {u'acquisition_era_name': 'CRAB',
u'start_date': 0}
###add processing_era
block_dump[u'processing_era'] = {u'processing_version': 1,
u'description': 'grid-control'}
yield DBS3InputValidation.validate_json_input('blockBulk', block_dump)
from gcSupport import Options, Plugin, getConfig, scriptOptions
from grid_control.utils.webservice import readJSON
from grid_control_cms.provider_sitedb import SiteDB
def lfn2pfn(node, lfn):
return readJSON('https://cmsweb.cern.ch/phedex/datasvc/json/prod/lfn2pfn',
{'node': node, 'protocol': 'srmv2', 'lfn': lfn})['phedex']['mapping'][0]['pfn']
parser = Options()
parser.addText(None, 's', 'SE', default = None, help = 'Resolve LFN on CMS SE into PFN')
parser.addText(None, ' ', 'se-prot', default = 'srmv2', help = 'Name of default SE protocol')
parser.addText(None, ' ', 'lfn', default = '/store/user/<hypernews name>', help = 'Name of default LFN')
options = scriptOptions(parser)
if options.opts.SE:
if '<hypernews name>' in options.opts.lfn:
token = Plugin.getClass('AccessToken').createInstance('VomsProxy', getConfig(), None)
site_db = SiteDB()
hnName = site_db.dn_to_username(dn=token.getFQUsername())
if not hnName:
raise Exception('Unable to map grid certificate to hypernews name!')
options.opts.lfn = options.opts.lfn.replace('<hypernews name>', hnName)
tmp = readJSON('https://cmsweb.cern.ch/phedex/datasvc/json/prod/lfn2pfn',
{'node': options.opts.SE, 'protocol': options.opts.se_prot, 'lfn': options.opts.lfn})['phedex']['mapping']
for entry in tmp:
if len(tmp) > 1:
print(entry['node'] + ' ' + entry['pfn'])
print(entry['pfn'])
