def main(opts, args):
if len(args) == 0:
utils.exitWithUsage('Dataset path not specified!')
datasetPath = args[0]
if '*' in datasetPath:
dbs3 = Plugin.createInstance('DBS3Provider', getConfig(), datasetPath, None)
toProcess = dbs3.getCMSDatasetsImpl(datasetPath)
else:
toProcess = [datasetPath]
nProd = Plugin.getClass('NickNameProducer').createInstance(opts.producer, getConfig())
utils.printTabular(
[(0, 'Nickname'), (1, 'Dataset')],
lmap(lambda ds: {0: nProd.getName('', ds, None), 1: ds}, toProcess), 'll')
def discoverDataset(providerName, config_dict):
config = getConfig(configDict = {'dataset': config_dict})
DataProvider = Plugin.getClass('DataProvider')
provider = DataProvider.createInstance(providerName, config, config_dict['dataset'], None)
if config_dict['output']:
return DataProvider.saveToFile(config_dict['output'], provider.getBlocks(), config_dict['strip'])
return DataProvider.saveToStream(sys.stdout, provider.getBlocks(), config_dict['strip'])
def main(opts, args):
if len(args) == 0:
utils.exitWithUsage('Dataset path not specified!')
datasetPath = args[0]
if '*' in datasetPath:
dbs3 = Plugin.createInstance('DBS3Provider', getConfig(), datasetPath,
None)
toProcess = dbs3.getCMSDatasetsImpl(datasetPath)
else:
toProcess = [datasetPath]
nProd = Plugin.getClass('NickNameProducer').createInstance(
opts.producer, getConfig())
utils.printTabular([(0, 'Nickname'), (1, 'Dataset')],
lmap(lambda ds: {
0: nProd.getName('', ds, None),
1: ds
}, toProcess), 'll')
def discoverDataset(providerName, config_dict):
config = getConfig(configDict={'dataset': config_dict})
DataProvider = Plugin.getClass('DataProvider')
provider = DataProvider.createInstance(providerName, config,
config_dict['dataset'], None)
if config_dict['output']:
return DataProvider.saveToFile(config_dict['output'],
provider.getBlocks(),
config_dict['strip'])
return DataProvider.saveToStream(sys.stdout, provider.getBlocks(),
config_dict['strip'])
def discoverDataset(providerName, config_dict):
config = getConfig(configDict = {'dataset': config_dict})
if config_dict['dump config'] == 'True':
config.write(sys.stdout, printDefault = False, printMinimal = True)
return
DataProvider = Plugin.getClass('DataProvider')
provider = DataProvider.createInstance(providerName, config, config_dict['dataset'], None)
stripMetadata = config_dict['strip'] == 'True'
if config_dict['output']:
return DataProvider.saveToFile(config_dict['output'], provider.getBlocks(), stripMetadata)
return DataProvider.saveToStream(sys.stdout, provider.getBlocks(), stripMetadata)
def discoverDataset(providerName, config_dict):
config = getConfig(configDict={'dataset': config_dict})
if config_dict['dump config'] == 'True':
config.write(sys.stdout, printDefault=False, printMinimal=True)
return
DataProvider = Plugin.getClass('DataProvider')
provider = DataProvider.createInstance(providerName, config,
config_dict['dataset'], None)
stripMetadata = config_dict['strip'] == 'True'
if config_dict['output']:
return DataProvider.saveToFile(config_dict['output'],
provider.getBlocks(), stripMetadata)
return DataProvider.saveToStream(sys.stdout, provider.getBlocks(),
stripMetadata)
def loop_download(opts, args):
# Init everything in each loop to pick up changes
(config, jobDB) = gcSupport.initGC(args)
token = Plugin.getClass('AccessToken').createInstance(
opts.token, config, 'access') #, OSLayer.create(config))
workDir = config.getWorkPath()
jobList = jobDB.getJobs(ClassSelector(JobClass.SUCCESS))
# Create SE output dir
if not opts.output:
opts.output = os.path.join(workDir, 'se_output')
if '://' not in opts.output:
opts.output = 'file:///%s' % os.path.abspath(opts.output)
infos = {}
def incInfo(x):
infos[x] = infos.get(x, 0) + 1
if opts.shuffle:
random.shuffle(jobList)
else:
jobList.sort()
if int(opts.threads):
download_multithreaded(opts, workDir, jobList, incInfo, jobDB, token)
else:
download_sequential(opts, workDir, jobList, incInfo, jobDB, token)
# Print overview
if infos:
print('\nStatus overview:')
for (state, num) in infos.items():
if num > 0:
print('\t%20s: [%d/%d]' % (state, num, len(jobList)))
# return True if download is finished
return ('Downloaded' in infos) and (infos['Downloaded'] == len(jobDB))
def loop_download(opts, args):
# Init everything in each loop to pick up changes
(config, jobDB) = gcSupport.initGC(args)
token = Plugin.getClass('AccessToken').createInstance(opts.token, config, 'access')#, OSLayer.create(config))
workDir = config.getWorkPath()
jobList = jobDB.getJobs(ClassSelector(JobClass.SUCCESS))
# Create SE output dir
if not opts.output:
opts.output = os.path.join(workDir, 'se_output')
if '://' not in opts.output:
opts.output = 'file:///%s' % os.path.abspath(opts.output)
infos = {}
def incInfo(x):
infos[x] = infos.get(x, 0) + 1
if opts.shuffle:
random.shuffle(jobList)
else:
jobList.sort()
if int(opts.threads):
download_multithreaded(opts, workDir, jobList, incInfo, jobDB, token)
else:
download_sequential(opts, workDir, jobList, incInfo, jobDB, token)
# Print overview
if infos:
print('\nStatus overview:')
for (state, num) in infos.items():
if num > 0:
print('\t%20s: [%d/%d]' % (state, num, len(jobList)))
# return True if download is finished
return ('Downloaded' in infos) and (infos['Downloaded'] == len(jobDB))
parser.section('data', 'Dataset debugging', '%s <dataset file> <dataset file> ...')
parser.addText('data', '', 'dataset-show-diff', default='', help='Show difference between datasets')
parser.addText('data', '', 'dataset-show-removed', default='', help='Find removed dataset blocks')
parser.addText(None, 'd', 'logfile-decode', default='', help='Decode log files')
options = scriptOptions(parser)
(opts, args) = (options.opts, options.args)
########################################################
# BACKEND
if opts.backend_list_nodes or opts.backend_list_queues:
config = getConfig()
backend = str.join(' ', args) or 'local'
wms = Plugin.getClass('WMS').createInstance(backend, config, backend)
if opts.backend_list_nodes:
logging.info(repr(wms.getNodes()))
if opts.backend_list_queues:
logging.info(repr(wms.getQueues()))
########################################################
# DATASET PARTITION
def partition_invalid(splitter):
for jobNum in irange(splitter.getMaxJobs()):
splitInfo = splitter.getSplitInfo(jobNum)
if splitInfo.get(DataSplitter.Invalid, False):
yield {0: jobNum}
def partition_list(splitter, keyList):
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
import sys
from gcSupport import Activity, JobSelector, Options, Plugin, displayPluginList, getConfig, getPluginList, scriptOptions, utils
parser = Options(usage = '%s [OPTIONS] <config file>')
parser.addBool(None, 'L', 'report-list', default = False, help = 'List available report classes')
parser.addBool(None, 'T', 'use-task', default = False, help = 'Forward task information to report')
parser.addText(None, 'R', 'report', default = 'GUIReport')
parser.addText(None, 'J', 'job-selector', default = None)
parser.addText(None, ' ', 'string', default = '')
options = scriptOptions(parser)
Report = Plugin.getClass('Report')
if options.opts.report_list:
sys.stderr.write('Available report classes:\n')
displayPluginList(getPluginList('Report'))
if len(options.args) != 1:
utils.exitWithUsage(parser.usage())
def main(opts, args):
# try to open config file
config = getConfig(args[0], section = 'global')
# Initialise task module
task = None
if opts.use_task:
parser.addBool(None,
'L',
'report-list',
default=False,
help='List available report classes')
parser.addBool(None,
'T',
'use-task',
default=False,
help='Forward task information to report')
parser.addText(None, 'R', 'report', default='GUIReport')
parser.addText(None, 'J', 'job-selector', default=None)
parser.addText(None, ' ', 'string', default='')
options = scriptOptions(parser)
Report = Plugin.getClass('Report')
if options.opts.report_list:
sys.stderr.write('Available report classes:\n')
displayPluginList(getPluginList('Report'))
if len(options.args) != 1:
utils.exitWithUsage(parser.usage())
def main(opts, args):
# try to open config file
config = getConfig(args[0], section='global')
# Initialise task module
task = None
aliasList = lmap(lambda d_a: d_a[1], by_depth_name)
aliasList.remove(new_name)
entry = {'Name': new_name, 'Alias': str.join(', ', aliasList)}
if ('Multi' not in name) and ('Base' not in name):
tableList.append(entry)
return tableList
def displayList(clsList):
header = [('Name', 'Name')]
fmtString = 'l'
for entry in clsList:
if entry['Alias']:
header.append(('Alias', 'Alias'))
fmtString = 'rl'
break
utils.printTabular(header,
sorted(clsList, key=lambda x: x['Name'].lower()),
fmtString=fmtString)
if __name__ == '__main__':
BasePlugin = Plugin.getClass(options.args[0])
aliasDict = {}
for entry in BasePlugin.getClassList():
depth = entry.pop('depth', 0)
(alias, name) = entry.popitem()
aliasDict.setdefault(name, []).append((depth, alias))
aliasDict.pop(options.args[0])
displayList(getDisplayList(aliasDict))
# | limitations under the License.
from gcSupport import Options, Plugin, getConfig, scriptOptions
from grid_control.utils.webservice import JSONRestClient
from grid_control_cms.sitedb import SiteDB
def lfn2pfn(node, lfn, prot = 'srmv2'):
return JSONRestClient().get(url = 'https://cmsweb.cern.ch/phedex/datasvc/json/prod/lfn2pfn',
params = {'node': node, 'protocol': prot, 'lfn': lfn})['phedex']['mapping']
parser = Options()
parser.addText(None, 's', 'se', default = None, help = 'Resolve LFN on CMS SE into PFN')
parser.addText(None, ' ', 'se-prot', default = 'srmv2', help = 'Name of default SE protocol')
parser.addText(None, ' ', 'lfn', default = '/store/user/<hypernews name>', help = 'Name of default LFN')
options = scriptOptions(parser)
if options.opts.se:
if '<hypernews name>' in options.opts.lfn:
token = Plugin.getClass('AccessToken').createInstance('VomsProxy', getConfig(), 'token')
site_db = SiteDB()
hnName = site_db.dn_to_username(dn=token.getFQUsername())
if not hnName:
raise Exception('Unable to map grid certificate to hypernews name!')
options.opts.lfn = options.opts.lfn.replace('<hypernews name>', hnName)
tmp = lfn2pfn(node = options.opts.se, prot = options.opts.se_prot, lfn = options.opts.lfn)
for entry in tmp:
if len(tmp) > 1:
print(entry['node'] + ' ' + entry['pfn'])
print(entry['pfn'])
by_len_depth = sorted(aliasDict[name], key = lambda d_a: (-len(d_a[1]), d_a[0]))
# sorted by depth and name
by_depth_name = sorted(aliasDict[name], key = lambda d_a: (d_a[0], d_a[1]))
new_name = by_len_depth.pop()[1]
aliasList = lmap(lambda d_a: d_a[1], by_depth_name)
aliasList.remove(new_name)
entry = {'Name': new_name, 'Alias': str.join(', ', aliasList)}
if ('Multi' not in name) and ('Base' not in name):
tableList.append(entry)
return tableList
def displayList(clsList):
header = [('Name', 'Name')]
fmtString = 'l'
for entry in clsList:
if entry['Alias']:
header.append(('Alias', 'Alias'))
fmtString = 'rl'
break
utils.printTabular(header, sorted(clsList, key = lambda x: x['Name'].lower()), fmtString = fmtString)
if __name__ == '__main__':
BasePlugin = Plugin.getClass(options.args[0])
aliasDict = {}
for entry in BasePlugin.getClassList():
depth = entry.pop('depth', 0)
(alias, name) = entry.popitem()
aliasDict.setdefault(name, []).append((depth, alias))
aliasDict.pop(options.args[0])
displayList(getDisplayList(aliasDict))