def discover_blocks(options):
# Get work directory, create dbs dump directory
if os.path.isdir(options.args[0]):
workDir = os.path.abspath(os.path.normpath(options.args[0]))
else:
workDir = getConfig(configFile=options.args[0]).getWorkPath()
if not options.opts.tempdir:
options.opts.tempdir = os.path.join(workDir, 'dbs')
if not os.path.exists(options.opts.tempdir):
os.mkdir(options.opts.tempdir)
# get provider with dataset information
if options.opts.input_file:
provider = DataProvider.createInstance('ListProvider', getConfig(),
options.opts.input_file, None)
else:
config = getConfig(configDict={'dataset': options.config_dict})
provider = DataProvider.createInstance('DBSInfoProvider', config,
options.args[0], None)
blocks = provider.getBlocks(show_stats=False)
DataProvider.saveToFile(os.path.join(options.opts.tempdir, 'dbs.dat'),
blocks)
if options.opts.discovery:
sys.exit(os.EX_OK)
return blocks
def getEntries(self, path, metadata, events, seList, objStore):
datacachePath = os.path.join(objStore.get('GC_WORKDIR', ''),
'datacache.dat')
source = utils.QM((self._source == '')
and os.path.exists(datacachePath), datacachePath,
self._source)
if source and (source not in self._lfnMap):
pSource = DataProvider.createInstance('ListProvider',
createConfig(), source)
for (n, fl) in imap(
lambda b:
(b[DataProvider.Dataset], b[DataProvider.FileList]),
pSource.getBlocks()):
self._lfnMap.setdefault(source, {}).update(
dict(
imap(
lambda fi:
(self.lfnTrans(fi[DataProvider.URL]), n), fl)))
pList = set()
for key in ifilter(lambda k: k in metadata, self._parentKeys):
pList.update(
imap(
lambda pPath: self._lfnMap.get(source, {}).get(
self.lfnTrans(pPath)), metadata[key]))
metadata['PARENT_PATH'] = lfilter(identity, pList)
yield (path, metadata, events, seList, objStore)
def getEntries(self, path, metadata, events, seList, objStore):
datacachePath = os.path.join(objStore.get('GC_WORKDIR', ''), 'datacache.dat')
source = utils.QM((self._source == '') and os.path.exists(datacachePath), datacachePath, self._source)
if source and (source not in self._lfnMap):
pSource = DataProvider.createInstance('ListProvider', createConfig(), source)
for (n, fl) in imap(lambda b: (b[DataProvider.Dataset], b[DataProvider.FileList]), pSource.getBlocks()):
self._lfnMap.setdefault(source, {}).update(dict(imap(lambda fi: (self.lfnTrans(fi[DataProvider.URL]), n), fl)))
pList = set()
for key in ifilter(lambda k: k in metadata, self._parentKeys):
pList.update(imap(lambda pPath: self._lfnMap.get(source, {}).get(self.lfnTrans(pPath)), metadata[key]))
metadata['PARENT_PATH'] = lfilter(identity, pList)
yield (path, metadata, events, seList, objStore)
def discover_blocks(options):
# Get work directory, create dbs dump directory
if os.path.isdir(options.args[0]):
workDir = os.path.abspath(os.path.normpath(options.args[0]))
else:
workDir = getConfig(configFile = options.args[0]).getWorkPath()
if not options.opts.tempdir:
options.opts.tempdir = os.path.join(workDir, 'dbs')
if not os.path.exists(options.opts.tempdir):
os.mkdir(options.opts.tempdir)
# get provider with dataset information
if options.opts.input_file:
provider = DataProvider.createInstance('ListProvider', getConfig(), options.opts.input_file, None)
else:
config = getConfig(configDict = {'dataset': options.config_dict})
provider = DataProvider.createInstance('DBSInfoProvider', config, options.args[0], None)
blocks = provider.getBlocks(show_stats = False)
DataProvider.saveToFile(os.path.join(options.opts.tempdir, 'dbs.dat'), blocks)
if options.opts.discovery:
sys.exit(os.EX_OK)
return blocks
selected = JobSelector.create(opts.job_selector, task = task)
logging.info('Matching jobs: ' + str.join(' ', imap(str, jobDB.getJobsIter(selected))))
if opts.job_reset_attempts:
jobs_reset_attempts(jobDB, selected)
if opts.job_force_state:
jobs_force_state(opts, jobDB, selected)
if opts.job_show_jdl:
jobs_show_jdl(jobDB, selected)
########################################################
# DATASET INFOS
if opts.dataset_show_diff:
if len(args) != 2:
utils.exitWithUsage('%s <dataset source 1> <dataset source 2>' % sys.argv[0])
a = DataProvider.createInstance('ListProvider', config, args[0], None)
b = DataProvider.createInstance('ListProvider', config, args[1], None)
(blocksAdded, blocksMissing, blocksChanged) = DataProvider.resyncSources(a.getBlocks(show_stats = False), b.getBlocks(show_stats = False))
utils.printTabular([(DataProvider.Dataset, 'Dataset'), (DataProvider.BlockName, 'Block')], blocksMissing)
if opts.dataset_show_removed:
if len(args) < 2:
utils.exitWithUsage('%s <dataset source 1> <dataset source 2> ... <dataset source N> ' % sys.argv[0])
removed = []
oldDP = DataProvider.createInstance('ListProvider', config, args[0], None)
for new in args[1:]:
newDP = DataProvider.createInstance('ListProvider', config, new, None)
(blocksAdded, blocksMissing, blocksChanged) = DataProvider.resyncSources(oldDP.getBlocks(show_stats = False), newDP.getBlocks(show_stats = False))
for block in blocksMissing:
tmp = dict(block)
tmp[-1] = new
def __init__(self, config):
InfoScanner.__init__(self, config)
dsPath = config.get('source dataset path')
self._source = DataProvider.createInstance('ListProvider', config,
dsPath)
def __init__(self, config):
InfoScanner.__init__(self, config)
dsPath = config.get('source dataset path')
self._source = DataProvider.createInstance('ListProvider', config, dsPath)
def __init__(self, config):
InfoScanner.__init__(self, config)
dsPath = config.get('source dataset path', onChange=triggerDataResync)
self._source = DataProvider.createInstance('ListProvider', config,
dsPath)
