def testFilterData(self):
data = list(
DataCache.filterData(filterDict={}, maskList=['RequestType']))
self.assertEqual(20, len(data))
self.assertItemsEqual([
'ReReco', 'MonteCarlo', 'StepChain', 'MonteCarloFromGEN', 'ReDigi',
'TaskChain', 'DQMHarvest'
], set(list(data)))
data = list(
DataCache.filterData(filterDict={},
maskList=['Campaign', 'RequestType']))
self.assertEqual(40, len(data))
data = list(
DataCache.filterData(filterDict={'IncludeParents': 'True'},
maskList=['Campaign']))
self.assertEqual(2, len(data))
data = list(
DataCache.filterData(
filterDict={'Campaign': 'CMSSW_9_4_0__test2inwf-1510737328'},
maskList=['RequestName']))
self.assertItemsEqual([
"amaltaro_TaskChain_InclParents_HG1812_Validation_181203_121005_1483"
], data)
def testFilterData(self):
data = list(DataCache.filterData(filterDict={}, maskList=['RequestType']))
self.assertEqual(20, len(data))
self.assertItemsEqual(['ReReco', 'MonteCarlo', 'StepChain', 'MonteCarloFromGEN',
'ReDigi', 'TaskChain', 'DQMHarvest'], set(list(data)))
data = list(DataCache.filterData(filterDict={}, maskList=['Campaign', 'RequestType']))
self.assertEqual(40, len(data))
data = list(DataCache.filterData(filterDict={'IncludeParents': 'True'}, maskList=['Campaign']))
self.assertEqual(2, len(data))
data = list(DataCache.filterData(filterDict={'Campaign': 'CMSSW_9_4_0__test2inwf-1510737328'},
maskList=['RequestName']))
self.assertItemsEqual(["amaltaro_TaskChain_InclParents_HG1812_Validation_181203_121005_1483"],
data)
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows(
DataCache.filterData(
ACTIVE_NO_CLOSEOUT_FILTER,
["InputDataset", "OutputDatasets", "MCPileup", "DataPileup"]))
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
if DataCache.isEmpty():
raise DataCacheEmpty()
else:
return rows(
DataCache.filterData(ACTIVE_STATUS_FILTER,
["OutputModulesLFNBases"]))
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
if DataCache.isEmpty():
raise DataCacheEmpty()
else:
return rows(
DataCache.filterData(ACTIVE_STATUS_FILTER, [
"InputDataset", "OutputDatasets", "MCPileup", "DataPileup"
]))
def fetchIncludeParentsRequests(self, config):
"""
Fetch active requests from the DataCache that have IncludeParents=True
"""
# use this boolean to signal whether there were datasets that failed
# to get their parentage resolved
incompleteParentage = False
setDsets = set()
setParents = set()
self.logger.info("Executing parent lock cherrypy thread")
for inputDset in DataCache.filterData(ACTIVE_NO_CLOSEOUT_PARENT_FILTER,
["InputDataset"]):
setDsets.add(inputDset)
self.logger.info(
"Found %d unique datasets requiring the parent dataset",
len(setDsets))
for dset in setDsets:
try:
res = self.dbs.listDatasetParents(dset)
except Exception as exc:
self.logger.warning(
"Failed to resolve parentage for: %s. Error: %s", dset,
str(exc))
incompleteParentage = True
continue
self.logger.info("Resolved parentage for: %s", res)
if res:
setParents.add(res[0]['parent_dataset'])
if not incompleteParentage:
DataCache.setParentDatasetList(list(setParents))
self.logger.info("Parentage lookup complete and cache renewed")
else:
# then don't replace any data for the moment, simply add new parents
previousData = set(DataCache.getParentDatasetList())
setParents = setParents | previousData
DataCache.setParentDatasetList(list(setParents))
self.logger.info("Parentage lookup complete and cache updated")
return
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows(DataCache.filterData(ACTIVE_STATUS_FILTER, ["OutputModulesLFNBases"]))
def get(self):
# This assumes DataCahe is periodically updated.
# If data is not updated, need to check, dataCacheUpdate log
return rows(DataCache.filterData(ACTIVE_NO_CLOSEOUT_FILTER,
["InputDataset", "OutputDatasets", "MCPileup", "DataPileup"]))
