def testFilterData(self): data = list( DataCache.filterData(filterDict={}, maskList=['RequestType'])) self.assertEqual(20, len(data)) self.assertItemsEqual([ 'ReReco', 'MonteCarlo', 'StepChain', 'MonteCarloFromGEN', 'ReDigi', 'TaskChain', 'DQMHarvest' ], set(list(data))) data = list( DataCache.filterData(filterDict={}, maskList=['Campaign', 'RequestType'])) self.assertEqual(40, len(data)) data = list( DataCache.filterData(filterDict={'IncludeParents': 'True'}, maskList=['Campaign'])) self.assertEqual(2, len(data)) data = list( DataCache.filterData( filterDict={'Campaign': 'CMSSW_9_4_0__test2inwf-1510737328'}, maskList=['RequestName'])) self.assertItemsEqual([ "amaltaro_TaskChain_InclParents_HG1812_Validation_181203_121005_1483" ], data)
def testFilterData(self): data = list(DataCache.filterData(filterDict={}, maskList=['RequestType'])) self.assertEqual(20, len(data)) self.assertItemsEqual(['ReReco', 'MonteCarlo', 'StepChain', 'MonteCarloFromGEN', 'ReDigi', 'TaskChain', 'DQMHarvest'], set(list(data))) data = list(DataCache.filterData(filterDict={}, maskList=['Campaign', 'RequestType'])) self.assertEqual(40, len(data)) data = list(DataCache.filterData(filterDict={'IncludeParents': 'True'}, maskList=['Campaign'])) self.assertEqual(2, len(data)) data = list(DataCache.filterData(filterDict={'Campaign': 'CMSSW_9_4_0__test2inwf-1510737328'}, maskList=['RequestName'])) self.assertItemsEqual(["amaltaro_TaskChain_InclParents_HG1812_Validation_181203_121005_1483"], data)
def get(self): # This assumes DataCahe is periodically updated. # If data is not updated, need to check, dataCacheUpdate log return rows( DataCache.filterData( ACTIVE_NO_CLOSEOUT_FILTER, ["InputDataset", "OutputDatasets", "MCPileup", "DataPileup"]))
def get(self): # This assumes DataCahe is periodically updated. # If data is not updated, need to check, dataCacheUpdate log if DataCache.isEmpty(): raise DataCacheEmpty() else: return rows( DataCache.filterData(ACTIVE_STATUS_FILTER, ["OutputModulesLFNBases"]))
def get(self): # This assumes DataCahe is periodically updated. # If data is not updated, need to check, dataCacheUpdate log if DataCache.isEmpty(): raise DataCacheEmpty() else: return rows( DataCache.filterData(ACTIVE_STATUS_FILTER, [ "InputDataset", "OutputDatasets", "MCPileup", "DataPileup" ]))
def fetchIncludeParentsRequests(self, config): """ Fetch active requests from the DataCache that have IncludeParents=True """ # use this boolean to signal whether there were datasets that failed # to get their parentage resolved incompleteParentage = False setDsets = set() setParents = set() self.logger.info("Executing parent lock cherrypy thread") for inputDset in DataCache.filterData(ACTIVE_NO_CLOSEOUT_PARENT_FILTER, ["InputDataset"]): setDsets.add(inputDset) self.logger.info( "Found %d unique datasets requiring the parent dataset", len(setDsets)) for dset in setDsets: try: res = self.dbs.listDatasetParents(dset) except Exception as exc: self.logger.warning( "Failed to resolve parentage for: %s. Error: %s", dset, str(exc)) incompleteParentage = True continue self.logger.info("Resolved parentage for: %s", res) if res: setParents.add(res[0]['parent_dataset']) if not incompleteParentage: DataCache.setParentDatasetList(list(setParents)) self.logger.info("Parentage lookup complete and cache renewed") else: # then don't replace any data for the moment, simply add new parents previousData = set(DataCache.getParentDatasetList()) setParents = setParents | previousData DataCache.setParentDatasetList(list(setParents)) self.logger.info("Parentage lookup complete and cache updated") return
def get(self): # This assumes DataCahe is periodically updated. # If data is not updated, need to check, dataCacheUpdate log return rows(DataCache.filterData(ACTIVE_STATUS_FILTER, ["OutputModulesLFNBases"]))
def get(self): # This assumes DataCahe is periodically updated. # If data is not updated, need to check, dataCacheUpdate log return rows(DataCache.filterData(ACTIVE_NO_CLOSEOUT_FILTER, ["InputDataset", "OutputDatasets", "MCPileup", "DataPileup"]))