def get(self):
# This assumes both the DataCache and the parentage cache list
# get periodically updated.
# In case of problems, the WMStats cherrypy threads logs need to be checked
if DataCache.isEmpty():
raise DataCacheEmpty()
else:
return rows(DataCache.getParentDatasetList())
def fetchIncludeParentsRequests(self, config):
"""
Fetch active requests from the DataCache that have IncludeParents=True
"""
# use this boolean to signal whether there were datasets that failed
# to get their parentage resolved
incompleteParentage = False
setDsets = set()
setParents = set()
self.logger.info("Executing parent lock cherrypy thread")
for inputDset in DataCache.filterData(ACTIVE_NO_CLOSEOUT_PARENT_FILTER,
["InputDataset"]):
setDsets.add(inputDset)
self.logger.info(
"Found %d unique datasets requiring the parent dataset",
len(setDsets))
for dset in setDsets:
try:
res = self.dbs.listDatasetParents(dset)
except Exception as exc:
self.logger.warning(
"Failed to resolve parentage for: %s. Error: %s", dset,
str(exc))
incompleteParentage = True
continue
self.logger.info("Resolved parentage for: %s", res)
if res:
setParents.add(res[0]['parent_dataset'])
if not incompleteParentage:
DataCache.setParentDatasetList(list(setParents))
self.logger.info("Parentage lookup complete and cache renewed")
else:
# then don't replace any data for the moment, simply add new parents
previousData = set(DataCache.getParentDatasetList())
setParents = setParents | previousData
DataCache.setParentDatasetList(list(setParents))
self.logger.info("Parentage lookup complete and cache updated")
return
