def getPileupLockedAndAvailable(self, container, account, scope="cms"):
"""
Mock method to resolve where the pileup container (and all its blocks)
is locked and available.
"""
logging.info("%s: calling mock getPileupLockedAndAvailable",
self.__class__.__name__)
result = dict()
if not self.isContainer(container):
raise WMRucioException(
"Pileup location needs to be resolved for a container DID type"
)
kwargs = dict(name=container, account=account, scope=scope)
try:
DBS3Reader(PROD_DBS).checkDatasetPath(kwargs['name'])
blocks = DBS3Reader(PROD_DBS).listFileBlocks(
dataset=kwargs['name'])
for block in blocks:
result[block] = self.sitesByBlock(block)
except DBSReaderError:
logging.error("%s: Failed to fetch blocks from DBS",
self.__class__.__name__)
return result
def makePhEDExDrop(dbsUrl, datasetPath, *blockNames):
"""
_makePhEDExDrop_
Given a DBS Url, dataset name and list of blockNames,
generate an XML structure for injection
"""
from WMCore.Services.DBS.DBS3Reader import DBS3Reader
reader = DBS3Reader(dbsUrl)
spec = XMLInjectionSpec(dbsUrl)
dataset = spec.getDataset(datasetPath)
for block in blockNames:
blockContent = reader.getFileBlock(block)
if blockContent['IsOpen']:
xmlBlock = dataset.getFileblock(block, "y")
else:
xmlBlock = dataset.getFileblock(block, "n")
# Any Checksum from DBS is type cksum
for x in blockContent[block]['Files']:
checksums = {'cksum': x['Checksum']}
if x.get('Adler32') not in (None, ''):
checksums['adler32'] = x['Adler32']
xmlBlock.addFile(x['LogicalFileName'], checksums, x['FileSize'])
xml = spec.save()
return xml
def getFromDBS(dataset, logger):
"""
Uses the WMCore DBS3Reader object to fetch all the blocks and files
for a given container.
Returns a dictionary key'ed by the block name, and an inner dictionary
with the number of valid and invalid files. It also returns a total counter
for the number of valid and invalid files in the dataset.
"""
dbsReader = DBS3Reader(DBS_URL, logger)
result = dict()
dbsFilesCounter = Counter({'valid': 0, 'invalid': 0})
blocks = dbsReader.listFileBlocks(dataset)
for block in blocks:
data = dbsReader.dbs.listFileArray(block_name=block,
validFileOnly=0,
detail=True)
result.setdefault(block, Counter({'valid': 0, 'invalid': 0}))
for fileInfo in data:
if fileInfo['is_file_valid'] == 1:
result[block]['valid'] += 1
dbsFilesCounter['valid'] += 1
else:
result[block]['invalid'] += 1
dbsFilesCounter['invalid'] += 1
return result, dbsFilesCounter
def getReplicaPhEDExNodesForBlocks(self,
block=None,
dataset=None,
complete='y'):
"""
Args:
block: the name of the block
dataset: the name of the dataset
complete: ??
Returns:
a fake list of blocks and the fakes sites they are at
"""
if isinstance(dataset, list):
dataset = dataset[0] # Dataset is a list in these tests
if dataset:
# TODO: Generalize this and maybe move dataset detection into sitesByBlock
if dataset == PILEUP_DATASET:
return {
'%s#0fcb2b12-d27e-11e0-91b1-003048caaace' % dataset:
['T2_XX_SiteA', 'T2_XX_SiteB', 'T2_XX_SiteC']
}
else:
try:
DBS3Reader(PROD_DBS).checkDatasetPath(dataset)
blocks = DBS3Reader(PROD_DBS).dbs.listBlocks(
dataset=dataset)
singleBlock = blocks[0]['block_name']
return {singleBlock: self.sitesByBlock(singleBlock)}
except DBSReaderError:
return {
'%s#0fcb2b12-d27e-11e0-91b1-003048caaace' % dataset:
[]
}
replicas = {}
for oneBlock in block:
if oneBlock.split('#')[0] == PILEUP_DATASET:
# Pileup is at a single site
sites = ['T2_XX_SiteC']
_BLOCK_LOCATIONS[oneBlock] = sites
else:
sites = self.sitesByBlock(block=oneBlock)
_BLOCK_LOCATIONS[oneBlock] = sites
replicas.update({oneBlock: sites})
return replicas
def _queryAndCompareWithDBS(self, pileupDict, defaultArguments, dbsUrl):
"""
pileupDict is a Python dictionary containing particular pileup
configuration information. Query DBS on given dataset contained
now in both input defaultArguments as well as in the pileupDict
and compare values.
"""
reader = DBS3Reader(dbsUrl)
phedex = PhEDEx()
inputArgs = defaultArguments["PileupConfig"]
self.assertEqual(len(inputArgs), len(pileupDict),
"Number of pileup types different.")
for pileupType in inputArgs:
m = ("pileup type '%s' not in PileupFetcher-produced pileup "
"configuration: '%s'" % (pileupType, pileupDict))
self.assertTrue(pileupType in pileupDict, m)
# now query DBS for compare actual results on files lists for each
# pileup type and dataset and location (storage element names)
# pileupDict is saved in the file and now comparing items of this
# configuration with actual DBS results, the structure of pileupDict:
# {"pileupTypeA": {"BlockA": {"FileList": [], "PhEDExNodeNames": []},
# "BlockB": {"FileList": [], "PhEDExNodeNames": []}, ....}
for pileupType, datasets in inputArgs.items():
# this is from the pileup configuration produced by PileupFetcher
blockDict = pileupDict[pileupType]
for dataset in datasets:
dbsFileBlocks = reader.listFileBlocks(dataset=dataset)
blocksLocation = phedex.getReplicaPhEDExNodesForBlocks(
dataset=dataset, complete='y')
for dbsFileBlockName in dbsFileBlocks:
fileList = []
pnns = set()
for pnn in blocksLocation[dbsFileBlockName]:
pnns.add(pnn)
# now get list of files in the block
dbsFiles = reader.listFilesInBlock(dbsFileBlockName)
for dbsFile in dbsFiles:
fileList.append(dbsFile["LogicalFileName"])
# now compare the sets:
m = ("PNNs don't agree for pileup type '%s', "
"dataset '%s' in configuration: '%s'" %
(pileupType, dataset, pileupDict))
self.assertEqual(
set(blockDict[dbsFileBlockName]["PhEDExNodeNames"]),
pnns, m)
m = (
"FileList don't agree for pileup type '%s', dataset '%s' "
" in configuration: '%s'" %
(pileupType, dataset, pileupDict))
storedFileList = [
item['logical_file_name']
for item in blockDict[dbsFileBlockName]["FileList"]
]
self.assertItemsEqual(storedFileList, fileList, m)
def checkDBSUrl(dbsUrl):
if dbsUrl:
try:
DBS3Reader(dbsUrl).dbs.serverinfo()
except:
raise WMWorkloadToolsException("DBS is not responding: %s" %
dbsUrl)
return True
def DBSReader(endpoint, **kwargs):
"""Function to find and instantiate desired DBSReader object"""
try:
dbs = DBS3Reader(endpoint, **kwargs)
# if this doesn't throw endpoint is dbs3
dbs.dbs.serverinfo()
return dbs
except Exception as ex:
msg = 'Instantiating DBS3Reader failed with %s\n' % str(ex)
raise DBSReaderError("Can't contact DBS at %s, got errors %s" %
(endpoint, msg))
def _validateInputDataset(arguments):
inputdataset = arguments.get("InputDataset", None)
dbsURL = arguments.get("DbsUrl", None)
if inputdataset != None and dbsURL != None:
#import DBS3Reader here, since Runtime code import this module and worker node doesn't have dbs3 client
from WMCore.Services.DBS.DBS3Reader import DBS3Reader
from WMCore.Services.DBS.DBSErrors import DBSReaderError
try:
DBS3Reader(dbsURL).checkDatasetPath(inputdataset)
except DBSReaderError as ex:
# we need to Wrap the exception to WMSpecFactoryException to be caught in reqmgr validation
raise WMSpecFactoryException(str(ex))
return
def getContainerLockedAndAvailable(self, **kwargs):
"""
Mock the method to discover where container data is locked and available.
Note that, by default, it will not return any Tape RSEs.
:return: a unique list of RSEs
"""
logging.info("%s: Calling mock getContainerLockedAndAvailable",
self.__class__.__name__)
if 'name' not in kwargs:
raise WMRucioException(
"A DID name must be provided to the getContainerLockedAndAvailable API"
)
kwargs.setdefault("scope", "cms")
if kwargs['name'] == PILEUP_DATASET:
return ['T2_XX_SiteA', 'T2_XX_SiteB', 'T2_XX_SiteC']
try:
DBS3Reader(PROD_DBS).checkDatasetPath(kwargs['name'])
blocks = DBS3Reader(PROD_DBS).dbs.listBlocks(
dataset=kwargs['name'])
singleBlock = blocks[0]['block_name']
return self.sitesByBlock(singleBlock)
except DBSReaderError:
return []
def validateInputDatasSetAndParentFlag(arguments):
inputdataset = arguments.get("InputDataset", None)
if strToBool(arguments.get("IncludeParents", False)):
if inputdataset == None:
msg = "IncludeParent flag is True but there is no inputdataset"
raise WMSpecFactoryException(msg)
else:
dbsURL = arguments.get("DbsUrl", None)
if dbsURL != None:
#import DBS3Reader here, since Runtime code import this module and worker node doesn't have dbs3 client
from WMCore.Services.DBS.DBS3Reader import DBS3Reader
result = DBS3Reader(dbsURL).listDatasetParents(inputdataset)
if len(result) == 0:
msg = "IncludeParent flag is True but inputdataset %s doesn't have parents" % (inputdataset)
raise WMSpecFactoryException(msg)
else:
_validateInputDataset(arguments)
return
def validateInputDatasSetAndParentFlag(arguments):
"""
Check if the InputDataset value provided corresponds to an actual dataset in DBS.
If parent flag is provided, then check whether the input dataset has a parent.
the InputDataset existence in DBS and its parent, if needed.
"""
inputdataset = _getChainKey(arguments, "InputDataset")
mcpileup = _getChainKey(arguments, "MCPileup")
datapileup = _getChainKey(arguments, "DataPileup")
includeParents = _getChainKey(arguments, "IncludeParents")
# TODO: this replace can be removed in one year from now, thus March 2022
dbsURL = arguments.get("DbsUrl")
if dbsURL:
dbsURL = dbsURL.replace("cmsweb.cern.ch", "cmsweb-prod.cern.ch")
if includeParents and not inputdataset:
msg = "IncludeParents flag is True but InputDataset value has not been provided"
raise WMSpecFactoryException(msg)
if dbsURL and inputdataset or mcpileup or datapileup:
# import DBS3Reader here, since Runtime code import this module and worker
# node doesn't have dbs3 client
from WMCore.Services.DBS.DBS3Reader import DBS3Reader
from WMCore.Services.DBS.DBSErrors import DBSReaderError
dbsInst = DBS3Reader(dbsURL)
try:
_datasetExists(dbsInst, inputdataset)
_datasetExists(dbsInst, mcpileup)
_datasetExists(dbsInst, datapileup)
except DBSReaderError as ex:
# we need to Wrap the exception to WMSpecFactoryException to be caught in reqmgr validation
raise WMSpecFactoryException(str(ex))
if includeParents:
try:
result = dbsInst.listDatasetParents(inputdataset)
if len(result) == 0:
msg = "IncludeParents flag is True but the input dataset %s has no parents" % inputdataset
raise DBSReaderError(msg)
except DBSReaderError as ex:
raise WMSpecFactoryException(str(ex))
return
def _queryAndCompareWithDBS(self, pileupDict, pileupConfig, dbsUrl):
"""
pileupDict is a Python dictionary containing particular pileup
configuration information. Query DBS on given dataset contained
now in both input pileupConfig as well as in the pileupDict
and compare values.
"""
self.assertItemsEqual(list(pileupDict), list(pileupConfig))
reader = DBS3Reader(dbsUrl)
rucioObj = Rucio(self.rucioAcct)
# now query DBS and compare the blocks and files from DBS
# against those returned by the PileupFetcher
for pileupType, datasets in viewitems(pileupConfig):
# this is from the pileup configuration produced by PileupFetcher
blockDict = pileupDict[pileupType]
for dataset in datasets:
dbsBlocks = reader.listFileBlocks(dataset=dataset)
rucioBlocksLocation = rucioObj.getPileupLockedAndAvailable(dataset,
account=self.rucioAcct)
# first, validate the number of blocks and their names
self.assertItemsEqual(list(blockDict), dbsBlocks)
self.assertItemsEqual(list(blockDict), list(rucioBlocksLocation))
# now validate the block location between Rucio and PileupFetcher
for block, blockLocation in viewitems(blockDict):
self.assertItemsEqual(blockLocation['PhEDExNodeNames'], rucioBlocksLocation[block])
# finally, validate the files
fileList = []
# now get list of files in the block
dbsFiles = reader.listFilesInBlock(block)
for dbsFile in dbsFiles:
fileList.append(dbsFile["LogicalFileName"])
self.assertItemsEqual(blockDict[block]["FileList"], fileList)
def checkDBSURL(url):
# import DBS3Reader here, since Runtime code import this module and worker node doesn't have dbs3 client
from WMCore.Services.DBS.DBS3Reader import DBS3Reader
return DBS3Reader(url).checkDBSServer()
def main():
"""
_main_
"""
if 'WMAGENT_CONFIG' not in os.environ:
os.environ[
'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
if 'manage' not in os.environ:
os.environ[
'manage'] = '/data/srv/wmagent/current/config/wmagent/manage'
connectToDB()
myPhEDEx = PhEDEx()
myDBS = DBS3Reader('https://cmsweb.cern.ch/dbs/prod/global/DBSReader/')
myThread = threading.currentThread()
print "Shutting down PhEDExInjector..."
subprocess.call([
os.environ['manage'], "execute-agent", "wmcoreD", "--shutdown",
"--component=PhEDExInjector"
],
stdout=open(os.devnull, 'wb'))
time.sleep(5)
## TASK1: query DB for files not injected in phedex yet
# Get the files that the PhEDExInjector would look for
formatter = DBFormatter(logging, myThread.dbi)
formatter.sql = getQuery
results = formatter.execute()
fileList = []
fileList = [lfn[0] for lfn in results]
## TASK2: makes lfns a bit shorter to sort and uniq them
reducedLfns = [lfn.rsplit('/', 2)[0] for lfn in fileList]
reducedLfns = list(set(reducedLfns))
## TASK3: build uniq dataset names and check whether PhEDEx and DBS contain
## the same number of files. If so, then those lfns are healthy
print "Checking %d dataset in both PhEDEx and DBS ..." % len(reducedLfns)
crippleLfns, healthyLfns = [], []
i = 0
n = len(reducedLfns)
for lfn in reducedLfns:
try:
lfnAux = lfn.split('/')
dset = '/' + lfnAux[4] + '/' + lfnAux[3] + '-' + lfnAux[
6] + '/' + lfnAux[5]
result = myPhEDEx._getResult('blockreplicas',
args={'dataset': dset},
verb='GET')
phedexFiles = 0
for item in result["phedex"]["block"]:
phedexFiles += item['files']
## TODO: ValidFile is only available for > 0.9.95pre5. Once all agents are
## upgraded, then we can start using this new query.
#result = myDBS.listDatasetFileDetails(dset)
#dbsFiles = 0
#for item in result.itervalues():
# dbsFiles += 1 if item['ValidFile'] else 0
# This call returns valid+invalid number of filesfiles
result = myDBS.listDatasetFiles(dset)
dbsFiles = len(result)
if phedexFiles == dbsFiles:
healthyLfns.append(lfn)
else:
crippleLfns.append(lfn)
except:
print "Error with:", lfn
i += 1
if i % 100 == 0:
print '%d/%d files processed' % (i, n)
## TASK4: map the short cripple and healthy lists to the full original lfns
## TODO: this code looks terrible... IMPROVE IT!
if crippleLfns:
filesToCheck = []
for lfn in crippleLfns:
#filesToCheck = [file for file in fileList if lfn in file]
for file in fileList:
if lfn in file:
filesToCheck.append(file)
else:
filesToCheck = []
if healthyLfns:
filesInPhedex = []
for lfn in healthyLfns:
#filesInPhedex = [file for file in fileList if lfn in file]
for file in fileList:
if lfn in file:
filesInPhedex.append(file)
else:
filesInPhedex = []
## TASK5: query PhEDEx for each cripple file (filesToCheck)
## and build the final file lists
missingFiles = []
i = 0
n = len(filesToCheck)
for file in filesToCheck:
try:
result = myPhEDEx._getResult('data',
args={'file': file},
verb='GET')
if len(result['phedex']['dbs']):
filesInPhedex.append(file)
else:
missingFiles.append(file)
except:
print "Error contacting Phedex", file
i += 1
if i % 100 == 0:
print '%d/%d files processed' % (i, n)
if not filesInPhedex:
print "There are no files to be updated in the buffer. Contact a developer."
print "Starting PhEDExInjector now ..."
subprocess.call([
os.environ['manage'], "execute-agent", "wmcoreD", "--start",
"--component=PhEDExInjector"
],
stdout=open(os.devnull, 'wb'))
return 0
print "Found %d out of %d files that are already registered in PhEDEx \
but buffer doesn't know" % (len(filesInPhedex), len(fileList))
print "Fixing them now, it may take several minutes ..."
## TASK6: time to actually fix these files
binds = []
for file in filesInPhedex:
binds.append({'lfn': file})
formatter.dbi.processData(setQuery,
binds,
conn=None,
transaction=False,
returnCursor=False)
print "Rows were successfully updated! Good job!"
print "Starting PhEDExInjector now ..."
subprocess.call([
os.environ['manage'], "execute-agent", "wmcoreD", "--start",
"--component=PhEDExInjector"
],
stdout=open(os.devnull, 'wb'))
print "Done!"
return 0
def __init__(self, rest, config):
super(StepChainParentageFixTask, self).__init__(config)
self.reqmgrDB = RequestDBWriter(config.reqmgrdb_url)
self.dbsSvc = DBS3Reader(config.dbs_url, logger=self.logger)
self.statusToCheck = ["closed-out", "announced", "normal-archived"]
def main():
BASE_URL = "https://cmsweb.cern.ch"
COUCH_URL = "%s/couchdb" % BASE_URL
dbs_url = "https://cmsweb.cern.ch/dbs/prod/global/DBSWriter"
reqmgrdb_url = "%s/reqmgr_workload_cache" % (COUCH_URL)
statusToCheck = ['closed-out', 'announced']
logger = setupLogger()
dbsSvc = DBS3Reader(dbs_url, logger=logger)
reqmgrDB = RequestDBWriter(reqmgrdb_url)
logger.info("Running fixStepChainParentage thread for statuses")
childDatasets = set()
requests = set()
requestsByChildDataset = {}
for status in statusToCheck:
reqByChildDS = getChildDatasetsForStepChainMissingParent(
reqmgrDB, status, logger)
logger.info("Retrieved %d datasets to fix parentage, in status: %s",
len(reqByChildDS), status)
childDatasets = childDatasets.union(set(reqByChildDS.keys()))
# We need to just get one of the StepChain workflow if multiple workflow contains the same datasets. (i.e. ACDC)
requestsByChildDataset.update(reqByChildDS)
for wfs in reqByChildDS.values():
requests = requests.union(wfs)
logger.info(" datasets are: %s", reqByChildDS)
failedRequests = set()
totalChildDS = len(childDatasets)
fixCount = 0
for childDS in childDatasets:
logger.info("Resolving parentage for dataset: %s", childDS)
try:
failedBlocks = dbsSvc.fixMissingParentageDatasets(childDS,
insertFlag=True)
except Exception as exc:
logger.exception(
"Failed to resolve parentage data for dataset: %s. Error: %s",
childDS, str(exc))
failedRequests = failedRequests.union(
requestsByChildDataset[childDS])
else:
if failedBlocks:
logger.warning(
"These blocks failed to be resolved and will be retried later: %s",
failedBlocks)
failedRequests = failedRequests.union(
requestsByChildDataset[childDS])
else:
fixCount += 1
logger.info(
"Parentage for '%s' successfully updated. Processed %s out of %s datasets.",
childDS, fixCount, totalChildDS)
logger.info(" dataset sorted: %s\n", childDS)
requestsToUpdate = requests - failedRequests
### FIXME: disable the block below if you do NOT want to update the
# workflow in ReqMgr2
for request in requestsToUpdate:
try:
reqmgrDB.updateRequestProperty(request,
{"ParentageResolved": True})
logger.info("Marked ParentageResolved=True for request: %s",
request)
except Exception as exc:
logger.error(
"Failed to update 'ParentageResolved' flag to True for request: %s",
request)
msg = "A total of %d requests have been processed, where %d will have to be retried in the next cycle."
logger.info(msg, len(requestsToUpdate), len(failedRequests))
sys.exit(0)