Script.registerSwitch('', 'FixIt', ' Take action to fix the catalogs')
Script.setUsageMessage('\n'.join([__doc__,
'Usage:',
' %s [option|cfgfile] [values]' % Script.scriptName, ]))
dmScript = DMScript()
dmScript.registerDMSwitches() # Directory
Script.parseCommandLine(ignoreErrors=True)
fixIt = False
for opt, val in Script.getUnprocessedSwitches():
if opt == 'FixIt':
fixIt = True
# imports
from DIRAC import gLogger
from LHCbDIRAC.DataManagementSystem.Client.ConsistencyChecks import ConsistencyChecks
cc = ConsistencyChecks()
cc.directories = dmScript.getOption('Directory', [])
cc.lfns = dmScript.getOption('LFNs', []) + [lfn for arg in Script.getPositionalArgs() for lfn in arg.split(',')]
bkQuery = dmScript.getBKQuery(visible='All')
if bkQuery.getQueryDict() != {'Visible': 'All'}:
bkQuery.setOption('ReplicaFlag', 'All')
cc.bkQuery = bkQuery
seList = dmScript.getOption('SEs', [])
if not seList:
dmScript.setSEs('Tier1-Archive')
seList = dmScript.getOption('SEs', [])
from LHCbDIRAC.DataManagementSystem.Client.CheckExecutors import doCheckSE
doCheckSE(cc, seList, fixIt)
if prod != prodList[0]:
gLogger.notice("====================")
gLogger.notice("Processing %s production %d" % (cc.transType, cc.prod))
if status:
res = tr.getTransformationFiles({'TransformationID': prod, 'Status': status})
if res['OK']:
lfnList = [trFile['LFN'] for trFile in res['Value']]
gLogger.notice('Found %d files with status %s' % (len(lfnList), status))
else:
gLogger.fatal("Error getting files %s" % status, res['Message'])
DIRAC.exit(2)
if not lfnList:
continue
cc.lfns = lfnList
if not fileType:
bkQuery = BKQuery({'Production': prod, 'FileType': 'ALL', 'Visible': 'All'})
cc.fileType = bkQuery.getBKFileTypes()
gLogger.notice("Looking for descendants of type %s" % str(cc.fileType))
notAllFileTypes = False
else:
cc.fileType = fileType
cc.fileTypesExcluded = ['LOG']
notAllFileTypes = True
cc.runsList = runsList
cc.runStatus = 'Active'
cc.fromProd = fromProd
cc.checkTS2BK()
# Print out the results
def doCheckFC2BK(cc, fixFC=False, fixBK=False, listAffectedRuns=False):
"""
Method actually calling for the the check using ConsistencyChecks module
It prints out results and calls corrective actions if required
"""
cc.checkFC2BK()
maxFiles = 10
suffix = ''
nb = 0
baseName = 'CheckFC2BK' + ('-%s' % cc.prod if cc.prod else '')
while True:
fileName = baseName + '%s.txt' % suffix
if not os.path.exists(fileName):
break
nb += 1
suffix = '-%d' % nb
fp = None
if cc.existLFNsBKRepNo:
gLogger.notice('>>>>')
affectedRuns = list(
set(str(run) for run in cc.existLFNsBKRepNo.itervalues()))
gLogger.error("%d files are in the FC but have replica = NO in BK" %
len(cc.existLFNsBKRepNo))
from LHCbDIRAC.DataManagementSystem.Client.ConsistencyChecks import ConsistencyChecks
ccAux = ConsistencyChecks()
gLogger.notice("====== Now checking %d files from FC to SE ======" %
len(cc.existLFNsBKRepNo))
ccAux.lfns = cc.existLFNsBKRepNo.keys()
doCheckFC2SE(ccAux, bkCheck=False, fixIt=fixFC, fixOption='FixFC')
cc.existLFNsBKRepNo = sorted(
set(cc.existLFNsBKRepNo) - set(ccAux.existLFNsNoSE) -
set(ccAux.existLFNsNotExisting) - set(ccAux.existLFNsBadFiles))
if cc.existLFNsBKRepNo:
gLogger.notice(
"====== Completed, %d files are in the FC and SE but have replica = NO in BK ======"
% len(cc.existLFNsBKRepNo))
if fp is None:
fp = open(fileName, 'w')
fp.write('\nInFCButBKNo '.join([''] + sorted(cc.existLFNsBKRepNo)))
res = cc.bkClient.getFileMetadata(cc.existLFNsBKRepNo)
if not res['OK']:
gLogger.fatal("Unable to get file metadata", res['Message'])
return
if res['Value']['Failed']:
gLogger.error("No metadata found for some files",
'%d files' % len(res['Value']['Failed']))
success = res['Value']['Successful']
filesInvisible = set(lfn for lfn, meta in success.iteritems()
if meta['VisibilityFlag'][0].upper() == 'N')
filesVisible = set(success) - filesInvisible
gLogger.notice('%d files are visible, %d files are invisible' %
(len(filesVisible), len(filesInvisible)))
# Try and print the whole as INFO (in case --Verbose was used).
# If nothing printed, print a limited number of files as ERROR
if not gLogger.info('\n'.join(
'%s : Visi %s' %
(lfn, success.get(lfn, {}).get('VisibilityFlag', '?'))
for lfn in sorted(cc.existLFNsBKRepNo))):
if len(cc.existLFNsBKRepNo) > maxFiles:
gLogger.notice('First %d files:' % maxFiles)
gLogger.error('\n'.join(
'%s : Visi %s' %
(lfn, success.get(lfn, {}).get('VisibilityFlag', '?'))
for lfn in sorted(cc.existLFNsBKRepNo)[0:maxFiles]))
if listAffectedRuns:
gLogger.notice('Affected runs: %s' % ','.join(affectedRuns))
gLogger.notice("Full list of files: grep InFCButBKNo %s" %
fileName)
if fixBK:
gLogger.notice("Going to fix them, setting the replica flag")
res = cc.bkClient.addFiles(list(success))
if res['OK']:
gLogger.notice(
"\tSuccessfully added replica flag to %d files" %
len(success))
else:
gLogger.error('Failed to set the replica flag',
res['Message'])
elif fixFC:
gLogger.notice(
"Going to fix them, by removing from the FC and storage")
__removeFile(success)
else:
gLogger.notice(
"Use --FixBK to fix it (set the replica flag) or --FixFC (for removing from FC and storage)"
)
else:
gLogger.notice(
"====== Completed, no files in the FC with replica = NO in BK ======"
)
gLogger.notice('<<<<')
else:
gLogger.notice("No files in FC with replica = NO in BK -> OK!")
if cc.existLFNsNotInBK:
gLogger.notice('>>>>')
gLogger.error("%d files are in the FC but are NOT in BK:" %
len(cc.existLFNsNotInBK))
if fp is None:
fp = open(fileName, 'w')
fp.write('\nInFCNotInBK '.join([''] + sorted(cc.existLFNsNotInBK)))
if not gLogger.info('\n'.join(sorted(cc.existLFNsNotInBK))):
if len(cc.existLFNsNotInBK) > maxFiles:
gLogger.notice('First %d files:' % maxFiles)
gLogger.error('\n'.join(sorted(cc.existLFNsNotInBK[0:maxFiles])))
gLogger.notice("Full list of files: grep InFCNotInBK %s" % fileName)
if fixFC:
gLogger.notice(
"Going to fix them, by removing from the FC and storage")
__removeFile(cc.existLFNsNotInBK)
else:
gLogger.notice(
"Use --FixFC to fix it (remove from FC and storage)")
gLogger.notice('<<<<')
else:
gLogger.notice("No files in FC not in BK -> OK!")
if fp is not None:
fp.close()
' Consider also files with replica flag NO')
Script.parseCommandLine(ignoreErrors=True)
fixIt = False
checkAll = False
production = 0
for opt, val in Script.getUnprocessedSwitches():
if opt == 'FixIt':
fixIt = True
elif opt == 'CheckAllFlags':
checkAll = True
# imports
from LHCbDIRAC.DataManagementSystem.Client.ConsistencyChecks import ConsistencyChecks
gLogger.setLevel('INFO')
cc = ConsistencyChecks()
bkQuery = dmScript.getBKQuery(visible='All')
cc.bkQuery = bkQuery
cc.lfns = dmScript.getOption('LFNs', [])
productions = dmScript.getOption('Productions', [])
from LHCbDIRAC.DataManagementSystem.Client.CheckExecutors import doCheckBK2FC
if productions:
for prod in productions:
cc.prod = prod
gLogger.always("Processing production %d" % cc.prod)
doCheckBK2FC(cc, checkAll, fixIt)
gLogger.always("Processed production %d" % cc.prod)
else:
doCheckBK2FC(cc, checkAll, fixIt)