%s <lfn | fileContainingLfns> <SE> <status>
""" % Script.scriptName)
Script.parseCommandLine()
for opt, val in Script.getUnprocessedSwitches():
if opt == 'FixIt':
fixIt = True
from DIRAC import gLogger
gLogger.setLevel('INFO')
from LHCbDIRAC.DataManagementSystem.Client.DataIntegrityClient import DataIntegrityClient
for lfn in Script.getPositionalArgs():
dmScript.setLFNsFromFile(lfn)
lfns = dmScript.getOption('LFNs')
if not lfns:
print "No LFNs given..."
Script.showHelp()
DIRAC.exit(0)
integrityClient = DataIntegrityClient()
res = integrityClient.catalogFileToBK(lfns)
if not res['OK']:
gLogger.error(res['Message'])
DIRAC.exit(1)
replicas = res['Value']['CatalogReplicas']
metadata = res['Value']['CatalogMetadata']
res = integrityClient.checkPhysicalFiles(replicas, metadata, fixIt=fixIt)
if not res['OK']:
gLogger.error(res['Message'])
Script.parseCommandLine(ignoreErrors=True)
Script.setUsageMessage('\n'.join([
__doc__,
'Usage:',
' %s [option|cfgfile] ...' % Script.scriptName,
]))
runInfo = True
userGroup = None
transList = __getTransformations(Script.getPositionalArgs())
if not transList:
DIRAC.exit(1)
requestedLFNs = dmScript.getOption('LFNs')
if not requestedLFNs:
gLogger.always('No files to add')
DIRAC.exit(1)
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
trClient = TransformationClient()
rc = 0
for transID in transList:
res = trClient.setFileStatusForTransformation(transID,
'Removed',
requestedLFNs,
force=True)
if res['OK']:
gLogger.always(
'Successfully set %d files%s Removed in transformation %d' %
listAffectedRuns = False
for switch in Script.getUnprocessedSwitches():
if switch[0] == 'FixFC':
fixFC = True
if switch[0] == 'FixBK':
fixBK = True
elif switch[0] == 'AffectedRuns':
listAffectedRuns = True
if fixFC and fixBK:
gLogger.notice("Can't fix both FC and BK, please choose")
DIRAC.exit(0)
# imports
from LHCbDIRAC.DataManagementSystem.Client.ConsistencyChecks import ConsistencyChecks
cc = ConsistencyChecks()
cc.directories = dmScript.getOption('Directory', [])
cc.lfns = dmScript.getOption('LFNs', []) + [
lfn for arg in Script.getPositionalArgs() for lfn in arg.split(',')
]
productions = dmScript.getOption('Productions', [])
runs = dmScript.getOption('Runs', [])
from LHCbDIRAC.DataManagementSystem.Client.CheckExecutors import doCheckFC2BK
if productions and not runs:
fileType = dmScript.getOption('FileType', [])
if fileType:
cc.fileType = fileType
for prod in productions:
cc.prod = prod
gLogger.always("Processing production %d" % cc.prod)
doCheckFC2BK(cc, fixFC, fixBK, listAffectedRuns)
Script.registerSwitch('', 'FixIt', ' Take action to fix the catalogs')
Script.setUsageMessage('\n'.join([__doc__,
'Usage:',
' %s [option|cfgfile] [values]' % Script.scriptName, ]))
dmScript = DMScript()
dmScript.registerDMSwitches() # Directory
Script.parseCommandLine(ignoreErrors=True)
fixIt = False
for opt, val in Script.getUnprocessedSwitches():
if opt == 'FixIt':
fixIt = True
# imports
from DIRAC import gLogger
from LHCbDIRAC.DataManagementSystem.Client.ConsistencyChecks import ConsistencyChecks
cc = ConsistencyChecks()
cc.directories = dmScript.getOption('Directory', [])
cc.lfns = dmScript.getOption('LFNs', []) + [lfn for arg in Script.getPositionalArgs() for lfn in arg.split(',')]
bkQuery = dmScript.getBKQuery(visible='All')
if bkQuery.getQueryDict() != {'Visible': 'All'}:
bkQuery.setOption('ReplicaFlag', 'All')
cc.bkQuery = bkQuery
seList = dmScript.getOption('SEs', [])
if not seList:
dmScript.setSEs('Tier1-Archive')
seList = dmScript.getOption('SEs', [])
from LHCbDIRAC.DataManagementSystem.Client.CheckExecutors import doCheckSE
doCheckSE(cc, seList, fixIt)
for id_o in ids:
r = id_o.split(':')
if len(r) > 1:
for i in xrange(int(r[0]), int(r[1]) + 1):
idList.append(i)
else:
idList.append(int(r[0]))
except Exception:
print "Invalid set of transformationIDs..."
DIRAC.exit(1)
if len(args) == 2:
status = args[1].split(',')
elif not status:
status = ['Unknown']
lfnsExplicit = dmScript.getOption('LFNs')
transClient = TransformationClient()
for transID in idList:
lfns = lfnsExplicit
if not lfns:
res = transClient.getTransformation(transID)
if not res['OK']:
print "Failed to get transformation information: %s" % res[
'Message']
DIRAC.exit(2)
selectDict = {
'TransformationID': res['Value']['TransformationID'],
'Status': status
else:
ids = args[0].split(",")
prodList = []
try:
for id in ids:
r = id.split(':')
if len(r) > 1:
for i in range(int(r[0]), int(r[1]) + 1):
prodList.append(i)
else:
prodList.append(int(eval(r[0])))
except (ValueError, NameError) as e:
gLogger.exception("Bad production list: %s" % args[0], lException=e)
# In case the user asked for specific LFNs
if not status:
lfnList = dmScript.getOption('LFNs', [])
if not status and not lfnList and not runsList and not fromProd and not force:
gLogger.fatal("You are about to check descendants for all files in a production")
gLogger.fatal("If you really want to do so, use --Force")
DIRAC.exit(0)
from LHCbDIRAC.DataManagementSystem.Client.ConsistencyChecks import ConsistencyChecks
from LHCbDIRAC.BookkeepingSystem.Client.BKQuery import BKQuery
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
tr = TransformationClient()
for prod in prodList:
startTime = time.time()
cc = ConsistencyChecks()
# Setting the prod also sets its type
try:
Script.registerSwitch('', 'List', ' Print out the list of LFNs')
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1],
'Usage:',
' %s [option|cfgfile]' % Script.scriptName,
]))
Script.parseCommandLine(ignoreErrors=False)
dumpList = False
for switch in Script.getUnprocessedSwitches():
if switch[0] == 'List':
dumpList = True
bkQuery = dmScript.getBKQuery()
lfns = dmScript.getOption('LFNs', [])
if not bkQuery and not lfns:
gLogger.error("No BKQuery and no files given...")
dExit(1)
# Invert the visibility flag as want to set Invisible those that are visible and vice-versa
from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient
bk = BookkeepingClient()
visibilityFlag = dmScript.getOption('Visibility', None)
if visibilityFlag is None:
gLogger.error('Visibility option should be given')
dExit(2)
visibilityFlag = str(visibilityFlag).lower() == 'yes'
if bkQuery:
# Query with visibility opposite to what is requested to be set ;-)
bkQuery.setOption('Visible', 'No' if visibilityFlag else 'Yes')
outputFiles = False
for switch in Script.getUnprocessedSwitches():
if switch[0] == 'InputFiles':
inputFiles = True
if switch[0] == 'OutputFiles':
outputFiles = True
if not inputFiles and not outputFiles:
inputFiles = True
outputFiles = True
jobidList = []
for jobid in args:
if os.path.exists(jobid):
bkScript.setJobidsFromFile(jobid)
else:
jobidList += jobid.split(',')
jobidList += bkScript.getOption('JobIDs', [])
if not jobidList:
print "No jobID provided!"
Script.showHelp()
DIRAC.exit(0)
from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient
retVal = BookkeepingClient().getJobInputOutputFiles(jobidList)
if retVal['OK']:
success = retVal['Value']['Successful']
for job in success:
# Remove from input the files that are also output! This happens because the output of step 1 can be the input of step 2...
# only worth if input files are requested though
if inputFiles:
success[job]['InputFiles'] = sorted(
set(success[job]['InputFiles']) -
' Consider also files with replica flag NO')
Script.parseCommandLine(ignoreErrors=True)
fixIt = False
checkAll = False
production = 0
for opt, val in Script.getUnprocessedSwitches():
if opt == 'FixIt':
fixIt = True
elif opt == 'CheckAllFlags':
checkAll = True
# imports
from LHCbDIRAC.DataManagementSystem.Client.ConsistencyChecks import ConsistencyChecks
gLogger.setLevel('INFO')
cc = ConsistencyChecks()
bkQuery = dmScript.getBKQuery(visible='All')
cc.bkQuery = bkQuery
cc.lfns = dmScript.getOption('LFNs', [])
productions = dmScript.getOption('Productions', [])
from LHCbDIRAC.DataManagementSystem.Client.CheckExecutors import doCheckBK2FC
if productions:
for prod in productions:
cc.prod = prod
gLogger.always("Processing production %d" % cc.prod)
doCheckBK2FC(cc, checkAll, fixIt)
gLogger.always("Processed production %d" % cc.prod)
else:
doCheckBK2FC(cc, checkAll, fixIt)