Check if all files have a replica in a certain (set of) SE )Tier1-Archive default)
List the files that don't have a replica in the specified SE (group)
'''
__RCSID__ = "$Id$"
if __name__ == '__main__':
# Script initialization
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
Script.registerSwitch('', 'FixIt', ' Take action to fix the catalogs')
Script.setUsageMessage('\n'.join([__doc__,
'Usage:',
' %s [option|cfgfile] [values]' % Script.scriptName, ]))
dmScript = DMScript()
dmScript.registerDMSwitches() # Directory
Script.parseCommandLine(ignoreErrors=True)
fixIt = False
for opt, val in Script.getUnprocessedSwitches():
if opt == 'FixIt':
fixIt = True
# imports
from DIRAC import gLogger
from LHCbDIRAC.DataManagementSystem.Client.ConsistencyChecks import ConsistencyChecks
cc = ConsistencyChecks()
cc.directories = dmScript.getOption('Directory', [])
cc.lfns = dmScript.getOption('LFNs', []) + [lfn for arg in Script.getPositionalArgs() for lfn in arg.split(',')]
bkQuery = dmScript.getBKQuery(visible='All')
if bkQuery.getQueryDict() != {'Visible': 'All'}:
return failed
def prettyMsg(msg, msgList):
areIs = 's are' if len(msgList) > 1 else ' is'
gLogger.always('The following file%s %s:\n%s' %
(areIs, msg, '\n'.join(msgList)))
#====================================
if __name__ == "__main__":
from DIRAC.Core.Base import Script
import sys
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, ProgressBar
dmScript = DMScript()
Script.registerSwitch('v', 'Verbose', ' Set verbose mode')
Script.registerSwitch(
'', 'Production=',
' Select a production from which jobs in IDR will be used')
Script.registerSwitch('', 'User='******' Select a user')
Script.parseCommandLine(ignoreErrors=True)
import DIRAC
from DIRAC import gLogger, gConfig
from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
verbose = False
production = None
userName = None
#! /usr/bin/env python
"""
Get statistical information on a dataset
"""
__RCSID__ = "$Id$"
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerBKSwitches()
dmScript.registerFileSwitches()
Script.registerSwitch('', 'ByStream', ' Get rejection for each stream')
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1],
'Usage:',
' %s [option|cfgfile]' % Script.scriptName,
]))
Script.parseCommandLine(ignoreErrors=False)
from LHCbDIRAC.BookkeepingSystem.Client.ScriptExecutors import executeRejectionStats
executeRejectionStats(dmScript)
gLogger.fatal('Error changing directory owner', res['Message'])
exit(2)
startTime = time()
if recursive and subDirectories:
res = chown(subDirectories,
user,
group='lhcb_user',
mode=0o755,
recursive=True,
ndirs=1,
fcClient=dfc)
if not res['OK']:
gLogger.fatal('Error changing directory owner', res['Message'])
exit(2)
gLogger.always(
'Successfully changed owner in %d directories in %.1f seconds' %
(res['Value'], time() - startTime))
else:
gLogger.always(
'Successfully changed owner in directory %s in %.1f seconds' %
(baseDir, time() - startTime))
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeLfnMetadata
dmScript = DMScript()
dmScript.setLFNs(baseDir)
sys.stdout.write('Directory metadata: ')
sys.stdout.flush()
executeLfnMetadata(dmScript)
exit(0)
if seList and siteSEs:
seList = list(set(seList) & set(siteSEs))
else:
seList += siteSEs
return seList
if __name__ == '__main__':
# Script initialization
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
Script.setUsageMessage('\n'.join([__doc__,
'Usage:',
' %s [option|cfgfile] [values]' % Script.scriptName, ]))
dmScript = DMScript()
dmScript.registerDMSwitches()
maxFiles = 20
Script.registerSwitch('', 'FixIt', ' Take action to fix the catalogs and storage')
Script.registerSwitch('', 'Replace', ' Replace bad or missing replicas (default=False)')
Script.registerSwitch('', 'NoBK', ' Do not check with BK')
Script.registerSwitch('', 'Verbose', ' Set logging mode to INFO')
Script.registerSwitch('', 'MaxFiles=', ' Set maximum number of files to be printed (default %d)' % maxFiles)
Script.parseCommandLine(ignoreErrors=True)
from DIRAC import gLogger
fixIt = False
bkCheck = True
replace = False
verbose = False
########################################################################
"""
Get the list of all the user files.
"""
__RCSID__ = "$Id$"
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
if __name__ == "__main__":
days = 0
months = 0
years = 0
depth = 1
wildcard = '*'
dmScript = DMScript()
dmScript.registerNamespaceSwitches()
Script.registerSwitch("", "Days=",
"Match files older than number of days [%s]" % days)
Script.registerSwitch(
"", "Months=", "Match files older than number of months [%s]" % months)
Script.registerSwitch(
"", "Years=", "Match files older than number of years [%s]" % years)
Script.registerSwitch("", "Wildcard=",
"Wildcard for matching filenames [%s]" % wildcard)
Script.registerSwitch('', 'Output', 'Write list to an output file')
Script.registerSwitch("", "EmptyDirs",
"Create a list of empty directories")
Script.registerSwitch(
"", "Depth=",
"Depth to which recursively browse (default = %d)" % depth)
__RCSID__ = "$Id$"
# Code
if __name__ == '__main__':
# Script initialization
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
import DIRAC
Script.setUsageMessage('\n'.join([
__doc__,
'Usage:',
' %s [option|cfgfile] [values]' % Script.scriptName,
]))
dmScript = DMScript()
dmScript.registerNamespaceSwitches() # Directory
dmScript.registerFileSwitches() # File, LFNs
dmScript.registerBKSwitches()
Script.registerSwitch('', 'FixBK', ' Take action to fix the BK')
Script.registerSwitch('', 'FixFC', ' Take action to fix the FC')
Script.registerSwitch(
'', 'AffectedRuns',
' List the runs affected by the encountered problem')
Script.parseCommandLine(ignoreErrors=True)
from DIRAC import gLogger
fixBK = False
fixFC = False
listAffectedRuns = False
for switch in Script.getUnprocessedSwitches():
#!/usr/bin/env python
""" Get statistics on productions related to a given processing pass
"""
__RCSID__ = "$Id$"
from LHCbDIRAC.ProductionManagementSystem.Client.ProcessingProgress import ProcessingProgress, HTMLProgressTable
if __name__ == "__main__":
import DIRAC
from DIRAC.Core.Base import Script
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript
from LHCbDIRAC.BookkeepingSystem.Client.BKQuery import BKQuery
dmScript = DMScript()
dmScript.registerBKSwitches()
Script.registerSwitch('', 'Conditions=',
' comma separated list of DataTakingConditions')
Script.registerSwitch('', 'HTML=',
' <file> : Output in html format to <file>')
Script.registerSwitch('', 'NoPrint', ' No printout')
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1],
'Usage:',
' %s [option|cfgfile] [<LFN>] [<LFN>...]' % Script.scriptName,
]))
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1],
'Usage:',
#! /usr/bin/env python
"""
Set the status (default Unused) of a list of LFNs or files in status <Status> of Transformation <TransID>
"""
import DIRAC
from DIRAC.Core.Base import Script
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC.Core.Utilities.List import breakListIntoChunks
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerFileSwitches()
newStatus = 'Unused'
statusList = ("Unused", "Assigned", "Done", "Problematic", "MissingLFC",
"MissingInFC", "MaxReset", "Processed", "NotProcessed",
"Removed", 'ProbInFC')
Script.registerSwitch(
'', 'Status=',
"Select files with a given status from %s" % str(statusList))
Script.registerSwitch('', 'NewStatus=',
"New status to be set (default: %s)" % newStatus)
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1],
'Usage:',
' %s [option|cfgfile] <TransID> <Status>' % Script.scriptName,
]))
Script.parseCommandLine()
#!/usr/bin/env python
########################################################################
# File : dirac-dms-get-file
# Author : Stuart Paterson
########################################################################
"""
Retrieve a single file or list of files from Grid storage to the current directory.
"""
__RCSID__ = "$Id$"
import os
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerFileSwitches()
dmScript.registerNamespaceSwitches('download to (default = %s)' %
os.path.realpath('.'))
Script.setUsageMessage('\n'.join([
__doc__,
'Usage:',
' %s [option|cfgfile] [<LFN>] [<LFN>...] [SourceSE]' %
Script.scriptName,
]))
Script.parseCommandLine(ignoreErrors=False)
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeGetFile
from DIRAC import exit
def __compareFileListWithCatalog( self ):
successful = {}
failed = {}
failedHostKey = {}
dmScript = DMScript()
lfnFileDict = {}
lfnDict = {}
# for each file we have one or more http entries, but they all ahve the same lfn
# We save the lfns in a dict because FileCatalog.getReplicas needs a dictionary to work
# Also for each lfn key we assign the corresponding urlList.
for urlList in self.fileList:
lfn = dmScript.getLFNsFromList( urlList )
if not len(lfn):
self.log.error( "readFederation.__compareFileListWithCatalog: can't get LFN from HTTP url %s" % urlList )
continue
lfn = lfn[0]
lfnDict[lfn] = True
lfnFileDict[lfn] = urlList
res = self.__getSEListFromReplicas(lfnDict)
if not res:
errMsg = "readFederation:.__compareFileListWithCatalog: Failed to get SEs from replicas."
self.log.error(errMsg)
return S_ERROR( errMsg )
# all the possible storage elements
fullSEList = res
for lfn, urlList in lfnFileDict.items():
SEListPerLFN = []
for url in urlList:
# find on which SEs the file should be stored.
SEs = []
res = pfnparse(url)
if not res['OK']:
continue
parsed_dict = res['Value']
host = parsed_dict['Host']
try:
for SEName in self.SEDict[host]:
# some SEs have identical configuration - so if they have the same host address
# we will check if the file is at least in one of these SEs according to the
# catalog
SEs.append(SEName)
except KeyError:
failedHostKey[url] = "readFederation.__compareFileListWithCatalog: self.SEDict has no key %s. Check if SE is defined in config." % host
SEListPerLFN.append(SEs)
confirmedSE = []
# If checking for dedicated SEs clean up the SEListPerLFN so only dedicated SEs
# will be checked
if self.dedicatedSE:
for subList in SEListPerLFN:
if any(dedicatedSE in subList for dedicatedSE in self.dedicatedSE):
confirmedSE.append(subList)
else:
confirmedSE = SEListPerLFN
if fullSEList.has_key(lfn):
SEList = fullSEList[lfn].keys()
else:
failedHostKey[lfn] = 'No SEList available for this LFN'
continue
for SESubList in confirmedSE:
if not any(SE in SEList for SE in SESubList):
if lfn in failed:
failed[lfn].append('Failed to find match in catalog for %s' % SESubList)
else:
failed[lfn] = ['Failed to find match in catalog for %s' % SESubList]
else:
successful[lfn] = True
self.fileList = []
return S_OK( { 'Successful' : successful, 'Failed' : failed, 'Failed Host' : failedHostKey } )
# imports
import sys
import os
import time
import DIRAC
from DIRAC import gLogger
# Code
if __name__ == '__main__':
# Script initialization
from DIRAC.Core.Base import Script
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript
dmScript = DMScript()
dmScript.registerFileSwitches()
depth = 1
Script.setUsageMessage('\n'.join([__doc__,
'Usage:',
' %s [option|cfgfile] [ProdIDs]' % Script.scriptName, ]))
Script.registerSwitch('', 'Runs=', ' Specify the run range')
Script.registerSwitch('', 'ActiveRunsProduction=', ' Specify the production from which the runs should be derived')
Script.registerSwitch('', 'FileType=', 'S pecify the descendants file type')
Script.registerSwitch('', 'NoFC', ' Trust the BK replica flag, no LFC check')
Script.registerSwitch('', 'FixIt', ' Fix the files in transformation table')
Script.registerSwitch('', 'Verbose', ' Print full list of files with error')
Script.registerSwitch('', 'Status=', ' Select files with a given status in the production')
Script.registerSwitch('', 'Depth=', ' Depth to which to check descendants (default=%d)' % depth)
Script.registerSwitch('', 'Force', ' Use this flag to force checking a whole production')
#! /usr/bin/env python
"""
Set the visibility flag to a dataset
"""
__RCSID__ = "$Id$"
from DIRAC import exit as dExit, gLogger
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerBKSwitches()
dmScript.registerFileSwitches()
Script.registerSwitch('', 'List', ' Print out the list of LFNs')
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1],
'Usage:',
' %s [option|cfgfile]' % Script.scriptName,
]))
Script.parseCommandLine(ignoreErrors=False)
dumpList = False
for switch in Script.getUnprocessedSwitches():
if switch[0] == 'List':
dumpList = True
bkQuery = dmScript.getBKQuery()
lfns = dmScript.getOption('LFNs', [])
# File : dirac-bookkeeping-job-input-output
# Author : Zoltan Mathe
########################################################################
"""
It returns the input and output files of a given list of DIRAC Jobids
"""
__RCSID__ = "$Id$"
from DIRAC.Core.Base import Script
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, printDMResult
import DIRAC
import os
if __name__ == "__main__":
bkScript = DMScript()
bkScript.registerJobsSwitches()
Script.registerSwitch('', 'InputFiles', ' Only input files')
Script.registerSwitch('', 'OutputFiles', ' Only output files')
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... [DIRACJobid|File]' % Script.scriptName,
'Arguments:', ' DIRACJobid: DIRAC Jobids',
' File: Name of the file with contains a list of DIRACJobids'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
inputFiles = False
outputFiles = False
for switch in Script.getUnprocessedSwitches():
#!/usr/bin/env python
########################################################################
__RCSID__ = "$Id$"
import DIRAC
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript
from DIRAC.Core.Base import Script
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerBKSwitches()
dmScript.registerFileSwitches()
fixIt = False
Script.registerSwitch('', 'FixIt', 'Set replicas problematic if needed')
Script.setUsageMessage("""
Check the integrity of the state of the storages and information in the File Catalogs
for a given file or a collection of files.
Usage:
%s <lfn | fileContainingLfns> <SE> <status>
""" % Script.scriptName)
Script.parseCommandLine()
for opt, val in Script.getUnprocessedSwitches():
if opt == 'FixIt':
fixIt = True
from DIRAC import gLogger
#!/usr/bin/env python
########################################################################
# File : dirac-bookkeeping-productions-summary
# Author : Zoltan Mathe
########################################################################
"""
Retrieve production summary from the Bookkeeping
"""
__RCSID__ = "$Id$"
import DIRAC
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
dmScript = DMScript()
dmScript.registerBKSwitches()
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ...' % Script.scriptName
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
exitCode = 0
from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient
bk = BookkeepingClient()
bkQuery = dmScript.getBKQuery()
if not bkQuery:
print "No BKQuery given..."
else:
transList.append(int(r[0]))
except Exception as e:
gLogger.exception("Invalid transformation", lException=e)
transList = []
return transList
if __name__ == "__main__":
import os
import DIRAC
from DIRAC import gLogger
from DIRAC.Core.Base import Script
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript
dmScript = DMScript()
dmScript.registerFileSwitches()
Script.parseCommandLine(ignoreErrors=True)
Script.setUsageMessage('\n'.join([
__doc__,
'Usage:',
' %s [option|cfgfile] ...' % Script.scriptName,
]))
runInfo = True
userGroup = None
transList = __getTransformations(Script.getPositionalArgs())
if not transList:
__RCSID__ = "$Id$"
# Code
if __name__ == '__main__':
# Script initialization
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
from DIRAC import gLogger
Script.setUsageMessage('\n'.join([
__doc__,
'Usage:',
' %s [option|cfgfile] [values]' % Script.scriptName,
]))
dmScript = DMScript()
dmScript.registerBKSwitches()
dmScript.registerFileSwitches()
Script.registerSwitch('', 'FixIt', ' Take action to fix the catalogs')
Script.registerSwitch('', 'CheckAllFlags',
' Consider also files with replica flag NO')
Script.parseCommandLine(ignoreErrors=True)
fixIt = False
checkAll = False
production = 0
for opt, val in Script.getUnprocessedSwitches():
if opt == 'FixIt':
fixIt = True
elif opt == 'CheckAllFlags':
checkAll = True
