Check if all files have a replica in a certain (set of) SE )Tier1-Archive default) List the files that don't have a replica in the specified SE (group) ''' __RCSID__ = "$Id$" if __name__ == '__main__': # Script initialization from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script Script.registerSwitch('', 'FixIt', ' Take action to fix the catalogs') Script.setUsageMessage('\n'.join([__doc__, 'Usage:', ' %s [option|cfgfile] [values]' % Script.scriptName, ])) dmScript = DMScript() dmScript.registerDMSwitches() # Directory Script.parseCommandLine(ignoreErrors=True) fixIt = False for opt, val in Script.getUnprocessedSwitches(): if opt == 'FixIt': fixIt = True # imports from DIRAC import gLogger from LHCbDIRAC.DataManagementSystem.Client.ConsistencyChecks import ConsistencyChecks cc = ConsistencyChecks() cc.directories = dmScript.getOption('Directory', []) cc.lfns = dmScript.getOption('LFNs', []) + [lfn for arg in Script.getPositionalArgs() for lfn in arg.split(',')] bkQuery = dmScript.getBKQuery(visible='All') if bkQuery.getQueryDict() != {'Visible': 'All'}:
return failed def prettyMsg(msg, msgList): areIs = 's are' if len(msgList) > 1 else ' is' gLogger.always('The following file%s %s:\n%s' % (areIs, msg, '\n'.join(msgList))) #==================================== if __name__ == "__main__": from DIRAC.Core.Base import Script import sys from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, ProgressBar dmScript = DMScript() Script.registerSwitch('v', 'Verbose', ' Set verbose mode') Script.registerSwitch( '', 'Production=', ' Select a production from which jobs in IDR will be used') Script.registerSwitch('', 'User='******' Select a user') Script.parseCommandLine(ignoreErrors=True) import DIRAC from DIRAC import gLogger, gConfig from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient verbose = False production = None userName = None
#! /usr/bin/env python """ Get statistical information on a dataset """ __RCSID__ = "$Id$" from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script if __name__ == "__main__": dmScript = DMScript() dmScript.registerBKSwitches() dmScript.registerFileSwitches() Script.registerSwitch('', 'ByStream', ' Get rejection for each stream') Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile]' % Script.scriptName, ])) Script.parseCommandLine(ignoreErrors=False) from LHCbDIRAC.BookkeepingSystem.Client.ScriptExecutors import executeRejectionStats executeRejectionStats(dmScript)
gLogger.fatal('Error changing directory owner', res['Message']) exit(2) startTime = time() if recursive and subDirectories: res = chown(subDirectories, user, group='lhcb_user', mode=0o755, recursive=True, ndirs=1, fcClient=dfc) if not res['OK']: gLogger.fatal('Error changing directory owner', res['Message']) exit(2) gLogger.always( 'Successfully changed owner in %d directories in %.1f seconds' % (res['Value'], time() - startTime)) else: gLogger.always( 'Successfully changed owner in directory %s in %.1f seconds' % (baseDir, time() - startTime)) from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeLfnMetadata dmScript = DMScript() dmScript.setLFNs(baseDir) sys.stdout.write('Directory metadata: ') sys.stdout.flush() executeLfnMetadata(dmScript) exit(0)
if seList and siteSEs: seList = list(set(seList) & set(siteSEs)) else: seList += siteSEs return seList if __name__ == '__main__': # Script initialization from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script Script.setUsageMessage('\n'.join([__doc__, 'Usage:', ' %s [option|cfgfile] [values]' % Script.scriptName, ])) dmScript = DMScript() dmScript.registerDMSwitches() maxFiles = 20 Script.registerSwitch('', 'FixIt', ' Take action to fix the catalogs and storage') Script.registerSwitch('', 'Replace', ' Replace bad or missing replicas (default=False)') Script.registerSwitch('', 'NoBK', ' Do not check with BK') Script.registerSwitch('', 'Verbose', ' Set logging mode to INFO') Script.registerSwitch('', 'MaxFiles=', ' Set maximum number of files to be printed (default %d)' % maxFiles) Script.parseCommandLine(ignoreErrors=True) from DIRAC import gLogger fixIt = False bkCheck = True replace = False verbose = False
######################################################################## """ Get the list of all the user files. """ __RCSID__ = "$Id$" from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script if __name__ == "__main__": days = 0 months = 0 years = 0 depth = 1 wildcard = '*' dmScript = DMScript() dmScript.registerNamespaceSwitches() Script.registerSwitch("", "Days=", "Match files older than number of days [%s]" % days) Script.registerSwitch( "", "Months=", "Match files older than number of months [%s]" % months) Script.registerSwitch( "", "Years=", "Match files older than number of years [%s]" % years) Script.registerSwitch("", "Wildcard=", "Wildcard for matching filenames [%s]" % wildcard) Script.registerSwitch('', 'Output', 'Write list to an output file') Script.registerSwitch("", "EmptyDirs", "Create a list of empty directories") Script.registerSwitch( "", "Depth=", "Depth to which recursively browse (default = %d)" % depth)
__RCSID__ = "$Id$" # Code if __name__ == '__main__': # Script initialization from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script import DIRAC Script.setUsageMessage('\n'.join([ __doc__, 'Usage:', ' %s [option|cfgfile] [values]' % Script.scriptName, ])) dmScript = DMScript() dmScript.registerNamespaceSwitches() # Directory dmScript.registerFileSwitches() # File, LFNs dmScript.registerBKSwitches() Script.registerSwitch('', 'FixBK', ' Take action to fix the BK') Script.registerSwitch('', 'FixFC', ' Take action to fix the FC') Script.registerSwitch( '', 'AffectedRuns', ' List the runs affected by the encountered problem') Script.parseCommandLine(ignoreErrors=True) from DIRAC import gLogger fixBK = False fixFC = False listAffectedRuns = False for switch in Script.getUnprocessedSwitches():
#!/usr/bin/env python """ Get statistics on productions related to a given processing pass """ __RCSID__ = "$Id$" from LHCbDIRAC.ProductionManagementSystem.Client.ProcessingProgress import ProcessingProgress, HTMLProgressTable if __name__ == "__main__": import DIRAC from DIRAC.Core.Base import Script from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript from LHCbDIRAC.BookkeepingSystem.Client.BKQuery import BKQuery dmScript = DMScript() dmScript.registerBKSwitches() Script.registerSwitch('', 'Conditions=', ' comma separated list of DataTakingConditions') Script.registerSwitch('', 'HTML=', ' <file> : Output in html format to <file>') Script.registerSwitch('', 'NoPrint', ' No printout') Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] [<LFN>] [<LFN>...]' % Script.scriptName, ])) Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:',
#! /usr/bin/env python """ Set the status (default Unused) of a list of LFNs or files in status <Status> of Transformation <TransID> """ import DIRAC from DIRAC.Core.Base import Script from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient from DIRAC.Core.Utilities.List import breakListIntoChunks from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript if __name__ == "__main__": dmScript = DMScript() dmScript.registerFileSwitches() newStatus = 'Unused' statusList = ("Unused", "Assigned", "Done", "Problematic", "MissingLFC", "MissingInFC", "MaxReset", "Processed", "NotProcessed", "Removed", 'ProbInFC') Script.registerSwitch( '', 'Status=', "Select files with a given status from %s" % str(statusList)) Script.registerSwitch('', 'NewStatus=', "New status to be set (default: %s)" % newStatus) Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] <TransID> <Status>' % Script.scriptName, ])) Script.parseCommandLine()
#!/usr/bin/env python ######################################################################## # File : dirac-dms-get-file # Author : Stuart Paterson ######################################################################## """ Retrieve a single file or list of files from Grid storage to the current directory. """ __RCSID__ = "$Id$" import os from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script if __name__ == "__main__": dmScript = DMScript() dmScript.registerFileSwitches() dmScript.registerNamespaceSwitches('download to (default = %s)' % os.path.realpath('.')) Script.setUsageMessage('\n'.join([ __doc__, 'Usage:', ' %s [option|cfgfile] [<LFN>] [<LFN>...] [SourceSE]' % Script.scriptName, ])) Script.parseCommandLine(ignoreErrors=False) from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeGetFile from DIRAC import exit
def __compareFileListWithCatalog( self ): successful = {} failed = {} failedHostKey = {} dmScript = DMScript() lfnFileDict = {} lfnDict = {} # for each file we have one or more http entries, but they all ahve the same lfn # We save the lfns in a dict because FileCatalog.getReplicas needs a dictionary to work # Also for each lfn key we assign the corresponding urlList. for urlList in self.fileList: lfn = dmScript.getLFNsFromList( urlList ) if not len(lfn): self.log.error( "readFederation.__compareFileListWithCatalog: can't get LFN from HTTP url %s" % urlList ) continue lfn = lfn[0] lfnDict[lfn] = True lfnFileDict[lfn] = urlList res = self.__getSEListFromReplicas(lfnDict) if not res: errMsg = "readFederation:.__compareFileListWithCatalog: Failed to get SEs from replicas." self.log.error(errMsg) return S_ERROR( errMsg ) # all the possible storage elements fullSEList = res for lfn, urlList in lfnFileDict.items(): SEListPerLFN = [] for url in urlList: # find on which SEs the file should be stored. SEs = [] res = pfnparse(url) if not res['OK']: continue parsed_dict = res['Value'] host = parsed_dict['Host'] try: for SEName in self.SEDict[host]: # some SEs have identical configuration - so if they have the same host address # we will check if the file is at least in one of these SEs according to the # catalog SEs.append(SEName) except KeyError: failedHostKey[url] = "readFederation.__compareFileListWithCatalog: self.SEDict has no key %s. Check if SE is defined in config." % host SEListPerLFN.append(SEs) confirmedSE = [] # If checking for dedicated SEs clean up the SEListPerLFN so only dedicated SEs # will be checked if self.dedicatedSE: for subList in SEListPerLFN: if any(dedicatedSE in subList for dedicatedSE in self.dedicatedSE): confirmedSE.append(subList) else: confirmedSE = SEListPerLFN if fullSEList.has_key(lfn): SEList = fullSEList[lfn].keys() else: failedHostKey[lfn] = 'No SEList available for this LFN' continue for SESubList in confirmedSE: if not any(SE in SEList for SE in SESubList): if lfn in failed: failed[lfn].append('Failed to find match in catalog for %s' % SESubList) else: failed[lfn] = ['Failed to find match in catalog for %s' % SESubList] else: successful[lfn] = True self.fileList = [] return S_OK( { 'Successful' : successful, 'Failed' : failed, 'Failed Host' : failedHostKey } )
# imports import sys import os import time import DIRAC from DIRAC import gLogger # Code if __name__ == '__main__': # Script initialization from DIRAC.Core.Base import Script from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript dmScript = DMScript() dmScript.registerFileSwitches() depth = 1 Script.setUsageMessage('\n'.join([__doc__, 'Usage:', ' %s [option|cfgfile] [ProdIDs]' % Script.scriptName, ])) Script.registerSwitch('', 'Runs=', ' Specify the run range') Script.registerSwitch('', 'ActiveRunsProduction=', ' Specify the production from which the runs should be derived') Script.registerSwitch('', 'FileType=', 'S pecify the descendants file type') Script.registerSwitch('', 'NoFC', ' Trust the BK replica flag, no LFC check') Script.registerSwitch('', 'FixIt', ' Fix the files in transformation table') Script.registerSwitch('', 'Verbose', ' Print full list of files with error') Script.registerSwitch('', 'Status=', ' Select files with a given status in the production') Script.registerSwitch('', 'Depth=', ' Depth to which to check descendants (default=%d)' % depth) Script.registerSwitch('', 'Force', ' Use this flag to force checking a whole production')
#! /usr/bin/env python """ Set the visibility flag to a dataset """ __RCSID__ = "$Id$" from DIRAC import exit as dExit, gLogger from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script if __name__ == "__main__": dmScript = DMScript() dmScript.registerBKSwitches() dmScript.registerFileSwitches() Script.registerSwitch('', 'List', ' Print out the list of LFNs') Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile]' % Script.scriptName, ])) Script.parseCommandLine(ignoreErrors=False) dumpList = False for switch in Script.getUnprocessedSwitches(): if switch[0] == 'List': dumpList = True bkQuery = dmScript.getBKQuery() lfns = dmScript.getOption('LFNs', [])
# File : dirac-bookkeeping-job-input-output # Author : Zoltan Mathe ######################################################################## """ It returns the input and output files of a given list of DIRAC Jobids """ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, printDMResult import DIRAC import os if __name__ == "__main__": bkScript = DMScript() bkScript.registerJobsSwitches() Script.registerSwitch('', 'InputFiles', ' Only input files') Script.registerSwitch('', 'OutputFiles', ' Only output files') Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... [DIRACJobid|File]' % Script.scriptName, 'Arguments:', ' DIRACJobid: DIRAC Jobids', ' File: Name of the file with contains a list of DIRACJobids' ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() inputFiles = False outputFiles = False for switch in Script.getUnprocessedSwitches():
#!/usr/bin/env python ######################################################################## __RCSID__ = "$Id$" import DIRAC from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript from DIRAC.Core.Base import Script if __name__ == "__main__": dmScript = DMScript() dmScript.registerBKSwitches() dmScript.registerFileSwitches() fixIt = False Script.registerSwitch('', 'FixIt', 'Set replicas problematic if needed') Script.setUsageMessage(""" Check the integrity of the state of the storages and information in the File Catalogs for a given file or a collection of files. Usage: %s <lfn | fileContainingLfns> <SE> <status> """ % Script.scriptName) Script.parseCommandLine() for opt, val in Script.getUnprocessedSwitches(): if opt == 'FixIt': fixIt = True from DIRAC import gLogger
#!/usr/bin/env python ######################################################################## # File : dirac-bookkeeping-productions-summary # Author : Zoltan Mathe ######################################################################## """ Retrieve production summary from the Bookkeeping """ __RCSID__ = "$Id$" import DIRAC from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script dmScript = DMScript() dmScript.registerBKSwitches() Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ...' % Script.scriptName ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() exitCode = 0 from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient bk = BookkeepingClient() bkQuery = dmScript.getBKQuery() if not bkQuery: print "No BKQuery given..."
else: transList.append(int(r[0])) except Exception as e: gLogger.exception("Invalid transformation", lException=e) transList = [] return transList if __name__ == "__main__": import os import DIRAC from DIRAC import gLogger from DIRAC.Core.Base import Script from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript dmScript = DMScript() dmScript.registerFileSwitches() Script.parseCommandLine(ignoreErrors=True) Script.setUsageMessage('\n'.join([ __doc__, 'Usage:', ' %s [option|cfgfile] ...' % Script.scriptName, ])) runInfo = True userGroup = None transList = __getTransformations(Script.getPositionalArgs()) if not transList:
__RCSID__ = "$Id$" # Code if __name__ == '__main__': # Script initialization from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script from DIRAC import gLogger Script.setUsageMessage('\n'.join([ __doc__, 'Usage:', ' %s [option|cfgfile] [values]' % Script.scriptName, ])) dmScript = DMScript() dmScript.registerBKSwitches() dmScript.registerFileSwitches() Script.registerSwitch('', 'FixIt', ' Take action to fix the catalogs') Script.registerSwitch('', 'CheckAllFlags', ' Consider also files with replica flag NO') Script.parseCommandLine(ignoreErrors=True) fixIt = False checkAll = False production = 0 for opt, val in Script.getUnprocessedSwitches(): if opt == 'FixIt': fixIt = True elif opt == 'CheckAllFlags': checkAll = True