def compareFile(dfcUrl):
rstDist = dm.getReplicas(dfcUrl)
if rstDist['OK'] == True:
if dfcUrl in rstDist['Value']['Successful'].keys():
if SITE in rstDist['Value']['Successful'][dfcUrl].keys():
return True
else:
fileUnlinkedList.append(dfcUrl)
return False
else:
fileDarkList.append(dfcUrl)
return False
else:
gLogger.error("Dirac API Error!")
exit(1)
return False
def execute():
tr = TransformationClient()
for switch in Script.getUnprocessedSwitches():
pass
bkQuery = dmScript.getBKQuery()
if not bkQuery:
gLogger.notice("No BKQuery given...")
exit(1)
startTime = time.time()
prods = bkQuery.getBKProductions() # visible = 'All' )
parents = {}
productions = {}
for prod in prods:
type = tr.getTransformation(prod).get('Value',
{}).get('Type', 'Unknown')
productions[prod] = type
parent = tr.getBookkeepingQuery(prod).get('Value',
{}).get('ProductionID', '')
if parent:
type = tr.getTransformation(parent).get('Value',
{}).get('Type', 'Unknown')
parents[parent] = type
gLogger.notice("For BK path %s:" % bkQuery.getPath())
if not prods:
gLogger.notice('No productions found!')
else:
printProds('Productions found', productions)
if parents:
printProds('Parent productions', parents)
gLogger.notice('Completed in %.1f seconds' % (time.time() - startTime))
'Specify transformation type')
Script.registerSwitch('g:', 'groupSize=', 'Group size for each task')
Script.registerSwitch('p:', 'plugin=', 'Plugin, default to "Broadcast"')
Script.registerSwitch('f:', 'sourceSE=', 'SourceSE')
Script.parseCommandLine(ignoreErrors=False)
from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
args = Script.getPositionalArgs()
if len(args) != 3:
Script.showHelp()
exit(1)
transformationName = args[0]
metaTransfer = args[1]
targetSE = args[2]
sourceSE = ''
groupSize = 100
transformationType = 'Transfer-JUNO'
plugin = 'Broadcast'
switches = Script.getUnprocessedSwitches()
for switch in switches:
if switch[0] == 'g' or switch[0] == 'groupSize':
groupSize = int(switch[1])
if switch[0] == 't' or switch[0] == 'transformationType':
result = dm.putAndRegister(lfn, pfn, se)
uploadTime = time.time() - start
if result['OK']:
log += 'Succeed to upload file to SE %s.\n' % se
log += 'Upload Time : %ss\n' % uploadTime
start = time.time()
result = dm.getReplica(lfn, se, tempfile.gettempdir())
downloadTime = time.time() - start
if result['OK']:
log += 'Succeed to download file from SE %s.\n' % se
log += 'Download Time : %ss\n' % downloadTime
else:
exit_code = 1
log += 'Failed to download file from SE %s : %s\n' % (
se, result['Message'])
result = dm.removeFile(lfn)
if result['OK']:
log += 'Succeed to delete file from SE %s.\n' % se
else:
log += 'Faile to delete file from SE %s : %s\n' % (se,
result['Message'])
else:
exit_code = 1
log += 'Failed to upload file to SE %s : %s\n' % (se, result['Message'])
print(log)
exit(exit_code)
switches = [
("n:", "datasetName=", "a dataset that contain a set of files."),
("r:", "DFCDir=", "The logical dir in DFC."),
("m:", "metequery=", "a set of query condition"),
]
for switch in switches:
Script.registerSwitch(*switch)
Script.setUsageMessage(__doc__)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getUnprocessedSwitches()
if not args:
Script.showHelp()
exit(1)
setNameFlag = False
dfcDirFlag = False
queryFlag = False
for switch in args:
if switch[0].lower() == "n" or switch[0].lower() == "datasetName":
setNameFlag = True
setName = switch[1]
elif switch[0].lower() == "r" or switch[0].lower() == "DFCDir":
dfcDirFlag = True
dfcDir = switch[1]
elif switch[0].lower() == "m" or switch[0].lower() == "metequery":
queryFlag = True
setQuery = switch[1]
from IHEPDIRAC.Badger.API.Badger import Badger
from DIRAC import S_OK, S_ERROR, gLogger, gConfig, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
Script.registerSwitch("m:", "method=", "Downloading method")
Script.registerSwitch("D:", "dir=", "Output directory")
Script.registerSwitch("w:", "wait=", "Waiting interval (s)")
Script.registerSwitch("t:", "thread=", "Simultaneously downloading thread number")
Script.parseCommandLine(ignoreErrors=True)
options = Script.getUnprocessedSwitches()
args = Script.getPositionalArgs()
if not args:
Script.showHelp()
exit(1)
setName = args[0]
destDir = '.' #localdir that file download to
#if len(args)>1:
# destDir = args[1]
##print destDir
rsync_se = 'IHEPD-USER'
rsync_url = gConfig.getValue('/Resources/Applications/RsyncEndpoints/%s/Url'%rsync_se, 'rsync://localhost/bes-srm')
method = 'rsync'
output_dir = '.'
interval = 300
for option in options:
(switch, val) = option
if switch == 'm' or switch == 'method':
fileTuple = (lfn, f, size, _se, guid, adler32)
fileTupleBuffer.append(fileTuple)
gLogger.debug('Register to lfn: %s' % lfn)
gLogger.debug('fileTuple: %s' % (fileTuple, ))
if len(fileTupleBuffer) >= _bufferSize:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
if not result['OK']:
gLogger.error('Register file failed')
return 1
del fileTupleBuffer[:]
gLogger.debug('%s files registered' % counter)
if fileTupleBuffer:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
if not result['OK']:
gLogger.error('Register file failed')
return 1
del fileTupleBuffer[:]
gLogger.info('Totally %s files registered' % counter)
return 0
if __name__ == '__main__':
exit(main())
# File : dirac-dms-pfn-metadata.py
# Author : Ph. Charpentier
########################################################################
"""
Check existence of a (list of) LHCb LFNs/PFNs given a valid DIRAC SE (or for all replicas)
Only the LFN contained in the PFN is considered, unlike the DIRAC similar script
"""
__RCSID__ = "$Id$"
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script, printDMResult
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerFileSwitches()
dmScript.registerSiteSwitches()
Script.registerSwitch('', 'Summary',
' Only prints a summary on existing files')
Script.setUsageMessage('\n'.join([
__doc__, 'Usage:',
' %s [option|cfgfile] ... [URL[,URL2[,URL3...]]] SE[ SE2...]' %
Script.scriptName, 'Arguments:',
' URL: Logical/Physical File Name or file containing URLs',
' SE: Valid DIRAC SE'
]))
Script.parseCommandLine(ignoreErrors=True)
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executePfnMetadata
from DIRAC import exit
exit(executePfnMetadata(dmScript, check=True, exists=True))
def setNumOfPingsToDo( self, value ):
try:
self.pingsToDo = max( 1, int( value ) )
except ValueError:
return S_ERROR( "Number of pings to do has to be a number" )
return S_OK()
# Instantiate the params class
cliParams = Params()
# Register accepted switches and their callbacks
Script.registerSwitch( "r", "showRaw", "show raw result from the query", cliParams.setRawResult )
Script.registerSwitch( "p:", "numPings=", "Number of pings to do (by default 1)", cliParams.setNumOfPingsToDo )
# Define a help message
Script.setUsageMessage( '\n'.join( [ __doc__,
'Usage:',
' %s [option|cfgfile] <system name to ping>+' % Script.scriptName,
' Specifying a system is mandatory' ] ) )
# Parse the command line and initialize DIRAC
Script.parseCommandLine( ignoreErrors = False )
# Get the list of services
servicesList = Script.getPositionalArgs()
# Check and process the command line switches and options
if len( servicesList ) == 0:
Script.showHelp()
exit(1)
Only the LFN contained in the PFN is considered, unlike the DIRAC similar script
"""
__RCSID__ = "$Id$"
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script, printDMResult
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerFileSwitches()
dmScript.registerSiteSwitches()
Script.registerSwitch('', 'Check',
' Checks the PFN metadata vs LFN metadata')
Script.registerSwitch(
'', 'Exists',
' Only reports if the file exists (and checks the checksum)')
Script.registerSwitch('', 'Summary',
' Only prints a summary on existing files')
Script.setUsageMessage('\n'.join([
__doc__, 'Usage:',
' %s [option|cfgfile] ... [URL[,URL2[,URL3...]]] SE[ SE2...]' %
Script.scriptName, 'Arguments:',
' URL: Logical/Physical File Name or file containing URLs',
' SE: Valid DIRAC SE'
]))
Script.parseCommandLine(ignoreErrors=True)
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executePfnMetadata
from DIRAC import exit
exit(executePfnMetadata(dmScript))
elif switch[0] == 'Mode':
mode = int(switch[1], base=8)
elif switch[0] == 'Create':
create = True
args = Script.getPositionalArgs()
if dirList is None and args:
dirList = args
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from LHCbDIRAC.DataManagementSystem.Utilities.FCUtilities import chown
res = getProxyInfo()
if not res['OK']:
gLogger.fatal("Can't get proxy info", res['Message'])
exit(1)
properties = res['Value'].get('groupProperties', [])
if not 'FileCatalogManagement' in properties:
gLogger.error(
"You need to use a proxy from a group with FileCatalogManagement")
exit(5)
if group and not group.startswith('lhcb_'):
gLogger.fatal("This is not a valid group", group)
exit(1)
directories = set()
for baseDir in dirList:
if os.path.exists(baseDir):
f = open(baseDir)
directories.update([
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerFileSwitches()
dmScript.registerSiteSwitches()
Script.registerSwitch(
"n", "NoFC",
" use this option to force the removal from storage of replicas not in FC"
)
Script.registerSwitch(
'', 'ReduceReplicas=',
' specify the number of replicas you want to keep (default SE: Tier1-USER)'
)
Script.registerSwitch(
"", "Force",
" use this option for force the removal of replicas even if last one")
Script.setUsageMessage('\n'.join(
__doc__.split('\n') + [
'Usage:',
' %s [option|cfgfile] ... [LFN[,LFN2[,LFN3...]]] SE[,SE2...]' %
Script.scriptName, 'Arguments:',
' LFN: Logical File Name or file containing LFNs',
' SE: Valid DIRAC SE'
]))
Script.parseCommandLine()
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeRemoveReplicas
from DIRAC import exit
exit(executeRemoveReplicas(dmScript))
switches = [
("n:","datasetName=","a dataset that contain a set of files."),
("r:","DFCDir=","The logical dir in DFC."),
("m:","metequery=","a set of query condition"),
("f:","fileName=","a fileList in this file"),
]
for switch in switches:
Script.registerSwitch(*switch)
Script.setUsageMessage(__doc__)
Script.parseCommandLine(ignoreErrors=True)
#args = Script.getPositionalArgs()
args = Script.getUnprocessedSwitches()
if not args:
Script.showHelp()
exit(-1)
setNameFlag = False
dfcDirFlag = False
queryFlag = False
filenameFlag = False
for switch in args:
if switch[0].lower() == "n" or switch[0].lower() == "datasetName":
setNameFlag = True
setName = switch[1]
if switch[0].lower() == "r" or switch[0].lower() == "DFCDir":
dfcDirFlag = True
dfcDir = switch[1]
if switch[0].lower() == "m" or switch[0].lower() == "metequery":
queryFlag = True
setQuery = switch[1]
if switch[0].lower() == "f" or switch[0].lower() == "fileName":
gLogger.notice(
'Would fix the discrepancy by replicating files from other SEs.')
gLogger.notice("\n\n")
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
fcc = FileCatalogClient('DataManagement/FileCatalog')
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
#get all files in dfcDir
gf_result = dm.getFilesFromDirectory(dfcDir)
if not gf_result['OK']:
gLogger.error('Failed to list directory %s: %s' %
(dfcDir, gf_result['Message']))
exit()
files = gf_result['Value']
fileNumber = len(files)
gLogger.notice('There are {0} files in dir {1}'.format(fileNumber, dfcDir))
#get replicas in the SE
gr_result = dm.getReplicasFromDirectory(dfcDir)
if not gr_result['OK']:
gLogger.error('Failed to get replicas from directory %s: %s' %
(dfcDir, gr_result['Message']))
exit()
replicas = {SE: [] for SE in SE_list}
for LFN in gr_result['Value'].keys():
repInfo = gr_result['Value'][LFN]
for SE in SE_list:
Script.registerSwitch("", "Days=",
"Match files older than number of days [%s]" % days)
Script.registerSwitch(
"", "Months=", "Match files older than number of months [%s]" % months)
Script.registerSwitch(
"", "Years=", "Match files older than number of years [%s]" % years)
Script.registerSwitch("", "Wildcard=",
"Wildcard for matching filenames [%s]" % wildcard)
Script.registerSwitch('', 'Output', 'Write list to an output file')
Script.registerSwitch("", "EmptyDirs",
"Create a list of empty directories")
Script.registerSwitch(
"", "Depth=",
"Depth to which recursively browse (default = %d)" % depth)
Script.registerSwitch("r", "Recursive", "Set depth to infinite")
Script.registerSwitch('', 'NoDirectories',
'Only print out only files, not subdirectories')
dmScript.registerBKSwitches()
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1],
'Usage:',
' %s [option|cfgfile] ...' % Script.scriptName,
]))
Script.parseCommandLine(ignoreErrors=False)
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeListDirectory
from DIRAC import exit
exit(executeListDirectory(dmScript, days, months, years, wildcard, depth))
start = time.time()
result = dm.removeFile( lfn )
result = dm.putAndRegister( lfn, pfn, se )
uploadTime = time.time() - start
if result[ 'OK' ]:
log += 'Succeed to upload file to SE %s.\n' % se
log += 'Upload Time : %ss\n' % uploadTime
start = time.time()
result = dm.getReplica( lfn, se, tempfile.gettempdir() )
downloadTime = time.time() - start
if result[ 'OK' ]:
log += 'Succeed to download file from SE %s.\n' % se
log += 'Download Time : %ss\n' % downloadTime
else:
exit_code = 1
log += 'Failed to download file from SE %s : %s\n' % ( se, result[ 'Message' ] )
result = dm.removeFile( lfn )
if result[ 'OK' ]:
log += 'Succeed to delete file from SE %s.\n' % se
else:
log += 'Faile to delete file from SE %s : %s\n' % ( se, result[ 'Message' ] )
else:
exit_code = 1
log += 'Failed to upload file to SE %s : %s\n' % ( se, result[ 'Message' ] )
print(log)
exit(exit_code)
#!/usr/bin/env python
"""
Retrieve an access URL for an LFN replica given a valid DIRAC SE.
"""
__RCSID__ = "$Id$"
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerFileSwitches()
dmScript.registerSiteSwitches()
Script.registerSwitch(
'', 'Protocol=',
' Define the protocol for which a tURL is requested (default:root)')
Script.setUsageMessage('\n'.join(
__doc__.split('\n') + [
'Usage:',
' %s [option|cfgfile] ... [LFN[,LFN2[,LFN3...]]] SE[,SE2...]' %
Script.scriptName, 'Arguments:',
' LFN: Logical File Name or file containing LFNs',
' SE: Valid DIRAC SE'
]))
Script.parseCommandLine(ignoreErrors=True)
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeAccessURL
from DIRAC import exit
exit(executeAccessURL(dmScript))
#!/usr/bin/env python
"""
Remove replicas of a (list of) LFNs from all non-ARCHIVE storage elements
"""
__RCSID__ = "$Id$"
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerFileSwitches()
dmScript.registerBKSwitches()
Script.registerSwitch("", "Force", " use this option for force the removal of files without ARCHIVE")
Script.setUsageMessage('\n'.join(__doc__.split('\n') + [
'Usage:',
' %s [option|cfgfile] ... [LFN[,LFN2[,LFN3...]]] SE[,SE2...]' % Script.scriptName]))
Script.parseCommandLine()
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeRemoveReplicas
from DIRAC import exit
exit(executeRemoveReplicas(dmScript, allDisk=True))
"""
Show replicas for a (set of) LFNs
"""
__RCSID__ = "$Id$"
__VERSION__ = "$Revision: 86918 $"
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerFileSwitches()
Script.registerSwitch('a', "All", " Also show inactive replicas")
Script.registerSwitch('', 'DiskOnly', ' Show only disk replicas')
Script.registerSwitch('', 'PreferDisk',
" If disk replica, don't show tape replicas")
Script.registerSwitch('', 'ForJobs',
' Select only replicas that can be used for jobs')
Script.setUsageMessage('\n'.join([
__doc__,
'Usage:',
' %s [option|cfgfile] [<LFN>] [<LFN>...]' % Script.scriptName,
]))
Script.parseCommandLine(ignoreErrors=False)
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeLfnReplicas
from DIRAC import exit
exit(executeLfnReplicas(dmScript))
#!/usr/bin/env python
"""
Set a (set of) LFNs as problematic in the FC and in the BK and transformation system if all replicas are problematic
"""
__RCSID__ = "$Id$"
__VERSION__ = "$Revision: 87258 $"
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerFileSwitches()
dmScript.registerSiteSwitches()
Script.registerSwitch('', 'Reset', ' Reset files to OK')
Script.registerSwitch('', 'Full', ' Give full list of files')
Script.registerSwitch('', 'NoAction', ' No action taken, just give stats')
Script.setUsageMessage('\n'.join([__doc__.split('\n')[1],
'Usage:',
' %s [option|cfgfile] [<LFN>] [<LFN>...]' % Script.scriptName, ]))
Script.parseCommandLine(ignoreErrors=False)
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeSetProblematicFiles
from DIRAC import exit
exit(executeSetProblematicFiles(dmScript))
#!/usr/bin/env python
########################################################################
# File : dirac-dms-replicate-to-run-destination
# Author : Philippe Charpentier
########################################################################
"""
Replicate a (list of) existing LFN(s) to Ses defined by the run destination
"""
__RCSID__ = "$Id$"
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerFileSwitches()
dmScript.registerSiteSwitches()
Script.registerSwitch('', 'RemoveSource',
' If set, the source replica(s) will be removed')
Script.setUsageMessage('\n'.join([
__doc__, 'Usage:',
' %s [option|cfgfile] ... [LFN1[,LFN2,[...]]] Dest[,Dest2[,...]] [Source [Cache]]'
% Script.scriptName, 'Arguments:', ' Dest: Valid DIRAC SE(s)'
]))
Script.parseCommandLine(ignoreErrors=True)
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeReplicateToRunDestination
from DIRAC import exit
exit(executeReplicateToRunDestination(dmScript))
# File : dirac-dms-replicate-lfn
# Author : Stuart Paterson
########################################################################
"""
Replicate a (list of) existing LFN(s) to (set of) Storage Element(s)
"""
__RCSID__ = "$Id$"
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerFileSwitches()
dmScript.registerSiteSwitches()
Script.registerSwitch('', 'RemoveSource',
' If set, the source replica(s) will be removed')
Script.setUsageMessage('\n'.join([
__doc__, 'Usage:',
' %s [option|cfgfile] ... [LFN1[,LFN2,[...]]] Dest[,Dest2[,...]] [Source [Cache]]'
% Script.scriptName, 'Arguments:', ' Dest: Valid DIRAC SE(s)',
' Source: Valid DIRAC SE',
' Cache: Local directory to be used as cache'
]))
Script.parseCommandLine(ignoreErrors=True)
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeReplicateLfn
from DIRAC import exit
exit(executeReplicateLfn(dmScript))
#!/usr/bin/env python
#mtime:2013/12/09
"""
besdirac-dms-dataset-filelist
get the filelist of the given dataset
Usage:
besdirac-dms-dataset-filelist<datasetname>
"""
__RCSID__ = "$Id$"
import pprint
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp()
datasetName = args[0]
from IHEPDIRAC.Badger.API.Badger import Badger
badger = Badger()
result = badger.getFilesByDatasetName(datasetName)
if result['OK']:
fileList = result['Value']
pprint.pprint(fileList)
exit(0)
# File : dirac-dms-get-file
# Author : Stuart Paterson
########################################################################
"""
Retrieve a single file or list of files from Grid storage to the current directory.
"""
__RCSID__ = "$Id$"
import os
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerFileSwitches()
dmScript.registerNamespaceSwitches('download to (default = %s)' %
os.path.realpath('.'))
Script.setUsageMessage('\n'.join([
__doc__,
'Usage:',
' %s [option|cfgfile] [<LFN>] [<LFN>...] [SourceSE]' %
Script.scriptName,
]))
Script.parseCommandLine(ignoreErrors=False)
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeGetFile
from DIRAC import exit
exit(executeGetFile(dmScript))
#!/usr/bin/env python
# mtime:2013/12/09
"""
besdirac-dms-dataset-check
check if the dataset changed
Usage:
besdirac-dms-dataset-check <datasetname>
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp()
datasetName = args[0]
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
badger.checkDataset(datasetName)
exit(0)
#!/usr/bin/env python
"""
Remove the given file or a list of files from the File Catalog and from the storage
"""
__RCSID__ = "$Id$"
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerBKSwitches()
dmScript.registerFileSwitches()
Script.registerSwitch(
'', 'IncludeProcessedFiles',
' Forced to set Removed the files in status Processed (default:not reset)'
)
Script.setUsageMessage('\n'.join([
__doc__,
'Usage:',
' %s [option|cfgfile] [<LFN>] [<LFN>...]' % Script.scriptName,
]))
Script.parseCommandLine()
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeRemoveFiles
from DIRAC import exit
exit(executeRemoveFiles(dmScript))
# $HeadURL$
# File : dirac-dms-add-file
########################################################################
"""
Upload a file to the grid storage and register it in the File Catalog
"""
__RCSID__ = "$Id$"
if __name__ == "__main__":
from DIRAC.Core.Base import Script
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... LFN Path SE [GUID]' % Script.scriptName,
'Arguments:', ' LFN: Logical File Name',
' Path: Local path of the file',
' SE: DIRAC Storage Element',
' GUID: GUID to use in the registration (optional)', '',
' ++ OR ++', '', 'Usage:',
' %s [option|cfgfile] ... LocalFile' % Script.scriptName,
'Arguments:',
' LocalFile: Path to local file containing all the above, i.e.:',
' lfn1 localfile1 SE [GUID1]', ' lfn2 localfile2 SE [GUID2]'
]))
Script.parseCommandLine(ignoreErrors=True)
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeAddFile
from DIRAC import exit
exit(executeAddFile())
"""
__RCSID__ = "$Id$"
import time
import os
import anydbm
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.registerSwitch("r", "dir", "the directory that dataset files located")
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
if len(args) == 0:
Script.showHelp()
exit(-1)
ePoint = ''
if len(args) > 1:
energyPoint = args[1]
from IHEPDIRAC.Badger.API.Badger import Badger
from IHEPDIRAC.Badger.API.multiworker import IWorker, MultiWorker
def getDB(name, function):
"""return a db instance,the db contain the file list.
default value is 0,means the file is not transfer yet,if 2,means OK.
"""
dbname = "db_" + name[1] + '_' + str(name[-3:])
if not os.path.exists(dbname):
fileList = function(name)
path: the path as the root path
conditions: meta query conditions
Example:
besdirac-dms-dataset-create name /zhanggang_test/ "runL>111 runH<200 resonance=jpsi"
"""
__RCSID__ = "$Id$"
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage(__doc__)
args = Script.getPositionalArgs()
# print len(args)
if len(args) < 3:
Script.showHelp()
exit(-1)
datasetName = args[0]
path = args[1]
strArg = args[2]
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
prefix = badger.getDatasetNamePrefix()
datasetName = prefix + datasetName
# print datasetName
result = badger.registerDataset(datasetName, path, strArg)
if result["OK"]:
resVal = badger.getFilesByDatasetName(datasetName)
switches = [
("n:","datasetName=","a dataset that contain a set of files."),
("r:","DFCDir=","The logical dir in DFC."),
("m:","metequery=","a set of query condition"),
("f:","fileName=","a fileList in this file"),
]
for switch in switches:
Script.registerSwitch(*switch)
Script.setUsageMessage(__doc__)
Script.parseCommandLine(ignoreErrors=True)
#args = Script.getPositionalArgs()
args = Script.getUnprocessedSwitches()
if not args:
Script.showHelp()
exit(-1)
setNameFlag = False
dfcDirFlag = False
queryFlag = False
filenameFlag = False
for switch in args:
if switch[0].lower() == "n" or switch[0].lower() == "datasetName":
setNameFlag = True
setName = switch[1]
if switch[0].lower() == "r" or switch[0].lower() == "DFCDir":
dfcDirFlag = True
dfcDir = switch[1]
if switch[0].lower() == "m" or switch[0].lower() == "metequery":
queryFlag = True
setQuery = switch[1]
if switch[0].lower() == "f" or switch[0].lower() == "fileName":
import sys
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.Core.Base import Script
usageMsg = '''Remove specified replica from SE under specified directory
{0} [option|cfgfile] DFCDir SE'''.format(Script.scriptName)
Script.setUsageMessage(usageMsg)
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if len(args) != 2:
Script.showHelp()
exit(1)
dfcDir = args[0]
SE = args[1]
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
fcc = FileCatalogClient('DataManagement/FileCatalog')
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
counterFile = 0
counterDir = 0
def removeFromDir(d):
#!/usr/bin/env python
########################################################################
# File : dirac-dms-register-bk2fc
# Author : Philippe Charpentier
########################################################################
"""
Given a (list of) LFNs and SEs, check for existence of a file and register in the FC if the file exists
"""
__RCSID__ = "$Id$"
from LHCbDIRAC.DataManagementSystem.Client.DMScript import DMScript, Script
if __name__ == "__main__":
dmScript = DMScript()
dmScript.registerFileSwitches()
dmScript.registerSiteSwitches()
Script.setUsageMessage('\n'.join([
__doc__, 'Usage:',
' %s [option|cfgfile] ... [LFN1[,LFN2,[...]]] [--SE] Dest[,Dest2[,...]] '
% Script.scriptName, 'Arguments:', ' Dest: Valid DIRAC SE(s)'
]))
Script.parseCommandLine(ignoreErrors=True)
from LHCbDIRAC.DataManagementSystem.Client.ScriptExecutors import executeRegisterBK2FC
from DIRAC import exit
exit(executeRegisterBK2FC(dmScript))
