def _getOutputs():
repoLocation = ''
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine( ignoreErrors = False )
repoLocation = clip.repo
if not repoLocation:
Script.showHelp()
dexit(1)
from DIRAC import gLogger
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac(True, repoLocation)
exitCode = 0
res = dirac.monitorRepository(False)
if not res['OK']:
gLogger.error("Failed because %s" % res['Message'])
dexit(1)
res = dirac.retrieveRepositorySandboxes()
if not res['OK']:
gLogger.error("Failed because %s" % res['Message'])
dexit(1)
if clip.outputdata:
res = dirac.retrieveRepositoryData()
if not res['OK']:
gLogger.error("Failed because %s" % res['Message'])
exit(1)
dexit(exitCode)
def _runFSTAgent():
""" read commands line params and run FST agent for a given transformation ID """
params = _Params()
params.registerSwitches()
Script.parseCommandLine()
if not params.checkSettings()['OK']:
Script.showHelp()
dexit(1)
from ILCDIRAC.ILCTransformationSystem.Agent.FileStatusTransformationAgent import FileStatusTransformationAgent
fstAgent = FileStatusTransformationAgent('ILCTransformation/FileStatusTransformationAgent',
'ILCTransformation/FileStatusTransformationAgent',
'dirac-ilc-filestatus-transformation')
fstAgent.log = gLogger
fstAgent.enabled = params.enabled
res = fstAgent.getTransformations(transID=params.transID)
if not res['OK']:
dexit(1)
if not res['Value']:
print("Transformation Not Found")
dexit(1)
trans = res['Value'][0]
res = fstAgent.processTransformation(
int(params.transID), trans['SourceSE'], trans['TargetSE'], trans['DataTransType'])
if not res["OK"]:
dexit(1)
dexit(0)
def addProperty( arg ):
global groupProperties
if not arg:
Script.showHelp()
DIRAC.exit( -1 )
if not arg in groupProperties:
groupProperties.append( arg )
def _createTrafo():
"""reads command line parameters, makes check and creates replication transformation"""
clip = Params()
clip.registerSwitches(Script)
registerSwitches(Script)
Script.parseCommandLine()
if not clip.checkSettings(Script)['OK']:
LOG.error("ERROR: Missing settings")
return 1
for prodID in clip.metaValues:
tGroup = getTransformationGroup(prodID, clip.groupName)
parDict = dict(flavour='Replication',
targetSE=clip.targetSE,
sourceSE=clip.sourceSE,
metaKey=clip.metaKey,
metaValue=prodID,
extraData={'Datatype': clip.datatype},
extraname=clip.extraname,
plugin=clip.plugin,
groupSize=clip.groupSize,
tGroup=tGroup,
enable=clip.enable,
)
LOG.debug("Parameters: %s" % pformat(parDict))
resCreate = createDataTransformation(**parDict)
if not resCreate['OK']:
return 1
return 0
def _createTrafo():
"""reads command line parameters, makes check and creates replication transformation"""
clip = Params()
clip.registerSwitches(Script)
registerSwitches(clip, Script)
Script.parseCommandLine()
if not clip.checkSettings(Script)['OK']:
LOG.error("ERROR: Missing settings")
return 1
for index, prodID in enumerate(clip.metaValues):
datatype = clip.datatype if clip.datatype else ['GEN', 'SIM', 'REC'][index % 3]
plugin = 'Broadcast' if clip.forcemoving or clip.flavour != 'Moving' else 'BroadcastProcessed'
retData = checkDatatype(prodID, datatype)
if not retData['OK']:
LOG.error("ERROR: %s" % retData['Message'])
return 1
tGroup = getTransformationGroup(prodID, clip.groupName)
parDict = dict(flavour='Moving',
targetSE=clip.targetSE,
sourceSE=clip.sourceSE,
metaKey=clip.metaKey,
metaValue=prodID,
extraData={'Datatype': datatype},
extraname=clip.extraname,
plugin=plugin,
groupSize=clip.groupSize,
tGroup=tGroup,
enable=clip.enable,
)
LOG.debug("Parameters: %s" % pformat(parDict))
resCreate = createDataTransformation(**parDict)
if not resCreate['OK']:
return 1
return 0
def _showSoftware():
"""Show available software"""
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine()
from DIRAC import gConfig, gLogger
base = '/Operations/Defaults/AvailableTarBalls'
platforms = gConfig.getSections(base)
for platform in platforms['Value']:
gLogger.notice("For platform %s, here are the available software" % platform)
apps = gConfig.getSections(base + "/" + platform)
for app in apps['Value']:
if clip.software and app.lower() != clip.software.lower():
continue
gLogger.notice(" - %s" % app)
versions = gConfig.getSections(base + "/" + platform + "/" + app)
if clip.appsOnly:
continue
for vers in versions['Value']:
gLogger.notice(" * %s" % vers)
depsb = gConfig.getSections(base + "/" + platform + "/" + app + "/" + vers)
if 'Dependencies' in depsb['Value']:
gLogger.notice(" Depends on")
deps = gConfig.getSections( os.path.join( base, platform, app, vers , "Dependencies") )
for dep in deps['Value']:
depversions = gConfig.getOption(base + "/" + platform + "/" + app + "/" + vers + "/Dependencies/" + dep + "/version")
gLogger.notice(" %s %s" % (dep, depversions['Value']))
if not len(versions['Value']):
gLogger.notice(" No version available")
def main():
if len(args) < 1:
Script.showHelp()
return
fixedStatuses = ['Total', 'Done', 'Failed', 'Running', 'Waiting', 'Deleted']
for taskID in args:
taskID = int(taskID)
progress = getTaskEventProgress(taskID)
if not progress:
continue
print '='*80
print 'Task %s progress:' % taskID
print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12
print '%-12s %-12s %-12s' % ('Status', 'Job Number', 'Event Number')
print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12
printALine(progress, 'Total')
print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12
for fixedStatus in fixedStatuses:
if fixedStatus != 'Total':
printALine(progress, fixedStatus)
for p in progress:
if p not in fixedStatuses:
printALine(progress, p)
print '-'*12 + ' ' + '-'*12 + ' ' + '-'*12
print ''
def addProperty( arg ):
global hostProperties
if not arg:
Script.showHelp()
DIRAC.exit( -1 )
if not arg in hostProperties:
hostProperties.append( arg )
def addUserGroup(arg):
global userGroups
if not arg:
Script.showHelp()
DIRAC.exit(-1)
if not arg in userGroups:
userGroups.append(arg)
def _createTrafo():
"""reads command line parameters, makes check and creates replication transformation"""
clip = Params()
clip.registerSwitches(Script)
Script.parseCommandLine()
if not clip.checkSettings(Script)['OK']:
gLogger.error("ERROR: Missing settings")
return 1
for metaValue in clip.metaValues:
resCreate = createDataTransformation(flavour=clip.flavour,
targetSE=clip.targetSE,
sourceSE=clip.sourceSE,
metaKey=clip.metaKey,
metaValue=metaValue,
extraData=clip.extraData,
extraname=clip.extraname,
groupSize=clip.groupSize,
plugin=clip.plugin,
enable=clip.enable,
)
if not resCreate['OK']:
gLogger.error("Failed to create Transformation", resCreate['Message'])
return 1
return 0
def addUserName( arg ):
global userNames
if not arg:
Script.showHelp()
DIRAC.exit( -1 )
if not arg in userNames:
userNames.append( arg )
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if args:
subLogger.error( "Found the following positional args '%s', but we only accept switches" % args )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
switches = dict( Script.getUnprocessedSwitches() )
switches.setdefault( 'statusType' , None )
switches.setdefault( 'releaseToken', False )
for key in ( 'element', 'name', 'reason' ):
if not key in switches:
subLogger.error( "%s Switch missing" % key )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
if not switches[ 'element' ] in ( 'Site', 'Resource', 'Node' ):
subLogger.error( "Found %s as element switch" % switches[ 'element' ] )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return switches
def runTests():
"""runs the tests"""
clip = CLIParams()
clip.registerSwitches()
Script.parseCommandLine()
overlayrun = clip.testOverlay
myMarlinSteeringFile = "clic_ild_cdr_steering_overlay_1400.0.xml" if overlayrun else "clic_ild_cdr_steering.xml"
myLCSimPreSteeringFile = "clic_cdr_prePandoraOverlay_1400.0.lcsim" if overlayrun else "clic_cdr_prePandora.lcsim"
myLCSimPostSteeringFile = "clic_cdr_postPandoraOverlay.lcsim"
parameterDict = dict( mokkaVersion="0706P08",
mokkaSteeringFile="clic_ild_cdr.steer",
detectorModel="CLIC_ILD_CDR",
steeringFileVersion="V22",
machine="clic_cdr",
backgroundType="gghad",
energy=1400,
marlinVersion="v0111Prod",
rootVersion="5.34",
marlinSteeringFile=myMarlinSteeringFile,
marlinInputdata = "/ilc/user/s/sailer/testFiles/prod_clic_ild_e2e2_o_sim_2214_26.slcio",
gearFile='clic_ild_cdr.gear',
lcsimPreSteeringFile=myLCSimPreSteeringFile,
lcsimPostSteeringFile=myLCSimPostSteeringFile
)
myTests = TestCreater(clip, parameterDict)
res = myTests.checkForTests()
if not res['OK']:
dexit(1)
myTests.run()
return
def getBoolean( value ):
if value.lower() == 'true':
return True
elif value.lower() == 'false':
return False
else:
Script.showHelp()
def registerSwitches(self):
Script.registerSwitch('P:', "Platform=", 'Platform to use', self.setPlatform)
Script.registerSwitch('p:', "Path=", "Path to the Whizard results directory", self.setPath)
Script.registerSwitch("V:", "Version=", "Whizard version", self.setVersion)
Script.registerSwitch('b:', 'BeamSpectra=', 'Beam spectra version', self.setBeamSpectra)
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'\nUsage:',
' %s [option|cfgfile] ...\n' % Script.scriptName ] ) )
def setUp( self ): from DIRAC.Core.Base import Script Script.parseCommandLine() from DIRAC.FrameworkSystem.Client.Logger import gLogger gLogger.showHeaders( True ) self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.processPool = ProcessPool( 4, 8, 8 ) self.processPool.daemonize()
def __init__( self, taskID, timeWait, raiseException=False ): from DIRAC.Core.Base import Script Script.parseCommandLine() from DIRAC.FrameworkSystem.Client.Logger import gLogger self.log = gLogger.getSubLogger( self.__class__.__name__ + "/%s" % taskID ) self.taskID = taskID self.timeWait = timeWait self.raiseException = raiseException
def main():
if len(args) < 1:
Script.showHelp()
return
for taskID in args:
taskID = int(taskID)
deleteTask(taskID)
print ""
def setUserMail(arg):
global userMail
if userMail or not arg:
Script.showHelp()
DIRAC.exit(-1)
if not arg.find("@") > 0:
Script.gLogger.error("Not a valid mail address", arg)
DIRAC.exit(-1)
userMail = arg
def registerUsageMessage():
'''
Takes the script __doc__ and adds the DIRAC version to it
'''
usageMessage = 'DIRAC version: %s \n' % version
usageMessage += __doc__
Script.setUsageMessage( usageMessage )
def registerUsageMessage():
"""
Takes the script __doc__ and adds the DIRAC version to it
"""
usageMessage = "DIRAC version: %s \n" % version
usageMessage += __doc__
Script.setUsageMessage(usageMessage)
def error( msg ):
'''
Format error messages
'''
subLogger.error( "\nERROR:" )
subLogger.error( "\t" + msg )
subLogger.error( "\tPlease, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
def isOK(self):
if not self.cliParams.releasesToBuild:
gLogger.error("Missing releases to build!")
Script.showHelp()
return False
if not self.cliParams.destination:
self.cliParams.destination = tempfile.mkdtemp('DiracDist')
else:
mkDir(self.cliParams.destination)
gLogger.notice("Will generate tarballs in %s" % self.cliParams.destination)
return True
def __init__( self, taskID, timeWait, raiseException=False ):
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from DIRAC.FrameworkSystem.Client.Logger import gLogger
self.log = gLogger.getSubLogger( self.__class__.__name__ + "/%s" % taskID )
self.taskID = taskID
self.log.always( "pid=%s task=%s I'm locked" % ( os.getpid(), self.taskID ) )
gLock.acquire()
self.log.always("you can't see that line, object is stuck by gLock" )
self.timeWait = timeWait
self.raiseException = raiseException
gLock.release()
def main():
if len(args) < 1:
Script.showHelp()
return
jobIDs = []
for jobID in args:
jobIDs.append(int(jobID))
showJobs(jobIDs)
print 'Totally %s job(s) displayed' % len(jobIDs)
def main():
"""
main program entry point
"""
options = Params()
options.registerCLISwitches()
Script.parseCommandLine( ignoreErrors = True )
if options.delete_all and options.vos:
gLogger.error( "-a and -v options are mutually exclusive. Please pick one or the other." )
return 1
proxyLoc = Locations.getDefaultProxyLocation()
if not os.path.exists( proxyLoc ):
gLogger.error( "No local proxy found in %s, exiting." % proxyLoc )
return 1
result = ProxyInfo.getProxyInfo( proxyLoc, True )
if not result[ 'OK' ]:
raise RuntimeError( 'Failed to get local proxy info.' )
if result[ 'Value' ][ 'secondsLeft' ] < 60 and options.needsValidProxy():
raise RuntimeError( 'Lifetime of local proxy too short, please renew proxy.' )
userDN=result[ 'Value' ][ 'identity' ]
if options.delete_all:
# delete remote proxies
remote_groups = getProxyGroups()
if not remote_groups:
gLogger.notice( 'No remote proxies found.' )
for vo_group in remote_groups:
deleteRemoteProxy( userDN, vo_group )
# delete local proxy
deleteLocalProxy( proxyLoc )
elif options.vos:
vo_groups = set()
for voname in options.vos:
vo_groups.update(mapVoToGroups( voname ) )
# filter set of all groups to only contain groups for which there is a user proxy
user_groups = getProxyGroups()
vo_groups.intersection_update( user_groups )
if not vo_groups:
gLogger.notice( 'You have no proxies registered for any of the specified VOs.' )
for group in vo_groups:
deleteRemoteProxy( userDN, group )
else:
deleteLocalProxy( proxyLoc )
return 0
def setUpClass( cls ): # destroy kerberos token #try: # subprocess.call(['kdestroy']) #except subprocess.CalledProcessError as err: # print "WARNING: kdestroy did not succeed." # print err.output # Constants for the tests Script.parseCommandLine() user = ProxyInfo.getProxyInfo()['Value']['username'] SETestCase.lfntestfilepath += '%s/%s/setests/%s/' % (user[0], user, randomFolder()) print "Using lfn %s" % SETestCase.lfntestfilepath SETestCase.lfntestfile = SETestCase.lfntestfilepath + SETestCase.lfntestfilename
def _createTrafo():
"""reads command line parameters, makes check and creates replication transformation"""
from DIRAC import exit as dexit
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine()
if not clip.checkSettings()['OK']:
gLogger.error("ERROR: Missing settings")
dexit(1)
resCreate = _createReplication( clip.targetSE, clip.sourceSE, clip.prodID, clip.datatype, clip.extraname )
if not resCreate['OK']:
dexit(1)
dexit(0)
def registerSwitches():
'''
Registers all switches that can be used while calling the script from the
command line interface.
'''
switches = (
( 'init', 'Initialize the element to the status in the CS ( applicable for StorageElements )' ),
( 'element=', 'Element family to be Synchronized ( Site, Resource or Node ) or `all`' ),
)
for switch in switches:
Script.registerSwitch( '', switch[ 0 ], switch[ 1 ] )
def registerUsageMessage():
'''
Takes the script __doc__ and adds the DIRAC version to it
'''
hLine = ' ' + '='*78 + '\n'
usageMessage = hLine
usageMessage += ' DIRAC %s\n' % version
usageMessage += __doc__
usageMessage += '\n' + hLine
Script.setUsageMessage( usageMessage )
#!/usr/bin/env python
########################################################################
# $HeadURL$
# File : dirac-production-job-delete
# Author : Stuart Paterson
########################################################################
"""
Delete DIRAC job from WMS, if running it will be killed
"""
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Base import Script
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... JobID ...' % Script.scriptName, 'Arguments:',
' JobID: DIRAC Job ID'
]))
Script.registerSwitch("f:", "File=",
"Get output for jobs with IDs from the file")
Script.registerSwitch("g:", "JobGroup=",
"Get output for jobs in the given group")
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
import os.path
if __name__ == "__main__":
#!/usr/bin/env python
########################################################################
# $HeadURL$
# File : dirac-jobexec
# Author : Stuart Paterson
########################################################################
__RCSID__ = "$Id$"
""" The dirac-jobexec script is equipped to execute workflows that
are specified via their XML description. The main client of
this script is the Job Wrapper.
"""
import DIRAC
from DIRAC.Core.Base import Script
# Register workflow parameter switch
Script.registerSwitch('p:', 'parameter=',
'Parameters that are passed directly to the workflow')
Script.parseCommandLine()
from DIRAC.Core.Workflow.Parameter import *
from DIRAC.Core.Workflow.Module import *
from DIRAC.Core.Workflow.Step import *
from DIRAC.Core.Workflow.Workflow import *
from DIRAC.Core.Workflow.WorkflowReader import *
from DIRAC import S_OK, S_ERROR, gConfig, gLogger
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient
from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer
import DIRAC
import os, os.path, sys, string
#! /usr/bin/env python
########################################################################
# $HeadURL: $
########################################################################
__RCSID__ = "$Id: $"
from DIRAC.Core.Base import Script
Script.setUsageMessage("""
Resolve problematic files for the specified transformations
Usage:
%s [options] TransID [TransID]
""" % Script.scriptName)
Script.parseCommandLine()
import DIRAC
from DIRAC import gLogger, S_OK, S_ERROR
from DIRAC.Core.Utilities.List import sortList
from LHCbDIRAC.DataManagementSystem.Client.DataIntegrityClient import DataIntegrityClient
integrityClient = DataIntegrityClient()
def resolveTransforamtionProblematics(transID):
gLogger.notice("Obtaining problematic files for transformation %d" %
transID)
res = integrityClient.getTransformationProblematics(transID)
if not res['OK']:
gLogger.error("Failed to get transformation problematic files",
res['Message'])
def setHostDN(arg):
global hostDN
if hostDN or not arg:
Script.showHelp()
DIRAC.exit(-1)
hostDN = arg
hostDN = arg
def addProperty(arg):
global hostProperties
if not arg:
Script.showHelp()
DIRAC.exit(-1)
if not arg in hostProperties:
hostProperties.append(arg)
Script.setUsageMessage('\n'.join([
'Add or Modify a Host info in DIRAC',
'\nUsage:\n',
' %s [option|cfgfile] ... Property=<Value> ...' % Script.scriptName,
'\nArguments:\n',
' Property=<Value>: Other properties to be added to the User like (Responsible=XXXX)',
]))
Script.registerSwitch('H:', 'HostName:', 'Name of the Host (Mandatory)',
setHostName)
Script.registerSwitch('D:', 'HostDN:',
'DN of the Host Certificate (Mandatory)', setHostDN)
Script.registerSwitch(
'P:', 'Property:',
'Property to be added to the Host (Allow Multiple instances or None)',
addProperty)
Script.parseCommandLine(ignoreErrors=True)
#!/usr/bin/env python
########################################################################
# File : dirac-admin-get-job-pilot-output
# Author : Stuart Paterson
########################################################################
"""
Retrieve the output of the pilot that executed a given job
"""
from __future__ import print_function
__RCSID__ = "$Id$"
from DIRAC.Core.Base import Script
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... JobID ...' % Script.scriptName, 'Arguments:',
' JobID: DIRAC ID of the Job'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
from DIRAC import exit as DIRACExit
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
for job in args:
#!/usr/bin/env python
########################################################################
# $Header: $
########################################################################
__RCSID__ = "$Id$"
from DIRAC import exit as dexit
from DIRAC.Core.Base import Script
from DIRAC import gLogger
Script.setUsageMessage("""
Remove the given file replica or a list of file replicas from the File Catalog
This script should be used with great care as it may leave dark data in the storage!
Use dirac-dms-remove-replicas instead
Usage:
%s <LFN | fileContainingLFNs> <SE>
""" % Script.scriptName)
Script.parseCommandLine()
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
res = getProxyInfo()
if not res['OK']:
gLogger.fatal("Can't get proxy info", res['Message'])
dexit(1)
properties = res['Value'].get('groupProperties', [])
if not 'FileCatalogManagement' in properties:
gLogger.error(
"You need to use a proxy from a group with FileCatalogManagement")
dexit(5)
from DIRAC.Core.Base import Script
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC import exit as DIRACExit, gConfig, gLogger
from DIRAC.Core.Utilities.List import intListToString
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getPropertiesForGroup
from DIRAC.Core.Utilities.SitesDIRACGOCDBmapping import getDIRACSiteName
if __name__ == "__main__":
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... DIRACSiteName GridSiteName CE [CE] ...' %
Script.scriptName, 'Arguments:',
' DIRACSiteName: Name of the site for DIRAC in the form GRID.LOCATION.COUNTRY (ie:LCG.CERN.ch)',
' GridSiteName: Name of the site in the Grid (ie: CERN-PROD)',
' CE: Name of the CE to be included in the site (ie: ce111.cern.ch)'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
csAPI = CSAPI()
if len(args) < 3:
Script.showHelp()
DIRACExit(-1)
diracSiteName = args[0]
gridSiteName = args[1]
ces = args[2:]
#/usr/bin/env python
#############################################
#$HeadURL:$
#data:2013/08/07
#author:gang
#############################################
"""
upload a set of file to SE and register them in DFC.
"""
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Base import Script
Script.registerSwitch("r", "dir", "the directory that dataset files located")
Script.setUsageMessage('\n'.join([
__doc__, 'Usage:',
'%s dir' % Script.scriptName, 'Arguments:'
' dir: dir is a logical directory in DFC'
]))
Script.parseCommandLine(ignoreErrors=True)
dir = Script.getPositionalArgs()
#print dir
if len(dir) != 1:
Script.showHelp()
from BESDIRAC.Badger.API.Badger import Badger
badger = Badger()
localdir = dir[0]
exitCode = 0
for tableName in [
'MessageRepository', 'FixedTextMessages', 'SubSystems',
'Systems', 'AgentPersistentData', 'ClientIPs', 'Sites',
'UserDNs'
]:
result = db._update('DROP TABLE `%s`' % tableName)
assert result['OK']
gLogger.info('\n OK\n')
except AssertionError:
print 'ERROR ',
if not result['OK']:
print result['Message']
else:
print result
sys.exit(1)
if __name__ == '__main__':
from DIRAC.Core.Base import Script
Script.parseCommandLine()
gLogger.setLevel('VERBOSE')
if 'PYTHONOPTIMIZE' in os.environ and os.environ['PYTHONOPTIMIZE']:
gLogger.info('Unset pyhthon optimization "PYTHONOPTIMIZE"')
sys.exit(0)
testSystemLoggingDB()
########################################################################
# File : dirac-externals-requirements
# Author : Adri/Federico
########################################################################
""" If /RequiredExternals/ is found in releases.cfg, then some packages to install with pip may be found. This will do it.
"""
import os
import sys
from pip.index import PackageFinder
from pip.req import InstallRequirement, RequirementSet
from pip.locations import user_dir, src_prefix
from pip.download import PipSession
from DIRAC.Core.Base import Script
Script.disableCS()
from DIRAC import gLogger, rootPath, S_OK
from DIRAC.Core.Utilities.CFG import CFG
__RCSID__ = "$Id$"
instType = "server"
def setInstallType(val):
global instType
instType = val
return S_OK()
def registerSwitches(self):
Script.registerSwitch("P:", "Platform=",
"Platform ex. %s" % self.platform,
self.setPlatform)
Script.registerSwitch("A:", "Applications=",
"Comma separated list of applications",
self.setName)
Script.registerSwitch("V:", "Version=", "Version name",
self.setVersion)
Script.registerSwitch("C:", "Comment=", "Comment", self.setComment)
Script.registerSwitch("S:", "Script=", "Full path to initScript",
self.setInitScript)
Script.registerSwitch("B:", "Base=", "Path to Installation Base",
self.setBasePath)
Script.registerSwitch(
"O:", "Config=",
"Path To [Clic|ILD]Config (if it is in ApplicationPath)",
self.setConfig)
Script.registerSwitch(
"", "ILDConfig=",
"Path To ILDConfig (if one is in ApplicationPath) [DEPRECATED]",
self.setConfig)
Script.registerSwitch("Q:", "DBSlice=", "Path to Mokka DB Slice",
self.setDBSlice)
Script.registerSwitch("N", "dry-run", "DryRun: do not commit to CS",
self.dryRun)
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], '\nUsage:',
' %s [option|cfgfile] ...\n' % Script.scriptName
]))
#! /usr/bin/env python
########################################################################
# $HeadURL$
# File : dirac-wms-jobs-select-output-search
# Author : Vladimir Romanovsky
########################################################################
"""
Retrieve output sandbox for DIRAC Jobs for the given selection and search for a string in their std.out
"""
__RCSID__ = "$Id$"
import os
from shutil import rmtree
import DIRAC
from DIRAC.Core.Base import Script
Script.registerSwitch("", "Status=", "Primary status")
Script.registerSwitch("", "MinorStatus=", "Secondary status")
Script.registerSwitch("", "ApplicationStatus=", "Application status")
Script.registerSwitch("", "Site=", "Execution site")
Script.registerSwitch("", "Owner=", "Owner (DIRAC nickname)")
Script.registerSwitch("", "JobGroup=", "Select jobs for specified job group")
Script.registerSwitch(
"", "Date=",
"Date in YYYY-MM-DD format, if not specified default is today")
Script.registerSwitch("", "File=",
"File name,if not specified default is std.out ")
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... String ...' % Script.scriptName, 'Arguments:',
' String: string to search for'
]))
""" Prod3 MC Pipe Script to create a Transformation
JB, LA December 2016
"""
from DIRAC.Core.Base import Script
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1],
'Usage:',
' %s array_layout site particle pointing_dir zenith_angle nShower' %
Script.scriptName,
'Arguments:',
' array_layout: hex or square',
' site: Paranal, Aar, Armazones_2K',
' particle: gamma, proton, electron',
' pointing_dir: North or South',
' zenith_agle: 20',
' nShower: from 5 to 25000',
'\ne.g: %s hex Paranal gamma South 20 5' % Script.scriptName,
]))
Script.parseCommandLine()
import DIRAC
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.Core.Workflow.Parameter import Parameter
#from CTADIRAC.Interfaces.API.Prod3MCPipeJob import Prod3MCPipeJob
from Prod3MCPipeJob import Prod3MCPipeJob
#from DIRAC.Interfaces.API.Dirac import Dirac
def submitTS(job):
import json
import sys
from DIRAC import gLogger
from DIRAC.Core.Base import Script
from DIRAC.Core.Security.X509Chain import X509Chain
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
class CertEncoder(json.JSONEncoder):
"""
JSON encoder for data structures possibly including certificate objects.
"""
def default(self, obj):
if isinstance(obj, X509Chain):
return obj.dumpAllToString()
return json.JSONEncoder.default(self, obj)
if __name__ == "__main__":
# Suppress error messages, since stdout of this script is expected to be in JSON format
gLogger.setLevel("FATAL")
Script.enableCS() # Required so dict includes username
ProxyInfo = getProxyInfo()
if not ProxyInfo["OK"]:
sys.exit(1)
print(json.dumps(ProxyInfo["Value"], cls=CertEncoder))
#!/usr/bin/env python
"""
Get VM instances available in the configured cloud sites
"""
__RCSID__ = "$Id$"
from DIRAC.Core.Base import Script
from DIRAC import gLogger, exit as DIRACExit
Script.setUsageMessage('\n'.join([
'Get VM nodes information', 'Usage:',
'%s site ce node [option]... [cfgfile]' % Script.scriptName, 'Arguments:',
' cfgfile: DIRAC Cfg with description of the configuration (optional)'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
from VMDIRAC.WorkloadManagementSystem.Client.VMClient import VMClient
from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup
if len(args) != 3:
print Script.showHelp()
DIRACExit(-1)
site, ce, node = args
vmClient = VMClient()
result = vmClient.stopInstance(site, ce, node)
if not result['OK']:
from DIRAC import gLogger
from DIRAC import S_OK, S_ERROR, exit
from DIRAC.Core.Base import Script
Script.setUsageMessage("""
Get and merge output files of the task
Usage:
%(script)s [options] TaskID
Examples:
# Download all output files in task 165 to the current directory
%(script)s 165
# Download all output files in task 165 and merge all the files into larger ones, where the merged file size will not exceed 1GB
%(script)s -g 1G 165
# Download all dst files in task 237 to directory "/some/dir"
%(script)s -f '*.dst' -D /some/dir 237
# List all root file names in task 46. Do NOT download
%(script)s -l -f '*.root' 46
# Download all root files in task 329 to directory "output" and merge to files smaller than 800MB, reserve the root files after merge
%(script)s -g 800M -k -f '*.root' -D output 329
""" % {'script': Script.scriptName})
Script.registerSwitch(
"m:", "method=", "Downloading method: local_rsync, dfc, cp, daemon_rsync")
Script.registerSwitch("l", "list", "List all output files. Do NOT download")
Script.registerSwitch("D:", "dir=", "Output directory")
#!/usr/bin/env python
########################################################################
# $Header: $
########################################################################
from __future__ import print_function
__RCSID__ = "$Id: $"
import DIRAC
from DIRAC.Core.Base import Script
from DIRAC import exit as dexit
from DIRAC import gLogger
Script.setUsageMessage("""
Remove the given file or a list of files from the File Catalog
Usage:
%s <LFN | fileContainingLFNs>
""" % Script.scriptName)
Script.parseCommandLine()
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
res = getProxyInfo()
if not res['OK']:
gLogger.fatal("Can't get proxy info", res['Message'])
dexit(1)
properties = res['Value'].get('groupProperties', [])
if not 'FileCatalogManagement' in properties:
gLogger.error(
"You need to use a proxy from a group with FileCatalogManagement")
dexit(5)
#!/usr/bin/env python
""" Simple data management script for PROD3 MC
create DFC MetaData structure
put and register files in DFC
"""
__RCSID__ = "$Id$"
# generic imports
import os, glob, json
# DIRAC imports
import DIRAC
from DIRAC.Core.Base import Script
Script.parseCommandLine()
# Specific DIRAC imports
from CTADIRAC.Core.Workflow.Modules.Prod3DataManager import Prod3DataManager
####################################################
def putAndRegisterPROD3(args):
""" simple wrapper to put and register all PROD3 files
Keyword arguments:
args -- a list of arguments in order []
"""
metadata = args[0]
metadatafield = args[1]
filemetadata = args[2]
inputpath = args[3]
def setSites(optVal_):
global sites
sites = optVal_.split(',')
return S_OK()
description = """Get computing resources capable to execute a job with the given description.
Note that only statically defined computing resource parameters are considered although sites
can fail matching due to their dynamic state, e.g. occupancy by other jobs. Also input data
proximity is not taken into account.
"""
Script.setUsageMessage('\n'.join(
[description, 'Usage:',
' %s [option]... <job_JDL>' % Script.scriptName]))
Script.registerSwitch("F", "full-match", "Check all the matching criteria",
setFullMatch)
Script.registerSwitch("S:", "site=",
"Check matching for these sites (comma separated list)",
setSites)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) == 0:
gLogger.error("Error: No job description provided")
Script.showHelp(exitCode=1)
def setHostName(arg):
global hostName
if hostName or not arg:
Script.showHelp()
DIRAC.exit(-1)
hostName = arg
Staged GRIDKA-RDST 1 4.5535
StageSubmitted GRIDKA-RDST 5 22.586
Waiting PIC-RDST 3 13.6478
WARNING: the Size for files with Status=New is not yet determined at the point of selection!
--------------------- current status of the SE Caches from the DB-----------
GRIDKA-RDST : 6 replicas with a size of 29.141 GB.
"""
__RCSID__ = "$Id$"
from DIRAC.Core.Base import Script
from DIRAC import gConfig, gLogger, exit as DIRACExit, S_OK, version
Script.setUsageMessage(__doc__)
Script.parseCommandLine(ignoreErrors=False)
from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient
client = StorageManagerClient()
res = client.getCacheReplicasSummary()
if not res['OK']:
gLogger.fatal(res['Message'])
DIRACExit(2)
stagerInfo = res['Value']
outStr = "\n"
outStr += " %s" % ("Status".ljust(20))
outStr += " %s" % ("SE".ljust(20))
outStr += " %s" % ("NumberOfFiles".ljust(20))
outStr += " %s" % ("Size(GB)".ljust(20))
outStr += " \n--------------------------------------------------------------------------\n"
######################################################################## # $HeadURL$ # File : DiracAdmin.py # Author : Stuart Paterson ######################################################################## from DIRAC.Core.Base import Script Script.initialize() """DIRAC Administrator API Class All administrative functionality is exposed through the DIRAC Admin API. Examples include site banning and unbanning, WMS proxy uploading etc. """ __RCSID__ = "$Id$" import DIRAC from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC.ConfigurationSystem.Client.Helpers import getVO from DIRAC.Core.DISET.RPCClient import RPCClient from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager from DIRAC.Core.Utilities.SiteCEMapping import getCEsForSite, getSiteCEMapping from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient from DIRAC.Core.Security.X509Chain import X509Chain from DIRAC.Core.Security import Locations, CS from DIRAC import gConfig, gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities.Grid import ldapSite, ldapCluster, ldapCE, ldapService, ldapCEState, ldapCEVOView, ldapSA import re, os, sys, string, time, shutil, types import pprint
#!/usr/bin/env python
"""
Start a given production
"""
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Base import Script
Script.setUsageMessage('\n'.join([__doc__.split('\n')[1],
'Usage:',
' %s prodID' % Script.scriptName,
'Arguments:',
' prodID: Production ID (mandatory)'
]))
Script.parseCommandLine()
from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp(exitCode=1)
# get arguments
prodID = args[0]
prodClient = ProductionClient()
#!/usr/bin/env python
"""
Find files in the FileCatalog using file metadata
"""
from __future__ import print_function
import DIRAC
if __name__ == "__main__":
from DIRAC.Core.Base import Script
Script.registerSwitch('', 'Path=', ' Path to search for')
Script.registerSwitch('', 'SE=', ' (comma-separated list of) SEs/SE-groups to be searched')
Script.setUsageMessage(
'\n'.join(
[
__doc__.split('\n')[1],
'Usage:',
' %s [options] metaspec [metaspec ...]' %
Script.scriptName,
'Arguments:',
' metaspec: metadata index specification \
(of the form: "meta=value" or "meta<value", "meta!=value", etc.)',
'',
'Examples:',
' $ dirac-dms-find-lfns Path=/lhcb/user "Size>1000" "CreationDate<2015-05-15"',
]))
Script.parseCommandLine(ignoreErrors=True)
def setVO(optionValue):
global vo
vo = optionValue
Script.localCfg.addDefaultEntry('/DIRAC/VirtualOrganization', vo)
DIRAC.gConfig.setOptionValue(cfgInstallPath('VirtualOrganization'), vo)
return DIRAC.S_OK()
def forceUpdate(optionValue):
global update
update = True
return DIRAC.S_OK()
Script.disableCS()
Script.registerSwitch("S:", "Setup=", "Set <setup> as DIRAC setup", setSetup)
Script.registerSwitch("C:", "ConfigurationServer=",
"Set <server> as DIRAC configuration server", setServer)
Script.registerSwitch("I", "IncludeAllServers",
"include all Configuration Servers", setAllServers)
Script.registerSwitch("n:", "SiteName=", "Set <sitename> as DIRAC Site Name",
setSiteName)
Script.registerSwitch("N:", "CEName=", "Determiner <sitename> from <cename>",
setCEName)
Script.registerSwitch("V:", "VO=", "Set the VO name", setVO)
Script.registerSwitch("W:", "gateway=",
"Configure <gateway> as DIRAC Gateway for the site",
setGateway)
#!/usr/bin/env python
########################################################################
# File : dirac-admin-list-users
# Author : Adrian Casajus
########################################################################
from __future__ import print_function
__RCSID__ = "$Id$"
from DIRAC.Core.Base import Script
Script.registerSwitch("e", "extended", "Show extended info")
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
from DIRAC import exit as DIRACExit
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
extendedInfo = False
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ('e', 'extended'):
extendedInfo = True
if not extendedInfo:
result = diracAdmin.csListHosts()
for host in result['Value']:
print(" %s" % host)
else:
#!/usr/bin/env python
########################################################################
# $HeadURL$
# File : dirac-admin-list-users
# Author : Adrian Casajus
########################################################################
"""
Lists the users in the Configuration. If no group is specified return all users.
"""
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Base import Script
Script.registerSwitch("e", "extended", "Show extended info")
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... [Group] ...' % Script.scriptName, 'Arguments:',
' Group: Only users from this group (default: all)'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) == 0:
args = ['all']
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
extendedInfo = False
# Author : Ricardo Graciani
########################################################################
"""
Report Normalized CPU length of queue
This script is used by the dirac-pilot script to set the CPUTime limit for the matching
"""
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Base import Script
from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import queueNormalizedCPU
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... Queue ...' % Script.scriptName, 'Arguments:',
' Queue: GlueCEUniqueID of the Queue (ie, juk.nikhef.nl:8443/cream-pbs-lhcb)'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
exitCode = 0
for ceUniqueID in args:
normCPU = queueNormalizedCPU(ceUniqueID)
if not normCPU['OK']:
#!/usr/bin/env python
"""
Get summary informations of all productions
"""
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Base import Script
from DIRAC.Core.Utilities.PrettyPrint import printTable
Script.setUsageMessage('\n'.join([__doc__.split('\n')[1]]))
Script.parseCommandLine()
from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient
prodClient = ProductionClient()
res = prodClient.getProductions()
fields = [
'ProductionName',
'Status',
'ProductionID',
'CreationDate',
'LastUpdate',
'AuthorDN',
'AuthorGroup']
records = []
