def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if args:
subLogger.error( "Found the following positional args '%s', but we only accept switches" % args )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
switches = dict( Script.getUnprocessedSwitches() )
for key in ( 'status', 'se','limit'):
if not key in switches:
print "You're not using switch --%s, query may take long!" % key
if 'status' in switches.keys():
if not switches[ 'status' ] in ( 'New', 'Offline', 'Waiting','Failed','StageSubmitted','Staged' ):
subLogger.error( "Found \"%s\" as Status value. Incorrect value used!" % switches[ 'status' ] )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return switches
def _runFSTAgent():
""" read commands line params and run FST agent for a given transformation ID """
params = _Params()
params.registerSwitches()
Script.parseCommandLine()
if not params.checkSettings()['OK']:
Script.showHelp()
dexit(1)
from ILCDIRAC.ILCTransformationSystem.Agent.FileStatusTransformationAgent import FileStatusTransformationAgent
fstAgent = FileStatusTransformationAgent('ILCTransformation/FileStatusTransformationAgent',
'ILCTransformation/FileStatusTransformationAgent',
'dirac-ilc-filestatus-transformation')
fstAgent.log = gLogger
fstAgent.enabled = params.enabled
res = fstAgent.getTransformations(transID=params.transID)
if not res['OK']:
dexit(1)
if not res['Value']:
print("Transformation Not Found")
dexit(1)
trans = res['Value'][0]
res = fstAgent.processTransformation(
int(params.transID), trans['SourceSE'], trans['TargetSE'], trans['DataTransType'])
if not res["OK"]:
dexit(1)
dexit(0)
def processScriptSwitches():
global vo, dry, doCEs, doSEs
Script.registerSwitch( "V:", "vo=", "Virtual Organization" )
Script.registerSwitch( "D", "dry", "Dry run" )
Script.registerSwitch( "C", "ce", "Process Computing Elements" )
Script.registerSwitch( "S", "se", "Process Storage Elements" )
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile]' % Script.scriptName ] ) )
Script.parseCommandLine( ignoreErrors = True )
vo = ''
dry = False
doCEs = False
doSEs = False
for sw in Script.getUnprocessedSwitches():
if sw[0] in ( "V", "vo" ):
vo = sw[1]
if sw[0] in ( "D", "dry" ):
dry = True
if sw[0] in ( "C", "ce" ):
doCEs = True
if sw[0] in ( "S", "se" ):
doSEs = True
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if args:
subLogger.error( "Found the following positional args '%s', but we only accept switches" % args )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
switches = dict( Script.getUnprocessedSwitches() )
# Default values
switches.setdefault( 'element', None )
if not switches[ 'element' ] in ( 'all', 'Site', 'Resource', 'Node', None ):
subLogger.error( "Found %s as element switch" % switches[ 'element' ] )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return switches
def _createTrafo():
"""reads command line parameters, makes check and creates replication transformation"""
clip = Params()
clip.registerSwitches(Script)
Script.parseCommandLine()
if not clip.checkSettings(Script)['OK']:
gLogger.error("ERROR: Missing settings")
return 1
for metaValue in clip.metaValues:
resCreate = createDataTransformation(flavour=clip.flavour,
targetSE=clip.targetSE,
sourceSE=clip.sourceSE,
metaKey=clip.metaKey,
metaValue=metaValue,
extraData=clip.extraData,
extraname=clip.extraname,
groupSize=clip.groupSize,
plugin=clip.plugin,
enable=clip.enable,
)
if not resCreate['OK']:
gLogger.error("Failed to create Transformation", resCreate['Message'])
return 1
return 0
def runTests():
"""runs the tests"""
clip = CLIParams()
clip.registerSwitches()
Script.parseCommandLine()
overlayrun = clip.testOverlay
myMarlinSteeringFile = "clic_ild_cdr_steering_overlay_1400.0.xml" if overlayrun else "clic_ild_cdr_steering.xml"
myLCSimPreSteeringFile = "clic_cdr_prePandoraOverlay_1400.0.lcsim" if overlayrun else "clic_cdr_prePandora.lcsim"
myLCSimPostSteeringFile = "clic_cdr_postPandoraOverlay.lcsim"
parameterDict = dict( mokkaVersion="0706P08",
mokkaSteeringFile="clic_ild_cdr.steer",
detectorModel="CLIC_ILD_CDR",
steeringFileVersion="V22",
machine="clic_cdr",
backgroundType="gghad",
energy=1400,
marlinVersion="v0111Prod",
rootVersion="5.34",
marlinSteeringFile=myMarlinSteeringFile,
marlinInputdata = "/ilc/user/s/sailer/testFiles/prod_clic_ild_e2e2_o_sim_2214_26.slcio",
gearFile='clic_ild_cdr.gear',
lcsimPreSteeringFile=myLCSimPreSteeringFile,
lcsimPostSteeringFile=myLCSimPostSteeringFile
)
myTests = TestCreater(clip, parameterDict)
res = myTests.checkForTests()
if not res['OK']:
dexit(1)
myTests.run()
return
def _createTrafo():
"""reads command line parameters, makes check and creates replication transformation"""
clip = Params()
clip.registerSwitches(Script)
registerSwitches(clip, Script)
Script.parseCommandLine()
if not clip.checkSettings(Script)['OK']:
LOG.error("ERROR: Missing settings")
return 1
for index, prodID in enumerate(clip.metaValues):
datatype = clip.datatype if clip.datatype else ['GEN', 'SIM', 'REC'][index % 3]
plugin = 'Broadcast' if clip.forcemoving or clip.flavour != 'Moving' else 'BroadcastProcessed'
retData = checkDatatype(prodID, datatype)
if not retData['OK']:
LOG.error("ERROR: %s" % retData['Message'])
return 1
tGroup = getTransformationGroup(prodID, clip.groupName)
parDict = dict(flavour='Moving',
targetSE=clip.targetSE,
sourceSE=clip.sourceSE,
metaKey=clip.metaKey,
metaValue=prodID,
extraData={'Datatype': datatype},
extraname=clip.extraname,
plugin=plugin,
groupSize=clip.groupSize,
tGroup=tGroup,
enable=clip.enable,
)
LOG.debug("Parameters: %s" % pformat(parDict))
resCreate = createDataTransformation(**parDict)
if not resCreate['OK']:
return 1
return 0
def processScriptSwitches():
global vo, dry, doCEs, doSEs, hostURL, glue2
Script.registerSwitch("V:", "vo=", "Virtual Organization")
Script.registerSwitch("D", "dry", "Dry run")
Script.registerSwitch("C", "ce", "Process Computing Elements")
Script.registerSwitch("S", "se", "Process Storage Elements")
Script.registerSwitch("H:", "host=", "use this url for information querying")
Script.registerSwitch("G", "glue2", "query GLUE2 information schema")
Script.setUsageMessage('\n'.join([__doc__.split('\n')[1],
'Usage:',
' %s [option|cfgfile]' % Script.scriptName]))
Script.parseCommandLine(ignoreErrors=True)
vo = ''
dry = False
doCEs = False
doSEs = False
hostURL = None
glue2 = False
for sw in Script.getUnprocessedSwitches():
if sw[0] in ("V", "vo"):
vo = sw[1]
if sw[0] in ("D", "dry"):
dry = True
if sw[0] in ("C", "ce"):
doCEs = True
if sw[0] in ("S", "se"):
doSEs = True
if sw[0] in ("H", "host"):
hostURL = sw[1]
if sw[0] in ("G", "glue2"):
glue2 = True
def _createTrafo():
"""reads command line parameters, makes check and creates replication transformation"""
clip = Params()
clip.registerSwitches(Script)
registerSwitches(Script)
Script.parseCommandLine()
if not clip.checkSettings(Script)['OK']:
LOG.error("ERROR: Missing settings")
return 1
for prodID in clip.metaValues:
tGroup = getTransformationGroup(prodID, clip.groupName)
parDict = dict(flavour='Replication',
targetSE=clip.targetSE,
sourceSE=clip.sourceSE,
metaKey=clip.metaKey,
metaValue=prodID,
extraData={'Datatype': clip.datatype},
extraname=clip.extraname,
plugin=clip.plugin,
groupSize=clip.groupSize,
tGroup=tGroup,
enable=clip.enable,
)
LOG.debug("Parameters: %s" % pformat(parDict))
resCreate = createDataTransformation(**parDict)
if not resCreate['OK']:
return 1
return 0
def _showSoftware():
"""Show available software"""
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine()
from DIRAC import gConfig, gLogger
base = '/Operations/Defaults/AvailableTarBalls'
platforms = gConfig.getSections(base)
for platform in platforms['Value']:
gLogger.notice("For platform %s, here are the available software" % platform)
apps = gConfig.getSections(base + "/" + platform)
for app in apps['Value']:
if clip.software and app.lower() != clip.software.lower():
continue
gLogger.notice(" - %s" % app)
versions = gConfig.getSections(base + "/" + platform + "/" + app)
if clip.appsOnly:
continue
for vers in versions['Value']:
gLogger.notice(" * %s" % vers)
depsb = gConfig.getSections(base + "/" + platform + "/" + app + "/" + vers)
if 'Dependencies' in depsb['Value']:
gLogger.notice(" Depends on")
deps = gConfig.getSections( os.path.join( base, platform, app, vers , "Dependencies") )
for dep in deps['Value']:
depversions = gConfig.getOption(base + "/" + platform + "/" + app + "/" + vers + "/Dependencies/" + dep + "/version")
gLogger.notice(" %s %s" % (dep, depversions['Value']))
if not len(versions['Value']):
gLogger.notice(" No version available")
def _getOutputs():
repoLocation = ''
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine( ignoreErrors = False )
repoLocation = clip.repo
if not repoLocation:
Script.showHelp()
dexit(1)
from DIRAC import gLogger
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac(True, repoLocation)
exitCode = 0
res = dirac.monitorRepository(False)
if not res['OK']:
gLogger.error("Failed because %s" % res['Message'])
dexit(1)
res = dirac.retrieveRepositorySandboxes()
if not res['OK']:
gLogger.error("Failed because %s" % res['Message'])
dexit(1)
if clip.outputdata:
res = dirac.retrieveRepositoryData()
if not res['OK']:
gLogger.error("Failed because %s" % res['Message'])
exit(1)
dexit(exitCode)
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if args:
subLogger.error( "Found the following positional args '%s', but we only accept switches" % args )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
switches = dict( Script.getUnprocessedSwitches() )
switches.setdefault( 'statusType' , None )
switches.setdefault( 'releaseToken', False )
for key in ( 'element', 'name', 'reason' ):
if not key in switches:
subLogger.error( "%s Switch missing" % key )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
if not switches[ 'element' ] in ( 'Site', 'Resource', 'Node' ):
subLogger.error( "Found %s as element switch" % switches[ 'element' ] )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return switches
def setUp( self ): from DIRAC.Core.Base import Script Script.parseCommandLine() from DIRAC.FrameworkSystem.Client.Logger import gLogger gLogger.showHeaders( True ) self.log = gLogger.getSubLogger( self.__class__.__name__ ) self.processPool = ProcessPool( 4, 8, 8 ) self.processPool.daemonize()
def __init__( self, taskID, timeWait, raiseException=False ): from DIRAC.Core.Base import Script Script.parseCommandLine() from DIRAC.FrameworkSystem.Client.Logger import gLogger self.log = gLogger.getSubLogger( self.__class__.__name__ + "/%s" % taskID ) self.taskID = taskID self.timeWait = timeWait self.raiseException = raiseException
def __init__( self, taskID, timeWait, raiseException=False ):
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from DIRAC.FrameworkSystem.Client.Logger import gLogger
self.log = gLogger.getSubLogger( self.__class__.__name__ + "/%s" % taskID )
self.taskID = taskID
self.log.always( "pid=%s task=%s I'm locked" % ( os.getpid(), self.taskID ) )
gLock.acquire()
self.log.always("you can't see that line, object is stuck by gLock" )
self.timeWait = timeWait
self.raiseException = raiseException
gLock.release()
def main():
"""
main program entry point
"""
options = Params()
options.registerCLISwitches()
Script.parseCommandLine( ignoreErrors = True )
if options.delete_all and options.vos:
gLogger.error( "-a and -v options are mutually exclusive. Please pick one or the other." )
return 1
proxyLoc = Locations.getDefaultProxyLocation()
if not os.path.exists( proxyLoc ):
gLogger.error( "No local proxy found in %s, exiting." % proxyLoc )
return 1
result = ProxyInfo.getProxyInfo( proxyLoc, True )
if not result[ 'OK' ]:
raise RuntimeError( 'Failed to get local proxy info.' )
if result[ 'Value' ][ 'secondsLeft' ] < 60 and options.needsValidProxy():
raise RuntimeError( 'Lifetime of local proxy too short, please renew proxy.' )
userDN=result[ 'Value' ][ 'identity' ]
if options.delete_all:
# delete remote proxies
remote_groups = getProxyGroups()
if not remote_groups:
gLogger.notice( 'No remote proxies found.' )
for vo_group in remote_groups:
deleteRemoteProxy( userDN, vo_group )
# delete local proxy
deleteLocalProxy( proxyLoc )
elif options.vos:
vo_groups = set()
for voname in options.vos:
vo_groups.update(mapVoToGroups( voname ) )
# filter set of all groups to only contain groups for which there is a user proxy
user_groups = getProxyGroups()
vo_groups.intersection_update( user_groups )
if not vo_groups:
gLogger.notice( 'You have no proxies registered for any of the specified VOs.' )
for group in vo_groups:
deleteRemoteProxy( userDN, group )
else:
deleteLocalProxy( proxyLoc )
return 0
def setUpClass( cls ): # destroy kerberos token #try: # subprocess.call(['kdestroy']) #except subprocess.CalledProcessError as err: # print "WARNING: kdestroy did not succeed." # print err.output # Constants for the tests Script.parseCommandLine() user = ProxyInfo.getProxyInfo()['Value']['username'] SETestCase.lfntestfilepath += '%s/%s/setests/%s/' % (user[0], user, randomFolder()) print "Using lfn %s" % SETestCase.lfntestfilepath SETestCase.lfntestfile = SETestCase.lfntestfilepath + SETestCase.lfntestfilename
def _createTrafo():
"""reads command line parameters, makes check and creates replication transformation"""
from DIRAC import exit as dexit
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine()
if not clip.checkSettings()['OK']:
gLogger.error("ERROR: Missing settings")
dexit(1)
resCreate = _createReplication( clip.targetSE, clip.sourceSE, clip.prodID, clip.datatype, clip.extraname )
if not resCreate['OK']:
dexit(1)
dexit(0)
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if len( args ) == 0:
error( "Argument is missing, you should enter either 'test', 'update', 'view', 'remove', 'restore'" )
else:
cmd = args[0].lower()
switches = dict( Script.getUnprocessedSwitches() )
diracConfigFile = CSHelpers.gConfig.diracConfigFilePath
# Default values
switches.setdefault( 'name', None )
switches.setdefault( 'element', None )
switches.setdefault( 'elementType', None )
switches.setdefault( 'setup', "Defaults" )
switches.setdefault( 'file', diracConfigFile )
#switches.setdefault( 'statusType', None )
#switches.setdefault( 'status', None )
# when it's a add/modify query and status/reason/statusType are not specified
#then some specific defaults are set up
if cmd == 'test':
if switches['elementType'] is None and switches['element'] is None and switches['name'] is None:
error( "to test, you should enter at least one switch: either element, elmentType, or name" )
else:
if switches[ 'element' ] != None:
switches[ 'element' ] = switches[ 'element' ].title()
if switches[ 'element' ] not in ( 'Resource', 'Site' ):
error( "you should enter either 'Site' or 'Resource' for switch 'element'" )
if switches[ 'elementType' ] != None:
switches[ 'elementType' ] = switches[ 'elementType' ].title()
if switches[ 'file' ] == None:
error("Enter a fullpath dirac config file location when using 'file' option")
elif cmd == 'remove' :
if 'policy' not in switches or switches['policy'] is None:
error( "to remove, you should enter a policy" )
elif cmd == 'update' or cmd == 'view' or cmd == 'restore':
pass
else:
error( "Incorrect argument: you should enter either 'test', 'update', 'view', 'remove', 'restore'" )
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return args, switches
def setUp( self ):
"""c'tor
:param self: self reference
"""
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from DIRAC.FrameworkSystem.Client.Logger import gLogger
gLogger.showHeaders( True )
self.log = gLogger.getSubLogger( self.__class__.__name__ )
self.processPool = ProcessPool( 4, 8, 8,
poolCallback = self.poolCallback,
poolExceptionCallback = self.poolExceptionCallback )
self.processPool.daemonize()
def _findInFC():
"""Find something in the FileCatalog"""
from DIRAC import exit as dexit
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args)<2:
Script.showHelp('ERROR: Not enough arguments')
gLogger.error("Run %s --help" % SCRIPTNAME )
dexit(1)
path = args[0]
if path == '.':
path = '/'
## Check that the first argument is not a MetaQuery
if any( op in path for op in OPLIST ):
gLogger.error("ERROR: Path '%s' is not a valid path! The first argument must be a path" % path)
gLogger.error("Run %s --help" % SCRIPTNAME )
dexit(1)
gLogger.verbose("Path:", path)
metaQuery = args[1:]
metaDataDict = _createQueryDict(metaQuery)
gLogger.verbose("Query:",str(metaDataDict))
if not metaDataDict:
gLogger.info("No query")
dexit(1)
fc = FileCatalogClient()
res = fc.findFilesByMetadata(metaDataDict, path)
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
if not res['Value']:
gLogger.notice("No files found")
listToPrint = None
if clip.printOnlyDirectories:
listToPrint = set( "/".join(fullpath.split("/")[:-1]) for fullpath in res['Value'] )
else:
listToPrint = res['Value']
for entry in listToPrint:
print entry
dexit(0)
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if len( args ) < 3:
error( "Missing all mandatory 'query', 'element', 'tableType' arguments" )
elif not args[0].lower() in ( 'select', 'add', 'modify', 'delete' ):
error( "Incorrect 'query' argument" )
elif not args[1].lower() in ( 'site', 'resource', 'component', 'node' ):
error( "Incorrect 'element' argument" )
elif not args[2].lower() in ( 'status', 'log', 'history' ):
error( "Incorrect 'tableType' argument" )
else:
query = args[0].lower()
switches = dict( Script.getUnprocessedSwitches() )
# Default values
switches.setdefault( 'name', None )
switches.setdefault( 'statusType', None )
switches.setdefault( 'status', None )
switches.setdefault( 'elementType', None )
switches.setdefault( 'reason', None )
switches.setdefault( 'lastCheckTime', None )
switches.setdefault( 'tokenOwner', None )
if 'status' in switches and switches[ 'status' ] is not None:
switches[ 'status' ] = switches[ 'status' ].title()
if not switches[ 'status' ] in ( 'Active', 'Probing', 'Degraded', 'Banned', 'Unknown', 'Error' ):
error( "'%s' is an invalid argument for switch 'status'" % switches[ 'status' ] )
# when it's a add/modify query and status/reason/statusType are not specified
#then some specific defaults are set up
if query == 'add' or query == 'modify':
if not 'status' in switches or switches[ 'status' ] is None:
switches[ 'status' ] = 'Unknown'
if not 'reason' in switches or switches[ 'reason' ] is None:
switches[ 'reason' ] = 'Unknown reason'
if not 'statusType' in switches or switches[ 'statusType' ] is None:
switches[ 'statusType' ] = 'all'
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return args, switches
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 3:
error("Missing all mandatory 'query', 'element', 'tableType' arguments")
elif not args[0].lower() in ("select", "add", "modify", "delete"):
error("Incorrect 'query' argument")
elif not args[1].lower() in ("site", "resource", "component", "node"):
error("Incorrect 'element' argument")
elif not args[2].lower() in ("status", "log", "history"):
error("Incorrect 'tableType' argument")
else:
query = args[0].lower()
switches = dict(Script.getUnprocessedSwitches())
# Default values
switches.setdefault("name", None)
switches.setdefault("statusType", None)
switches.setdefault("status", None)
switches.setdefault("elementType", None)
switches.setdefault("reason", None)
switches.setdefault("lastCheckTime", None)
switches.setdefault("tokenOwner", None)
if "status" in switches and switches["status"] is not None:
switches["status"] = switches["status"].title()
if not switches["status"] in ("Active", "Probing", "Degraded", "Banned", "Unknown", "Error"):
error("'%s' is an invalid argument for switch 'status'" % switches["status"])
# when it's a add/modify query and status/reason/statusType are not specified
# then some specific defaults are set up
if query == "add" or query == "modify":
if not "status" in switches or switches["status"] is None:
switches["status"] = "Unknown"
if not "reason" in switches or switches["reason"] is None:
switches["reason"] = "Unknown reason"
if not "statusType" in switches or switches["statusType"] is None:
switches["statusType"] = "all"
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.iteritems())
return args, switches
def _getOutputData():
cliParams = _Params()
cliParams.registerSwitches()
Script.parseCommandLine( ignoreErrors = False )
if not cliParams.repo:
Script.showHelp()
dexit(2)
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac(True, cliParams.repo)
exitCode = 0
dirac.monitorRepository(False)
dirac.retrieveRepositoryData()
dexit(exitCode)
def _setSwitches(self):
Script.registerSwitch("S:", "system=", "Systems to check, by default all of them are checked", self._setSystems)
Script.registerSwitch("M", "modified", "Show entries which differ from the default", self._setShowModified)
Script.registerSwitch("A", "added", "Show entries which do not exist in ConfigTemplate", self._setShowAdded)
Script.registerSwitch("U", "missingSection", "Show sections which do not exist in the current configuration",
self._setShowMissingSections)
Script.registerSwitch("O", "missingOption", "Show options which do not exist in the current configuration",
self._setShowMissingOptions)
Script.setUsageMessage('\n'.join([self.__doc__,
'Usage:',
' %s [option|cfgfile] -[MAUO] [-S <system]' % Script.scriptName]))
Script.parseCommandLine(ignoreErrors=True)
if not any([self.showModified, self.showAdded, self.showMissingSections, self.showMissingOptions]):
LOG.error("\nERROR: Set at least one of the flags M A U O")
Script.showHelp()
def addUser():
"""Add user to configuration service and other things"""
clip = Params()
clip.registerSwitches()
Script.parseCommandLine()
if not ( clip.certCN and clip.groups and clip.certDN and clip.uname):
gLogger.error("Username, DN, CN, and groups have to be given")
Script.showHelp()
gLogger.notice("Add User to Egroup")
addUserToEgroup(clip)
if not clip.email:
gLogger.fatal("No email defined and not found in phonebook, you have to provide it: -E<email>")
dexit(1)
userProps = {'DN': clip.certDN, 'Email': clip.email, 'CN': clip.certCN, 'Groups': clip.groups}
gLogger.notice("Add User to CS")
addUserToCS(clip, userProps)
gLogger.notice("Add User to FC")
addUserToFC(clip)
gLogger.notice("Done")
def addSoftware():
"""uploads, registers, and sends email about new software package"""
cliParams = Params()
cliParams.registerSwitches()
Script.parseCommandLine( ignoreErrors = True )
platform = cliParams.platform
appName = cliParams.name
appVersion = cliParams.version
comment = cliParams.comment
tarball_loc = cliParams.tarball
if not platform or not appName or not comment:
Script.showHelp()
dexit(2)
softAdder = SoftwareAdder(platform, appName, tarball_loc, appVersion, comment)
softAdder.addSoftware()
gLogger.notice("All done!")
dexit(0)
def printUsers():
"""Print the list of users in the VO"""
clip = Params()
clip.registerSwitches()
Script.parseCommandLine()
clip.setURLs()
from DIRAC.Core.Security.VOMSService import VOMSService
voms = VOMSService(clip.adminUrl, clip.attributeUrl)
res = voms.admListMembers()
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
users = res['Value']
for user in users:
if not clip.username:
printUser(user, clip.addPrint)
else:
if user['DN'].lower().count(clip.username.lower()):
printUser(user, clip.addPrint)
def _getProdLogs():
"""get production log files from LogSE"""
clip = _Params()
clip.registerSwitch()
Script.parseCommandLine()
if not ( clip.logF or clip.logD or clip.prodid ):
Script.showHelp()
dexit(1)
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
ops = Operations()
storageElementName = ops.getValue('/LogStorage/LogSE', 'LogSE')
from DIRAC.Resources.Storage.StorageElement import StorageElementItem as StorageElement
logSE = StorageElement(storageElementName)
if clip.prodid and not ( clip.logD or clip.logF ):
result = _getLogFolderFromID( clip )
if not result['OK']:
gLogger.error( result['Message'] )
dexit(1)
if clip.logD:
if not clip.noPromptBeforeDL:
res = promptUser('Are you sure you want to get ALL the files in this directory?')
if not res['OK']:
dexit()
choice = res['Value']
if choice.lower()=='n':
dexit(0)
if isinstance(clip.logD, str):
res = logSE.getDirectory(clip.logD, localPath=clip.outputdir)
_printErrorReport(res)
elif isinstance(clip.logD, list):
for logdir in clip.logD:
gLogger.notice('Getting log files from '+str(logdir))
res = logSE.getDirectory(logdir, localPath=clip.outputdir)
_printErrorReport(res)
if clip.logF:
res = logSE.getFile(clip.logF, localPath = clip.outputdir)
_printErrorReport(res)
def _createLFNList():
"""create the LFnList"""
cliparams = _Params()
cliparams.registerSwitches()
Script.parseCommandLine( ignoreErrors = False )
repoLocation = cliparams.repo
if not repoLocation:
Script.showHelp()
dexit(2)
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
dirac = DiracILC(True, repoLocation)
dirac.monitorRepository(False)
lfns = []
lfns = dirac.retrieveRepositoryOutputDataLFNs()
LOG.notice("lfnlist=[")
for lfn in lfns :
LOG.notice('"LFN:%s",' % lfn)
LOG.notice("]")
dexit(0)
import DIRAC
from DIRAC import S_OK, S_ERROR
from DIRAC.Core.Base import Script
Script.setUsageMessage("""
Show task detailed information
Usage:
%s [option] ... [TaskID] ...
""" % Script.scriptName)
Script.registerSwitch("j", "job", "Show job numbers")
Script.parseCommandLine(ignoreErrors=False)
options = Script.getUnprocessedSwitches()
args = Script.getPositionalArgs()
from IHEPDIRAC.WorkloadManagementSystem.Client.TaskClient import TaskClient
taskClient = TaskClient()
from DIRAC.Core.DISET.RPCClient import RPCClient
jobmonClient = RPCClient('WorkloadManagement/JobMonitoring')
def showPairs(pairs):
width = 0
for pair in pairs:
width = max(width, len(pair[0]))
format = '%%-%ds : %%s' % width
import os
import random
import re
import sys
import threading
import time
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import *
from ILCDIRAC.Interfaces.API.NewInterface.Applications import *
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from GridTools import *
### ----------------------------------------------------------------------------------------------------
### Start of SubmitJob function
### ----------------------------------------------------------------------------------------------------
def SubmitJob(jobInfo):
#########################
# Unpack job information
#########################
eventType = jobInfo['eventType']
energy = jobInfo['energy']
detectorModel = jobInfo['detectorModel']
reconstructionVariant = jobInfo['reconstructionVariant']
slcioFile = jobInfo['slcioFile']
pandoraSettingsFileLocal = jobInfo['pandoraSettingsFileLocal']
"""
Print Configuration information for a given Site
"""
from __future__ import print_function
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Base import Script
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... Site ...' % Script.scriptName, 'Arguments:',
' Site: Name of the Site'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
for site in args:
result = diracAdmin.getSiteSection(site, printOutput=True)
if not result['OK']:
errorList.append((site, result['Message']))
def doTheWhizardInstallation():
"""Do the instalation for new whizard version Copy libraries, create tarball,
upload processList file add entry in configuration system
"""
res = checkSLCVersion()
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
res = checkGFortranVersion()
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
cliParams = Params()
cliParams.registerSwitches()
Script.parseCommandLine( ignoreErrors= False)
whizardResultFolder = cliParams.path
platform = cliParams.platform
whizard_version = cliParams.version
appVersion = whizard_version
beam_spectra_version = cliParams.beam_spectra
if not whizardResultFolder or not whizard_version or not beam_spectra_version:
Script.showHelp()
dexit(2)
from ILCDIRAC.Core.Utilities.ProcessList import ProcessList
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from ILCDIRAC.Core.Utilities.FileUtils import upload
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
diracAdmin = DiracAdmin()
modifiedCS = False
softwareSection = "/Operations/Defaults/AvailableTarBalls"
processlistLocation = "ProcessList/Location"
appName = "whizard"
ops = Operations()
path_to_process_list = ops.getValue(processlistLocation, "")
if not path_to_process_list:
gLogger.error("Could not find process list location in CS")
dexit(2)
gLogger.verbose("Getting process list from file catalog")
datMan = DataManager()
res = datMan.getFile(path_to_process_list)
if not res['OK']:
gLogger.error("Error while getting process list from storage")
dexit(2)
gLogger.verbose("done")
##just the name of the local file in current working directory
processlist = os.path.basename(path_to_process_list)
if not os.path.exists(processlist):
gLogger.error("Process list does not exist locally")
dexit(2)
pl = ProcessList(processlist)
startDir = os.getcwd()
inputlist = {}
os.chdir(whizardResultFolder)
folderlist = os.listdir(whizardResultFolder)
whiz_here = folderlist.count("whizard")
if whiz_here == 0:
gLogger.error("whizard executable not found in %s, please check" % whizardResultFolder)
os.chdir(startDir)
dexit(2)
whizprc_here = folderlist.count("whizard.prc")
if whizprc_here == 0:
gLogger.error("whizard.prc not found in %s, please check" % whizardResultFolder)
os.chdir(startDir)
dexit(2)
whizmdl_here = folderlist.count("whizard.mdl")
if whizmdl_here == 0:
gLogger.error("whizard.mdl not found in %s, please check" % whizardResultFolder)
os.chdir(startDir)
dexit(2)
gLogger.verbose("Preparing process list")
## FIXME:: What is this doing exactly? Is this necessary? -- APS, JFS
for f in folderlist:
if f.count(".in"):
infile = open(f, "r")
found_detail = False
for line in infile:
if line.count("decay_description"):
currprocess = f.split(".template.in")[0]
inputlist[currprocess] = {}
inputlist[currprocess]["InFile"] = f.rstrip("~")
inputlist[currprocess]["Detail"] = line.split("\"")[1]
found_detail = True
if line.count("process_id") and found_detail:
process_id = line.split("\"")[1]
inputlist[currprocess]["Model"] = ""
inputlist[currprocess]["Generator"] = ""
inputlist[currprocess]["Restrictions"] = ""
for process in process_id.split():
print "Looking for detail of process %s" % (process)
process_detail = getDetailsFromPRC("whizard.prc", process)
inputlist[currprocess]["Model"] = process_detail["Model"]
inputlist[currprocess]["Generator"] = process_detail["Generator"]
if len(inputlist[currprocess]["Restrictions"]):
inputlist[currprocess]["Restrictions"] = inputlist[currprocess]["Restrictions"] + ", " + process_detail["Restrictions"]
else:
inputlist[currprocess]["Restrictions"] = process_detail["Restrictions"]
#if len(inputlist[currprocess].items()):
# inputlist.append(processdict)
## END FIXEME
##Update inputlist with what was found looking in the prc file
processes = readPRCFile("whizard.prc")
inputlist.update(processes)
##get from cross section files the cross sections for the processes in inputlist
#Need full process list
for f in folderlist:
if f.count("cross_sections_"):
crossfile = open(f, "r")
for line in crossfile:
line = line.rstrip().lstrip()
if not len(line):
continue
if line[0] == "#" or line[0] == "!":
continue
if len(line.split()) < 2:
continue
currprocess = line.split()[0]
if currprocess in inputlist:
inputlist[currprocess]['CrossSection'] = line.split()[1]
gLogger.notice("Preparing Tarball")
##Make a folder in the current directory of the user to store the whizard libraries, executable et al.
localWhizardFolderRel = ("whizard" + whizard_version) # relative path
localWhizardFolder = os.path.join(startDir, localWhizardFolderRel)
if not os.path.exists(localWhizardFolder):
os.makedirs(localWhizardFolder)
localWhizardLibFolder = os.path.join(localWhizardFolder,'lib')
if os.path.exists(localWhizardLibFolder):
shutil.rmtree(localWhizardLibFolder)
os.makedirs(localWhizardLibFolder) ##creates the lib folder
whizardLibraries = getListOfLibraries(os.path.join(whizardResultFolder, "whizard"))
copyLibsCall = ["rsync","-avzL"]
for lib in whizardLibraries:
copyLibsCall.append(lib)
copyLibsCall.append(localWhizardLibFolder)
subprocess.Popen(copyLibsCall, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
for fileName in folderlist:
shutil.copy(fileName, localWhizardFolder)
##Get the list of md5 sums for all the files in the folder to be tarred
os.chdir( localWhizardFolder )
subprocess.call(["find . -type f -exec md5sum {} > ../md5_checksum.md5 \\; && mv ../md5_checksum.md5 ."], shell=True)
os.chdir(startDir)
##Create the Tarball
gLogger.notice("Creating Tarball...")
appTar = localWhizardFolder + ".tgz"
myappTar = tarfile.open(appTar, "w:gz")
myappTar.add(localWhizardFolderRel)
myappTar.close()
md5sum = md5.md5(open( appTar, 'r' ).read()).hexdigest()
gLogger.notice("...Done")
gLogger.notice("Registering new Tarball in CS")
tarballurl = {}
av_platforms = gConfig.getSections(softwareSection, [])
if av_platforms['OK']:
if platform not in av_platforms['Value']:
gLogger.error("Platform %s unknown, available are %s." % (platform, ", ".join(av_platforms['Value'])))
gLogger.error("If yours is missing add it in CS")
dexit(255)
else:
gLogger.error("Could not find all platforms available in CS")
dexit(255)
av_apps = gConfig.getSections("%s/%s" % (softwareSection, platform), [])
if not av_apps['OK']:
gLogger.error("Could not find all applications available in CS")
dexit(255)
if appName.lower() in av_apps['Value']:
versions = gConfig.getSections("%s/%s/%s" % (softwareSection, platform, appName.lower()),
[])
if not versions['OK']:
gLogger.error("Could not find all versions available in CS")
dexit(255)
if appVersion in versions['Value']:
gLogger.error('Application %s %s for %s already in CS, nothing to do' % (appName.lower(), appVersion, platform))
dexit(0)
else:
result = diracAdmin.csSetOption("%s/%s/%s/%s/TarBall" % (softwareSection, platform, appName.lower(), appVersion),
os.path.basename(appTar))
if result['OK']:
modifiedCS = True
tarballurl = gConfig.getOption("%s/%s/%s/TarBallURL" % (softwareSection, platform, appName.lower()), "")
if len(tarballurl['Value']) > 0:
res = upload(tarballurl['Value'], appTar)
if not res['OK']:
gLogger.error("Upload to %s failed" % tarballurl['Value'])
dexit(255)
result = diracAdmin.csSetOption("%s/%s/%s/%s/Md5Sum" % (softwareSection, platform, appName.lower(), appVersion),
md5sum)
if result['OK']:
modifiedCS = True
result = diracAdmin.csSetOption("%s/%s/%s/%s/Dependencies/beam_spectra/version" % (softwareSection,
platform,
appName.lower(),
appVersion),
beam_spectra_version)
else:
result = diracAdmin.csSetOption("%s/%s/%s/%s/TarBall" % (softwareSection, platform,
appName.lower(), appVersion),
os.path.basename(appTar))
if result['OK']:
modifiedCS = True
tarballurl = gConfig.getOption("%s/%s/%s/TarBallURL" % (softwareSection, platform, appName.lower()),
"")
if len(tarballurl['Value']) > 0:
res = upload(tarballurl['Value'], appTar)
if not res['OK']:
gLogger.error("Upload to %s failed" % tarballurl['Value'])
dexit(255)
result = diracAdmin.csSetOption("%s/%s/%s/%s/Md5Sum" % (softwareSection, platform, appName.lower(), appVersion),
md5sum)
result = diracAdmin.csSetOption("%s/%s/%s/%s/Dependencies/beam_spectra/version" % (softwareSection,
platform,
appName.lower(),
appVersion),
beam_spectra_version)
gLogger.verbose("Done uploading the tar ball")
os.remove(appTar)
#Set for all new processes the TarBallURL
for process in inputlist.keys():
inputlist[process]['TarBallCSPath'] = tarballurl['Value'] + os.path.basename(appTar)
pl.updateProcessList(inputlist)
pl.writeProcessList()
raw_input("Do you want to upload the process list? Press ENTER to proceed or CTRL-C to abort!")
pl.uploadProcessListToFileCatalog(path_to_process_list, appVersion)
#Commit the changes if nothing has failed and the CS has been modified
if modifiedCS:
result = diracAdmin.csCommitChanges(False)
gLogger.verbose(result)
gLogger.notice('All done OK!')
dexit(0)
def main():
import os
import DIRAC
from DIRAC import gConfig
from DIRAC.Core.Base import Script
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
from DIRAC.Core.Utilities.PrettyPrint import printTable
def version(arg):
Script.disableCS()
print(DIRAC.version)
DIRAC.exit(0)
def platform(arg):
Script.disableCS()
print(DIRAC.getPlatform())
DIRAC.exit(0)
Script.registerSwitch("v", "version",
"print version of current DIRAC installation",
version)
Script.registerSwitch("p", "platform",
"print platform of current DIRAC installation",
platform)
Script.parseCommandLine(ignoreErrors=True)
records = []
records.append(('Setup', gConfig.getValue('/DIRAC/Setup', 'Unknown')))
records.append(('ConfigurationServer',
gConfig.getValue('/DIRAC/Configuration/Servers', [])))
records.append(('Installation path', DIRAC.rootPath))
if os.path.exists(
os.path.join(DIRAC.rootPath, DIRAC.getPlatform(), 'bin', 'mysql')):
records.append(('Installation type', 'server'))
else:
records.append(('Installation type', 'client'))
records.append(('Platform', DIRAC.getPlatform()))
ret = getProxyInfo(disableVOMS=True)
if ret['OK']:
if 'group' in ret['Value']:
vo = getVOForGroup(ret['Value']['group'])
else:
vo = getVOForGroup('')
if not vo:
vo = "None"
records.append(('VirtualOrganization', vo))
if 'identity' in ret['Value']:
records.append(('User DN', ret['Value']['identity']))
if 'secondsLeft' in ret['Value']:
records.append(('Proxy validity, secs', {
'Value': str(ret['Value']['secondsLeft']),
'Just': 'L'
}))
if gConfig.getValue('/DIRAC/Security/UseServerCertificate', True):
records.append(('Use Server Certificate', 'Yes'))
else:
records.append(('Use Server Certificate', 'No'))
if gConfig.getValue('/DIRAC/Security/SkipCAChecks', False):
records.append(('Skip CA Checks', 'Yes'))
else:
records.append(('Skip CA Checks', 'No'))
records.append(('DIRAC version', DIRAC.version))
fields = ['Option', 'Value']
print()
printTable(fields, records, numbering=False)
print()
def main():
Script.localCfg.addDefaultEntry("LogLevel", "fatal")
Script.parseCommandLine()
CSCLI().start()
