def processScriptSwitches():
global vo, dry, doCEs, doSEs
Script.registerSwitch( "V:", "vo=", "Virtual Organization" )
Script.registerSwitch( "D", "dry", "Dry run" )
Script.registerSwitch( "C", "ce", "Process Computing Elements" )
Script.registerSwitch( "S", "se", "Process Storage Elements" )
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile]' % Script.scriptName ] ) )
Script.parseCommandLine( ignoreErrors = True )
vo = ''
dry = False
doCEs = False
doSEs = False
for sw in Script.getUnprocessedSwitches():
if sw[0] in ( "V", "vo" ):
vo = sw[1]
if sw[0] in ( "D", "dry" ):
dry = True
if sw[0] in ( "C", "ce" ):
doCEs = True
if sw[0] in ( "S", "se" ):
doSEs = True
def processScriptSwitches():
global vo, dry, doCEs, doSEs, hostURL, glue2
Script.registerSwitch("V:", "vo=", "Virtual Organization")
Script.registerSwitch("D", "dry", "Dry run")
Script.registerSwitch("C", "ce", "Process Computing Elements")
Script.registerSwitch("S", "se", "Process Storage Elements")
Script.registerSwitch("H:", "host=", "use this url for information querying")
Script.registerSwitch("G", "glue2", "query GLUE2 information schema")
Script.setUsageMessage('\n'.join([__doc__.split('\n')[1],
'Usage:',
' %s [option|cfgfile]' % Script.scriptName]))
Script.parseCommandLine(ignoreErrors=True)
vo = ''
dry = False
doCEs = False
doSEs = False
hostURL = None
glue2 = False
for sw in Script.getUnprocessedSwitches():
if sw[0] in ("V", "vo"):
vo = sw[1]
if sw[0] in ("D", "dry"):
dry = True
if sw[0] in ("C", "ce"):
doCEs = True
if sw[0] in ("S", "se"):
doSEs = True
if sw[0] in ("H", "host"):
hostURL = sw[1]
if sw[0] in ("G", "glue2"):
glue2 = True
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if args:
subLogger.error( "Found the following positional args '%s', but we only accept switches" % args )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
switches = dict( Script.getUnprocessedSwitches() )
# Default values
switches.setdefault( 'element', None )
if not switches[ 'element' ] in ( 'all', 'Site', 'Resource', 'Node', None ):
subLogger.error( "Found %s as element switch" % switches[ 'element' ] )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return switches
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if args:
subLogger.error( "Found the following positional args '%s', but we only accept switches" % args )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
switches = dict( Script.getUnprocessedSwitches() )
for key in ( 'status', 'se','limit'):
if not key in switches:
print "You're not using switch --%s, query may take long!" % key
if 'status' in switches.keys():
if not switches[ 'status' ] in ( 'New', 'Offline', 'Waiting','Failed','StageSubmitted','Staged' ):
subLogger.error( "Found \"%s\" as Status value. Incorrect value used!" % switches[ 'status' ] )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return switches
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if args:
subLogger.error( "Found the following positional args '%s', but we only accept switches" % args )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
switches = dict( Script.getUnprocessedSwitches() )
switches.setdefault( 'statusType' , None )
switches.setdefault( 'releaseToken', False )
for key in ( 'element', 'name', 'reason' ):
if not key in switches:
subLogger.error( "%s Switch missing" % key )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
if not switches[ 'element' ] in ( 'Site', 'Resource', 'Node' ):
subLogger.error( "Found %s as element switch" % switches[ 'element' ] )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return switches
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if len( args ) == 0:
error( "Argument is missing, you should enter either 'test', 'update', 'view', 'remove', 'restore'" )
else:
cmd = args[0].lower()
switches = dict( Script.getUnprocessedSwitches() )
diracConfigFile = CSHelpers.gConfig.diracConfigFilePath
# Default values
switches.setdefault( 'name', None )
switches.setdefault( 'element', None )
switches.setdefault( 'elementType', None )
switches.setdefault( 'setup', "Defaults" )
switches.setdefault( 'file', diracConfigFile )
#switches.setdefault( 'statusType', None )
#switches.setdefault( 'status', None )
# when it's a add/modify query and status/reason/statusType are not specified
#then some specific defaults are set up
if cmd == 'test':
if switches['elementType'] is None and switches['element'] is None and switches['name'] is None:
error( "to test, you should enter at least one switch: either element, elmentType, or name" )
else:
if switches[ 'element' ] != None:
switches[ 'element' ] = switches[ 'element' ].title()
if switches[ 'element' ] not in ( 'Resource', 'Site' ):
error( "you should enter either 'Site' or 'Resource' for switch 'element'" )
if switches[ 'elementType' ] != None:
switches[ 'elementType' ] = switches[ 'elementType' ].title()
if switches[ 'file' ] == None:
error("Enter a fullpath dirac config file location when using 'file' option")
elif cmd == 'remove' :
if 'policy' not in switches or switches['policy'] is None:
error( "to remove, you should enter a policy" )
elif cmd == 'update' or cmd == 'view' or cmd == 'restore':
pass
else:
error( "Incorrect argument: you should enter either 'test', 'update', 'view', 'remove', 'restore'" )
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return args, switches
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if len( args ) < 3:
error( "Missing all mandatory 'query', 'element', 'tableType' arguments" )
elif not args[0].lower() in ( 'select', 'add', 'modify', 'delete' ):
error( "Incorrect 'query' argument" )
elif not args[1].lower() in ( 'site', 'resource', 'component', 'node' ):
error( "Incorrect 'element' argument" )
elif not args[2].lower() in ( 'status', 'log', 'history' ):
error( "Incorrect 'tableType' argument" )
else:
query = args[0].lower()
switches = dict( Script.getUnprocessedSwitches() )
# Default values
switches.setdefault( 'name', None )
switches.setdefault( 'statusType', None )
switches.setdefault( 'status', None )
switches.setdefault( 'elementType', None )
switches.setdefault( 'reason', None )
switches.setdefault( 'lastCheckTime', None )
switches.setdefault( 'tokenOwner', None )
if 'status' in switches and switches[ 'status' ] is not None:
switches[ 'status' ] = switches[ 'status' ].title()
if not switches[ 'status' ] in ( 'Active', 'Probing', 'Degraded', 'Banned', 'Unknown', 'Error' ):
error( "'%s' is an invalid argument for switch 'status'" % switches[ 'status' ] )
# when it's a add/modify query and status/reason/statusType are not specified
#then some specific defaults are set up
if query == 'add' or query == 'modify':
if not 'status' in switches or switches[ 'status' ] is None:
switches[ 'status' ] = 'Unknown'
if not 'reason' in switches or switches[ 'reason' ] is None:
switches[ 'reason' ] = 'Unknown reason'
if not 'statusType' in switches or switches[ 'statusType' ] is None:
switches[ 'statusType' ] = 'all'
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return args, switches
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 3:
error("Missing all mandatory 'query', 'element', 'tableType' arguments")
elif not args[0].lower() in ("select", "add", "modify", "delete"):
error("Incorrect 'query' argument")
elif not args[1].lower() in ("site", "resource", "component", "node"):
error("Incorrect 'element' argument")
elif not args[2].lower() in ("status", "log", "history"):
error("Incorrect 'tableType' argument")
else:
query = args[0].lower()
switches = dict(Script.getUnprocessedSwitches())
# Default values
switches.setdefault("name", None)
switches.setdefault("statusType", None)
switches.setdefault("status", None)
switches.setdefault("elementType", None)
switches.setdefault("reason", None)
switches.setdefault("lastCheckTime", None)
switches.setdefault("tokenOwner", None)
if "status" in switches and switches["status"] is not None:
switches["status"] = switches["status"].title()
if not switches["status"] in ("Active", "Probing", "Degraded", "Banned", "Unknown", "Error"):
error("'%s' is an invalid argument for switch 'status'" % switches["status"])
# when it's a add/modify query and status/reason/statusType are not specified
# then some specific defaults are set up
if query == "add" or query == "modify":
if not "status" in switches or switches["status"] is None:
switches["status"] = "Unknown"
if not "reason" in switches or switches["reason"] is None:
switches["reason"] = "Unknown reason"
if not "statusType" in switches or switches["statusType"] is None:
switches["statusType"] = "all"
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.iteritems())
return args, switches
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if not len(args) == 2:
Script.showHelp()
params = {}
params['ce'] = None
params['site'] = None
params['host'] = None
params['vo'] = None
params['info'] = args[0]
ret = getProxyInfo(disableVOMS=True)
if ret['OK'] and 'group' in ret['Value']:
params['vo'] = getVOForGroup(ret['Value']['group'])
else:
Script.gLogger.error('Could not determine VO')
Script.showHelp()
if params['info'] in ['ce', 'ce-state', 'ce-cluster', 'ce-vo']:
params['ce'] = args[1]
elif params['info']in ['site', 'site-se']:
params['site'] = args[1]
else:
Script.gLogger.error('Wrong argument value')
Script.showHelp()
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("H", "host"):
params['host'] = unprocSw[1]
if unprocSw[0] in ("V", "vo"):
params['vo'] = unprocSw[1]
return params
def parseSwitches():
'''
Parse switches and positional arguments given to the script
'''
# Parse the command line and initialize DIRAC
Script.parseCommandLine(ignoreErrors=False)
# Get the list of services
servicesList = Script.getPositionalArgs()
gLogger.info('This is the servicesList %s:' % servicesList)
# Gets the rest of the
switches = dict(Script.getUnprocessedSwitches())
gLogger.debug("The switches used are:")
map(gLogger.debug, switches.iteritems())
switches['servicesList'] = servicesList
return switches
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if args:
subLogger.error("Found the following positional args '%s', but we only accept switches" % args)
subLogger.error("Please, check documentation below")
Script.showHelp()
DIRACExit(1)
switches = dict(Script.getUnprocessedSwitches())
# Default values
switches.setdefault("elementType", None)
switches.setdefault("name", None)
switches.setdefault("tokenOwner", None)
switches.setdefault("statusType", None)
switches.setdefault("status", None)
if not "element" in switches:
subLogger.error("element Switch missing")
subLogger.error("Please, check documentation below")
Script.showHelp()
DIRACExit(1)
if not switches["element"] in ("Site", "Resource", "Node"):
subLogger.error("Found %s as element switch" % switches["element"])
subLogger.error("Please, check documentation below")
Script.showHelp()
DIRACExit(1)
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.iteritems())
return switches
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if args:
subLogger.error( "Found the following positional args '%s', but we only accept switches" % args )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
switches = dict( Script.getUnprocessedSwitches() )
# Default values
switches.setdefault( 'elementType', None )
switches.setdefault( 'name', None )
switches.setdefault( 'tokenOwner', None )
switches.setdefault( 'statusType', None )
switches.setdefault( 'status', None )
if not 'element' in switches:
subLogger.error( "element Switch missing" )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
if not switches[ 'element' ] in ( 'Site', 'Resource', 'Node' ):
subLogger.error( "Found %s as element switch" % switches[ 'element' ] )
subLogger.error( "Please, check documentation below" )
Script.showHelp()
DIRACExit( 1 )
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return switches
def main():
Script.parseCommandLine(ignoreErrors=False)
Script.registerSwitch('D:', 'Download=', 'Defines data acquisition as DownloadInputData')
Script.registerSwitch('P:', 'Protocol=', 'Defines data acquisition as InputDataByProtocol')
_downloadinputdata = False
_jobID = None
for switch in Script.getUnprocessedSwitches():
if switch[0] in ('D', 'Download'):
_downloadinputdata = True
_jobID = switch[1]
if switch[0] in ('I', 'Protocol'):
_downloadinputdata = False
_jobID = switch[1]
from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import Extensions
ext = Extensions()
_vo = ext.getCSExtensions()[0]
_diracPath = Extensions().getExtensionPath('DIRAC')
_dir = os.path.expanduser('~') + os.path.sep
try:
_path = __runSystemDefaults(_jobID, _vo)
__downloadJobDescriptionXML(_jobID, _path)
__modifyJobDescription(_jobID, _path, _downloadinputdata)
__downloadPilotScripts(_path, _diracPath)
__configurePilot(_path, _vo)
__runJobLocally(_jobID, _path, _vo)
finally:
os.chdir(_dir)
os.rename(_dir + '.dirac.cfg.old', _dir + '.dirac.cfg')
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if not args:
error("Missing mandatory 'query' argument")
elif not args[0].lower() in ('select', 'add', 'delete'):
error("Missing mandatory argument")
else:
query = args[0].lower()
switches = dict(Script.getUnprocessedSwitches())
# Default values
switches.setdefault('downtimeID', None)
switches.setdefault('element', None)
switches.setdefault('name', None)
switches.setdefault('startDate', None)
switches.setdefault('endDate', None)
switches.setdefault('severity', None)
switches.setdefault('description', None)
switches.setdefault('link', None)
if query in ('add', 'delete') and switches['downtimeID'] is None:
error("'downtimeID' switch is mandatory for '%s' but found missing" %
query)
if query in ('add', 'delete') and 'ongoing' in switches:
error("'ongoing' switch can be used only with 'select'")
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.items())
return (args, switches)
def execute():
tr = TransformationClient()
for switch in Script.getUnprocessedSwitches():
pass
bkQuery = dmScript.getBKQuery()
if not bkQuery:
gLogger.notice("No BKQuery given...")
exit(1)
startTime = time.time()
prods = bkQuery.getBKProductions() # visible = 'All' )
parents = {}
productions = {}
for prod in prods:
type = tr.getTransformation(prod).get('Value',
{}).get('Type', 'Unknown')
productions[prod] = type
parent = tr.getBookkeepingQuery(prod).get('Value',
{}).get('ProductionID', '')
if parent:
type = tr.getTransformation(parent).get('Value',
{}).get('Type', 'Unknown')
parents[parent] = type
gLogger.notice("For BK path %s:" % bkQuery.getPath())
if not prods:
gLogger.notice('No productions found!')
else:
printProds('Productions found', productions)
if parents:
printProds('Parent productions', parents)
gLogger.notice('Completed in %.1f seconds' % (time.time() - startTime))
def registerSwitchesAndParseCommandLine(self):
"""Register the default plus additional parameters and parse options.
:param list options: list of three tuple for options to add to the script
:param list flags: list of three tuple for flags to add to the script
:param str opName
"""
for short, longOption, doc in self.options:
Script.registerSwitch(short + ':' if short else '',
longOption + '=', doc)
for short, longOption, doc in self.flags:
Script.registerSwitch(short, longOption, doc)
self.switches[longOption] = False
Script.parseCommandLine()
if Script.getPositionalArgs():
Script.showHelp()
DIRAC.exit(1)
for switch in Script.getUnprocessedSwitches():
for short, longOption, doc in self.options:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
LOG.verbose('Found switch %r with value %r' %
(longOption, switch[1]))
self.switches[longOption] = switch[1]
break
for short, longOption, doc in self.flags:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
self.switches[longOption] = True
break
self.checkSwitches()
self.switches['DryRun'] = not self.switches.get('Execute', False)
self.switches['SourceSE'] = self.switches.get('SourceSE',
'').split(',')
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if len( args ) == 0:
error( "Missing mandatory 'query' argument" )
elif not args[0].lower() in ( 'select', 'add', 'delete' ):
error( "Missing mandatory argument" )
else:
query = args[0].lower()
switches = dict( Script.getUnprocessedSwitches() )
# Default values
switches.setdefault( 'downtimeID', None )
switches.setdefault( 'element', None )
switches.setdefault( 'name', None )
switches.setdefault( 'startDate', None )
switches.setdefault( 'endDate', None )
switches.setdefault( 'severity', None )
switches.setdefault( 'description', None )
switches.setdefault( 'link', None )
if query in ( 'add', 'delete' ) and switches['downtimeID'] is None:
error( "'downtimeID' switch is mandatory for '%s' but found missing" % query )
if query in ( 'add', 'delete' ) and 'ongoing' in switches:
error( "'ongoing' switch can be used only with 'select'" )
subLogger.debug( "The switches used are:" )
map( subLogger.debug, switches.iteritems() )
return ( args, switches )
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if args:
subLogger.error(
"Found the following positional args '%s', but we only accept switches"
% args)
subLogger.error("Please, check documentation below")
Script.showHelp()
DIRACExit(1)
switches = dict(Script.getUnprocessedSwitches())
for key in ('status', 'se', 'limit'):
if not key in switches:
print "You're not using switch --%s, query may take long!" % key
if 'status' in switches.keys():
if not switches['status'] in ('New', 'Offline', 'Waiting', 'Failed',
'StageSubmitted', 'Staged'):
subLogger.error(
"Found \"%s\" as Status value. Incorrect value used!" %
switches['status'])
subLogger.error("Please, check documentation below")
Script.showHelp()
DIRACExit(1)
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.iteritems())
return switches
import DIRAC
from DIRAC import gLogger
from DIRAC.Core.Base import Script
unit = 'GB'
Script.registerSwitch( "u:", "Unit=", " Unit to use [default %s] (MB,GB,TB,PB)" % unit )
Script.setUsageMessage( """
Get the size of the given file or a list of files
Usage:
%s <lfn | fileContainingLfns> <SE> <status>
""" % Script.scriptName )
Script.parseCommandLine( ignoreErrors = False )
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "u" or switch[0].lower() == "unit":
unit = switch[1]
scaleDict = { 'MB' : 1000 * 1000.0,
'GB' : 1000 * 1000 * 1000.0,
'TB' : 1000 * 1000 * 1000 * 1000.0,
'PB' : 1000 * 1000 * 1000 * 1000 * 1000.0}
if not unit in scaleDict.keys():
gLogger.error( "Unit must be one of MB,GB,TB,PB" )
DIRAC.exit( 2 )
scaleFactor = scaleDict[unit]
args = Script.getPositionalArgs()
lfns = []
for inputFileName in args:
if os.path.exists( inputFileName ):
'Arguments:',
' CE: Name of the CE(ie: ce111.cern.ch)'] ) )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
if not len( args ) == 1:
Script.showHelp()
ce = args[0]
host = None
vo = getVO( 'lhcb' )
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ( "H", "host" ):
host = unprocSw[1]
if unprocSw[0] in ( "V", "vo" ):
vo = unprocSw[1]
diracAdmin = DiracAdmin()
result = diracAdmin.getBDIICEState( ce, useVO = vo, host = host )
if not result['OK']:
print result['Message']
DIRAC.exit( 2 )
ces = result['Value']
for ce in ces:
for name, value in wfParameters.items():
workflow.setValue( name, value )
result = workflow.execute()
return result
positionalArgs = Script.getPositionalArgs()
if len( positionalArgs ) != 1:
gLogger.debug( 'Positional arguments were %s' % ( positionalArgs ) )
DIRAC.abort( 1, "Must specify the Job XML file description" )
if os.environ.has_key( 'JOBID' ):
gLogger.info( 'JobID: %s' % ( os.environ['JOBID'] ) )
jobXMLfile = positionalArgs[0]
parList = Script.getUnprocessedSwitches()
parDict = {}
for switch, parameter in parList:
if switch == "p":
name, value = parameter.split( '=' )
value = value.strip()
# The comma separated list in curly brackets is interpreted as a list
if value.startswith("{"):
value = value[1:-1].replace('"','').replace(" ",'').split(',')
value = ';'.join(value)
parDict[name] = value
gLogger.debug( 'PYTHONPATH:\n%s' % ( string.join( sys.path, '\n' ) ) )
result = jobexec( jobXMLfile, parDict )
Script.registerSwitch( 't:', 'time=', 'CPU time limit per job in seconds (default %s)'%(cpuLimit) )
Script.registerSwitch( 'T:', 'title=', 'job title (default %s)'%(jobTitle) )
Script.registerSwitch( 'v:', 'verbose', 'switches off the verbose mode' )
Script.registerSwitch( 'k:', 'marlinxml=', 'name of marlin steering file (default %s)'%(marlinXml) )
Script.registerSwitch( 'K:', 'marlin=', 'Marlin version to use (default %s)'%(marlinVer) )
Script.registerSwitch( 'x:', 'settings=', 'name of pandora settings file (default %s)'%(settingsFile) )
Script.registerSwitch( 'y:', 'strategy=', 'name of tracking strategy file to use (default %s)'%(strategyFile) )
Script.registerSwitch( 'z:', 'maxfiles=', 'maximum number of files to process (default %s)'%(maxFiles) )
Script.registerSwitch( 'g:', 'pandoradetector=', 'name of pandora geometry xml file or just detector name (default %s)'%(slicPandoraDetector) )
Script.registerSwitch( 'G:', 'gearxml=', 'name of gear geometry xml file (default %s)'%(gearFile) )
Script.registerSwitch( 'Z:', 'postlcsimxml=', 'name of the lcsim steering file used in the finalization step (default %s)'%(xmlPostPandora) )
Script.setUsageMessage( sys.argv[0]+'-n <nEvts> (-i <inputList> OR -I <prodID> AND/OR -m <macro>) (<additional options>)\\A process name needs to be defined using -p NAME for the output path. Only when using a production ID as input the process name can be obtained from the file catalog meta data.' )
Script.parseCommandLine()
switches = Script.getUnprocessedSwitches()
for switch in switches:
opt = switch[0]
arg = switch[1]
if opt in ('a','alias'):
aliasFile = arg
if opt in ('A','agent'):
agentMode = True
if opt in ('b','banlist'):
banlistFile = arg
if opt in ('C','destination'):
destination = arg
if opt in ('D','detector'):
detector = arg
if opt in ('f','file'):
dmScript.registerBKSwitches()
dmScript.registerFileSwitches()
fixIt = False
Script.registerSwitch('', 'FixIt', 'Set replicas problematic if needed')
Script.setUsageMessage("""
Check the integrity of the state of the storages and information in the File Catalogs
for a given file or a collection of files.
Usage:
%s <lfn | fileContainingLfns> <SE> <status>
""" % Script.scriptName)
Script.parseCommandLine()
for opt, val in Script.getUnprocessedSwitches():
if opt == 'FixIt':
fixIt = True
from DIRAC import gLogger
gLogger.setLevel('INFO')
from LHCbDIRAC.DataManagementSystem.Client.DataIntegrityClient import DataIntegrityClient
for lfn in Script.getPositionalArgs():
dmScript.setLFNsFromFile(lfn)
lfns = dmScript.getOption('LFNs')
if not lfns:
print "No LFNs given..."
Script.showHelp()
DIRAC.exit(0)
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("p:", "inputfile=", "Input File")
Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig")
Script.registerSwitch("V:", "version=", "Version (Use setVersion)")
Script.registerSwitch("C:", "comp=", "Compile (True/False)")
Script.parseCommandLine(ignoreErrors=True)
## default values ##############
simtelConfig = None
version = None
comp = True
### set switch values ###
for switch in Script.getUnprocessedSwitches():
if switch[0] == "inputfile" or switch[0] == "p":
corsikaFileLFN = switch[1].split('ParametricInputData=LFN:')[1]
elif switch[0] == "simtelConfig" or switch[0] == "S":
simtelConfig = switch[1]
elif switch[0] == "version" or switch[0] == "V":
version = switch[1]
elif switch[0] == "comp" or switch[0] == "C":
comp = switch[1]
if version == None:
Script.showHelp()
jobReport.setApplicationStatus('Missing options')
DIRAC.exit(-1)
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
############ Producing SimTel File
if comp == True:
install_CorsikaSimtelPack(version)
else:
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
res = installSoftwarePackage(CorsikaSimtelPack, workingArea())
if not res['OK']:
DIRAC.gLogger.error('Failed to execute installSoftwarePackage',
res)
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
cfg_dict = {
"4MSST": 'cta-prod2-4m-dc',
"SCSST": 'cta-prod2-sc-sst',
"STD": 'cta-prod2',
"NSBX3": 'cta-prod2',
"ASTRI": 'cta-prod2-astri',
"NORTH": 'cta-prod2n',
"ASTRISLN": 'cta-astri-sln'
}
#if simtelConfig=="6INROW":
# all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD","SCMST"]
#elif simtelConfig=="5INROW":
# all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD"]
#else:
# all_configs=[simtelConfig]
all_configs = [simtelConfig]
for current_conf in all_configs:
DIRAC.gLogger.notice('current conf is', current_conf)
current_version = version
DIRAC.gLogger.notice('current version is', current_version)
#if current_conf == "SCMST":
# current_version = version + '_sc3'
# DIRAC.gLogger.notice('current version is', current_version)
# if os.path.isdir('sim_telarray'):
# DIRAC.gLogger.notice('Package found in the local area. Removing package...')
# cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run'
# if(os.system(cmd)):
# DIRAC.exit( -1 )
# install_CorsikaSimtelPack(current_version)
#else:
# current_version = version
# DIRAC.gLogger.notice('current version is', current_version)
#### execute simtelarray ################
fd = open('run_sim.sh', 'w')
fd.write("""#! /bin/sh
export SVNPROD2=$PWD
export SVNTAG=SVN-PROD2
export CORSIKA_IO_BUFFER=800MB
./grid_prod2-repro.sh %s %s""" %
(os.path.basename(corsikaFileLFN), current_conf))
fd.close()
####################################
os.system('chmod u+x run_sim.sh')
os.system('chmod u+x grid_prod2-repro.sh')
cmdTuple = ['./run_sim.sh']
ret = systemCall(0, cmdTuple, sendSimtelOutput)
simtelReturnCode, stdout, stderr = ret['Value']
if (os.system('grep Broken simtel.log') == 0):
DIRAC.gLogger.error('Broken string found in simtel.log')
jobReport.setApplicationStatus('Broken pipe')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute run_sim.sh')
DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode)
DIRAC.exit(-1)
## check simtel data/log/histo Output File exist
cfg = cfg_dict[current_conf]
simtelFileName = os.path.basename(corsikaFileLFN).replace(
'corsika.gz', 'simtel.gz')
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Data/*.simtel.gz ' + simtelFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelLogFileName = simtelFileName.replace('simtel.gz', 'log.gz')
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Log/*.log.gz ' + simtelLogFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelHistFileName = simtelFileName.replace('simtel.gz', 'hdata.gz')
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName
if (os.system(cmd)):
DIRAC.exit(-1)
################################################################
DIRAC.exit()
# Instantiate the params class
cliParams = Params()
# Register accepted switches and their callbacks
Script.registerSwitch("sp:", "split=", "Number of files the original will be split into.", cliParams.setSplit)
Script.registerSwitch("t:", "total=", "Total number of files (split + EC generated ones).", cliParams.setTotal)
Script.registerSwitch("rd:", "remote_directory=", "Direcory where the files will be uploaded.", cliParams.setRemDir)
Script.registerSwitch("td:", "temporary_directory=", "Location of the temporary files until they are uploaded.", cliParams.setTempDir)
Script.registerSwitch("i:", "input_file=", "Location of the file to be uploaded.", cliParams.setInputFile)
Script.registerSwitch("se:", "se_list=", "A file with names of usable SEs.", cliParams.setSEList)
Script.registerSwitch("pr:", "processes=", "Number of processes to run concurrently.", cliParams.setProcesses)
# Parse the command line and initialize DIRAC
Script.parseCommandLine(ignoreErrors = False)
switches = dict(Script.getUnprocessedSwitches())
# Get the list of services
servicesList = Script.getPositionalArgs()
import DIRAC.Interfaces.API.Dirac as dirac_api
import DIRAC.Resources.Catalog.FileCatalogClient as FCC
def get_se_status(testdir):
# A function that tests the SEs visible by dirac-dms-show-se-status
# by adding a file to them to see if they work.
# Name of the file on the SE
testfile_remote = '1'
# Create a file to upload for testing
def main():
Script.registerSwitch(
"", "hosts=",
"Comma separated list of hosts or file containing row wise list of hosts"
" targeted for update (leave empty for all)")
Script.registerSwitch(
"", "retry=",
"Number of retry attempts on hosts that have failed to update")
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if len(args) < 1 or len(args) > 2:
Script.showHelp()
version = args[0]
retry = 0
hosts = []
for switch in Script.getUnprocessedSwitches():
if switch[0] == "hosts":
hosts = switch[1]
if switch[0] == "retry":
retry = int(switch[1])
try:
with open(hosts, 'r') as f:
hosts = f.read().splitlines()
hosts = [str(host) for host in hosts]
except Exception:
pass
if not isinstance(hosts, list):
hosts = hosts.split(',')
from concurrent.futures import ThreadPoolExecutor, as_completed
from DIRAC import S_OK, S_ERROR
from DIRAC import gLogger
from DIRAC.FrameworkSystem.Client.SystemAdministratorClient import SystemAdministratorClient
from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient
def getListOfHosts():
"""
Return the list of all hosts that constitute a DIRAC instance
"""
client = ComponentMonitoringClient()
result = client.getHosts({}, False, False)
if result['OK']:
hosts = [host['HostName'] for host in result['Value']]
return S_OK(hosts)
return S_ERROR('Cannot get list of hosts: %s' % result['Message'])
def parseHostname(hostName):
"""
Separate the hostname from the port
:param str hostName: hostname you want to parse
"""
hostList = hostName.split(':')
host = hostList[0]
if len(hostList) == 2:
port = hostList[1]
else:
port = None
return host, port
def updateHost(hostName, version):
"""
Apply update to specific host
:param str hostName: name of the host you want to update
:param str version: version vArBpC you want to update to
"""
host, port = parseHostname(hostName)
client = SystemAdministratorClient(host, port)
result = client.ping()
if not result['OK']:
gLogger.error("Cannot connect to %s" % host)
return result
gLogger.notice(
"Initiating software update of %s, this can take a while, please be patient ..."
% host)
result = client.updateSoftware(version, '', '', timeout=600)
if not result['OK']:
return result
return S_OK()
def updateHosts(version, hosts=[]):
"""
Apply update to all hosts
:param str version: version vArBpC you want to update to
:param list[str] hosts: list of hosts to be updated, defaults to []
"""
if not hosts:
result = getListOfHosts()
if not result['OK']:
return result
hosts = result['Value']
updateSuccess = []
updateFail = []
executor = ThreadPoolExecutor(max_workers=len(hosts))
futureUpdate = {
executor.submit(updateHost, host, version): host
for host in hosts
}
for future in as_completed(futureUpdate):
host = futureUpdate[future]
result = future.result()
if result['OK']:
updateSuccess.append(host)
else:
updateFail.append(host)
if not updateFail:
gLogger.notice("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
gLogger.notice("!!! Successfully updated all hosts !!!")
gLogger.notice("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
return S_OK([updateSuccess, updateFail])
if not updateSuccess:
gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
gLogger.notice("XXXXX Failed to update all hosts XXXXX")
gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
return S_OK([updateSuccess, updateFail])
gLogger.notice("X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!")
gLogger.notice("X!X!X Partially updated hosts X!X!X!")
gLogger.notice("Succeeded to update:")
for host in updateSuccess:
gLogger.notice(" + %s" % host)
gLogger.notice("Failed to update:")
for host in updateFail:
gLogger.notice(" - %s" % host)
gLogger.notice("X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!X!")
return S_OK([updateSuccess, updateFail])
def restartHost(hostName):
"""
Restart all systems and components of a host
:param str hostName: name of the host you want to restart
"""
host, port = parseHostname(hostName)
gLogger.notice("Pinging %s ..." % host)
client = SystemAdministratorClient(host, port)
result = client.ping()
if not result['OK']:
gLogger.error("Could not connect to %s: %s" %
(host, result['Message']))
return result
gLogger.notice("Host %s is active" % host)
gLogger.notice("Initiating restart of all systems and components")
# This restart call will always return S_ERROR because of SystemAdministrator restart
# Connection will be lost to the host
result = client.restartComponent('*', '*')
if result['Message'] == "Peer closed connection":
gLogger.notice(
"Restarted all systems on %s : connection to SystemAdministrator lost"
% host)
return S_OK(result['Message'])
gLogger.error("Received unxpected message: %s" % result['Message'])
return result
def updateInstance(version, hosts, retry):
"""
Update each server of an instance and restart them
:param str version: version vArBpC you want to update to
:param list[str] hosts: list of hosts to be updated
:param int retry: number of retry attempts on hosts that have failed to update
"""
result = updateHosts(version, hosts)
if not result['OK']:
return result
updateSuccess = result['Value'][0]
updateFail = result['Value'][1]
restartSuccess = []
restartFail = []
for host in updateSuccess:
result = restartHost(host)
if result['OK']:
restartSuccess.append(host)
else:
restartFail.append(host)
if not restartFail and not updateFail:
return S_OK("Successfully updated and restarted all hosts")
gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
gLogger.notice("XXXXX There were problems in the update process XXXXX")
gLogger.notice("Succeeded to update:")
for host in updateSuccess:
gLogger.notice(" + %s" % host)
gLogger.notice("Succeeded to restart:")
for host in restartSuccess:
gLogger.notice(" + %s" % host)
gLogger.notice("Failed to update:")
for host in updateFail:
gLogger.notice(" - %s" % host)
gLogger.notice("Failed to restart:")
for host in restartFail:
gLogger.notice(" - %s" % host)
gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
if retry > 0:
retryHosts = list(set(updateFail + restartFail))
gLogger.notice("Retrying update on (%s attempts remaining):" %
retry)
for host in retryHosts:
gLogger.notice(" - %s" % host)
gLogger.notice(
"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
return updateInstance(version, retryHosts, retry - 1)
return S_ERROR("Update failed!")
result = updateInstance(version, hosts, retry)
if not result['OK']:
gLogger.fatal(result['Message'])
DIRAC.exit(1)
gLogger.notice(result['Value'])
DIRAC.exit(0)
def main():
parameters = ['OwnerDN', 'StartExecTime', 'EndExecTime']
Script.registerSwitch(
'', 'Parameters=',
' List of strings to be matched by job parameters or attributes')
Script.parseCommandLine(ignoreErrors=True)
for switch in Script.getUnprocessedSwitches():
if switch[0] == 'Parameters':
parameters += [par for par in switch[1].split(',')]
parameters = [(i, par.lower()) for i, par in enumerate(parameters) if par]
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.Interfaces.API.Dirac import Dirac
diracAdmin = DiracAdmin()
dirac = Dirac()
errorList = []
for gridID in args:
result = {}
res = diracAdmin.getPilotInfo(gridID)
if not res['OK']:
errorList.append((gridID, res['Message']))
else:
jobIDs = set(
[int(jobID) for jobID in res['Value'][gridID]['Jobs']])
totCPU = 0
totWall = 0
effRequested = False
for jobID in sorted(jobIDs):
result.setdefault(jobID, {})
for func in (dirac.getJobParameters, dirac.getJobAttributes):
res = func(jobID)
if not res['OK']:
errorList.append(('Job %d' % jobID, res['Message']))
else:
params = res['Value']
if 'TotalCPUTime(s)' in params:
totCPU += float(params['TotalCPUTime(s)'])
totWall += float(params['WallClockTime(s)'])
params['CPUEfficiency'] = '%s %%' % (
100. * params['TotalCPUTime(s)'] /
params['WallClockTime(s)'])
for i, par in parameters:
for param in [
p for p in _stringInList(
str(par), str(params))
if not _stringInList(
str(p), str(result[jobID]))
]:
if param == 'CPUEfficiency':
effRequested = True
result[jobID]['%d.%s' %
(i, param)] = params[param]
if effRequested:
result['CPUEfficiency'] = '%s %%' % (100. * totCPU /
totWall)
print(diracAdmin.pPrint.pformat({gridID: result}))
for error in errorList:
print("ERROR %s: %s" % error)
def main():
args = Script.getPositionalArgs()
switches = Script.getUnprocessedSwitches()
name = 'jsub'
job_group = 'jsub-job'
task_id = 0
sub_ids = []
backend_ids = []
input_sandbox = []
output_sandbox = []
executable = ''
site = None
banned_site = None
backend_task_id = 0
job_status = []
for k, v in switches:
if k == 'name':
name = v
elif k == 'job-group':
job_group = v
elif k == 'task-id':
task_id = int(v)
elif k == 'sub-ids':
sub_ids = getListArg(v)
elif k == 'backend-ids':
backend_ids = getListArg(v)
elif k == 'input-sandbox':
input_sandbox = getListArg(v)
elif k == 'output-sandbox':
output_sandbox = getListArg(v)
elif k == 'executable':
executable = v
elif k == 'site':
site = getListArg(v)
elif k == 'banned-site':
banned_site = getListArg(v)
elif k == 'backend-task-id':
backend_task_id = v
elif k == 'job-status':
job_status = getListArg(v)
if args[0] == 'submit':
result = submit(name,
job_group,
task_id,
input_sandbox,
output_sandbox,
executable,
site,
banned_site,
sub_ids=sub_ids)
elif args[0] == 'delete':
result = delete(int(backend_task_id), job_status)
elif args[0] == 'reschedule':
result = reschedule(int(backend_task_id), job_status, sub_ids,
backend_ids)
elif args[0] == 'status':
result = status(int(backend_task_id), job_status)
print(json.dumps(result))
return 0
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch('', 'Job=', ' JobID[,jobID2,...]')
Script.registerSwitch('', 'Transformation=', ' transformation ID')
Script.registerSwitch(
'', 'Tasks=', ' Associated to --Transformation, list of taskIDs')
Script.registerSwitch('', 'Verbose', ' Print more information')
Script.registerSwitch('', 'Terse', ' Only print request status')
Script.registerSwitch('', 'Full', ' Print full request content')
Script.registerSwitch('', 'Status=',
' Select all requests in a given status')
Script.registerSwitch(
'', 'Since=',
' Associated to --Status, start date yyyy-mm-dd or nb of days (default= -one day'
)
Script.registerSwitch(
'', 'Until=', ' Associated to --Status, end date (default= now')
Script.registerSwitch(
'', 'Maximum=',
' Associated to --Status, max number of requests ')
Script.registerSwitch('', 'Reset',
' Reset Failed files to Waiting if any')
Script.registerSwitch('', 'Force', ' Force reset even if not Failed')
Script.registerSwitch(
'', 'All',
' (if --Status Failed) all requests, otherwise exclude irrecoverable failures'
)
Script.registerSwitch('', 'FixJob',
' Set job Done if the request is Done')
Script.registerSwitch('', 'Cancel', ' Cancel the request')
Script.registerSwitch('', 'ListJobs', ' List the corresponding jobs')
Script.registerSwitch(
'', 'TargetSE=', ' Select request only if that SE is in the targetSEs')
from DIRAC.Core.Base.Script import parseCommandLine
parseCommandLine()
import DIRAC
from DIRAC import gLogger
jobs = []
requestID = 0
transID = None
taskIDs = None
tasks = None
requests = []
full = False
verbose = False
status = None
until = None
since = None
terse = False
allR = False
reset = False
fixJob = False
maxRequests = 999999999999
cancel = False
listJobs = False
force = False
targetSE = set()
for switch in Script.getUnprocessedSwitches():
if switch[0] == 'Job':
jobs = []
job = "Unknown"
try:
for arg in switch[1].split(','):
if os.path.exists(arg):
with open(arg, 'r') as fp:
lines = fp.readlines()
for line in lines:
for job in line.split(','):
jobs += [int(job.strip())]
gLogger.notice("Found %d jobs in file %s" %
(len(jobs), arg))
else:
jobs.append(int(arg))
except TypeError:
gLogger.fatal("Invalid jobID", job)
elif switch[0] == 'Transformation':
try:
transID = int(switch[1])
except Exception:
gLogger.fatal('Invalid transID', switch[1])
elif switch[0] == 'Tasks':
try:
taskIDs = [int(task) for task in switch[1].split(',')]
except Exception:
gLogger.fatal('Invalid tasks', switch[1])
elif switch[0] == 'Full':
full = True
elif switch[0] == 'Verbose':
verbose = True
elif switch[0] == 'Terse':
terse = True
elif switch[0] == 'All':
allR = True
elif switch[0] == 'Reset':
reset = True
elif switch[0] == 'Force':
force = True
elif switch[0] == 'Status':
status = switch[1].capitalize()
elif switch[0] == 'Since':
since = convertDate(switch[1])
elif switch[0] == 'Until':
until = convertDate(switch[1])
elif switch[0] == 'FixJob':
fixJob = True
elif switch[0] == 'Cancel':
cancel = True
elif switch[0] == 'ListJobs':
listJobs = True
elif switch[0] == 'Maximum':
try:
maxRequests = int(switch[1])
except Exception:
pass
elif switch[0] == 'TargetSE':
targetSE = set(switch[1].split(','))
if reset and not force:
status = 'Failed'
if fixJob:
status = 'Done'
if terse:
verbose = True
if status:
if not until:
until = datetime.datetime.utcnow()
if not since:
since = until - datetime.timedelta(hours=24)
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.RequestManagementSystem.Client.ReqClient import printRequest, recoverableRequest
reqClient = ReqClient()
if transID:
if not taskIDs:
gLogger.fatal(
"If Transformation is set, a list of Tasks should also be set")
Script.showHelp(exitCode=2)
# In principle, the task name is unique, so the request name should be unique as well
# If ever this would not work anymore, we would need to use the transformationClient
# to fetch the ExternalID
requests = ['%08d_%08d' % (transID, task) for task in taskIDs]
allR = True
elif not jobs:
requests = []
# Get full list of arguments, with and without comma
for arg in [
x.strip() for arg in Script.getPositionalArgs()
for x in arg.split(',')
]:
if os.path.exists(arg):
lines = open(arg, 'r').readlines()
requests += [
reqID.strip() for line in lines
for reqID in line.split(',')
]
gLogger.notice("Found %d requests in file" % len(requests))
else:
requests.append(arg)
allR = True
else:
res = reqClient.getRequestIDsForJobs(jobs)
if not res['OK']:
gLogger.fatal("Error getting request for jobs", res['Message'])
DIRAC.exit(2)
if res['Value']['Failed']:
gLogger.error(
"No request found for jobs %s" %
','.join(sorted(str(job) for job in res['Value']['Failed'])))
requests = sorted(res['Value']['Successful'].values())
if requests:
allR = True
else:
DIRAC.exit(0)
if status and not requests:
allR = allR or status != 'Failed'
res = reqClient.getRequestIDsList([status],
limit=maxRequests,
since=since,
until=until)
if not res['OK']:
gLogger.error("Error getting requests:", res['Message'])
DIRAC.exit(2)
requests = [
reqID for reqID, _st, updTime in res['Value']
if updTime > since and updTime <= until and reqID
]
gLogger.notice('Obtained %d requests %s between %s and %s' %
(len(requests), status, since, until))
if not requests:
gLogger.notice('No request selected....')
Script.showHelp(exitCode=2)
okRequests = []
warningPrinted = False
jobIDList = []
for reqID in requests:
# We allow reqID to be the requestName if it is unique
try:
requestID = int(reqID)
except ValueError:
requestID = reqClient.getRequestIDForName(reqID)
if not requestID['OK']:
gLogger.notice(requestID['Message'])
continue
requestID = requestID['Value']
request = reqClient.peekRequest(requestID)
if not request["OK"]:
gLogger.error(request["Message"])
DIRAC.exit(-1)
request = request["Value"]
if not request:
gLogger.error("no such request %s" % requestID)
continue
# If no operation as the targetSE, skip
if targetSE:
found = False
for op in request:
if op.TargetSE and targetSE.intersection(
op.TargetSE.split(',')):
found = True
break
if not found:
continue
# keep a list of jobIDs if requested
if request.JobID and listJobs:
jobIDList.append(request.JobID)
if status and request.Status != status:
gLogger.notice(
"Request %s is not in requested status %s%s" %
(reqID, status, ' (cannot be reset)' if reset else ''))
continue
if fixJob and request.Status == 'Done' and request.JobID:
# The request is for a job and is Done, verify that the job is in the proper status
result = reqClient.finalizeRequest(request.RequestID,
request.JobID,
useCertificates=False)
if not result['OK']:
gLogger.error("Error finalizing job", result['Message'])
else:
gLogger.notice("Job %d updated to %s" %
(request.JobID, result['Value']))
continue
if cancel:
if request.Status not in ('Done', 'Failed'):
ret = reqClient.cancelRequest(requestID)
if not ret['OK']:
gLogger.error("Error canceling request %s" % reqID,
ret['Message'])
else:
gLogger.notice("Request %s cancelled" % reqID)
else:
gLogger.notice("Request %s is in status %s, not cancelled" %
(reqID, request.Status))
elif allR or recoverableRequest(request):
okRequests.append(str(requestID))
if reset:
gLogger.notice('============ Request %s =============' %
requestID)
ret = reqClient.resetFailedRequest(requestID, allR=allR)
if not ret['OK']:
gLogger.error("Error resetting request %s" % requestID,
ret['Message'])
else:
if len(requests) > 1:
gLogger.notice('\n===================================')
dbStatus = reqClient.getRequestStatus(requestID).get(
'Value', 'Unknown')
printRequest(request,
status=dbStatus,
full=full,
verbose=verbose,
terse=terse)
if listJobs:
gLogger.notice("List of %d jobs:\n" % len(jobIDList),
','.join(str(jobID) for jobID in jobIDList))
if status and okRequests:
from DIRAC.Core.Utilities.List import breakListIntoChunks
gLogger.notice('\nList of %d selected requests:' % len(okRequests))
for reqs in breakListIntoChunks(okRequests, 100):
gLogger.notice(','.join(reqs))
def main():
Script.registerSwitch(
"v:", "vo=",
"Location of pilot version in CS /Operations/<vo>/Pilot/Version"
" (default value specified in CS under /DIRAC/DefaultSetup)")
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if len(args) < 1 or len(args) > 2:
Script.showHelp()
version = args[0]
vo = None
for switch in Script.getUnprocessedSwitches():
if switch[0] == "v" or switch[0] == "vo":
vo = switch[1]
from DIRAC import S_OK, S_ERROR
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
def updatePilot(version, vo):
"""
Update in the CS the pilot version used,
If only one version present in CS it's overwritten.
If two versions present, the new one is added and the last removed
:param version: version vArBpC of pilot you want to use
:param vo: Location of pilot version in CS /Operations/<vo>/Pilot/Version
"""
setup = vo
if not vo:
setup = gConfig.getValue('/DIRAC/DefaultSetup')
if not setup:
return S_ERROR("No value set for /DIRAC/DefaultSetup in CS")
pilotVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup,
[])
if not pilotVersion:
return S_ERROR(
"No pilot version set under Operations/%s/Pilot/Version in CS"
% setup)
pilotVersion.pop()
pilotVersion.insert(0, version)
api = CSAPI()
api.setOption('Operations/%s/Pilot/Version' % setup,
", ".join(pilotVersion))
result = api.commit()
if not result['OK']:
gLogger.fatal('Could not commit new version of pilot!')
return result
newVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup)
return S_OK("New version of pilot set to %s" % newVersion)
result = updatePilot(version, vo)
if not result['OK']:
gLogger.fatal(result['Message'])
DIRAC.exit(1)
gLogger.notice(result['Value'])
DIRAC.exit(0)
from DIRAC import S_OK, S_ERROR
from DIRAC.Core.Base import Script
Script.setUsageMessage(
"""
Delete task and all jobs in the task
Usage:
%s [option] ... [TaskID] ...
"""
% Script.scriptName
)
Script.parseCommandLine(ignoreErrors=False)
options = Script.getUnprocessedSwitches()
args = Script.getPositionalArgs()
from BESDIRAC.WorkloadManagementSystem.Client.TaskClient import TaskClient
taskClient = TaskClient()
def deleteTask(taskID):
result = taskClient.deleteTask(taskID)
if not result["OK"]:
print "Delete task error: %s" % result["Message"]
return
print "Task %s deleted" % taskID
'Arguments:',
' metaspec: metadata index specification (of the form: "meta=value" or "meta<value", "meta!=value", etc.)',
'', 'Examples:',
' $ dirac-dms-find-lfns Path=/lhcb/user "Size>1000" "CreationDate<2015-05-15"',
] )
)
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS
from DIRAC import gLogger
path = '/'
for opt, val in Script.getUnprocessedSwitches():
if opt == 'Path':
path = val
fc = FileCatalog()
result = fc.getMetadataFields()
if not result['OK']:
gLogger.error( 'Can not access File Catalog:', result['Message'] )
DIRAC.exit( -1 )
typeDict = result['Value']['FileMetaFields']
typeDict.update( result['Value']['DirectoryMetaFields'] )
# Special meta tags
typeDict.update( FILE_STANDARD_METAKEYS )
if len( args ) < 1:
print "Error: No argument provided\n%s:" % Script.scriptName
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("T:", "template=", "Corsika Template")
Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig")
Script.registerSwitch("p:", "run_number=",
"Do not use: Run Number automatically set")
Script.registerSwitch("E:", "executable=",
"Executable (Use SetExecutable)")
Script.registerSwitch("V:", "version=", "Version (Use setVersion)")
Script.registerSwitch("M:", "mode=",
"Mode (corsika_standalone/corsika_simtelarray)")
Script.parseCommandLine(ignoreErrors=True)
## default values ##############
run_number = None
corsikaTemplate = None
simtelConfig = None
executable = None
version = None
mode = 'corsika_simtelarray'
### set switch values ###
for switch in Script.getUnprocessedSwitches():
if switch[0] == "run_number" or switch[0] == "p":
run_number = switch[1].split('ParametricParameters=')[1]
elif switch[0] == "template" or switch[0] == "T":
corsikaTemplate = switch[1]
elif switch[0] == "simtelConfig" or switch[0] == "S":
simtelConfig = switch[1]
elif switch[0] == "executable" or switch[0] == "E":
executable = switch[1]
elif switch[0] == "version" or switch[0] == "V":
version = switch[1]
elif switch[0] == "mode" or switch[0] == "M":
mode = switch[1]
if version == None or executable == None or run_number == None or corsikaTemplate == None:
Script.showHelp()
jobReport.setApplicationStatus('Missing options')
DIRAC.exit(-1)
from CTADIRAC.Core.Workflow.Modules.CorsikaApp import CorsikaApp
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
createGlobalsFromConfigFiles(corsikaTemplate, version)
############ Producing Corsika File
install_CorsikaSimtelPack(version)
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
cs = CorsikaApp()
cs.setSoftwarePackage(CorsikaSimtelPack)
cs.csExe = executable
cs.csArguments = [
'--run-number', run_number, '--run', 'corsika', corsikaTemplate
]
corsikaReturnCode = cs.execute()
if corsikaReturnCode != 0:
DIRAC.gLogger.error('Corsika Application: Failed')
jobReport.setApplicationStatus('Corsika Application: Failed')
DIRAC.exit(-1)
###################### rename of corsika output file #######################
rundir = 'run' + run_number
filein = rundir + '/' + corsikaOutputFileName
corsikaFileName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.gz'
mv_cmd = 'mv ' + filein + ' ' + corsikaFileName
if (os.system(mv_cmd)):
DIRAC.exit(-1)
### create corsika tar luisa ####################
corsikaTarName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.tar.gz'
filetar1 = rundir + '/' + 'input'
filetar2 = rundir + '/' + 'DAT' + run_number + '.dbase'
filetar3 = rundir + '/run' + str(int(run_number)) + '.log'
cmdTuple = [
'/bin/tar', 'zcf', corsikaTarName, filetar1, filetar2, filetar3
]
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute tar')
DIRAC.exit(-1)
if (mode == 'corsika_standalone'):
DIRAC.exit()
############ Producing SimTel File
######################Building simtel Directory Metadata #######################
cfg_dict = {
"4MSST": 'cta-prod2-4m-dc',
"SCSST": 'cta-prod2-sc-sst',
"STD": 'cta-prod2',
"NSBX3": 'cta-prod2',
"ASTRI": 'cta-prod2-astri',
"NORTH": 'cta-prod2n',
"SCMST": 'cta-prod2-sc3'
}
#if simtelConfig=="6INROW":
# all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD","SCMST"]
#elif simtelConfig=="5INROW":
# all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD"]
#else:
# all_configs=[simtelConfig]
all_configs = [simtelConfig]
for current_conf in all_configs:
DIRAC.gLogger.notice('current conf is', current_conf)
current_version = version
DIRAC.gLogger.notice('current version is', current_version)
#if current_conf == "SCMST":
# current_version = version + '_sc3'
# DIRAC.gLogger.notice('current version is', current_version)
# if os.path.isdir('sim_telarray'):
# DIRAC.gLogger.notice('Package found in the local area. Removing package...')
# cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run'
# if(os.system(cmd)):
# DIRAC.exit( -1 )
# install_CorsikaSimtelPack(current_version)
#else:
# current_version = version
# DIRAC.gLogger.notice('current version is', current_version)
#### execute simtelarray ################
fd = open('run_sim.sh', 'w')
fd.write("""#! /bin/sh
export SVNPROD2=$PWD
export SVNTAG=SVN-PROD2
export CORSIKA_IO_BUFFER=800MB
./grid_prod2-repro.sh %s %s""" % (corsikaFileName, current_conf))
fd.close()
####################################
os.system('chmod u+x run_sim.sh')
os.system('chmod u+x grid_prod2-repro.sh')
cmdTuple = ['./run_sim.sh']
ret = systemCall(0, cmdTuple, sendSimtelOutput)
simtelReturnCode, stdout, stderr = ret['Value']
if (os.system('grep Broken simtel.log') == 0):
DIRAC.gLogger.error('Broken string found in simtel.log')
jobReport.setApplicationStatus('Broken pipe')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute run_sim.sh')
DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode)
DIRAC.exit(-1)
## check simtel data/log/histo Output File exist
cfg = cfg_dict[current_conf]
simtelFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.simtel.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/*/Data/*.simtel.gz ' + simtelFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelLogFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/*/Log/*.log.gz ' + simtelLogFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelHistFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.hdata.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/*/Histograms/*.hdata.gz ' + simtelHistFileName
if (os.system(cmd)):
DIRAC.exit(-1)
################################################################
DIRAC.exit()
def main():
days = 0
months = 0
years = 0
wildcard = None
baseDir = ''
emptyDirsFlag = False
Script.registerSwitch("D:", "Days=",
"Match files older than number of days [%s]" % days)
Script.registerSwitch(
"M:", "Months=",
"Match files older than number of months [%s]" % months)
Script.registerSwitch(
"Y:", "Years=", "Match files older than number of years [%s]" % years)
Script.registerSwitch("w:", "Wildcard=",
"Wildcard for matching filenames [All]")
Script.registerSwitch(
"b:", "BaseDir=",
"Base directory to begin search (default /[vo]/user/[initial]/[username])"
)
Script.registerSwitch("e", "EmptyDirs",
"Create a list of empty directories")
Script.parseCommandLine(ignoreErrors=False)
for switch in Script.getUnprocessedSwitches():
if switch[0] == "D" or switch[0].lower() == "days":
days = int(switch[1])
if switch[0] == "M" or switch[0].lower() == "months":
months = int(switch[1])
if switch[0] == "Y" or switch[0].lower() == "years":
years = int(switch[1])
if switch[0].lower() == "w" or switch[0].lower() == "wildcard":
wildcard = '*' + switch[1]
if switch[0].lower() == "b" or switch[0].lower() == "basedir":
baseDir = switch[1]
if switch[0].lower() == "e" or switch[0].lower() == "emptydirs":
emptyDirsFlag = True
import DIRAC
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from datetime import datetime, timedelta
import sys
import os
import time
import fnmatch
fc = FileCatalog()
def isOlderThan(cTimeStruct, days):
timeDelta = timedelta(days=days)
maxCTime = datetime.utcnow() - timeDelta
if cTimeStruct < maxCTime:
return True
return False
withMetadata = False
if days or months or years:
withMetadata = True
totalDays = 0
if years:
totalDays += 365 * years
if months:
totalDays += 30 * months
if days:
totalDays += days
res = getProxyInfo(False, False)
if not res['OK']:
gLogger.error("Failed to get client proxy information.",
res['Message'])
DIRAC.exit(2)
proxyInfo = res['Value']
if proxyInfo['secondsLeft'] == 0:
gLogger.error("Proxy expired")
DIRAC.exit(2)
username = proxyInfo['username']
vo = ''
if 'group' in proxyInfo:
vo = getVOForGroup(proxyInfo['group'])
if not baseDir:
if not vo:
gLogger.error('Could not determine VO')
Script.showHelp()
baseDir = '/%s/user/%s/%s' % (vo, username[0], username)
baseDir = baseDir.rstrip('/')
gLogger.notice('Will search for files in %s%s' %
(baseDir, (' matching %s' % wildcard) if wildcard else ''))
activeDirs = [baseDir]
allFiles = []
emptyDirs = []
while len(activeDirs) > 0:
currentDir = activeDirs.pop()
res = fc.listDirectory(currentDir, withMetadata, timeout=360)
if not res['OK']:
gLogger.error("Error retrieving directory contents",
"%s %s" % (currentDir, res['Message']))
elif currentDir in res['Value']['Failed']:
gLogger.error(
"Error retrieving directory contents",
"%s %s" % (currentDir, res['Value']['Failed'][currentDir]))
else:
dirContents = res['Value']['Successful'][currentDir]
subdirs = dirContents['SubDirs']
files = dirContents['Files']
if not subdirs and not files:
emptyDirs.append(currentDir)
gLogger.notice('%s: empty directory' % currentDir)
else:
for subdir in sorted(subdirs, reverse=True):
if (not withMetadata) or isOlderThan(
subdirs[subdir]['CreationDate'], totalDays):
activeDirs.append(subdir)
for filename in sorted(files):
fileOK = False
if (not withMetadata) or isOlderThan(
files[filename]['MetaData']['CreationDate'],
totalDays):
if wildcard is None or fnmatch.fnmatch(
filename, wildcard):
fileOK = True
if not fileOK:
files.pop(filename)
allFiles += sorted(files)
if len(files) or len(subdirs):
gLogger.notice(
"%s: %d files%s, %d sub-directories" %
(currentDir, len(files), ' matching'
if withMetadata or wildcard else '', len(subdirs)))
outputFileName = '%s.lfns' % baseDir.replace('/%s' % vo,
'%s' % vo).replace('/', '-')
outputFile = open(outputFileName, 'w')
for lfn in sorted(allFiles):
outputFile.write(lfn + '\n')
outputFile.close()
gLogger.notice('%d matched files have been put in %s' %
(len(allFiles), outputFileName))
if emptyDirsFlag:
outputFileName = '%s.emptydirs' % baseDir.replace(
'/%s' % vo, '%s' % vo).replace('/', '-')
outputFile = open(outputFileName, 'w')
for dir in sorted(emptyDirs):
outputFile.write(dir + '\n')
outputFile.close()
gLogger.notice('%d empty directories have been put in %s' %
(len(emptyDirs), outputFileName))
DIRAC.exit(0)
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if not args:
error(
"Argument is missing, you should enter either 'test', 'update', 'view', 'remove', 'restore'"
)
else:
cmd = args[0].lower()
switches = dict(Script.getUnprocessedSwitches())
diracConfigFile = CSHelpers.gConfig.diracConfigFilePath
# Default values
switches.setdefault('name', None)
switches.setdefault('element', None)
switches.setdefault('elementType', None)
switches.setdefault('setup', "Defaults")
switches.setdefault('file', diracConfigFile)
#switches.setdefault( 'statusType', None )
#switches.setdefault( 'status', None )
# when it's a add/modify query and status/reason/statusType are not specified
# then some specific defaults are set up
if cmd == 'test':
if switches['elementType'] is None and switches[
'element'] is None and switches['name'] is None:
error(
"to test, you should enter at least one switch: either element, elmentType, or name"
)
else:
if switches['element'] is not None:
switches['element'] = switches['element'].title()
if switches['element'] not in ('Resource', 'Site'):
error(
"you should enter either 'Site' or 'Resource' for switch 'element'"
)
if switches['elementType'] is not None:
switches['elementType'] = switches['elementType'].title()
if switches['file'] is None:
error(
"Enter a fullpath dirac config file location when using 'file' option"
)
elif cmd == 'remove':
if 'policy' not in switches or switches['policy'] is None:
error("to remove, you should enter a policy")
elif cmd == 'update' or cmd == 'view' or cmd == 'restore':
pass
else:
error(
"Incorrect argument: you should enter either 'test', 'update', 'view', 'remove', 'restore'"
)
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.iteritems())
return args, switches
def main():
read = True
write = True
check = True
remove = True
site = ''
mute = False
Script.registerSwitch("r", "BanRead", " Ban only reading from the storage element")
Script.registerSwitch("w", "BanWrite", " Ban writing to the storage element")
Script.registerSwitch("k", "BanCheck", " Ban check access to the storage element")
Script.registerSwitch("v", "BanRemove", " Ban remove access to the storage element")
Script.registerSwitch("a", "All", " Ban all access to the storage element")
Script.registerSwitch("m", "Mute", " Do not send email")
Script.registerSwitch(
"S:",
"Site=",
" Ban all SEs associate to site (note that if writing is allowed, check is always allowed)")
Script.parseCommandLine(ignoreErrors=True)
ses = Script.getPositionalArgs()
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() in ("r", "banread"):
write = False
check = False
remove = False
if switch[0].lower() in ("w", "banwrite"):
read = False
check = False
remove = False
if switch[0].lower() in ("k", "bancheck"):
read = False
write = False
remove = False
if switch[0].lower() in ("v", "banremove"):
read = False
write = False
check = False
if switch[0].lower() in ("a", "all"):
pass
if switch[0].lower() in ("m", "mute"):
mute = True
if switch[0].lower() in ("s", "site"):
site = switch[1]
# from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
ses = resolveSEGroup(ses)
diracAdmin = DiracAdmin()
errorList = []
setup = gConfig.getValue('/DIRAC/Setup', '')
if not setup:
print('ERROR: Could not contact Configuration Service')
DIRAC.exit(2)
res = getProxyInfo()
if not res['OK']:
gLogger.error('Failed to get proxy information', res['Message'])
DIRAC.exit(2)
userName = res['Value'].get('username')
if not userName:
gLogger.error('Failed to get username for proxy')
DIRAC.exit(2)
if site:
res = getSites()
if not res['OK']:
gLogger.error(res['Message'])
DIRAC.exit(-1)
if site not in res['Value']:
gLogger.error('The provided site (%s) is not known.' % site)
DIRAC.exit(-1)
ses.extend(res['Value']['SE'].replace(' ', '').split(','))
if not ses:
gLogger.error('There were no SEs provided')
DIRAC.exit(-1)
readBanned = []
writeBanned = []
checkBanned = []
removeBanned = []
resourceStatus = ResourceStatus()
res = resourceStatus.getElementStatus(ses, "StorageElement")
if not res['OK']:
gLogger.error("Storage Element %s does not exist" % ses)
DIRAC.exit(-1)
reason = 'Forced with dirac-admin-ban-se by %s' % userName
for se, seOptions in res['Value'].items():
resW = resC = resR = {'OK': False}
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if read and 'ReadAccess' in seOptions:
if not seOptions['ReadAccess'] in ['Active', 'Degraded', 'Probing']:
gLogger.notice('Read option for %s is %s, instead of %s' %
(se, seOptions['ReadAccess'], ['Active', 'Degraded', 'Probing']))
gLogger.notice('Try specifying the command switches')
else:
resR = resourceStatus.setElementStatus(se, 'StorageElement', 'ReadAccess', 'Banned', reason, userName)
# res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" )
if not resR['OK']:
gLogger.error('Failed to update %s read access to Banned' % se)
else:
gLogger.notice('Successfully updated %s read access to Banned' % se)
readBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if write and 'WriteAccess' in seOptions:
if not seOptions['WriteAccess'] in ['Active', 'Degraded', 'Probing']:
gLogger.notice('Write option for %s is %s, instead of %s' %
(se, seOptions['WriteAccess'], ['Active', 'Degraded', 'Probing']))
gLogger.notice('Try specifying the command switches')
else:
resW = resourceStatus.setElementStatus(se, 'StorageElement', 'WriteAccess', 'Banned', reason, userName)
# res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" )
if not resW['OK']:
gLogger.error("Failed to update %s write access to Banned" % se)
else:
gLogger.notice("Successfully updated %s write access to Banned" % se)
writeBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if check and 'CheckAccess' in seOptions:
if not seOptions['CheckAccess'] in ['Active', 'Degraded', 'Probing']:
gLogger.notice('Check option for %s is %s, instead of %s' %
(se, seOptions['CheckAccess'], ['Active', 'Degraded', 'Probing']))
gLogger.notice('Try specifying the command switches')
else:
resC = resourceStatus.setElementStatus(se, 'StorageElement', 'CheckAccess', 'Banned', reason, userName)
# res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" )
if not resC['OK']:
gLogger.error("Failed to update %s check access to Banned" % se)
else:
gLogger.notice("Successfully updated %s check access to Banned" % se)
checkBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if remove and 'RemoveAccess' in seOptions:
if not seOptions['RemoveAccess'] in ['Active', 'Degraded', 'Probing']:
gLogger.notice('Remove option for %s is %s, instead of %s' %
(se, seOptions['RemoveAccess'], ['Active', 'Degraded', 'Probing']))
gLogger.notice('Try specifying the command switches')
else:
resC = resourceStatus.setElementStatus(se, 'StorageElement', 'RemoveAccess', 'Banned', reason, userName)
# res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" )
if not resC['OK']:
gLogger.error("Failed to update %s remove access to Banned" % se)
else:
gLogger.notice("Successfully updated %s remove access to Banned" % se)
removeBanned.append(se)
if not(resR['OK'] or resW['OK'] or resC['OK']):
DIRAC.exit(-1)
if not (writeBanned or readBanned or checkBanned or removeBanned):
gLogger.notice("No storage elements were banned")
DIRAC.exit(-1)
if mute:
gLogger.notice('Email is muted by script switch')
DIRAC.exit(0)
subject = '%s storage elements banned for use' % len(writeBanned + readBanned + checkBanned + removeBanned)
addressPath = 'EMail/Production'
address = Operations().getValue(addressPath, '')
body = ''
if read:
body = "%s\n\nThe following storage elements were banned for reading:" % body
for se in readBanned:
body = "%s\n%s" % (body, se)
if write:
body = "%s\n\nThe following storage elements were banned for writing:" % body
for se in writeBanned:
body = "%s\n%s" % (body, se)
if check:
body = "%s\n\nThe following storage elements were banned for check access:" % body
for se in checkBanned:
body = "%s\n%s" % (body, se)
if remove:
body = "%s\n\nThe following storage elements were banned for remove access:" % body
for se in removeBanned:
body = "%s\n%s" % (body, se)
if not address:
gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body)
DIRAC.exit(0)
res = diracAdmin.sendMail(address, subject, body)
gLogger.notice('Notifying %s' % address)
if res['OK']:
gLogger.notice(res['Value'])
else:
gLogger.notice(res['Message'])
DIRAC.exit(0)
from DIRAC import gLogger
from DIRAC.Core.Base import Script
from DIRAC.Core.DISET.RPCClient import RPCClient
from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient
from LHCbDIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
Script.setUsageMessage('\n'.join(
[__doc__.split('\n')[1], 'Usage:',
' %s eventType ' % Script.scriptName]))
Script.registerSwitch('p', 'production',
"Obtain the paths in ``Production format'' for Ganga")
Script.parseCommandLine(ignoreErrors=True)
productionFormat = False
for p, _v in Script.getUnprocessedSwitches():
if p.lower() in ('p', 'production'):
productionFormat = True
break
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp()
DIRAC.exit(1)
eventType = args[0]
bkClient = BookkeepingClient()
# # get productions for given event type
res = bkClient.getProductionSummaryFromView({
def main():
# Register workflow parameter switch
Script.registerSwitch(
'p:', 'parameter=',
'Parameters that are passed directly to the workflow')
Script.parseCommandLine()
# from DIRAC.Core.Workflow.Parameter import *
from DIRAC import gLogger
from DIRAC.Core.Workflow.Workflow import fromXMLFile
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient
from DIRAC.RequestManagementSystem.Client.Request import Request
# Forcing the current directory to be the first in the PYTHONPATH
sys.path.insert(0, os.path.realpath('.'))
gLogger.showHeaders(True)
def jobexec(jobxml, wfParameters):
jobfile = os.path.abspath(jobxml)
if not os.path.exists(jobfile):
gLogger.warn('Path to specified workflow %s does not exist' %
(jobfile))
sys.exit(1)
workflow = fromXMLFile(jobfile)
gLogger.debug(workflow)
code = workflow.createCode()
gLogger.debug(code)
jobID = 0
if 'JOBID' in os.environ:
jobID = os.environ['JOBID']
gLogger.info('DIRAC JobID %s is running at site %s' %
(jobID, DIRAC.siteName()))
workflow.addTool('JobReport', JobReport(jobID))
workflow.addTool('AccountingReport', DataStoreClient())
workflow.addTool('Request', Request())
# Propagate the command line parameters to the workflow if any
for pName, pValue in wfParameters.items():
workflow.setValue(pName, pValue)
# Propagate the command line parameters to the workflow module instances of each step
for stepdefinition in workflow.step_definitions.values():
for moduleInstance in stepdefinition.module_instances:
for pName, pValue in wfParameters.items():
if moduleInstance.parameters.find(pName):
moduleInstance.parameters.setValue(pName, pValue)
return workflow.execute()
positionalArgs = Script.getPositionalArgs()
if len(positionalArgs) != 1:
gLogger.debug('Positional arguments were %s' % (positionalArgs))
DIRAC.abort(1, "Must specify the Job XML file description")
if 'JOBID' in os.environ:
gLogger.info('JobID: %s' % (os.environ['JOBID']))
jobXMLfile = positionalArgs[0]
parList = Script.getUnprocessedSwitches()
parDict = {}
for switch, parameter in parList:
if switch == "p":
name, value = parameter.split('=')
value = value.strip()
# The comma separated list in curly brackets is interpreted as a list
if value.startswith("{"):
value = value[1:-1].replace('"', '').replace(" ",
'').split(',')
value = ';'.join(value)
parDict[name] = value
gLogger.debug('PYTHONPATH:\n%s' % ('\n'.join(sys.path)))
jobExec = jobexec(jobXMLfile, parDict)
if not jobExec['OK']:
gLogger.debug('Workflow execution finished with errors, exiting')
if jobExec['Errno']:
sys.exit(jobExec['Errno'])
else:
sys.exit(1)
else:
gLogger.debug('Workflow execution successful, exiting')
sys.exit(0)
#!/usr/bin/env python
from DIRAC.Core.Base import Script
Script.parseCommandLine( ignoreErrors = False )
options = Script.getUnprocessedSwitches()
args = Script.getPositionalArgs()
from DIRAC.Core.DISET.RPCClient import RPCClient
jobMonitor = RPCClient('WorkloadManagement/JobMonitoring')
if len(args) < 2:
print 'Need user name and status'
user = args[0]
status = args[1]
result = jobMonitor.getJobs( { 'Owner': user, 'Status': status } )
if not result['OK']:
print 'getjobs error: %s' % result['Message']
print len(result['Value'])
Script.registerSwitch( "g:", "JobGroup=", "Get output for jobs in the given group" )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
from DIRAC.Core.Utilities.Time import toString, date, day
dirac = Dirac()
exitCode = 0
errorList = []
outputDir = None
group = None
jobs = []
for sw, value in Script.getUnprocessedSwitches():
if sw in ( 'D', 'Dir' ):
outputDir = value
elif sw.lower() in ( 'f', 'file' ):
if os.path.exists( value ):
jFile = open( value )
jobs += jFile.read().split()
jFile.close()
elif sw.lower() in ( 'g', 'jobgroup' ):
group = value
jobDate = toString( date() - 30 * day )
# Choose jobs in final state, no more than 30 days old
result = dirac.selectJobs( jobGroup = value, date = jobDate, status = 'Done' )
if not result['OK']:
if not "No jobs selected" in result['Message']:
'Usage:',
' %s test directory' % Script.scriptName
]))
Script.registerSwitch("", "UseFilter=", "e.g. True/False")
Script.parseCommandLine()
from DIRAC import gLogger
from DIRAC.Interfaces.API.Job import Job
from DIRAC.TransformationSystem.Client.Transformation import Transformation
# Needed to test transformations with Filters
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
# Parse the arguments
args = Script.getUnprocessedSwitches()
if len(args) != 1:
Script.showHelp()
directory = args[0]
UseFilter = None
for switch, switchValue in Script.getUnprocessedSwitches():
if switch == "UseFilter":
UseFilter = True if switchValue.lower() == "true" else False
# Let's first create the prodJobuction
prodJobType = 'Merge'
transName = 'testProduction_' + str(int(time.time()))
desc = 'just test'
prodJob = Job()
Script.registerSwitch('S:', 'setup=',
"set the software dist module to update.")
Script.registerSwitch('D:', 'softwareDistModule=',
"set the software dist module to update.")
Script.parseCommandLine()
args = Script.getPositionalArgs()
from DIRAC import gConfig
cFile = ''
sMod = ''
vo = ''
setup = ''
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("F", "file"):
cFile = unprocSw[1]
if unprocSw[0] in ("V", "vo"):
vo = unprocSw[1]
if unprocSw[0] in ("D", "softwareDistModule"):
sMod = unprocSw[1]
if unprocSw[0] in ("S", "setup"):
setup = unprocSw[1]
from DIRAC.Core.Utilities.CFG import CFG
localCfg = CFG()
if cFile:
localConfigFile = cFile
else:
def main():
Script.registerSwitch("E:", "email=",
"Boolean True/False (True by default)")
Script.parseCommandLine(ignoreErrors=True)
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC import exit as DIRACExit, gConfig, gLogger
def getBoolean(value):
if value.lower() == 'true':
return True
elif value.lower() == 'false':
return False
else:
Script.showHelp()
email = True
for switch in Script.getUnprocessedSwitches():
if switch[0] == "email":
email = getBoolean(switch[1])
args = Script.getPositionalArgs()
if len(args) < 2:
Script.showHelp()
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
setup = gConfig.getValue('/DIRAC/Setup', '')
if not setup:
print('ERROR: Could not contact Configuration Service')
exitCode = 2
DIRACExit(exitCode)
# result = promptUser(
# 'All the elements that are associated with this site will be active, '
# 'are you sure about this action?'
# )
# if not result['OK'] or result['Value'] is 'n':
# print 'Script stopped'
# DIRACExit( 0 )
site = args[0]
comment = args[1]
result = diracAdmin.allowSite(site, comment, printOutput=True)
if not result['OK']:
errorList.append((site, result['Message']))
exitCode = 2
else:
if email:
userName = diracAdmin._getCurrentUser()
if not userName['OK']:
print('ERROR: Could not obtain current username from proxy')
exitCode = 2
DIRACExit(exitCode)
userName = userName['Value']
subject = '%s is added in site mask for %s setup' % (site, setup)
body = 'Site %s is added to the site mask for %s setup by %s on %s.\n\n' % (
site, setup, userName, time.asctime())
body += 'Comment:\n%s' % comment
addressPath = 'EMail/Production'
address = Operations().getValue(addressPath, '')
if not address:
gLogger.notice(
"'%s' not defined in Operations, can not send Mail\n" %
addressPath, body)
else:
result = diracAdmin.sendMail(address, subject, body)
else:
print('Automatic email disabled by flag.')
for error in errorList:
print("ERROR %s: %s" % error)
DIRACExit(exitCode)
def main():
Script.registerSwitch("D:", "Dir=", "Store the output in this directory")
Script.registerSwitch("f:", "File=", "Get output for jobs with IDs from the file")
Script.registerSwitch("g:", "JobGroup=", "Get output for jobs in the given group")
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
from DIRAC.Core.Utilities.Time import toString, date, day
from DIRAC.Core.Utilities.File import mkDir
dirac = Dirac()
exitCode = 0
errorList = []
outputDir = None
group = None
jobs = []
for sw, value in Script.getUnprocessedSwitches():
if sw in ('D', 'Dir'):
outputDir = value
elif sw.lower() in ('f', 'file'):
if os.path.exists(value):
jFile = open(value)
jobs += jFile.read().split()
jFile.close()
elif sw.lower() in ('g', 'jobgroup'):
group = value
jobDate = toString(date() - 30 * day)
# Choose jobs in final state, no more than 30 days old
result = dirac.selectJobs(jobGroup=value, date=jobDate, status='Done')
if not result['OK']:
if "No jobs selected" not in result['Message']:
print("Error:", result['Message'])
DIRAC.exit(-1)
else:
jobs += result['Value']
result = dirac.selectJobs(jobGroup=value, date=jobDate, status='Failed')
if not result['OK']:
if "No jobs selected" not in result['Message']:
print("Error:", result['Message'])
DIRAC.exit(-1)
else:
jobs += result['Value']
for arg in parseArguments(args):
if os.path.isdir(arg):
print("Output for job %s already retrieved, remove the output directory to redownload" % arg)
else:
jobs.append(arg)
if not jobs:
print("No jobs selected")
DIRAC.exit(0)
if group:
if outputDir:
outputDir = os.path.join(outputDir, group)
else:
outputDir = group
if outputDir:
mkDir(outputDir)
else:
outputDir = os.getcwd()
jobs = [str(job) for job in jobs]
doneJobs = os.listdir(outputDir)
todoJobs = [job for job in jobs if job not in doneJobs]
for job in todoJobs:
result = dirac.getOutputSandbox(job, outputDir=outputDir)
jobDir = str(job)
if outputDir:
jobDir = os.path.join(outputDir, job)
if result['OK']:
if os.path.exists(jobDir):
print('Job output sandbox retrieved in %s/' % (jobDir))
else:
if os.path.exists('%s' % jobDir):
shutil.rmtree(jobDir)
errorList.append((job, result['Message']))
exitCode = 2
for error in errorList:
print("ERROR %s: %s" % error)
DIRAC.exit(exitCode)
%s <dataset name>
""" % Script.scriptName)
Script.registerSwitch("", "save=", "The directory which save files.")
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if (len(args) != 1):
gLogger.error("Please support the dataset name")
DIRAC.exit(-1)
dataset = args[0]
dir_save = args[0]
for k,v in Script.getUnprocessedSwitches():
if k.lower() in ["save"]:
dir_save = v
gLogger.info("Dataset Name: ", dataset)
gLogger.info("Save in: ", dir_save)
# Get the list of LFNs in one dataset
from DIRAC.Core.DISET.RPCClient import RPCClient
transferRequest = RPCClient("Transfer/Dataset")
res = transferRequest.list(dataset)
if not res["OK"]:
gLogger.error(res)
DIRAC.exit(-1)
Script.registerSwitch("w:", "Wildcard=",
"Wildcard for matching filenames [%s]" % wildcard)
Script.registerSwitch(
"b:", "BaseDir=",
"Base directory to begin search (default /[vo]/user/[initial]/[username])")
Script.registerSwitch("e", "EmptyDirs", "Create a list of empty directories")
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1],
'Usage:',
' %s [option|cfgfile] ...' % Script.scriptName,
]))
Script.parseCommandLine(ignoreErrors=False)
for switch in Script.getUnprocessedSwitches():
if switch[0] == "D" or switch[0].lower() == "days":
days = int(switch[1])
if switch[0] == "M" or switch[0].lower() == "months":
months = int(switch[1])
if switch[0] == "Y" or switch[0].lower() == "years":
years = int(switch[1])
if switch[0].lower() == "w" or switch[0].lower() == "wildcard":
wildcard = switch[1]
if switch[0].lower() == "b" or switch[0].lower() == "basedir":
baseDir = switch[1]
if switch[0].lower() == "e" or switch[0].lower() == "emptydirs":
emptyDirsFlag = True
import DIRAC
from DIRAC import gLogger
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("T:", "template=", "Corsika Template")
Script.registerSwitch("p:", "run_number=",
"Do not use: Run Number automatically set")
Script.registerSwitch("E:", "executable=",
"Executable (Use SetExecutable)")
Script.registerSwitch("v:", "version=", "Version (Use setVersion)")
Script.registerSwitch("D:", "dcta=", "dcta")
Script.registerSwitch("I:", "icta=", "icta")
Script.registerSwitch("C:", "c_cta=", "c_cta")
Script.parseCommandLine(ignoreErrors=False)
## default values ##############
run_number = None
template = None
executable = None
version = None
### set switch values ###
for switch in Script.getUnprocessedSwitches():
if switch[0] == "run_number" or switch[0] == "p":
run_number = switch[1].split('ParametricParameters=')[1]
elif switch[0] == "template" or switch[0] == "T":
template = switch[1]
elif switch[0] == "executable" or switch[0] == "E":
executable = switch[1]
elif switch[0] == "version" or switch[0] == "v":
version = switch[1]
if version == None or executable == None or run_number == None or template == None:
Script.showHelp()
jobReport.setApplicationStatus('Missing options')
DIRAC.exit(-1)
from CTADIRAC.Core.Workflow.Modules.CorsikaApp import CorsikaApp
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
packs = [CorsikaSimtelPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
installSoftwareEnviron(package, workingArea())
packageTuple = package.split('/')
corsika_subdir = sharedArea(
) + '/' + packageTuple[0] + '/' + version
cmd = 'cp -u -r ' + corsika_subdir + '/* .'
os.system(cmd)
continue
if workingArea:
if checkSoftwarePackage(package, workingArea())['OK']:
DIRAC.gLogger.notice('Package found in Local Area:', package)
continue
if installSoftwarePackage(package, workingArea())['OK']:
############## compile #############################
cmdTuple = ['./build_all', 'prod2', 'qgs2']
######### special case for Astri ############################
if version == 'prod-2_08072014_to':
############## compile #############################
fd = open('run_compile.sh', 'w')
fd.write("""#! /bin/sh
source ./build_all prod2 qgs2
#
echo " Let's check that build_all did its work "
ls -alFsh
echo "+++++++++++++++++++++++++++++++++++++++++++++++++"
echo " Let's see what files are in the corsika-run directory "
ls -alFsh ./corsika-run
#
if [ ! -x ./corsika-run/corsika ]
then
echo " ERROR: Corsika executable found. Exit "
exit 1
fi
echo "+++++++++++++++++++++++++++++++++++++++++++++++++"
#
echo " Now let's try to compile hessio according to Federico's recipe "
cd ./hessioxxx
make clean
make EXTRA_DEFINES="-DCTA_PROD2 -DWITH_LOW_GAIN_CHANNEL"
#
echo " Let's see what files are in the lib directory "
ls -alFsh ./lib
#
if [ ! -f ./lib/libhessio.so ]
then
echo " ERROR: libhessio library not found. Exit "
exit 1
fi
echo "+++++++++++++++++++++++++++++++++++++++++++++++++"
#
cd .. # come back to original dir
#
echo " Now let's try to compile simtel according to Federico's recipe "
cd ./sim_telarray
make clean
make EXTRA_DEFINES="-DCTA_PROD2 -DWITH_LOW_GAIN_CHANNEL"
make install
#
echo " Let's see what files are in the bin directory "
ls -alFsh ./bin
#
if [ ! -x ./bin/sim_telarray ]
then
echo " ERROR: sim_telarray excutable not found. Exit "
exit 1
fi
echo "+++++++++++++++++++++++++++++++++++++++++++++++++"
#
echo " Everything was compiled and linked properly" """)
fd.close()
os.system('chmod u+x run_compile.sh')
cmdTuple = ['./run_compile.sh']
##########################################################################
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute build')
DIRAC.exit(-1)
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
###### execute corsika ###############
cs = CorsikaApp()
cs.setSoftwarePackage(CorsikaSimtelPack)
cs.csExe = executable
cs.csArguments = ['--run-number', run_number, '--run', 'corsika', template]
corsikaReturnCode = cs.execute()
if corsikaReturnCode != 0:
DIRAC.gLogger.error('Failed to execute corsika Application')
jobReport.setApplicationStatus('Corsika Application: Failed')
DIRAC.exit(-1)
###### rename corsika file #################################
rundir = 'run' + run_number
corsikaKEYWORDS = ['TELFIL']
dictCorsikaKW = fileToKWDict(template, corsikaKEYWORDS)
corsikafilename = rundir + '/' + dictCorsikaKW['TELFIL'][0]
destcorsikafilename = 'corsika_run' + run_number + '.corsika.gz'
cmd = 'mv ' + corsikafilename + ' ' + destcorsikafilename
os.system(cmd)
### create corsika tar ####################
corsika_tar = 'corsika_run' + run_number + '.tar.gz'
filetar1 = rundir + '/' + 'input'
filetar2 = rundir + '/' + 'DAT' + run_number + '.dbase'
filetar3 = rundir + '/run' + str(int(run_number)) + '.log'
cmdTuple = ['/bin/tar', 'zcf', corsika_tar, filetar1, filetar2, filetar3]
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute tar')
DIRAC.exit(-1)
###### execute sim_telarray ###############
ret = getSoftwareEnviron(CorsikaSimtelPack)
if not ret['OK']:
error = ret['Message']
DIRAC.gLogger.error(error, CorsikaSimtelPack)
DIRAC.exit(-1)
corsikaEnviron = ret['Value']
cmdTuple = ['sim_telarray']
# add input file argument for sim_telarray ###################
inputfile = 'input_file=' + destcorsikafilename
inputfileopt = ['-C', inputfile]
cmdTuple.extend(inputfileopt)
# add output file argument for sim_telarray
destsimtelfilename = 'simtel_run' + run_number + '.simtel.gz'
outputfile = 'output_file=' + destsimtelfilename
outputfileopt = ['-C', outputfile]
cmdTuple.extend(outputfileopt)
# add histo argument for sim_telarray
desthistofilename = 'simtel_run' + run_number + '.hdata.gz'
histofile = 'histogram_file=' + desthistofilename
histofileopt = ['-C', histofile]
cmdTuple.extend(histofileopt)
# add other arguments for sim_telarray specified by user ######
simtelparfile = open('simtel.par', 'r').readlines()
for line in simtelparfile:
for word in line.split():
cmdTuple.append(word)
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendSimtelOutput, env=corsikaEnviron)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute:', simexe)
DIRAC.exit(-1)
DIRAC.exit()
if moduleInstance.parameters.find(pName):
moduleInstance.parameters.setValue(pName, pValue)
return workflow.execute()
positionalArgs = Script.getPositionalArgs()
if len(positionalArgs) != 1:
gLogger.debug('Positional arguments were %s' % (positionalArgs))
DIRAC.abort(1, "Must specify the Job XML file description")
if 'JOBID' in os.environ:
gLogger.info('JobID: %s' % (os.environ['JOBID']))
jobXMLfile = positionalArgs[0]
parList = Script.getUnprocessedSwitches()
parDict = {}
for switch, parameter in parList:
if switch == "p":
name, value = parameter.split('=')
value = value.strip()
# The comma separated list in curly brackets is interpreted as a list
if value.startswith("{"):
value = value[1:-1].replace('"', '').replace(" ", '').split(',')
value = ';'.join(value)
parDict[name] = value
gLogger.debug('PYTHONPATH:\n%s' % ('\n'.join(sys.path)))
jobExec = jobexec(jobXMLfile, parDict)
Script.registerSwitch("f:", "File=",
"Get output for jobs with IDs from the file")
Script.registerSwitch("g:", "JobGroup=",
"Get output for jobs in the given group")
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if __name__ == "__main__":
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
from DIRAC.Core.Utilities.Time import toString, date, day
dirac = Dirac()
jobs = []
for sw, value in Script.getUnprocessedSwitches():
if sw.lower() in ('f', 'file'):
if os.path.exists(value):
jFile = open(value)
jobs += jFile.read().split()
jFile.close()
elif sw.lower() in ('g', 'jobgroup'):
group = value
jobDate = toString(date() - 30 * day)
result = dirac.selectJobs(jobGroup=value, date=jobDate)
if not result['OK']:
if "No jobs selected" not in result['Message']:
print("Error:", result['Message'])
DIRAC.exit(-1)
else:
jobs += result['Value']
"""
Obtain replica information from file catalogue client.
"""
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Base import Script
Script.registerSwitch('a', "All", " Also show inactive replicas")
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... LFN ...' % Script.scriptName, 'Arguments:',
' LFN: Logical File Name or file containing LFNs'
]))
Script.parseCommandLine(ignoreErrors=True)
lfns = Script.getPositionalArgs()
switches = Script.getUnprocessedSwitches()
active = True
for switch in switches:
opt = switch[0].lower()
if opt in ("a", "all"):
active = False
if len(lfns) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
if len(lfns) == 1:
try:
def main():
read = False
write = False
check = False
remove = False
site = ''
mute = False
Script.registerSwitch("r", "AllowRead",
" Allow only reading from the storage element")
Script.registerSwitch("w", "AllowWrite",
" Allow only writing to the storage element")
Script.registerSwitch(
"k", "AllowCheck",
" Allow only check access to the storage element")
Script.registerSwitch(
"v", "AllowRemove",
" Allow only remove access to the storage element")
Script.registerSwitch("a", "All",
" Allow all access to the storage element")
Script.registerSwitch("m", "Mute", " Do not send email")
Script.registerSwitch("S:", "Site=",
" Allow all SEs associated to site")
Script.parseCommandLine(ignoreErrors=True)
ses = Script.getPositionalArgs()
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() in ("r", "allowread"):
read = True
if switch[0].lower() in ("w", "allowwrite"):
write = True
if switch[0].lower() in ("k", "allowcheck"):
check = True
if switch[0].lower() in ("v", "allowremove"):
remove = True
if switch[0].lower() in ("a", "all"):
read = True
write = True
check = True
remove = True
if switch[0].lower() in ("m", "mute"):
mute = True
if switch[0].lower() in ("s", "site"):
site = switch[1]
# imports
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
if not (read or write or check or remove):
# No switch was specified, means we need all of them
gLogger.notice(
"No option given, all accesses will be allowed if they were not")
read = True
write = True
check = True
remove = True
ses = resolveSEGroup(ses)
diracAdmin = DiracAdmin()
errorList = []
setup = gConfig.getValue('/DIRAC/Setup', '')
if not setup:
print('ERROR: Could not contact Configuration Service')
DIRAC.exit(2)
res = getProxyInfo()
if not res['OK']:
gLogger.error('Failed to get proxy information', res['Message'])
DIRAC.exit(2)
userName = res['Value'].get('username')
if not userName:
gLogger.error('Failed to get username for proxy')
DIRAC.exit(2)
if site:
res = getSites()
if not res['OK']:
gLogger.error(res['Message'])
DIRAC.exit(-1)
if site not in res['Value']:
gLogger.error('The provided site (%s) is not known.' % site)
DIRAC.exit(-1)
ses.extend(res['Value']['SE'].replace(' ', '').split(','))
if not ses:
gLogger.error('There were no SEs provided')
DIRAC.exit()
STATUS_TYPES = ["ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess"]
ALLOWED_STATUSES = ["Unknown", "InActive", "Banned", "Probing", "Degraded"]
statusAllowedDict = {}
for statusType in STATUS_TYPES:
statusAllowedDict[statusType] = []
statusFlagDict = {}
statusFlagDict['ReadAccess'] = read
statusFlagDict['WriteAccess'] = write
statusFlagDict['CheckAccess'] = check
statusFlagDict['RemoveAccess'] = remove
resourceStatus = ResourceStatus()
res = resourceStatus.getElementStatus(ses, "StorageElement")
if not res['OK']:
gLogger.error('Storage Element %s does not exist' % ses)
DIRAC.exit(-1)
reason = 'Forced with dirac-admin-allow-se by %s' % userName
for se, seOptions in res['Value'].items():
# InActive is used on the CS model, Banned is the equivalent in RSS
for statusType in STATUS_TYPES:
if statusFlagDict[statusType]:
if seOptions.get(statusType) == "Active":
gLogger.notice('%s status of %s is already Active' %
(statusType, se))
continue
if statusType in seOptions:
if not seOptions[statusType] in ALLOWED_STATUSES:
gLogger.notice(
'%s option for %s is %s, instead of %s' %
(statusType, se, seOptions['ReadAccess'],
ALLOWED_STATUSES))
gLogger.notice('Try specifying the command switches')
else:
resR = resourceStatus.setElementStatus(
se, "StorageElement", statusType, 'Active', reason,
userName)
if not resR['OK']:
gLogger.fatal(
"Failed to update %s %s to Active, exit -" %
(se, statusType), resR['Message'])
DIRAC.exit(-1)
else:
gLogger.notice(
"Successfully updated %s %s to Active" %
(se, statusType))
statusAllowedDict[statusType].append(se)
totalAllowed = 0
totalAllowedSEs = []
for statusType in STATUS_TYPES:
totalAllowed += len(statusAllowedDict[statusType])
totalAllowedSEs += statusAllowedDict[statusType]
totalAllowedSEs = list(set(totalAllowedSEs))
if not totalAllowed:
gLogger.info("No storage elements were allowed")
DIRAC.exit(-1)
if mute:
gLogger.notice('Email is muted by script switch')
DIRAC.exit(0)
subject = '%s storage elements allowed for use' % len(totalAllowedSEs)
addressPath = 'EMail/Production'
address = Operations().getValue(addressPath, '')
body = ''
if read:
body = "%s\n\nThe following storage elements were allowed for reading:" % body
for se in statusAllowedDict['ReadAccess']:
body = "%s\n%s" % (body, se)
if write:
body = "%s\n\nThe following storage elements were allowed for writing:" % body
for se in statusAllowedDict['WriteAccess']:
body = "%s\n%s" % (body, se)
if check:
body = "%s\n\nThe following storage elements were allowed for checking:" % body
for se in statusAllowedDict['CheckAccess']:
body = "%s\n%s" % (body, se)
if remove:
body = "%s\n\nThe following storage elements were allowed for removing:" % body
for se in statusAllowedDict['RemoveAccess']:
body = "%s\n%s" % (body, se)
if not address:
gLogger.notice(
"'%s' not defined in Operations, can not send Mail\n" %
addressPath, body)
DIRAC.exit(0)
res = diracAdmin.sendMail(address, subject, body)
gLogger.notice('Notifying %s' % address)
if res['OK']:
gLogger.notice(res['Value'])
else:
gLogger.notice(res['Message'])
DIRAC.exit(0)
