def addUserName( arg ):
global userNames
if not arg:
Script.showHelp()
DIRAC.exit( -1 )
if not arg in userNames:
userNames.append( arg )
def showHelp( self, dummy = False ):
"""
Printout help message including a Usage message if defined via setUsageMessage method
"""
if self.__usageMessage:
gLogger.notice( self.__usageMessage )
else:
gLogger.notice( "Usage:" )
gLogger.notice( " %s (<options>|<cfgFile>)*" % os.path.basename( sys.argv[0] ) )
if dummy:
gLogger.notice( dummy )
gLogger.notice( "General options:" )
iLastOpt = 0
for iPos in range( len( self.commandOptionList ) ):
optionTuple = self.commandOptionList[ iPos ]
gLogger.notice( " %s --%s : %s" % ( ( "-" + optionTuple[0].ljust( 3 ) if optionTuple[0] else " " * 4 ),
optionTuple[1].ljust( 15 ),
optionTuple[2] ) )
iLastOpt = iPos
if optionTuple[0] == 'h':
#Last general opt is always help
break
if iLastOpt + 1 < len( self.commandOptionList ):
gLogger.notice( " \nOptions:" )
for iPos in range( iLastOpt + 1, len( self.commandOptionList ) ):
optionTuple = self.commandOptionList[ iPos ]
if optionTuple[0]:
gLogger.notice( " -%s --%s : %s" % ( optionTuple[0].ljust( 3 ), optionTuple[1].ljust( 15 ), optionTuple[2] ) )
else:
gLogger.notice( " %s --%s : %s" % ( ' ', optionTuple[1].ljust( 15 ), optionTuple[2] ) )
DIRAC.exit( 0 )
def addProperty( arg ):
global groupProperties
if not arg:
Script.showHelp()
DIRAC.exit( -1 )
if not arg in groupProperties:
groupProperties.append( arg )
def addProperty( arg ):
global hostProperties
if not arg:
Script.showHelp()
DIRAC.exit( -1 )
if not arg in hostProperties:
hostProperties.append( arg )
def getInfo(params):
'''
Retrieve information from BDII
'''
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
if params['info'] == 'ce':
result = diracAdmin.getBDIICE(params['ce'], host=params['host'])
if params['info'] == 'ce-state':
result = diracAdmin.getBDIICEState(params['ce'], useVO=params['vo'], host=params['host'])
if params['info'] == 'ce-cluster':
result = diracAdmin.getBDIICluster(params['ce'], host=params['host'])
if params['info'] == 'ce-vo':
result = diracAdmin.getBDIICEVOView(params['ce'], useVO=params['vo'], host=params['host'])
if params['info'] == 'site':
result = diracAdmin.getBDIISite(params['site'], host=params['host'])
if params['info'] == 'site-se':
result = diracAdmin.getBDIISE(params['site'], useVO=params['vo'], host=params['host'])
if not result['OK']:
print result['Message']
DIRAC.exit(2)
return result
def run( parameters , delete, nthreads ):
"""
The main user interface
"""
source_dir = parameters[0]
dest_dir = parameters[1]
upload = False
storage = None
if len( parameters ) == 3:
storage = parameters[2]
source_dir = os.path.abspath(source_dir)
dest_dir = dest_dir.rstrip('/')
upload = True
if not os.path.isdir(source_dir):
gLogger.fatal("Source directory does not exist")
DIRAC.exit( 1 )
if len (parameters ) == 2:
dest_dir = os.path.abspath(dest_dir)
source_dir = source_dir.rstrip('/')
if not os.path.isdir(dest_dir):
gLogger.fatal("Destination directory does not exist")
DIRAC.exit( 1 )
res = syncDestinations( upload, source_dir, dest_dir, storage, delete, nthreads )
if not res['OK']:
return S_ERROR(res['Message'])
return S_OK("Successfully mirrored " + source_dir + " into " + dest_dir)
def addUserGroup(arg):
global userGroups
if not arg:
Script.showHelp()
DIRAC.exit(-1)
if not arg in userGroups:
userGroups.append(arg)
def setUserMail(arg):
global userMail
if userMail or not arg:
Script.showHelp()
DIRAC.exit(-1)
if not arg.find("@") > 0:
Script.gLogger.error("Not a valid mail address", arg)
DIRAC.exit(-1)
userMail = arg
def showLicense( self, dummy = False ):
"""
Print license
"""
lpath = os.path.join( DIRAC.rootPath, "DIRAC", "LICENSE" )
sys.stdout.write( " - DIRAC is GPLv3 licensed\n\n" )
try:
with open( lpath ) as fd:
sys.stdout.write( fd.read() )
except IOError:
sys.stdout.write( "Can't find GPLv3 license at %s. Somebody stole it!\n" % lpath )
sys.stdout.write( "Please check out http://www.gnu.org/licenses/gpl-3.0.html for more info\n" )
DIRAC.exit(0)
def uploadListOfFiles(dm, source_dir, dest_dir, storage, listOfFiles, tID):
"""
Wrapper for multithreaded uploading of a list of files
"""
log = gLogger.getSubLogger("[Thread %s] " % tID)
threadLine = "[Thread %s]" % tID
for filename in listOfFiles:
res = uploadLocalFile(dm, dest_dir+"/"+filename, source_dir+"/"+filename, storage)
if not res['OK']:
log.fatal(threadLine + ' Uploading ' + filename + ' -X- [FAILED] ' + res['Message'])
DIRAC.exit( 1 )
else:
log.notice(threadLine+ " Uploading " + filename + " -> [DONE]")
def downloadListOfFiles(dm, source_dir, dest_dir, listOfFiles, tID):
"""
Wrapper for multithreaded downloading of a list of files
"""
log = gLogger.getSubLogger("[Thread %s] " % tID)
threadLine = "[Thread %s]" % tID
for filename in listOfFiles:
res = downloadRemoteFile(dm, source_dir + "/" + filename, dest_dir + ("/" + filename).rsplit("/", 1)[0])
if not res['OK']:
log.fatal(threadLine + ' Downloading ' + filename + ' -X- [FAILED] ' + res['Message'])
DIRAC.exit( 1 )
else:
log.notice(threadLine+ " Downloading " + filename + " -> [DONE]")
def _getTSFiles(self):
""" Helper function - get files from the TS
"""
selectDict = {'TransformationID': self.prod}
if self._lfns:
selectDict['LFN'] = self._lfns
elif self.runStatus and self.fromProd:
res = self.transClient.getTransformationRuns(
{'TransformationID': self.fromProd, 'Status': self.runStatus})
if not res['OK']:
gLogger.error("Failed to get runs for transformation %d" % self.prod)
else:
if res['Value']:
self.runsList.extend(
[run['RunNumber'] for run in res['Value'] if run['RunNumber'] not in self.runsList])
gLogger.notice("%d runs selected" % len(res['Value']))
elif not self.runsList:
gLogger.notice("No runs selected, check completed")
DIRAC.exit(0)
if not self._lfns and self.runsList:
selectDict['RunNumber'] = self.runsList
res = self.transClient.getTransformation(self.prod)
if not res['OK']:
gLogger.error("Failed to find transformation %s" % self.prod)
return [], [], []
status = res['Value']['Status']
if status not in ('Active', 'Stopped', 'Completed', 'Idle'):
gLogger.notice("Transformation %s in status %s, will not check if files are processed" % (
self.prod, status))
processedLFNs = []
nonProcessedLFNs = []
nonProcessedStatuses = []
if self._lfns:
processedLFNs = self._lfns
else:
res = self.transClient.getTransformationFiles(selectDict)
if not res['OK']:
gLogger.error("Failed to get files for transformation %d" %
self.prod, res['Message'])
return [], [], []
else:
processedLFNs = [item['LFN']
for item in res['Value'] if item['Status'] == 'Processed']
nonProcessedLFNs = [item['LFN']
for item in res['Value'] if item['Status'] != 'Processed']
nonProcessedStatuses = list(
set(item['Status'] for item in res['Value'] if item['Status'] != 'Processed'))
return processedLFNs, nonProcessedLFNs, nonProcessedStatuses
def run( self, lfn, metas ):
retVal = self.catalog.getMeta( lfn )
if not retVal[ "OK" ]:
print "Error:", retVal[ "Message" ]
DIRAC.exit( -1 )
metadict = retVal[ "Value" ]
if not metas:
for k, v in metadict.items( ):
print k+"="+str( v )
else:
for meta in metas:
if meta in metadict.keys( ):
print meta+"="+metadict[ meta ]
def checkFunction():
""" gets CPU normalisation from MFJ or calculate itself """
from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import getPowerFromMJF
from ILCDIRAC.Core.Utilities.CPUNormalization import getCPUNormalization
from DIRAC import gLogger, gConfig
result = getCPUNormalization()
if not result['OK']:
gLogger.error( result['Message'] )
norm = round( result['Value']['NORM'], 1 )
gLogger.notice( 'Estimated CPU power is %.1f %s' % ( norm, result['Value']['UNIT'] ) )
mjfPower = getPowerFromMJF()
if mjfPower:
gLogger.notice( 'CPU power from MJF is %.1f HS06' % mjfPower )
else:
gLogger.notice( 'MJF not available on this node' )
if update and not configFile:
gConfig.setOptionValue( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm )
gConfig.setOptionValue( '/LocalSite/CPUNormalizationFactor', norm )
gConfig.dumpLocalCFGToFile( gConfig.diracConfigFilePath )
if configFile:
from DIRAC.Core.Utilities.CFG import CFG
cfg = CFG()
try:
# Attempt to open the given file
cfg.loadFromFile( configFile )
except:
pass
# Create the section if it does not exist
if not cfg.existsKey( 'LocalSite' ):
cfg.createNewSection( 'LocalSite' )
cfg.setOption( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm )
cfg.setOption( '/LocalSite/CPUNormalizationFactor', norm )
cfg.writeToFile( configFile )
DIRAC.exit()
def getSEInfo(lfns) :
result = fileCatalog.getReplicas( lfns )
if not result['OK']:
print 'ERROR: ', result['Message']
exitCode = 2
DIRAC.exit( exitCode )
replicas=result['Value']['Successful'][lfns]
nreplicas = len(replicas)
ralsrm=''
selist=''
for se in replicas.keys() :
selist=selist+se+','
# if se == target :
# ralsrm=replicas[se]
print str(nreplicas)+' '+lfns+' '+selist[:-1]
def export_insertOpBD( self ):
print "server, insertOpBD"
db = OperationFileDB()
res = db.createTables()
if not res["OK"]:
gLogger.error( ' error' )
DIRAC.exit( -1 )
gLogger.verbose( 'ok creation table' )
args = dict()
args['LFN'] = "testLFN"
args['SESource'] = "testSESource"
args['SEDestination'] = "testSEDestination"
args['blob'] = "testBlob"
operationFile = OperationFile( args )
res = db.putOperationFile( operationFile )
return res
def showHelp( self, dummy = False, exitCode = 0 ):
"""
Printout help message including a Usage message if defined via setUsageMessage method
"""
if self.__usageMessage:
gLogger.notice( '\n'+self.__usageMessage.lstrip() )
else:
gLogger.notice( "\nUsage:" )
gLogger.notice( "\n %s (<options>|<cfgFile>)*" % os.path.basename( sys.argv[0] ) )
if dummy:
gLogger.notice( dummy )
gLogger.notice( "\nGeneral options:" )
iLastOpt = 0
for iPos in range( len( self.commandOptionList ) ):
optionTuple = self.commandOptionList[ iPos ]
if optionTuple[0].endswith( ':' ):
line = " -%s --%s : %s" % ( optionTuple[0][:-1].ljust( 2 ),
(optionTuple[1][:-1] + ' <value> ').ljust( 22 ),
optionTuple[2] )
gLogger.notice( line )
else:
gLogger.notice( " -%s --%s : %s" % ( optionTuple[0].ljust( 2 ), optionTuple[1].ljust( 22 ), optionTuple[2] ) )
iLastOpt = iPos
if optionTuple[0] == 'h':
#Last general opt is always help
break
if iLastOpt + 1 < len( self.commandOptionList ):
gLogger.notice( " \nOptions:" )
for iPos in range( iLastOpt + 1, len( self.commandOptionList ) ):
optionTuple = self.commandOptionList[ iPos ]
if optionTuple[0].endswith( ':' ):
line = " -%s --%s : %s" % ( optionTuple[0][:-1].ljust( 2 ),
(optionTuple[1][:-1] + ' <value> ').ljust( 22 ),
optionTuple[2] )
gLogger.notice( line )
else:
gLogger.notice( " -%s --%s : %s" % ( optionTuple[0].ljust( 2 ), optionTuple[1].ljust( 22 ), optionTuple[2] ) )
gLogger.notice( "" )
DIRAC.exit( exitCode )
def syncStorageElementsInit():
ses = gConfig.getSections( '/Resources/StorageElements' )
if not ses[ 'OK' ]:
gLogger.error( ses[ 'Message' ] )
DIRAC.exit( 2 )
statuses = rsc.getValidStatuses()[ 'Value' ]
for se in ses[ 'Value' ]:
opts = gConfig.getOptions( '/Resources/StorageElements/%s' % se )[ 'Value' ]
statusTypes = [ 'Read', 'Write', 'Check', 'Remove' ]
for opt in opts:
if not opt.endswith( 'Access' ):
continue
status = gConfig.getValue( '/Resources/StorageElements/%s/%s' % ( se, opt ) )
if status in [ 'NotAllowed', 'InActive' ]:
status = 'Banned'
if not status in statuses:
gLogger.error( '%s not a valid status for %s - %s' % ( status, se, statusType ) )
continue
statusType = opt.replace( 'Access', '' )
statusTypes.remove( statusType )
rsc.modifyElementStatus( 'StorageElement', se, statusType,
status = status, reason = 'Init sync' )
for sType in statusTypes:
# If there is nothing on the CS, we set the statusType to DEFAULT_STATUS
DEFAULT_STATUS = 'Active'
rsc.modifyElementStatus( 'StorageElement', se, sType,
status = DEFAULT_STATUS, reason = 'Default status' )
def doUpload(fc, dm, result, source_dir, dest_dir, storage, delete, nthreads):
"""
Wrapper for uploading files
"""
if delete:
lfns = [dest_dir+"/"+filename for filename in result['Value']['Delete']['Files']]
if len(lfns)>0:
res = removeRemoteFiles(dm,lfns)
if not res['OK']:
gLogger.fatal('Deleting of files: ' + lfns + " -X- [FAILED]" +res['Message'])
DIRAC.exit( 1 )
else:
gLogger.notice("Deleting "+ ', '.join(lfns) + " -> [DONE]")
for directoryname in result['Value']['Delete']['Directories']:
res = removeRemoteDirectory(fc, dest_dir + "/" + directoryname)
if not res['OK']:
gLogger.fatal('Deleting of directory: ' + directoryname + " -X- [FAILED] " + res['Message'])
DIRAC.exit( 1 )
else:
gLogger.notice("Deleting "+ directoryname + " -> [DONE]")
for directoryname in result['Value']['Create']['Directories']:
res = createRemoteDirectory(fc, dest_dir+"/"+ directoryname)
if not res['OK']:
gLogger.fatal('Creation of directory: ' + directoryname + " -X- [FAILED] " + res['Message'])
DIRAC.exit( 1 )
else:
gLogger.notice("Creating " + directoryname + " -> [DONE]")
listOfFiles = result['Value']['Create']['Files']
#Chech that we do not have to many threads
if nthreads > len(listOfFiles):
nthreads = len(listOfFiles)
if nthreads == 0:
return S_OK('Upload finished successfully')
listOfListOfFiles = chunkList( listOfFiles, nthreads )
res = runInParallel(arguments=[dm, source_dir, dest_dir, storage], listOfLists=listOfListOfFiles, function=uploadListOfFiles )
if not res['OK']:
return S_ERROR("Upload of files failed")
return S_OK('Upload finished successfully')
def doDownload(dm, result, source_dir, dest_dir, delete, nthreads):
"""
Wrapper for downloading files
"""
if delete:
for filename in result['Value']['Delete']['Files']:
res = removeLocalFile(dest_dir+"/"+ filename)
if not res['OK']:
gLogger.fatal('Deleting of file: ' + filename + ' -X- [FAILED] ' + res['Message'])
DIRAC.exit( 1 )
else:
gLogger.notice("Deleting "+ filename + " -> [DONE]")
for directoryname in result['Value']['Delete']['Directories']:
res = removeLocaDirectory( dest_dir + "/" + directoryname )
if not res['OK']:
gLogger.fatal('Deleting of directory: ' + directoryname + ' -X- [FAILED] ' + res['Message'])
DIRAC.exit( 1 )
else:
gLogger.notice("Deleting "+ directoryname + " -> [DONE]")
for directoryname in result['Value']['Create']['Directories']:
res = createLocalDirectory( dest_dir+"/"+ directoryname )
if not res['OK']:
gLogger.fatal('Creation of directory: ' + directoryname + ' -X- [FAILED] ' + res['Message'])
DIRAC.exit( 1 )
else:
gLogger.notice("Creating " + directoryname + " -> [DONE]")
listOfFiles = result['Value']['Create']['Files']
#Chech that we do not have to many threads
if nthreads > len(listOfFiles):
nthreads = len(listOfFiles)
if nthreads==0:
return S_OK('Upload finished successfully')
listOfListOfFiles = chunkList( listOfFiles, nthreads )
res = runInParallel(arguments=[dm, source_dir, dest_dir], listOfLists=listOfListOfFiles, function=downloadListOfFiles )
if not res['OK']:
return S_ERROR("Download of files failed")
return S_OK('Upload finished successfully')
def generateProxy(params):
if params.checkClock:
result = getClockDeviation()
if result['OK']:
deviation = result['Value']
if deviation > 600:
print "Error: Your host clock seems to be off by more than TEN MINUTES! Thats really bad."
print "We're cowardly refusing to generate a proxy. Please fix your system time"
DIRAC.exit(1)
elif deviation > 180:
print "Error: Your host clock seems to be off by more than THREE minutes! Thats bad."
print "Warn : We'll generate the proxy but please fix your system time"
elif deviation > 60:
print "Error: Your host clock seems to be off by more than a minute! Thats not good."
print "Warn : We'll generate the proxy but please fix your system time"
certLoc = params.certLoc
keyLoc = params.keyLoc
if not certLoc or not keyLoc:
cakLoc = Locations.getCertificateAndKeyLocation()
if not cakLoc:
return S_ERROR("Can't find user certificate and key")
if not certLoc:
certLoc = cakLoc[0]
if not keyLoc:
keyLoc = cakLoc[1]
testChain = X509Chain()
retVal = testChain.loadKeyFromFile(keyLoc, password=params.userPasswd)
if not retVal['OK']:
passwdPrompt = "Enter Certificate password:"******"\n")
else:
userPasswd = getpass.getpass(passwdPrompt)
params.userPasswd = userPasswd
proxyLoc = params.proxyLoc
if not proxyLoc:
proxyLoc = Locations.getDefaultProxyLocation()
if params.debug:
h = int(params.proxyLifeTime / 3600)
m = int(params.proxyLifeTime / 60) - h * 60
print "Proxy lifetime will be %02d:%02d" % (h, m)
print "User cert is %s" % certLoc
print "User key is %s" % keyLoc
print "Proxy will be written to %s" % proxyLoc
if params.diracGroup:
print "DIRAC Group will be set to %s" % params.diracGroup
else:
print "No DIRAC Group will be set"
print "Proxy strength will be %s" % params.proxyStrength
if params.limitedProxy:
print "Proxy will be limited"
chain = X509Chain()
#Load user cert and key
retVal = chain.loadChainFromFile(certLoc)
if not retVal['OK']:
params.debugMsg("ERROR: %s" % retVal['Message'])
return S_ERROR("Can't load %s" % certLoc)
retVal = chain.loadKeyFromFile(keyLoc, password=params.userPasswd)
if not retVal['OK']:
params.debugMsg("ERROR: %s" % retVal['Message'])
return S_ERROR("Can't load %s" % keyLoc)
if params.checkWithCS and params.diracGroup:
retVal = chain.generateProxyToFile(proxyLoc,
params.proxyLifeTime,
strength=params.proxyStrength,
limited=params.limitedProxy)
params.debugMsg("Contacting CS...")
retVal = Script.enableCS()
if not retVal['OK']:
params.debugMsg("ERROR: %s" % retVal['Message'])
return S_ERROR("Can't contact DIRAC CS: %s" % retVal['Message'])
if not params.diracGroup:
params.diracGroup = CS.getDefaultUserGroup()
userDN = chain.getCertInChain(-1)['Value'].getSubjectDN()['Value']
params.debugMsg("Checking DN %s" % userDN)
retVal = CS.getUsernameForDN(userDN)
if not retVal['OK']:
params.debugMsg("ERROR: %s" % retVal['Message'])
return S_ERROR("DN %s is not registered" % userDN)
username = retVal['Value']
params.debugMsg("Username is %s" % username)
retVal = CS.getGroupsForUser(username)
if not retVal['OK']:
params.debugMsg("ERROR: %s" % retVal['Message'])
return S_ERROR("User %s has no groups defined" % username)
groups = retVal['Value']
if params.diracGroup not in groups:
return S_ERROR("Requested group %s is not valid for user %s" %
(params.diracGroup, username))
params.debugMsg("Creating proxy for %s@%s (%s)" %
(username, params.diracGroup, userDN))
retVal = chain.generateProxyToFile(proxyLoc,
params.proxyLifeTime,
params.diracGroup,
strength=params.proxyStrength,
limited=params.limitedProxy)
if not retVal['OK']:
params.debugMsg("ERROR: %s" % retVal['Message'])
return S_ERROR("Couldn't generate proxy: %s" % retVal['Message'])
return S_OK(proxyLoc)
import os
import DIRAC
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient
stageLfns = {}
if os.path.exists( fileName ):
try:
lfnFile = open( fileName )
lfns = [ k.strip() for k in lfnFile.readlines() ]
lfnFile.close()
except Exception:
DIRAC.gLogger.exception( 'Can not open file', fileName )
DIRAC.exit( -1 )
else:
lfns = args[1:]
stageLfns[seName] = lfns
stagerClient = StorageManagerClient()
res = stagerClient.setRequest( stageLfns, 'WorkloadManagement',
'updateJobFromStager@WorkloadManagement/JobStateUpdate',
0 ) # fake JobID = 0
if not res['OK']:
DIRAC.gLogger.error( res['Message'] )
DIRAC.exit( -1 )
else:
print "Stage request submitted for LFNs:\n %s" %lfns
print "SE= %s" %seName
def main():
"""
Main executive code
"""
Script.registerSwitch("", "Job=", " JobID[,jobID2,...]")
Script.registerSwitch("", "Transformation=", " transformation ID")
Script.registerSwitch("", "Tasks=", " Associated to --Transformation, list of taskIDs")
Script.registerSwitch("", "Verbose", " Print more information")
Script.registerSwitch("", "Terse", " Only print request status")
Script.registerSwitch("", "Full", " Print full request content")
Script.registerSwitch("", "Status=", " Select all requests in a given status")
Script.registerSwitch(
"", "Since=", " Associated to --Status, start date yyyy-mm-dd or nb of days (default= -one day"
)
Script.registerSwitch("", "Until=", " Associated to --Status, end date (default= now")
Script.registerSwitch("", "Maximum=", " Associated to --Status, max number of requests ")
Script.registerSwitch("", "Reset", " Reset Failed files to Waiting if any")
Script.registerSwitch("", "Force", " Force reset even if not Failed")
Script.registerSwitch(
"", "All", " (if --Status Failed) all requests, otherwise exclude irrecoverable failures"
)
Script.registerSwitch("", "FixJob", " Set job Done if the request is Done")
Script.registerSwitch("", "Cancel", " Cancel the request")
Script.registerSwitch("", "ListJobs", " List the corresponding jobs")
Script.registerSwitch("", "TargetSE=", " Select request only if that SE is in the targetSEs")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
(
"file: a file containing a list of requests (Comma-separated on each line)",
"request: a request ID or a unique request name",
),
mandatory=False,
)
Script.registerArgument(["request: a request ID or a unique request name"], mandatory=False)
Script.parseCommandLine()
import DIRAC
from DIRAC import gLogger
jobs = []
requestID = 0
transID = None
taskIDs = None
tasks = None
requests = []
full = False
verbose = False
status = None
until = None
since = None
terse = False
allR = False
reset = False
fixJob = False
maxRequests = 999999999999
cancel = False
listJobs = False
force = False
targetSE = set()
for switch in Script.getUnprocessedSwitches():
if switch[0] == "Job":
jobs = []
job = "Unknown"
try:
for arg in switch[1].split(","):
if os.path.exists(arg):
with open(arg, "r") as fp:
lines = fp.readlines()
for line in lines:
for job in line.split(","):
jobs += [int(job.strip())]
gLogger.notice("Found %d jobs in file %s" % (len(jobs), arg))
else:
jobs.append(int(arg))
except TypeError:
gLogger.fatal("Invalid jobID", job)
elif switch[0] == "Transformation":
try:
transID = int(switch[1])
except Exception:
gLogger.fatal("Invalid transID", switch[1])
elif switch[0] == "Tasks":
try:
taskIDs = [int(task) for task in switch[1].split(",")]
except Exception:
gLogger.fatal("Invalid tasks", switch[1])
elif switch[0] == "Full":
full = True
elif switch[0] == "Verbose":
verbose = True
elif switch[0] == "Terse":
terse = True
elif switch[0] == "All":
allR = True
elif switch[0] == "Reset":
reset = True
elif switch[0] == "Force":
force = True
elif switch[0] == "Status":
status = switch[1].capitalize()
elif switch[0] == "Since":
since = convertDate(switch[1])
elif switch[0] == "Until":
until = convertDate(switch[1])
elif switch[0] == "FixJob":
fixJob = True
elif switch[0] == "Cancel":
cancel = True
elif switch[0] == "ListJobs":
listJobs = True
elif switch[0] == "Maximum":
try:
maxRequests = int(switch[1])
except Exception:
pass
elif switch[0] == "TargetSE":
targetSE = set(switch[1].split(","))
if reset and not force:
status = "Failed"
if fixJob:
status = "Done"
if terse:
verbose = True
if status:
if not until:
until = datetime.datetime.utcnow()
if not since:
since = until - datetime.timedelta(hours=24)
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.RequestManagementSystem.Client.ReqClient import printRequest, recoverableRequest
reqClient = ReqClient()
if transID:
if not taskIDs:
gLogger.fatal("If Transformation is set, a list of Tasks should also be set")
Script.showHelp(exitCode=2)
# In principle, the task name is unique, so the request name should be unique as well
# If ever this would not work anymore, we would need to use the transformationClient
# to fetch the ExternalID
requests = ["%08d_%08d" % (transID, task) for task in taskIDs]
allR = True
elif not jobs:
requests = []
# Get full list of arguments, with and without comma
for arg in [x.strip() for ar in Script.getPositionalArgs() for x in ar.split(",")]:
if os.path.exists(arg):
lines = open(arg, "r").readlines()
requests += [reqID.strip() for line in lines for reqID in line.split(",")]
gLogger.notice("Found %d requests in file" % len(requests))
else:
requests.append(arg)
allR = True
else:
res = reqClient.getRequestIDsForJobs(jobs)
if not res["OK"]:
gLogger.fatal("Error getting request for jobs", res["Message"])
DIRAC.exit(2)
if res["Value"]["Failed"]:
gLogger.error("No request found for jobs %s" % ",".join(sorted(str(job) for job in res["Value"]["Failed"])))
requests = sorted(res["Value"]["Successful"].values())
if requests:
allR = True
else:
DIRAC.exit(0)
if status and not requests:
allR = allR or status != "Failed"
res = reqClient.getRequestIDsList([status], limit=maxRequests, since=since, until=until)
if not res["OK"]:
gLogger.error("Error getting requests:", res["Message"])
DIRAC.exit(2)
requests = [reqID for reqID, _st, updTime in res["Value"] if updTime > since and updTime <= until and reqID]
gLogger.notice("Obtained %d requests %s between %s and %s" % (len(requests), status, since, until))
if not requests:
gLogger.notice("No request selected....")
Script.showHelp(exitCode=2)
okRequests = []
jobIDList = []
for reqID in requests:
# We allow reqID to be the requestName if it is unique
try:
# PEP-515 allows for underscore in numerical literals
# So a request name 00123_00456
# is interpreted as a requestID 12300456
# Using an exception here for non-string is not an option
if isinstance(reqID, str) and not reqID.isdigit():
raise ValueError()
requestID = int(reqID)
except (ValueError, TypeError):
requestID = reqClient.getRequestIDForName(reqID)
if not requestID["OK"]:
gLogger.notice(requestID["Message"])
continue
requestID = requestID["Value"]
request = reqClient.peekRequest(requestID)
if not request["OK"]:
gLogger.error(request["Message"])
DIRAC.exit(-1)
request = request["Value"]
if not request:
gLogger.error("no such request %s" % requestID)
continue
# If no operation as the targetSE, skip
if targetSE:
found = False
for op in request:
if op.TargetSE and targetSE.intersection(op.TargetSE.split(",")):
found = True
break
if not found:
continue
# keep a list of jobIDs if requested
if request.JobID and listJobs:
jobIDList.append(request.JobID)
if status and request.Status != status:
gLogger.notice(
"Request %s is not in requested status %s%s" % (reqID, status, " (cannot be reset)" if reset else "")
)
continue
if fixJob and request.Status == "Done" and request.JobID:
# The request is for a job and is Done, verify that the job is in the proper status
result = reqClient.finalizeRequest(request.RequestID, request.JobID, useCertificates=False)
if not result["OK"]:
gLogger.error("Error finalizing job", result["Message"])
else:
gLogger.notice("Job %d updated to %s" % (request.JobID, result["Value"]))
continue
if cancel:
if request.Status not in ("Done", "Failed"):
ret = reqClient.cancelRequest(requestID)
if not ret["OK"]:
gLogger.error("Error canceling request %s" % reqID, ret["Message"])
else:
gLogger.notice("Request %s cancelled" % reqID)
else:
gLogger.notice("Request %s is in status %s, not cancelled" % (reqID, request.Status))
elif allR or recoverableRequest(request):
okRequests.append(str(requestID))
if reset:
gLogger.notice("============ Request %s =============" % requestID)
ret = reqClient.resetFailedRequest(requestID, allR=allR)
if not ret["OK"]:
gLogger.error("Error resetting request %s" % requestID, ret["Message"])
else:
if len(requests) > 1:
gLogger.notice("\n===================================")
dbStatus = reqClient.getRequestStatus(requestID).get("Value", "Unknown")
printRequest(request, status=dbStatus, full=full, verbose=verbose, terse=terse)
if listJobs:
gLogger.notice("List of %d jobs:\n" % len(jobIDList), ",".join(str(jobID) for jobID in jobIDList))
if status and okRequests:
from DIRAC.Core.Utilities.List import breakListIntoChunks
gLogger.notice("\nList of %d selected requests:" % len(okRequests))
for reqs in breakListIntoChunks(okRequests, 100):
gLogger.notice(",".join(reqs))
def main():
from DIRAC.Core.Base import Script
### DoCtaIrf options ##########################################################
Script.registerSwitch("A:", "analysis=", "Analysis Type", setAnalysisType)
Script.registerSwitch("C:", "cuts=", "Cuts Config", setCutsConfig)
Script.registerSwitch("R:", "runlist=", "Runlist", setRunlist)
Script.registerSwitch("Z:", "zenith=", "Zenith", setZenith)
Script.registerSwitch("O:", "offset=", "Offset", setOffset)
Script.registerSwitch("M:", "energy=", "Energy Method", setEnergyMethod)
Script.registerSwitch("T:", "arrayconfig=", "Array Configuration",
setArrayConfig)
Script.registerSwitch("P:", "particle=", "Particle Type", setParticleType)
## other options
Script.registerSwitch("V:", "version=", "HAP version", setVersion)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
from CTADIRAC.Core.Workflow.Modules.HapApplication import HapApplication
from CTADIRAC.Core.Workflow.Modules.HapRootMacro import HapRootMacro
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
ha = HapApplication()
HapPack = 'HAP/' + version + '/HAP'
packs = ['HESS/v0.2/lib', 'HESS/v0.3/root', HapPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
continue
if localArea:
if checkSoftwarePackage(package, localArea())['OK']:
DIRAC.gLogger.notice('Package found in Local Area:', package)
continue
if installSoftwarePackage(package, localArea())['OK']:
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
ha.setSoftwarePackage(HapPack)
ha.hapExecutable = 'DoCtaIrf'
runlistdir = os.environ['PWD']
build_infile(runlist)
ha.hapArguments = [
analysistype, cutsconfig, runlistdir, runlist, zenith, offset,
arrayconfig, energymethod, particle
]
DIRAC.gLogger.notice('Executing Hap Application')
res = ha.execute()
if not res['OK']:
DIRAC.gLogger.error('Failed to execute Hap Application')
jobReport.setApplicationStatus('Hap Application: Failed')
DIRAC.exit(-1)
###################### Check TTree Output File #######################
outfile = 'MVAFile_' + runlist + ".root"
if not os.path.isfile(outfile):
error = 'TTree file was not created:'
DIRAC.gLogger.error(error, outfile)
jobReport.setApplicationStatus('DoCtaIrf: TTree file not created')
DIRAC.exit(-1)
###################### Quality Check for TTree Output File: step0######################
hr = HapRootMacro()
hr.setSoftwarePackage(HapPack)
DIRAC.gLogger.notice('Executing TTree check step0')
hr.rootMacro = '/hapscripts/mva/Open_TT.C+'
outfilestr = '"' + outfile + '"'
args = [outfilestr]
DIRAC.gLogger.notice('Open_TT macro Arguments:', args)
hr.rootArguments = args
DIRAC.gLogger.notice('Executing Hap Open_TT macro')
res = hr.execute()
if not res['OK']:
DIRAC.gLogger.error('Open_TT: Failed')
DIRAC.exit(-1)
#########################Quality Check for TTree Output File: step1####################
DIRAC.gLogger.notice('Executing TTree check step1')
ret = getSoftwareEnviron(HapPack)
if not ret['OK']:
error = ret['Message']
DIRAC.gLogger.error(error, HapPack)
DIRAC.exit(-1)
hapEnviron = ret['Value']
hessroot = hapEnviron['HESSROOT']
check_script = hessroot + '/hapscripts/mva/check_TT.csh'
cmdTuple = [check_script]
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute TTree Check step1')
jobReport.setApplicationStatus('Check_TTree: Failed')
DIRAC.exit(-1)
#############################################
DIRAC.exit()
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("T:", "template=", "Corsika Template")
Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig")
Script.registerSwitch("p:", "run_number=",
"Do not use: Run Number automatically set")
Script.registerSwitch("E:", "executable=",
"Executable (Use SetExecutable)")
Script.registerSwitch("V:", "version=", "Version (Use setVersion)")
Script.registerSwitch("M:", "mode=",
"Mode (corsika_standalone/corsika_simtelarray)")
Script.parseCommandLine(ignoreErrors=True)
## default values ##############
run_number = None
corsikaTemplate = None
simtelConfig = None
executable = None
version = None
mode = 'corsika_simtelarray'
### set switch values ###
for switch in Script.getUnprocessedSwitches():
if switch[0] == "run_number" or switch[0] == "p":
run_number = switch[1].split('ParametricParameters=')[1]
elif switch[0] == "template" or switch[0] == "T":
corsikaTemplate = switch[1]
elif switch[0] == "simtelConfig" or switch[0] == "S":
simtelConfig = switch[1]
elif switch[0] == "executable" or switch[0] == "E":
executable = switch[1]
elif switch[0] == "version" or switch[0] == "V":
version = switch[1]
elif switch[0] == "mode" or switch[0] == "M":
mode = switch[1]
if version == None or executable == None or run_number == None or corsikaTemplate == None:
Script.showHelp()
jobReport.setApplicationStatus('Missing options')
DIRAC.exit(-1)
from CTADIRAC.Core.Workflow.Modules.CorsikaApp import CorsikaApp
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
createGlobalsFromConfigFiles(corsikaTemplate, version)
############ Producing Corsika File
install_CorsikaSimtelPack(version)
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
cs = CorsikaApp()
cs.setSoftwarePackage(CorsikaSimtelPack)
cs.csExe = executable
cs.csArguments = [
'--run-number', run_number, '--run', 'corsika', corsikaTemplate
]
corsikaReturnCode = cs.execute()
if corsikaReturnCode != 0:
DIRAC.gLogger.error('Corsika Application: Failed')
jobReport.setApplicationStatus('Corsika Application: Failed')
DIRAC.exit(-1)
###################### rename of corsika output file #######################
rundir = 'run' + run_number
filein = rundir + '/' + corsikaOutputFileName
corsikaFileName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.gz'
mv_cmd = 'mv ' + filein + ' ' + corsikaFileName
if (os.system(mv_cmd)):
DIRAC.exit(-1)
### create corsika tar luisa ####################
corsikaTarName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.tar.gz'
filetar1 = rundir + '/' + 'input'
filetar2 = rundir + '/' + 'DAT' + run_number + '.dbase'
filetar3 = rundir + '/run' + str(int(run_number)) + '.log'
cmdTuple = [
'/bin/tar', 'zcf', corsikaTarName, filetar1, filetar2, filetar3
]
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute tar')
DIRAC.exit(-1)
if (mode == 'corsika_standalone'):
DIRAC.exit()
############ Producing SimTel File
######################Building simtel Directory Metadata #######################
cfg_dict = {
"4MSST": 'cta-prod2-4m-dc',
"SCSST": 'cta-prod2-sc-sst',
"STD": 'cta-prod2',
"NSBX3": 'cta-prod2',
"ASTRI": 'cta-prod2-astri',
"NORTH": 'cta-prod2n',
"SCMST": 'cta-prod2-sc3'
}
#if simtelConfig=="6INROW":
# all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD","SCMST"]
#elif simtelConfig=="5INROW":
# all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD"]
#else:
# all_configs=[simtelConfig]
all_configs = [simtelConfig]
for current_conf in all_configs:
DIRAC.gLogger.notice('current conf is', current_conf)
current_version = version
DIRAC.gLogger.notice('current version is', current_version)
#if current_conf == "SCMST":
# current_version = version + '_sc3'
# DIRAC.gLogger.notice('current version is', current_version)
# if os.path.isdir('sim_telarray'):
# DIRAC.gLogger.notice('Package found in the local area. Removing package...')
# cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run'
# if(os.system(cmd)):
# DIRAC.exit( -1 )
# install_CorsikaSimtelPack(current_version)
#else:
# current_version = version
# DIRAC.gLogger.notice('current version is', current_version)
#### execute simtelarray ################
fd = open('run_sim.sh', 'w')
fd.write("""#! /bin/sh
export SVNPROD2=$PWD
export SVNTAG=SVN-PROD2
export CORSIKA_IO_BUFFER=800MB
./grid_prod2-repro.sh %s %s""" % (corsikaFileName, current_conf))
fd.close()
####################################
os.system('chmod u+x run_sim.sh')
os.system('chmod u+x grid_prod2-repro.sh')
cmdTuple = ['./run_sim.sh']
ret = systemCall(0, cmdTuple, sendSimtelOutput)
simtelReturnCode, stdout, stderr = ret['Value']
if (os.system('grep Broken simtel.log') == 0):
DIRAC.gLogger.error('Broken string found in simtel.log')
jobReport.setApplicationStatus('Broken pipe')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute run_sim.sh')
DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode)
DIRAC.exit(-1)
## check simtel data/log/histo Output File exist
cfg = cfg_dict[current_conf]
simtelFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.simtel.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/*/Data/*.simtel.gz ' + simtelFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelLogFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/*/Log/*.log.gz ' + simtelLogFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelHistFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.hdata.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/*/Histograms/*.hdata.gz ' + simtelHistFileName
if (os.system(cmd)):
DIRAC.exit(-1)
################################################################
DIRAC.exit()
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("p:", "inputfile=", "Input File")
Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig")
Script.registerSwitch("V:", "version=", "Version (Use setVersion)")
Script.registerSwitch("C:", "comp=", "Compile (True/False)")
Script.parseCommandLine(ignoreErrors=True)
## default values ##############
simtelConfig = None
version = None
comp = True
### set switch values ###
for switch in Script.getUnprocessedSwitches():
if switch[0] == "inputfile" or switch[0] == "p":
corsikaFileLFN = switch[1].split('ParametricInputData=LFN:')[1]
elif switch[0] == "simtelConfig" or switch[0] == "S":
simtelConfig = switch[1]
elif switch[0] == "version" or switch[0] == "V":
version = switch[1]
elif switch[0] == "comp" or switch[0] == "C":
comp = switch[1]
if version == None:
Script.showHelp()
jobReport.setApplicationStatus('Missing options')
DIRAC.exit(-1)
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
############ Producing SimTel File
if comp == True:
install_CorsikaSimtelPack(version)
else:
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
res = installSoftwarePackage(CorsikaSimtelPack, workingArea())
if not res['OK']:
DIRAC.gLogger.error('Failed to execute installSoftwarePackage',
res)
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
cfg_dict = {
"4MSST": 'cta-prod2-4m-dc',
"SCSST": 'cta-prod2-sc-sst',
"STD": 'cta-prod2',
"NSBX3": 'cta-prod2',
"ASTRI": 'cta-prod2-astri',
"NORTH": 'cta-prod2n',
"ASTRISLN": 'cta-astri-sln'
}
#if simtelConfig=="6INROW":
# all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD","SCMST"]
#elif simtelConfig=="5INROW":
# all_configs=["4MSST","SCSST","ASTRI","NSBX3","STD"]
#else:
# all_configs=[simtelConfig]
all_configs = [simtelConfig]
for current_conf in all_configs:
DIRAC.gLogger.notice('current conf is', current_conf)
current_version = version
DIRAC.gLogger.notice('current version is', current_version)
#if current_conf == "SCMST":
# current_version = version + '_sc3'
# DIRAC.gLogger.notice('current version is', current_version)
# if os.path.isdir('sim_telarray'):
# DIRAC.gLogger.notice('Package found in the local area. Removing package...')
# cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run'
# if(os.system(cmd)):
# DIRAC.exit( -1 )
# install_CorsikaSimtelPack(current_version)
#else:
# current_version = version
# DIRAC.gLogger.notice('current version is', current_version)
#### execute simtelarray ################
fd = open('run_sim.sh', 'w')
fd.write("""#! /bin/sh
export SVNPROD2=$PWD
export SVNTAG=SVN-PROD2
export CORSIKA_IO_BUFFER=800MB
./grid_prod2-repro.sh %s %s""" %
(os.path.basename(corsikaFileLFN), current_conf))
fd.close()
####################################
os.system('chmod u+x run_sim.sh')
os.system('chmod u+x grid_prod2-repro.sh')
cmdTuple = ['./run_sim.sh']
ret = systemCall(0, cmdTuple, sendSimtelOutput)
simtelReturnCode, stdout, stderr = ret['Value']
if (os.system('grep Broken simtel.log') == 0):
DIRAC.gLogger.error('Broken string found in simtel.log')
jobReport.setApplicationStatus('Broken pipe')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute run_sim.sh')
DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode)
DIRAC.exit(-1)
## check simtel data/log/histo Output File exist
cfg = cfg_dict[current_conf]
simtelFileName = os.path.basename(corsikaFileLFN).replace(
'corsika.gz', 'simtel.gz')
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Data/*.simtel.gz ' + simtelFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelLogFileName = simtelFileName.replace('simtel.gz', 'log.gz')
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Log/*.log.gz ' + simtelLogFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelHistFileName = simtelFileName.replace('simtel.gz', 'hdata.gz')
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName
if (os.system(cmd)):
DIRAC.exit(-1)
################################################################
DIRAC.exit()
import DIRAC
from DIRAC.Core.Base import Script
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... PilotID ...' % Script.scriptName, 'Arguments:',
' PilotID: Grid ID of the pilot'
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
for gridID in args:
result = diracAdmin.getPilotOutput(gridID)
if not result['OK']:
errorList.append((gridID, result['Message']))
exitCode = 2
for error in errorList:
print "ERROR %s: %s" % error
DIRAC.exit(exitCode)
#from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC import gConfig, gLogger
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
#csAPI = CSAPI()
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
setup = gConfig.getValue('/DIRAC/Setup', '')
if not setup:
print 'ERROR: Could not contact Configuration Service'
exitCode = 2
DIRAC.exit(exitCode)
res = getProxyInfo()
if not res['OK']:
gLogger.error('Failed to get proxy information', res['Message'])
DIRAC.exit(2)
userName = res['Value'].get('username')
if not userName:
gLogger.error('Failed to get username for proxy')
DIRAC.exit(2)
if site:
res = gConfig.getOptionsDict('/Resources/Sites/LCG/%s' % site)
if not res['OK']:
gLogger.error('The provided site (%s) is not known.' % site)
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("p:", "inputfile=", "Input File", setInputFile)
Script.registerSwitch("E:", "simtelExecName=", "SimtelExecName",
setExecutable)
Script.registerSwitch("V:", "version=", "Version", setVersion)
Script.registerSwitch("D:", "storage_element=", "Storage Element",
setStorageElement)
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
Script.parseCommandLine()
global fcc, fcL, storage_element
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
packs = [CorsikaSimtelPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
installSoftwareEnviron(package, workingArea())
packageTuple = package.split('/')
corsika_subdir = sharedArea(
) + '/' + packageTuple[0] + '/' + version
cmd = 'cp -u -r ' + corsika_subdir + '/* .'
os.system(cmd)
continue
if workingArea:
if checkSoftwarePackage(package, workingArea())['OK']:
DIRAC.gLogger.notice('Package found in Local Area:', package)
continue
if installSoftwarePackage(package, workingArea())['OK']:
############## compile #############################
if version == 'prod-2_21122012':
cmdTuple = ['./build_all', 'prod2', 'qgs2']
else:
cmdTuple = ['./build_all', 'ultra', 'qgs2']
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to compile')
DIRAC.exit(-1)
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
###########
## Checking MD coherence
fc = FileCatalog('LcgFileCatalog')
res = fc._getCatalogConfigDetails('DIRACFileCatalog')
print 'DFC CatalogConfigDetails:', res
res = fc._getCatalogConfigDetails('LcgFileCatalog')
print 'LCG CatalogConfigDetails:', res
fcc = FileCatalogClient()
fcL = FileCatalog('LcgFileCatalog')
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
############################
#############
# CLAUDIA: simtelConfigFile should be built from ???
simtelConfigFilesPath = 'sim_telarray/multi'
simtelConfigFile = simtelConfigFilesPath + '/multi_cta-ultra5.cfg'
createGlobalsFromConfigFiles(simtelConfigFile)
#######################
## files spread in 1000-runs subDirectories
corsikaFileName = os.path.basename(corsikaFileLFN)
run_number = corsikaFileName.split('run')[1].split('.corsika.gz')[
0] # run001412.corsika.gz
runNum = int(run_number)
subRunNumber = '%03d' % runNum
runNumModMille = runNum % 1000
runNumTrunc = (runNum - runNumModMille) / 1000
runNumSeriesDir = '%03dxxx' % runNumTrunc
print 'runNumSeriesDir=', runNumSeriesDir
f = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w')
f.close()
##### If storage element is IN2P3-tape save simtel file on disk ###############
if storage_element == 'CC-IN2P3-Tape':
storage_element = 'CC-IN2P3-Disk'
############ Producing SimTel File
######################Building simtel Directory Metadata #######################
resultCreateSimtelDirMD = createSimtelFileSystAndMD()
if not resultCreateSimtelDirMD['OK']:
DIRAC.gLogger.error('Failed to create simtelArray Directory MD')
jobReport.setApplicationStatus(
'Failed to create simtelArray Directory MD')
DIRAC.gLogger.error(
'Metadata coherence problem, no simtelArray File produced')
DIRAC.exit(-1)
else:
print 'simtel Directory MD successfully created'
#### execute simtelarray ################
fd = open('run_sim.sh', 'w')
fd.write(
"""#! /bin/sh
echo "go for sim_telarray"
. ./examples_common.sh
export CORSIKA_IO_BUFFER=800MB
zcat %s | $SIM_TELARRAY_PATH/run_sim_%s""" % (corsikaFileName, simtelExecName))
fd.close()
os.system('chmod u+x run_sim.sh')
cmdTuple = ['./run_sim.sh']
ret = systemCall(0, cmdTuple, sendOutputSimTel)
simtelReturnCode, stdout, stderr = ret['Value']
if (os.system('grep Broken simtel.log')):
print 'not broken'
else:
print 'broken'
# Tag corsika File if Broken Pipe
corsikaTagMD = {}
corsikaTagMD['CorsikaToReprocess'] = 'CorsikaToReprocess'
result = fcc.setMetadata(corsikaFileLFN, corsikaTagMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
jobReport.setApplicationStatus('Broken pipe')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute run_sim.sh')
DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode)
DIRAC.exit(-1)
## putAndRegister simtel data/log/histo Output File:
simtelFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.simtel.gz'
cmd = 'mv Data/sim_telarray/' + simtelExecName + '/0.0deg/Data/*.simtel.gz ' + simtelFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutFileDir = os.path.join(simtelDirPath, 'Data', runNumSeriesDir)
simtelOutFileLFN = os.path.join(simtelOutFileDir, simtelFileName)
simtelRunNumberSeriesDirExist = fcc.isDirectory(
simtelOutFileDir)['Value']['Successful'][simtelOutFileDir]
newSimtelRunFileSeriesDir = (
simtelRunNumberSeriesDirExist != True
) # if new runFileSeries, will need to add new MD
simtelLogFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz'
cmd = 'mv Data/sim_telarray/' + simtelExecName + '/0.0deg/Log/*.log.gz ' + simtelLogFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutLogFileDir = os.path.join(simtelDirPath, 'Log', runNumSeriesDir)
simtelOutLogFileLFN = os.path.join(simtelOutLogFileDir, simtelLogFileName)
simtelHistFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.hdata.gz'
cmd = 'mv Data/sim_telarray/' + simtelExecName + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutHistFileDir = os.path.join(simtelDirPath, 'Histograms',
runNumSeriesDir)
simtelOutHistFileLFN = os.path.join(simtelOutHistFileDir,
simtelHistFileName)
################################################
DIRAC.gLogger.notice('Put and register simtel File in LFC and DFC:',
simtelOutFileLFN)
ret = dirac.addFile(simtelOutFileLFN, simtelFileName, storage_element)
res = CheckCatalogCoherence(simtelOutFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
######################################################################
DIRAC.gLogger.notice('Put and register simtel Log File in LFC and DFC:',
simtelOutLogFileLFN)
ret = dirac.addFile(simtelOutLogFileLFN, simtelLogFileName,
storage_element)
res = CheckCatalogCoherence(simtelOutLogFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
######################################################################
DIRAC.gLogger.notice('Put and register simtel Histo File in LFC and DFC:',
simtelOutHistFileLFN)
ret = dirac.addFile(simtelOutHistFileLFN, simtelHistFileName,
storage_element)
res = CheckCatalogCoherence(simtelOutHistFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
######################################################################
if newSimtelRunFileSeriesDir:
insertRunFileSeriesMD(simtelOutFileDir, runNumTrunc)
insertRunFileSeriesMD(simtelOutLogFileDir, runNumTrunc)
insertRunFileSeriesMD(simtelOutHistFileDir, runNumTrunc)
###### simtel File level metadata ############################################
simtelFileMD = {}
simtelFileMD['runNumber'] = int(run_number)
simtelFileMD['jobID'] = jobID
simtelFileMD['simtelReturnCode'] = simtelReturnCode
result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(simtelOutLogFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(simtelOutHistFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.addFileAncestors(
{simtelOutFileLFN: {
'Ancestors': [corsikaFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{simtelOutLogFileLFN: {
'Ancestors': [corsikaFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{simtelOutHistFileLFN: {
'Ancestors': [corsikaFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD)
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
DIRAC.exit()
def setHostDN(arg):
global hostDN
if hostDN or not arg:
Script.showHelp()
DIRAC.exit(-1)
hostDN = arg
def setHostName(arg):
global hostName
if hostName or not arg:
Script.showHelp()
DIRAC.exit(-1)
hostName = arg
def main():
Script.registerSwitch(
"v:", "vo=",
"Location of pilot version in CS /Operations/<vo>/Pilot/Version"
" (default value specified in CS under /DIRAC/DefaultSetup)")
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if len(args) < 1 or len(args) > 2:
Script.showHelp()
version = args[0]
vo = None
for switch in Script.getUnprocessedSwitches():
if switch[0] == "v" or switch[0] == "vo":
vo = switch[1]
from DIRAC import S_OK, S_ERROR
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
def updatePilot(version, vo):
"""
Update in the CS the pilot version used,
If only one version present in CS it's overwritten.
If two versions present, the new one is added and the last removed
:param version: version vArBpC of pilot you want to use
:param vo: Location of pilot version in CS /Operations/<vo>/Pilot/Version
"""
setup = vo
if not vo:
setup = gConfig.getValue('/DIRAC/DefaultSetup')
if not setup:
return S_ERROR("No value set for /DIRAC/DefaultSetup in CS")
pilotVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup,
[])
if not pilotVersion:
return S_ERROR(
"No pilot version set under Operations/%s/Pilot/Version in CS"
% setup)
pilotVersion.pop()
pilotVersion.insert(0, version)
api = CSAPI()
api.setOption('Operations/%s/Pilot/Version' % setup,
", ".join(pilotVersion))
result = api.commit()
if not result['OK']:
gLogger.fatal('Could not commit new version of pilot!')
return result
newVersion = gConfig.getValue('Operations/%s/Pilot/Version' % setup)
return S_OK("New version of pilot set to %s" % newVersion)
result = updatePilot(version, vo)
if not result['OK']:
gLogger.fatal(result['Message'])
DIRAC.exit(1)
gLogger.notice(result['Value'])
DIRAC.exit(0)
def main():
from DIRAC import gLogger
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import createSharedArea
from DIRAC.Core.Utilities.Subprocess import systemCall
DIRAC.gLogger.notice('Platform is:')
os.system('dirac-platform')
args = Script.getPositionalArgs()
version = args[0]
area = sharedArea()
if area:
gLogger.notice('Using Shared Area at:', area)
# if defined, check that it really exists
if not os.path.isdir(area):
gLogger.error('Missing Shared Area Directory:', area)
if createSharedArea() == True:
gLogger.notice('Shared Area created')
else:
gLogger.error('Failed to create Shared Area Directory:', area)
DIRAC.exit(-1)
else:
if createSharedArea() == True:
gLogger.notice('Shared Area created')
else:
gLogger.error('Failed to create Shared Area Directory:', area)
DIRAC.exit(-1)
EvnDispSimtelPack = os.path.join('evndisplay', version, 'evndisplay')
packs = [EvnDispSimtelPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
packageTuple = package.split('/')
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
continue
if installSoftwarePackage(package, sharedArea())['OK']:
if not os.path.isdir(
os.path.join(sharedArea(), packageTuple[0],
packageTuple[1])):
DIRAC.gLogger.error(
'Software package missing in the shared area')
DIRAC.exit(-1)
continue
DIRAC.gLogger.error('Software package not correctly installed')
DIRAC.exit(-1)
DIRAC.exit()
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(" sourceSE: source SE")
Script.registerArgument(" LFN: LFN or file containing a List of LFNs")
Script.registerArgument(["targetSE: target SEs"])
Script.parseCommandLine()
import DIRAC
from DIRAC import gLogger
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
args = Script.getPositionalArgs()
sourceSE = args[0]
lfnList = getLFNList(args[1])
targetSEs = list(set([se for targetSE in args[2:] for se in targetSE.split(",")]))
gLogger.info(
"Will create request with 'MoveReplica' "
"operation using %s lfns and %s target SEs" % (len(lfnList), len(targetSEs))
)
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.Core.Utilities.List import breakListIntoChunks
lfnChunks = breakListIntoChunks(lfnList, 100)
multiRequests = len(lfnChunks) > 1
error = 0
count = 0
reqClient = ReqClient()
fc = FileCatalog()
for lfnChunk in lfnChunks:
metaDatas = fc.getFileMetadata(lfnChunk)
if not metaDatas["OK"]:
gLogger.error("unable to read metadata for lfns: %s" % metaDatas["Message"])
error = -1
continue
metaDatas = metaDatas["Value"]
for failedLFN, reason in metaDatas["Failed"].items():
gLogger.error("skipping %s: %s" % (failedLFN, reason))
lfnChunk = set(metaDatas["Successful"])
if not lfnChunk:
gLogger.error("LFN list is empty!!!")
error = -1
continue
if len(lfnChunk) > Operation.MAX_FILES:
gLogger.error("too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES)
error = -1
continue
count += 1
request = Request()
request.RequestName = "%s_%s" % (
md5(repr(time.time()).encode()).hexdigest()[:16],
md5(repr(time.time()).encode()).hexdigest()[:16],
)
moveReplica = Operation()
moveReplica.Type = "MoveReplica"
moveReplica.SourceSE = sourceSE
moveReplica.TargetSE = ",".join(targetSEs)
for lfn in lfnChunk:
metaDict = metaDatas["Successful"][lfn]
opFile = File()
opFile.LFN = lfn
opFile.Size = metaDict["Size"]
if "Checksum" in metaDict:
# # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
opFile.Checksum = metaDict["Checksum"]
opFile.ChecksumType = "ADLER32"
moveReplica.addFile(opFile)
request.addOperation(moveReplica)
result = reqClient.putRequest(request)
if not result["OK"]:
gLogger.error("Failed to submit Request: %s" % (result["Message"]))
error = -1
continue
if not multiRequests:
gLogger.always("Request %d submitted successfully" % result["Value"])
if multiRequests:
gLogger.always("%d requests have been submitted" % (count))
DIRAC.exit(error)
def main():
read = True
write = True
check = True
remove = True
sites = []
mute = False
Script.registerSwitch("r", "BanRead",
" Ban only reading from the storage element")
Script.registerSwitch("w", "BanWrite",
" Ban writing to the storage element")
Script.registerSwitch("k", "BanCheck",
" Ban check access to the storage element")
Script.registerSwitch("v", "BanRemove",
" Ban remove access to the storage element")
Script.registerSwitch("a", "All",
" Ban all access to the storage element")
Script.registerSwitch("m", "Mute", " Do not send email")
Script.registerSwitch(
"S:", "Site=",
" Ban all SEs associate to site (note that if writing is allowed, check is always allowed)"
)
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
["seGroupList: list of SEs or comma-separated SEs"])
switches, ses = Script.parseCommandLine(ignoreErrors=True)
for switch in switches:
if switch[0].lower() in ("r", "banread"):
write = False
check = False
remove = False
if switch[0].lower() in ("w", "banwrite"):
read = False
check = False
remove = False
if switch[0].lower() in ("k", "bancheck"):
read = False
write = False
remove = False
if switch[0].lower() in ("v", "banremove"):
read = False
write = False
check = False
if switch[0].lower() in ("a", "all"):
pass
if switch[0].lower() in ("m", "mute"):
mute = True
if switch[0].lower() in ("s", "site"):
sites = switch[1].split(",")
# from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup, DMSHelpers
ses = resolveSEGroup(ses)
diracAdmin = DiracAdmin()
setup = gConfig.getValue("/DIRAC/Setup", "")
if not setup:
print("ERROR: Could not contact Configuration Service")
DIRAC.exit(2)
res = getProxyInfo()
if not res["OK"]:
gLogger.error("Failed to get proxy information", res["Message"])
DIRAC.exit(2)
userName = res["Value"].get("username")
if not userName:
gLogger.error("Failed to get username for proxy")
DIRAC.exit(2)
for site in sites:
res = DMSHelpers().getSEsForSite(site)
if not res["OK"]:
gLogger.error(res["Message"], site)
DIRAC.exit(-1)
ses.extend(res["Value"])
if not ses:
gLogger.error("There were no SEs provided")
DIRAC.exit(-1)
readBanned = []
writeBanned = []
checkBanned = []
removeBanned = []
resourceStatus = ResourceStatus()
res = resourceStatus.getElementStatus(ses, "StorageElement")
if not res["OK"]:
gLogger.error("Storage Element %s does not exist" % ses)
DIRAC.exit(-1)
reason = "Forced with dirac-admin-ban-se by %s" % userName
for se, seOptions in res["Value"].items():
resW = resC = resR = {"OK": False}
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if read and "ReadAccess" in seOptions:
if seOptions["ReadAccess"] == "Banned":
gLogger.notice("Read access already banned", se)
resR["OK"] = True
elif not seOptions["ReadAccess"] in [
"Active", "Degraded", "Probing"
]:
gLogger.notice("Read option for %s is %s, instead of %s" %
(se, seOptions["ReadAccess"],
["Active", "Degraded", "Probing"]))
gLogger.notice("Try specifying the command switches")
else:
resR = resourceStatus.setElementStatus(se, "StorageElement",
"ReadAccess", "Banned",
reason, userName)
# res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" )
if not resR["OK"]:
gLogger.error("Failed to update %s read access to Banned" %
se)
else:
gLogger.notice(
"Successfully updated %s read access to Banned" % se)
readBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if write and "WriteAccess" in seOptions:
if seOptions["WriteAccess"] == "Banned":
gLogger.notice("Write access already banned", se)
resW["OK"] = True
elif not seOptions["WriteAccess"] in [
"Active", "Degraded", "Probing"
]:
gLogger.notice("Write option for %s is %s, instead of %s" %
(se, seOptions["WriteAccess"],
["Active", "Degraded", "Probing"]))
gLogger.notice("Try specifying the command switches")
else:
resW = resourceStatus.setElementStatus(se, "StorageElement",
"WriteAccess", "Banned",
reason, userName)
# res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" )
if not resW["OK"]:
gLogger.error(
"Failed to update %s write access to Banned" % se)
else:
gLogger.notice(
"Successfully updated %s write access to Banned" % se)
writeBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if check and "CheckAccess" in seOptions:
if seOptions["CheckAccess"] == "Banned":
gLogger.notice("Check access already banned", se)
resC["OK"] = True
elif not seOptions["CheckAccess"] in [
"Active", "Degraded", "Probing"
]:
gLogger.notice("Check option for %s is %s, instead of %s" %
(se, seOptions["CheckAccess"],
["Active", "Degraded", "Probing"]))
gLogger.notice("Try specifying the command switches")
else:
resC = resourceStatus.setElementStatus(se, "StorageElement",
"CheckAccess", "Banned",
reason, userName)
# res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" )
if not resC["OK"]:
gLogger.error(
"Failed to update %s check access to Banned" % se)
else:
gLogger.notice(
"Successfully updated %s check access to Banned" % se)
checkBanned.append(se)
# Eventually, we will get rid of the notion of InActive, as we always write Banned.
if remove and "RemoveAccess" in seOptions:
if seOptions["RemoveAccess"] == "Banned":
gLogger.notice("Remove access already banned", se)
resC["OK"] = True
elif not seOptions["RemoveAccess"] in [
"Active", "Degraded", "Probing"
]:
gLogger.notice("Remove option for %s is %s, instead of %s" %
(se, seOptions["RemoveAccess"],
["Active", "Degraded", "Probing"]))
gLogger.notice("Try specifying the command switches")
else:
resC = resourceStatus.setElementStatus(se, "StorageElement",
"RemoveAccess",
"Banned", reason,
userName)
# res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" )
if not resC["OK"]:
gLogger.error(
"Failed to update %s remove access to Banned" % se)
else:
gLogger.notice(
"Successfully updated %s remove access to Banned" % se)
removeBanned.append(se)
if not (resR["OK"] or resW["OK"] or resC["OK"]):
DIRAC.exit(-1)
if not (writeBanned or readBanned or checkBanned or removeBanned):
gLogger.notice("No storage elements were banned")
DIRAC.exit(-1)
if mute:
gLogger.notice("Email is muted by script switch")
DIRAC.exit(0)
subject = "%s storage elements banned for use" % len(
writeBanned + readBanned + checkBanned + removeBanned)
addressPath = "EMail/Production"
address = Operations().getValue(addressPath, "")
body = ""
if read:
body = "%s\n\nThe following storage elements were banned for reading:" % body
for se in readBanned:
body = "%s\n%s" % (body, se)
if write:
body = "%s\n\nThe following storage elements were banned for writing:" % body
for se in writeBanned:
body = "%s\n%s" % (body, se)
if check:
body = "%s\n\nThe following storage elements were banned for check access:" % body
for se in checkBanned:
body = "%s\n%s" % (body, se)
if remove:
body = "%s\n\nThe following storage elements were banned for remove access:" % body
for se in removeBanned:
body = "%s\n%s" % (body, se)
if not address:
gLogger.notice(
"'%s' not defined in Operations, can not send Mail\n" %
addressPath, body)
DIRAC.exit(0)
res = diracAdmin.sendMail(address, subject, body)
gLogger.notice("Notifying %s" % address)
if res["OK"]:
gLogger.notice(res["Value"])
else:
gLogger.notice(res["Message"])
DIRAC.exit(0)
def setupWorkflow(self, debug=False):
""" Override the base class job workflow to adapt to NSB test simulations
All parameters shall have been defined before that method is called.
"""
# step 1 - debug only
iStep = 1
if debug:
lsStep = self.setExecutable('/bin/ls -alhtr',
logFile='LS_Init_Log.txt')
lsStep['Value']['name'] = 'Step%i_LS_Init' % iStep
lsStep['Value']['descr_short'] = 'list files in working directory'
iStep += 1
# step 2
swStep = self.setExecutable( 'cta-prod3-setupsw',
arguments='%s %s simulations sl6-gcc44'% (self.package, self.version),\
logFile='SetupSoftware_Log.txt')
swStep['Value']['name'] = 'Step%i_SetupSoftware' % iStep
swStep['Value']['descr_short'] = 'Setup software'
iStep += 1
# step 3
if self.cta_site == 'Paranal':
prod_script = './dirac_prod3_paranal_baseline'
elif self.cta_site == 'LaPalma':
prod_script = './dirac_prod3_lapalma_baseline'
else:
DIRAC.gLogger.error('Unknown site: %s' % self.cta_site)
DIRAC.gLogger.error('No shell script associated')
DIRAC.exit(-1)
csStep = self.setExecutable( prod_script, \
arguments = '--start_run %s --run %s %s %s %s %s %s' % \
( self.start_run_number, self.run_number, \
self.extra_tag, self.cta_site,\
self.particle, self.pointing_dir, self.zenith_angle ), \
logFile='CorsikaSimtel_Log.txt')
csStep['Value']['name'] = 'Step%i_CorsikaSimtel' % iStep
csStep['Value']['descr_short'] = 'Run Corsika piped into simtel'
iStep += 1
# step 4 verify merged data
mgvStep = self.setExecutable( 'cta-prod3-verifysteps', \
arguments = "generic %0d 1000 '%s/Data/*.zst'"%\
(self.N_output_files, self.inputpath),\
logFile='Verify_Simtel_Log.txt')
mgvStep['Value']['name'] = 'Step%i_VerifySimtel' % iStep
mgvStep['Value']['descr_short'] = 'Verify simtel files'
iStep += 1
# step 5 - debug only
if debug:
lsStep = self.setExecutable('/bin/ls -Ralhtr',
logFile='LS_End_Log.txt')
lsStep['Value']['name'] = 'Step%i_LS_End' % iStep
lsStep['Value'][
'descr_short'] = 'list files in working directory and sub-directory'
iStep += 1
# step 6
# meta data
# ## the order of the metadata dictionary is important, since it's used to build the directory structure
metadata = collections.OrderedDict()
metadata['array_layout'] = self.array_layout
metadata['site'] = self.cta_site
metadata['particle'] = self.particle
if self.pointing_dir == 'North':
metadata['phiP'] = 180
if self.pointing_dir == 'South':
metadata['phiP'] = 0
metadata['thetaP'] = float(self.zenith_angle)
metadata[self.program_category + '_prog'] = 'simtel'
metadata[self.program_category + '_prog_version'] = self.version
metadata['data_level'] = self.output_data_level
metadata['configuration_id'] = self.configuration_id
mdjson = json.dumps(metadata)
# meta data field
metadatafield = {
'array_layout': 'VARCHAR(128)',
'site': 'VARCHAR(128)',
'particle': 'VARCHAR(128)',
'phiP': 'float',
'thetaP': 'float',
self.program_category + '_prog': 'VARCHAR(128)',
self.program_category + '_prog_version': 'VARCHAR(128)',
'data_level': 'int',
'configuration_id': 'int'
}
mdfieldjson = json.dumps(metadatafield)
# register Data
## file meta data
#filemetadata = {'runNumber': self.run_number, 'nsb':1}
filemetadata = {'runNumber': self.run_number}
file_md_json = json.dumps(filemetadata)
outputpattern = './Data/sim_telarray/*/*/Data/*baseline.simtel.zst'
dmStep = self.setExecutable('../CTADIRAC/Core/scripts/cta-analysis-managedata.py',
arguments = "'%s' '%s' '%s' %s '%s' %s %s '%s'" %\
(mdjson, mdfieldjson, file_md_json, self.basepath,
outputpattern, self.package, self.program_category, self.catalogs),
logFile = 'Data_DataManagement_Log.txt')
dmStep['Value']['name'] = 'Step%i_DataManagement_1' % iStep
dmStep['Value'][
'descr_short'] = 'Save data files to SE and register them in DFC'
iStep += 1
## log file
outputpattern = './Data/sim_telarray/*/*/Log/*baseline.log.gz'
dmStep = self.setExecutable('../CTADIRAC/Core/scripts/cta-analysis-managedata.py',
arguments = "'%s' '%s' '%s' %s '%s' %s %s '%s' Log" % \
(mdjson, mdfieldjson, file_md_json, self.basepath,
outputpattern, self.package, self.program_category, self.catalogs),
logFile = 'Log_DataManagement_Log.txt')
dmStep['Value']['name'] = 'Step%i_Log_DataManagement' % iStep
dmStep['Value'][
'descr_short'] = 'Save log files to SE and register them in DFC'
iStep += 1
## histogram
# outputpattern = './Data/sim_telarray/*/*/Histograms/*baseline.hdata.gz'
# dmStep = self.setExecutable('../CTADIRAC/Core/scripts/cta-analysis-managedata.py',
# arguments = "'%s' '%s' '%s' %s '%s' %s %s '%s' Histograms" % \
# (mdjson, mdfieldjson, file_md_json, self.basepath,
# outputpattern, self.package, self.program_category, self.catalogs),
# logFile = 'Histo_DataManagement_Log.txt')
# dmStep['Value']['name'] = 'Step%i_Histo_DataManagement' % iStep
# dmStep['Value']['descr_short'] = 'Save hitograms files to SE and register them in DFC'
# iStep += 1
# Number of showers is passed via an environment variable
self.setExecutionEnv({'NSHOW': '%s' % self.nShower})
def install_CorsikaSimtelPack(version):
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
packs = [CorsikaSimtelPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
installSoftwareEnviron(package, workingArea())
packageTuple = package.split('/')
corsika_subdir = sharedArea(
) + '/' + packageTuple[0] + '/' + version
cmd = 'cp -u -r ' + corsika_subdir + '/* .'
os.system(cmd)
continue
if workingArea:
if checkSoftwarePackage(package, workingArea())['OK']:
DIRAC.gLogger.notice('Package found in Local Area:', package)
continue
if installSoftwarePackage(package, workingArea())['OK']:
############## compile #############################
cmdTuple = ['./build_all', 'prod2', 'qgs2']
######### special case for Astri ############################
if version == 'prod-2_08072014_to':
############## compile #############################
fd = open('run_compile.sh', 'w')
fd.write("""#! /bin/sh
source ./build_all prod2 qgs2
#
echo " Let's check that build_all did its work "
ls -alFsh
echo "+++++++++++++++++++++++++++++++++++++++++++++++++"
echo " Let's see what files are in the corsika-run directory "
ls -alFsh ./corsika-run
#
if [ ! -x ./corsika-run/corsika ]
then
echo " ERROR: Corsika executable found. Exit "
exit 1
fi
echo "+++++++++++++++++++++++++++++++++++++++++++++++++"
#
echo " Now let's try to compile hessio according to Federico's recipe "
cd ./hessioxxx
make clean
make EXTRA_DEFINES="-DCTA_PROD2 -DWITH_LOW_GAIN_CHANNEL"
#
echo " Let's see what files are in the lib directory "
ls -alFsh ./lib
#
if [ ! -f ./lib/libhessio.so ]
then
echo " ERROR: libhessio library not found. Exit "
exit 1
fi
echo "+++++++++++++++++++++++++++++++++++++++++++++++++"
#
cd .. # come back to original dir
#
echo " Now let's try to compile simtel according to Federico's recipe "
cd ./sim_telarray
make clean
make EXTRA_DEFINES="-DCTA_PROD2 -DWITH_LOW_GAIN_CHANNEL"
make install
#
echo " Let's see what files are in the bin directory "
ls -alFsh ./bin
#
if [ ! -x ./bin/sim_telarray ]
then
echo " ERROR: sim_telarray excutable not found. Exit "
exit 1
fi
echo "+++++++++++++++++++++++++++++++++++++++++++++++++"
#
echo " Everything was compiled and linked properly" """)
fd.close()
os.system('chmod u+x run_compile.sh')
cmdTuple = ['./run_compile.sh']
##########################################################################
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute build')
DIRAC.exit(-1)
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
mjfPower = getPowerFromMJF()
if mjfPower:
gLogger.notice( 'CPU power from MJF is %.1f HS06' % mjfPower )
else:
gLogger.notice( 'MJF not available on this node' )
if update and not configFile:
gConfig.setOptionValue( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm )
gConfig.setOptionValue( '/LocalSite/CPUNormalizationFactor', norm )
gConfig.dumpLocalCFGToFile( gConfig.diracConfigFilePath )
if configFile:
from DIRAC.Core.Utilities.CFG import CFG
cfg = CFG()
try:
# Attempt to open the given file
cfg.loadFromFile( configFile )
except:
pass
# Create the section if it does not exist
if not cfg.existsKey( 'LocalSite' ):
cfg.createNewSection( 'LocalSite' )
cfg.setOption( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm )
cfg.setOption( '/LocalSite/CPUNormalizationFactor', norm )
cfg.writeToFile( configFile )
DIRAC.exit()
def platform(arg):
Script.disableCS()
print(DIRAC.getPlatform())
DIRAC.exit(0)
reqClient = ReqClient()
fc = FileCatalog()
for lfnList in breakListIntoChunks( lfns, 100 ):
oRequest = Request()
oRequest.RequestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] )
replicateAndRegister = Operation()
replicateAndRegister.Type = 'ReplicateAndRegister'
replicateAndRegister.TargetSE = targetSE
res = fc.getFileMetadata( lfnList )
if not res['OK']:
print "Can't get file metadata: %s" % res['Message']
DIRAC.exit( 1 )
if res['Value']['Failed']:
print "Could not get the file metadata of the following, so skipping them:"
for fFile in res['Value']['Failed']:
print fFile
lfnMetadata = res['Value']['Successful']
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
rarFile.Size = lfnMetadata[lfn]['Size']
rarFile.Checksum = lfnMetadata[lfn]['Checksum']
rarFile.GUID = lfnMetadata[lfn]['GUID']
rarFile.ChecksumType = 'ADLER32'
replicateAndRegister.addFile( rarFile )
def version(arg):
Script.disableCS()
print(DIRAC.version)
DIRAC.exit(0)
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if len( args ) < 2:
Script.showHelp()
if len( args ) > 2:
print 'Only one LFN SE pair will be considered'
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
lfn = args[0]
seName = args[1]
try:
f = open( lfn, 'r' )
lfns = f.read().splitlines()
f.close()
except:
lfns = [lfn]
for lfn in lfns:
result = dirac.removeReplica( lfn, seName, printOutput = True )
if not result['OK']:
print 'ERROR: ', result['Message']
exitCode = 2
DIRAC.exit( exitCode )
outStr = "%s\n------SRM staging request info--------------" % outStr
for srid in stageRequests.keys():
outStr = "%s\n%s: %s" % ( outStr, 'SRM RequestID'.ljust( 8 ), stageRequests[srid]['RequestID'].ljust( 100 ) )
outStr = "%s\n%s: %s" % ( outStr, 'SRM StageStatus'.ljust( 8 ), stageRequests[srid]['StageStatus'].ljust( 100 ) )
outStr = "%s\n%s: %s" % ( outStr, 'SRM StageRequestSubmitTime'.ljust( 8 ), str(stageRequests[srid]['StageRequestSubmitTime']).ljust( 100 ) )
outStr = "%s\n%s: %s" % ( outStr, 'SRM StageRequestCompletedTime'.ljust( 8 ), str(stageRequests[srid]['StageRequestCompletedTime']).ljust( 100 ) )
outStr = "%s\n%s: %s" % ( outStr, 'SRM PinExpiryTime'.ljust( 8 ), str(stageRequests[srid]['PinExpiryTime']).ljust( 100 ) )
outStr = "%s\n%s: %s sec" % ( outStr, 'SRM PinLength'.ljust( 8 ), str(stageRequests[srid]['PinLength']).ljust( 100 ) )
else:
outStr = '%s\nThere are no staging requests submitted to the site yet.'.ljust( 8) % outStr
else:
outStr = "\nThere is no such file requested for staging. Check for typo's!"
#Script.showHelp()
print outStr
DIRAC.exit( 0 )
''' Example:
dirac-stager-monitor-file.py /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056603_1.full.dst GRIDKA-RDST
--------------------
LFN : /lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056603_1.full.dst
SE : GRIDKA-RDST
PFN : srm://gridka-dCache.fzk.de/pnfs/gridka.de/lhcb/LHCb/Collision12/FULL.DST/00020846/0005/00020846_00056603_1.full.dst
Status : StageSubmitted
LastUpdate: 2013-06-11 18:13:40
Reason : None
Jobs requesting this file to be staged: 48518896
------SRM staging request info--------------
SRM RequestID: -1768636375
SRM StageStatus: StageSubmitted
SRM StageRequestSubmitTime: 2013-06-11 18:13:38
def CorsikaSimtelExample(args=None):
from CTADIRAC.Interfaces.API.CorsikaSimtelJob import CorsikaSimtelJob
from DIRAC.Interfaces.API.Dirac import Dirac
j = CorsikaSimtelJob()
j.setVersion('prod-2_15122013')
j.setExecutable('corsika_autoinputs')
if len(args) != 4:
Script.showHelp()
if args[3] not in [
'STD', '4MSST', 'SCSST', 'ASTRI', 'NSBX3', 'NORTH', 'SCMST'
]:
Script.gLogger.notice("reprocessing configuration is incorrect:",
args[3])
DIRAC.exit(-1)
runMin = int(args[0])
runMax = int(args[1])
cfgfile = args[2]
simtelArrayConfig = args[3]
ilist = []
for i in range(runMin, runMax + 1):
run_number = '%06d' % i
ilist.append(run_number)
j.setGenericParametricInput(ilist)
j.setName('run%s')
j.setInputSandbox([
cfgfile, 'grid_prod2-repro.sh',
'LFN:/vo.cta.in2p3.fr/MC/PROD2/SVN-PROD2.tar.gz'
])
j.setParameters([
'--template', cfgfile, '--mode', 'corsika_simtel', '-S',
simtelArrayConfig
])
j.setOutputSandbox(
['corsika_autoinputs.log', 'simtel.log', 'applicationLog.txt'])
# Retrieve your Output Data
corsika_out = '*.corsika.gz'
corsikatar_out = '*.corsika.tar.gz'
sim_out = '*.simtel.gz'
log_out = '*.log.gz'
hist_out = '*.hdata.gz'
j.setOutputData([corsika_out, corsikatar_out, sim_out, log_out, hist_out])
j.setCPUTime(200000)
Script.gLogger.info(j._toJDL())
res = Dirac().submit(j)
print res
def setUserDN(arg):
global userDN
if userDN or not arg:
Script.showHelp()
DIRAC.exit(-1)
userDN = arg
gLogger.notice(" + %s" % host)
gLogger.notice("Succeeded to restart:")
for host in restartSuccess:
gLogger.notice(" + %s" % host)
gLogger.notice("Failed to update:")
for host in updateFail:
gLogger.notice(" - %s" % host)
gLogger.notice("Failed to restart:")
for host in restartFail:
gLogger.notice(" - %s" % host)
gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
if retry > 0:
retryHosts = list(set(updateFail + restartFail))
gLogger.notice("Retrying update on (%s attempts remaining):" % retry)
for host in retryHosts:
gLogger.notice(" - %s" % host)
gLogger.notice("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
return updateInstance(version, retryHosts, retry - 1)
return S_ERROR("Update failed!")
if __name__ == "__main__":
result = updateInstance(version, hosts, retry)
if not result['OK']:
gLogger.fatal(result['Message'])
DIRAC.exit(1)
gLogger.notice(result['Value'])
DIRAC.exit(0)
def setUserName(arg):
global userName
if userName or not arg:
Script.showHelp()
DIRAC.exit(-1)
userName = arg
DIRAC.gLogger.notice('Checking:', package)
packageTuple = package.split('/')
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
continue
if installSoftwarePackage(package, sharedArea())['OK']:
if not os.path.isdir(
os.path.join(sharedArea(), packageTuple[0],
packageTuple[1])):
DIRAC.gLogger.error(
'Software package missing in the shared area')
DIRAC.exit(-1)
continue
DIRAC.gLogger.error('Software package not correctly installed')
DIRAC.exit(-1)
DIRAC.exit()
if __name__ == '__main__':
try:
main()
except Exception:
DIRAC.gLogger.exception()
DIRAC.exit(-1)
mjfPower = getPowerFromMJF()
if mjfPower:
gLogger.notice('CPU power from MJF is %.1f HS06' % mjfPower)
else:
gLogger.notice('MJF not available on this node')
if update and not configFile:
gConfig.setOptionValue('/LocalSite/CPUScalingFactor',
mjfPower if mjfPower else norm)
gConfig.setOptionValue('/LocalSite/CPUNormalizationFactor', norm)
gConfig.dumpLocalCFGToFile(gConfig.diracConfigFilePath)
if configFile:
from DIRAC.Core.Utilities.CFG import CFG
cfg = CFG()
try:
# Attempt to open the given file
cfg.loadFromFile(configFile)
except:
pass
# Create the section if it does not exist
if not cfg.existsKey('LocalSite'):
cfg.createNewSection('LocalSite')
cfg.setOption('/LocalSite/CPUScalingFactor',
mjfPower if mjfPower else norm)
cfg.setOption('/LocalSite/CPUNormalizationFactor', norm)
cfg.writeToFile(configFile)
DIRAC.exit()
def install_CorsikaSimtelPack(version):
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
packs = [CorsikaSimtelPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
installSoftwareEnviron(package, workingArea())
packageTuple = package.split('/')
corsika_subdir = sharedArea(
) + '/' + packageTuple[0] + '/' + version
cmd = 'cp -u -r ' + corsika_subdir + '/* .'
os.system(cmd)
continue
if workingArea:
if checkSoftwarePackage(package, workingArea())['OK']:
DIRAC.gLogger.notice('Package found in Local Area:', package)
continue
if installSoftwarePackage(package, workingArea())['OK']:
############## compile #############################
if version == 'prod-2_22072013_tox':
############### compile tox ################
fd = open('run_compile.sh', 'w')
fd.write("""#! /bin/sh
./build_all prod2 qgs2
# If the code was already there, we just clean but do not remove it.
if [ -d "hessioxxx" ]; then
(cd hessioxxx && make clean && make EXTRA_DEFINES='-DH_MAX_TEL=55 -DH_MAX_PIX=1984 -DH_MAX_SECTORS=13100 -DNO_LOW_GAIN')
fi
# If the code was already there, we just clean but do not remove it.
if [ -d "sim_telarray" ]; then
(cd sim_telarray && make clean && make EXTRA_DEFINES='-DH_MAX_TEL=55 -DH_MAX_PIX=1984 -DH_MAX_SECTORS=13100 -DNO_LOW_GAIN' install)
fi""")
fd.close()
os.system('chmod u+x run_compile.sh')
cmdTuple = ['./run_compile.sh']
##########################################
else:
cmdTuple = ['./build_all', 'prod2', 'qgs2']
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute build')
DIRAC.exit(-1)
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
outputDir = value
elif sw.lower() in ( 'f', 'file' ):
if os.path.exists( value ):
jFile = open( value )
jobs += jFile.read().split()
jFile.close()
elif sw.lower() in ( 'g', 'jobgroup' ):
group = value
jobDate = toString( date() - 30 * day )
# Choose jobs in final state, no more than 30 days old
result = dirac.selectJobs( jobGroup = value, date = jobDate, status = 'Done' )
if not result['OK']:
if not "No jobs selected" in result['Message']:
print "Error:", result['Message']
DIRAC.exit( -1 )
else:
jobs += result['Value']
result = dirac.selectJobs( jobGroup = value, date = jobDate, status = 'Failed' )
if not result['OK']:
if not "No jobs selected" in result['Message']:
print "Error:", result['Message']
DIRAC.exit( -1 )
else:
jobs += result['Value']
for arg in parseArguments( args ):
if os.path.isdir( arg ):
print "Output for job %s already retrieved, remove the output directory to redownload" % arg
else:
jobs.append( arg )
def main():
days = 0
months = 0
years = 0
wildcard = None
baseDir = ""
emptyDirsFlag = False
Script.registerSwitch("D:", "Days=",
"Match files older than number of days [%s]" % days)
Script.registerSwitch(
"M:", "Months=",
"Match files older than number of months [%s]" % months)
Script.registerSwitch(
"Y:", "Years=", "Match files older than number of years [%s]" % years)
Script.registerSwitch("w:", "Wildcard=",
"Wildcard for matching filenames [All]")
Script.registerSwitch(
"b:", "BaseDir=",
"Base directory to begin search (default /[vo]/user/[initial]/[username])"
)
Script.registerSwitch("e", "EmptyDirs",
"Create a list of empty directories")
Script.parseCommandLine(ignoreErrors=False)
for switch in Script.getUnprocessedSwitches():
if switch[0] == "D" or switch[0].lower() == "days":
days = int(switch[1])
if switch[0] == "M" or switch[0].lower() == "months":
months = int(switch[1])
if switch[0] == "Y" or switch[0].lower() == "years":
years = int(switch[1])
if switch[0].lower() == "w" or switch[0].lower() == "wildcard":
wildcard = "*" + switch[1]
if switch[0].lower() == "b" or switch[0].lower() == "basedir":
baseDir = switch[1]
if switch[0].lower() == "e" or switch[0].lower() == "emptydirs":
emptyDirsFlag = True
import DIRAC
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from datetime import datetime, timedelta
import sys
import os
import time
import fnmatch
fc = FileCatalog()
def isOlderThan(cTimeStruct, days):
timeDelta = timedelta(days=days)
maxCTime = datetime.utcnow() - timeDelta
if cTimeStruct < maxCTime:
return True
return False
withMetadata = False
if days or months or years:
withMetadata = True
totalDays = 0
if years:
totalDays += 365 * years
if months:
totalDays += 30 * months
if days:
totalDays += days
res = getProxyInfo(False, False)
if not res["OK"]:
gLogger.error("Failed to get client proxy information.",
res["Message"])
DIRAC.exit(2)
proxyInfo = res["Value"]
if proxyInfo["secondsLeft"] == 0:
gLogger.error("Proxy expired")
DIRAC.exit(2)
username = proxyInfo["username"]
vo = ""
if "group" in proxyInfo:
vo = getVOForGroup(proxyInfo["group"])
if not baseDir:
if not vo:
gLogger.error("Could not determine VO")
Script.showHelp()
baseDir = "/%s/user/%s/%s" % (vo, username[0], username)
baseDir = baseDir.rstrip("/")
gLogger.notice("Will search for files in %s%s" %
(baseDir, (" matching %s" % wildcard) if wildcard else ""))
activeDirs = [baseDir]
allFiles = []
emptyDirs = []
while len(activeDirs) > 0:
currentDir = activeDirs.pop()
res = fc.listDirectory(currentDir, withMetadata, timeout=360)
if not res["OK"]:
gLogger.error("Error retrieving directory contents",
"%s %s" % (currentDir, res["Message"]))
elif currentDir in res["Value"]["Failed"]:
gLogger.error(
"Error retrieving directory contents",
"%s %s" % (currentDir, res["Value"]["Failed"][currentDir]))
else:
dirContents = res["Value"]["Successful"][currentDir]
subdirs = dirContents["SubDirs"]
files = dirContents["Files"]
if not subdirs and not files:
emptyDirs.append(currentDir)
gLogger.notice("%s: empty directory" % currentDir)
else:
for subdir in sorted(subdirs, reverse=True):
if (not withMetadata) or isOlderThan(
subdirs[subdir]["CreationDate"], totalDays):
activeDirs.append(subdir)
for filename in sorted(files):
fileOK = False
if (not withMetadata) or isOlderThan(
files[filename]["MetaData"]["CreationDate"],
totalDays):
if wildcard is None or fnmatch.fnmatch(
filename, wildcard):
fileOK = True
if not fileOK:
files.pop(filename)
allFiles += sorted(files)
if len(files) or len(subdirs):
gLogger.notice(
"%s: %d files%s, %d sub-directories" %
(currentDir, len(files), " matching"
if withMetadata or wildcard else "", len(subdirs)))
outputFileName = "%s.lfns" % baseDir.replace("/%s" % vo,
"%s" % vo).replace("/", "-")
outputFile = open(outputFileName, "w")
for lfn in sorted(allFiles):
outputFile.write(lfn + "\n")
outputFile.close()
gLogger.notice("%d matched files have been put in %s" %
(len(allFiles), outputFileName))
if emptyDirsFlag:
outputFileName = "%s.emptydirs" % baseDir.replace(
"/%s" % vo, "%s" % vo).replace("/", "-")
outputFile = open(outputFileName, "w")
for dir in sorted(emptyDirs):
outputFile.write(dir + "\n")
outputFile.close()
gLogger.notice("%d empty directories have been put in %s" %
(len(emptyDirs), outputFileName))
DIRAC.exit(0)
# # execution
if __name__ == "__main__":
from DIRAC.Core.Base.Script import parseCommandLine
parseCommandLine()
import DIRAC
from DIRAC import gLogger
args = Script.getPositionalArgs()
requestName = None
targetSEs = None
if len( args ) < 3:
Script.showHelp()
DIRAC.exit( 1 )
requestName = args[0]
lfnList = getLFNList( args[1] )
targetSEs = list( set( [ se for targetSE in args[2:] for se in targetSE.split( ',' ) ] ) )
gLogger.info( "Will create request '%s' with 'ReplicateAndRegister' "\
"operation using %s lfns and %s target SEs" % ( requestName, len( lfnList ), len( targetSEs ) ) )
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.Core.Utilities.List import breakListIntoChunks
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch('', 'Job=', ' JobID[,jobID2,...]')
Script.registerSwitch('', 'Transformation=', ' transformation ID')
Script.registerSwitch(
'', 'Tasks=', ' Associated to --Transformation, list of taskIDs')
Script.registerSwitch('', 'Verbose', ' Print more information')
Script.registerSwitch('', 'Terse', ' Only print request status')
Script.registerSwitch('', 'Full', ' Print full request content')
Script.registerSwitch('', 'Status=',
' Select all requests in a given status')
Script.registerSwitch(
'', 'Since=',
' Associated to --Status, start date yyyy-mm-dd or nb of days (default= -one day'
)
Script.registerSwitch(
'', 'Until=', ' Associated to --Status, end date (default= now')
Script.registerSwitch(
'', 'Maximum=',
' Associated to --Status, max number of requests ')
Script.registerSwitch('', 'Reset',
' Reset Failed files to Waiting if any')
Script.registerSwitch('', 'Force', ' Force reset even if not Failed')
Script.registerSwitch(
'', 'All',
' (if --Status Failed) all requests, otherwise exclude irrecoverable failures'
)
Script.registerSwitch('', 'FixJob',
' Set job Done if the request is Done')
Script.registerSwitch('', 'Cancel', ' Cancel the request')
Script.registerSwitch('', 'ListJobs', ' List the corresponding jobs')
Script.registerSwitch(
'', 'TargetSE=', ' Select request only if that SE is in the targetSEs')
from DIRAC.Core.Base.Script import parseCommandLine
parseCommandLine()
import DIRAC
from DIRAC import gLogger
jobs = []
requestID = 0
transID = None
taskIDs = None
tasks = None
requests = []
full = False
verbose = False
status = None
until = None
since = None
terse = False
allR = False
reset = False
fixJob = False
maxRequests = 999999999999
cancel = False
listJobs = False
force = False
targetSE = set()
for switch in Script.getUnprocessedSwitches():
if switch[0] == 'Job':
jobs = []
job = "Unknown"
try:
for arg in switch[1].split(','):
if os.path.exists(arg):
with open(arg, 'r') as fp:
lines = fp.readlines()
for line in lines:
for job in line.split(','):
jobs += [int(job.strip())]
gLogger.notice("Found %d jobs in file %s" %
(len(jobs), arg))
else:
jobs.append(int(arg))
except TypeError:
gLogger.fatal("Invalid jobID", job)
elif switch[0] == 'Transformation':
try:
transID = int(switch[1])
except Exception:
gLogger.fatal('Invalid transID', switch[1])
elif switch[0] == 'Tasks':
try:
taskIDs = [int(task) for task in switch[1].split(',')]
except Exception:
gLogger.fatal('Invalid tasks', switch[1])
elif switch[0] == 'Full':
full = True
elif switch[0] == 'Verbose':
verbose = True
elif switch[0] == 'Terse':
terse = True
elif switch[0] == 'All':
allR = True
elif switch[0] == 'Reset':
reset = True
elif switch[0] == 'Force':
force = True
elif switch[0] == 'Status':
status = switch[1].capitalize()
elif switch[0] == 'Since':
since = convertDate(switch[1])
elif switch[0] == 'Until':
until = convertDate(switch[1])
elif switch[0] == 'FixJob':
fixJob = True
elif switch[0] == 'Cancel':
cancel = True
elif switch[0] == 'ListJobs':
listJobs = True
elif switch[0] == 'Maximum':
try:
maxRequests = int(switch[1])
except Exception:
pass
elif switch[0] == 'TargetSE':
targetSE = set(switch[1].split(','))
if reset and not force:
status = 'Failed'
if fixJob:
status = 'Done'
if terse:
verbose = True
if status:
if not until:
until = datetime.datetime.utcnow()
if not since:
since = until - datetime.timedelta(hours=24)
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.RequestManagementSystem.Client.ReqClient import printRequest, recoverableRequest
reqClient = ReqClient()
if transID:
if not taskIDs:
gLogger.fatal(
"If Transformation is set, a list of Tasks should also be set")
Script.showHelp(exitCode=2)
# In principle, the task name is unique, so the request name should be unique as well
# If ever this would not work anymore, we would need to use the transformationClient
# to fetch the ExternalID
requests = ['%08d_%08d' % (transID, task) for task in taskIDs]
allR = True
elif not jobs:
requests = []
# Get full list of arguments, with and without comma
for arg in [
x.strip() for arg in Script.getPositionalArgs()
for x in arg.split(',')
]:
if os.path.exists(arg):
lines = open(arg, 'r').readlines()
requests += [
reqID.strip() for line in lines
for reqID in line.split(',')
]
gLogger.notice("Found %d requests in file" % len(requests))
else:
requests.append(arg)
allR = True
else:
res = reqClient.getRequestIDsForJobs(jobs)
if not res['OK']:
gLogger.fatal("Error getting request for jobs", res['Message'])
DIRAC.exit(2)
if res['Value']['Failed']:
gLogger.error(
"No request found for jobs %s" %
','.join(sorted(str(job) for job in res['Value']['Failed'])))
requests = sorted(res['Value']['Successful'].values())
if requests:
allR = True
else:
DIRAC.exit(0)
if status and not requests:
allR = allR or status != 'Failed'
res = reqClient.getRequestIDsList([status],
limit=maxRequests,
since=since,
until=until)
if not res['OK']:
gLogger.error("Error getting requests:", res['Message'])
DIRAC.exit(2)
requests = [
reqID for reqID, _st, updTime in res['Value']
if updTime > since and updTime <= until and reqID
]
gLogger.notice('Obtained %d requests %s between %s and %s' %
(len(requests), status, since, until))
if not requests:
gLogger.notice('No request selected....')
Script.showHelp(exitCode=2)
okRequests = []
warningPrinted = False
jobIDList = []
for reqID in requests:
# We allow reqID to be the requestName if it is unique
try:
requestID = int(reqID)
except ValueError:
requestID = reqClient.getRequestIDForName(reqID)
if not requestID['OK']:
gLogger.notice(requestID['Message'])
continue
requestID = requestID['Value']
request = reqClient.peekRequest(requestID)
if not request["OK"]:
gLogger.error(request["Message"])
DIRAC.exit(-1)
request = request["Value"]
if not request:
gLogger.error("no such request %s" % requestID)
continue
# If no operation as the targetSE, skip
if targetSE:
found = False
for op in request:
if op.TargetSE and targetSE.intersection(
op.TargetSE.split(',')):
found = True
break
if not found:
continue
# keep a list of jobIDs if requested
if request.JobID and listJobs:
jobIDList.append(request.JobID)
if status and request.Status != status:
gLogger.notice(
"Request %s is not in requested status %s%s" %
(reqID, status, ' (cannot be reset)' if reset else ''))
continue
if fixJob and request.Status == 'Done' and request.JobID:
# The request is for a job and is Done, verify that the job is in the proper status
result = reqClient.finalizeRequest(request.RequestID,
request.JobID,
useCertificates=False)
if not result['OK']:
gLogger.error("Error finalizing job", result['Message'])
else:
gLogger.notice("Job %d updated to %s" %
(request.JobID, result['Value']))
continue
if cancel:
if request.Status not in ('Done', 'Failed'):
ret = reqClient.cancelRequest(requestID)
if not ret['OK']:
gLogger.error("Error canceling request %s" % reqID,
ret['Message'])
else:
gLogger.notice("Request %s cancelled" % reqID)
else:
gLogger.notice("Request %s is in status %s, not cancelled" %
(reqID, request.Status))
elif allR or recoverableRequest(request):
okRequests.append(str(requestID))
if reset:
gLogger.notice('============ Request %s =============' %
requestID)
ret = reqClient.resetFailedRequest(requestID, allR=allR)
if not ret['OK']:
gLogger.error("Error resetting request %s" % requestID,
ret['Message'])
else:
if len(requests) > 1:
gLogger.notice('\n===================================')
dbStatus = reqClient.getRequestStatus(requestID).get(
'Value', 'Unknown')
printRequest(request,
status=dbStatus,
full=full,
verbose=verbose,
terse=terse)
if listJobs:
gLogger.notice("List of %d jobs:\n" % len(jobIDList),
','.join(str(jobID) for jobID in jobIDList))
if status and okRequests:
from DIRAC.Core.Utilities.List import breakListIntoChunks
gLogger.notice('\nList of %d selected requests:' % len(okRequests))
for reqs in breakListIntoChunks(okRequests, 100):
gLogger.notice(','.join(reqs))
""" % Script.scriptName) Script.parseCommandLine() import DIRAC from DIRAC import gConfig,gLogger from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.Core.Utilities.List import sortList storageCFGBase = "/Resources/StorageElements" res = gConfig.getSections( storageCFGBase, True ) if not res[ 'OK' ]: gLogger.error( 'Failed to get storage element info' ) gLogger.error( res[ 'Message' ] ) DIRAC.exit( -1 ) gLogger.info( "%s %s %s" % ( 'Storage Element'.ljust( 25 ), 'Read Status'.rjust( 15 ), 'Write Status'.rjust( 15 ) ) ) seList = sortList( res[ 'Value' ] ) resourceStatus = ResourceStatus() res = resourceStatus.getStorageElementStatus( seList ) if not res[ 'OK' ]: gLogger.error( "Failed to get StorageElement status for %s" % str( seList ) ) for k,v in res[ 'Value' ].items(): readState, writeState = 'Active', 'Active'
Script.parseCommandLine(ignoreErrors=True)
runRanges = []
for arg in Script.getPositionalArgs():
runRanges += arg.split(',')
runSet = set()
for run in runRanges:
try:
if ':' in arg:
run1, run2 = run.split(':')
runSet.update(range(int(run1), int(run2) + 1))
else:
runSet.add(int(run))
except (ValueError, IndexError) as e:
gLogger.exception("Invalid run number", arg, lException=e)
DIRAC.exit(1)
production = None
item = None
byValue = False
active = False
for switch in Script.getUnprocessedSwitches():
if switch[0] == 'Production':
try:
production = [int(prod) for prod in switch[1].split(',')]
except ValueError as e:
gLogger.exception('Bad production ID', lException=e)
DIRAC.exit(1)
elif switch[0] == 'Information':
item = switch[1]
elif switch[0] == 'ByValue':
if not len( args ) == 1:
Script.showHelp()
ce = args[0]
host = None
vo = getVO( 'lhcb' )
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ( "H", "host" ):
host = unprocSw[1]
if unprocSw[0] in ( "V", "vo" ):
vo = unprocSw[1]
diracAdmin = DiracAdmin()
result = diracAdmin.getBDIICEState( ce, useVO = vo, host = host )
if not result['OK']:
print result['Message']
DIRAC.exit( 2 )
ces = result['Value']
for ce in ces:
print "CE: %s {" % ce.get( 'GlueCEUniqueID', 'Unknown' )
for item in ce.iteritems():
print "%s: %s" % item
print "}"
verbose = False
if days or months or years:
verbose = True
totalDays = 0
if years:
totalDays += 365 * years
if months:
totalDays += 30 * months
if days:
totalDays += days
res = getProxyInfo(False, False)
if not res['OK']:
gLogger.error("Failed to get client proxy information.", res['Message'])
DIRAC.exit(2)
proxyInfo = res['Value']
username = proxyInfo['username']
vo = ''
if 'group' in proxyInfo:
vo = getVOForGroup(proxyInfo['group'])
if not baseDir:
if not vo:
gLogger.error('Could not determine VO')
Script.showHelp()
baseDir = '/%s/user/%s/%s' % (vo, username[0], username)
baseDir = baseDir.rstrip('/')
gLogger.info('Will search for files in %s' % baseDir)
activeDirs = [baseDir]
Usage:
%s <lfn | fileContainingLfns> <SE> <status>
""" % Script.scriptName )
Script.parseCommandLine( ignoreErrors = False )
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "u" or switch[0].lower() == "unit":
unit = switch[1]
scaleDict = { 'MB' : 1000 * 1000.0,
'GB' : 1000 * 1000 * 1000.0,
'TB' : 1000 * 1000 * 1000 * 1000.0,
'PB' : 1000 * 1000 * 1000 * 1000 * 1000.0}
if not unit in scaleDict.keys():
gLogger.error( "Unit must be one of MB,GB,TB,PB" )
DIRAC.exit( 2 )
scaleFactor = scaleDict[unit]
args = Script.getPositionalArgs()
lfns = []
for inputFileName in args:
if os.path.exists( inputFileName ):
inputFile = open( inputFileName, 'r' )
string = inputFile.read()
inputFile.close()
lfns.extend( [ lfn.strip() for lfn in string.splitlines() ] )
else:
lfns.append( inputFileName )
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
res = FileCatalog().getFileSize( lfns )
string = inputFile.read()
inputFile.close()
lfns.extend( [ lfn.strip() for lfn in string.splitlines() ] )
else:
lfns.append( inputFileName )
from DIRAC.Core.Utilities.List import breakListIntoChunks
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
errorReasons = {}
successfullyRemoved = 0
for lfnList in breakListIntoChunks( lfns, 100 ):
res = dm.removeFile( lfnList )
if not res['OK']:
gLogger.error( "Failed to remove data", res['Message'] )
DIRAC.exit( -2 )
for lfn, r in res['Value']['Failed'].items():
reason = str( r )
if not reason in errorReasons.keys():
errorReasons[reason] = []
errorReasons[reason].append( lfn )
successfullyRemoved += len( res['Value']['Successful'].keys() )
for reason, lfns in errorReasons.items():
gLogger.notice( "Failed to remove %d files with error: %s" % ( len( lfns ), reason ) )
if successfullyRemoved > 0:
gLogger.notice( "Successfully removed %d files" % successfullyRemoved )
DIRAC.exit( 0 )
