def execute(self):
"""
The method called by the Workflow framework
"""
from DIRAC.Core.Utilities.Subprocess import systemCall
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
ret = self.__checkInputs()
if not ret['OK']:
return ret
ret = getSoftwareEnviron(self.softwarePackage)
if not ret['OK']:
error = ret['Message']
self.log.error(error, self.softwarePackage)
return DIRAC.S_ERROR(' '.join([error, str(self.softwarePackage)]))
envdispEnviron = ret['Value']
cmdTuple = [self.edExe]
cmdTuple.extend(self.edArguments)
self.log.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, self.sendOutput, env=envdispEnviron)
if not ret['OK']:
self.log.error('Failed to execute evndisp:', ret['Message'])
return DIRAC.S_ERROR('Can not execute evndisp')
status, stdout, stderr = ret['Value']
self.log.notice('evndisp status is:', status)
return status
def execute(self):
"""
The method called by the Workflow framework
"""
from DIRAC.Core.Utilities.Subprocess import systemCall
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
ret = self.__checkInputs()
if not ret['OK']:
return ret
ret = getSoftwareEnviron(self.softwarePackage)
if not ret['OK']:
error = ret['Message']
self.log.error(error, self.softwarePackage)
return DIRAC.S_ERROR(' '.join([error, str(self.softwarePackage)]))
hapEnviron = ret['Value']
hessroot = hapEnviron['HESSROOT']
rootlogon_file = hessroot + '/rootlogon.C'
cp_cmd = 'cp ' + rootlogon_file + ' .'
os.system(cp_cmd)
fileName = hessroot + self.rootMacro
if fileName[-1] == '+':
# If the macro has to be compiled there is an extra "+" at the end of its name
fileName = fileName[:-1]
if not os.path.isfile(fileName):
error = 'Hap Root macro file does not exist:'
self.log.error(error, fileName)
return DIRAC.S_ERROR(' '.join([error, fileName]))
fileName = hessroot + self.rootMacro
cmdTuple = ['root', '-b', '-q']
cmdTuple += [
'%s( %s )' %
(fileName, ', '.join(self.rootArguments).replace("'", '"'))
]
self.log.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, self.sendOutput, env=hapEnviron)
if not ret['OK']:
self.log.error('Failed to execute Root:', ret['Message'])
return DIRAC.S_ERROR('Can not execute Hap root macro')
return DIRAC.S_OK()
def execute( self ):
"""
The method called by the Workflow framework
"""
from DIRAC.Core.Utilities.Subprocess import systemCall
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
ret = self.__checkInputs()
if not ret['OK']:
return ret
ret = getSoftwareEnviron( self.softwarePackage )
if not ret['OK']:
error = ret['Message']
self.log.error( error, self.softwarePackage )
return DIRAC.S_ERROR( ' '.join( [ error, str( self.softwarePackage ) ] ) )
rootEnviron = ret['Value']
fileName = self.rootMacro
if fileName[-1] == '+':
# If the macro has to be compiled there is an extra "+" at the end of its name
fileName = fileName[:-1]
if not os.path.isfile( fileName ):
error = 'Root macro file does not exist:'
self.log.error( error, fileName )
return DIRAC.S_ERROR( ' '.join( [ error, fileName ] ) )
cmdTuple = ['root', '-b', '-q']
cmdTuple += ['%s( %s )' % ( self.rootMacro, ', '.join( self.rootArguments ).replace( "'", '"' ) ) ]
self.log.info( 'Executing command tuple:', cmdTuple )
ret = systemCall( 0, cmdTuple, env = rootEnviron )
if not ret['OK']:
self.log.error( 'Failed to execute Root:', ret['Message'] )
return DIRAC.S_ERROR( 'Can not execute root' )
status, stdout, stderr = ret['Value']
if status:
self.log.error( 'Root execution reports Error:', status )
self.log.error( stdout )
self.log.error( stderr )
return DIRAC.S_ERROR( 'Failed root Execution' )
self.log.info( 'Root stdout:' )
self.log.info( stdout )
return DIRAC.S_OK()
def execute(self):
"""
The method called by the Workflow framework
"""
from DIRAC.Core.Utilities.Subprocess import systemCall
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
ret = self.__checkInputs()
if not ret['OK']:
return ret
ret = getSoftwareEnviron(self.softwarePackage)
if not ret['OK']:
error = ret['Message']
self.log.error(error, self.softwarePackage)
return DIRAC.S_ERROR(' '.join([error, str(self.softwarePackage)]))
hapEnviron = ret['Value']
cmdTuple = ['eventio_cta']
cmdTuple.extend(self.hapArguments)
self.log.info('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, self.sendOutput, env=hapEnviron)
if not ret['OK']:
self.log.error('Failed to execute hap:', ret['Message'])
return DIRAC.S_ERROR('Can not execute hap')
status, stdout, stderr = ret['Value']
if status:
self.log.error('Hap execution reports Error:', status)
self.log.error(stdout)
self.log.error(stderr)
return DIRAC.S_ERROR('Failed hap Execution')
self.log.info('Hap stdout:')
self.log.info(stdout)
return DIRAC.S_OK()
def main():
from DIRAC.Core.Base import Script
#### eventio_cta options ##########################################
Script.registerSwitch("T:", "tellist=", "Tellist", setTellist)
Script.registerSwitch("F:", "Nfirst_mcevt=", "Nfirst_mcevt",
setNfirst_mcevt)
Script.registerSwitch("L:", "Nlast_mcevt=", "Nlast_mcevt", setNlast_mcevt)
## add other eventio_cta options ################################
# Script.registerSwitch( "N:", "num=", "Num", setNum)
## Script.registerSwitch( "L:", "limitmc=", "Limitmc", setLimitmc)
# Script.registerSwitch( "S:", "telidoffset=", "Telidoffset", setTelidoffset)
Script.registerSwitch("P:", "pixelslices=", "setPixelslices (true/false)",
setPixelslices)
Script.registerSwitch("p:", "run_number=",
"Run Number (set automatically)", setRunNumber)
### other options ###############################################
Script.registerSwitch("V:", "version=", "HAP version", setVersion)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
if tellist == None or version == None:
Script.showHelp()
jobReport.setApplicationStatus('Options badly specified')
DIRAC.exit(-1)
from CTADIRAC.Core.Workflow.Modules.HapApplication import HapApplication
from CTADIRAC.Core.Workflow.Modules.HapRootMacro import HapRootMacro
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
HapPack = 'HAP/' + version + '/HAP'
packs = ['HESS/v0.2/lib', 'HESS/v0.3/root', HapPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
continue
if localArea:
if checkSoftwarePackage(package, localArea())['OK']:
DIRAC.gLogger.notice('Package found in Local Area:', package)
continue
if installSoftwarePackage(package, localArea())['OK']:
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
telconf = os.path.join(localArea(),
'HAP/%s/config/%s' % (version, tellist))
ha = HapApplication()
ha.setSoftwarePackage(HapPack)
ha.hapExecutable = 'eventio_cta'
fileout = 'raw_' + part_type + '_run' + run_number + '.root'
infile = build_infile()
ha.hapArguments = ['-file', infile, '-o', fileout, '-tellist', telconf]
try:
ha.hapArguments.extend(
['-Nfirst_mcevt', Nfirst_mcevt, '-Nlast_mcevt', Nlast_mcevt])
except NameError:
DIRAC.gLogger.info('Nfirst_mcevt/Nlast_mcevt options are not used')
try:
if (pixelslices == 'true'):
ha.hapArguments.extend(['-pixelslices'])
except NameError:
DIRAC.gLogger.info('pixelslices option is not used')
DIRAC.gLogger.notice('Executing Hap Converter Application')
res = ha.execute()
if not res['OK']:
DIRAC.gLogger.error('Failed to execute eventio_cta Application')
jobReport.setApplicationStatus('eventio_cta: Failed')
DIRAC.exit(-1)
if not os.path.isfile(fileout):
error = 'raw file was not created:'
DIRAC.gLogger.error(error, fileout)
jobReport.setApplicationStatus('eventio_cta: RawData not created')
DIRAC.exit(-1)
###################### Check RAW DATA: step0 #######################
hr = HapRootMacro()
hr.setSoftwarePackage(HapPack)
DIRAC.gLogger.notice('Executing RAW check step0')
hr.rootMacro = '/hapscripts/dst/Open_Raw.C+'
outfilestr = '"' + fileout + '"'
args = [outfilestr]
DIRAC.gLogger.notice('Open_Raw macro Arguments:', args)
hr.rootArguments = args
DIRAC.gLogger.notice('Executing Hap Open_Raw macro')
res = hr.execute()
if not res['OK']:
DIRAC.gLogger.error('Open_Raw: Failed')
DIRAC.exit(-1)
#####################Check RAW DATA: step1 ##################
DIRAC.gLogger.notice('Executing Raw Check step1')
ret = getSoftwareEnviron(HapPack)
if not ret['OK']:
error = ret['Message']
DIRAC.gLogger.error(error, HapPack)
DIRAC.exit(-1)
hapEnviron = ret['Value']
hessroot = hapEnviron['HESSROOT']
check_script = hessroot + '/hapscripts/dst/check_raw.csh'
cmdTuple = [check_script]
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute RAW Check step1')
jobReport.setApplicationStatus('Check_raw: Failed')
DIRAC.exit(-1)
status, stdout, stderr = ret['Value']
if status == 1:
jobReport.setApplicationStatus(
'RAW Check step1: Big problem during RAW production')
DIRAC.gLogger.error('Check_raw: Big problem during RAW production')
DIRAC.exit(-1)
DIRAC.exit()
def main():
from DIRAC.Core.Base import Script
### make_CTA_DST options ###############################################
Script.registerSwitch("T:", "tellist=", "Tellist", setTellist)
Script.registerSwitch("N:", "nevent=", "Nevent", setNevent)
Script.registerSwitch("p:", "run_number=",
"Run Number (set automatically)", setRunNumber)
### other options ###############################################
Script.registerSwitch("V:", "version=", "HAP version", setVersion)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
if tellist == None or version == None:
Script.showHelp()
jobReport.setApplicationStatus('Options badly specified')
DIRAC.exit(-1)
from CTADIRAC.Core.Workflow.Modules.HapRootMacro import HapRootMacro
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
HapPack = 'HAP/' + version + '/HAP'
packs = ['HESS/v0.2/lib', 'HESS/v0.3/root', HapPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
continue
if localArea:
if checkSoftwarePackage(package, localArea())['OK']:
DIRAC.gLogger.notice('Package found in Local Area:', package)
continue
if installSoftwarePackage(package, localArea())['OK']:
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
hr = HapRootMacro()
hr.setSoftwarePackage(HapPack)
telconf = os.path.join(localArea(),
'HAP/%s/config/%s' % (version, tellist))
infile = build_infile()
infilestr = '"' + infile + '"'
telconfstr = '"' + telconf + '"'
args = [str(int(run_number)), infilestr, telconfstr]
try:
args.extend([nevent])
except NameError:
DIRAC.gLogger.info('nevent arg not used')
DIRAC.gLogger.notice('make_CTA_DST macro Arguments:', args)
hr.rootMacro = '/hapscripts/dst/make_CTA_DST.C+'
hr.rootArguments = args
DIRAC.gLogger.notice('Executing Hap make_CTA_DST macro')
res = hr.execute()
if not res['OK']:
DIRAC.gLogger.error('Failed to execute make_CTA_DST macro')
jobReport.setApplicationStatus('Failure during make_CTA_DST')
DIRAC.exit(-1)
############ check existance of output file ####
filedst = 'dst_CTA_%08d' % int(run_number) + '.root'
if not os.path.isfile(filedst):
DIRAC.gLogger.error('dst file not found:', filedst)
jobReport.setApplicationStatus('make_CTA_DST.C: DST file not created')
DIRAC.exit(-1)
fileout = 'dst_' + part_type + '_run' + run_number + '.root'
cmd = 'mv ' + filedst + ' ' + fileout
os.system(cmd)
###################Check std out #############################
DIRAC.gLogger.notice('Executing DST Check step0')
ret = getSoftwareEnviron(HapPack)
if not ret['OK']:
error = ret['Message']
DIRAC.gLogger.error(error, HapPack)
DIRAC.exit(-1)
hapEnviron = ret['Value']
hessroot = hapEnviron['HESSROOT']
check_script = hessroot + '/hapscripts/dst/check_dst0.csh'
cmdTuple = [check_script]
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute DST Check step0')
jobReport.setApplicationStatus('Check_dst0: Failed')
DIRAC.exit(-1)
status, stdout, stderr = ret['Value']
if status == 1:
jobReport.setApplicationStatus(
'Check_dst0: Big problem during the DST production')
DIRAC.gLogger.error(
'DST Check step0 reports: Big problem during the DST production')
DIRAC.exit(-1)
if status == 2:
jobReport.setApplicationStatus('Check_dst0: No triggered events')
DIRAC.gLogger.notice('DST Check step0 reports: No triggered events')
DIRAC.exit()
############# run the CheckDST macro #################
DIRAC.gLogger.notice('Executing DST check step1')
hr.rootMacro = '/hapscripts/dst/CheckDST.C+'
fileoutstr = '"' + fileout + '"'
args = [fileoutstr]
DIRAC.gLogger.notice('CheckDST macro Arguments:', args)
hr.rootArguments = args
DIRAC.gLogger.notice('Executing Hap CheckDST macro')
res = hr.execute()
if not res['OK']:
DIRAC.gLogger.error('Failure during DST Check step1')
jobReport.setApplicationStatus('Check_dst1: Failed')
DIRAC.exit(-1)
#######################Check std out of CheckDST macro ##########################
DIRAC.gLogger.notice('Executing DST Check step2')
check_script = hessroot + '/hapscripts/dst/check_dst2.csh'
cmdTuple = [check_script]
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute DST Check step2')
jobReport.setApplicationStatus('Check_dst2: Failed')
DIRAC.exit(-1)
status, stdout, stderr = ret['Value']
if status == 1:
jobReport.setApplicationStatus(
'DST Check step2: Big problem during the DST production')
DIRAC.gLogger.error(
'DST Check step2 reports: Big problem during the DST production')
DIRAC.exit(-1)
if status == 2:
jobReport.setApplicationStatus('DST Check step2: No triggered events')
DIRAC.gLogger.notice('DST Check step2 reports: No triggered events')
DIRAC.exit()
DIRAC.exit()
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("T:", "template=", "Corsika Template")
Script.registerSwitch("p:", "run_number=",
"Do not use: Run Number automatically set")
Script.registerSwitch("E:", "executable=",
"Executable (Use SetExecutable)")
Script.registerSwitch("v:", "version=", "Version (Use setVersion)")
Script.registerSwitch("D:", "dcta=", "dcta")
Script.registerSwitch("I:", "icta=", "icta")
Script.registerSwitch("C:", "c_cta=", "c_cta")
Script.parseCommandLine(ignoreErrors=False)
## default values ##############
run_number = None
template = None
executable = None
version = None
### set switch values ###
for switch in Script.getUnprocessedSwitches():
if switch[0] == "run_number" or switch[0] == "p":
run_number = switch[1].split('ParametricParameters=')[1]
elif switch[0] == "template" or switch[0] == "T":
template = switch[1]
elif switch[0] == "executable" or switch[0] == "E":
executable = switch[1]
elif switch[0] == "version" or switch[0] == "v":
version = switch[1]
if version == None or executable == None or run_number == None or template == None:
Script.showHelp()
jobReport.setApplicationStatus('Missing options')
DIRAC.exit(-1)
from CTADIRAC.Core.Workflow.Modules.CorsikaApp import CorsikaApp
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
packs = [CorsikaSimtelPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
installSoftwareEnviron(package, workingArea())
packageTuple = package.split('/')
corsika_subdir = sharedArea(
) + '/' + packageTuple[0] + '/' + version
cmd = 'cp -u -r ' + corsika_subdir + '/* .'
os.system(cmd)
continue
if workingArea:
if checkSoftwarePackage(package, workingArea())['OK']:
DIRAC.gLogger.notice('Package found in Local Area:', package)
continue
if installSoftwarePackage(package, workingArea())['OK']:
############## compile #############################
cmdTuple = ['./build_all', 'prod2', 'qgs2']
######### special case for Astri ############################
if version == 'prod-2_08072014_to':
############## compile #############################
fd = open('run_compile.sh', 'w')
fd.write("""#! /bin/sh
source ./build_all prod2 qgs2
#
echo " Let's check that build_all did its work "
ls -alFsh
echo "+++++++++++++++++++++++++++++++++++++++++++++++++"
echo " Let's see what files are in the corsika-run directory "
ls -alFsh ./corsika-run
#
if [ ! -x ./corsika-run/corsika ]
then
echo " ERROR: Corsika executable found. Exit "
exit 1
fi
echo "+++++++++++++++++++++++++++++++++++++++++++++++++"
#
echo " Now let's try to compile hessio according to Federico's recipe "
cd ./hessioxxx
make clean
make EXTRA_DEFINES="-DCTA_PROD2 -DWITH_LOW_GAIN_CHANNEL"
#
echo " Let's see what files are in the lib directory "
ls -alFsh ./lib
#
if [ ! -f ./lib/libhessio.so ]
then
echo " ERROR: libhessio library not found. Exit "
exit 1
fi
echo "+++++++++++++++++++++++++++++++++++++++++++++++++"
#
cd .. # come back to original dir
#
echo " Now let's try to compile simtel according to Federico's recipe "
cd ./sim_telarray
make clean
make EXTRA_DEFINES="-DCTA_PROD2 -DWITH_LOW_GAIN_CHANNEL"
make install
#
echo " Let's see what files are in the bin directory "
ls -alFsh ./bin
#
if [ ! -x ./bin/sim_telarray ]
then
echo " ERROR: sim_telarray excutable not found. Exit "
exit 1
fi
echo "+++++++++++++++++++++++++++++++++++++++++++++++++"
#
echo " Everything was compiled and linked properly" """)
fd.close()
os.system('chmod u+x run_compile.sh')
cmdTuple = ['./run_compile.sh']
##########################################################################
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute build')
DIRAC.exit(-1)
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
###### execute corsika ###############
cs = CorsikaApp()
cs.setSoftwarePackage(CorsikaSimtelPack)
cs.csExe = executable
cs.csArguments = ['--run-number', run_number, '--run', 'corsika', template]
corsikaReturnCode = cs.execute()
if corsikaReturnCode != 0:
DIRAC.gLogger.error('Failed to execute corsika Application')
jobReport.setApplicationStatus('Corsika Application: Failed')
DIRAC.exit(-1)
###### rename corsika file #################################
rundir = 'run' + run_number
corsikaKEYWORDS = ['TELFIL']
dictCorsikaKW = fileToKWDict(template, corsikaKEYWORDS)
corsikafilename = rundir + '/' + dictCorsikaKW['TELFIL'][0]
destcorsikafilename = 'corsika_run' + run_number + '.corsika.gz'
cmd = 'mv ' + corsikafilename + ' ' + destcorsikafilename
os.system(cmd)
### create corsika tar ####################
corsika_tar = 'corsika_run' + run_number + '.tar.gz'
filetar1 = rundir + '/' + 'input'
filetar2 = rundir + '/' + 'DAT' + run_number + '.dbase'
filetar3 = rundir + '/run' + str(int(run_number)) + '.log'
cmdTuple = ['/bin/tar', 'zcf', corsika_tar, filetar1, filetar2, filetar3]
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute tar')
DIRAC.exit(-1)
###### execute sim_telarray ###############
ret = getSoftwareEnviron(CorsikaSimtelPack)
if not ret['OK']:
error = ret['Message']
DIRAC.gLogger.error(error, CorsikaSimtelPack)
DIRAC.exit(-1)
corsikaEnviron = ret['Value']
cmdTuple = ['sim_telarray']
# add input file argument for sim_telarray ###################
inputfile = 'input_file=' + destcorsikafilename
inputfileopt = ['-C', inputfile]
cmdTuple.extend(inputfileopt)
# add output file argument for sim_telarray
destsimtelfilename = 'simtel_run' + run_number + '.simtel.gz'
outputfile = 'output_file=' + destsimtelfilename
outputfileopt = ['-C', outputfile]
cmdTuple.extend(outputfileopt)
# add histo argument for sim_telarray
desthistofilename = 'simtel_run' + run_number + '.hdata.gz'
histofile = 'histogram_file=' + desthistofilename
histofileopt = ['-C', histofile]
cmdTuple.extend(histofileopt)
# add other arguments for sim_telarray specified by user ######
simtelparfile = open('simtel.par', 'r').readlines()
for line in simtelparfile:
for word in line.split():
cmdTuple.append(word)
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendSimtelOutput, env=corsikaEnviron)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute:', simexe)
DIRAC.exit(-1)
DIRAC.exit()
def main():
from DIRAC.Core.Base import Script
### DoCtaIrf options ##########################################################
Script.registerSwitch("A:", "analysis=", "Analysis Type", setAnalysisType)
Script.registerSwitch("C:", "cuts=", "Cuts Config", setCutsConfig)
Script.registerSwitch("R:", "runlist=", "Runlist", setRunlist)
Script.registerSwitch("Z:", "zenith=", "Zenith", setZenith)
Script.registerSwitch("O:", "offset=", "Offset", setOffset)
Script.registerSwitch("M:", "energy=", "Energy Method", setEnergyMethod)
Script.registerSwitch("T:", "arrayconfig=", "Array Configuration",
setArrayConfig)
Script.registerSwitch("P:", "particle=", "Particle Type", setParticleType)
## other options
Script.registerSwitch("V:", "version=", "HAP version", setVersion)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp()
from CTADIRAC.Core.Workflow.Modules.HapApplication import HapApplication
from CTADIRAC.Core.Workflow.Modules.HapRootMacro import HapRootMacro
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
ha = HapApplication()
HapPack = 'HAP/' + version + '/HAP'
packs = ['HESS/v0.2/lib', 'HESS/v0.3/root', HapPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
continue
if localArea:
if checkSoftwarePackage(package, localArea())['OK']:
DIRAC.gLogger.notice('Package found in Local Area:', package)
continue
if installSoftwarePackage(package, localArea())['OK']:
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
ha.setSoftwarePackage(HapPack)
ha.hapExecutable = 'DoCtaIrf'
runlistdir = os.environ['PWD']
build_infile(runlist)
ha.hapArguments = [
analysistype, cutsconfig, runlistdir, runlist, zenith, offset,
arrayconfig, energymethod, particle
]
DIRAC.gLogger.notice('Executing Hap Application')
res = ha.execute()
if not res['OK']:
DIRAC.gLogger.error('Failed to execute Hap Application')
jobReport.setApplicationStatus('Hap Application: Failed')
DIRAC.exit(-1)
###################### Check TTree Output File #######################
outfile = 'MVAFile_' + runlist + ".root"
if not os.path.isfile(outfile):
error = 'TTree file was not created:'
DIRAC.gLogger.error(error, outfile)
jobReport.setApplicationStatus('DoCtaIrf: TTree file not created')
DIRAC.exit(-1)
###################### Quality Check for TTree Output File: step0######################
hr = HapRootMacro()
hr.setSoftwarePackage(HapPack)
DIRAC.gLogger.notice('Executing TTree check step0')
hr.rootMacro = '/hapscripts/mva/Open_TT.C+'
outfilestr = '"' + outfile + '"'
args = [outfilestr]
DIRAC.gLogger.notice('Open_TT macro Arguments:', args)
hr.rootArguments = args
DIRAC.gLogger.notice('Executing Hap Open_TT macro')
res = hr.execute()
if not res['OK']:
DIRAC.gLogger.error('Open_TT: Failed')
DIRAC.exit(-1)
#########################Quality Check for TTree Output File: step1####################
DIRAC.gLogger.notice('Executing TTree check step1')
ret = getSoftwareEnviron(HapPack)
if not ret['OK']:
error = ret['Message']
DIRAC.gLogger.error(error, HapPack)
DIRAC.exit(-1)
hapEnviron = ret['Value']
hessroot = hapEnviron['HESSROOT']
check_script = hessroot + '/hapscripts/mva/check_TT.csh'
cmdTuple = [check_script]
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute TTree Check step1')
jobReport.setApplicationStatus('Check_TTree: Failed')
DIRAC.exit(-1)
#############################################
DIRAC.exit()
def main():
from DIRAC.Core.Base import Script
Script.initialize()
DIRAC.gLogger.notice('Platform is:')
os.system('dirac-platform')
from CTADIRAC.Core.Workflow.Modules.Read_CtaApp import Read_CtaApp
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
version = sys.argv[3]
DIRAC.gLogger.notice('Version:', version)
install_CorsikaSimtelPack(version)
######### run read_cta #######################################
rcta = Read_CtaApp()
CorsikaSimtelPack = os.path.join('corsika_simhessarray', version,
'corsika_simhessarray')
rcta.setSoftwarePackage(CorsikaSimtelPack)
rcta.rctaExe = 'read_cta'
# add arguments for read_cta specified by user ######
args = []
rctaparfile = open('read_cta.par', 'r').readlines()
for line in rctaparfile:
for word in line.split():
args.append(word)
simtelFileLFN = sys.argv[-1].split('ParametricInputData=LFN:')[1]
simtelFileName = os.path.basename(simtelFileLFN)
dstFileName = simtelFileName.replace('simtel.gz', 'simtel-dst0.gz')
dstHistoFileName = simtelFileName.replace('simtel.gz', 'hdata-dst0.gz')
args.extend([
'--dst-file', dstFileName, '--histogram-file', dstHistoFileName,
simtelFileName
])
rcta.rctaArguments = args
rctaReturnCode = rcta.execute()
if rctaReturnCode != 0:
DIRAC.gLogger.error('read_cta Application: Failed')
jobReport.setApplicationStatus('read_cta Application: Failed')
DIRAC.exit(-1)
#################################################################
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
ret = getSoftwareEnviron(CorsikaSimtelPack)
if not ret['OK']:
error = ret['Message']
DIRAC.gLogger.error(error, CorsikaSimtelPack)
DIRAC.exit(-1)
read_ctaEnviron = ret['Value']
######## run dst quality checks ######################################
fd = open('check_dst_histo.sh', 'w')
fd.write("""#! /bin/sh
dsthistfilename=%s
dstfile=%s
n6="$(list_histograms -h 6 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
n12001="$(list_histograms -h 12001 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
if [ $n6 -ne $n12001 ]; then
echo 'n6 found:' $n6
echo 'n12001 found:' $n12001
exit 1
else
echo 'n6 found:' $n6
echo 'n12001 found:' $n12001
fi
n12002="$(list_histograms -h 12002 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
nev="$(statio ${dstfile} | egrep '^2010' | cut -f2)"
if [ -z "$nev" ]; then nev="0"; fi
if [ $nev -ne $n12002 ]; then
echo 'nev found:' $nev
echo 'n12002 found:' $n12002
exit 1
else
echo 'nev found:' $nev
echo 'n12002 found:' $n12002
fi
""" % (dstHistoFileName, dstFileName))
fd.close()
os.system('chmod u+x check_dst_histo.sh')
cmdTuple = ['./check_dst_histo.sh']
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput, env=read_ctaEnviron)
checkHistoReturnCode, stdout, stderr = ret['Value']
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute check_dst_histo.sh')
DIRAC.gLogger.error('check_dst_histo.sh status is:',
checkHistoReturnCode)
DIRAC.exit(-1)
if (checkHistoReturnCode != 0):
DIRAC.gLogger.error('Failure during check_dst_histo.sh')
DIRAC.gLogger.error('check_dst_histo.sh status is:',
checkHistoReturnCode)
jobReport.setApplicationStatus('Histo check Failed')
DIRAC.exit(-1)
DIRAC.exit()
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("p:", "run_number=", "Run Number", setRunNumber)
Script.registerSwitch("T:", "template=", "Template", setCorsikaTemplate)
Script.registerSwitch("E:", "executable=", "Executable", setExecutable)
Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig", setConfig)
Script.registerSwitch("V:", "version=", "Version", setVersion)
Script.registerSwitch("M:", "mode=", "Mode", setMode)
Script.registerSwitch("C:", "savecorsika=", "Save Corsika", setSaveCorsika)
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
Script.parseCommandLine()
global fcc, fcL, storage_element
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from CTADIRAC.Core.Workflow.Modules.CorsikaApp import CorsikaApp
from CTADIRAC.Core.Workflow.Modules.Read_CtaApp import Read_CtaApp
from DIRAC.Core.Utilities.Subprocess import systemCall
jobID = os.environ['JOBID']
jobID = int(jobID)
global jobReport
jobReport = JobReport(jobID)
###########
## Checking MD coherence
fc = FileCatalog('LcgFileCatalog')
res = fc._getCatalogConfigDetails('DIRACFileCatalog')
print 'DFC CatalogConfigDetails:', res
res = fc._getCatalogConfigDetails('LcgFileCatalog')
print 'LCG CatalogConfigDetails:', res
fcc = FileCatalogClient()
fcL = FileCatalog('LcgFileCatalog')
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
#############
simtelConfigFilesPath = 'sim_telarray/multi'
simtelConfigFile = simtelConfigFilesPath + '/multi_cta-ultra5.cfg'
#simtelConfigFile = simtelConfigFilesPath + '/multi_cta-prod1s.cfg'
createGlobalsFromConfigFiles('prodConfigFile', corsikaTemplate, version)
######################Building prod Directory Metadata #######################
resultCreateProdDirMD = createProdFileSystAndMD()
if not resultCreateProdDirMD['OK']:
DIRAC.gLogger.error('Failed to create prod Directory MD')
jobReport.setApplicationStatus('Failed to create prod Directory MD')
DIRAC.gLogger.error('Metadata coherence problem, no file produced')
DIRAC.exit(-1)
else:
print 'prod Directory MD successfully created'
######################Building corsika Directory Metadata #######################
resultCreateCorsikaDirMD = createCorsikaFileSystAndMD()
if not resultCreateCorsikaDirMD['OK']:
DIRAC.gLogger.error('Failed to create corsika Directory MD')
jobReport.setApplicationStatus('Failed to create corsika Directory MD')
DIRAC.gLogger.error(
'Metadata coherence problem, no corsikaFile produced')
DIRAC.exit(-1)
else:
print 'corsika Directory MD successfully created'
############ Producing Corsika File
global CorsikaSimtelPack
CorsikaSimtelPack = os.path.join('corsika_simhessarray', version,
'corsika_simhessarray')
install_CorsikaSimtelPack(version, 'sim')
cs = CorsikaApp()
cs.setSoftwarePackage(CorsikaSimtelPack)
cs.csExe = executable
cs.csArguments = [
'--run-number', run_number, '--run', 'corsika', corsikaTemplate
]
corsikaReturnCode = cs.execute()
if corsikaReturnCode != 0:
DIRAC.gLogger.error('Corsika Application: Failed')
jobReport.setApplicationStatus('Corsika Application: Failed')
DIRAC.exit(-1)
###################### rename of corsika output file #######################
rundir = 'run' + run_number
filein = rundir + '/' + corsikaOutputFileName
corsikaFileName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.gz'
mv_cmd = 'mv ' + filein + ' ' + corsikaFileName
if (os.system(mv_cmd)):
DIRAC.exit(-1)
########################
########################
## files spread in 1000-runs subDirectories
runNum = int(run_number)
subRunNumber = '%03d' % runNum
runNumModMille = runNum % 1000
runNumTrunc = (runNum - runNumModMille) / 1000
runNumSeriesDir = '%03dxxx' % runNumTrunc
print 'runNumSeriesDir=', runNumSeriesDir
### create corsika tar luisa ####################
corsikaTarName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.tar.gz'
filetar1 = rundir + '/' + 'input'
filetar2 = rundir + '/' + 'DAT' + run_number + '.dbase'
filetar3 = rundir + '/run' + str(int(run_number)) + '.log'
cmdTuple = [
'/bin/tar', 'zcf', corsikaTarName, filetar1, filetar2, filetar3
]
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute tar')
DIRAC.exit(-1)
######################################################
corsikaOutFileDir = os.path.join(corsikaDirPath, particle, 'Data',
runNumSeriesDir)
corsikaOutFileLFN = os.path.join(corsikaOutFileDir, corsikaFileName)
corsikaRunNumberSeriesDirExist = fcc.isDirectory(
corsikaOutFileDir)['Value']['Successful'][corsikaOutFileDir]
newCorsikaRunNumberSeriesDir = (
corsikaRunNumberSeriesDirExist != True
) # if new runFileSeries, will need to add new MD
#### create a file to DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK ################
f = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w')
f.close()
if savecorsika == 'True':
DIRAC.gLogger.notice('Put and register corsika File in LFC and DFC:',
corsikaOutFileLFN)
ret = dirac.addFile(corsikaOutFileLFN, corsikaFileName,
storage_element)
res = CheckCatalogCoherence(corsikaOutFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
# put and register corsikaTarFile:
corsikaTarFileDir = os.path.join(corsikaDirPath, particle, 'Log',
runNumSeriesDir)
corsikaTarFileLFN = os.path.join(corsikaTarFileDir, corsikaTarName)
##### If storage element is IN2P3-tape save simtel file on disk ###############
if storage_element == 'CC-IN2P3-Tape':
storage_element = 'CC-IN2P3-Disk'
DIRAC.gLogger.notice(
'Put and register corsikaTar File in LFC and DFC:',
corsikaTarFileLFN)
ret = dirac.addFile(corsikaTarFileLFN, corsikaTarName, storage_element)
####Checking and restablishing catalog coherence #####################
res = CheckCatalogCoherence(corsikaTarFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
######################################################################
if newCorsikaRunNumberSeriesDir:
insertRunFileSeriesMD(corsikaOutFileDir, runNumTrunc)
insertRunFileSeriesMD(corsikaTarFileDir, runNumTrunc)
###### insert corsika File Level metadata ############################################
corsikaFileMD = {}
corsikaFileMD['runNumber'] = int(run_number)
corsikaFileMD['jobID'] = jobID
corsikaFileMD['corsikaReturnCode'] = corsikaReturnCode
corsikaFileMD['nbShowers'] = nbShowers
result = fcc.setMetadata(corsikaOutFileLFN, corsikaFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(corsikaTarFileLFN, corsikaFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
##### Exit now if only corsika simulation required
if (mode == 'corsika_standalone'):
DIRAC.exit()
############ Producing SimTel File
######################Building simtel Directory Metadata #######################
cfg_dict = {
"4MSST": 'cta-prod2-4m-dc',
"SCSST": 'cta-prod2-sc-sst',
"STD": 'cta-prod2',
"NSBX3": 'cta-prod2',
"ASTRI": 'cta-prod2-astri',
"SCMST": 'cta-prod2-sc3',
"NORTH": 'cta-prod2n'
}
if simtelConfig == "6INROW":
all_configs = ["4MSST", "SCSST", "ASTRI", "NSBX3", "STD", "SCMST"]
elif simtelConfig == "5INROW":
all_configs = ["4MSST", "SCSST", "ASTRI", "NSBX3", "STD"]
elif simtelConfig == "3INROW":
all_configs = ["SCSST", "STD", "SCMST"]
else:
all_configs = [simtelConfig]
############################################
#for current_conf in all_configs:
#DIRAC.gLogger.notice('current conf is',current_conf)
#if current_conf == "SCMST":
#current_version = version + '_sc3'
#DIRAC.gLogger.notice('current version is', current_version)
#if os.path.isdir('sim_telarray'):
#DIRAC.gLogger.notice('Package found in the local area. Removing package...')
#cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run'
#if(os.system(cmd)):
#DIRAC.exit( -1 )
#install_CorsikaSimtelPack(current_version)
#else:
#current_version = version
#DIRAC.gLogger.notice('current version is', current_version)
#############################################################
for current_conf in all_configs:
DIRAC.gLogger.notice('current conf is', current_conf)
if current_conf == "SCMST":
current_version = version + '_sc3'
DIRAC.gLogger.notice('current version is', current_version)
installSoftwareEnviron(CorsikaSimtelPack, workingArea(), 'sim-sc3')
else:
current_version = version
DIRAC.gLogger.notice('current version is', current_version)
########################################################
global simtelDirPath
global simtelProdVersion
simtelProdVersion = current_version + '_simtel'
simtelDirPath = os.path.join(corsikaParticleDirPath, simtelProdVersion)
resultCreateSimtelDirMD = createSimtelFileSystAndMD(current_conf)
if not resultCreateSimtelDirMD['OK']:
DIRAC.gLogger.error('Failed to create simtelArray Directory MD')
jobReport.setApplicationStatus(
'Failed to create simtelArray Directory MD')
DIRAC.gLogger.error(
'Metadata coherence problem, no simtelArray File produced')
DIRAC.exit(-1)
else:
DIRAC.gLogger.notice('simtel Directory MD successfully created')
############## check simtel data file LFN exists ########################
simtelFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.simtel.gz'
simtelDirPath_conf = simtelDirPath + '_' + current_conf
simtelOutFileDir = os.path.join(simtelDirPath_conf, 'Data',
runNumSeriesDir)
simtelOutFileLFN = os.path.join(simtelOutFileDir, simtelFileName)
res = CheckCatalogCoherence(simtelOutFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('Current conf already done', current_conf)
continue
#### execute simtelarray ################
fd = open('run_sim.sh', 'w')
fd.write("""#! /bin/sh
source ./Corsika_simhessarrayEnv.sh
export SVNPROD2=$PWD
export SVNTAG=SVN-PROD2_rev10503
export CORSIKA_IO_BUFFER=800MB
cp ../grid_prod2-repro.sh .
ln -s ../%s
ln -s ../$SVNTAG
./grid_prod2-repro.sh %s %s""" %
(corsikaFileName, corsikaFileName, current_conf))
fd.close()
####################################
os.system('chmod u+x run_sim.sh')
cmdTuple = ['./run_sim.sh']
ret = systemCall(0, cmdTuple, sendOutputSimTel)
simtelReturnCode, stdout, stderr = ret['Value']
if (os.system('grep Broken simtel.log') == 0):
DIRAC.gLogger.error('Broken string found in simtel.log')
jobReport.setApplicationStatus('Broken pipe')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute run_sim.sh')
DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode)
DIRAC.exit(-1)
## check simtel data/log/histo Output File exist
cfg = cfg_dict[current_conf]
#cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Data/*.simtel.gz ' + simtelFileName
if current_conf == "SCMST":
cmdprefix = 'mv sim-sc3/Data/sim_telarray/' + cfg + '/0.0deg/'
else:
cmdprefix = 'mv sim/Data/sim_telarray/' + cfg + '/0.0deg/'
cmd = cmdprefix + 'Data/*' + cfg + '_*.simtel.gz ' + simtelFileName
if (os.system(cmd)):
DIRAC.exit(-1)
############################################
simtelRunNumberSeriesDirExist = fcc.isDirectory(
simtelOutFileDir)['Value']['Successful'][simtelOutFileDir]
newSimtelRunFileSeriesDir = (
simtelRunNumberSeriesDirExist != True
) # if new runFileSeries, will need to add new MD
simtelLogFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz'
#cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Log/*.log.gz ' + simtelLogFileName
cmd = cmdprefix + 'Log/*' + cfg + '_*.log.gz ' + simtelLogFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutLogFileDir = os.path.join(simtelDirPath_conf, 'Log',
runNumSeriesDir)
simtelOutLogFileLFN = os.path.join(simtelOutLogFileDir,
simtelLogFileName)
simtelHistFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.hdata.gz'
#cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName
cmd = cmdprefix + 'Histograms/*' + cfg + '_*.hdata.gz ' + simtelHistFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutHistFileDir = os.path.join(simtelDirPath_conf, 'Histograms',
runNumSeriesDir)
simtelOutHistFileLFN = os.path.join(simtelOutHistFileDir,
simtelHistFileName)
########### quality check on Histo #############################################
fd = open('check_histo.sh', 'w')
fd.write("""#! /bin/sh
nsim=$(list_histograms %s|fgrep 'Histogram 6 '|sed 's/^.*contents: //'| sed 's:/.*$::')
nevents=%d
if [ $nsim -lt $(( $nevents - 20 )) ]; then
echo 'nsim found:' $nsim
echo 'nsim expected:' $nevents
exit 1
else
echo 'nsim found:' $nsim
echo 'nsim expected:' $nevents
fi
""" % (simtelHistFileName, int(nbShowers) * int(cscat)))
fd.close()
ret = getSoftwareEnviron(CorsikaSimtelPack)
if not ret['OK']:
error = ret['Message']
DIRAC.gLogger.error(error, CorsikaSimtelPack)
DIRAC.exit(-1)
corsikaEnviron = ret['Value']
os.system('chmod u+x check_histo.sh')
cmdTuple = ['./check_histo.sh']
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput, env=corsikaEnviron)
checkHistoReturnCode, stdout, stderr = ret['Value']
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute check_histo.sh')
DIRAC.gLogger.error('check_histo.sh status is:',
checkHistoReturnCode)
DIRAC.exit(-1)
if (checkHistoReturnCode != 0):
DIRAC.gLogger.error('Failure during check_histo.sh')
DIRAC.gLogger.error('check_histo.sh status is:',
checkHistoReturnCode)
jobReport.setApplicationStatus('Histo check Failed')
DIRAC.exit(-1)
########## quality check on Log #############################
cmd = 'zcat %s | grep Finished.' % simtelLogFileName
DIRAC.gLogger.notice('Executing system call:', cmd)
if (os.system(cmd)):
jobReport.setApplicationStatus('Log check Failed')
DIRAC.exit(-1)
################################################
from DIRAC.Core.Utilities import List
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
opsHelper = Operations()
global seList
seList = opsHelper.getValue('ProductionOutputs/SimtelProd', [])
seList = List.randomize(seList)
DIRAC.gLogger.notice('SeList is:', seList)
######### Upload simtel data/log/histo ##############################################
res = upload_to_seList(simtelOutFileLFN, simtelFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('OutputData Upload Error', simtelOutFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
res = CheckCatalogCoherence(simtelOutLogFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('Log file already exists. Removing:',
simtelOutLogFileLFN)
ret = dirac.removeFile(simtelOutLogFileLFN)
res = upload_to_seList(simtelOutLogFileLFN, simtelLogFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Upload simtel Log Error', simtelOutLogFileLFN)
DIRAC.gLogger.notice('Removing simtel data file:',
simtelOutFileLFN)
ret = dirac.removeFile(simtelOutFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
res = CheckCatalogCoherence(simtelOutHistFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('Histo file already exists. Removing:',
simtelOutHistFileLFN)
ret = dirac.removeFile(simtelOutHistFileLFN)
res = upload_to_seList(simtelOutHistFileLFN, simtelHistFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Upload simtel Histo Error',
simtelOutHistFileLFN)
DIRAC.gLogger.notice('Removing simtel data file:',
simtelOutFileLFN)
ret = dirac.removeFile(simtelOutFileLFN)
DIRAC.gLogger.notice('Removing simtel log file:',
simtelOutLogFileLFN)
ret = dirac.removeFile(simtelOutLogFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
# simtelRunNumberSeriesDirExist = fcc.isDirectory(simtelOutFileDir)['Value']['Successful'][simtelOutFileDir]
# newSimtelRunFileSeriesDir = (simtelRunNumberSeriesDirExist != True) # if new runFileSeries, will need to add new MD
if newSimtelRunFileSeriesDir:
print 'insertRunFileSeriesMD'
insertRunFileSeriesMD(simtelOutFileDir, runNumTrunc)
insertRunFileSeriesMD(simtelOutLogFileDir, runNumTrunc)
insertRunFileSeriesMD(simtelOutHistFileDir, runNumTrunc)
else:
print 'NotinsertRunFileSeriesMD'
###### simtel File level metadata ############################################
simtelFileMD = {}
simtelFileMD['runNumber'] = int(run_number)
simtelFileMD['jobID'] = jobID
simtelFileMD['simtelReturnCode'] = simtelReturnCode
result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(simtelOutLogFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(simtelOutHistFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
if savecorsika == 'True':
result = fcc.addFileAncestors(
{simtelOutFileLFN: {
'Ancestors': [corsikaOutFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{simtelOutLogFileLFN: {
'Ancestors': [corsikaOutFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{simtelOutHistFileLFN: {
'Ancestors': [corsikaOutFileLFN]
}})
print 'result addFileAncestor:', result
##### Exit now if only corsika simulation required
if (mode == 'corsika_simtel'):
continue
######### run read_cta #######################################
rcta = Read_CtaApp()
rcta.setSoftwarePackage(CorsikaSimtelPack)
rcta.rctaExe = 'read_cta'
powerlaw_dict = {
'gamma': '-2.57',
'gamma_ptsrc': '-2.57',
'proton': '-2.70',
'electron': '-3.21'
}
dstFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.simtel-dst0.gz'
dstHistoFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.hdata-dst0.gz'
## added some options starting from Armazones_2K prod.
rcta.rctaArguments = [
'-r', '4', '-u', '--integration-scheme', '4',
'--integration-window', '7,3', '--tail-cuts', '6,8', '--min-pix',
'2', '--min-amp', '20', '--type', '1,0,0,400', '--tail-cuts',
'9,12', '--min-amp', '20', '--type', '2,0,0,100', '--tail-cuts',
'8,11', '--min-amp', '19', '--type', '3,0,0,40', '--tail-cuts',
'6,9', '--min-amp', '15', '--type', '4,0,0,15', '--tail-cuts',
'3.7,5.5', '--min-amp', '8', '--type', '5,0,0,70,5.6',
'--tail-cuts', '2.4,3.2', '--min-amp', '5.6', '--dst-level', '0',
'--dst-file', dstFileName, '--histogram-file', dstHistoFileName,
'--powerlaw', powerlaw_dict[particle], simtelFileName
]
rctaReturnCode = rcta.execute()
if rctaReturnCode != 0:
DIRAC.gLogger.error('read_cta Application: Failed')
jobReport.setApplicationStatus('read_cta Application: Failed')
DIRAC.exit(-1)
######## run dst quality checks ######################################
fd = open('check_dst_histo.sh', 'w')
fd.write("""#! /bin/sh
dsthistfilename=%s
dstfile=%s
n6="$(list_histograms -h 6 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
n12001="$(list_histograms -h 12001 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
if [ $n6 -ne $n12001 ]; then
echo 'n6 found:' $n6
echo 'n12001 found:' $n12001
exit 1
else
echo 'n6 found:' $n6
echo 'n12001 found:' $n12001
fi
n12002="$(list_histograms -h 12002 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
nev="$(statio ${dstfile} | egrep '^2010' | cut -f2)"
if [ -z "$nev" ]; then nev="0"; fi
if [ $nev -ne $n12002 ]; then
echo 'nev found:' $nev
echo 'n12002 found:' $n12002
exit 1
else
echo 'nev found:' $nev
echo 'n12002 found:' $n12002
fi
""" % (dstHistoFileName, dstFileName))
fd.close()
os.system('chmod u+x check_dst_histo.sh')
cmdTuple = ['./check_dst_histo.sh']
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput, env=corsikaEnviron)
checkHistoReturnCode, stdout, stderr = ret['Value']
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute check_dst_histo.sh')
DIRAC.gLogger.error('check_dst_histo.sh status is:',
checkHistoReturnCode)
DIRAC.exit(-1)
if (checkHistoReturnCode != 0):
DIRAC.gLogger.error('Failure during check_dst_histo.sh')
DIRAC.gLogger.error('check_dst_histo.sh status is:',
checkHistoReturnCode)
jobReport.setApplicationStatus('Histo check Failed')
DIRAC.exit(-1)
############create MD and upload dst data/histo ##########################################################
global dstDirPath
global dstProdVersion
dstProdVersion = current_version + '_dst'
dstDirPath = os.path.join(simtelDirPath_conf, dstProdVersion)
dstOutFileDir = os.path.join(dstDirPath, 'Data', runNumSeriesDir)
dstOutFileLFN = os.path.join(dstOutFileDir, dstFileName)
resultCreateDstDirMD = createDstFileSystAndMD()
if not resultCreateDstDirMD['OK']:
DIRAC.gLogger.error('Failed to create Dst Directory MD')
jobReport.setApplicationStatus('Failed to create Dst Directory MD')
DIRAC.gLogger.error(
'Metadata coherence problem, no Dst File produced')
DIRAC.exit(-1)
else:
DIRAC.gLogger.notice('Dst Directory MD successfully created')
############################################################
res = CheckCatalogCoherence(dstOutFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('dst file already exists. Removing:',
dstOutFileLFN)
ret = dirac.removeFile(dstOutFileLFN)
res = upload_to_seList(dstOutFileLFN, dstFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Upload dst Error', dstOutFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
##############################################################
dstHistoFileDir = os.path.join(dstDirPath, 'Histograms',
runNumSeriesDir)
dstHistoFileLFN = os.path.join(dstHistoFileDir, dstHistoFileName)
res = CheckCatalogCoherence(dstHistoFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('dst histo file already exists. Removing:',
dstHistoFileLFN)
ret = dirac.removeFile(dstHistoFileLFN)
res = upload_to_seList(dstHistoFileLFN, dstHistoFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Upload dst Error', dstHistoFileName)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
########### Insert RunNumSeries MD ##########################
dstRunNumberSeriesDirExist = fcc.isDirectory(
dstOutFileDir)['Value']['Successful'][dstOutFileDir]
newDstRunFileSeriesDir = (
dstRunNumberSeriesDirExist != True
) # if new runFileSeries, will need to add new MD
if newDstRunFileSeriesDir:
insertRunFileSeriesMD(dstOutFileDir, runNumTrunc)
insertRunFileSeriesMD(dstHistoFileDir, runNumTrunc)
####### dst File level metadata ###############################################
dstFileMD = {}
dstFileMD['runNumber'] = int(run_number)
dstFileMD['jobID'] = jobID
dstFileMD['rctaReturnCode'] = rctaReturnCode
result = fcc.setMetadata(dstOutFileLFN, dstFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(dstHistoFileLFN, dstFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
########## set the ancestors for dst #####################################
result = fcc.addFileAncestors(
{dstOutFileLFN: {
'Ancestors': [simtelOutFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{dstHistoFileLFN: {
'Ancestors': [simtelOutFileLFN]
}})
print 'result addFileAncestor:', result
######################################################
DIRAC.exit()
def main():
from DIRAC import gLogger
from DIRAC.Core.Base import Script
Script.parseCommandLine(ignoreErrors=True)
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
args = Script.getPositionalArgs()
version = args[0]
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
packs = [CorsikaSimtelPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
installSoftwareEnviron(package, workingArea())
packageTuple = package.split('/')
corsika_subdir = sharedArea(
) + '/' + packageTuple[0] + '/' + version
cmd = 'cp -u -r ' + corsika_subdir + '/* .'
os.system(cmd)
continue
if workingArea:
if checkSoftwarePackage(package, workingArea())['OK']:
DIRAC.gLogger.notice('Package found in Local Area:', package)
continue
if installSoftwarePackage(package, workingArea())['OK']:
############## compile #############################
if version == 'clean_23012012':
cmdTuple = ['./build_all', 'ultra', 'qgs2']
elif version in [
'prod-2_21122012', 'prod-2_08032013', 'prod-2_06052013'
]:
cmdTuple = ['./build_all', 'prod2', 'qgs2']
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to compile')
DIRAC.exit(-1)
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
ret = getSoftwareEnviron(CorsikaSimtelPack)
if not ret['OK']:
error = ret['Message']
DIRAC.gLogger.error(error, CorsikaSimtelPack)
DIRAC.exit(-1)
corsikaEnviron = ret['Value']
executable_file = args[1]
cmd = 'chmod u+x ' + executable_file
os.system(cmd)
cmdTuple = args[1:]
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput, env=corsikaEnviron)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute read_hess:', ret['Message'])
DIRAC.exit(-1)
status, stdout, stderr = ret['Value']
if status:
DIRAC.gLogger.error('read_hess execution reports Error:', status)
DIRAC.gLogger.error(stdout)
DIRAC.gLogger.error(stderr)
DIRAC.exit(-1)
DIRAC.exit()
def execute(self):
"""
The method called by the Workflow framework
"""
from DIRAC.Core.Utilities.Subprocess import systemCall
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
ret = self.__checkInputs()
if not ret['OK']:
return ret
ret = getSoftwareEnviron(self.softwarePackage)
if not ret['OK']:
error = ret['Message']
self.log.error(error, self.softwarePackage)
return DIRAC.S_ERROR(' '.join([error, str(self.softwarePackage)]))
hapEnviron = ret['Value']
hessroot = hapEnviron['HESSROOT']
rootlogon_file = hessroot + '/rootlogon.C'
cp_cmd = 'cp ' + rootlogon_file + ' .'
os.system(cp_cmd)
fileName = hessroot + '/hapscripts/dst/' + self.rootMacro
if fileName[-1] == '+':
# If the macro has to be compiled there is an extra "+" at the end of its name
fileName = fileName[:-1]
if not os.path.isfile(fileName):
error = 'make_CTA_DST.C file does not exist:'
self.log.error(error, fileName)
return DIRAC.S_ERROR(' '.join([error, fileName]))
fileName = hessroot + '/hapscripts/dst/' + self.rootMacro
cmdTuple = ['root', '-b', '-q']
configpath = hessroot + '/config/array'
self.rootArguments[2] = self.rootArguments[2].replace(
"array", configpath)
cmdTuple += [
'%s( %s )' %
(fileName, ', '.join(self.rootArguments).replace("'", '"'))
]
self.log.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, self.sendOutput, env=hapEnviron)
if not ret['OK']:
self.log.error('Failed to execute Root:', ret['Message'])
return DIRAC.S_ERROR('Can not execute root')
status, stdout, stderr = ret['Value']
if status == 0:
self.log.error('make_CTA_DST.C execution reports Error:', status)
self.log.error(stdout)
self.log.error(stderr)
return DIRAC.S_ERROR('Failed root Execution')
fileout = '/tmp/dst_CTA_%08d' % int(self.rootArguments[0]) + '.root'
mv_cmd = 'mv ' + fileout + ' .'
os.system(mv_cmd)
return DIRAC.S_OK()
