def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("p:", "inputfile=", "Input File", setInputFile)
Script.registerSwitch("E:", "simtelExecName=", "SimtelExecName",
setExecutable)
Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig", setConfig)
Script.registerSwitch("V:", "version=", "Version", setVersion)
Script.registerSwitch("D:", "storage_element=", "Storage Element",
setStorageElement)
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
Script.parseCommandLine()
DIRAC.gLogger.setLevel('INFO')
global fcc, fcL, storage_element
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray'
packs = [CorsikaSimtelPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if sharedArea:
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
installSoftwareEnviron(package, workingArea())
packageTuple = package.split('/')
corsika_subdir = sharedArea(
) + '/' + packageTuple[0] + '/' + version
cmd = 'cp -u -r ' + corsika_subdir + '/* .'
os.system(cmd)
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
###########
## Checking MD coherence
fc = FileCatalog('LcgFileCatalog')
res = fc._getCatalogConfigDetails('DIRACFileCatalog')
print 'DFC CatalogConfigDetails:', res
res = fc._getCatalogConfigDetails('LcgFileCatalog')
print 'LCG CatalogConfigDetails:', res
fcc = FileCatalogClient()
fcL = FileCatalog('LcgFileCatalog')
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
############################
#############
# CLAUDIA: simtelConfigFile should be built from ???
#simtelConfigFilesPath = 'sim_telarray/multi'
#simtelConfigFile = simtelConfigFilesPath + '/multi_cta-ultra5.cfg'
#createGlobalsFromConfigFiles(simtelConfigFile)
createGlobalsFromConfigFiles()
#######################
## files spread in 1000-runs subDirectories
corsikaFileName = os.path.basename(corsikaFileLFN)
run_number = corsikaFileName.split('run')[1].split('.corsika.gz')[
0] # run001412.corsika.gz
runNum = int(run_number)
subRunNumber = '%03d' % runNum
runNumModMille = runNum % 1000
runNumTrunc = (runNum - runNumModMille) / 1000
runNumSeriesDir = '%03dxxx' % runNumTrunc
print 'runNumSeriesDir=', runNumSeriesDir
f = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w')
f.close()
##### If storage element is IN2P3-tape save simtel file on disk ###############
if storage_element == 'CC-IN2P3-Tape':
storage_element = 'CC-IN2P3-Disk'
############ Producing SimTel File
######################Building simtel Directory Metadata #######################
resultCreateSimtelDirMD = createSimtelFileSystAndMD()
if not resultCreateSimtelDirMD['OK']:
DIRAC.gLogger.error('Failed to create simtelArray Directory MD')
jobReport.setApplicationStatus(
'Failed to create simtelArray Directory MD')
DIRAC.gLogger.error(
'Metadata coherence problem, no simtelArray File produced')
DIRAC.exit(-1)
else:
print 'simtel Directory MD successfully created'
#### execute simtelarray ################
# fd = open('run_sim.sh', 'w' )
# fd.write( """#! /bin/sh
#echo "go for sim_telarray"
#. ./examples_common.sh
#export CORSIKA_IO_BUFFER=800MB
#zcat %s | $SIM_TELARRAY_PATH/run_sim_%s""" % (corsikaFileName, simtelExecName))
# fd.close()
#### execute simtelarray ################
fd = open('run_sim.sh', 'w')
fd.write("""#! /bin/sh
export SVNPROD2=$PWD
export SVNTAG=SVN-PROD2
export CORSIKA_IO_BUFFER=800MB
./grid_prod2-repro.sh %s %s""" % (corsikaFileName, simtelConfig))
fd.close()
os.system('chmod u+x run_sim.sh')
cmdTuple = ['./run_sim.sh']
ret = systemCall(0, cmdTuple, sendOutputSimTel)
simtelReturnCode, stdout, stderr = ret['Value']
if (os.system('grep Broken simtel.log')):
print 'not broken'
else:
print 'broken'
# Tag corsika File if Broken Pipe
corsikaTagMD = {}
corsikaTagMD['CorsikaToReprocess'] = 'CorsikaToReprocess'
result = fcc.setMetadata(corsikaFileLFN, corsikaTagMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
jobReport.setApplicationStatus('Broken pipe')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute run_sim.sh')
DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode)
DIRAC.exit(-1)
## putAndRegister simtel data/log/histo Output File:
simtelFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.simtel.gz'
cfg = simtelExecName
if simtelExecName == "cta-prod2-nsbx3":
cfg = "cta-prod2"
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Data/*.simtel.gz ' + simtelFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutFileDir = os.path.join(simtelDirPath, 'Data', runNumSeriesDir)
simtelOutFileLFN = os.path.join(simtelOutFileDir, simtelFileName)
simtelRunNumberSeriesDirExist = fcc.isDirectory(
simtelOutFileDir)['Value']['Successful'][simtelOutFileDir]
newSimtelRunFileSeriesDir = (
simtelRunNumberSeriesDirExist != True
) # if new runFileSeries, will need to add new MD
simtelLogFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Log/*.log.gz ' + simtelLogFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutLogFileDir = os.path.join(simtelDirPath, 'Log', runNumSeriesDir)
simtelOutLogFileLFN = os.path.join(simtelOutLogFileDir, simtelLogFileName)
simtelHistFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.hdata.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutHistFileDir = os.path.join(simtelDirPath, 'Histograms',
runNumSeriesDir)
simtelOutHistFileLFN = os.path.join(simtelOutHistFileDir,
simtelHistFileName)
################################################
DIRAC.gLogger.notice('Put and register simtel File in LFC and DFC:',
simtelOutFileLFN)
ret = dirac.addFile(simtelOutFileLFN, simtelFileName, storage_element)
res = CheckCatalogCoherence(simtelOutFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
######################################################################
DIRAC.gLogger.notice('Put and register simtel Log File in LFC and DFC:',
simtelOutLogFileLFN)
ret = dirac.addFile(simtelOutLogFileLFN, simtelLogFileName,
storage_element)
res = CheckCatalogCoherence(simtelOutLogFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
######################################################################
DIRAC.gLogger.notice('Put and register simtel Histo File in LFC and DFC:',
simtelOutHistFileLFN)
ret = dirac.addFile(simtelOutHistFileLFN, simtelHistFileName,
storage_element)
res = CheckCatalogCoherence(simtelOutHistFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
######################################################################
if newSimtelRunFileSeriesDir:
insertRunFileSeriesMD(simtelOutFileDir, runNumTrunc)
insertRunFileSeriesMD(simtelOutLogFileDir, runNumTrunc)
insertRunFileSeriesMD(simtelOutHistFileDir, runNumTrunc)
###### simtel File level metadata ############################################
simtelFileMD = {}
simtelFileMD['runNumber'] = int(run_number)
simtelFileMD['jobID'] = jobID
simtelFileMD['simtelReturnCode'] = simtelReturnCode
result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(simtelOutLogFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(simtelOutHistFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.addFileAncestors(
{simtelOutFileLFN: {
'Ancestors': [corsikaFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{simtelOutLogFileLFN: {
'Ancestors': [corsikaFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{simtelOutHistFileLFN: {
'Ancestors': [corsikaFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD)
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
DIRAC.exit()
class ILDRegisterOutputData(ModuleBase):
""" Register output data in the FC for the ILD productions
"""
def __init__(self):
super(ILDRegisterOutputData, self).__init__()
self.version = "ILDRegisterOutputData v1"
self.log = gLogger.getSubLogger("ILDRegisterOutputData")
self.commandTimeOut = 10 * 60
self.enable = True
self.prodOutputLFNs = []
self.nbofevents = 0
self.filecatalog = FileCatalogClient()
self.ildconfig = ''
self.swpackages = []
def applicationSpecificInputs(self):
if 'Enable' in self.step_commons:
self.enable = self.step_commons['Enable']
if not type(self.enable) == type(True):
self.log.warn(
'Enable flag set to non-boolean value %s, setting to False'
% self.enable)
self.enable = False
if 'ProductionOutputData' in self.workflow_commons:
self.prodOutputLFNs = self.workflow_commons[
'ProductionOutputData'].split(";")
else:
self.prodOutputLFNs = []
if 'SoftwarePackages' in self.workflow_commons:
self.swpackages = self.workflow_commons['SoftwarePackages'].split(
";")
self.nbofevents = self.NumberOfEvents #comes from ModuleBase
if 'ILDConfigPackage' in self.workflow_commons:
self.ildconfig = self.workflow_commons['ILDConfigPackage']
return S_OK('Parameters resolved')
def execute(self):
self.log.info('Initializing %s' % self.version)
result = self.resolveInputVariables()
if not result['OK']:
self.log.error("Failed to resolve input parameters:",
result['Message'])
return result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('No registration of output data metadata attempted')
if len(self.prodOutputLFNs) == 0:
self.log.info(
'No production data found, so no metadata registration to be done'
)
return S_OK("No files' metadata to be registered")
self.log.verbose(
"Will try to set the metadata for the following files: \n %s" %
"\n".join(self.prodOutputLFNs))
#TODO: What meta data should be stored at file level?
for files in self.prodOutputLFNs:
meta = {}
if self.nbofevents:
nbevts = {}
nbevts['NumberOfEvents'] = self.nbofevents
if 'file_number_of_event_relation' in self.workflow_commons:
if os.path.basename(files) in self.workflow_commons[
'file_number_of_event_relation']:
nbevts['NumberOfEvents'] = self.workflow_commons[
'file_number_of_event_relation'][os.path.basename(
files)]
meta.update(nbevts)
if 'CrossSection' in self.inputdataMeta:
xsec = {'CrossSection': self.inputdataMeta['CrossSection']}
meta.update(xsec)
if 'CrossSectionError' in self.inputdataMeta:
xsec = {
'CrossSectionError':
self.inputdataMeta['CrossSectionError']
}
meta.update(xsec)
if 'GenProcessID' in self.inputdataMeta:
fmeta = {'GenProcessID': self.inputdataMeta['GenProcessID']}
meta.update(fmeta)
if 'GenProcessType' in self.inputdataMeta:
fmeta = {
'GenProcessType': self.inputdataMeta['GenProcessType']
}
meta.update(fmeta)
if 'GenProcessName' in self.inputdataMeta:
fmeta = {
'GenProcessName': self.inputdataMeta['GenProcessName']
}
meta.update(fmeta)
if 'Luminosity' in self.inputdataMeta:
fmeta = {'Luminosity': self.inputdataMeta['Luminosity']}
meta.update(fmeta)
if 'BeamParticle1' in self.inputdataMeta:
fmeta = {
'BeamParticle1': self.inputdataMeta['BeamParticle1'],
'BeamParticle2': self.inputdataMeta['BeamParticle2']
}
meta.update(fmeta)
if 'PolarizationB1' in self.inputdataMeta:
fmeta = {
'PolarizationB1': self.inputdataMeta['PolarizationB1'],
'PolarizationB2': self.inputdataMeta['PolarizationB2']
}
meta.update(fmeta)
if self.ildconfig:
fmeta = {'ILDConfig': self.ildconfig}
meta.update(fmeta)
if self.WorkflowStartFrom:
meta.update({"FirstEventFromInput": self.WorkflowStartFrom})
if self.enable:
res = self.filecatalog.setMetadata(files, meta)
if not res['OK']:
self.log.error('Could not register metadata:',
res['Message'])
return res
self.log.info("Registered %s with tags %s" % (files, meta))
###Now, set the ancestors
if self.InputData:
inputdata = self.InputData
if self.enable:
res = self.filecatalog.addFileAncestors(
{files: {
'Ancestors': inputdata
}})
if not res['OK']:
self.log.error('Registration of Ancestors failed for:',
str(files))
self.log.error('because of', res['Message'])
return res
return S_OK('Output data metadata registered in catalog')
class RegisterOutputData( ModuleBase ):
""" At the end of a production Job, we need to register meta data info for the files.
"""
def __init__(self):
super(RegisterOutputData, self).__init__()
self.version = "RegisterOutputData v1"
self.log = gLogger.getSubLogger( "RegisterOutputData" )
self.commandTimeOut = 10*60
self.enable = True
self.prodOutputLFNs = []
self.nbofevents = 0
self.luminosity = 0
self.sel_eff = 0
self.cut_eff = 0
self.add_info = ''
self.filecatalog = FileCatalogClient()
self.filemeta = {}
def applicationSpecificInputs(self):
if self.step_commons.has_key('Enable'):
self.enable = self.step_commons['Enable']
if not type(self.enable) == type(True):
self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable)
self.enable = False
if self.workflow_commons.has_key('ProductionOutputData'):
self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";")
else:
self.prodOutputLFNs = []
self.nbofevents = self.NumberOfEvents
if self.workflow_commons.has_key('Luminosity'):
self.luminosity = self.workflow_commons['Luminosity']
##Additional info: cross section only for the time being, comes from WHIZARD
if self.workflow_commons.has_key('Info'):
if 'stdhepcut' in self.workflow_commons['Info']:
self.sel_eff = self.workflow_commons['Info']['stdhepcut']['Reduction']
self.cut_eff = self.workflow_commons['Info']['stdhepcut']['CutEfficiency']
del self.workflow_commons['Info']['stdhepcut']
self.add_info = DEncode.encode(self.workflow_commons['Info'])
return S_OK('Parameters resolved')
def execute(self):
""" Run the module
"""
self.log.info('Initializing %s' % self.version)
result = self.resolveInputVariables()
if not result['OK']:
self.log.error("failed to resolve input parameters:", result['Message'])
return result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('No registration of output data metadata attempted')
if len(self.prodOutputLFNs) == 0:
self.log.info('No production data found, so no metadata registration to be done')
return S_OK("No files' metadata to be registered")
self.log.verbose("Will try to set the metadata for the following files: \n %s" % "\n".join(self.prodOutputLFNs))
for files in self.prodOutputLFNs:
metafiles = {}
if self.nbofevents:
nbevts = {}
nbevts['NumberOfEvents'] = self.nbofevents
if self.workflow_commons.has_key('file_number_of_event_relation'):
if self.workflow_commons['file_number_of_event_relation'].has_key(os.path.basename(files)):
nbevts['NumberOfEvents'] = self.workflow_commons['file_number_of_event_relation'][os.path.basename(files)]
metafiles.update(nbevts)
if self.luminosity:
metafiles.update({'Luminosity': self.luminosity})
if self.sel_eff:
metafiles.update({'Reduction':self.sel_eff})
if self.cut_eff:
metafiles.update({'CutEfficiency': self.cut_eff})
if self.add_info:
metafiles.update({'AdditionalInfo':self.add_info})
elif 'AdditionalInfo' in self.inputdataMeta:
metafiles.update({'AdditionalInfo':self.inputdataMeta['AdditionalInfo']})
if 'CrossSection' in self.inputdataMeta:
metafiles.update({'CrossSection':self.inputdataMeta['CrossSection']})
if self.WorkflowStartFrom:
metafiles.update({"FirstEventFromInput":self.WorkflowStartFrom})
if self.enable:
res = self.filecatalog.setMetadata(files, metafiles)
if not res['OK']:
self.log.error(res['Message'])
self.log.error('Could not register metadata for %s' % files)
return res
self.log.info("Registered %s with tags %s" % (files, metafiles))
###Now, set the ancestors
if self.InputData:
if self.enable:
res = self.filecatalog.addFileAncestors({ files : {'Ancestors' : self.InputData } })
if not res['OK']:
self.log.error('Registration of Ancestors for %s failed' % files)
self.log.error('Because of ', res['Message'])
return res
return S_OK('Output data metadata registered in catalog')
class ILDRegisterOutputData(ModuleBase):
""" Register output data in the FC for the ILD productions
"""
def __init__(self):
super(ILDRegisterOutputData, self).__init__()
self.version = "ILDRegisterOutputData v1"
self.commandTimeOut = 10 * 60
self.enable = True
self.prodOutputLFNs = []
self.nbofevents = 0
self.filecatalog = FileCatalogClient()
self.ildconfig = ''
self.swpackages = []
def applicationSpecificInputs(self):
if 'Enable' in self.step_commons:
self.enable = self.step_commons['Enable']
if not type(self.enable) == type(True):
LOG.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable)
self.enable = False
if 'ProductionOutputData' in self.workflow_commons:
self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";")
else:
self.prodOutputLFNs = []
if 'SoftwarePackages' in self.workflow_commons:
self.swpackages = self.workflow_commons['SoftwarePackages'].split(";")
self.nbofevents = self.NumberOfEvents #comes from ModuleBase
if 'ILDConfigPackage' in self.workflow_commons:
self.ildconfig = self.workflow_commons['ILDConfigPackage']
return S_OK('Parameters resolved')
def execute(self):
LOG.info('Initializing %s' % self.version)
result = self.resolveInputVariables()
if not result['OK']:
LOG.error("Failed to resolve input parameters:", result['Message'])
return result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('No registration of output data metadata attempted')
if len(self.prodOutputLFNs) == 0:
LOG.info('No production data found, so no metadata registration to be done')
return S_OK("No files' metadata to be registered")
LOG.verbose("Will try to set the metadata for the following files: \n %s" % "\n".join(self.prodOutputLFNs))
#TODO: What meta data should be stored at file level?
for files in self.prodOutputLFNs:
meta = {}
if self.nbofevents:
nbevts = {}
nbevts['NumberOfEvents'] = self.nbofevents
if 'file_number_of_event_relation' in self.workflow_commons:
if os.path.basename(files) in self.workflow_commons['file_number_of_event_relation']:
nbevts['NumberOfEvents'] = self.workflow_commons['file_number_of_event_relation'][os.path.basename(files)]
meta.update(nbevts)
if 'CrossSection' in self.inputdataMeta:
xsec = {'CrossSection':self.inputdataMeta['CrossSection']}
meta.update(xsec)
if 'CrossSectionError' in self.inputdataMeta:
xsec = {'CrossSectionError':self.inputdataMeta['CrossSectionError']}
meta.update(xsec)
if 'GenProcessID' in self.inputdataMeta:
fmeta = {'GenProcessID':self.inputdataMeta['GenProcessID']}
meta.update(fmeta)
if 'GenProcessType' in self.inputdataMeta:
fmeta = {'GenProcessType':self.inputdataMeta['GenProcessType']}
meta.update(fmeta)
if 'GenProcessName' in self.inputdataMeta:
fmeta = {'GenProcessName':self.inputdataMeta['GenProcessName']}
meta.update(fmeta)
if 'Luminosity' in self.inputdataMeta:
fmeta = {'Luminosity':self.inputdataMeta['Luminosity']}
meta.update(fmeta)
if 'BeamParticle1' in self.inputdataMeta:
fmeta = {'BeamParticle1':self.inputdataMeta['BeamParticle1'],
'BeamParticle2':self.inputdataMeta['BeamParticle2']}
meta.update(fmeta)
if 'PolarizationB1' in self.inputdataMeta:
fmeta = {'PolarizationB1':self.inputdataMeta['PolarizationB1'],
'PolarizationB2':self.inputdataMeta['PolarizationB2']}
meta.update(fmeta)
if self.ildconfig:
fmeta = {'ILDConfig' : self.ildconfig}
meta.update(fmeta)
if self.WorkflowStartFrom:
meta.update({"FirstEventFromInput":self.WorkflowStartFrom})
if self.enable:
res = self.filecatalog.setMetadata(files, meta)
if not res['OK']:
LOG.error('Could not register metadata:', res['Message'])
return res
LOG.info("Registered %s with tags %s" % (files, meta))
###Now, set the ancestors
if self.InputData:
inputdata = self.InputData
if self.enable:
res = self.filecatalog.addFileAncestors({files : {'Ancestors' : inputdata}})
if not res['OK']:
LOG.error('Registration of Ancestors failed for:', str(files))
LOG.error('because of', res['Message'])
return res
return S_OK('Output data metadata registered in catalog')
class RegisterOutputData(ModuleBase):
""" At the end of a production Job, we need to register meta data info for the files.
"""
def __init__(self):
super(RegisterOutputData, self).__init__()
self.version = "RegisterOutputData v1"
self.log = gLogger.getSubLogger("RegisterOutputData")
self.commandTimeOut = 10 * 60
self.enable = True
self.prodOutputLFNs = []
self.nbofevents = 0
self.luminosity = 0
self.sel_eff = 0
self.cut_eff = 0
self.add_info = ''
self.filecatalog = FileCatalogClient()
self.filemeta = {}
def applicationSpecificInputs(self):
if 'Enable' in self.step_commons:
self.enable = self.step_commons['Enable']
if not isinstance(self.enable, bool):
self.log.warn(
'Enable flag set to non-boolean value %s, setting to False'
% self.enable)
self.enable = False
if 'ProductionOutputData' in self.workflow_commons:
self.prodOutputLFNs = self.workflow_commons[
'ProductionOutputData'].split(";")
else:
self.prodOutputLFNs = []
self.nbofevents = self.NumberOfEvents
if 'Luminosity' in self.workflow_commons:
self.luminosity = self.workflow_commons['Luminosity']
##Additional info: cross section only for the time being, comes from WHIZARD
if 'Info' in self.workflow_commons:
if 'stdhepcut' in self.workflow_commons['Info']:
self.sel_eff = self.workflow_commons['Info']['stdhepcut'][
'Reduction']
self.cut_eff = self.workflow_commons['Info']['stdhepcut'][
'CutEfficiency']
del self.workflow_commons['Info']['stdhepcut']
self.add_info = DEncode.encode(self.workflow_commons['Info'])
return S_OK('Parameters resolved')
def execute(self):
""" Run the module
"""
self.log.info('Initializing %s' % self.version)
result = self.resolveInputVariables()
if not result['OK']:
self.log.error("failed to resolve input parameters:",
result['Message'])
return result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('No registration of output data metadata attempted')
if len(self.prodOutputLFNs) == 0:
self.log.info(
'No production data found, so no metadata registration to be done'
)
return S_OK("No files' metadata to be registered")
self.log.verbose(
"Will try to set the metadata for the following files: \n %s" %
"\n".join(self.prodOutputLFNs))
for files in self.prodOutputLFNs:
metafiles = {}
if self.nbofevents:
nbevts = {}
nbevts['NumberOfEvents'] = self.nbofevents
if 'file_number_of_event_relation' in self.workflow_commons:
if os.path.basename(files) in self.workflow_commons[
'file_number_of_event_relation']:
nbevts['NumberOfEvents'] = self.workflow_commons[
'file_number_of_event_relation'][os.path.basename(
files)]
metafiles.update(nbevts)
if self.luminosity:
metafiles.update({'Luminosity': self.luminosity})
if self.sel_eff:
metafiles.update({'Reduction': self.sel_eff})
if self.cut_eff:
metafiles.update({'CutEfficiency': self.cut_eff})
if self.add_info:
metafiles.update({'AdditionalInfo': self.add_info})
elif 'AdditionalInfo' in self.inputdataMeta:
metafiles.update(
{'AdditionalInfo': self.inputdataMeta['AdditionalInfo']})
if 'CrossSection' in self.inputdataMeta:
metafiles.update(
{'CrossSection': self.inputdataMeta['CrossSection']})
if self.WorkflowStartFrom:
metafiles.update(
{"FirstEventFromInput": self.WorkflowStartFrom})
if self.enable:
res = self.filecatalog.setMetadata(files, metafiles)
if not res['OK']:
self.log.error(res['Message'])
self.log.error('Could not register metadata for %s' %
files)
return res
self.log.info("Registered %s with tags %s" % (files, metafiles))
###Now, set the ancestors
if self.InputData:
if self.enable:
res = self.filecatalog.addFileAncestors(
{files: {
'Ancestors': self.InputData
}})
if not res['OK']:
self.log.error(
'Registration of Ancestors for %s failed' % files)
self.log.error('Because of ', res['Message'])
return res
return S_OK('Output data metadata registered in catalog')
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig", setConfig)
Script.registerSwitch("V:", "version=", "Version", setVersion)
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
Script.parseCommandLine()
DIRAC.gLogger.setLevel('INFO')
global fcc, fcL
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
global jobID
jobID = os.environ['JOBID']
jobReport = JobReport(int(jobID))
###########
## Checking MD coherence
fc = FileCatalog('LcgFileCatalog')
res = fc._getCatalogConfigDetails('DIRACFileCatalog')
print 'DFC CatalogConfigDetails:', res
res = fc._getCatalogConfigDetails('LcgFileCatalog')
print 'LCG CatalogConfigDetails:', res
fcc = FileCatalogClient()
fcL = FileCatalog('LcgFileCatalog')
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
############################
install_CorsikaSimtelPack(version)
#############
# simtelConfigFile should be built from ???
#simtelConfigFilesPath = 'sim_telarray/multi'
#simtelConfigFile = simtelConfigFilesPath + '/multi_cta-ultra5.cfg'
#createGlobalsFromConfigFiles(simtelConfigFile)
#createGlobalsFromConfigFiles(current_version)
#######################
## files spread in 1000-runs subDirectories
global corsikaFileLFN
corsikaFileLFN = dirac.getJobJDL(jobID)['Value']['InputData']
print 'corsikaFileLFN is ' + corsikaFileLFN
corsikaFileName = os.path.basename(corsikaFileLFN)
run_number = corsikaFileName.split('run')[1].split('.corsika.gz')[
0] # run001412.corsika.gz
runNum = int(run_number)
subRunNumber = '%03d' % runNum
runNumModMille = runNum % 1000
runNumTrunc = (runNum - runNumModMille) / 1000
runNumSeriesDir = '%03dxxx' % runNumTrunc
print 'runNumSeriesDir=', runNumSeriesDir
f = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w')
f.close()
############ Producing SimTel File
######################Building simtel Directory Metadata #######################
cfg_dict = {
"4MSST": 'cta-prod2-4m-dc',
"SCSST": 'cta-prod2-sc-sst',
"STD": 'cta-prod2',
"NSBX3": 'cta-prod2',
"ASTRI": 'cta-prod2-astri',
"SCMST": 'cta-prod2-sc3',
"NORTH": 'cta-prod2n'
}
if simtelConfig == "6INROW":
all_configs = ["4MSST", "SCSST", "ASTRI", "NSBX3", "STD", "SCMST"]
elif simtelConfig == "5INROW":
all_configs = ["4MSST", "SCSST", "ASTRI", "NSBX3", "STD"]
elif simtelConfig == "3INROW":
all_configs = ["SCSST", "STD", "SCMST"]
else:
all_configs = [simtelConfig]
for current_conf in all_configs:
DIRAC.gLogger.notice('current conf is', current_conf)
if current_conf == "SCMST":
current_version = version + '_sc3'
DIRAC.gLogger.notice('current version is', current_version)
if os.path.isdir('sim_telarray'):
DIRAC.gLogger.notice(
'Package found in the local area. Removing package...')
cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run'
if (os.system(cmd)):
DIRAC.exit(-1)
install_CorsikaSimtelPack(current_version)
else:
current_version = version
DIRAC.gLogger.notice('current version is', current_version)
########################################################
createGlobalsFromConfigFiles(current_version)
resultCreateSimtelDirMD = createSimtelFileSystAndMD(
current_conf, current_version)
if not resultCreateSimtelDirMD['OK']:
DIRAC.gLogger.error('Failed to create simtelArray Directory MD')
jobReport.setApplicationStatus(
'Failed to create simtelArray Directory MD')
DIRAC.gLogger.error(
'Metadata coherence problem, no simtelArray File produced')
DIRAC.exit(-1)
else:
print 'simtel Directory MD successfully created'
############## introduce file existence check here ########################
simtelFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.simtel.gz'
simtelDirPath_conf = simtelDirPath + '_' + current_conf
simtelOutFileDir = os.path.join(simtelDirPath_conf, 'Data',
runNumSeriesDir)
simtelOutFileLFN = os.path.join(simtelOutFileDir, simtelFileName)
res = CheckCatalogCoherence(simtelOutFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('Current conf already done', current_conf)
continue
#### execute simtelarray ################
fd = open('run_sim.sh', 'w')
fd.write("""#! /bin/sh
export SVNPROD2=$PWD
export SVNTAG=SVN-PROD2
export CORSIKA_IO_BUFFER=800MB
./grid_prod2-repro.sh %s %s""" % (corsikaFileName, current_conf))
fd.close()
os.system('chmod u+x grid_prod2-repro.sh')
os.system('chmod u+x run_sim.sh')
cmdTuple = ['./run_sim.sh']
ret = systemCall(0, cmdTuple, sendOutputSimTel)
simtelReturnCode, stdout, stderr = ret['Value']
if (os.system('grep Broken simtel.log')):
DIRAC.gLogger.notice('not broken')
else:
DIRAC.gLogger.notice('broken')
jobReport.setApplicationStatus('Broken pipe')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute run_sim.sh')
DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode)
DIRAC.exit(-1)
## putAndRegister simtel data/log/histo Output File:
cfg = cfg_dict[current_conf]
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Data/*.simtel.gz ' + simtelFileName
if (os.system(cmd)):
DIRAC.exit(-1)
############################################
simtelRunNumberSeriesDirExist = fcc.isDirectory(
simtelOutFileDir)['Value']['Successful'][simtelOutFileDir]
newSimtelRunFileSeriesDir = (
simtelRunNumberSeriesDirExist != True
) # if new runFileSeries, will need to add new MD
simtelLogFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Log/*.log.gz ' + simtelLogFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutLogFileDir = os.path.join(simtelDirPath_conf, 'Log',
runNumSeriesDir)
simtelOutLogFileLFN = os.path.join(simtelOutLogFileDir,
simtelLogFileName)
simtelHistFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.hdata.gz'
cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutHistFileDir = os.path.join(simtelDirPath_conf, 'Histograms',
runNumSeriesDir)
simtelOutHistFileLFN = os.path.join(simtelOutHistFileDir,
simtelHistFileName)
########### quality check on Histo Missing because it needs the NSHOW #############################################
########## quality check on Log #############################
cmd = 'zcat %s | grep Finished.' % simtelLogFileName
DIRAC.gLogger.notice('Executing system call:', cmd)
if (os.system(cmd)):
jobReport.setApplicationStatus('Log check Failed')
DIRAC.exit(-1)
################################################
from DIRAC.Core.Utilities import List
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
opsHelper = Operations()
global seList
seList = opsHelper.getValue('ProductionOutputs/SimtelProd', [])
seList = List.randomize(seList)
DIRAC.gLogger.notice('SeList is:', seList)
######### Upload simtel data/log/histo ##############################################
res = upload_to_seList(simtelOutFileLFN, simtelFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('OutputData Upload Error', simtelOutFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
res = CheckCatalogCoherence(simtelOutLogFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('Log file already exists. Removing:',
simtelOutLogFileLFN)
ret = dirac.removeFile(simtelOutLogFileLFN)
res = upload_to_seList(simtelOutLogFileLFN, simtelLogFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Upload simtel Log Error', simtelOutLogFileLFN)
DIRAC.gLogger.notice('Removing simtel data file:',
simtelOutFileLFN)
ret = dirac.removeFile(simtelOutFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
res = CheckCatalogCoherence(simtelOutHistFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('Histo file already exists. Removing:',
simtelOutHistFileLFN)
ret = dirac.removeFile(simtelOutHistFileLFN)
res = upload_to_seList(simtelOutHistFileLFN, simtelHistFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Upload simtel Histo Error',
simtelOutHistFileLFN)
DIRAC.gLogger.notice('Removing simtel data file:',
simtelOutFileLFN)
ret = dirac.removeFile(simtelOutFileLFN)
DIRAC.gLogger.notice('Removing simtel log file:',
simtelOutLogFileLFN)
ret = dirac.removeFile(simtelOutLogFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
####################################################################
if newSimtelRunFileSeriesDir:
insertRunFileSeriesMD(simtelOutFileDir, runNumTrunc)
insertRunFileSeriesMD(simtelOutLogFileDir, runNumTrunc)
insertRunFileSeriesMD(simtelOutHistFileDir, runNumTrunc)
###### simtel File level metadata ############################################
simtelFileMD = {}
simtelFileMD['runNumber'] = int(run_number)
simtelFileMD['jobID'] = jobID
simtelFileMD['simtelReturnCode'] = simtelReturnCode
result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(simtelOutLogFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(simtelOutHistFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.addFileAncestors(
{simtelOutFileLFN: {
'Ancestors': [corsikaFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{simtelOutLogFileLFN: {
'Ancestors': [corsikaFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{simtelOutHistFileLFN: {
'Ancestors': [corsikaFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD)
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
DIRAC.exit()
class SIDRegisterOutputData(ModuleBase):
""" Register output data in the FC for the SID productions
"""
def __init__(self):
super(SIDRegisterOutputData, self).__init__()
self.version = "SIDRegisterOutputData v1"
self.log = gLogger.getSubLogger( "SIDRegisterOutputData" )
self.commandTimeOut = 10 * 60
self.enable = True
self.prodOutputLFNs = []
self.swpackages = []
self.nbofevents = 0
self.luminosity = 0
self.filecatalog = FileCatalogClient()
def applicationSpecificInputs(self):
if self.step_commons.has_key('Enable'):
self.enable = self.step_commons['Enable']
if not type(self.enable) == type(True):
self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable)
self.enable = False
if self.workflow_commons.has_key('ProductionOutputData'):
self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";")
else:
self.prodOutputLFNs = []
if self.workflow_commons.has_key('SoftwarePackages'):
self.swpackages = self.workflow_commons['SoftwarePackages'].split(";")
self.nbofevents = self.NumberOfEvents #comes from ModuleBase
if self.workflow_commons.has_key('Luminosity'):
self.luminosity = self.workflow_commons['Luminosity']
return S_OK('Parameters resolved')
def execute(self):
self.log.info('Initializing %s' % self.version)
result = self.resolveInputVariables()
if not result['OK']:
self.log.error(result['Message'])
return result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('No registration of output data metadata attempted')
if len(self.prodOutputLFNs) == 0:
self.log.info('No production data found, so no metadata registration to be done')
return S_OK("No files' metadata to be registered")
self.log.verbose("Will try to set the metadata for the following files: \n %s"% string.join(self.prodOutputLFNs,
"\n"))
for files in self.prodOutputLFNs:
# elements = files.split("/")
# metaprodid = {}
# metaforfiles = {}
meta = {}
# metaen={}
# metaen['Energy']=elements[5]
# meta.update(metaen)
# energy = string.join(elements[0:6],"/")
# if self.enable:
# res = self.filecatalog.setMetadata(energy,metaen)
# if not res['OK']:
# self.log.error('Could not register metadata Energy, with value %s for %s'%(elements[4],energy))
# return res
# metaevt={}
# metaevt['EvtType']=elements[6]
# meta.update(metaevt)
# evttype = string.join(elements[0:7],"/")
# if self.enable:
# res = self.filecatalog.setMetadata(evttype,metaevt)
# if not res['OK']:
# self.log.error('Could not register metadata EvtType, with value %s for %s'%(elements[5],evttype))
# return res
# prodid = ''
#
# metadat={}
# metadat['Datatype']=elements[7]
# datatype = string.join(elements[0:8],"/")
# if self.enable:
# res = self.filecatalog.setMetadata(datatype,metadat)
# if not res['OK']:
# self.log.error('Could not register metadata Datatype, with value %s for %s'%(elements[7],datatype))
# return res
# metaprodid['ProdID'] = elements[8]
# prodid = string.join(elements[0:9],"/")
# if self.enable:
# res = self.filecatalog.setMetadata(prodid,metaprodid)
# if not res['OK']:
# self.log.error('Could not register metadata ProdID, with value %s for %s'%(elements[8],prodid))
# return res
#
if self.nbofevents:
nbevts = {}
nbevts['NumberOfEvents'] = self.nbofevents
if self.enable:
res = self.filecatalog.setMetadata(files, nbevts)
if not res['OK']:
self.log.error('Could not register metadata NumberOfEvents, with value %s for %s' % (self.nbofevents,
files))
return res
meta.update(nbevts)
if self.luminosity:
lumi = {}
lumi['Luminosity'] = self.luminosity
if self.enable:
res = self.filecatalog.setMetadata(files, lumi)
if not res['OK']:
self.log.error('Could not register metadata Luminosity, with value %s for %s'%(self.luminosity, files))
return res
meta.update(lumi)
# meta.update(metaprodid)
self.log.info("Registered %s with tags %s"%(files, meta))
###Now, set the ancestors
if self.InputData:
inputdata = self.InputData
if self.enable:
res = self.filecatalog.addFileAncestors({files : {'Ancestors' : inputdata}})
if not res['OK']:
self.log.error('Registration of Ancestors for %s failed' % files)
return res
# FIXME: in next DIRAC release, remove loop and replace key,value below by meta
#res = self.filecatalog.setMetadata(os.path.dirname(files),meta)
#if not res['OK']:
# self.log.error('Could not register metadata %s for %s'%(meta, files))
# return res
return S_OK('Output data metadata registered in catalog')
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch("p:", "run_number=", "Run Number", setRunNumber)
Script.registerSwitch("T:", "template=", "Template", setCorsikaTemplate)
Script.registerSwitch("E:", "executable=", "Executable", setExecutable)
Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig", setConfig)
Script.registerSwitch("V:", "version=", "Version", setVersion)
Script.registerSwitch("M:", "mode=", "Mode", setMode)
Script.registerSwitch("C:", "savecorsika=", "Save Corsika", setSaveCorsika)
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
Script.parseCommandLine()
global fcc, fcL, storage_element
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from CTADIRAC.Core.Workflow.Modules.CorsikaApp import CorsikaApp
from CTADIRAC.Core.Workflow.Modules.Read_CtaApp import Read_CtaApp
from DIRAC.Core.Utilities.Subprocess import systemCall
jobID = os.environ['JOBID']
jobID = int(jobID)
global jobReport
jobReport = JobReport(jobID)
###########
## Checking MD coherence
fc = FileCatalog('LcgFileCatalog')
res = fc._getCatalogConfigDetails('DIRACFileCatalog')
print 'DFC CatalogConfigDetails:', res
res = fc._getCatalogConfigDetails('LcgFileCatalog')
print 'LCG CatalogConfigDetails:', res
fcc = FileCatalogClient()
fcL = FileCatalog('LcgFileCatalog')
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
#############
simtelConfigFilesPath = 'sim_telarray/multi'
simtelConfigFile = simtelConfigFilesPath + '/multi_cta-ultra5.cfg'
#simtelConfigFile = simtelConfigFilesPath + '/multi_cta-prod1s.cfg'
createGlobalsFromConfigFiles('prodConfigFile', corsikaTemplate, version)
######################Building prod Directory Metadata #######################
resultCreateProdDirMD = createProdFileSystAndMD()
if not resultCreateProdDirMD['OK']:
DIRAC.gLogger.error('Failed to create prod Directory MD')
jobReport.setApplicationStatus('Failed to create prod Directory MD')
DIRAC.gLogger.error('Metadata coherence problem, no file produced')
DIRAC.exit(-1)
else:
print 'prod Directory MD successfully created'
######################Building corsika Directory Metadata #######################
resultCreateCorsikaDirMD = createCorsikaFileSystAndMD()
if not resultCreateCorsikaDirMD['OK']:
DIRAC.gLogger.error('Failed to create corsika Directory MD')
jobReport.setApplicationStatus('Failed to create corsika Directory MD')
DIRAC.gLogger.error(
'Metadata coherence problem, no corsikaFile produced')
DIRAC.exit(-1)
else:
print 'corsika Directory MD successfully created'
############ Producing Corsika File
global CorsikaSimtelPack
CorsikaSimtelPack = os.path.join('corsika_simhessarray', version,
'corsika_simhessarray')
install_CorsikaSimtelPack(version, 'sim')
cs = CorsikaApp()
cs.setSoftwarePackage(CorsikaSimtelPack)
cs.csExe = executable
cs.csArguments = [
'--run-number', run_number, '--run', 'corsika', corsikaTemplate
]
corsikaReturnCode = cs.execute()
if corsikaReturnCode != 0:
DIRAC.gLogger.error('Corsika Application: Failed')
jobReport.setApplicationStatus('Corsika Application: Failed')
DIRAC.exit(-1)
###################### rename of corsika output file #######################
rundir = 'run' + run_number
filein = rundir + '/' + corsikaOutputFileName
corsikaFileName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.gz'
mv_cmd = 'mv ' + filein + ' ' + corsikaFileName
if (os.system(mv_cmd)):
DIRAC.exit(-1)
########################
########################
## files spread in 1000-runs subDirectories
runNum = int(run_number)
subRunNumber = '%03d' % runNum
runNumModMille = runNum % 1000
runNumTrunc = (runNum - runNumModMille) / 1000
runNumSeriesDir = '%03dxxx' % runNumTrunc
print 'runNumSeriesDir=', runNumSeriesDir
### create corsika tar luisa ####################
corsikaTarName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.tar.gz'
filetar1 = rundir + '/' + 'input'
filetar2 = rundir + '/' + 'DAT' + run_number + '.dbase'
filetar3 = rundir + '/run' + str(int(run_number)) + '.log'
cmdTuple = [
'/bin/tar', 'zcf', corsikaTarName, filetar1, filetar2, filetar3
]
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute tar')
DIRAC.exit(-1)
######################################################
corsikaOutFileDir = os.path.join(corsikaDirPath, particle, 'Data',
runNumSeriesDir)
corsikaOutFileLFN = os.path.join(corsikaOutFileDir, corsikaFileName)
corsikaRunNumberSeriesDirExist = fcc.isDirectory(
corsikaOutFileDir)['Value']['Successful'][corsikaOutFileDir]
newCorsikaRunNumberSeriesDir = (
corsikaRunNumberSeriesDirExist != True
) # if new runFileSeries, will need to add new MD
#### create a file to DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK ################
f = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w')
f.close()
if savecorsika == 'True':
DIRAC.gLogger.notice('Put and register corsika File in LFC and DFC:',
corsikaOutFileLFN)
ret = dirac.addFile(corsikaOutFileLFN, corsikaFileName,
storage_element)
res = CheckCatalogCoherence(corsikaOutFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
# put and register corsikaTarFile:
corsikaTarFileDir = os.path.join(corsikaDirPath, particle, 'Log',
runNumSeriesDir)
corsikaTarFileLFN = os.path.join(corsikaTarFileDir, corsikaTarName)
##### If storage element is IN2P3-tape save simtel file on disk ###############
if storage_element == 'CC-IN2P3-Tape':
storage_element = 'CC-IN2P3-Disk'
DIRAC.gLogger.notice(
'Put and register corsikaTar File in LFC and DFC:',
corsikaTarFileLFN)
ret = dirac.addFile(corsikaTarFileLFN, corsikaTarName, storage_element)
####Checking and restablishing catalog coherence #####################
res = CheckCatalogCoherence(corsikaTarFileLFN)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Job failed: Catalog Coherence problem found')
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Error during addFile call:', ret['Message'])
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
######################################################################
if newCorsikaRunNumberSeriesDir:
insertRunFileSeriesMD(corsikaOutFileDir, runNumTrunc)
insertRunFileSeriesMD(corsikaTarFileDir, runNumTrunc)
###### insert corsika File Level metadata ############################################
corsikaFileMD = {}
corsikaFileMD['runNumber'] = int(run_number)
corsikaFileMD['jobID'] = jobID
corsikaFileMD['corsikaReturnCode'] = corsikaReturnCode
corsikaFileMD['nbShowers'] = nbShowers
result = fcc.setMetadata(corsikaOutFileLFN, corsikaFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(corsikaTarFileLFN, corsikaFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
##### Exit now if only corsika simulation required
if (mode == 'corsika_standalone'):
DIRAC.exit()
############ Producing SimTel File
######################Building simtel Directory Metadata #######################
cfg_dict = {
"4MSST": 'cta-prod2-4m-dc',
"SCSST": 'cta-prod2-sc-sst',
"STD": 'cta-prod2',
"NSBX3": 'cta-prod2',
"ASTRI": 'cta-prod2-astri',
"SCMST": 'cta-prod2-sc3',
"NORTH": 'cta-prod2n'
}
if simtelConfig == "6INROW":
all_configs = ["4MSST", "SCSST", "ASTRI", "NSBX3", "STD", "SCMST"]
elif simtelConfig == "5INROW":
all_configs = ["4MSST", "SCSST", "ASTRI", "NSBX3", "STD"]
elif simtelConfig == "3INROW":
all_configs = ["SCSST", "STD", "SCMST"]
else:
all_configs = [simtelConfig]
############################################
#for current_conf in all_configs:
#DIRAC.gLogger.notice('current conf is',current_conf)
#if current_conf == "SCMST":
#current_version = version + '_sc3'
#DIRAC.gLogger.notice('current version is', current_version)
#if os.path.isdir('sim_telarray'):
#DIRAC.gLogger.notice('Package found in the local area. Removing package...')
#cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run'
#if(os.system(cmd)):
#DIRAC.exit( -1 )
#install_CorsikaSimtelPack(current_version)
#else:
#current_version = version
#DIRAC.gLogger.notice('current version is', current_version)
#############################################################
for current_conf in all_configs:
DIRAC.gLogger.notice('current conf is', current_conf)
if current_conf == "SCMST":
current_version = version + '_sc3'
DIRAC.gLogger.notice('current version is', current_version)
installSoftwareEnviron(CorsikaSimtelPack, workingArea(), 'sim-sc3')
else:
current_version = version
DIRAC.gLogger.notice('current version is', current_version)
########################################################
global simtelDirPath
global simtelProdVersion
simtelProdVersion = current_version + '_simtel'
simtelDirPath = os.path.join(corsikaParticleDirPath, simtelProdVersion)
resultCreateSimtelDirMD = createSimtelFileSystAndMD(current_conf)
if not resultCreateSimtelDirMD['OK']:
DIRAC.gLogger.error('Failed to create simtelArray Directory MD')
jobReport.setApplicationStatus(
'Failed to create simtelArray Directory MD')
DIRAC.gLogger.error(
'Metadata coherence problem, no simtelArray File produced')
DIRAC.exit(-1)
else:
DIRAC.gLogger.notice('simtel Directory MD successfully created')
############## check simtel data file LFN exists ########################
simtelFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.simtel.gz'
simtelDirPath_conf = simtelDirPath + '_' + current_conf
simtelOutFileDir = os.path.join(simtelDirPath_conf, 'Data',
runNumSeriesDir)
simtelOutFileLFN = os.path.join(simtelOutFileDir, simtelFileName)
res = CheckCatalogCoherence(simtelOutFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('Current conf already done', current_conf)
continue
#### execute simtelarray ################
fd = open('run_sim.sh', 'w')
fd.write("""#! /bin/sh
source ./Corsika_simhessarrayEnv.sh
export SVNPROD2=$PWD
export SVNTAG=SVN-PROD2_rev10503
export CORSIKA_IO_BUFFER=800MB
cp ../grid_prod2-repro.sh .
ln -s ../%s
ln -s ../$SVNTAG
./grid_prod2-repro.sh %s %s""" %
(corsikaFileName, corsikaFileName, current_conf))
fd.close()
####################################
os.system('chmod u+x run_sim.sh')
cmdTuple = ['./run_sim.sh']
ret = systemCall(0, cmdTuple, sendOutputSimTel)
simtelReturnCode, stdout, stderr = ret['Value']
if (os.system('grep Broken simtel.log') == 0):
DIRAC.gLogger.error('Broken string found in simtel.log')
jobReport.setApplicationStatus('Broken pipe')
DIRAC.exit(-1)
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute run_sim.sh')
DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode)
DIRAC.exit(-1)
## check simtel data/log/histo Output File exist
cfg = cfg_dict[current_conf]
#cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Data/*.simtel.gz ' + simtelFileName
if current_conf == "SCMST":
cmdprefix = 'mv sim-sc3/Data/sim_telarray/' + cfg + '/0.0deg/'
else:
cmdprefix = 'mv sim/Data/sim_telarray/' + cfg + '/0.0deg/'
cmd = cmdprefix + 'Data/*' + cfg + '_*.simtel.gz ' + simtelFileName
if (os.system(cmd)):
DIRAC.exit(-1)
############################################
simtelRunNumberSeriesDirExist = fcc.isDirectory(
simtelOutFileDir)['Value']['Successful'][simtelOutFileDir]
newSimtelRunFileSeriesDir = (
simtelRunNumberSeriesDirExist != True
) # if new runFileSeries, will need to add new MD
simtelLogFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz'
#cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Log/*.log.gz ' + simtelLogFileName
cmd = cmdprefix + 'Log/*' + cfg + '_*.log.gz ' + simtelLogFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutLogFileDir = os.path.join(simtelDirPath_conf, 'Log',
runNumSeriesDir)
simtelOutLogFileLFN = os.path.join(simtelOutLogFileDir,
simtelLogFileName)
simtelHistFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.hdata.gz'
#cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName
cmd = cmdprefix + 'Histograms/*' + cfg + '_*.hdata.gz ' + simtelHistFileName
if (os.system(cmd)):
DIRAC.exit(-1)
simtelOutHistFileDir = os.path.join(simtelDirPath_conf, 'Histograms',
runNumSeriesDir)
simtelOutHistFileLFN = os.path.join(simtelOutHistFileDir,
simtelHistFileName)
########### quality check on Histo #############################################
fd = open('check_histo.sh', 'w')
fd.write("""#! /bin/sh
nsim=$(list_histograms %s|fgrep 'Histogram 6 '|sed 's/^.*contents: //'| sed 's:/.*$::')
nevents=%d
if [ $nsim -lt $(( $nevents - 20 )) ]; then
echo 'nsim found:' $nsim
echo 'nsim expected:' $nevents
exit 1
else
echo 'nsim found:' $nsim
echo 'nsim expected:' $nevents
fi
""" % (simtelHistFileName, int(nbShowers) * int(cscat)))
fd.close()
ret = getSoftwareEnviron(CorsikaSimtelPack)
if not ret['OK']:
error = ret['Message']
DIRAC.gLogger.error(error, CorsikaSimtelPack)
DIRAC.exit(-1)
corsikaEnviron = ret['Value']
os.system('chmod u+x check_histo.sh')
cmdTuple = ['./check_histo.sh']
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput, env=corsikaEnviron)
checkHistoReturnCode, stdout, stderr = ret['Value']
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute check_histo.sh')
DIRAC.gLogger.error('check_histo.sh status is:',
checkHistoReturnCode)
DIRAC.exit(-1)
if (checkHistoReturnCode != 0):
DIRAC.gLogger.error('Failure during check_histo.sh')
DIRAC.gLogger.error('check_histo.sh status is:',
checkHistoReturnCode)
jobReport.setApplicationStatus('Histo check Failed')
DIRAC.exit(-1)
########## quality check on Log #############################
cmd = 'zcat %s | grep Finished.' % simtelLogFileName
DIRAC.gLogger.notice('Executing system call:', cmd)
if (os.system(cmd)):
jobReport.setApplicationStatus('Log check Failed')
DIRAC.exit(-1)
################################################
from DIRAC.Core.Utilities import List
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
opsHelper = Operations()
global seList
seList = opsHelper.getValue('ProductionOutputs/SimtelProd', [])
seList = List.randomize(seList)
DIRAC.gLogger.notice('SeList is:', seList)
######### Upload simtel data/log/histo ##############################################
res = upload_to_seList(simtelOutFileLFN, simtelFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('OutputData Upload Error', simtelOutFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
res = CheckCatalogCoherence(simtelOutLogFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('Log file already exists. Removing:',
simtelOutLogFileLFN)
ret = dirac.removeFile(simtelOutLogFileLFN)
res = upload_to_seList(simtelOutLogFileLFN, simtelLogFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Upload simtel Log Error', simtelOutLogFileLFN)
DIRAC.gLogger.notice('Removing simtel data file:',
simtelOutFileLFN)
ret = dirac.removeFile(simtelOutFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
res = CheckCatalogCoherence(simtelOutHistFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('Histo file already exists. Removing:',
simtelOutHistFileLFN)
ret = dirac.removeFile(simtelOutHistFileLFN)
res = upload_to_seList(simtelOutHistFileLFN, simtelHistFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Upload simtel Histo Error',
simtelOutHistFileLFN)
DIRAC.gLogger.notice('Removing simtel data file:',
simtelOutFileLFN)
ret = dirac.removeFile(simtelOutFileLFN)
DIRAC.gLogger.notice('Removing simtel log file:',
simtelOutLogFileLFN)
ret = dirac.removeFile(simtelOutLogFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
# simtelRunNumberSeriesDirExist = fcc.isDirectory(simtelOutFileDir)['Value']['Successful'][simtelOutFileDir]
# newSimtelRunFileSeriesDir = (simtelRunNumberSeriesDirExist != True) # if new runFileSeries, will need to add new MD
if newSimtelRunFileSeriesDir:
print 'insertRunFileSeriesMD'
insertRunFileSeriesMD(simtelOutFileDir, runNumTrunc)
insertRunFileSeriesMD(simtelOutLogFileDir, runNumTrunc)
insertRunFileSeriesMD(simtelOutHistFileDir, runNumTrunc)
else:
print 'NotinsertRunFileSeriesMD'
###### simtel File level metadata ############################################
simtelFileMD = {}
simtelFileMD['runNumber'] = int(run_number)
simtelFileMD['jobID'] = jobID
simtelFileMD['simtelReturnCode'] = simtelReturnCode
result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(simtelOutLogFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(simtelOutHistFileLFN, simtelFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
if savecorsika == 'True':
result = fcc.addFileAncestors(
{simtelOutFileLFN: {
'Ancestors': [corsikaOutFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{simtelOutLogFileLFN: {
'Ancestors': [corsikaOutFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{simtelOutHistFileLFN: {
'Ancestors': [corsikaOutFileLFN]
}})
print 'result addFileAncestor:', result
##### Exit now if only corsika simulation required
if (mode == 'corsika_simtel'):
continue
######### run read_cta #######################################
rcta = Read_CtaApp()
rcta.setSoftwarePackage(CorsikaSimtelPack)
rcta.rctaExe = 'read_cta'
powerlaw_dict = {
'gamma': '-2.57',
'gamma_ptsrc': '-2.57',
'proton': '-2.70',
'electron': '-3.21'
}
dstFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.simtel-dst0.gz'
dstHistoFileName = particle + '_' + str(thetaP) + '_' + str(
phiP) + '_alt' + str(
obslev) + '_' + 'run' + run_number + '.hdata-dst0.gz'
## added some options starting from Armazones_2K prod.
rcta.rctaArguments = [
'-r', '4', '-u', '--integration-scheme', '4',
'--integration-window', '7,3', '--tail-cuts', '6,8', '--min-pix',
'2', '--min-amp', '20', '--type', '1,0,0,400', '--tail-cuts',
'9,12', '--min-amp', '20', '--type', '2,0,0,100', '--tail-cuts',
'8,11', '--min-amp', '19', '--type', '3,0,0,40', '--tail-cuts',
'6,9', '--min-amp', '15', '--type', '4,0,0,15', '--tail-cuts',
'3.7,5.5', '--min-amp', '8', '--type', '5,0,0,70,5.6',
'--tail-cuts', '2.4,3.2', '--min-amp', '5.6', '--dst-level', '0',
'--dst-file', dstFileName, '--histogram-file', dstHistoFileName,
'--powerlaw', powerlaw_dict[particle], simtelFileName
]
rctaReturnCode = rcta.execute()
if rctaReturnCode != 0:
DIRAC.gLogger.error('read_cta Application: Failed')
jobReport.setApplicationStatus('read_cta Application: Failed')
DIRAC.exit(-1)
######## run dst quality checks ######################################
fd = open('check_dst_histo.sh', 'w')
fd.write("""#! /bin/sh
dsthistfilename=%s
dstfile=%s
n6="$(list_histograms -h 6 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
n12001="$(list_histograms -h 12001 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
if [ $n6 -ne $n12001 ]; then
echo 'n6 found:' $n6
echo 'n12001 found:' $n12001
exit 1
else
echo 'n6 found:' $n6
echo 'n12001 found:' $n12001
fi
n12002="$(list_histograms -h 12002 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
nev="$(statio ${dstfile} | egrep '^2010' | cut -f2)"
if [ -z "$nev" ]; then nev="0"; fi
if [ $nev -ne $n12002 ]; then
echo 'nev found:' $nev
echo 'n12002 found:' $n12002
exit 1
else
echo 'nev found:' $nev
echo 'n12002 found:' $n12002
fi
""" % (dstHistoFileName, dstFileName))
fd.close()
os.system('chmod u+x check_dst_histo.sh')
cmdTuple = ['./check_dst_histo.sh']
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput, env=corsikaEnviron)
checkHistoReturnCode, stdout, stderr = ret['Value']
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute check_dst_histo.sh')
DIRAC.gLogger.error('check_dst_histo.sh status is:',
checkHistoReturnCode)
DIRAC.exit(-1)
if (checkHistoReturnCode != 0):
DIRAC.gLogger.error('Failure during check_dst_histo.sh')
DIRAC.gLogger.error('check_dst_histo.sh status is:',
checkHistoReturnCode)
jobReport.setApplicationStatus('Histo check Failed')
DIRAC.exit(-1)
############create MD and upload dst data/histo ##########################################################
global dstDirPath
global dstProdVersion
dstProdVersion = current_version + '_dst'
dstDirPath = os.path.join(simtelDirPath_conf, dstProdVersion)
dstOutFileDir = os.path.join(dstDirPath, 'Data', runNumSeriesDir)
dstOutFileLFN = os.path.join(dstOutFileDir, dstFileName)
resultCreateDstDirMD = createDstFileSystAndMD()
if not resultCreateDstDirMD['OK']:
DIRAC.gLogger.error('Failed to create Dst Directory MD')
jobReport.setApplicationStatus('Failed to create Dst Directory MD')
DIRAC.gLogger.error(
'Metadata coherence problem, no Dst File produced')
DIRAC.exit(-1)
else:
DIRAC.gLogger.notice('Dst Directory MD successfully created')
############################################################
res = CheckCatalogCoherence(dstOutFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('dst file already exists. Removing:',
dstOutFileLFN)
ret = dirac.removeFile(dstOutFileLFN)
res = upload_to_seList(dstOutFileLFN, dstFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Upload dst Error', dstOutFileLFN)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
##############################################################
dstHistoFileDir = os.path.join(dstDirPath, 'Histograms',
runNumSeriesDir)
dstHistoFileLFN = os.path.join(dstHistoFileDir, dstHistoFileName)
res = CheckCatalogCoherence(dstHistoFileLFN)
if res == DIRAC.S_OK:
DIRAC.gLogger.notice('dst histo file already exists. Removing:',
dstHistoFileLFN)
ret = dirac.removeFile(dstHistoFileLFN)
res = upload_to_seList(dstHistoFileLFN, dstHistoFileName)
if res != DIRAC.S_OK:
DIRAC.gLogger.error('Upload dst Error', dstHistoFileName)
jobReport.setApplicationStatus('OutputData Upload Error')
DIRAC.exit(-1)
########### Insert RunNumSeries MD ##########################
dstRunNumberSeriesDirExist = fcc.isDirectory(
dstOutFileDir)['Value']['Successful'][dstOutFileDir]
newDstRunFileSeriesDir = (
dstRunNumberSeriesDirExist != True
) # if new runFileSeries, will need to add new MD
if newDstRunFileSeriesDir:
insertRunFileSeriesMD(dstOutFileDir, runNumTrunc)
insertRunFileSeriesMD(dstHistoFileDir, runNumTrunc)
####### dst File level metadata ###############################################
dstFileMD = {}
dstFileMD['runNumber'] = int(run_number)
dstFileMD['jobID'] = jobID
dstFileMD['rctaReturnCode'] = rctaReturnCode
result = fcc.setMetadata(dstOutFileLFN, dstFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
result = fcc.setMetadata(dstHistoFileLFN, dstFileMD)
print "result setMetadata=", result
if not result['OK']:
print 'ResultSetMetadata:', result['Message']
########## set the ancestors for dst #####################################
result = fcc.addFileAncestors(
{dstOutFileLFN: {
'Ancestors': [simtelOutFileLFN]
}})
print 'result addFileAncestor:', result
result = fcc.addFileAncestors(
{dstHistoFileLFN: {
'Ancestors': [simtelOutFileLFN]
}})
print 'result addFileAncestor:', result
######################################################
DIRAC.exit()
