def getSoftwareEnviron(package, environ=None):
"""
Check shared and local Area for the given package and return
the new environ dictionary
"""
if environ == None:
environ = dict(os.environ)
elif type(environ) == DictType:
environ = dict(environ)
else:
return DIRAC.S_ERROR('environ argument must be a dictionary')
gLogger.notice('Getting environment for', package)
for area in [workingArea(), sharedArea(), localArea()]:
if area:
if not checkSoftwarePackage(package, area)['OK']:
continue
fileName = _getEnvFileName(package, area)
base = os.path.splitext(fileName)[0]
gLogger.info('Sourcing file to get environment:', fileName)
result = DIRAC.sourceEnv(TIMEOUT, [base], environ)
if not result['OK']:
return result
if result['stdout']:
gLogger.info(result['stdout'])
return DIRAC.S_OK(result['outputEnv'])
# If we get here it means the package has not been found
return DIRAC.S_ERROR('Failed to get Environment')
def setupWorkflow(self, debug=False):
""" Setup job workflow by defining the sequence of all executables
All parameters shall have been defined before that method is called.
"""
# step 1 -- to be removed -- debug only
iStep = 1
if debug:
lsStep = self.setExecutable( '/bin/ls -alhtr', logFile = 'LS_Init_Log.txt' )
lsStep['Value']['name']='Step%i_LS_Init'%iStep
lsStep['Value']['descr_short']='list files in working directory'
iStep+=1
# step 2
swStep = self.setExecutable( '$DIRACROOT/scripts/cta-prod3-setupsw',
arguments='%s %s'% (self.package, self.version),\
logFile='SetupSoftware_Log.txt')
swStep['Value']['name'] = 'Step%i_SetupSoftware' % iStep
swStep['Value']['descr_short'] = 'Setup software'
iStep+=1
# step 2bis
# arguments are nbFiles=0 (not used) and fileSize=100kB
inStep = self.setExecutable( '$DIRACROOT/scripts/cta-prod3-verifysteps', \
arguments = 'analysisinputs 0 100', \
logFile = 'Verify_ReadCtaInputs_Log.txt' )
inStep['Value']['name'] = 'Step%i_VerifyReadCtaInputs' % iStep
inStep['Value']['descr_short'] = 'Verify ReadCta Inputs'
iStep += 1
# step 3
res = DIRAC.sourceEnv(600, ['prod3_types'], {} )
read_cta_opts=res['outputEnv']['read_cta_opts']
rctaStep = self.setExecutable( './dirac_prod3_read_cta', \
arguments = "-q -r 4 -u --min-trg-tel 2 %s" % (read_cta_opts),
logFile = 'ReadCta_Log.txt' )
rctaStep['Value']['name'] = 'Step%i_ReadCta' % iStep
rctaStep['Value']['descr_short'] = 'Run ReadCta'
iStep += 1
# step 4
# ## the order of the metadata dictionary is important, since it's used to build the directory structure
mdjson = json.dumps( self.metadata )
metadatafield = {'array_layout':'VARCHAR(128)', 'site':'VARCHAR(128)', 'particle':'VARCHAR(128)', \
'phiP':'float', 'thetaP': 'float', 'analysis_prog':'VARCHAR(128)', 'analysis_prog_version':'VARCHAR(128)'}
mdfieldjson = json.dumps( metadatafield )
fmdjson = json.dumps( self.filemetadata )
## Upload Data files
dmStep = self.setExecutable( '$DIRACROOT/CTADIRAC/Core/scripts/cta-analysis-managedata.py',
arguments = "'%s' '%s' '%s' %s '%s' %s" % ( mdjson, mdfieldjson, fmdjson, self.basepath, self.outputpattern, self.package ),
logFile = 'DataManagement_Log.txt' )
dmStep['Value']['name'] = 'Step%i_DataManagement' % iStep
dmStep['Value']['descr_short'] = 'Save files to SE and register them in DFC'
iStep += 1
# Upload Histogram files and use 'Log' as outputType
self.outputpattern = './*hdata-dst0.gz'
dmStep = self.setExecutable( '$DIRACROOT/CTADIRAC/Core/scripts/cta-analysis-managedata.py',
arguments = "'%s' '%s' '%s' %s '%s' %s %s" % ( mdjson, mdfieldjson, fmdjson, self.basepath, self.outputpattern, self.package, 'Log'),
logFile = 'DataManagement_Log.txt' )
dmStep['Value']['name'] = 'Step%i_DataManagement' % iStep
dmStep['Value']['descr_short'] = 'Save files to SE and register them in DFC'
iStep += 1
