def __init__(self, **kwargs):
GridBackend.__init__(self, catalogue_prefix='', **kwargs)
from DIRAC.Core.Base import Script
Script.initialize()
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient
self.pm = ProxyManagerClient()
proxy = self.pm.getUserProxiesInfo()
if not proxy['OK']:
raise BackendException("Proxy error.")
from DIRAC.Interfaces.API.Dirac import Dirac
self.dirac = Dirac()
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
self.fc = FileCatalog()
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
self.dm = DataManager()
self._xattr_cmd = sh.Command('gfal-xattr').bake(_tty_out=False)
self._replica_checksum_cmd = sh.Command('gfal-sum').bake(_tty_out=False)
self._bringonline_cmd = sh.Command('gfal-legacy-bringonline').bake(_tty_out=False)
self._cp_cmd = sh.Command('gfal-copy').bake(_tty_out=False)
self._ls_se_cmd = sh.Command('gfal-ls').bake(color='never', _tty_out=False)
self._move_cmd = sh.Command('gfal-rename').bake(_tty_out=False)
self._mkdir_cmd = sh.Command('gfal-mkdir').bake(_tty_out=False)
self._replicate_cmd = sh.Command('dirac-dms-replicate-lfn').bake(_tty_out=False)
self._add_cmd = sh.Command('dirac-dms-add-file').bake(_tty_out=False)
def read_site_summary():
active_sites = {}
from DIRAC.Core.Base import Script
Script.initialize( False, False, False, False )
JobMonitoring = RPCClient('WorkloadManagement/JobMonitoring', timeout=600)
result = JobMonitoring.getSiteSummary()
if not result['OK']:
sys.exit("Failed to get SiteSummary: %s" % str(result['Message']))
sitesummary=result['Value']
#return sitesummary
#print sitesummary
sitelists = getSiteList()
# Write the coordinates in a temporal file
coord_file = open('sites.tmp','w')
coord_file.write("Sites,x,y")
# For now, for sites without coordinates info in DIRAC
coord_file.write("\nAdelaide,-34.910836,138.57121")
coord_file.write("\nIPHC,48.6056,7.7095")
coord_file.write("\nKEK,36.06,140.2303")
for site in sorted(sitelists):
ele = site.split('.')
result = gConfig.getOptionsDict('Resources/Sites/%s/%s' % ( ele[0], site ))
if not result['OK']:
continue
optionsdict=result['Value']
status = "production"
if 'Status' in optionsdict:
status = optionsdict['Status']
if status not in ['production', 'commissioning']:
continue
if site not in sitesummary:
continue
active_sites[site] = sitesummary[site]
x='0'
y='0'
if 'Coordinates' in optionsdict:
[y,x] = optionsdict['Coordinates'].split(':')
if x.replace('-','').replace('.','').isdigit() and y.replace('-','').replace('.','').isdigit():
active_sites[site]['Coordinates'] = [float(x),float(y)]
coord_file.write('\n'+ site.split('.')[1]+','+ str(x) + ',' + str(y))
else:
print "[WARNING:] No location info for",site
coord_file.close()
return active_sites
########################################################################
# $HeadURL$
# File : API.py
########################################################################
""" DIRAC API Base Class """
__RCSID__ = "$Id$"
from DIRAC.Core.Base import Script
Script.initialize()
from DIRAC import gLogger, S_OK, S_ERROR
from DIRAC.Core.Utilities.List import sortList
from DIRAC.Core.Security.Misc import getProxyInfo, formatProxyInfoAsString
from DIRAC.Core.Security.CS import getDNForUsername
import pprint, sys
COMPONENT_NAME = 'API'
def _printFormattedDictList(dictList, fields, uniqueField, orderBy):
""" Will print ordered the supplied field of a list of dictionaries """
orderDict = {}
fieldWidths = {}
dictFields = {}
for myDict in dictList:
for field in fields:
fieldValue = myDict[field]
if not fieldWidths.has_key(field):
fieldWidths[field] = len(str(field))
if len(str(fieldValue)) > fieldWidths[field]:
def initDIRAC( rootPath, enableDebug = False ):
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
configDict = { 'webConfig' : {} }
configDict[ 'webConfig' ]['dirac.webroot'] = rootPath
diracRootPath = os.path.realpath( os.path.dirname( os.path.dirname( rootPath ) ) )
configDict[ 'webConfig' ]['dirac.root'] = diracRootPath
if diracRootPath not in sys.path:
sys.path.append( diracRootPath )
from DIRAC.FrameworkSystem.Client.Logger import gLogger
gLogger.registerBackends( [ 'stderr' ] )
from DIRAC.Core.Base import Script
Script.registerSwitch( "r", "reload", "Reload for pylons" )
Script.localCfg.addDefaultEntry( "/DIRAC/Security/UseServerCertificate", "yes" )
Script.localCfg.addDefaultEntry( "LogColor", True )
Script.initialize( script = "Website", ignoreErrors = True, initializeMonitor = False )
gLogger._systemName = "Framework"
gLogger.initialize( "Web", "/Website" )
gLogger.setLevel( "VERBOSE" )
from DIRAC import gMonitor, gConfig, rootPath as droot
from DIRAC.Core.Utilities import CFG
from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions
gMonitor.setComponentType( gMonitor.COMPONENT_WEB )
gMonitor.initialize()
gMonitor.registerActivity( "pagesServed", "Pages served", "Framework", "pages", gMonitor.OP_SUM )
gLogger.info( "DIRAC Initialized" )
configDict['portalVersion'] = portalVersion( rootPath )
gLogger.info( "DIRAC portal version: %s" % configDict['portalVersion'] )
extModules = [ '%sDIRAC' % module for module in getCSExtensions() ]
#Load web.cfg of modules
cfgFilePaths = [ os.path.join( droot, "etc", "web.cfg" ) ]
for extModule in extModules:
gLogger.info( "Adding web.cfg for %s extension" % extModule )
extModulePath = os.path.join( diracRootPath, extModule )
webCFGPath = os.path.join( extModulePath, "Web", "web.cfg" )
cfgFilePaths.append( webCFGPath )
for systemDir in os.listdir( extModulePath ):
webCFGSystemPath = os.path.join( extModulePath, systemDir, "Web", "web.cfg" )
cfgFilePaths.append( webCFGSystemPath )
webCFG = CFG.CFG()
for webCFGPath in cfgFilePaths:
if not os.path.isfile( webCFGPath ):
gLogger.warn( "%s does not exist" % webCFGPath )
else:
gLogger.info( "Loading %s" % webCFGPath )
modCFG = CFG.CFG().loadFromFile( webCFGPath )
if modCFG.getOption( 'Website/AbsoluteDefinition', False ):
gLogger.info( "CFG %s is absolute" % webCFGPath )
webCFG = modCFG
else:
webCFG = webCFG.mergeWith( modCFG )
gConfig.loadCFG( webCFG )
gLogger.showHeaders( True )
gLogger._gLogger__initialized = False
gLogger.initialize( "Web", "/Website" )
#Define the controllers, templates and public directories
for type in ( 'controllers', 'templates', 'public' ):
configDict[ type ] = []
for extModule in extModules:
extModulePath = os.path.join( diracRootPath, extModule )
typePath = os.path.join( extModulePath, "Web", type )
if os.path.isdir( typePath ):
gLogger.info( "Adding %s path for module %s" % ( type, extModule ) )
configDict[ type ].append( typePath )
for systemDir in os.listdir( extModulePath ):
systemTypePath = os.path.join( extModulePath, systemDir, "Web", type )
if os.path.isdir( systemTypePath ):
gLogger.info( "Adding %s path for system %s in module %s" % ( type, systemDir, extModule ) )
configDict[ type ].append( systemTypePath )
#End of extensions
configDict[ type ].append( os.path.join( rootPath, type ) )
#Load debug.cfg?
if enableDebug:
debugCFGPath = os.path.join( rootPath, "debug.cfg" )
if os.path.isfile( debugCFGPath ):
gLogger.info( "Loading debug cfg file at %s" % debugCFGPath )
gConfig.loadFile( debugCFGPath )
gLogger.info( "Extension modules loaded" )
return configDict
######################################################################## # $HeadURL$ # File : DiracAdmin.py # Author : Stuart Paterson ######################################################################## """DIRAC Administrator API Class All administrative functionality is exposed through the DIRAC Admin API. Examples include site banning and unbanning, WMS proxy uploading etc. """ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script Script.initialize() import DIRAC from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup from DIRAC.Core.DISET.RPCClient import RPCClient from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager from DIRAC.Core.Utilities.SiteCEMapping import getSiteCEMapping from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient from DIRAC.Core.Security.X509Chain import X509Chain from DIRAC.Core.Security import Locations, CS from DIRAC import gConfig, gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities.Grid import ldapSite, ldapCluster, ldapCE, ldapService from DIRAC.Core.Utilities.Grid import ldapCEState, ldapCEVOView, ldapSA
def initDIRAC(rootPath, enableDebug=False):
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
configDict = {'webConfig': {}}
configDict['webConfig']['dirac.webroot'] = rootPath
diracRootPath = os.path.realpath(os.path.dirname(
os.path.dirname(rootPath)))
configDict['webConfig']['dirac.root'] = diracRootPath
if diracRootPath not in sys.path:
sys.path.append(diracRootPath)
from DIRAC.FrameworkSystem.Client.Logger import gLogger
gLogger.registerBackends(['stderr'])
from DIRAC.Core.Base import Script
Script.registerSwitch("r", "reload", "Reload for pylons")
Script.localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate",
"yes")
Script.localCfg.addDefaultEntry("LogColor", True)
Script.initialize(script="Website",
ignoreErrors=True,
initializeMonitor=False)
gLogger._systemName = "Framework"
gLogger.initialize("Web", "/Website")
gLogger.setLevel("VERBOSE")
from DIRAC import gMonitor, gConfig, rootPath as droot
from DIRAC.Core.Utilities import CFG
from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions
gMonitor.setComponentType(gMonitor.COMPONENT_WEB)
gMonitor.initialize()
gMonitor.registerActivity("pagesServed", "Pages served", "Framework",
"pages", gMonitor.OP_SUM)
gLogger.info("DIRAC Initialized")
configDict['portalVersion'] = portalVersion(rootPath)
gLogger.info("DIRAC portal version: %s" % configDict['portalVersion'])
extModules = ['%sDIRAC' % module for module in getCSExtensions()]
#Load web.cfg of modules
cfgFilePaths = [os.path.join(droot, "etc", "web.cfg")]
for extModule in extModules:
gLogger.info("Adding web.cfg for %s extension" % extModule)
extModulePath = os.path.join(diracRootPath, extModule)
webCFGPath = os.path.join(extModulePath, "Web", "web.cfg")
cfgFilePaths.append(webCFGPath)
for systemDir in os.listdir(extModulePath):
webCFGSystemPath = os.path.join(extModulePath, systemDir, "Web",
"web.cfg")
cfgFilePaths.append(webCFGSystemPath)
webCFG = CFG.CFG()
for webCFGPath in cfgFilePaths:
if not os.path.isfile(webCFGPath):
gLogger.warn("%s does not exist" % webCFGPath)
else:
gLogger.info("Loading %s" % webCFGPath)
modCFG = CFG.CFG().loadFromFile(webCFGPath)
if modCFG.getOption('Website/AbsoluteDefinition', False):
gLogger.info("CFG %s is absolute" % webCFGPath)
webCFG = modCFG
else:
webCFG = webCFG.mergeWith(modCFG)
gConfig.loadCFG(webCFG)
gLogger.showHeaders(True)
gLogger._gLogger__initialized = False
gLogger.initialize("Web", "/Website")
#Define the controllers, templates and public directories
for type in ('controllers', 'templates', 'public'):
configDict[type] = []
for extModule in extModules:
extModulePath = os.path.join(diracRootPath, extModule)
typePath = os.path.join(extModulePath, "Web", type)
if os.path.isdir(typePath):
gLogger.info("Adding %s path for module %s" %
(type, extModule))
configDict[type].append(typePath)
for systemDir in os.listdir(extModulePath):
systemTypePath = os.path.join(extModulePath, systemDir, "Web",
type)
if os.path.isdir(systemTypePath):
gLogger.info("Adding %s path for system %s in module %s" %
(type, systemDir, extModule))
configDict[type].append(systemTypePath)
#End of extensions
configDict[type].append(os.path.join(rootPath, type))
#Load debug.cfg?
if enableDebug:
debugCFGPath = os.path.join(rootPath, "debug.cfg")
if os.path.isfile(debugCFGPath):
gLogger.info("Loading debug cfg file at %s" % debugCFGPath)
gConfig.loadFile(debugCFGPath)
gLogger.info("Extension modules loaded")
return configDict
def main():
from DIRAC.Core.Base import Script
Script.initialize()
DIRAC.gLogger.notice('Platform is:')
os.system('dirac-platform')
from CTADIRAC.Core.Workflow.Modules.Read_CtaApp import Read_CtaApp
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
version = sys.argv[3]
DIRAC.gLogger.notice('Version:', version)
install_CorsikaSimtelPack(version)
######### run read_cta #######################################
rcta = Read_CtaApp()
CorsikaSimtelPack = os.path.join('corsika_simhessarray', version,
'corsika_simhessarray')
rcta.setSoftwarePackage(CorsikaSimtelPack)
rcta.rctaExe = 'read_cta'
# add arguments for read_cta specified by user ######
args = []
rctaparfile = open('read_cta.par', 'r').readlines()
for line in rctaparfile:
for word in line.split():
args.append(word)
simtelFileLFN = sys.argv[-1].split('ParametricInputData=LFN:')[1]
simtelFileName = os.path.basename(simtelFileLFN)
dstFileName = simtelFileName.replace('simtel.gz', 'simtel-dst0.gz')
dstHistoFileName = simtelFileName.replace('simtel.gz', 'hdata-dst0.gz')
args.extend([
'--dst-file', dstFileName, '--histogram-file', dstHistoFileName,
simtelFileName
])
rcta.rctaArguments = args
rctaReturnCode = rcta.execute()
if rctaReturnCode != 0:
DIRAC.gLogger.error('read_cta Application: Failed')
jobReport.setApplicationStatus('read_cta Application: Failed')
DIRAC.exit(-1)
#################################################################
from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron
ret = getSoftwareEnviron(CorsikaSimtelPack)
if not ret['OK']:
error = ret['Message']
DIRAC.gLogger.error(error, CorsikaSimtelPack)
DIRAC.exit(-1)
read_ctaEnviron = ret['Value']
######## run dst quality checks ######################################
fd = open('check_dst_histo.sh', 'w')
fd.write("""#! /bin/sh
dsthistfilename=%s
dstfile=%s
n6="$(list_histograms -h 6 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
n12001="$(list_histograms -h 12001 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
if [ $n6 -ne $n12001 ]; then
echo 'n6 found:' $n6
echo 'n12001 found:' $n12001
exit 1
else
echo 'n6 found:' $n6
echo 'n12001 found:' $n12001
fi
n12002="$(list_histograms -h 12002 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')"
nev="$(statio ${dstfile} | egrep '^2010' | cut -f2)"
if [ -z "$nev" ]; then nev="0"; fi
if [ $nev -ne $n12002 ]; then
echo 'nev found:' $nev
echo 'n12002 found:' $n12002
exit 1
else
echo 'nev found:' $nev
echo 'n12002 found:' $n12002
fi
""" % (dstHistoFileName, dstFileName))
fd.close()
os.system('chmod u+x check_dst_histo.sh')
cmdTuple = ['./check_dst_histo.sh']
DIRAC.gLogger.notice('Executing command tuple:', cmdTuple)
ret = systemCall(0, cmdTuple, sendOutput, env=read_ctaEnviron)
checkHistoReturnCode, stdout, stderr = ret['Value']
if not ret['OK']:
DIRAC.gLogger.error('Failed to execute check_dst_histo.sh')
DIRAC.gLogger.error('check_dst_histo.sh status is:',
checkHistoReturnCode)
DIRAC.exit(-1)
if (checkHistoReturnCode != 0):
DIRAC.gLogger.error('Failure during check_dst_histo.sh')
DIRAC.gLogger.error('check_dst_histo.sh status is:',
checkHistoReturnCode)
jobReport.setApplicationStatus('Histo check Failed')
DIRAC.exit(-1)
DIRAC.exit()
#!/usr/bin/env python
# requires DIRAC UI to be setup and valid proxy
# takes output from listsolid.py as input
# returns a list of files that are on the SE,
# but not in the catalogue
# optional: registers any files that are present on the SE
# but missing from the catalogue
import sys
import uuid
import getopt
# DIRAC does not work otherwise
from DIRAC.Core.Base import Script
# Script.parseCommandLine( ignoreErrors = True )
Script.initialize() # this lets me use getopt
# end of DIRAC setup
DIRAC_SE_NAME="BEgrid-ULB-VUB-disk"
# DIRAC_SE_NAME="UKI-LT2-IC-HEP-disk"
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
def add_catalog_entry(fc, fileDict, logfile):
result = fc.addFile(fileDict)
# TODO: handle the return code as an exception
if not result["OK"]:
logfile.write(result)
print result
return
def main():
from DIRAC.Core.Base import Script
Script.initialize()
DIRAC.gLogger.notice('Platform is:')
os.system('dirac-platform')
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from CTADIRAC.Core.Workflow.Modules.EvnDispApp import EvnDispApp
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int(jobID)
jobReport = JobReport(jobID)
version = sys.argv[3]
DIRAC.gLogger.notice('Version:', version)
EvnDispPack = os.path.join('evndisplay', version, 'evndisplay')
packs = [EvnDispPack]
for package in packs:
DIRAC.gLogger.notice('Checking:', package)
if checkSoftwarePackage(package, sharedArea())['OK']:
DIRAC.gLogger.notice('Package found in Shared Area:', package)
installSoftwareEnviron(package, sharedArea())
continue
else:
installSoftwarePackage(package, workingArea())
DIRAC.gLogger.notice('Package found in workingArea:', package)
continue
DIRAC.gLogger.error('Check Failed for software package:', package)
DIRAC.gLogger.error('Software package not available')
DIRAC.exit(-1)
ed = EvnDispApp()
ed.setSoftwarePackage(EvnDispPack)
dstFileLFNList = sys.argv[-1].split('ParametricParameters={')[1].split(
'}')[0].replace(',', ' ')
args = []
i = 0
for word in dstFileLFNList.split():
i = i + 1
dstfile = os.path.basename(word)
###### execute evndisplay stage1 ###############
executable = sys.argv[5]
logfileName = executable + '_' + str(i) + '.log'
args = ['-sourcefile', dstfile, '-outputdirectory', 'outdir']
# add other arguments for evndisp specified by user ######
evndispparfile = open('evndisp.par', 'r').readlines()
for line in evndispparfile:
for word in line.split():
args.append(word)
execute_module(ed, executable, args)
for name in glob.glob('outdir/*.root'):
evndispOutFile = name.split('.root')[0] + '_' + str(
jobID) + '_evndisp.root'
cmd = 'mv ' + name + ' ' + os.path.basename(evndispOutFile)
if (os.system(cmd)):
DIRAC.exit(-1)
########### quality check on Log #############################################
cmd = 'mv ' + executable + '.log' + ' ' + logfileName
if (os.system(cmd)):
DIRAC.exit(-1)
fd = open('check_log.sh', 'w')
fd.write("""#! /bin/sh
if grep -i "error" %s; then
exit 1
fi
if grep "Final checks on result file (seems to be OK):" %s; then
exit 0
else
exit 1
fi
""" % (logfileName, logfileName))
fd.close()
os.system('chmod u+x check_log.sh')
cmd = './check_log.sh'
DIRAC.gLogger.notice('Executing system call:', cmd)
if (os.system(cmd)):
jobReport.setApplicationStatus('EvnDisp Log Check Failed')
DIRAC.exit(-1)
##################################################################
########### remove the dst file #############################################
cmd = 'rm ' + dstfile
if (os.system(cmd)):
DIRAC.exit(-1)
DIRAC.exit()
#!/usr/bin/env python
import os
import sys
from DIRAC.Core.Base import Script
Script.initialize(ignoreErrors=True)
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.Job import Job
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ.get('DIRACJOBID', '0')
if not jobID:
print 'DIRAC job ID not found'
sys.exit(1)
jobReport = JobReport(jobID, 'JUNO_JobScript')
result = jobReport.setApplicationStatus(', '.join(sys.argv[1:]))
if not result['OK']:
print 'Set application status error: %s' % result
def main():
from DIRAC.Core.Base import Script
Script.initialize()
DIRAC.gLogger.notice('Platform is:')
os.system('dirac-platform')
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from CTADIRAC.Core.Workflow.Modules.EvnDispApp import EvnDispApp
from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage
from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron
from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea
from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea
from DIRAC.Core.Utilities.Subprocess import systemCall
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
jobID = os.environ['JOBID']
jobID = int( jobID )
jobReport = JobReport( jobID )
version = sys.argv[3]
DIRAC.gLogger.notice( 'Version:', version )
EvnDispPack = os.path.join('evndisplay',version,'evndisplay')
packs = [EvnDispPack]
for package in packs:
DIRAC.gLogger.notice( 'Checking:', package )
if checkSoftwarePackage( package, sharedArea() )['OK']:
DIRAC.gLogger.notice( 'Package found in Shared Area:', package )
installSoftwareEnviron( package, sharedArea() )
# cmd = 'cp -r ' + os.path.join(sharedArea(),'evndisplay',version,'EVNDISP.CTA.runparameter') + ' .'
# if(os.system(cmd)):
# DIRAC.exit( -1 )
# cmd = 'cp -r ' + os.path.join(sharedArea(),'evndisplay',version,'Calibration') + ' .'
# if(os.system(cmd)):
# DIRAC.exit( -1 )
continue
else:
installSoftwarePackage( package, workingArea() )
DIRAC.gLogger.notice( 'Package found in workingArea:', package )
continue
DIRAC.gLogger.error( 'Check Failed for software package:', package )
DIRAC.gLogger.error( 'Software package not available')
DIRAC.exit( -1 )
ed = EvnDispApp()
ed.setSoftwarePackage(EvnDispPack)
########## Use of trg mask file #######################
usetrgfile = sys.argv[7]
DIRAC.gLogger.notice( 'Usetrgfile:', usetrgfile )
####### Use of multiple inputs per job ###
simtelFileLFNList = sys.argv[-1].split('ParametricParameters={')[1].split('}')[0].replace(',',' ')
# first element of the list
simtelFileLFN = simtelFileLFNList.split(' ')[0]
## convert the string into a list and get the basename
simtelFileList = []
for word in simtelFileLFNList.split():
simtelFileList.append(os.path.basename(word))
#### Parse the Layout List #################
layoutList = parseLayoutList(sys.argv[9])
#############################################
#### Loop over the Layout List #################
for layout in layoutList:
args = []
########## download trg mask file #######################
if usetrgfile == 'True':
trgmaskFileLFN = simtelFileLFN.replace( 'simtel.gz', 'trgmask.gz' )
DIRAC.gLogger.notice( 'Trying to download the trgmask File', trgmaskFileLFN )
result = DataManager().getFile( trgmaskFileLFN )
if not result['OK']:
DIRAC.gLogger.error( 'Failed to download trgmakfile:', result )
jobReport.setApplicationStatus( 'Trgmakfile download Error' )
DIRAC.exit( -1 )
args.extend( ['-t', os.path.basename( trgmaskFileLFN )] )
############################################################
###### execute evndisplay converter ##################
executable = sys.argv[5]
############ dst file Name ############################
run_number = simtelFileList[-1].split( 'run' )[1].split( '.simtel.gz' )[0]
runNum = int( run_number )
subRunNumber = '%06d' % runNum
particle = simtelFileList[-1].split( '_' )[0]
if 'ptsrc' in simtelFileList[-1]:
particle = particle + '_' + 'ptsrc'
dstfile = particle + '_run' + subRunNumber + '_' + str( jobID ) + '_' + os.path.basename( layout ) + '_dst.root'
###########################################
logfileName = executable + '_' + layout + '.log'
layout = os.path.join( 'EVNDISP.CTA.runparameter/DetectorGeometry', layout )
DIRAC.gLogger.notice( 'Layout is:', layout )
# add other arguments for evndisplay converter specified by user ######
converterparfile = open( 'converter.par', 'r' ).readlines()
for line in converterparfile:
for word in line.split():
args.append( word )
#########################################################
args.extend( ['-a', layout] )
args.extend( ['-o', dstfile] )
args.extend( simtelFileList )
execute_module( ed, executable, args )
########### check existence of DST file ###############
if not os.path.isfile( dstfile ):
DIRAC.gLogger.error( 'DST file Missing:', dstfile )
jobReport.setApplicationStatus( 'DST file Missing' )
DIRAC.exit( -1 )
########### quality check on Log #############################################
cmd = 'mv ' + executable + '.log' + ' ' + logfileName
if( os.system( cmd ) ):
DIRAC.exit( -1 )
fd = open( 'check_log.sh', 'w' )
fd.write( """#! /bin/sh
MCevts=$(grep writing %s | grep "MC events" | awk '{print $2}')
if [ $MCevts -gt 0 ]; then
exit 0
else
echo "MCevts is zero"
exit -1
fi
""" % (logfileName))
fd.close()
os.system( 'chmod u+x check_log.sh' )
cmd = './check_log.sh'
DIRAC.gLogger.notice( 'Executing system call:', cmd )
if( os.system( cmd ) ):
jobReport.setApplicationStatus( 'Converter Log Check Failed' )
DIRAC.exit( -1 )
#### Check the mode #################
mode = sys.argv[11]
if( mode == 'convert_standalone' ):
#DIRAC.exit()
continue
###### execute evndisplay stage1 ###############
executable = 'evndisp'
logfileName = executable + '_' + os.path.basename( layout ) + '.log'
args = ['-sourcefile', dstfile, '-outputdirectory', 'outdir']
# add other arguments for evndisp specified by user ######
evndispparfile = open( 'evndisp.par', 'r' ).readlines()
for line in evndispparfile:
for word in line.split():
args.append( word )
execute_module( ed, executable, args )
for name in glob.glob( 'outdir/*.root' ):
evndispOutFile = name.split( '.root' )[0] + '_' + str( jobID ) + '_' + os.path.basename( layout ) + '_evndisp.root'
cmd = 'mv ' + name + ' ' + os.path.basename( evndispOutFile )
if( os.system( cmd ) ):
DIRAC.exit( -1 )
########### quality check on Log #############################################
cmd = 'mv ' + executable + '.log' + ' ' + logfileName
if( os.system( cmd ) ):
DIRAC.exit( -1 )
fd = open( 'check_log.sh', 'w' )
fd.write( """#! /bin/sh
if grep -i "error" %s; then
exit 1
fi
if grep "Final checks on result file (seems to be OK):" %s; then
exit 0
else
exit 1
fi
""" % (logfileName,logfileName))
fd.close()
os.system( 'chmod u+x check_log.sh' )
cmd = './check_log.sh'
DIRAC.gLogger.notice( 'Executing system call:', cmd )
if( os.system( cmd ) ):
jobReport.setApplicationStatus( 'EvnDisp Log Check Failed' )
DIRAC.exit( -1 )
##################################################################
########### remove the converted dst file #############################################
cmd = 'rm ' + dstfile
if( os.system( cmd ) ):
DIRAC.exit( -1 )
DIRAC.exit()
