def __init__(self, **kwargs): GridBackend.__init__(self, catalogue_prefix='', **kwargs) from DIRAC.Core.Base import Script Script.initialize() from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient self.pm = ProxyManagerClient() proxy = self.pm.getUserProxiesInfo() if not proxy['OK']: raise BackendException("Proxy error.") from DIRAC.Interfaces.API.Dirac import Dirac self.dirac = Dirac() from DIRAC.Resources.Catalog.FileCatalog import FileCatalog self.fc = FileCatalog() from DIRAC.DataManagementSystem.Client.DataManager import DataManager self.dm = DataManager() self._xattr_cmd = sh.Command('gfal-xattr').bake(_tty_out=False) self._replica_checksum_cmd = sh.Command('gfal-sum').bake(_tty_out=False) self._bringonline_cmd = sh.Command('gfal-legacy-bringonline').bake(_tty_out=False) self._cp_cmd = sh.Command('gfal-copy').bake(_tty_out=False) self._ls_se_cmd = sh.Command('gfal-ls').bake(color='never', _tty_out=False) self._move_cmd = sh.Command('gfal-rename').bake(_tty_out=False) self._mkdir_cmd = sh.Command('gfal-mkdir').bake(_tty_out=False) self._replicate_cmd = sh.Command('dirac-dms-replicate-lfn').bake(_tty_out=False) self._add_cmd = sh.Command('dirac-dms-add-file').bake(_tty_out=False)
def read_site_summary(): active_sites = {} from DIRAC.Core.Base import Script Script.initialize( False, False, False, False ) JobMonitoring = RPCClient('WorkloadManagement/JobMonitoring', timeout=600) result = JobMonitoring.getSiteSummary() if not result['OK']: sys.exit("Failed to get SiteSummary: %s" % str(result['Message'])) sitesummary=result['Value'] #return sitesummary #print sitesummary sitelists = getSiteList() # Write the coordinates in a temporal file coord_file = open('sites.tmp','w') coord_file.write("Sites,x,y") # For now, for sites without coordinates info in DIRAC coord_file.write("\nAdelaide,-34.910836,138.57121") coord_file.write("\nIPHC,48.6056,7.7095") coord_file.write("\nKEK,36.06,140.2303") for site in sorted(sitelists): ele = site.split('.') result = gConfig.getOptionsDict('Resources/Sites/%s/%s' % ( ele[0], site )) if not result['OK']: continue optionsdict=result['Value'] status = "production" if 'Status' in optionsdict: status = optionsdict['Status'] if status not in ['production', 'commissioning']: continue if site not in sitesummary: continue active_sites[site] = sitesummary[site] x='0' y='0' if 'Coordinates' in optionsdict: [y,x] = optionsdict['Coordinates'].split(':') if x.replace('-','').replace('.','').isdigit() and y.replace('-','').replace('.','').isdigit(): active_sites[site]['Coordinates'] = [float(x),float(y)] coord_file.write('\n'+ site.split('.')[1]+','+ str(x) + ',' + str(y)) else: print "[WARNING:] No location info for",site coord_file.close() return active_sites
######################################################################## # $HeadURL$ # File : API.py ######################################################################## """ DIRAC API Base Class """ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script Script.initialize() from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities.List import sortList from DIRAC.Core.Security.Misc import getProxyInfo, formatProxyInfoAsString from DIRAC.Core.Security.CS import getDNForUsername import pprint, sys COMPONENT_NAME = 'API' def _printFormattedDictList(dictList, fields, uniqueField, orderBy): """ Will print ordered the supplied field of a list of dictionaries """ orderDict = {} fieldWidths = {} dictFields = {} for myDict in dictList: for field in fields: fieldValue = myDict[field] if not fieldWidths.has_key(field): fieldWidths[field] = len(str(field)) if len(str(fieldValue)) > fieldWidths[field]:
def initDIRAC( rootPath, enableDebug = False ): # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) configDict = { 'webConfig' : {} } configDict[ 'webConfig' ]['dirac.webroot'] = rootPath diracRootPath = os.path.realpath( os.path.dirname( os.path.dirname( rootPath ) ) ) configDict[ 'webConfig' ]['dirac.root'] = diracRootPath if diracRootPath not in sys.path: sys.path.append( diracRootPath ) from DIRAC.FrameworkSystem.Client.Logger import gLogger gLogger.registerBackends( [ 'stderr' ] ) from DIRAC.Core.Base import Script Script.registerSwitch( "r", "reload", "Reload for pylons" ) Script.localCfg.addDefaultEntry( "/DIRAC/Security/UseServerCertificate", "yes" ) Script.localCfg.addDefaultEntry( "LogColor", True ) Script.initialize( script = "Website", ignoreErrors = True, initializeMonitor = False ) gLogger._systemName = "Framework" gLogger.initialize( "Web", "/Website" ) gLogger.setLevel( "VERBOSE" ) from DIRAC import gMonitor, gConfig, rootPath as droot from DIRAC.Core.Utilities import CFG from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions gMonitor.setComponentType( gMonitor.COMPONENT_WEB ) gMonitor.initialize() gMonitor.registerActivity( "pagesServed", "Pages served", "Framework", "pages", gMonitor.OP_SUM ) gLogger.info( "DIRAC Initialized" ) configDict['portalVersion'] = portalVersion( rootPath ) gLogger.info( "DIRAC portal version: %s" % configDict['portalVersion'] ) extModules = [ '%sDIRAC' % module for module in getCSExtensions() ] #Load web.cfg of modules cfgFilePaths = [ os.path.join( droot, "etc", "web.cfg" ) ] for extModule in extModules: gLogger.info( "Adding web.cfg for %s extension" % extModule ) extModulePath = os.path.join( diracRootPath, extModule ) webCFGPath = os.path.join( extModulePath, "Web", "web.cfg" ) cfgFilePaths.append( webCFGPath ) for systemDir in os.listdir( extModulePath ): webCFGSystemPath = os.path.join( extModulePath, systemDir, "Web", "web.cfg" ) cfgFilePaths.append( webCFGSystemPath ) webCFG = CFG.CFG() for webCFGPath in cfgFilePaths: if not os.path.isfile( webCFGPath ): gLogger.warn( "%s does not exist" % webCFGPath ) else: gLogger.info( "Loading %s" % webCFGPath ) modCFG = CFG.CFG().loadFromFile( webCFGPath ) if modCFG.getOption( 'Website/AbsoluteDefinition', False ): gLogger.info( "CFG %s is absolute" % webCFGPath ) webCFG = modCFG else: webCFG = webCFG.mergeWith( modCFG ) gConfig.loadCFG( webCFG ) gLogger.showHeaders( True ) gLogger._gLogger__initialized = False gLogger.initialize( "Web", "/Website" ) #Define the controllers, templates and public directories for type in ( 'controllers', 'templates', 'public' ): configDict[ type ] = [] for extModule in extModules: extModulePath = os.path.join( diracRootPath, extModule ) typePath = os.path.join( extModulePath, "Web", type ) if os.path.isdir( typePath ): gLogger.info( "Adding %s path for module %s" % ( type, extModule ) ) configDict[ type ].append( typePath ) for systemDir in os.listdir( extModulePath ): systemTypePath = os.path.join( extModulePath, systemDir, "Web", type ) if os.path.isdir( systemTypePath ): gLogger.info( "Adding %s path for system %s in module %s" % ( type, systemDir, extModule ) ) configDict[ type ].append( systemTypePath ) #End of extensions configDict[ type ].append( os.path.join( rootPath, type ) ) #Load debug.cfg? if enableDebug: debugCFGPath = os.path.join( rootPath, "debug.cfg" ) if os.path.isfile( debugCFGPath ): gLogger.info( "Loading debug cfg file at %s" % debugCFGPath ) gConfig.loadFile( debugCFGPath ) gLogger.info( "Extension modules loaded" ) return configDict
######################################################################## # $HeadURL$ # File : DiracAdmin.py # Author : Stuart Paterson ######################################################################## """DIRAC Administrator API Class All administrative functionality is exposed through the DIRAC Admin API. Examples include site banning and unbanning, WMS proxy uploading etc. """ __RCSID__ = "$Id$" from DIRAC.Core.Base import Script Script.initialize() import DIRAC from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup from DIRAC.Core.DISET.RPCClient import RPCClient from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager from DIRAC.Core.Utilities.SiteCEMapping import getSiteCEMapping from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient from DIRAC.Core.Security.X509Chain import X509Chain from DIRAC.Core.Security import Locations, CS from DIRAC import gConfig, gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities.Grid import ldapSite, ldapCluster, ldapCE, ldapService from DIRAC.Core.Utilities.Grid import ldapCEState, ldapCEVOView, ldapSA
def initDIRAC(rootPath, enableDebug=False): # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) configDict = {'webConfig': {}} configDict['webConfig']['dirac.webroot'] = rootPath diracRootPath = os.path.realpath(os.path.dirname( os.path.dirname(rootPath))) configDict['webConfig']['dirac.root'] = diracRootPath if diracRootPath not in sys.path: sys.path.append(diracRootPath) from DIRAC.FrameworkSystem.Client.Logger import gLogger gLogger.registerBackends(['stderr']) from DIRAC.Core.Base import Script Script.registerSwitch("r", "reload", "Reload for pylons") Script.localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") Script.localCfg.addDefaultEntry("LogColor", True) Script.initialize(script="Website", ignoreErrors=True, initializeMonitor=False) gLogger._systemName = "Framework" gLogger.initialize("Web", "/Website") gLogger.setLevel("VERBOSE") from DIRAC import gMonitor, gConfig, rootPath as droot from DIRAC.Core.Utilities import CFG from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions gMonitor.setComponentType(gMonitor.COMPONENT_WEB) gMonitor.initialize() gMonitor.registerActivity("pagesServed", "Pages served", "Framework", "pages", gMonitor.OP_SUM) gLogger.info("DIRAC Initialized") configDict['portalVersion'] = portalVersion(rootPath) gLogger.info("DIRAC portal version: %s" % configDict['portalVersion']) extModules = ['%sDIRAC' % module for module in getCSExtensions()] #Load web.cfg of modules cfgFilePaths = [os.path.join(droot, "etc", "web.cfg")] for extModule in extModules: gLogger.info("Adding web.cfg for %s extension" % extModule) extModulePath = os.path.join(diracRootPath, extModule) webCFGPath = os.path.join(extModulePath, "Web", "web.cfg") cfgFilePaths.append(webCFGPath) for systemDir in os.listdir(extModulePath): webCFGSystemPath = os.path.join(extModulePath, systemDir, "Web", "web.cfg") cfgFilePaths.append(webCFGSystemPath) webCFG = CFG.CFG() for webCFGPath in cfgFilePaths: if not os.path.isfile(webCFGPath): gLogger.warn("%s does not exist" % webCFGPath) else: gLogger.info("Loading %s" % webCFGPath) modCFG = CFG.CFG().loadFromFile(webCFGPath) if modCFG.getOption('Website/AbsoluteDefinition', False): gLogger.info("CFG %s is absolute" % webCFGPath) webCFG = modCFG else: webCFG = webCFG.mergeWith(modCFG) gConfig.loadCFG(webCFG) gLogger.showHeaders(True) gLogger._gLogger__initialized = False gLogger.initialize("Web", "/Website") #Define the controllers, templates and public directories for type in ('controllers', 'templates', 'public'): configDict[type] = [] for extModule in extModules: extModulePath = os.path.join(diracRootPath, extModule) typePath = os.path.join(extModulePath, "Web", type) if os.path.isdir(typePath): gLogger.info("Adding %s path for module %s" % (type, extModule)) configDict[type].append(typePath) for systemDir in os.listdir(extModulePath): systemTypePath = os.path.join(extModulePath, systemDir, "Web", type) if os.path.isdir(systemTypePath): gLogger.info("Adding %s path for system %s in module %s" % (type, systemDir, extModule)) configDict[type].append(systemTypePath) #End of extensions configDict[type].append(os.path.join(rootPath, type)) #Load debug.cfg? if enableDebug: debugCFGPath = os.path.join(rootPath, "debug.cfg") if os.path.isfile(debugCFGPath): gLogger.info("Loading debug cfg file at %s" % debugCFGPath) gConfig.loadFile(debugCFGPath) gLogger.info("Extension modules loaded") return configDict
def main(): from DIRAC.Core.Base import Script Script.initialize() DIRAC.gLogger.notice('Platform is:') os.system('dirac-platform') from CTADIRAC.Core.Workflow.Modules.Read_CtaApp import Read_CtaApp from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport jobID = os.environ['JOBID'] jobID = int(jobID) jobReport = JobReport(jobID) version = sys.argv[3] DIRAC.gLogger.notice('Version:', version) install_CorsikaSimtelPack(version) ######### run read_cta ####################################### rcta = Read_CtaApp() CorsikaSimtelPack = os.path.join('corsika_simhessarray', version, 'corsika_simhessarray') rcta.setSoftwarePackage(CorsikaSimtelPack) rcta.rctaExe = 'read_cta' # add arguments for read_cta specified by user ###### args = [] rctaparfile = open('read_cta.par', 'r').readlines() for line in rctaparfile: for word in line.split(): args.append(word) simtelFileLFN = sys.argv[-1].split('ParametricInputData=LFN:')[1] simtelFileName = os.path.basename(simtelFileLFN) dstFileName = simtelFileName.replace('simtel.gz', 'simtel-dst0.gz') dstHistoFileName = simtelFileName.replace('simtel.gz', 'hdata-dst0.gz') args.extend([ '--dst-file', dstFileName, '--histogram-file', dstHistoFileName, simtelFileName ]) rcta.rctaArguments = args rctaReturnCode = rcta.execute() if rctaReturnCode != 0: DIRAC.gLogger.error('read_cta Application: Failed') jobReport.setApplicationStatus('read_cta Application: Failed') DIRAC.exit(-1) ################################################################# from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron ret = getSoftwareEnviron(CorsikaSimtelPack) if not ret['OK']: error = ret['Message'] DIRAC.gLogger.error(error, CorsikaSimtelPack) DIRAC.exit(-1) read_ctaEnviron = ret['Value'] ######## run dst quality checks ###################################### fd = open('check_dst_histo.sh', 'w') fd.write("""#! /bin/sh dsthistfilename=%s dstfile=%s n6="$(list_histograms -h 6 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')" n12001="$(list_histograms -h 12001 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')" if [ $n6 -ne $n12001 ]; then echo 'n6 found:' $n6 echo 'n12001 found:' $n12001 exit 1 else echo 'n6 found:' $n6 echo 'n12001 found:' $n12001 fi n12002="$(list_histograms -h 12002 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')" nev="$(statio ${dstfile} | egrep '^2010' | cut -f2)" if [ -z "$nev" ]; then nev="0"; fi if [ $nev -ne $n12002 ]; then echo 'nev found:' $nev echo 'n12002 found:' $n12002 exit 1 else echo 'nev found:' $nev echo 'n12002 found:' $n12002 fi """ % (dstHistoFileName, dstFileName)) fd.close() os.system('chmod u+x check_dst_histo.sh') cmdTuple = ['./check_dst_histo.sh'] DIRAC.gLogger.notice('Executing command tuple:', cmdTuple) ret = systemCall(0, cmdTuple, sendOutput, env=read_ctaEnviron) checkHistoReturnCode, stdout, stderr = ret['Value'] if not ret['OK']: DIRAC.gLogger.error('Failed to execute check_dst_histo.sh') DIRAC.gLogger.error('check_dst_histo.sh status is:', checkHistoReturnCode) DIRAC.exit(-1) if (checkHistoReturnCode != 0): DIRAC.gLogger.error('Failure during check_dst_histo.sh') DIRAC.gLogger.error('check_dst_histo.sh status is:', checkHistoReturnCode) jobReport.setApplicationStatus('Histo check Failed') DIRAC.exit(-1) DIRAC.exit()
#!/usr/bin/env python # requires DIRAC UI to be setup and valid proxy # takes output from listsolid.py as input # returns a list of files that are on the SE, # but not in the catalogue # optional: registers any files that are present on the SE # but missing from the catalogue import sys import uuid import getopt # DIRAC does not work otherwise from DIRAC.Core.Base import Script # Script.parseCommandLine( ignoreErrors = True ) Script.initialize() # this lets me use getopt # end of DIRAC setup DIRAC_SE_NAME="BEgrid-ULB-VUB-disk" # DIRAC_SE_NAME="UKI-LT2-IC-HEP-disk" from DIRAC.Resources.Catalog.FileCatalog import FileCatalog def add_catalog_entry(fc, fileDict, logfile): result = fc.addFile(fileDict) # TODO: handle the return code as an exception if not result["OK"]: logfile.write(result) print result return
def main(): from DIRAC.Core.Base import Script Script.initialize() DIRAC.gLogger.notice('Platform is:') os.system('dirac-platform') from DIRAC.DataManagementSystem.Client.DataManager import DataManager from CTADIRAC.Core.Workflow.Modules.EvnDispApp import EvnDispApp from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport jobID = os.environ['JOBID'] jobID = int(jobID) jobReport = JobReport(jobID) version = sys.argv[3] DIRAC.gLogger.notice('Version:', version) EvnDispPack = os.path.join('evndisplay', version, 'evndisplay') packs = [EvnDispPack] for package in packs: DIRAC.gLogger.notice('Checking:', package) if checkSoftwarePackage(package, sharedArea())['OK']: DIRAC.gLogger.notice('Package found in Shared Area:', package) installSoftwareEnviron(package, sharedArea()) continue else: installSoftwarePackage(package, workingArea()) DIRAC.gLogger.notice('Package found in workingArea:', package) continue DIRAC.gLogger.error('Check Failed for software package:', package) DIRAC.gLogger.error('Software package not available') DIRAC.exit(-1) ed = EvnDispApp() ed.setSoftwarePackage(EvnDispPack) dstFileLFNList = sys.argv[-1].split('ParametricParameters={')[1].split( '}')[0].replace(',', ' ') args = [] i = 0 for word in dstFileLFNList.split(): i = i + 1 dstfile = os.path.basename(word) ###### execute evndisplay stage1 ############### executable = sys.argv[5] logfileName = executable + '_' + str(i) + '.log' args = ['-sourcefile', dstfile, '-outputdirectory', 'outdir'] # add other arguments for evndisp specified by user ###### evndispparfile = open('evndisp.par', 'r').readlines() for line in evndispparfile: for word in line.split(): args.append(word) execute_module(ed, executable, args) for name in glob.glob('outdir/*.root'): evndispOutFile = name.split('.root')[0] + '_' + str( jobID) + '_evndisp.root' cmd = 'mv ' + name + ' ' + os.path.basename(evndispOutFile) if (os.system(cmd)): DIRAC.exit(-1) ########### quality check on Log ############################################# cmd = 'mv ' + executable + '.log' + ' ' + logfileName if (os.system(cmd)): DIRAC.exit(-1) fd = open('check_log.sh', 'w') fd.write("""#! /bin/sh if grep -i "error" %s; then exit 1 fi if grep "Final checks on result file (seems to be OK):" %s; then exit 0 else exit 1 fi """ % (logfileName, logfileName)) fd.close() os.system('chmod u+x check_log.sh') cmd = './check_log.sh' DIRAC.gLogger.notice('Executing system call:', cmd) if (os.system(cmd)): jobReport.setApplicationStatus('EvnDisp Log Check Failed') DIRAC.exit(-1) ################################################################## ########### remove the dst file ############################################# cmd = 'rm ' + dstfile if (os.system(cmd)): DIRAC.exit(-1) DIRAC.exit()
#!/usr/bin/env python import os import sys from DIRAC.Core.Base import Script Script.initialize(ignoreErrors=True) from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Interfaces.API.Job import Job from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport jobID = os.environ.get('DIRACJOBID', '0') if not jobID: print 'DIRAC job ID not found' sys.exit(1) jobReport = JobReport(jobID, 'JUNO_JobScript') result = jobReport.setApplicationStatus(', '.join(sys.argv[1:])) if not result['OK']: print 'Set application status error: %s' % result
def main(): from DIRAC.Core.Base import Script Script.initialize() DIRAC.gLogger.notice('Platform is:') os.system('dirac-platform') from DIRAC.DataManagementSystem.Client.DataManager import DataManager from CTADIRAC.Core.Workflow.Modules.EvnDispApp import EvnDispApp from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport jobID = os.environ['JOBID'] jobID = int( jobID ) jobReport = JobReport( jobID ) version = sys.argv[3] DIRAC.gLogger.notice( 'Version:', version ) EvnDispPack = os.path.join('evndisplay',version,'evndisplay') packs = [EvnDispPack] for package in packs: DIRAC.gLogger.notice( 'Checking:', package ) if checkSoftwarePackage( package, sharedArea() )['OK']: DIRAC.gLogger.notice( 'Package found in Shared Area:', package ) installSoftwareEnviron( package, sharedArea() ) # cmd = 'cp -r ' + os.path.join(sharedArea(),'evndisplay',version,'EVNDISP.CTA.runparameter') + ' .' # if(os.system(cmd)): # DIRAC.exit( -1 ) # cmd = 'cp -r ' + os.path.join(sharedArea(),'evndisplay',version,'Calibration') + ' .' # if(os.system(cmd)): # DIRAC.exit( -1 ) continue else: installSoftwarePackage( package, workingArea() ) DIRAC.gLogger.notice( 'Package found in workingArea:', package ) continue DIRAC.gLogger.error( 'Check Failed for software package:', package ) DIRAC.gLogger.error( 'Software package not available') DIRAC.exit( -1 ) ed = EvnDispApp() ed.setSoftwarePackage(EvnDispPack) ########## Use of trg mask file ####################### usetrgfile = sys.argv[7] DIRAC.gLogger.notice( 'Usetrgfile:', usetrgfile ) ####### Use of multiple inputs per job ### simtelFileLFNList = sys.argv[-1].split('ParametricParameters={')[1].split('}')[0].replace(',',' ') # first element of the list simtelFileLFN = simtelFileLFNList.split(' ')[0] ## convert the string into a list and get the basename simtelFileList = [] for word in simtelFileLFNList.split(): simtelFileList.append(os.path.basename(word)) #### Parse the Layout List ################# layoutList = parseLayoutList(sys.argv[9]) ############################################# #### Loop over the Layout List ################# for layout in layoutList: args = [] ########## download trg mask file ####################### if usetrgfile == 'True': trgmaskFileLFN = simtelFileLFN.replace( 'simtel.gz', 'trgmask.gz' ) DIRAC.gLogger.notice( 'Trying to download the trgmask File', trgmaskFileLFN ) result = DataManager().getFile( trgmaskFileLFN ) if not result['OK']: DIRAC.gLogger.error( 'Failed to download trgmakfile:', result ) jobReport.setApplicationStatus( 'Trgmakfile download Error' ) DIRAC.exit( -1 ) args.extend( ['-t', os.path.basename( trgmaskFileLFN )] ) ############################################################ ###### execute evndisplay converter ################## executable = sys.argv[5] ############ dst file Name ############################ run_number = simtelFileList[-1].split( 'run' )[1].split( '.simtel.gz' )[0] runNum = int( run_number ) subRunNumber = '%06d' % runNum particle = simtelFileList[-1].split( '_' )[0] if 'ptsrc' in simtelFileList[-1]: particle = particle + '_' + 'ptsrc' dstfile = particle + '_run' + subRunNumber + '_' + str( jobID ) + '_' + os.path.basename( layout ) + '_dst.root' ########################################### logfileName = executable + '_' + layout + '.log' layout = os.path.join( 'EVNDISP.CTA.runparameter/DetectorGeometry', layout ) DIRAC.gLogger.notice( 'Layout is:', layout ) # add other arguments for evndisplay converter specified by user ###### converterparfile = open( 'converter.par', 'r' ).readlines() for line in converterparfile: for word in line.split(): args.append( word ) ######################################################### args.extend( ['-a', layout] ) args.extend( ['-o', dstfile] ) args.extend( simtelFileList ) execute_module( ed, executable, args ) ########### check existence of DST file ############### if not os.path.isfile( dstfile ): DIRAC.gLogger.error( 'DST file Missing:', dstfile ) jobReport.setApplicationStatus( 'DST file Missing' ) DIRAC.exit( -1 ) ########### quality check on Log ############################################# cmd = 'mv ' + executable + '.log' + ' ' + logfileName if( os.system( cmd ) ): DIRAC.exit( -1 ) fd = open( 'check_log.sh', 'w' ) fd.write( """#! /bin/sh MCevts=$(grep writing %s | grep "MC events" | awk '{print $2}') if [ $MCevts -gt 0 ]; then exit 0 else echo "MCevts is zero" exit -1 fi """ % (logfileName)) fd.close() os.system( 'chmod u+x check_log.sh' ) cmd = './check_log.sh' DIRAC.gLogger.notice( 'Executing system call:', cmd ) if( os.system( cmd ) ): jobReport.setApplicationStatus( 'Converter Log Check Failed' ) DIRAC.exit( -1 ) #### Check the mode ################# mode = sys.argv[11] if( mode == 'convert_standalone' ): #DIRAC.exit() continue ###### execute evndisplay stage1 ############### executable = 'evndisp' logfileName = executable + '_' + os.path.basename( layout ) + '.log' args = ['-sourcefile', dstfile, '-outputdirectory', 'outdir'] # add other arguments for evndisp specified by user ###### evndispparfile = open( 'evndisp.par', 'r' ).readlines() for line in evndispparfile: for word in line.split(): args.append( word ) execute_module( ed, executable, args ) for name in glob.glob( 'outdir/*.root' ): evndispOutFile = name.split( '.root' )[0] + '_' + str( jobID ) + '_' + os.path.basename( layout ) + '_evndisp.root' cmd = 'mv ' + name + ' ' + os.path.basename( evndispOutFile ) if( os.system( cmd ) ): DIRAC.exit( -1 ) ########### quality check on Log ############################################# cmd = 'mv ' + executable + '.log' + ' ' + logfileName if( os.system( cmd ) ): DIRAC.exit( -1 ) fd = open( 'check_log.sh', 'w' ) fd.write( """#! /bin/sh if grep -i "error" %s; then exit 1 fi if grep "Final checks on result file (seems to be OK):" %s; then exit 0 else exit 1 fi """ % (logfileName,logfileName)) fd.close() os.system( 'chmod u+x check_log.sh' ) cmd = './check_log.sh' DIRAC.gLogger.notice( 'Executing system call:', cmd ) if( os.system( cmd ) ): jobReport.setApplicationStatus( 'EvnDisp Log Check Failed' ) DIRAC.exit( -1 ) ################################################################## ########### remove the converted dst file ############################################# cmd = 'rm ' + dstfile if( os.system( cmd ) ): DIRAC.exit( -1 ) DIRAC.exit()