def main(): original = False Script.registerSwitch('O', 'Original', 'Gets the original JDL') Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() for switch in Script.getUnprocessedSwitches(): if switch[0] == 'Original' or switch[0] == 'O': original = True for switch in Script.getUnprocessedSwitches(): if switch[0] == 'Original': original = True if len(args) < 1: Script.showHelp(exitCode=1) from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments dirac = Dirac() exitCode = 0 errorList = [] for job in parseArguments(args): result = dirac.getJobJDL(job, original=original, printOutput=True) if not result['OK']: errorList.append((job, result['Message'])) exitCode = 2 for error in errorList: print("ERROR %s: %s" % error) DIRAC.exit(exitCode)
def main(): original = False Script.registerSwitch("O", "Original", "Gets the original JDL") # Registering arguments will automatically add their description to the help menu Script.registerArgument(["JobID: DIRAC Job ID"]) sws, args = Script.parseCommandLine(ignoreErrors=True) for switch in sws: if switch[0] == "Original" or switch[0] == "O": original = True from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments dirac = Dirac() exitCode = 0 errorList = [] for job in parseArguments(args): result = dirac.getJobJDL(job, original=original, printOutput=True) if not result["OK"]: errorList.append((job, result["Message"])) exitCode = 2 for error in errorList: print("ERROR %s: %s" % error) DIRAC.exit(exitCode)
def main(): Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() from DIRAC.Interfaces.API.Dirac import Dirac # get arguments input_card = args[0] outprefix = "" if len(args) == 2: outprefix = args[1] #### get Parameter and set run_number ######### if os.environ.has_key('JOBID'): jobID = os.environ['JOBID'] dirac = Dirac() res = dirac.getJobJDL(jobID) run_number = res['Value']['Parameter.run'] # # run corsika os.system('chmod +x dirac_prod3_corsika_only') cmdTuple = ['./dirac_prod3_corsika_only', '--run', run_number, input_card] DIRAC.gLogger.notice('Executing command tuple:', cmdTuple) res = systemCall(0, cmdTuple, sendOutput) if not res['OK']: return res # ## rename output file #outfile = 'run%s.corsika.gz' % run_number outfile = '%srun%s.corsika.gz' % (outprefix, run_number) cmd = 'mv Data/corsika/*/*corsika.gz %s' % outfile if (os.system(cmd)): DIRAC.exit(-1) return DIRAC.S_OK()
def getHB9SCT(): """ Download SCT files corresponding to a list of HB9 merged files Keyword arguments: none -- none """ DIRAC.gLogger.info('Get HB9-SCT files') # get JDL dirac = Dirac() resJDL = dirac.getJobJDL(os.environ['JOBID'] ) # get list of output files idata=resJDL['Value']['InputData'] # dowload files for merged in idata: DIRAC.gLogger.debug("Input %s "%merged) sct=merged.strip('\n').replace('merged', 'SCT').replace('cta-prod3','cta-prod3-sct') downloadFile(sct) return DIRAC.S_OK()
def getSub5(): """ Download subarray-5 files corresponding to a list of subarray-2 files Keyword arguments: none -- none """ DIRAC.gLogger.info('Get Subarray-5 files') # get JDL dirac = Dirac() resJDL = dirac.getJobJDL(os.environ['JOBID'] ) # get list of output files idata=resJDL['Value']['InputData'] # dowload files for sub2 in idata: DIRAC.gLogger.debug("Input %s "%sub2) sub5=sub2.strip('\n').replace('subarray-2-nosct', 'subarray-5-nosct') downloadFile(sub5) return DIRAC.S_OK()
def createMDStructure( self, metadata, metadatafield, basepath, program_category ): """ create meta data structure """ # ## Add metadata fields to the DFC mdfield = json.loads( metadatafield ) for key, value in mdfield.items(): res = self.fc.addMetadataField( key, value ) if not res['OK']: return res # ## Create the directory structure md = json.loads( metadata , object_pairs_hook = collections.OrderedDict ) path = basepath process_program = program_category + '_prog' for key, value in collections.OrderedDict( ( k, md[k] ) for k in ( 'site', 'particle', process_program ) if k in md ).items(): path = os.path.join( path, self._formatPath( value ) ) res = self.fc.createDirectory( path ) if not res['OK']: return res # Set directory metadata for each subdir: 'site', 'particle', 'process_program' res = self.fcc.setMetadata( path, {key:value} ) if not res['OK']: return res # Create the TransformationID subdir and set MD # ## Get the TransformationID if os.environ.has_key( 'JOBID' ): jobID = os.environ['JOBID'] dirac = Dirac() res = dirac.getJobJDL( jobID ) TransformationID = '0000' if res['Value'].has_key( 'TransformationID' ): TransformationID = res['Value']['TransformationID'] path = os.path.join( path, TransformationID ) res = self.fc.createDirectory( path ) if not res['OK']: return res process_program_version = process_program + '_version' res = self.fcc.setMetadata( path, dict( ( k, md[k] ) for k in ( 'phiP', 'thetaP', 'array_layout', process_program_version ) if k in md ) ) if not res['OK']: return res # Create the Data and Log subdirs and set MD Transformation_path = path for subdir in ['Data', 'Log']: path = os.path.join( Transformation_path, subdir ) res = self.fc.createDirectory( path ) if not res['OK']: return res # Set metadata if not already defined res = self.fcc.getDirectoryUserMetadata( path ) if not res['OK']: return res if 'outputType' not in res['Value']: res = self.fcc.setMetadata( path, {'outputType':subdir} ) if not res['OK']: return res # MD for the Data directory - data_level and configuration_id path = os.path.join(Transformation_path, 'Data') # Set metadata if not already defined res = self.fcc.getDirectoryUserMetadata( path ) if not res['OK']: return res if 'data_level' and 'configuration_id' not in res['Value']: res = self.fcc.setMetadata(path, {'data_level': md['data_level'], 'configuration_id': md['configuration_id'] }) if not res['OK']: return res return DIRAC.S_OK( Transformation_path )
import DIRAC from DIRAC.Core.Base import Script Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1], 'Usage:', ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, 'Arguments:', ' JobID: DIRAC Job ID' ] ) ) Script.parseCommandLine( ignoreErrors = True ) args = Script.getPositionalArgs() if len( args ) < 1: Script.showHelp() from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 errorList = [] for job in args: result = dirac.getJobJDL( job, printOutput = True ) if not result['OK']: errorList.append( ( job, result['Message'] ) ) exitCode = 2 for error in errorList: print "ERROR %s: %s" % error DIRAC.exit( exitCode )
Script.parseCommandLine( ignoreErrors = True ) args = Script.getPositionalArgs() for switch in Script.getUnprocessedSwitches(): if switch[0] == 'Original' or switch[0] == 'O': original = True for switch in Script.getUnprocessedSwitches(): if switch[0] == 'Original': original = True if len( args ) < 1: Script.showHelp() from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments dirac = Dirac() exitCode = 0 errorList = [] for job in parseArguments( args ): result = dirac.getJobJDL( job, original = original, printOutput = True ) if not result['OK']: errorList.append( ( job, result['Message'] ) ) exitCode = 2 for error in errorList: print "ERROR %s: %s" % error DIRAC.exit( exitCode )
def main(): from DIRAC.Core.Base import Script Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig", setConfig) Script.registerSwitch("V:", "version=", "Version", setVersion) from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient from DIRAC.Resources.Catalog.FileCatalog import FileCatalog Script.parseCommandLine() DIRAC.gLogger.setLevel('INFO') global fcc, fcL from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport global jobID jobID = os.environ['JOBID'] jobReport = JobReport(int(jobID)) ########### ## Checking MD coherence fc = FileCatalog('LcgFileCatalog') res = fc._getCatalogConfigDetails('DIRACFileCatalog') print 'DFC CatalogConfigDetails:', res res = fc._getCatalogConfigDetails('LcgFileCatalog') print 'LCG CatalogConfigDetails:', res fcc = FileCatalogClient() fcL = FileCatalog('LcgFileCatalog') from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() ############################ install_CorsikaSimtelPack(version) ############# # simtelConfigFile should be built from ??? #simtelConfigFilesPath = 'sim_telarray/multi' #simtelConfigFile = simtelConfigFilesPath + '/multi_cta-ultra5.cfg' #createGlobalsFromConfigFiles(simtelConfigFile) #createGlobalsFromConfigFiles(current_version) ####################### ## files spread in 1000-runs subDirectories global corsikaFileLFN corsikaFileLFN = dirac.getJobJDL(jobID)['Value']['InputData'] print 'corsikaFileLFN is ' + corsikaFileLFN corsikaFileName = os.path.basename(corsikaFileLFN) run_number = corsikaFileName.split('run')[1].split('.corsika.gz')[ 0] # run001412.corsika.gz runNum = int(run_number) subRunNumber = '%03d' % runNum runNumModMille = runNum % 1000 runNumTrunc = (runNum - runNumModMille) / 1000 runNumSeriesDir = '%03dxxx' % runNumTrunc print 'runNumSeriesDir=', runNumSeriesDir f = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w') f.close() ############ Producing SimTel File ######################Building simtel Directory Metadata ####################### cfg_dict = { "4MSST": 'cta-prod2-4m-dc', "SCSST": 'cta-prod2-sc-sst', "STD": 'cta-prod2', "NSBX3": 'cta-prod2', "ASTRI": 'cta-prod2-astri', "SCMST": 'cta-prod2-sc3', "NORTH": 'cta-prod2n' } if simtelConfig == "6INROW": all_configs = ["4MSST", "SCSST", "ASTRI", "NSBX3", "STD", "SCMST"] elif simtelConfig == "5INROW": all_configs = ["4MSST", "SCSST", "ASTRI", "NSBX3", "STD"] elif simtelConfig == "3INROW": all_configs = ["SCSST", "STD", "SCMST"] else: all_configs = [simtelConfig] for current_conf in all_configs: DIRAC.gLogger.notice('current conf is', current_conf) if current_conf == "SCMST": current_version = version + '_sc3' DIRAC.gLogger.notice('current version is', current_version) if os.path.isdir('sim_telarray'): DIRAC.gLogger.notice( 'Package found in the local area. Removing package...') cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run' if (os.system(cmd)): DIRAC.exit(-1) install_CorsikaSimtelPack(current_version) else: current_version = version DIRAC.gLogger.notice('current version is', current_version) ######################################################## createGlobalsFromConfigFiles(current_version) resultCreateSimtelDirMD = createSimtelFileSystAndMD( current_conf, current_version) if not resultCreateSimtelDirMD['OK']: DIRAC.gLogger.error('Failed to create simtelArray Directory MD') jobReport.setApplicationStatus( 'Failed to create simtelArray Directory MD') DIRAC.gLogger.error( 'Metadata coherence problem, no simtelArray File produced') DIRAC.exit(-1) else: print 'simtel Directory MD successfully created' ############## introduce file existence check here ######################## simtelFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str( obslev) + '_' + 'run' + run_number + '.simtel.gz' simtelDirPath_conf = simtelDirPath + '_' + current_conf simtelOutFileDir = os.path.join(simtelDirPath_conf, 'Data', runNumSeriesDir) simtelOutFileLFN = os.path.join(simtelOutFileDir, simtelFileName) res = CheckCatalogCoherence(simtelOutFileLFN) if res == DIRAC.S_OK: DIRAC.gLogger.notice('Current conf already done', current_conf) continue #### execute simtelarray ################ fd = open('run_sim.sh', 'w') fd.write("""#! /bin/sh export SVNPROD2=$PWD export SVNTAG=SVN-PROD2 export CORSIKA_IO_BUFFER=800MB ./grid_prod2-repro.sh %s %s""" % (corsikaFileName, current_conf)) fd.close() os.system('chmod u+x grid_prod2-repro.sh') os.system('chmod u+x run_sim.sh') cmdTuple = ['./run_sim.sh'] ret = systemCall(0, cmdTuple, sendOutputSimTel) simtelReturnCode, stdout, stderr = ret['Value'] if (os.system('grep Broken simtel.log')): DIRAC.gLogger.notice('not broken') else: DIRAC.gLogger.notice('broken') jobReport.setApplicationStatus('Broken pipe') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Failed to execute run_sim.sh') DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode) DIRAC.exit(-1) ## putAndRegister simtel data/log/histo Output File: cfg = cfg_dict[current_conf] cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Data/*.simtel.gz ' + simtelFileName if (os.system(cmd)): DIRAC.exit(-1) ############################################ simtelRunNumberSeriesDirExist = fcc.isDirectory( simtelOutFileDir)['Value']['Successful'][simtelOutFileDir] newSimtelRunFileSeriesDir = ( simtelRunNumberSeriesDirExist != True ) # if new runFileSeries, will need to add new MD simtelLogFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz' cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Log/*.log.gz ' + simtelLogFileName if (os.system(cmd)): DIRAC.exit(-1) simtelOutLogFileDir = os.path.join(simtelDirPath_conf, 'Log', runNumSeriesDir) simtelOutLogFileLFN = os.path.join(simtelOutLogFileDir, simtelLogFileName) simtelHistFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str( obslev) + '_' + 'run' + run_number + '.hdata.gz' cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName if (os.system(cmd)): DIRAC.exit(-1) simtelOutHistFileDir = os.path.join(simtelDirPath_conf, 'Histograms', runNumSeriesDir) simtelOutHistFileLFN = os.path.join(simtelOutHistFileDir, simtelHistFileName) ########### quality check on Histo Missing because it needs the NSHOW ############################################# ########## quality check on Log ############################# cmd = 'zcat %s | grep Finished.' % simtelLogFileName DIRAC.gLogger.notice('Executing system call:', cmd) if (os.system(cmd)): jobReport.setApplicationStatus('Log check Failed') DIRAC.exit(-1) ################################################ from DIRAC.Core.Utilities import List from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations opsHelper = Operations() global seList seList = opsHelper.getValue('ProductionOutputs/SimtelProd', []) seList = List.randomize(seList) DIRAC.gLogger.notice('SeList is:', seList) ######### Upload simtel data/log/histo ############################################## res = upload_to_seList(simtelOutFileLFN, simtelFileName) if res != DIRAC.S_OK: DIRAC.gLogger.error('OutputData Upload Error', simtelOutFileLFN) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) res = CheckCatalogCoherence(simtelOutLogFileLFN) if res == DIRAC.S_OK: DIRAC.gLogger.notice('Log file already exists. Removing:', simtelOutLogFileLFN) ret = dirac.removeFile(simtelOutLogFileLFN) res = upload_to_seList(simtelOutLogFileLFN, simtelLogFileName) if res != DIRAC.S_OK: DIRAC.gLogger.error('Upload simtel Log Error', simtelOutLogFileLFN) DIRAC.gLogger.notice('Removing simtel data file:', simtelOutFileLFN) ret = dirac.removeFile(simtelOutFileLFN) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) res = CheckCatalogCoherence(simtelOutHistFileLFN) if res == DIRAC.S_OK: DIRAC.gLogger.notice('Histo file already exists. Removing:', simtelOutHistFileLFN) ret = dirac.removeFile(simtelOutHistFileLFN) res = upload_to_seList(simtelOutHistFileLFN, simtelHistFileName) if res != DIRAC.S_OK: DIRAC.gLogger.error('Upload simtel Histo Error', simtelOutHistFileLFN) DIRAC.gLogger.notice('Removing simtel data file:', simtelOutFileLFN) ret = dirac.removeFile(simtelOutFileLFN) DIRAC.gLogger.notice('Removing simtel log file:', simtelOutLogFileLFN) ret = dirac.removeFile(simtelOutLogFileLFN) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) #################################################################### if newSimtelRunFileSeriesDir: insertRunFileSeriesMD(simtelOutFileDir, runNumTrunc) insertRunFileSeriesMD(simtelOutLogFileDir, runNumTrunc) insertRunFileSeriesMD(simtelOutHistFileDir, runNumTrunc) ###### simtel File level metadata ############################################ simtelFileMD = {} simtelFileMD['runNumber'] = int(run_number) simtelFileMD['jobID'] = jobID simtelFileMD['simtelReturnCode'] = simtelReturnCode result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.setMetadata(simtelOutLogFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.setMetadata(simtelOutHistFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.addFileAncestors( {simtelOutFileLFN: { 'Ancestors': [corsikaFileLFN] }}) print 'result addFileAncestor:', result result = fcc.addFileAncestors( {simtelOutLogFileLFN: { 'Ancestors': [corsikaFileLFN] }}) print 'result addFileAncestor:', result result = fcc.addFileAncestors( {simtelOutHistFileLFN: { 'Ancestors': [corsikaFileLFN] }}) print 'result addFileAncestor:', result result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD) if not result['OK']: print 'ResultSetMetadata:', result['Message'] DIRAC.exit()
'Usage:', ' %s [option|cfgfile] ... JobID ...' % Script.scriptName, 'Arguments:', ' JobID: DIRAC Job ID' ] ) ) Script.parseCommandLine( ignoreErrors = True ) args = Script.getPositionalArgs() for switch in Script.getUnprocessedSwitches(): if switch[0] == 'Original' or switch[0] == 'O': original = True if len( args ) < 1: Script.showHelp() from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 errorList = [] for job in args: result = dirac.getJobJDL( job, original = original, printOutput = True ) if not result['OK']: errorList.append( ( job, result['Message'] ) ) exitCode = 2 for error in errorList: print "ERROR %s: %s" % error DIRAC.exit( exitCode )