def run_simtel_sst(args): """ Simple wrapper to create a Prod4SimtelSSTJob and setup parameters from positional arguments given on the command line. Parameters: args -- mode (trans_name dataset_name group_size) """ DIRAC.gLogger.notice('run_simtel_sst') # get arguments mode = args[0] if mode == 'TS': trans_name = args[1] dataset_name = args[2] group_size = int(args[3]) # job setup - 72 hours job = Prod4SimtelSSTJob(cpuTime=259200.) # override for testing job.setName('Prod4_Simtel') job.version='2018-11-07' # output job.setOutputSandbox(['*Log.txt']) # specific configuration if mode == 'WMS': job.base_path = '/vo.cta.in2p3.fr/user/b/bregeon' job.ts_task_id = '1' output_meta_data = {'array_layout': 'Baseline-SST-only', 'site': 'Paranal', 'particle': 'proton', 'phiP': 0.0, 'thetaP': 20.0, job.program_category + '_prog': 'simtel', job.program_category + '_prog_version': job.version, 'data_level': 0, 'configuration_id': 4} job.set_meta_data(output_meta_data) job.telconfig_list = 'sst-1m sst-gct sst-astri+chec-s' job.N_output_files = len(job.telconfig_list.split()) job.setupWorkflow(debug=True) # subtmit to the WMS for debug job.setDestination('LCG.IN2P3-CC.fr') result = submit_wms(job) elif mode == 'TS': input_meta_query = get_dataset_MQ(dataset_name) # refine output meta data if needed output_meta_data = copy(input_meta_query) job.set_meta_data(output_meta_data) #sst-1m sst-astri sst-astri+chec-s sst-astri+chec-s-7mm sst-gct job.telconfig_list = 'sst-1m sst-gct sst-astri+chec-s' job.N_output_files = len(job.telconfig_list.split()) job.ts_task_id = '@{JOB_ID}' # dynamic job.setupWorkflow(debug=False) job.setType('EvnDisp3') # mandatory *here* result = submit_trans(job, trans_name, input_meta_query, group_size) else: DIRAC.gLogger.error('1st argument should be the job mode: WMS or TS,\n\ not %s' % mode) return None return result
def build_evndisp_step(DL0_data_set, nsb=1): ''' Define a new EventDisplay analysis production step @return ProductionStep object ''' if nsb == 1: DIRAC.gLogger.notice('NSB1x Analysis') DL0_data_set_NSB = DL0_data_set elif nsb == 5: DIRAC.gLogger.notice('NSB5x Analysis') DL0_data_set_NSB = DL0_data_set.replace('NSB1x', 'NSB5x') prod_step_2 = ProductionStep() prod_step_2.Name = 'Analysis_'+DL0_data_set_NSB.replace('AdvancedBaseline_', '').replace('DL0', 'DL1') prod_step_2.Type = 'DataReprocessing' # This corresponds to the Transformation Type prod_step_2.Inputquery = get_dataset_MQ(DL0_data_set_NSB) prod_step_2.Outputquery = get_dataset_MQ(DL0_data_set_NSB.replace('DL0', 'DL1')) # Here define the job description to be associated to the second ProductionStep job2 = EvnDispProd5Job() job2.setName('Prod5_EvnDisp') # output job2.setOutputSandbox(['*Log.txt']) # refine output meta data if needed output_meta_data = copy(prod_step_2.Outputquery) job2.set_meta_data(output_meta_data) job2.set_file_meta_data(nsb=output_meta_data['nsb']['=']) # check if La Palma else use default that is Paranal if output_meta_data['site'] == 'LaPalma': job2.prefix = "CTA.prod5N" job2.layout_list = 'BL-0LSTs05MSTs-MSTF BL-0LSTs05MSTs-MSTN \ BL-4LSTs00MSTs-MSTN BL-4LSTs05MSTs-MSTF \ BL-4LSTs05MSTs-MSTN BL-4LSTs09MSTs-MSTF \ BL-4LSTs09MSTs-MSTN BL-4LSTs15MSTs-MSTF \ BL-4LSTs15MSTs-MSTN' DIRAC.gLogger.notice('LaPalma layouts:\n',job2.layout_list.split()) elif output_meta_data['site'] == 'Paranal': DIRAC.gLogger.notice('Paranal layouts:\n',job2.layout_list.split()) job2.ts_task_id = '@{JOB_ID}' # dynamic job2.setupWorkflow(debug=False) job2.setType('EvnDisp3') # mandatory *here* prod_step_2.Body = job2.workflow.toXML() prod_step_2.GroupSize = 5 return prod_step_2
def launch_job(args): """ Simple launcher to instanciate a Job and setup parameters from positional arguments given on the command line. Parameters: args -- mode (trans_name dataset_name group_size) """ DIRAC.gLogger.notice('Running EventDisplay jobs') # get arguments mode = args[0] if mode == 'TS': trans_name = args[1] dataset_name = args[2] group_size = int(args[3]) # job setup - 72 hours job = EvnDispProd5SingJob(cpuTime=259200.) job.version = 'eventdisplay-cta-dl1-prod5.v03' # override for testing job.setName('Prod5b_EvnDisp') # output job.setOutputSandbox(['*Log.txt']) # specific configuration if mode == 'WMS': job.base_path = '/vo.cta.in2p3.fr/user/b/bregeon' job.ts_task_id = '62' simtel_meta_data = { 'array_layout': 'Advanced-Baseline', 'site': 'LaPalma', 'particle': 'proton', 'phiP': 0.0, 'thetaP': 20.0 } job.set_meta_data(simtel_meta_data) job.set_file_meta_data(nsb=1) job.setupWorkflow(debug=True) # subtmit to the WMS for debug # job.setDestination('LCG.IN2P3-CC.fr') result = submit_wms(job) elif mode == 'TS': input_meta_query = get_dataset_MQ(dataset_name) # refine output meta data if needed output_meta_data = copy(input_meta_query) job.set_meta_data(output_meta_data) job.set_file_meta_data(nsb=output_meta_data['nsb']['=']) # adjust calibration file job.ts_task_id = '@{JOB_ID}' # dynamic job.group_size = group_size # for the input files verification job.setupWorkflow(debug=False) job.setType('EvnDisp3') # mandatory *here* result = submit_trans(job, trans_name, input_meta_query, group_size) else: DIRAC.gLogger.error('1st argument should be the job mode: WMS or TS,\n\ not %s' % mode) return None return result
def build_simulation_step(DL0_data_set, name_tag=''): ''' Setup Corsika + sim_telarray step Note that there is no InputQuery, since jobs created by this steps don't require any InputData @return ProductionStep object ''' # Note that there is no InputQuery, # since jobs created by this steps don't require any InputData DIRAC.gLogger.notice('MC Production step') prod_step_1 = ProductionStep() prod_step_1.Name = 'Simulation_%s' % DL0_data_set.replace('AdvancedBaseline_NSB1x_','') prod_step_1.Name += '%s' % name_tag prod_step_1.Type = 'MCSimulation' prod_step_1.Outputquery = get_dataset_MQ(DL0_data_set) prod_step_1.Outputquery['nsb'] = {'in': [1, 5]} # get meta data to be passed to simulation job site = prod_step_1.Outputquery['site'] particle = prod_step_1.Outputquery['particle'] if prod_step_1.Outputquery['phiP']['='] == 180: pointing_dir = 'North' elif prod_step_1.Outputquery['phiP']['='] == 0: pointing_dir = 'South' zenith_angle = prod_step_1.Outputquery['thetaP']['='] # Here define the job description (i.e. Name, Executable, etc.) # to be associated to the first ProductionStep, as done when using the TS job1 = Prod5bMCPipeNSBJob() job1.version = '2020-06-29b' job1.compiler = 'gcc83_matchcpu' # Initialize JOB_ID job1.workflow.addParameter(Parameter("JOB_ID", "000000", "string", "", "", True, False, "Temporary fix")) # configuration job1.setName('Prod5b_MC_Pipeline_NSB') job1.set_site(site) job1.set_particle(particle) job1.set_pointing_dir(pointing_dir) job1.zenith_angle = zenith_angle job1.n_shower = 50000 if particle == 'gamma': job1.n_shower = 20000 job1.setOutputSandbox(['*Log.txt']) job1.start_run_number = '0' job1.run_number = '@{JOB_ID}' # dynamic job1.setupWorkflow(debug=False) # Add the job description to the first ProductionStep prod_step_1.Body = job1.workflow.toXML() # return ProductionStep object return prod_step_1
def build_evndisp_step(DL0_data_set, nsb=1, name_tag=''): ''' Define a new EventDisplay analysis production step @return ProductionStep object ''' if nsb == 1: DIRAC.gLogger.notice('NSB1x Analysis') DL0_data_set_NSB = DL0_data_set elif nsb == 5: DIRAC.gLogger.notice('NSB5x Analysis') DL0_data_set_NSB = DL0_data_set.replace('NSB1x', 'NSB5x') prod_step_2 = ProductionStep() prod_step_2.Name = 'Analysis_'+DL0_data_set_NSB.replace('AdvancedBaseline_', '').replace('DL0', 'DL1') prod_step_2.Name += '%s' % name_tag prod_step_2.Type = 'DataReprocessing' # This corresponds to the Transformation Type prod_step_2.Inputquery = get_dataset_MQ(DL0_data_set_NSB) prod_step_2.Outputquery = get_dataset_MQ(DL0_data_set_NSB.replace('DL0', 'DL1')) # Here define the job description to be associated to the second ProductionStep job2 = EvnDispProd5SingJob(cpuTime=259200.) job2.version = 'eventdisplay-cta-dl1-prod5.v03' job2.setName('Prod5b_EvnDisp_Singularity') # output job2.setOutputSandbox(['*Log.txt']) # refine output meta data if needed output_meta_data = copy(prod_step_2.Outputquery) job2.set_meta_data(output_meta_data) job2.set_file_meta_data(nsb=output_meta_data['nsb']['=']) job2.ts_task_id = '@{JOB_ID}' # dynamic job2.group_size = 5 # for the input files verification job2.setupWorkflow(debug=False) job2.setType('EvnDisp3') # mandatory *here* prod_step_2.Body = job2.workflow.toXML() prod_step_2.GroupSize = 5 # have to match the above group size? # return ProductionStep object return prod_step_2
def runEvnDisp3(args): """ Simple wrapper to create a EvnDisp3RefJob and setup parameters from positional arguments given on the command line. Parameters: args -- (dataset_name group_size) """ DIRAC.gLogger.notice('runEvnDisp3') # get arguments mode = args[0] if mode == 'TS': dataset_name = args[1] group_size = int(args[2]) ################################ job = EvnDisp3RefJobC7(cpuTime=432000) # to be adjusted!! job.setName('EvnDisp3') job.setType('EvnDisp3') job.setOutputSandbox(['*Log.txt']) job.version = "prod3b_d20200521" # change here for Paranal or La Palma job.prefix = "CTA.prod4S" # don't mind reference to prod4, it's prod3 # set calibration file and parameters file job.calibration_file = "prod3b.Paranal-20171214.ped.root" job.configuration_id = 6 if mode == 'WMS': job.base_path = '/vo.cta.in2p3.fr/user/a/arrabito' job.ts_task_id = '1' output_meta_data = { 'array_layout': 'Baseline', 'site': 'Paranal', 'particle': 'gamma', 'phiP': 0.0, 'thetaP': 20.0, job.program_category + '_prog': 'simtel', job.program_category + '_prog_version': job.version, 'data_level': 0, 'configuration_id': job.configuration_id } job.set_meta_data(output_meta_data) job.setupWorkflow(debug=True) # subtmit to the WMS for debug #job.setDestination('LCG.CNAF.it') res = submit_wms(job) elif mode == 'TS': input_meta_query = get_dataset_MQ(dataset_name) output_meta_data = copy(input_meta_query) job.set_meta_data(output_meta_data) job.ts_task_id = '@{JOB_ID}' # dynamic job.setupWorkflow(debug=False) ### submit the workflow to the TS res = submit_trans(job, input_meta_query, group_size) else: DIRAC.gLogger.error( '1st argument should be the job mode: WMS or TS,\n\not %s' % mode) return None return res
def launch_job(args): """ Simple launcher to instanciate a Job and setup parameters from positional arguments given on the command line. Parameters: args -- mode (trans_name dataset_name group_size) """ DIRAC.gLogger.notice('launch_job') # get arguments mode = args[0] if mode == 'TS': trans_name = args[1] dataset_name = args[2] group_size = int(args[3]) # job setup - 72 hours job = Prod3DL1DataHandlerJob(cpuTime=259200.) # override for testing job.setName('Prod3_DL1DataHandler') # output job.setOutputSandbox(['*Log.txt']) # version job.version = 'v0.7.4' # configuration test or train "grid_config_test_02052019.yml" job.split_md = 'test' job.config_file_name = 'grid_config_%s_02052019.yml' % job.split_md # specific configuration if mode == 'WMS': job.base_path = '/vo.cta.in2p3.fr/user/b/bregeon' job.ts_task_id = '123' simtel_meta_data = { 'array_layout': 'Baseline', 'site': 'LaPalma', 'particle': 'proton', 'phiP': 180.0, 'thetaP': 20.0, 'nsb': 1, 'split': job.split_md } job.set_meta_data(simtel_meta_data) job.setupWorkflow(debug=True) job.setType('EvnDisp3') # mandatory *here* # subtmit to the WMS for debug # job.setDestination('LCG.IN2P3-CC.fr') result = submit_wms(job) elif mode == 'TS': input_meta_query = get_dataset_MQ(dataset_name) # refine output directory meta data if needed output_meta_data = copy(input_meta_query) job.set_meta_data(output_meta_data) job.ts_task_id = '@{JOB_ID}' # dynamic job.setupWorkflow(debug=False) job.setType('EvnDisp3') # mandatory *here* result = submit_trans(job, trans_name, json.dumps(input_meta_query), group_size, with_file_mask=True) else: DIRAC.gLogger.error('1st argument should be the job mode: WMS or TS,\n\ not %s' % mode) return None return result
def launch_job(args): """ Simple launcher to instanciate a Job and setup parameters from positional arguments given on the command line. Parameters: args -- mode (trans_name dataset_name group_size) """ DIRAC.gLogger.notice('Launching jobs') # get arguments mode = args[0] if mode == 'TS': trans_name = args[1] dataset_name = args[2] group_size = int(args[3]) # job setup - 72 hours job = EvnDispProd5Job(cpuTime=259200.) # override for testing job.setName('Prod5_EvnDisp') # output job.setOutputSandbox(['*Log.txt']) # specific configuration if mode == 'WMS': job.base_path = '/vo.cta.in2p3.fr/user/b/bregeon' job.ts_task_id = '111' simtel_meta_data = {'array_layout': 'Baseline-Advanced', 'site': 'LaPalma', 'particle': 'gamma', 'phiP': 0.0, 'thetaP': 20.0} job.prefix = 'CTA.prod5N' job.layout_list = 'BL-0LSTs05MSTs-MSTF BL-0LSTs05MSTs-MSTN \ BL-4LSTs00MSTs-MSTN BL-4LSTs05MSTs-MSTF \ BL-4LSTs05MSTs-MSTN BL-4LSTs09MSTs-MSTF \ BL-4LSTs09MSTs-MSTN BL-4LSTs15MSTs-MSTF \ BL-4LSTs15MSTs-MSTN' job.set_meta_data(simtel_meta_data) job.set_file_meta_data({'nsb':1}) job.setupWorkflow(debug=True) # subtmit to the WMS for debug # job.setDestination('LCG.IN2P3-CC.fr') result = submit_wms(job) elif mode == 'TS': input_meta_query = get_dataset_MQ(dataset_name) # refine output meta data if needed output_meta_data = copy(input_meta_query) job.set_meta_data(output_meta_data) job.set_file_meta_data(nsb=output_meta_data['nsb']['=']) if output_meta_data['site'] == 'LaPalma': job.prefix = 'CTA.prod5N' job.layout_list = 'BL-0LSTs05MSTs-MSTF BL-0LSTs05MSTs-MSTN \ BL-4LSTs00MSTs-MSTN BL-4LSTs05MSTs-MSTF \ BL-4LSTs05MSTs-MSTN BL-4LSTs09MSTs-MSTF \ BL-4LSTs09MSTs-MSTN BL-4LSTs15MSTs-MSTF \ BL-4LSTs15MSTs-MSTN' DIRAC.gLogger.notice('LaPalma layouts:\n',job.layout_list.split()) elif output_meta_data['site'] == 'Paranal': DIRAC.gLogger.notice('Paranal layouts:\n',job.layout_list.split()) # adjust calibration file if output_meta_data['nsb']['='] == 5: job.calibration_file = 'prod5/prod5-halfmoon-IPR.root' job.ts_task_id = '@{JOB_ID}' # dynamic job.setupWorkflow(debug=False) job.setType('EvnDisp3') # mandatory *here* result = submit_trans(job, trans_name, input_meta_query, group_size) else: DIRAC.gLogger.error('1st argument should be the job mode: WMS or TS,\n\ not %s' % mode) return None return result
def launch_job(args): """ Simple launcher to instanciate a Job and setup parameters from positional arguments given on the command line. Parameters: args -- mode (trans_name dataset_name group_size) """ DIRAC.gLogger.notice('run_evndisp_sst') # get arguments mode = args[0] if mode == 'TS': trans_name = args[1] dataset_name = args[2] group_size = int(args[3]) # job setup - 72 hours job = EvnDisp4SSTJob(cpuTime=259200.) # override for testing job.setName('Prod4_Evndisp') # output job.setOutputSandbox(['*Log.txt']) # specific configuration if mode == 'WMS': job.base_path = '/vo.cta.in2p3.fr/user/b/bregeon' job.ts_task_id = '123' simtel_meta_data = { 'array_layout': 'Baseline-SST-only', 'site': 'Paranal', 'particle': 'proton', 'phiP': 0.0, 'thetaP': 20.0 } job.set_meta_data(simtel_meta_data) job.set_file_meta_data({'tel_config': 'sst-1m'}) job.setupWorkflow(debug=True) # subtmit to the WMS for debug job.setDestination('LCG.IN2P3-CC.fr') result = submit_wms(job) elif mode == 'TS': input_meta_query = get_dataset_MQ(dataset_name) # refine output meta data if needed output_meta_data = copy(input_meta_query) job.set_meta_data(output_meta_data) file_meta_data = {key: output_meta_data[key] for key in ['tel_config']} job.set_file_meta_data(file_meta_data) if 'astri' in file_meta_data['tel_config']: job.layout = '3HB9-SST-A' job.ts_task_id = '@{JOB_ID}' # dynamic job.setupWorkflow(debug=False) job.setType('EvnDisp3') # mandatory *here* result = submit_trans(job, trans_name, json.dumps(input_meta_query), group_size) else: DIRAC.gLogger.error('1st argument should be the job mode: WMS or TS,\n\ not %s' % mode) return None return result
def launch_job(args): """ Simple launcher to instanciate a Job and setup parameters from positional arguments given on the command line. Parameters: args -- mode (trans_name dataset_name group_size) """ DIRAC.gLogger.notice('Launching jobs') # get arguments mode = args[0] if mode == 'TS': trans_name = args[1] dataset_name = args[2] group_size = int(args[3]) # job setup - 72 hours job = Prod3Stage1Job(cpuTime=259200.) job.stage1_config = 'stage1_config_Prod3_LaPalma_Baseline_NSB1x.json' # override for testing job.setName('Prod3_ctapipe_stage1') # output job.setOutputSandbox(['*Log.txt']) # specific configuration if mode == 'WMS': job.base_path = '/vo.cta.in2p3.fr/user/b/bregeon' job.ts_task_id = '2' simtel_meta_data = { 'array_layout': 'Baseline', 'site': 'LaPalma', 'particle': 'gamma', 'phiP': 180.0, 'thetaP': 20.0 } job.set_meta_data(simtel_meta_data) job.set_file_meta_data({'nsb': 1}) job.setupWorkflow(debug=True) # subtmit to the WMS for debug job.setDestination('LCG.IN2P3-CC.fr') result = submit_wms(job) elif mode == 'TS': job.base_path = '/vo.cta.in2p3.fr/MC/PROD3_Test' input_meta_query = get_dataset_MQ(dataset_name) # refine output meta data if needed output_meta_data = copy(input_meta_query) job.set_meta_data(output_meta_data) job.set_file_meta_data(nsb=output_meta_data['nsb']['='], split=output_meta_data['split']) input_meta_query = {} job.ts_task_id = '@{JOB_ID}' # dynamic job.setupWorkflow(debug=False) job.setType('EvnDisp3') # mandatory *here* result = submit_trans(job, trans_name, input_meta_query, group_size) else: DIRAC.gLogger.error('1st argument should be the job mode: WMS or TS,\n\ not %s' % mode) return None return result