def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) # Get container needed for this script WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.WSCLEAN_PATTERN) # Set names of the run and kill files, open run file for writing submit_file = 'submit_2GC_GC_continue_clean_job.sh' f = open(submit_file,'w') f.write('#!/usr/bin/env bash\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p','rb'),encoding='latin1') targets = project_info['target_list'] SLURM_WSCLEAN_MOD = cfg.SLURM_WSCLEAN SLURM_WSCLEAN_MOD['TIME'] = '18:00:00' PBS_WSCLEAN_MOD = cfg.PBS_WSCLEAN PBS_WSCLEAN_MOD['TIME'] = '18:00:00' # Loop over targets for target in targets: targetname = target[0] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) myms = target[2].rstrip('/') mask0 = sorted(glob.glob(IMAGES+'/*'+filename_targetname+'*.mask.fits')) print('------------------------------------------------------') print(gen.now()+'Target: '+targetname) print(gen.now()+'MS: '+myms) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' print(gen.now()+'Using mask: '+mask) corr_img_prefix = IMAGES+'/img_'+myms+'_pcalmask' # ------------------------------------------------------------------------------ # STEP 1: # Continue wsclean on CORRECTED_DATA column with additional scale id_wsclean = 'WSCM2'+code syscall = 'singularity exec '+WSCLEAN_CONTAINER+' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=corr_img_prefix, continueclean=True, datacol='CORRECTED_DATA', briggs = -1.5, multiscale = True, scales = '0,3,9,18', niter = 400000, bda=False, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean, infrastructure=INFRASTRUCTURE, slurm_config = SLURM_WSCLEAN_MOD, pbs_config = PBS_WSCLEAN_MOD) f.write(run_command+'\n') f.close()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) # Get containers needed for this script WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Get target information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') myms = project_info['master_ms'] pcals = project_info['secondary'] # Set names of the run file, open for writing submit_file = 'submit_image-cal_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') kill_file = SCRIPTS + '/kill_image-cal_jobs.sh' codes = [] ii = 1 # Initialise a list to hold all the job IDs id_list = [] # Loop over secondaries for pcal in pcals: pcalname = pcal[0] filename_pcalname = gen.scrub_target_name(pcalname) field = pcal[1] code = gen.get_target_code(pcalname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) # Image prefix img_prefix = IMAGES + '/img_' + myms + '_' + filename_pcalname + '_corrblind' # ------------------------------------------------------------------------------ # STEP 1: # wsclean with blind deconvolution syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=img_prefix, field=field, datacol='CORRECTED_DATA', imsize=8192, niter=100000, gain=0.2, mgain=0.9, bda=True, mask='none') if len(id_list) > 0: dependency = id_list[-1] else: dependency = None id_wsclean = 'WSCBL' + code id_list.append(id_wsclean) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean, infrastructure=INFRASTRUCTURE, dependency=dependency, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ if INFRASTRUCTURE in ['idia', 'chpc']: kill = 'echo "scancel "$' + '" "$'.join(id_list) + ' > ' + kill_file f.write(kill + '\n') f.write('\n') f.close() gen.make_executable(submit_file)
def main(): USE_SINGULARITY = cfg.USE_SINGULARITY gen.preamble() print(gen.col() + '1GC (referenced calibration) setup') gen.print_spacer() # ------------------------------------------------------------------------------ # # Setup paths, required containers, infrastructure # # ------------------------------------------------------------------------------ gen.setup_dir(cfg.LOGS) gen.setup_dir(cfg.SCRIPTS) gen.setup_dir(cfg.GAINTABLES) INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN, USE_SINGULARITY) MEQTREES_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.MEQTREES_PATTERN, USE_SINGULARITY) RAGAVI_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.RAGAVI_PATTERN, USE_SINGULARITY) SHADEMS_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.SHADEMS_PATTERN, USE_SINGULARITY) # ------------------------------------------------------------------------------ # # 1GC recipe definition # # ------------------------------------------------------------------------------ original_ms = glob.glob('*.ms')[0] code = gen.get_code(original_ms) myms = original_ms.replace('.ms', '_' + str(cfg.PRE_NCHANS) + 'ch.ms') steps = [] # _ ______ ___ _ _ ______ # | | | ___ \ / _ \ | \ | | | _ \ # | | ______ | |_/ / / /_\ \ | \| | | | | | # | | |______| | ___ \ | _ | | . ` | | | | | # | |____ | |_/ / | | | | | |\ | | |/ / # \_____/ \____/ \_| |_/ \_| \_/ |___/ if cfg.BAND[0].upper() == 'L': step = {} step['step'] = 0 step['comment'] = 'Split and average master MS' step['dependency'] = None step['id'] = 'SPPRE' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa( casascript=cfg.OXKAT + '/PRE_casa_average_to_1k_add_wtspec.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 1 step['comment'] = 'Run setup script to generate project_info pickle' step['dependency'] = 0 step['id'] = 'SETUP' + code syscall = CONTAINER_RUNNER + MEQTREES_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += ' python ' + cfg.OXKAT + '/1GC_00_setup.py ' + myms step['syscall'] = syscall steps.append(step) step = {} step['step'] = 2 step[ 'comment'] = 'Rephase primary calibrator visibilties in case of open-time offset problems' step['dependency'] = 1 step['id'] = 'UVFIX' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa( casascript=cfg.OXKAT + '/1GC_01_casa_rephase_primary_calibrator.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 3 step['comment'] = 'Apply basic flagging steps to all fields' step['dependency'] = 2 step['id'] = 'FGBAS' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT + '/1GC_02_casa_basic_flags.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 4 step['comment'] = 'Run setjy for primary calibrator' step['dependency'] = 3 step['id'] = 'SETJY' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT + '/1GC_04_casa_setjy.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 5 step['comment'] = 'Run auto-flaggers on calibrators' step['dependency'] = 4 step['id'] = 'FGCAL' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa( casascript=cfg.OXKAT + '/1GC_05_casa_autoflag_cals_DATA.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 6 step['comment'] = 'Split off calibrator MS with 8 SPWs' step['dependency'] = 5 step['id'] = 'SPCAL' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa( casascript=cfg.OXKAT + '/1GC_06_casa_split_calibrators.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 7 step['comment'] = 'Fit for intrinsic model of secondary calibrator' step['dependency'] = 6 step['id'] = 'CLMOD' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa( casascript=cfg.OXKAT + '/1GC_07_casa_get_secondary_model.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 8 step[ 'comment'] = 'Generate reference calibration solutions and apply to target(s)' step['dependency'] = 7 step['id'] = 'CL1GC' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa( casascript=cfg.OXKAT + '/1GC_08_casa_refcal_using_secondary_model.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 9 step['comment'] = 'Plot the gain solutions' step['dependency'] = 8 step['id'] = 'PLTAB' + code syscall = CONTAINER_RUNNER + RAGAVI_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += 'python3 ' + cfg.OXKAT + '/PLOT_gaintables.py cal_1GC_* cal_1GC_*calibrators.ms*' step['syscall'] = syscall steps.append(step) step = {} step['step'] = 10 step['comment'] = 'Split the corrected target data' step['dependency'] = 8 step['id'] = 'SPTRG' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT + '/1GC_09_casa_split_targets.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 11 step['comment'] = 'Plot the corrected calibrator visibilities' step['dependency'] = 10 step['id'] = 'PLVIS' + code syscall = CONTAINER_RUNNER + SHADEMS_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += 'python3 ' + cfg.OXKAT + '/1GC_10_plot_visibilities.py' step['syscall'] = syscall steps.append(step) # _ _ _ _ ______ # | | | | | | | | | ___| # | | | | | |_| | | |_ # | | | | | _ | | _| # | |_| | | | | | | | # \___/ \_| |_/ \_| elif cfg.BAND[0].upper() == 'U': step = {} step['step'] = 0 step['comment'] = 'Split and average master MS' step['dependency'] = None step['id'] = 'SPPRE' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa( casascript=cfg.OXKAT + '/PRE_casa_average_to_1k_add_wtspec.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 1 step['comment'] = 'Run setup script to generate project_info pickle' step['dependency'] = 0 step['id'] = 'SETUP' + code syscall = CONTAINER_RUNNER + MEQTREES_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += ' python ' + cfg.OXKAT + '/1GC_00_setup.py ' + myms step['syscall'] = syscall steps.append(step) step = {} step['step'] = 2 step['comment'] = 'Apply basic flagging steps to all fields' step['dependency'] = 1 step['id'] = 'FGBAS' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT + '/1GC_casa_UHF_basic_flags.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 3 step['comment'] = 'Run auto-flaggers on calibrators' step['dependency'] = 2 step['id'] = 'FGCAL' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa( casascript=cfg.OXKAT + '/1GC_casa_UHF_autoflag_cals_DATA.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 4 step[ 'comment'] = 'Generate reference calibration solutions and apply to target(s)' step['dependency'] = 3 step['id'] = 'CL1GC' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT + '/1GC_casa_UHF_refcal.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 5 step['comment'] = 'Plot the gain solutions' step['dependency'] = 4 step['id'] = 'PLTAB' + code syscall = CONTAINER_RUNNER + RAGAVI_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += 'python3 ' + cfg.OXKAT + '/PLOT_gaintables.py cal_1GC_*' step['syscall'] = syscall steps.append(step) step = {} step['step'] = 6 step['comment'] = 'Split the corrected target data' step['dependency'] = 4 step['id'] = 'SPTRG' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT + '/1GC_09_casa_split_targets.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 7 step['comment'] = 'Plot the corrected calibrator visibilities' step['dependency'] = 6 step['id'] = 'PLVIS' + code syscall = CONTAINER_RUNNER + SHADEMS_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += 'python3 ' + cfg.OXKAT + '/1GC_10_plot_visibilities.py' step['syscall'] = syscall steps.append(step) # ------------------------------------------------------------------------------ # # Write the run file and kill file based on the recipe # # ------------------------------------------------------------------------------ submit_file = 'submit_1GC_jobs.sh' kill_file = cfg.SCRIPTS + '/kill_1GC_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + cfg.BINDPATH + '\n') id_list = [] for step in steps: step_id = step['id'] id_list.append(step_id) if step['dependency'] is not None: dependency = steps[step['dependency']]['id'] else: dependency = None syscall = step['syscall'] if 'slurm_config' in step.keys(): slurm_config = step['slurm_config'] else: slurm_config = cfg.SLURM_DEFAULTS if 'pbs_config' in step.keys(): pbs_config = step['pbs_config'] else: pbs_config = cfg.PBS_DEFAULTS comment = step['comment'] run_command = gen.job_handler(syscall=syscall, jobname=step_id, infrastructure=INFRASTRUCTURE, dependency=dependency, slurm_config=slurm_config, pbs_config=pbs_config) f.write('\n# ' + comment + '\n') f.write(run_command) if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': kill = '\necho "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = '\necho "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file) gen.print_spacer() print(gen.col('Run file') + submit_file) gen.print_spacer()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if INFRASTRUCTURE == 'idia': myNCPU = 8 elif INFRASTRUCTURE == 'chpc': myNCPU = 23 else: myNCPU = 40 # Get paths from config and setup folders CWD = cfg.CWD OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) gen.setup_dir(GAINTABLES) # Enable running without containers if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' # Get containers needed for this script DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.DDFACET_PATTERN) KILLMS_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.KILLMS_PATTERN) # Set names of the run and kill files, open run file for writing submit_file = 'submit_3GC_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # Loop over targets codes = [] ii = 1 for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): print('------------------------------------------------------') print(gen.now() + myms + ' not found, skipping ' + targetname) else: filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) mask1 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask1.zoom*.fits')) if len(mask1) > 0: mask = mask1[0] else: mask = 'auto' region = glob.glob('*' + targetname + '*.reg') if len(region) == 0: print(gen.now() + 'Please provide a region file') print( "Reg file not supplied - will crash or complain on step 2") region = "noreg.reg" #import IPython #IPython.embed() #sys.exit() else: region = region[0] print('------------------------------------------------------') print(gen.now() + 'Target: ' + targetname) print(gen.now() + 'MS: ' + myms) print(gen.now() + 'Using mask: ' + mask) print(gen.now() + 'Using region: ' + region) f.write('\n# ' + targetname + '\n') kill_file = SCRIPTS + '/kill_3GC_jobs_' + filename_targetname + '.sh' ddf_img_prefix = IMAGES + '/img_' + myms + '_DDFpcal' kms_img_prefix = IMAGES + '/img_' + myms + '_DDFkMS' # Initialise a list to hold all the job IDs id_list = [] # ------------------------------------------------------------------------------ # STEP 1: # DDFacet on CORRECTED_DATA column id_ddfacet1 = 'DDCMA' + code id_list.append(id_ddfacet1) syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' ' syscall += gen.generate_syscall_ddfacet( mspattern=myms, imgname=ddf_img_prefix, ncpu=myNCPU, mask=mask, sparsification='100,30,10,2') run_command = gen.job_handler(syscall=syscall, jobname=id_ddfacet1, infrastructure=INFRASTRUCTURE, slurm_config=cfg.SLURM_HIGHMEM, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 2: # Run killMS id_killms = 'KILMS' + code id_list.append(id_killms) syscall = CONTAINER_RUNNER + KILLMS_CONTAINER + ' ' syscall += 'python3 ' + TOOLS + '/reg2npy.py ' + region + '\n ' syscall += CONTAINER_RUNNER + KILLMS_CONTAINER + ' ' syscall += gen.generate_syscall_killms(myms=myms, baseimg=ddf_img_prefix, ncpu=myNCPU, outsols='killms-cohjones', nodesfile=region + '.npy') run_command = gen.job_handler(syscall=syscall, jobname=id_killms, dependency=id_ddfacet1, infrastructure=INFRASTRUCTURE, slurm_config=cfg.SLURM_HIGHMEM, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 3: # DDFacet on CORRECTED_DATA column, apply killMS solutions id_ddfacet2 = 'DDKMA' + code id_list.append(id_ddfacet1) syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' ' syscall += gen.generate_syscall_ddfacet( mspattern=myms, imgname=kms_img_prefix, chunkhours=1, ncpu=myNCPU, initdicomodel=ddf_img_prefix + '.DicoModel', hogbom_maxmajoriter=0, hogbom_maxminoriter=1000, mask=mask, ddsols='killms-cohjones') run_command = gen.job_handler(syscall=syscall, jobname=id_ddfacet2, dependency=id_killms, infrastructure=INFRASTRUCTURE, slurm_config=cfg.SLURM_HIGHMEM, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ if INFRASTRUCTURE == 'idia': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD BIND = cfg.BIND OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS GAINTABLES = cfg.GAINTABLES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS BINDPATH = '$PWD,'+CWD+','+BIND gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(GAINTABLES) # Enable running without containers if CONTAINER_PATH is not None: CONTAINER_RUNNER='singularity exec ' else: CONTAINER_RUNNER='' # Get containers needed for this script CASA_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.CASA_PATTERN) RAGAVI_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.RAGAVI_PATTERN) SHADEMS_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.SHADEMS_PATTERN) MEQTREES_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.MEQTREES_PATTERN) # Set names of the run and kill files, open run file for writing submit_file = 'submit_1GC_jobs.sh' kill_file = SCRIPTS+'/kill_1GC_jobs.sh' f = open(submit_file,'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH='+BINDPATH+'\n') # Get the MS name original_ms = glob.glob('*.ms')[0] code = gen.get_code(original_ms) myms = original_ms.replace('.ms','_'+str(cfg.PRE_NCHANS)+'ch.ms') # Initialise a list to hold all the job IDs id_list = [] # ------------------------------------------------------------------------------ # Pre-processing: # Duplicate orignal MS, average to 1k channels if required id_average = 'SPPRE'+code id_list.append(id_average) casalog = LOGS+'/casa_1GC_'+id_average+'.log' syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' syscall += gen.generate_syscall_casa(casascript=OXKAT+'/PRE_casa_average_to_1k_add_wtspec.py', casalogfile=casalog) run_command = gen.job_handler(syscall=syscall, jobname=id_average, infrastructure=INFRASTRUCTURE) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 0: # Examine MS and store relevant deductions in project_info.p id_setup = 'SETUP'+code id_list.append(id_setup) syscall = CONTAINER_RUNNER+MEQTREES_CONTAINER+' ' syscall += 'python '+OXKAT+'/1GC_00_setup.py '+myms run_command = gen.job_handler(syscall=syscall, jobname=id_setup, infrastructure=INFRASTRUCTURE, dependency=id_average) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 1: # Rephase the primary calibrators to correct positions id_fixvis = 'UVFIX'+code id_list.append(id_fixvis) casalog = LOGS+'/casa_1GC_'+id_fixvis+'.log' syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' syscall += gen.generate_syscall_casa(casascript=OXKAT+'/1GC_01_casa_rephase_primary_calibrator.py', casalogfile=casalog) run_command = gen.job_handler(syscall=syscall, jobname=id_fixvis, infrastructure=INFRASTRUCTURE, dependency=id_setup) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 2: # Apply basic flagging steps to all fields id_basic = 'FGBAS'+code id_list.append(id_basic) casalog = LOGS+'/casa_1GC_'+id_basic+'.log' syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' syscall += gen.generate_syscall_casa(casascript=OXKAT+'/1GC_02_casa_basic_flags.py', casalogfile=casalog) run_command = gen.job_handler(syscall=syscall, jobname=id_basic, infrastructure=INFRASTRUCTURE, dependency=id_fixvis) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 3: # Run auto-flaggers on calibrators id_autoflagcals = 'FGCAL'+code id_list.append(id_autoflagcals) casalog = LOGS+'/casa_1GC_'+id_autoflagcals+'.log' syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' syscall += gen.generate_syscall_casa(casascript=OXKAT+'/1GC_03_casa_autoflag_cals_DATA.py', casalogfile=casalog) run_command = gen.job_handler(syscall=syscall, jobname=id_autoflagcals, infrastructure=INFRASTRUCTURE, dependency=id_basic) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 4: # Split calibrators into a MS with 8 spectral windows id_splitcals = 'SPCAL'+code id_list.append(id_splitcals) casalog = LOGS+'/casa_1GC_'+id_splitcals+'.log' syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' syscall += gen.generate_syscall_casa(casascript=OXKAT+'/1GC_04_casa_split_calibrators.py', casalogfile=casalog) run_command = gen.job_handler(syscall=syscall, jobname=id_splitcals, infrastructure=INFRASTRUCTURE, dependency=id_autoflagcals) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 5: # Derive an intrinsic spectral model for the secondary calibrator id_secondarymodel = 'CLMOD'+code id_list.append(id_secondarymodel) casalog = LOGS+'/casa_1GC_'+id_secondarymodel+'.log' syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' syscall += gen.generate_syscall_casa(casascript=OXKAT+'/1GC_05_casa_get_secondary_model.py', casalogfile=casalog) run_command = gen.job_handler(syscall=syscall, jobname=id_secondarymodel, infrastructure=INFRASTRUCTURE, dependency=id_splitcals) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 6: (1GC) # Perform reference calibration steps and apply to target(s) id_1GC = 'CL1GC'+code id_list.append(id_1GC) casalog = LOGS+'/casa_1GC_'+id_1GC+'.log' syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' syscall += gen.generate_syscall_casa(casascript=OXKAT+'/1GC_06_casa_refcal_using_secondary_model.py', casalogfile=casalog) run_command = gen.job_handler(syscall=syscall, jobname=id_1GC, infrastructure=INFRASTRUCTURE, dependency=id_secondarymodel) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 7: # Make gain table plots id_gainplots = 'PLTAB'+code id_list.append(id_gainplots) syscall = CONTAINER_RUNNER+RAGAVI_CONTAINER+' ' syscall += 'python3 '+OXKAT+'/PLOT_gaintables.py cal_1GC_* cal_1GC_*calibrators.ms*' run_command = gen.job_handler(syscall=syscall, jobname=id_gainplots, infrastructure=INFRASTRUCTURE, dependency=id_1GC) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 8: # Split the corrected target data into individual Measurement Sets id_splittargets = 'SPTRG'+code id_list.append(id_splittargets) casalog = LOGS+'/casa_1GC_'+id_splittargets+'.log' syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' syscall += gen.generate_syscall_casa(casascript=OXKAT+'/1GC_07_casa_split_targets.py', casalogfile=casalog) run_command = gen.job_handler(syscall=syscall, jobname=id_splittargets, infrastructure=INFRASTRUCTURE, dependency=id_1GC) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 9: # Make visibility plots id_visplots = 'PLVIS'+code id_list.append(id_visplots) syscall = CONTAINER_RUNNER+SHADEMS_CONTAINER+' ' syscall += 'python3 '+OXKAT+'/1GC_08_plot_visibilities.py' run_command = gen.job_handler(syscall=syscall, jobname=id_visplots, infrastructure=INFRASTRUCTURE, dependency=id_splittargets) f.write(run_command) # ------------------------------------------------------------------------------ if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': kill = 'echo "scancel "$'+'" "$'.join(id_list)+' > '+kill_file+'\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$'+'" "$'.join(id_list)+' > '+kill_file+'\n' f.write(kill) f.close() gen.make_executable(submit_file)
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD BIND = cfg.BIND OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS BINDPATH = '$PWD,' + CWD + ',' + BIND gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) gen.setup_dir(GAINTABLES) # Get containers needed for this script CUBICAL_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CUBICAL_PATTERN) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Set names of the run and kill files, open run file for writing submit_file = 'submit_3GC_peel_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + BINDPATH + '\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # Loop over targets codes = [] ii = 1 for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): print('------------------------------------------------------') print(gen.now() + myms + ' not found, skipping ' + targetname) else: filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) mask0 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask1.fits')) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' print('------------------------------------------------------') print(gen.now() + 'Target: ' + targetname) print(gen.now() + 'MS: ' + myms) print(gen.now() + 'Using mask: ' + mask) f.write('\n# ' + targetname + '\n') kill_file = SCRIPTS + '/kill_3GC_peel_jobs_' + filename_targetname + '.sh' prepeel_img_prefix = IMAGES + '/img_' + myms + '_prepeel' dir1_img_prefix = prepeel_img_prefix + '-' + cfg.CAL_3GC_PEEL_REGION.split( '/')[-1].split('.')[0] # Initialise a list to hold all the job IDs id_list = [] # ------------------------------------------------------------------------------ # STEP 1: # Masked wsclean on CORRECTED_DATA column with high (frequency) resolution id_wsclean = 'WSDMA' + code id_list.append(id_wsclean) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean( mslist=[myms], imgname=prepeel_img_prefix, datacol='CORRECTED_DATA', briggs=-0.6, chanout=cfg.CAL_3GC_PEEL_NCHAN, bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean, infrastructure=INFRASTRUCTURE, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 2: # Split model images id_imsplit = 'IMSPL' + code id_list.append(id_imsplit) syscall = 'singularity exec ' + CUBICAL_CONTAINER + ' ' syscall += 'python ' + OXKAT + '/3GC_split_model_images.py ' syscall += '--region ' + cfg.CAL_3GC_PEEL_REGION + ' ' syscall += '--prefix ' + prepeel_img_prefix + ' ' run_command = gen.job_handler(syscall=syscall, jobname=id_imsplit, infrastructure=INFRASTRUCTURE, dependency=id_wsclean, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 3: # Predict DIR1 model into MODEL_DATA id_predict1 = 'WS1PR' + code id_list.append(id_predict1) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict( msname=myms, imgbase=dir1_img_prefix, chanout=cfg.CAL_3GC_PEEL_NCHAN) run_command = gen.job_handler(syscall=syscall, jobname=id_predict1, infrastructure=INFRASTRUCTURE, dependency=id_imsplit, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 4: # Add extra data column id_addcol = 'ADCOL' + code id_list.append(id_addcol) syscall = 'singularity exec ' + CUBICAL_CONTAINER + ' ' syscall += 'python ' + TOOLS + '/add_MS_column.py ' syscall += '--colname ' + cfg.CAL_3GC_PEEL_DIR1COLNAME + ' ' syscall += myms run_command = gen.job_handler(syscall=syscall, jobname=id_addcol, infrastructure=INFRASTRUCTURE, dependency=id_predict1) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 5: # Copy MODEL_DATA to new column id_copycol = 'CPCOL' + code id_list.append(id_copycol) syscall = 'singularity exec ' + CUBICAL_CONTAINER + ' ' syscall += 'python ' + TOOLS + '/copy_MS_column.py ' syscall += '--fromcol MODEL_DATA ' syscall += '--tocol ' + cfg.CAL_3GC_PEEL_DIR1COLNAME + ' ' syscall += myms run_command = gen.job_handler(syscall=syscall, jobname=id_copycol, infrastructure=INFRASTRUCTURE, dependency=id_addcol) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 6: # Predict full sky model into MODEL_DATA id_predict2 = 'WS2PR' + code id_list.append(id_predict2) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict( msname=myms, imgbase=prepeel_img_prefix, chanout=cfg.CAL_3GC_PEEL_NCHAN) run_command = gen.job_handler(syscall=syscall, jobname=id_predict2, infrastructure=INFRASTRUCTURE, dependency=id_copycol, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 7: # Run CubiCal id_peel = 'CL3GC' + code id_list.append(id_peel) syscall = 'singularity exec ' + CUBICAL_CONTAINER + ' ' syscall += gen.generate_syscall_cubical( parset=cfg.CAL_3GC_PEEL_PARSET, myms=myms) run_command = gen.job_handler(syscall=syscall, jobname=id_peel, infrastructure=INFRASTRUCTURE, dependency=id_predict2, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ if INFRASTRUCTURE == 'idia': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD BIND = cfg.BIND OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS BINDPATH = '$PWD,' + CWD + ',' + BIND gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) gen.setup_dir(GAINTABLES) # Enable running without containers if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' # Get containers needed for this script CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN) DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.DDFACET_PATTERN) MAKEMASK_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.MAKEMASK_PATTERN) RAGAVI_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.RAGAVI_PATTERN) TRICOLOUR_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.TRICOLOUR_PATTERN) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Set names of the run and kill files, open run file for writing submit_file = 'submit_2GC_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + BINDPATH + '\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # Loop over targets codes = [] ii = 1 for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): print('------------------------------------------------------') print(gen.now() + myms + ' not found, skipping ' + targetname) else: filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) mask0 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask0.fits')) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' print('------------------------------------------------------') print(gen.now() + 'Target: ' + targetname) print(gen.now() + 'MS: ' + myms) print(gen.now() + 'Using mask: ' + mask) f.write('\n# ' + targetname + '\n') kill_file = SCRIPTS + '/kill_2GC_jobs_' + filename_targetname + '.sh' data_img_prefix = IMAGES + '/img_' + myms + '_datamask' corr_img_prefix = IMAGES + '/img_' + myms + '_pcalmask' # Initialise a list to hold all the job IDs id_list = [] # ------------------------------------------------------------------------------ # STEP 1: # Masked wsclean on DATA column id_wsclean1 = 'WSDMA' + code id_list.append(id_wsclean1) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=data_img_prefix, datacol='DATA', bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean1, infrastructure=INFRASTRUCTURE, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 2: # Predict MODEL_DATA id_predict1 = 'WSDPR' + code id_list.append(id_predict1) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict(msname=myms, imgbase=data_img_prefix) run_command = gen.job_handler(syscall=syscall, jobname=id_predict1, infrastructure=INFRASTRUCTURE, dependency=id_wsclean1, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 3: # Self-calibrate phases then amplitudes id_selfcal = 'CL2GC' + code id_list.append(id_selfcal) casalog = LOGS + '/casa_2GC_' + id_selfcal + '.log' syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' syscall += gen.generate_syscall_casa( casascript=OXKAT + '/2GC_casa_selfcal_target_amp_phases.py', casalogfile=casalog, extra_args='mslist=' + myms) run_command = gen.job_handler(syscall=syscall, jobname=id_selfcal, infrastructure=INFRASTRUCTURE, dependency=id_predict1) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 4: # Make gain table plots id_gainplots = 'PLTAB' + code id_list.append(id_gainplots) syscall = CONTAINER_RUNNER + RAGAVI_CONTAINER + ' ' syscall += 'python3 ' + OXKAT + '/PLOT_gaintables.py cal_2GC_*' + myms + '*' run_command = gen.job_handler(syscall=syscall, jobname=id_gainplots, infrastructure=INFRASTRUCTURE, dependency=id_selfcal) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 5: # Masked wsclean on CORRECTED_DATA column id_wsclean2 = 'WSCMA' + code id_list.append(id_wsclean2) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=corr_img_prefix, datacol='CORRECTED_DATA', bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean2, infrastructure=INFRASTRUCTURE, dependency=id_selfcal, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 6: # Make a FITS mask syscall = CONTAINER_RUNNER + MAKEMASK_CONTAINER + ' ' syscall += gen.generate_syscall_makemask( restoredimage=corr_img_prefix + '-MFS-image.fits', outfile=corr_img_prefix + '-MFS-image.mask1.fits', thresh=5.5, zoompix=cfg.DDF_NPIX)[0] id_makemask = 'MASK1' + code id_list.append(id_makemask) run_command = gen.job_handler(syscall=syscall, jobname=id_makemask, infrastructure=INFRASTRUCTURE, dependency=id_wsclean2) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 7: # Predict MODEL_DATA id_predict2 = 'WSCPR' + code id_list.append(id_predict2) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict(msname=myms, imgbase=corr_img_prefix) run_command = gen.job_handler(syscall=syscall, jobname=id_predict2, infrastructure=INFRASTRUCTURE, dependency=id_wsclean2, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ if INFRASTRUCTURE == 'idia': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file)
def main(): USE_SINGULARITY = cfg.USE_SINGULARITY gen.preamble() print(gen.col() + 'FLAG (target flagging) setup') gen.print_spacer() # ------------------------------------------------------------------------------ # # Setup paths, required containers, infrastructure # # ------------------------------------------------------------------------------ OXKAT = cfg.OXKAT DATA = cfg.DATA SCRIPTS = cfg.SCRIPTS TOOLS = cfg.TOOLS gen.setup_dir(cfg.LOGS) gen.setup_dir(cfg.SCRIPTS) INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN, USE_SINGULARITY) TRICOLOUR_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.TRICOLOUR_PATTERN, USE_SINGULARITY) # Get target information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # ------------------------------------------------------------------------------ # # FLAG recipe definition # # ------------------------------------------------------------------------------ target_steps = [] codes = [] ii = 1 # Loop over targets for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('MS') + 'not found, skipping') else: steps = [] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) # Target-specific kill file kill_file = SCRIPTS + '/kill_flag_jobs_' + filename_targetname + '.sh' gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('Measurement Set') + myms) print(gen.col('Code') + code) step = {} step['step'] = 0 step['comment'] = 'Run Tricolour on ' + myms step['dependency'] = None step['id'] = 'TRIC0' + code step['slurm_config'] = cfg.SLURM_TRICOLOUR step['pbs_config'] = cfg.PBS_TRICOLOUR syscall = CONTAINER_RUNNER + TRICOLOUR_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_tricolour( myms=myms, config=DATA + '/tricolour/target_flagging_1_narrow.yaml', datacol='DATA', fields='0', strategy='polarisation') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 1 step['comment'] = 'Backup flag table for ' + myms step['dependency'] = 0 step['id'] = 'SAVFG' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += 'casa -c ' + OXKAT + '/FLAG_casa_backup_flag_table.py --nologger --log2term --nogui ' syscall += 'versionname=tricolour1 mslist=' + myms step['syscall'] = syscall steps.append(step) target_steps.append((steps, kill_file, targetname)) # ------------------------------------------------------------------------------ # # Write the run file and kill file based on the recipe # # ------------------------------------------------------------------------------ submit_file = 'submit_flag_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + cfg.BINDPATH + '\n') for content in target_steps: steps = content[0] kill_file = content[1] targetname = content[2] id_list = [] f.write('\n#---------------------------------------\n') f.write('# ' + targetname) f.write('\n#---------------------------------------\n') for step in steps: step_id = step['id'] id_list.append(step_id) if step['dependency'] is not None: dependency = steps[step['dependency']]['id'] else: dependency = None syscall = step['syscall'] if 'slurm_config' in step.keys(): slurm_config = step['slurm_config'] else: slurm_config = cfg.SLURM_DEFAULTS if 'pbs_config' in step.keys(): pbs_config = step['pbs_config'] else: pbs_config = cfg.PBS_DEFAULTS comment = step['comment'] run_command = gen.job_handler(syscall=syscall, jobname=step_id, infrastructure=INFRASTRUCTURE, dependency=dependency, slurm_config=slurm_config, pbs_config=pbs_config) f.write('\n# ' + comment + '\n') f.write(run_command) if INFRASTRUCTURE != 'node': f.write('\n# Generate kill script for ' + targetname + '\n') if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file) gen.print_spacer() print(gen.col('Run file') + submit_file) gen.print_spacer()
def main(): USE_SINGULARITY = cfg.USE_SINGULARITY gen.preamble() print(gen.now() + 'Imaging secondary calibrators') # ------------------------------------------------------------------------------ # # Setup paths, required containers, infrastructure # # ------------------------------------------------------------------------------ OXKAT = cfg.OXKAT IMAGES = cfg.IMAGES SCRIPTS = cfg.SCRIPTS gen.setup_dir(IMAGES) gen.setup_dir(cfg.LOGS) gen.setup_dir(cfg.SCRIPTS) INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN, USE_SINGULARITY) # Get secondary calibrator information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') myms = project_info['master_ms'] pcal_names = project_info['secondary_names'] pcal_ids = project_info['secondary_ids'] # ------------------------------------------------------------------------------ # # Image calibrators -- recipe definition # # ------------------------------------------------------------------------------ cal_steps = [] codes = [] ii = 1 # Loop over secondaries for i in range(0, len(pcal_ids)): field = pcal_ids[i] calname = pcal_names[i] steps = [] filename_calname = gen.scrub_target_name(calname) code = gen.get_target_code(calname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) gen.print_spacer() print(gen.now() + 'Secondary | ' + calname) print(gen.now() + 'Code | ' + code) # Image prefix img_prefix = IMAGES + '/img_' + myms + '_' + filename_calname + '_corrblind' step = {} step['step'] = i step[ 'comment'] = 'Run wsclean, blind deconvolution of the CORRECTED_DATA for ' + calname if i == 0: step['dependency'] = None else: step['dependency'] = i - 1 step['id'] = 'WSCMA' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=img_prefix, field=field, datacol='CORRECTED_DATA', imsize=8192, niter=40000, gain=0.2, mgain=0.9, bda=True, mask='none') step['syscall'] = syscall steps.append(step) # ------------------------------------------------------------------------------ # # Write the run file and kill file based on the recipe # # ------------------------------------------------------------------------------ submit_file = 'submit_image_secondary_jobs.sh' kill_file = cfg.SCRIPTS + '/kill_image_secondary_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + cfg.BINDPATH + '\n') id_list = [] for step in steps: step_id = step['id'] id_list.append(step_id) if step['dependency'] is not None: dependency = steps[step['dependency']]['id'] else: dependency = None syscall = step['syscall'] if 'slurm_config' in step.keys(): slurm_config = step['slurm_config'] else: slurm_config = cfg.SLURM_DEFAULTS if 'pbs_config' in step.keys(): pbs_config = step['pbs_config'] else: pbs_config = cfg.PBS_DEFAULTS comment = step['comment'] run_command = gen.job_handler(syscall=syscall, jobname=step_id, infrastructure=INFRASTRUCTURE, dependency=dependency, slurm_config=slurm_config, pbs_config=pbs_config) f.write('\n# ' + comment + '\n') f.write(run_command) if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': kill = '\necho "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = '\necho "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file) gen.print_spacer() print(gen.now() + 'Created ' + submit_file) gen.print_spacer()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) # Get container needed for this script WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.WSCLEAN_PATTERN) # Bump up the walltimes SLURM_WSCLEAN_MOD = cfg.SLURM_WSCLEAN SLURM_WSCLEAN_MOD['TIME'] = '36:00:00' PBS_WSCLEAN_MOD = cfg.PBS_WSCLEAN PBS_WSCLEAN_MOD['WALLTIME'] = '36:00:00' # Set names of the run file, open for writing submit_file = 'submit_2GC_make_cube.sh' f = open(submit_file,'w') f.write('#!/usr/bin/env bash\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p','rb'),encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # Loop over targets codes = [] ii = 1 for tt in range(0,len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) mask0 = sorted(glob.glob(IMAGES+'/*'+filename_targetname+'*.mask.fits')) print('------------------------------------------------------') print(gen.now()+'Target: '+targetname) print(gen.now()+'MS: '+myms) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' print(gen.now()+'Using mask: '+mask) cube_prefix = IMAGES+'/img_'+myms+'_subbands' # ------------------------------------------------------------------------------ # STEP 1: # Continue wsclean on CORRECTED_DATA column with additional scale id_wsclean = 'WCUBE'+code syscall = 'singularity exec '+WSCLEAN_CONTAINER+' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=cube_prefix, datacol='CORRECTED_DATA', sourcelist = False, chanout = 16, briggs = -1.5, multiscale = True, scales = '0,3,9', minuvl = 164, tapergaussian = 8, joinchannels = False, fitspectralpol = 0, niter = 500000, bda = True, mask = mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean, infrastructure=INFRASTRUCTURE, slurm_config = SLURM_WSCLEAN_MOD, pbs_config = PBS_WSCLEAN_MOD) f.write(run_command+'\n') f.close()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) # Get container needed for this script WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Bump up the walltimes SLURM_WSCLEAN_MOD = cfg.SLURM_WSCLEAN SLURM_WSCLEAN_MOD['TIME'] = '18:00:00' PBS_WSCLEAN_MOD = cfg.PBS_WSCLEAN PBS_WSCLEAN_MOD['WALLTIME'] = '18:00:00' # Set names of the run file, open for writing submit_file = 'submit_2GC_continue_clean_no_maskjobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # Loop over targets codes = [] ii = 1 for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) print('------------------------------------------------------') print(gen.now() + 'Target: ' + targetname) print(gen.now() + 'MS: ' + myms) corr_img_prefix = IMAGES + '/img_' + myms + '_pcalmask' # ------------------------------------------------------------------------------ # STEP 1: # Continue wsclean on CORRECTED_DATA column with additional scale id_wsclean = 'WSCN3' + code syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=corr_img_prefix, continueclean=True, datacol='CORRECTED_DATA', briggs=-1.5, multiscale=True, scales='0,3,9,18', niter=300000, bda=False, mask='none') run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean, infrastructure=INFRASTRUCTURE, slurm_config=SLURM_WSCLEAN_MOD, pbs_config=PBS_WSCLEAN_MOD) f.write(run_command + '\n') f.close()
def main(): USE_SINGULARITY = cfg.USE_SINGULARITY gen.preamble() print(gen.col() + 'FLAG (target flagging & initial mask-making) setup') gen.print_spacer() # ------------------------------------------------------------------------------ # # Setup paths, required containers, infrastructure # # ------------------------------------------------------------------------------ OXKAT = cfg.OXKAT DATA = cfg.DATA IMAGES = cfg.IMAGES SCRIPTS = cfg.SCRIPTS TOOLS = cfg.TOOLS gen.setup_dir(IMAGES) gen.setup_dir(cfg.LOGS) gen.setup_dir(cfg.SCRIPTS) INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN, USE_SINGULARITY) CUBICAL_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CUBICAL_PATTERN, USE_SINGULARITY) MAKEMASK_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.MAKEMASK_PATTERN, USE_SINGULARITY) TRICOLOUR_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.TRICOLOUR_PATTERN, USE_SINGULARITY) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN, USE_SINGULARITY) # Get target information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # ------------------------------------------------------------------------------ # # FLAG recipe definition # # ------------------------------------------------------------------------------ target_steps = [] codes = [] ii = 1 # Loop over targets for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('MS') + 'not found, skipping') else: steps = [] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) # Image prefix img_prefix = IMAGES + '/img_' + myms + '_datablind' # Target-specific kill file kill_file = SCRIPTS + '/kill_flag_jobs_' + filename_targetname + '.sh' gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('Measurement Set') + myms) print(gen.col('Code') + code) step = {} step['step'] = 0 step['comment'] = 'Run Tricolour on ' + myms step['dependency'] = None step['id'] = 'TRIC0' + code step['slurm_config'] = cfg.SLURM_TRICOLOUR step['pbs_config'] = cfg.PBS_TRICOLOUR syscall = CONTAINER_RUNNER + TRICOLOUR_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_tricolour( myms=myms, config=DATA + '/tricolour/target_flagging_1_narrow.yaml', datacol='DATA', fields='0', strategy='polarisation') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 1 step['comment'] = 'Blind wsclean on DATA column of ' + myms step['dependency'] = 0 step['id'] = 'WSDBL' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN absmem = gen.absmem_helper(step, INFRASTRUCTURE, cfg.WSC_ABSMEM) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=img_prefix, datacol='DATA', bda=True, automask=False, autothreshold=False, localrms=False, mask=False, absmem=absmem) step['syscall'] = syscall steps.append(step) step = {} step['step'] = 2 step['comment'] = 'Make initial cleaning mask for ' + targetname step['dependency'] = 1 step['id'] = 'MASK0' + code syscall = CONTAINER_RUNNER + MAKEMASK_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_makemask( restoredimage=img_prefix + '-MFS-image.fits', outfile=img_prefix + '-MFS-image.mask0.fits', zoompix='')[0] step['syscall'] = syscall steps.append(step) step = {} step['step'] = 3 step['comment'] = 'Backup flag table for ' + myms step['dependency'] = 1 step['id'] = 'SAVFG' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += 'casa -c ' + OXKAT + '/FLAG_casa_backup_flag_table.py --nologger --log2term --nogui ' syscall += 'versionname=tricolour1 mslist=' + myms step['syscall'] = syscall steps.append(step) # step = {} # step['step'] = 4 # step['comment'] = 'Add CORRECTED_DATA column to '+myms # step['dependency'] = 3 # step['id'] = 'ADCOR'+code # syscall = CONTAINER_RUNNER+CUBICAL_CONTAINER+' ' if USE_SINGULARITY else '' # syscall += 'python '+TOOLS+'/add_MS_column.py ' # syscall += '--colname CORRECTED_DATA ' # syscall += myms # step['syscall'] = syscall # steps.append(step) # step = {} # step['step'] = 5 # step['comment'] = 'Copy DATA to CORRECTED_DATA for '+myms # step['dependency'] = 4 # step['id'] = 'CPDAT'+code # syscall = CONTAINER_RUNNER+CUBICAL_CONTAINER+' ' if USE_SINGULARITY else '' # syscall += 'python '+TOOLS+'/copy_MS_column.py ' # syscall += '--fromcol DATA ' # syscall += '--tocol CORRECTED_DATA ' # syscall += myms # step['syscall'] = syscall # steps.append(step) target_steps.append((steps, kill_file, targetname)) # ------------------------------------------------------------------------------ # # Write the run file and kill file based on the recipe # # ------------------------------------------------------------------------------ submit_file = 'submit_flag_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + cfg.BINDPATH + '\n') for content in target_steps: steps = content[0] kill_file = content[1] targetname = content[2] id_list = [] f.write('\n#---------------------------------------\n') f.write('# ' + targetname) f.write('\n#---------------------------------------\n') for step in steps: step_id = step['id'] id_list.append(step_id) if step['dependency'] is not None: dependency = steps[step['dependency']]['id'] else: dependency = None syscall = step['syscall'] if 'slurm_config' in step.keys(): slurm_config = step['slurm_config'] else: slurm_config = cfg.SLURM_DEFAULTS if 'pbs_config' in step.keys(): pbs_config = step['pbs_config'] else: pbs_config = cfg.PBS_DEFAULTS comment = step['comment'] run_command = gen.job_handler(syscall=syscall, jobname=step_id, infrastructure=INFRASTRUCTURE, dependency=dependency, slurm_config=slurm_config, pbs_config=pbs_config) f.write('\n# ' + comment + '\n') f.write(run_command) if INFRASTRUCTURE != 'node': f.write('\n# Generate kill script for ' + targetname + '\n') if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file) gen.print_spacer() print(gen.col('Run file') + submit_file) gen.print_spacer()
def main(): USE_SINGULARITY = cfg.USE_SINGULARITY gen.preamble() print(gen.col() + '2GC (direction independent selfcal) setup') gen.print_spacer() # ------------------------------------------------------------------------------ # # Setup paths, required containers, infrastructure # # ------------------------------------------------------------------------------ OXKAT = cfg.OXKAT DATA = cfg.DATA GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES SCRIPTS = cfg.SCRIPTS gen.setup_dir(GAINTABLES) gen.setup_dir(IMAGES) gen.setup_dir(cfg.LOGS) gen.setup_dir(cfg.SCRIPTS) INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN, USE_SINGULARITY) MAKEMASK_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.MAKEMASK_PATTERN, USE_SINGULARITY) RAGAVI_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.RAGAVI_PATTERN, USE_SINGULARITY) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN, USE_SINGULARITY) # Get target information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # ------------------------------------------------------------------------------ # # 2GC recipe definition # # ------------------------------------------------------------------------------ target_steps = [] codes = [] ii = 1 # Loop over targets for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('MS') + 'not found, skipping') else: steps = [] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) # Look for the FITS mask for this target mask0 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask0.fits')) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('Measurement Set') + myms) print(gen.col('Code') + code) print(gen.col('Mask') + mask) # Image prefixes data_img_prefix = IMAGES + '/img_' + myms + '_datamask' corr_img_prefix = IMAGES + '/img_' + myms + '_pcalmask' # Target-specific kill file kill_file = SCRIPTS + '/kill_2GC_jobs_' + filename_targetname + '.sh' step = {} step['step'] = 0 step[ 'comment'] = 'Run wsclean, masked deconvolution of the DATA column of ' + myms step['dependency'] = None step['id'] = 'WSDMA' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN absmem = gen.absmem_helper(step, INFRASTRUCTURE, cfg.WSC_ABSMEM) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=data_img_prefix, datacol='DATA', bda=True, automask=False, autothreshold=False, localrms=False, mask=mask, absmem=absmem) step['syscall'] = syscall steps.append(step) step = {} step['step'] = 1 step[ 'comment'] = 'Predict model visibilities from imaging of the DATA column' step['dependency'] = 0 step['id'] = 'WSDPR' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN absmem = gen.absmem_helper(step, INFRASTRUCTURE, cfg.WSC_ABSMEM) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_predict(msname=myms, imgbase=data_img_prefix, absmem=absmem) step['syscall'] = syscall steps.append(step) step = {} step['step'] = 2 step['comment'] = 'Run CASA self-calibration script' step['dependency'] = 1 step['id'] = 'CL2GC' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa( casascript=OXKAT + '/2GC_casa_selfcal_target_amp_phases.py', extra_args='mslist=' + myms) step['syscall'] = syscall steps.append(step) step = {} step['step'] = 3 step['comment'] = 'Plot the self-calibration gain solutions' step['dependency'] = 2 step['id'] = 'PLTAB' + code syscall = CONTAINER_RUNNER + RAGAVI_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += 'python3 ' + OXKAT + '/PLOT_gaintables.py cal_2GC_*' + myms + '*' step['syscall'] = syscall steps.append(step) step = {} step['step'] = 4 step[ 'comment'] = 'Run wsclean, masked deconvolution of the CORRECTED_DATA column of ' + myms step['dependency'] = 2 step['id'] = 'WSCMA' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN absmem = gen.absmem_helper(step, INFRASTRUCTURE, cfg.WSC_ABSMEM) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=corr_img_prefix, datacol='CORRECTED_DATA', bda=True, automask=False, autothreshold=False, localrms=False, mask=mask, absmem=absmem) step['syscall'] = syscall steps.append(step) step = {} step['step'] = 5 step[ 'comment'] = 'Refine the cleaning mask for ' + targetname + ', crop for use with DDFacet' step['dependency'] = 4 step['id'] = 'MASK1' + code syscall = CONTAINER_RUNNER + MAKEMASK_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_makemask( restoredimage=corr_img_prefix + '-MFS-image.fits', outfile=corr_img_prefix + '-MFS-image.mask1.fits', thresh=5.5, zoompix=cfg.DDF_NPIX)[0] step['syscall'] = syscall steps.append(step) step = {} step['step'] = 6 step[ 'comment'] = 'Predict model visibilities from imaging of the CORRECTED_DATA column' step['dependency'] = 4 step['id'] = 'WSCPR' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN absmem = gen.absmem_helper(step, INFRASTRUCTURE, cfg.WSC_ABSMEM) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_predict(msname=myms, imgbase=corr_img_prefix, absmem=absmem) step['syscall'] = syscall steps.append(step) target_steps.append((steps, kill_file, targetname)) # ------------------------------------------------------------------------------ # # Write the run file and kill file based on the recipe # # ------------------------------------------------------------------------------ submit_file = 'submit_2GC_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + cfg.BINDPATH + '\n') for content in target_steps: steps = content[0] kill_file = content[1] targetname = content[2] id_list = [] f.write('\n#---------------------------------------\n') f.write('# ' + targetname) f.write('\n#---------------------------------------\n') for step in steps: step_id = step['id'] id_list.append(step_id) if step['dependency'] is not None: dependency = steps[step['dependency']]['id'] else: dependency = None syscall = step['syscall'] if 'slurm_config' in step.keys(): slurm_config = step['slurm_config'] else: slurm_config = cfg.SLURM_DEFAULTS if 'pbs_config' in step.keys(): pbs_config = step['pbs_config'] else: pbs_config = cfg.PBS_DEFAULTS comment = step['comment'] run_command = gen.job_handler(syscall=syscall, jobname=step_id, infrastructure=INFRASTRUCTURE, dependency=dependency, slurm_config=slurm_config, pbs_config=pbs_config) f.write('\n# ' + comment + '\n') f.write(run_command) if INFRASTRUCTURE != 'node': f.write('\n# Generate kill script for ' + targetname + '\n') if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file) gen.print_spacer() print(gen.col('Run file') + submit_file) gen.print_spacer()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD BIND = cfg.BIND OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS BINDPATH = '$PWD,' + CWD + ',' + BIND gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) # Enable running without containers if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' # Get containers needed for this script CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN) DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.DDFACET_PATTERN) MAKEMASK_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.MAKEMASK_PATTERN) TRICOLOUR_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.TRICOLOUR_PATTERN) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Get target information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # Set names of the run file, open for writing submit_file = 'submit_flag_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + BINDPATH + '\n') codes = [] ii = 1 # Loop over targets for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): print('------------------------------------------------------') print(gen.now() + myms + ' not found, skipping ' + targetname) else: filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) # Image prefix img_prefix = IMAGES + '/img_' + myms + '_datablind' # Target-specific kill file kill_file = SCRIPTS + '/kill_flag_jobs_' + filename_targetname + '.sh' # Initialise a list to hold all the job IDs id_list = [] # ------------------------------------------------------------------------------ # STEP 1: # Run Tricolour on targets syscall = CONTAINER_RUNNER + TRICOLOUR_CONTAINER + ' ' syscall += gen.generate_syscall_tricolour( myms=myms, config=PARSETS + '/tricolour/target_flagging_1_narrow.yaml', datacol='DATA', fields='0', strategy='polarisation') id_tricolour = 'TRICO' + code id_list.append(id_tricolour) run_command = gen.job_handler(syscall=syscall, jobname=id_tricolour, infrastructure=INFRASTRUCTURE, slurm_config=cfg.SLURM_TRICOLOUR, pbs_config=cfg.PBS_TRICOLOUR) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 2: # wsclean with blind deconvolution syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=img_prefix, datacol='DATA', bda=True, niter=100000, autothreshold=1.0, mask='none') id_wsclean = 'WSDBL' + code id_list.append(id_wsclean) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean, infrastructure=INFRASTRUCTURE, dependency=id_tricolour, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 3: # Make a FITS mask syscall = CONTAINER_RUNNER + MAKEMASK_CONTAINER + ' ' syscall += gen.generate_syscall_makemask( restoredimage=img_prefix + '-MFS-image.fits', outfile=img_prefix + '-MFS-image.mask0.fits', zoompix='')[0] id_makemask = 'MASK0' + code id_list.append(id_makemask) run_command = gen.job_handler(syscall=syscall, jobname=id_makemask, infrastructure=INFRASTRUCTURE, dependency=id_wsclean) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 4: # Backup the flag table syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' syscall += 'casa -c ' + OXKAT + '/FLAG_casa_backup_flag_table.py --nologger --log2term --nogui ' syscall += 'versionname=tricolour1 ' id_saveflags = 'SAVFG' + code id_list.append(id_saveflags) run_command = gen.job_handler(syscall=syscall, jobname=id_saveflags, infrastructure=INFRASTRUCTURE, dependency=id_wsclean) f.write(run_command) # ------------------------------------------------------------------------------ if INFRASTRUCTURE in ['idia', 'chpc']: kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file f.write(kill + '\n') f.write('\n') f.close() gen.make_executable(submit_file)
def main(): USE_SINGULARITY = cfg.USE_SINGULARITY gen.preamble() print(gen.col() + '3GC (facet-based corrections) setup') gen.print_spacer() # ------------------------------------------------------------------------------ # # Setup paths, required containers, infrastructure # # ------------------------------------------------------------------------------ OXKAT = cfg.OXKAT DATA = cfg.DATA GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES SCRIPTS = cfg.SCRIPTS TOOLS = cfg.TOOLS gen.setup_dir(GAINTABLES) gen.setup_dir(IMAGES) gen.setup_dir(cfg.LOGS) gen.setup_dir(cfg.SCRIPTS) INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if INFRASTRUCTURE == 'idia': myNCPU = 8 # Dial back the parallelism for IDIA nodes elif INFRASTRUCTURE == 'chpc': myNCPU = 23 # Kind of meaningless as this stuff probably won't ever run on CHPC else: myNCPU = 40 # Assumed NCPU for standalone nodes if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.DDFACET_PATTERN, USE_SINGULARITY) KILLMS_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.KILLMS_PATTERN, USE_SINGULARITY) # Get target information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # ------------------------------------------------------------------------------ # # 3GC peeling recipe definition # # ------------------------------------------------------------------------------ target_steps = [] codes = [] ii = 1 # Loop over targets for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('MS') + 'not found, skipping') else: steps = [] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) # Look for the zoomed FITS mask for this target mask1 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask1.zoom*.fits')) if len(mask1) > 0: mask = mask1[0] else: mask = 'auto' # Look for the DS9 region file that defines the tessel centres for this target region = glob.glob('*' + targetname + '*.reg') if len(region) == 0: gen.print_spacer() print(gen.col() + 'Please provide a region file of the form:') print(gen.col() + ' *' + targetname + '*.reg') print(gen.col() + 'for this field.') gen.print_spacer() sys.exit() else: region = region[0] gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('Measurement Set') + myms) print(gen.col('Code') + code) print(gen.col('Mask') + mask) print(gen.col('Region') + region) # Image prefixes ddf_img_prefix = IMAGES + '/img_' + myms + '_DDFpcal' kms_img_prefix = IMAGES + '/img_' + myms + '_DDFkMS' # Target-specific kill file kill_file = SCRIPTS + '/kill_3GC_facet_jobs_' + filename_targetname + '.sh' step = {} step['step'] = 0 step[ 'comment'] = 'Run DDFacet, masked deconvolution of CORRECTED_DATA column of ' + myms step['dependency'] = None step['id'] = 'DDCMA' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_ddfacet(mspattern=myms, imgname=ddf_img_prefix, ncpu=myNCPU, mask=mask, sparsification='50,20,5,2') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 1 step[ 'comment'] = 'Convert the DS9 region into a numpy file that killMS will recognise' step['dependency'] = 0 step['id'] = 'RG2NP' + code syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += 'python3 ' + TOOLS + '/reg2npy.py ' + region step['syscall'] = syscall steps.append(step) step = {} step['step'] = 2 step['comment'] = 'Run killMS' step['dependency'] = 1 step['id'] = 'KILMS' + code syscall = CONTAINER_RUNNER + KILLMS_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_killms(myms=myms, baseimg=ddf_img_prefix, ncpu=myNCPU, outsols='killms-cohjones', nodesfile=region + '.npy') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 3 step[ 'comment'] = 'Run DDFacet on CORRECTED_DATA of ' + myms + ', applying killMS solutions' step['dependency'] = 2 step['id'] = 'DDKMA' + code syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_ddfacet( mspattern=myms, imgname=kms_img_prefix, chunkhours=1, ncpu=myNCPU, initdicomodel=ddf_img_prefix + '.DicoModel', hogbom_maxmajoriter=0, hogbom_maxminoriter=1000, mask=mask, ddsols='killms-cohjones') step['syscall'] = syscall steps.append(step) target_steps.append((steps, kill_file, targetname)) # ------------------------------------------------------------------------------ # # Write the run file and kill file based on the recipe # # ------------------------------------------------------------------------------ submit_file = 'submit_3GC_facet_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + cfg.BINDPATH + '\n') for content in target_steps: steps = content[0] kill_file = content[1] targetname = content[2] id_list = [] f.write('\n#---------------------------------------\n') f.write('# ' + targetname) f.write('\n#---------------------------------------\n') for step in steps: step_id = step['id'] id_list.append(step_id) if step['dependency'] is not None: dependency = steps[step['dependency']]['id'] else: dependency = None syscall = step['syscall'] if 'slurm_config' in step.keys(): slurm_config = step['slurm_config'] else: slurm_config = cfg.SLURM_DEFAULTS if 'pbs_config' in step.keys(): pbs_config = step['pbs_config'] else: pbs_config = cfg.PBS_DEFAULTS comment = step['comment'] run_command = gen.job_handler(syscall=syscall, jobname=step_id, infrastructure=INFRASTRUCTURE, dependency=dependency, slurm_config=slurm_config, pbs_config=pbs_config) f.write('\n# ' + comment + '\n') f.write(run_command) if INFRASTRUCTURE != 'node': f.write('\n# Generate kill script for ' + targetname + '\n') if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file) gen.print_spacer() print(gen.col('Run file') + submit_file) gen.print_spacer()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) gen.setup_dir(GAINTABLES) # Get containers needed for this script CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN) DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.DDFACET_PATTERN) TRICOLOUR_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.TRICOLOUR_PATTERN) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Set names of the run and kill files, open run file for writing submit_file = 'submit_2GC_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') targets = project_info['target_list'] SLURM_WSCLEAN_MOD = cfg.SLURM_WSCLEAN SLURM_WSCLEAN_MOD['TIME'] = '18:00:00' PBS_WSCLEAN_MOD = cfg.PBS_WSCLEAN PBS_WSCLEAN_MOD['TIME'] = '18:00:00' # Loop over targets for target in targets: targetname = target[0] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) myms = target[2].rstrip('/') mask0 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask.fits')) print('------------------------------------------------------') print(gen.now() + 'Target: ' + targetname) print(gen.now() + 'MS: ' + myms) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' print(gen.now() + 'Using mask: ' + mask) kill_file = SCRIPTS + '/kill_2GC_jobs_' + filename_targetname + '.sh' data_img_prefix = IMAGES + '/img_' + myms + '_datamask' corr_img_prefix = IMAGES + '/img_' + myms + '_pcalmask' # Initialise a list to hold all the job IDs id_list = [] # ------------------------------------------------------------------------------ # STEP 1: # Masked wsclean on DATA column id_wsclean1 = 'WSDMA' + code id_list.append(id_wsclean1) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=data_img_prefix, briggs=-1.5, multiscale=True, scales='0,3,9', niter=300000, datacol='DATA', bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean1, infrastructure=INFRASTRUCTURE, slurm_config=SLURM_WSCLEAN_MOD, pbs_config=PBS_WSCLEAN_MOD) f.write(run_command + '\n') # ------------------------------------------------------------------------------ # STEP 2: # Predict MODEL_DATA id_predict1 = 'WSDPR' + code id_list.append(id_predict1) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict(msname=myms, imgbase=data_img_prefix) run_command = gen.job_handler(syscall=syscall, jobname=id_predict1, infrastructure=INFRASTRUCTURE, dependency=id_wsclean1, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command + '\n') # ------------------------------------------------------------------------------ # STEP 3: # Self-calibrate phases then amplitudes id_selfcal = 'CLSLF' + code id_list.append(id_selfcal) casalog = LOGS + '/casa_2GC_' + id_selfcal + '.log' syscall = 'singularity exec ' + CASA_CONTAINER + ' ' syscall += gen.generate_syscall_casa( casascript=OXKAT + '/2GC_casa_selfcal_target_amp_phases.py', casalogfile=casalog, extra_args='mslist=' + myms + ' uvmin=300') run_command = gen.job_handler(syscall=syscall, jobname=id_selfcal, infrastructure=INFRASTRUCTURE, dependency=id_predict1) f.write(run_command + '\n') # ------------------------------------------------------------------------------ # STEP 4: # Masked wsclean on CORRECTED_DATA column id_wsclean2 = 'WSCMA' + code id_list.append(id_wsclean2) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=corr_img_prefix, datacol='CORRECTED_DATA', briggs=-1.5, multiscale=True, scales='0,3,9', niter=300000, bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean2, infrastructure=INFRASTRUCTURE, dependency=id_selfcal, slurm_config=SLURM_WSCLEAN_MOD, pbs_config=PBS_WSCLEAN_MOD) f.write(run_command + '\n') # ------------------------------------------------------------------------------ # STEP 5: # Predict MODEL_DATA id_predict2 = 'WSCPR' + code id_list.append(id_predict2) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict(msname=myms, imgbase=corr_img_prefix) run_command = gen.job_handler(syscall=syscall, jobname=id_predict2, infrastructure=INFRASTRUCTURE, dependency=id_wsclean2, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command + '\n') # ------------------------------------------------------------------------------ if INFRASTRUCTURE in ['idia', 'chpc']: kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file f.write(kill + '\n') f.close()
def main(): USE_SINGULARITY = cfg.USE_SINGULARITY if cfg.BAND[0].upper() == 'U': print(gen.col()+'Full-field primary models not available for UHF yet') gen.print_spacer() sys.exit() gen.preamble() print(gen.col()+'1GC (referenced calibration) setup') print(gen.col()+'Note that this recipe uses a full-field model for PKS B1934-638.') print(gen.col()+'This is somewhat experimental!') gen.print_spacer() # ------------------------------------------------------------------------------ # # Setup paths, required containers, infrastructure # # ------------------------------------------------------------------------------ gen.setup_dir(cfg.LOGS) gen.setup_dir(cfg.SCRIPTS) gen.setup_dir(cfg.GAINTABLES) INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if CONTAINER_PATH is not None: CONTAINER_RUNNER='singularity exec ' else: CONTAINER_RUNNER='' CASA_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.CASA_PATTERN,USE_SINGULARITY) MEQTREES_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.MEQTREES_PATTERN,USE_SINGULARITY) RAGAVI_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.RAGAVI_PATTERN,USE_SINGULARITY) SHADEMS_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.SHADEMS_PATTERN,USE_SINGULARITY) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.WSCLEAN_PATTERN,USE_SINGULARITY) # ------------------------------------------------------------------------------ # # 1GC recipe definition # # ------------------------------------------------------------------------------ original_ms = glob.glob('*.ms')[0] code = gen.get_code(original_ms) myms = original_ms.replace('.ms','_'+str(cfg.PRE_NCHANS)+'ch.ms') steps = [] step = {} step['step'] = 0 step['comment'] = 'Split and average master MS' step['dependency'] = None step['id'] = 'SPPRE'+code syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT+'/PRE_casa_average_to_1k_add_wtspec.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 1 step['comment'] = 'Run setup script to generate project_info pickle' step['dependency'] = 0 step['id'] = 'SETUP'+code syscall = CONTAINER_RUNNER+MEQTREES_CONTAINER+' ' if USE_SINGULARITY else '' syscall += 'python '+cfg.OXKAT+'/1GC_00_setup.py '+myms step['syscall'] = syscall steps.append(step) step = {} step['step'] = 2 step['comment'] = 'Rephase primary calibrator visibilties in case of open-time offset problems' step['dependency'] = 1 step['id'] = 'UVFIX'+code syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT+'/1GC_01_casa_rephase_primary_calibrator.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 3 step['comment'] = 'Apply basic flagging steps to all fields' step['dependency'] = 2 step['id'] = 'FGBAS'+code syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT+'/1GC_02_casa_basic_flags.py') step['syscall'] = syscall steps.append(step) # step = {} # step['step'] = 4 # step['comment'] = 'Add CORRECTED_DATA and MODEL_DATA columns' # step['dependency'] = 3 # step['id'] = 'ADCOL'+code # syscall = CONTAINER_RUNNER+MEQTREES_CONTAINER+' ' if USE_SINGULARITY else '' # syscall += 'python '+cfg.TOOLS+'/add_MS_column.py --colname CORRECTED_DATA,MODEL_DATA '+myms # step['syscall'] = syscall # steps.append(step) step = {} step['step'] = 4 step['comment'] = 'Predict field sources for primary if required' step['dependency'] = 3 # change back to 4 if above step is required step['id'] = 'SETCC'+code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN syscall = CONTAINER_RUNNER+WSCLEAN_CONTAINER+' ' if USE_SINGULARITY else '' syscall += 'python2 '+cfg.OXKAT+'/1GC_03_primary_cal_field_sources.py' step['syscall'] = syscall steps.append(step) step = {} step['step'] = 5 step['comment'] = 'Copy MODEL_DATA to CORRECTED_DATA (temp storage for primary field sources)' step['dependency'] = 4 step['id'] = 'CPCOL'+code syscall = CONTAINER_RUNNER+MEQTREES_CONTAINER+' ' if USE_SINGULARITY else '' syscall += 'python '+cfg.TOOLS+'/copy_MS_column.py --fromcol MODEL_DATA --tocol CORRECTED_DATA '+myms step['syscall'] = syscall steps.append(step) step = {} step['step'] = 6 step['comment'] = 'Run setjy for primary calibrator' step['dependency'] = 5 step['id'] = 'SETJY'+code syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT+'/1GC_04_casa_setjy.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 7 step['comment'] = 'Add field source model in CORRECTED_DATA to component model in MODEL_DATA' step['dependency'] = 6 step['id'] = 'SMCOL'+code syscall = CONTAINER_RUNNER+MEQTREES_CONTAINER+' ' if USE_SINGULARITY else '' syscall += 'python '+cfg.TOOLS+'/sum_MS_columns.py --src CORRECTED_DATA --dest MODEL_DATA '+myms step['syscall'] = syscall steps.append(step) step = {} step['step'] = 8 step['comment'] = 'Run auto-flaggers on calibrators' step['dependency'] = 7 step['id'] = 'FGCAL'+code syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT+'/1GC_05_casa_autoflag_cals_DATA.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 9 step['comment'] = 'Split off calibrator MS with 8 SPWs' step['dependency'] = 8 step['id'] = 'SPCAL'+code syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT+'/1GC_06_casa_split_calibrators.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 10 step['comment'] = 'Fit for intrinsic model of secondary calibrator' step['dependency'] = 9 step['id'] = 'CLMOD'+code syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT+'/1GC_07_casa_get_secondary_model.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 11 step['comment'] = 'Generate reference calibration solutions and apply to target(s)' step['dependency'] = 10 step['id'] = 'CL1GC'+code syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT+'/1GC_08_casa_refcal_using_secondary_model.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 12 step['comment'] = 'Plot the gain solutions' step['dependency'] = 11 step['id'] = 'PLTAB'+code syscall = CONTAINER_RUNNER+RAGAVI_CONTAINER+' ' if USE_SINGULARITY else '' syscall += 'python3 '+cfg.OXKAT+'/PLOT_gaintables.py cal_1GC_* cal_1GC_*calibrators.ms*' step['syscall'] = syscall steps.append(step) step = {} step['step'] = 13 step['comment'] = 'Split the corrected target data' step['dependency'] = 11 step['id'] = 'SPTRG'+code syscall = CONTAINER_RUNNER+CASA_CONTAINER+' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa(casascript=cfg.OXKAT+'/1GC_09_casa_split_targets.py') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 14 step['comment'] = 'Plot the corrected calibrator visibilities' step['dependency'] = 13 step['id'] = 'PLVIS'+code syscall = CONTAINER_RUNNER+SHADEMS_CONTAINER+' ' if USE_SINGULARITY else '' syscall += 'python3 '+cfg.OXKAT+'/1GC_10_plot_visibilities.py' step['syscall'] = syscall steps.append(step) # ------------------------------------------------------------------------------ # # Write the run file and kill file based on the recipe # # ------------------------------------------------------------------------------ submit_file = 'submit_1GC_jobs.sh' kill_file = cfg.SCRIPTS+'/kill_1GC_jobs.sh' f = open(submit_file,'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH='+cfg.BINDPATH+'\n') id_list = [] for step in steps: step_id = step['id'] id_list.append(step_id) if step['dependency'] is not None: dependency = steps[step['dependency']]['id'] else: dependency = None syscall = step['syscall'] if 'slurm_config' in step.keys(): slurm_config = step['slurm_config'] else: slurm_config = cfg.SLURM_DEFAULTS if 'pbs_config' in step.keys(): pbs_config = step['pbs_config'] else: pbs_config = cfg.PBS_DEFAULTS comment = step['comment'] run_command = gen.job_handler(syscall = syscall, jobname = step_id, infrastructure = INFRASTRUCTURE, dependency = dependency, slurm_config = slurm_config, pbs_config = pbs_config) f.write('\n# '+comment+'\n') f.write(run_command) if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': kill = '\necho "scancel "$'+'" "$'.join(id_list)+' > '+kill_file+'\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = '\necho "qdel "$'+'" "$'.join(id_list)+' > '+kill_file+'\n' f.write(kill) f.close() gen.make_executable(submit_file) gen.print_spacer() print(gen.col('Run file')+submit_file) gen.print_spacer()