def main(): remove_models = True DATA = cfg.DATA CALMODELPATH = DATA+'/calmodels/' # Uncomment when wsclean container comes with python3 as standard... # project_info = pickle.load(open('project_info.p','rb'), encoding = 'latin1') # myms = project_info['master_ms'] # primary_id = project_info['primary_id'] # primary_name = project_info['primary_name'] # primary_tag = project_info['primary_tag'] # ...using the python 2 pickle fix until then. myms = pp.myms primary_id = pp.bpcal primary_name = pp.bpcal_name primary_tag = pp.primary_tag if cfg.CAL_1GC_PRIMARY_MODEL == 'auto': caltar = glob.glob(CALMODELPATH+'*'+primary_tag+'*.tar.gz') if len(caltar) == 1: caltar = caltar[0] print('Found '+caltar+', untarring...') tf = tarfile.open(caltar) tf.extractall(path=CALMODELPATH) fitslist = sorted(glob.glob(CALMODELPATH+'*'+primary_tag+'*.fits')) nchan = len(fitslist) prefix = fitslist[0].split('-00')[0] print('Prefix '+prefix+' has '+str(nchan)+' frequency planes') else: print('No model images found for '+primary_name) elif cfg.CAL_1GC_PRIMARY_MODEL == 'setjy': print('Component model for setjy requested, no additional image-based model prediction will be done.') prefix = '' else: prefix = cfg.CAL_1GC_PRIMARY_MODEL fitslist = sorted(glob.glob(prefix+'*.fits')) if len(fitslist) == 0: print('No FITS images found matching '+prefix+'*.fits') prefix = '' else: nchan = len(fitslist) print('Prefix '+prefix+' has '+str(nchan)+' frequency planes') if prefix != '': syscall = gen.generate_syscall_predict(msname=myms,imgbase=prefix,field=primary_id,chanout=nchan) print(syscall) os.system(syscall) if remove_models: for item in fitslist: print('Removing '+item) os.remove(item)
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) gen.setup_dir(GAINTABLES) # Get containers needed for this script CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN) DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.DDFACET_PATTERN) TRICOLOUR_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.TRICOLOUR_PATTERN) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Set names of the run and kill files, open run file for writing submit_file = 'submit_2GC_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') targets = project_info['target_list'] SLURM_WSCLEAN_MOD = cfg.SLURM_WSCLEAN SLURM_WSCLEAN_MOD['TIME'] = '18:00:00' PBS_WSCLEAN_MOD = cfg.PBS_WSCLEAN PBS_WSCLEAN_MOD['TIME'] = '18:00:00' # Loop over targets for target in targets: targetname = target[0] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) myms = target[2].rstrip('/') mask0 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask.fits')) print('------------------------------------------------------') print(gen.now() + 'Target: ' + targetname) print(gen.now() + 'MS: ' + myms) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' print(gen.now() + 'Using mask: ' + mask) kill_file = SCRIPTS + '/kill_2GC_jobs_' + filename_targetname + '.sh' data_img_prefix = IMAGES + '/img_' + myms + '_datamask' corr_img_prefix = IMAGES + '/img_' + myms + '_pcalmask' # Initialise a list to hold all the job IDs id_list = [] # ------------------------------------------------------------------------------ # STEP 1: # Masked wsclean on DATA column id_wsclean1 = 'WSDMA' + code id_list.append(id_wsclean1) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=data_img_prefix, briggs=-1.5, multiscale=True, scales='0,3,9', niter=300000, datacol='DATA', bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean1, infrastructure=INFRASTRUCTURE, slurm_config=SLURM_WSCLEAN_MOD, pbs_config=PBS_WSCLEAN_MOD) f.write(run_command + '\n') # ------------------------------------------------------------------------------ # STEP 2: # Predict MODEL_DATA id_predict1 = 'WSDPR' + code id_list.append(id_predict1) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict(msname=myms, imgbase=data_img_prefix) run_command = gen.job_handler(syscall=syscall, jobname=id_predict1, infrastructure=INFRASTRUCTURE, dependency=id_wsclean1, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command + '\n') # ------------------------------------------------------------------------------ # STEP 3: # Self-calibrate phases then amplitudes id_selfcal = 'CLSLF' + code id_list.append(id_selfcal) casalog = LOGS + '/casa_2GC_' + id_selfcal + '.log' syscall = 'singularity exec ' + CASA_CONTAINER + ' ' syscall += gen.generate_syscall_casa( casascript=OXKAT + '/2GC_casa_selfcal_target_amp_phases.py', casalogfile=casalog, extra_args='mslist=' + myms + ' uvmin=300') run_command = gen.job_handler(syscall=syscall, jobname=id_selfcal, infrastructure=INFRASTRUCTURE, dependency=id_predict1) f.write(run_command + '\n') # ------------------------------------------------------------------------------ # STEP 4: # Masked wsclean on CORRECTED_DATA column id_wsclean2 = 'WSCMA' + code id_list.append(id_wsclean2) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=corr_img_prefix, datacol='CORRECTED_DATA', briggs=-1.5, multiscale=True, scales='0,3,9', niter=300000, bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean2, infrastructure=INFRASTRUCTURE, dependency=id_selfcal, slurm_config=SLURM_WSCLEAN_MOD, pbs_config=PBS_WSCLEAN_MOD) f.write(run_command + '\n') # ------------------------------------------------------------------------------ # STEP 5: # Predict MODEL_DATA id_predict2 = 'WSCPR' + code id_list.append(id_predict2) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict(msname=myms, imgbase=corr_img_prefix) run_command = gen.job_handler(syscall=syscall, jobname=id_predict2, infrastructure=INFRASTRUCTURE, dependency=id_wsclean2, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command + '\n') # ------------------------------------------------------------------------------ if INFRASTRUCTURE in ['idia', 'chpc']: kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file f.write(kill + '\n') f.close()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD BIND = cfg.BIND OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS BINDPATH = '$PWD,' + CWD + ',' + BIND gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) gen.setup_dir(GAINTABLES) # Get containers needed for this script CUBICAL_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CUBICAL_PATTERN) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Set names of the run and kill files, open run file for writing submit_file = 'submit_3GC_peel_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + BINDPATH + '\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # Loop over targets codes = [] ii = 1 for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): print('------------------------------------------------------') print(gen.now() + myms + ' not found, skipping ' + targetname) else: filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) mask0 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask1.fits')) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' print('------------------------------------------------------') print(gen.now() + 'Target: ' + targetname) print(gen.now() + 'MS: ' + myms) print(gen.now() + 'Using mask: ' + mask) f.write('\n# ' + targetname + '\n') kill_file = SCRIPTS + '/kill_3GC_peel_jobs_' + filename_targetname + '.sh' prepeel_img_prefix = IMAGES + '/img_' + myms + '_prepeel' dir1_img_prefix = prepeel_img_prefix + '-' + cfg.CAL_3GC_PEEL_REGION.split( '/')[-1].split('.')[0] # Initialise a list to hold all the job IDs id_list = [] # ------------------------------------------------------------------------------ # STEP 1: # Masked wsclean on CORRECTED_DATA column with high (frequency) resolution id_wsclean = 'WSDMA' + code id_list.append(id_wsclean) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean( mslist=[myms], imgname=prepeel_img_prefix, datacol='CORRECTED_DATA', briggs=-0.6, chanout=cfg.CAL_3GC_PEEL_NCHAN, bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean, infrastructure=INFRASTRUCTURE, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 2: # Split model images id_imsplit = 'IMSPL' + code id_list.append(id_imsplit) syscall = 'singularity exec ' + CUBICAL_CONTAINER + ' ' syscall += 'python ' + OXKAT + '/3GC_split_model_images.py ' syscall += '--region ' + cfg.CAL_3GC_PEEL_REGION + ' ' syscall += '--prefix ' + prepeel_img_prefix + ' ' run_command = gen.job_handler(syscall=syscall, jobname=id_imsplit, infrastructure=INFRASTRUCTURE, dependency=id_wsclean, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 3: # Predict DIR1 model into MODEL_DATA id_predict1 = 'WS1PR' + code id_list.append(id_predict1) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict( msname=myms, imgbase=dir1_img_prefix, chanout=cfg.CAL_3GC_PEEL_NCHAN) run_command = gen.job_handler(syscall=syscall, jobname=id_predict1, infrastructure=INFRASTRUCTURE, dependency=id_imsplit, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 4: # Add extra data column id_addcol = 'ADCOL' + code id_list.append(id_addcol) syscall = 'singularity exec ' + CUBICAL_CONTAINER + ' ' syscall += 'python ' + TOOLS + '/add_MS_column.py ' syscall += '--colname ' + cfg.CAL_3GC_PEEL_DIR1COLNAME + ' ' syscall += myms run_command = gen.job_handler(syscall=syscall, jobname=id_addcol, infrastructure=INFRASTRUCTURE, dependency=id_predict1) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 5: # Copy MODEL_DATA to new column id_copycol = 'CPCOL' + code id_list.append(id_copycol) syscall = 'singularity exec ' + CUBICAL_CONTAINER + ' ' syscall += 'python ' + TOOLS + '/copy_MS_column.py ' syscall += '--fromcol MODEL_DATA ' syscall += '--tocol ' + cfg.CAL_3GC_PEEL_DIR1COLNAME + ' ' syscall += myms run_command = gen.job_handler(syscall=syscall, jobname=id_copycol, infrastructure=INFRASTRUCTURE, dependency=id_addcol) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 6: # Predict full sky model into MODEL_DATA id_predict2 = 'WS2PR' + code id_list.append(id_predict2) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict( msname=myms, imgbase=prepeel_img_prefix, chanout=cfg.CAL_3GC_PEEL_NCHAN) run_command = gen.job_handler(syscall=syscall, jobname=id_predict2, infrastructure=INFRASTRUCTURE, dependency=id_copycol, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 7: # Run CubiCal id_peel = 'CL3GC' + code id_list.append(id_peel) syscall = 'singularity exec ' + CUBICAL_CONTAINER + ' ' syscall += gen.generate_syscall_cubical( parset=cfg.CAL_3GC_PEEL_PARSET, myms=myms) run_command = gen.job_handler(syscall=syscall, jobname=id_peel, infrastructure=INFRASTRUCTURE, dependency=id_predict2, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ if INFRASTRUCTURE == 'idia': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD BIND = cfg.BIND OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS BINDPATH = '$PWD,' + CWD + ',' + BIND gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) gen.setup_dir(GAINTABLES) # Enable running without containers if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' # Get containers needed for this script CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN) DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.DDFACET_PATTERN) MAKEMASK_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.MAKEMASK_PATTERN) RAGAVI_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.RAGAVI_PATTERN) TRICOLOUR_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.TRICOLOUR_PATTERN) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Set names of the run and kill files, open run file for writing submit_file = 'submit_2GC_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + BINDPATH + '\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # Loop over targets codes = [] ii = 1 for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): print('------------------------------------------------------') print(gen.now() + myms + ' not found, skipping ' + targetname) else: filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) mask0 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask0.fits')) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' print('------------------------------------------------------') print(gen.now() + 'Target: ' + targetname) print(gen.now() + 'MS: ' + myms) print(gen.now() + 'Using mask: ' + mask) f.write('\n# ' + targetname + '\n') kill_file = SCRIPTS + '/kill_2GC_jobs_' + filename_targetname + '.sh' data_img_prefix = IMAGES + '/img_' + myms + '_datamask' corr_img_prefix = IMAGES + '/img_' + myms + '_pcalmask' # Initialise a list to hold all the job IDs id_list = [] # ------------------------------------------------------------------------------ # STEP 1: # Masked wsclean on DATA column id_wsclean1 = 'WSDMA' + code id_list.append(id_wsclean1) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=data_img_prefix, datacol='DATA', bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean1, infrastructure=INFRASTRUCTURE, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 2: # Predict MODEL_DATA id_predict1 = 'WSDPR' + code id_list.append(id_predict1) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict(msname=myms, imgbase=data_img_prefix) run_command = gen.job_handler(syscall=syscall, jobname=id_predict1, infrastructure=INFRASTRUCTURE, dependency=id_wsclean1, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 3: # Self-calibrate phases then amplitudes id_selfcal = 'CL2GC' + code id_list.append(id_selfcal) casalog = LOGS + '/casa_2GC_' + id_selfcal + '.log' syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' syscall += gen.generate_syscall_casa( casascript=OXKAT + '/2GC_casa_selfcal_target_amp_phases.py', casalogfile=casalog, extra_args='mslist=' + myms) run_command = gen.job_handler(syscall=syscall, jobname=id_selfcal, infrastructure=INFRASTRUCTURE, dependency=id_predict1) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 4: # Make gain table plots id_gainplots = 'PLTAB' + code id_list.append(id_gainplots) syscall = CONTAINER_RUNNER + RAGAVI_CONTAINER + ' ' syscall += 'python3 ' + OXKAT + '/PLOT_gaintables.py cal_2GC_*' + myms + '*' run_command = gen.job_handler(syscall=syscall, jobname=id_gainplots, infrastructure=INFRASTRUCTURE, dependency=id_selfcal) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 5: # Masked wsclean on CORRECTED_DATA column id_wsclean2 = 'WSCMA' + code id_list.append(id_wsclean2) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=corr_img_prefix, datacol='CORRECTED_DATA', bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean2, infrastructure=INFRASTRUCTURE, dependency=id_selfcal, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 6: # Make a FITS mask syscall = CONTAINER_RUNNER + MAKEMASK_CONTAINER + ' ' syscall += gen.generate_syscall_makemask( restoredimage=corr_img_prefix + '-MFS-image.fits', outfile=corr_img_prefix + '-MFS-image.mask1.fits', thresh=5.5, zoompix=cfg.DDF_NPIX)[0] id_makemask = 'MASK1' + code id_list.append(id_makemask) run_command = gen.job_handler(syscall=syscall, jobname=id_makemask, infrastructure=INFRASTRUCTURE, dependency=id_wsclean2) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 7: # Predict MODEL_DATA id_predict2 = 'WSCPR' + code id_list.append(id_predict2) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict(msname=myms, imgbase=corr_img_prefix) run_command = gen.job_handler(syscall=syscall, jobname=id_predict2, infrastructure=INFRASTRUCTURE, dependency=id_wsclean2, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ if INFRASTRUCTURE == 'idia': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file)
def main(): USE_SINGULARITY = cfg.USE_SINGULARITY gen.preamble() print(gen.col() + '2GC (direction independent selfcal) setup') gen.print_spacer() # ------------------------------------------------------------------------------ # # Setup paths, required containers, infrastructure # # ------------------------------------------------------------------------------ OXKAT = cfg.OXKAT DATA = cfg.DATA GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES SCRIPTS = cfg.SCRIPTS gen.setup_dir(GAINTABLES) gen.setup_dir(IMAGES) gen.setup_dir(cfg.LOGS) gen.setup_dir(cfg.SCRIPTS) INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN, USE_SINGULARITY) MAKEMASK_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.MAKEMASK_PATTERN, USE_SINGULARITY) RAGAVI_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.RAGAVI_PATTERN, USE_SINGULARITY) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN, USE_SINGULARITY) # Get target information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # ------------------------------------------------------------------------------ # # 2GC recipe definition # # ------------------------------------------------------------------------------ target_steps = [] codes = [] ii = 1 # Loop over targets for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('MS') + 'not found, skipping') else: steps = [] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) # Look for the FITS mask for this target mask0 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask0.fits')) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('Measurement Set') + myms) print(gen.col('Code') + code) print(gen.col('Mask') + mask) # Image prefixes data_img_prefix = IMAGES + '/img_' + myms + '_datamask' corr_img_prefix = IMAGES + '/img_' + myms + '_pcalmask' # Target-specific kill file kill_file = SCRIPTS + '/kill_2GC_jobs_' + filename_targetname + '.sh' step = {} step['step'] = 0 step[ 'comment'] = 'Run wsclean, masked deconvolution of the DATA column of ' + myms step['dependency'] = None step['id'] = 'WSDMA' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN absmem = gen.absmem_helper(step, INFRASTRUCTURE, cfg.WSC_ABSMEM) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=data_img_prefix, datacol='DATA', bda=True, automask=False, autothreshold=False, localrms=False, mask=mask, absmem=absmem) step['syscall'] = syscall steps.append(step) step = {} step['step'] = 1 step[ 'comment'] = 'Predict model visibilities from imaging of the DATA column' step['dependency'] = 0 step['id'] = 'WSDPR' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN absmem = gen.absmem_helper(step, INFRASTRUCTURE, cfg.WSC_ABSMEM) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_predict(msname=myms, imgbase=data_img_prefix, absmem=absmem) step['syscall'] = syscall steps.append(step) step = {} step['step'] = 2 step['comment'] = 'Run CASA self-calibration script' step['dependency'] = 1 step['id'] = 'CL2GC' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa( casascript=OXKAT + '/2GC_casa_selfcal_target_amp_phases.py', extra_args='mslist=' + myms) step['syscall'] = syscall steps.append(step) step = {} step['step'] = 3 step['comment'] = 'Plot the self-calibration gain solutions' step['dependency'] = 2 step['id'] = 'PLTAB' + code syscall = CONTAINER_RUNNER + RAGAVI_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += 'python3 ' + OXKAT + '/PLOT_gaintables.py cal_2GC_*' + myms + '*' step['syscall'] = syscall steps.append(step) step = {} step['step'] = 4 step[ 'comment'] = 'Run wsclean, masked deconvolution of the CORRECTED_DATA column of ' + myms step['dependency'] = 2 step['id'] = 'WSCMA' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN absmem = gen.absmem_helper(step, INFRASTRUCTURE, cfg.WSC_ABSMEM) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=corr_img_prefix, datacol='CORRECTED_DATA', bda=True, automask=False, autothreshold=False, localrms=False, mask=mask, absmem=absmem) step['syscall'] = syscall steps.append(step) step = {} step['step'] = 5 step[ 'comment'] = 'Refine the cleaning mask for ' + targetname + ', crop for use with DDFacet' step['dependency'] = 4 step['id'] = 'MASK1' + code syscall = CONTAINER_RUNNER + MAKEMASK_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_makemask( restoredimage=corr_img_prefix + '-MFS-image.fits', outfile=corr_img_prefix + '-MFS-image.mask1.fits', thresh=5.5, zoompix=cfg.DDF_NPIX)[0] step['syscall'] = syscall steps.append(step) step = {} step['step'] = 6 step[ 'comment'] = 'Predict model visibilities from imaging of the CORRECTED_DATA column' step['dependency'] = 4 step['id'] = 'WSCPR' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN absmem = gen.absmem_helper(step, INFRASTRUCTURE, cfg.WSC_ABSMEM) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_predict(msname=myms, imgbase=corr_img_prefix, absmem=absmem) step['syscall'] = syscall steps.append(step) target_steps.append((steps, kill_file, targetname)) # ------------------------------------------------------------------------------ # # Write the run file and kill file based on the recipe # # ------------------------------------------------------------------------------ submit_file = 'submit_2GC_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + cfg.BINDPATH + '\n') for content in target_steps: steps = content[0] kill_file = content[1] targetname = content[2] id_list = [] f.write('\n#---------------------------------------\n') f.write('# ' + targetname) f.write('\n#---------------------------------------\n') for step in steps: step_id = step['id'] id_list.append(step_id) if step['dependency'] is not None: dependency = steps[step['dependency']]['id'] else: dependency = None syscall = step['syscall'] if 'slurm_config' in step.keys(): slurm_config = step['slurm_config'] else: slurm_config = cfg.SLURM_DEFAULTS if 'pbs_config' in step.keys(): pbs_config = step['pbs_config'] else: pbs_config = cfg.PBS_DEFAULTS comment = step['comment'] run_command = gen.job_handler(syscall=syscall, jobname=step_id, infrastructure=INFRASTRUCTURE, dependency=dependency, slurm_config=slurm_config, pbs_config=pbs_config) f.write('\n# ' + comment + '\n') f.write(run_command) if INFRASTRUCTURE != 'node': f.write('\n# Generate kill script for ' + targetname + '\n') if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file) gen.print_spacer() print(gen.col('Run file') + submit_file) gen.print_spacer()