def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) gen.setup_dir(GAINTABLES) # Get containers needed for this script CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN) DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.DDFACET_PATTERN) TRICOLOUR_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.TRICOLOUR_PATTERN) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Set names of the run and kill files, open run file for writing submit_file = 'submit_2GC_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') targets = project_info['target_list'] SLURM_WSCLEAN_MOD = cfg.SLURM_WSCLEAN SLURM_WSCLEAN_MOD['TIME'] = '18:00:00' PBS_WSCLEAN_MOD = cfg.PBS_WSCLEAN PBS_WSCLEAN_MOD['TIME'] = '18:00:00' # Loop over targets for target in targets: targetname = target[0] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) myms = target[2].rstrip('/') mask0 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask.fits')) print('------------------------------------------------------') print(gen.now() + 'Target: ' + targetname) print(gen.now() + 'MS: ' + myms) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' print(gen.now() + 'Using mask: ' + mask) kill_file = SCRIPTS + '/kill_2GC_jobs_' + filename_targetname + '.sh' data_img_prefix = IMAGES + '/img_' + myms + '_datamask' corr_img_prefix = IMAGES + '/img_' + myms + '_pcalmask' # Initialise a list to hold all the job IDs id_list = [] # ------------------------------------------------------------------------------ # STEP 1: # Masked wsclean on DATA column id_wsclean1 = 'WSDMA' + code id_list.append(id_wsclean1) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=data_img_prefix, briggs=-1.5, multiscale=True, scales='0,3,9', niter=300000, datacol='DATA', bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean1, infrastructure=INFRASTRUCTURE, slurm_config=SLURM_WSCLEAN_MOD, pbs_config=PBS_WSCLEAN_MOD) f.write(run_command + '\n') # ------------------------------------------------------------------------------ # STEP 2: # Predict MODEL_DATA id_predict1 = 'WSDPR' + code id_list.append(id_predict1) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict(msname=myms, imgbase=data_img_prefix) run_command = gen.job_handler(syscall=syscall, jobname=id_predict1, infrastructure=INFRASTRUCTURE, dependency=id_wsclean1, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command + '\n') # ------------------------------------------------------------------------------ # STEP 3: # Self-calibrate phases then amplitudes id_selfcal = 'CLSLF' + code id_list.append(id_selfcal) casalog = LOGS + '/casa_2GC_' + id_selfcal + '.log' syscall = 'singularity exec ' + CASA_CONTAINER + ' ' syscall += gen.generate_syscall_casa( casascript=OXKAT + '/2GC_casa_selfcal_target_amp_phases.py', casalogfile=casalog, extra_args='mslist=' + myms + ' uvmin=300') run_command = gen.job_handler(syscall=syscall, jobname=id_selfcal, infrastructure=INFRASTRUCTURE, dependency=id_predict1) f.write(run_command + '\n') # ------------------------------------------------------------------------------ # STEP 4: # Masked wsclean on CORRECTED_DATA column id_wsclean2 = 'WSCMA' + code id_list.append(id_wsclean2) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=corr_img_prefix, datacol='CORRECTED_DATA', briggs=-1.5, multiscale=True, scales='0,3,9', niter=300000, bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean2, infrastructure=INFRASTRUCTURE, dependency=id_selfcal, slurm_config=SLURM_WSCLEAN_MOD, pbs_config=PBS_WSCLEAN_MOD) f.write(run_command + '\n') # ------------------------------------------------------------------------------ # STEP 5: # Predict MODEL_DATA id_predict2 = 'WSCPR' + code id_list.append(id_predict2) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict(msname=myms, imgbase=corr_img_prefix) run_command = gen.job_handler(syscall=syscall, jobname=id_predict2, infrastructure=INFRASTRUCTURE, dependency=id_wsclean2, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command + '\n') # ------------------------------------------------------------------------------ if INFRASTRUCTURE in ['idia', 'chpc']: kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file f.write(kill + '\n') f.close()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) # Get containers needed for this script WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Get target information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') myms = project_info['master_ms'] pcals = project_info['secondary'] # Set names of the run file, open for writing submit_file = 'submit_image-cal_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') kill_file = SCRIPTS + '/kill_image-cal_jobs.sh' codes = [] ii = 1 # Initialise a list to hold all the job IDs id_list = [] # Loop over secondaries for pcal in pcals: pcalname = pcal[0] filename_pcalname = gen.scrub_target_name(pcalname) field = pcal[1] code = gen.get_target_code(pcalname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) # Image prefix img_prefix = IMAGES + '/img_' + myms + '_' + filename_pcalname + '_corrblind' # ------------------------------------------------------------------------------ # STEP 1: # wsclean with blind deconvolution syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=img_prefix, field=field, datacol='CORRECTED_DATA', imsize=8192, niter=100000, gain=0.2, mgain=0.9, bda=True, mask='none') if len(id_list) > 0: dependency = id_list[-1] else: dependency = None id_wsclean = 'WSCBL' + code id_list.append(id_wsclean) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean, infrastructure=INFRASTRUCTURE, dependency=dependency, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ if INFRASTRUCTURE in ['idia', 'chpc']: kill = 'echo "scancel "$' + '" "$'.join(id_list) + ' > ' + kill_file f.write(kill + '\n') f.write('\n') f.close() gen.make_executable(submit_file)
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD BIND = cfg.BIND OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS BINDPATH = '$PWD,' + CWD + ',' + BIND gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) # Enable running without containers if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' # Get containers needed for this script CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN) DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.DDFACET_PATTERN) MAKEMASK_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.MAKEMASK_PATTERN) TRICOLOUR_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.TRICOLOUR_PATTERN) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Get target information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # Set names of the run file, open for writing submit_file = 'submit_flag_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + BINDPATH + '\n') codes = [] ii = 1 # Loop over targets for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): print('------------------------------------------------------') print(gen.now() + myms + ' not found, skipping ' + targetname) else: filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) # Image prefix img_prefix = IMAGES + '/img_' + myms + '_datablind' # Target-specific kill file kill_file = SCRIPTS + '/kill_flag_jobs_' + filename_targetname + '.sh' # Initialise a list to hold all the job IDs id_list = [] # ------------------------------------------------------------------------------ # STEP 1: # Run Tricolour on targets syscall = CONTAINER_RUNNER + TRICOLOUR_CONTAINER + ' ' syscall += gen.generate_syscall_tricolour( myms=myms, config=PARSETS + '/tricolour/target_flagging_1_narrow.yaml', datacol='DATA', fields='0', strategy='polarisation') id_tricolour = 'TRICO' + code id_list.append(id_tricolour) run_command = gen.job_handler(syscall=syscall, jobname=id_tricolour, infrastructure=INFRASTRUCTURE, slurm_config=cfg.SLURM_TRICOLOUR, pbs_config=cfg.PBS_TRICOLOUR) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 2: # wsclean with blind deconvolution syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=img_prefix, datacol='DATA', bda=True, niter=100000, autothreshold=1.0, mask='none') id_wsclean = 'WSDBL' + code id_list.append(id_wsclean) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean, infrastructure=INFRASTRUCTURE, dependency=id_tricolour, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 3: # Make a FITS mask syscall = CONTAINER_RUNNER + MAKEMASK_CONTAINER + ' ' syscall += gen.generate_syscall_makemask( restoredimage=img_prefix + '-MFS-image.fits', outfile=img_prefix + '-MFS-image.mask0.fits', zoompix='')[0] id_makemask = 'MASK0' + code id_list.append(id_makemask) run_command = gen.job_handler(syscall=syscall, jobname=id_makemask, infrastructure=INFRASTRUCTURE, dependency=id_wsclean) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 4: # Backup the flag table syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' syscall += 'casa -c ' + OXKAT + '/FLAG_casa_backup_flag_table.py --nologger --log2term --nogui ' syscall += 'versionname=tricolour1 ' id_saveflags = 'SAVFG' + code id_list.append(id_saveflags) run_command = gen.job_handler(syscall=syscall, jobname=id_saveflags, infrastructure=INFRASTRUCTURE, dependency=id_wsclean) f.write(run_command) # ------------------------------------------------------------------------------ if INFRASTRUCTURE in ['idia', 'chpc']: kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file f.write(kill + '\n') f.write('\n') f.close() gen.make_executable(submit_file)
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if INFRASTRUCTURE == 'idia': myNCPU = 8 elif INFRASTRUCTURE == 'chpc': myNCPU = 23 else: myNCPU = 40 # Get paths from config and setup folders CWD = cfg.CWD OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) gen.setup_dir(GAINTABLES) # Enable running without containers if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' # Get containers needed for this script DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.DDFACET_PATTERN) KILLMS_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.KILLMS_PATTERN) # Set names of the run and kill files, open run file for writing submit_file = 'submit_3GC_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # Loop over targets codes = [] ii = 1 for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): print('------------------------------------------------------') print(gen.now() + myms + ' not found, skipping ' + targetname) else: filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) mask1 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask1.zoom*.fits')) if len(mask1) > 0: mask = mask1[0] else: mask = 'auto' region = glob.glob('*' + targetname + '*.reg') if len(region) == 0: print(gen.now() + 'Please provide a region file') print( "Reg file not supplied - will crash or complain on step 2") region = "noreg.reg" #import IPython #IPython.embed() #sys.exit() else: region = region[0] print('------------------------------------------------------') print(gen.now() + 'Target: ' + targetname) print(gen.now() + 'MS: ' + myms) print(gen.now() + 'Using mask: ' + mask) print(gen.now() + 'Using region: ' + region) f.write('\n# ' + targetname + '\n') kill_file = SCRIPTS + '/kill_3GC_jobs_' + filename_targetname + '.sh' ddf_img_prefix = IMAGES + '/img_' + myms + '_DDFpcal' kms_img_prefix = IMAGES + '/img_' + myms + '_DDFkMS' # Initialise a list to hold all the job IDs id_list = [] # ------------------------------------------------------------------------------ # STEP 1: # DDFacet on CORRECTED_DATA column id_ddfacet1 = 'DDCMA' + code id_list.append(id_ddfacet1) syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' ' syscall += gen.generate_syscall_ddfacet( mspattern=myms, imgname=ddf_img_prefix, ncpu=myNCPU, mask=mask, sparsification='100,30,10,2') run_command = gen.job_handler(syscall=syscall, jobname=id_ddfacet1, infrastructure=INFRASTRUCTURE, slurm_config=cfg.SLURM_HIGHMEM, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 2: # Run killMS id_killms = 'KILMS' + code id_list.append(id_killms) syscall = CONTAINER_RUNNER + KILLMS_CONTAINER + ' ' syscall += 'python3 ' + TOOLS + '/reg2npy.py ' + region + '\n ' syscall += CONTAINER_RUNNER + KILLMS_CONTAINER + ' ' syscall += gen.generate_syscall_killms(myms=myms, baseimg=ddf_img_prefix, ncpu=myNCPU, outsols='killms-cohjones', nodesfile=region + '.npy') run_command = gen.job_handler(syscall=syscall, jobname=id_killms, dependency=id_ddfacet1, infrastructure=INFRASTRUCTURE, slurm_config=cfg.SLURM_HIGHMEM, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 3: # DDFacet on CORRECTED_DATA column, apply killMS solutions id_ddfacet2 = 'DDKMA' + code id_list.append(id_ddfacet1) syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' ' syscall += gen.generate_syscall_ddfacet( mspattern=myms, imgname=kms_img_prefix, chunkhours=1, ncpu=myNCPU, initdicomodel=ddf_img_prefix + '.DicoModel', hogbom_maxmajoriter=0, hogbom_maxminoriter=1000, mask=mask, ddsols='killms-cohjones') run_command = gen.job_handler(syscall=syscall, jobname=id_ddfacet2, dependency=id_killms, infrastructure=INFRASTRUCTURE, slurm_config=cfg.SLURM_HIGHMEM, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ if INFRASTRUCTURE == 'idia': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) # Get container needed for this script WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.WSCLEAN_PATTERN) # Set names of the run and kill files, open run file for writing submit_file = 'submit_2GC_GC_continue_clean_job.sh' f = open(submit_file,'w') f.write('#!/usr/bin/env bash\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p','rb'),encoding='latin1') targets = project_info['target_list'] SLURM_WSCLEAN_MOD = cfg.SLURM_WSCLEAN SLURM_WSCLEAN_MOD['TIME'] = '18:00:00' PBS_WSCLEAN_MOD = cfg.PBS_WSCLEAN PBS_WSCLEAN_MOD['TIME'] = '18:00:00' # Loop over targets for target in targets: targetname = target[0] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) myms = target[2].rstrip('/') mask0 = sorted(glob.glob(IMAGES+'/*'+filename_targetname+'*.mask.fits')) print('------------------------------------------------------') print(gen.now()+'Target: '+targetname) print(gen.now()+'MS: '+myms) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' print(gen.now()+'Using mask: '+mask) corr_img_prefix = IMAGES+'/img_'+myms+'_pcalmask' # ------------------------------------------------------------------------------ # STEP 1: # Continue wsclean on CORRECTED_DATA column with additional scale id_wsclean = 'WSCM2'+code syscall = 'singularity exec '+WSCLEAN_CONTAINER+' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=corr_img_prefix, continueclean=True, datacol='CORRECTED_DATA', briggs = -1.5, multiscale = True, scales = '0,3,9,18', niter = 400000, bda=False, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean, infrastructure=INFRASTRUCTURE, slurm_config = SLURM_WSCLEAN_MOD, pbs_config = PBS_WSCLEAN_MOD) f.write(run_command+'\n') f.close()
def main(): USE_SINGULARITY = cfg.USE_SINGULARITY gen.preamble() print(gen.col() + '3GC (facet-based corrections) setup') gen.print_spacer() # ------------------------------------------------------------------------------ # # Setup paths, required containers, infrastructure # # ------------------------------------------------------------------------------ OXKAT = cfg.OXKAT DATA = cfg.DATA GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES SCRIPTS = cfg.SCRIPTS TOOLS = cfg.TOOLS gen.setup_dir(GAINTABLES) gen.setup_dir(IMAGES) gen.setup_dir(cfg.LOGS) gen.setup_dir(cfg.SCRIPTS) INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if INFRASTRUCTURE == 'idia': myNCPU = 8 # Dial back the parallelism for IDIA nodes elif INFRASTRUCTURE == 'chpc': myNCPU = 23 # Kind of meaningless as this stuff probably won't ever run on CHPC else: myNCPU = 40 # Assumed NCPU for standalone nodes if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.DDFACET_PATTERN, USE_SINGULARITY) KILLMS_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.KILLMS_PATTERN, USE_SINGULARITY) # Get target information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # ------------------------------------------------------------------------------ # # 3GC peeling recipe definition # # ------------------------------------------------------------------------------ target_steps = [] codes = [] ii = 1 # Loop over targets for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('MS') + 'not found, skipping') else: steps = [] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) # Look for the zoomed FITS mask for this target mask1 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask1.zoom*.fits')) if len(mask1) > 0: mask = mask1[0] else: mask = 'auto' # Look for the DS9 region file that defines the tessel centres for this target region = glob.glob('*' + targetname + '*.reg') if len(region) == 0: gen.print_spacer() print(gen.col() + 'Please provide a region file of the form:') print(gen.col() + ' *' + targetname + '*.reg') print(gen.col() + 'for this field.') gen.print_spacer() sys.exit() else: region = region[0] gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('Measurement Set') + myms) print(gen.col('Code') + code) print(gen.col('Mask') + mask) print(gen.col('Region') + region) # Image prefixes ddf_img_prefix = IMAGES + '/img_' + myms + '_DDFpcal' kms_img_prefix = IMAGES + '/img_' + myms + '_DDFkMS' # Target-specific kill file kill_file = SCRIPTS + '/kill_3GC_facet_jobs_' + filename_targetname + '.sh' step = {} step['step'] = 0 step[ 'comment'] = 'Run DDFacet, masked deconvolution of CORRECTED_DATA column of ' + myms step['dependency'] = None step['id'] = 'DDCMA' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_ddfacet(mspattern=myms, imgname=ddf_img_prefix, ncpu=myNCPU, mask=mask, sparsification='50,20,5,2') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 1 step[ 'comment'] = 'Convert the DS9 region into a numpy file that killMS will recognise' step['dependency'] = 0 step['id'] = 'RG2NP' + code syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += 'python3 ' + TOOLS + '/reg2npy.py ' + region step['syscall'] = syscall steps.append(step) step = {} step['step'] = 2 step['comment'] = 'Run killMS' step['dependency'] = 1 step['id'] = 'KILMS' + code syscall = CONTAINER_RUNNER + KILLMS_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_killms(myms=myms, baseimg=ddf_img_prefix, ncpu=myNCPU, outsols='killms-cohjones', nodesfile=region + '.npy') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 3 step[ 'comment'] = 'Run DDFacet on CORRECTED_DATA of ' + myms + ', applying killMS solutions' step['dependency'] = 2 step['id'] = 'DDKMA' + code syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_ddfacet( mspattern=myms, imgname=kms_img_prefix, chunkhours=1, ncpu=myNCPU, initdicomodel=ddf_img_prefix + '.DicoModel', hogbom_maxmajoriter=0, hogbom_maxminoriter=1000, mask=mask, ddsols='killms-cohjones') step['syscall'] = syscall steps.append(step) target_steps.append((steps, kill_file, targetname)) # ------------------------------------------------------------------------------ # # Write the run file and kill file based on the recipe # # ------------------------------------------------------------------------------ submit_file = 'submit_3GC_facet_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + cfg.BINDPATH + '\n') for content in target_steps: steps = content[0] kill_file = content[1] targetname = content[2] id_list = [] f.write('\n#---------------------------------------\n') f.write('# ' + targetname) f.write('\n#---------------------------------------\n') for step in steps: step_id = step['id'] id_list.append(step_id) if step['dependency'] is not None: dependency = steps[step['dependency']]['id'] else: dependency = None syscall = step['syscall'] if 'slurm_config' in step.keys(): slurm_config = step['slurm_config'] else: slurm_config = cfg.SLURM_DEFAULTS if 'pbs_config' in step.keys(): pbs_config = step['pbs_config'] else: pbs_config = cfg.PBS_DEFAULTS comment = step['comment'] run_command = gen.job_handler(syscall=syscall, jobname=step_id, infrastructure=INFRASTRUCTURE, dependency=dependency, slurm_config=slurm_config, pbs_config=pbs_config) f.write('\n# ' + comment + '\n') f.write(run_command) if INFRASTRUCTURE != 'node': f.write('\n# Generate kill script for ' + targetname + '\n') if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file) gen.print_spacer() print(gen.col('Run file') + submit_file) gen.print_spacer()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD BIND = cfg.BIND OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS BINDPATH = '$PWD,' + CWD + ',' + BIND gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) gen.setup_dir(GAINTABLES) # Get containers needed for this script CUBICAL_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CUBICAL_PATTERN) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Set names of the run and kill files, open run file for writing submit_file = 'submit_3GC_peel_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + BINDPATH + '\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # Loop over targets codes = [] ii = 1 for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): print('------------------------------------------------------') print(gen.now() + myms + ' not found, skipping ' + targetname) else: filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) mask0 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask1.fits')) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' print('------------------------------------------------------') print(gen.now() + 'Target: ' + targetname) print(gen.now() + 'MS: ' + myms) print(gen.now() + 'Using mask: ' + mask) f.write('\n# ' + targetname + '\n') kill_file = SCRIPTS + '/kill_3GC_peel_jobs_' + filename_targetname + '.sh' prepeel_img_prefix = IMAGES + '/img_' + myms + '_prepeel' dir1_img_prefix = prepeel_img_prefix + '-' + cfg.CAL_3GC_PEEL_REGION.split( '/')[-1].split('.')[0] # Initialise a list to hold all the job IDs id_list = [] # ------------------------------------------------------------------------------ # STEP 1: # Masked wsclean on CORRECTED_DATA column with high (frequency) resolution id_wsclean = 'WSDMA' + code id_list.append(id_wsclean) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean( mslist=[myms], imgname=prepeel_img_prefix, datacol='CORRECTED_DATA', briggs=-0.6, chanout=cfg.CAL_3GC_PEEL_NCHAN, bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean, infrastructure=INFRASTRUCTURE, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 2: # Split model images id_imsplit = 'IMSPL' + code id_list.append(id_imsplit) syscall = 'singularity exec ' + CUBICAL_CONTAINER + ' ' syscall += 'python ' + OXKAT + '/3GC_split_model_images.py ' syscall += '--region ' + cfg.CAL_3GC_PEEL_REGION + ' ' syscall += '--prefix ' + prepeel_img_prefix + ' ' run_command = gen.job_handler(syscall=syscall, jobname=id_imsplit, infrastructure=INFRASTRUCTURE, dependency=id_wsclean, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 3: # Predict DIR1 model into MODEL_DATA id_predict1 = 'WS1PR' + code id_list.append(id_predict1) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict( msname=myms, imgbase=dir1_img_prefix, chanout=cfg.CAL_3GC_PEEL_NCHAN) run_command = gen.job_handler(syscall=syscall, jobname=id_predict1, infrastructure=INFRASTRUCTURE, dependency=id_imsplit, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 4: # Add extra data column id_addcol = 'ADCOL' + code id_list.append(id_addcol) syscall = 'singularity exec ' + CUBICAL_CONTAINER + ' ' syscall += 'python ' + TOOLS + '/add_MS_column.py ' syscall += '--colname ' + cfg.CAL_3GC_PEEL_DIR1COLNAME + ' ' syscall += myms run_command = gen.job_handler(syscall=syscall, jobname=id_addcol, infrastructure=INFRASTRUCTURE, dependency=id_predict1) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 5: # Copy MODEL_DATA to new column id_copycol = 'CPCOL' + code id_list.append(id_copycol) syscall = 'singularity exec ' + CUBICAL_CONTAINER + ' ' syscall += 'python ' + TOOLS + '/copy_MS_column.py ' syscall += '--fromcol MODEL_DATA ' syscall += '--tocol ' + cfg.CAL_3GC_PEEL_DIR1COLNAME + ' ' syscall += myms run_command = gen.job_handler(syscall=syscall, jobname=id_copycol, infrastructure=INFRASTRUCTURE, dependency=id_addcol) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 6: # Predict full sky model into MODEL_DATA id_predict2 = 'WS2PR' + code id_list.append(id_predict2) syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict( msname=myms, imgbase=prepeel_img_prefix, chanout=cfg.CAL_3GC_PEEL_NCHAN) run_command = gen.job_handler(syscall=syscall, jobname=id_predict2, infrastructure=INFRASTRUCTURE, dependency=id_copycol, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 7: # Run CubiCal id_peel = 'CL3GC' + code id_list.append(id_peel) syscall = 'singularity exec ' + CUBICAL_CONTAINER + ' ' syscall += gen.generate_syscall_cubical( parset=cfg.CAL_3GC_PEEL_PARSET, myms=myms) run_command = gen.job_handler(syscall=syscall, jobname=id_peel, infrastructure=INFRASTRUCTURE, dependency=id_predict2, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ if INFRASTRUCTURE == 'idia': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) # Get container needed for this script WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH,cfg.WSCLEAN_PATTERN) # Bump up the walltimes SLURM_WSCLEAN_MOD = cfg.SLURM_WSCLEAN SLURM_WSCLEAN_MOD['TIME'] = '36:00:00' PBS_WSCLEAN_MOD = cfg.PBS_WSCLEAN PBS_WSCLEAN_MOD['WALLTIME'] = '36:00:00' # Set names of the run file, open for writing submit_file = 'submit_2GC_make_cube.sh' f = open(submit_file,'w') f.write('#!/usr/bin/env bash\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p','rb'),encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # Loop over targets codes = [] ii = 1 for tt in range(0,len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) mask0 = sorted(glob.glob(IMAGES+'/*'+filename_targetname+'*.mask.fits')) print('------------------------------------------------------') print(gen.now()+'Target: '+targetname) print(gen.now()+'MS: '+myms) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' print(gen.now()+'Using mask: '+mask) cube_prefix = IMAGES+'/img_'+myms+'_subbands' # ------------------------------------------------------------------------------ # STEP 1: # Continue wsclean on CORRECTED_DATA column with additional scale id_wsclean = 'WCUBE'+code syscall = 'singularity exec '+WSCLEAN_CONTAINER+' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=cube_prefix, datacol='CORRECTED_DATA', sourcelist = False, chanout = 16, briggs = -1.5, multiscale = True, scales = '0,3,9', minuvl = 164, tapergaussian = 8, joinchannels = False, fitspectralpol = 0, niter = 500000, bda = True, mask = mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean, infrastructure=INFRASTRUCTURE, slurm_config = SLURM_WSCLEAN_MOD, pbs_config = PBS_WSCLEAN_MOD) f.write(run_command+'\n') f.close()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD BIND = cfg.BIND OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS BINDPATH = '$PWD,' + CWD + ',' + BIND gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) gen.setup_dir(GAINTABLES) # Enable running without containers if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' # Get containers needed for this script CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN) DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.DDFACET_PATTERN) MAKEMASK_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.MAKEMASK_PATTERN) RAGAVI_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.RAGAVI_PATTERN) TRICOLOUR_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.TRICOLOUR_PATTERN) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Set names of the run and kill files, open run file for writing submit_file = 'submit_2GC_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + BINDPATH + '\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # Loop over targets codes = [] ii = 1 for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): print('------------------------------------------------------') print(gen.now() + myms + ' not found, skipping ' + targetname) else: filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) mask0 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask0.fits')) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' print('------------------------------------------------------') print(gen.now() + 'Target: ' + targetname) print(gen.now() + 'MS: ' + myms) print(gen.now() + 'Using mask: ' + mask) f.write('\n# ' + targetname + '\n') kill_file = SCRIPTS + '/kill_2GC_jobs_' + filename_targetname + '.sh' data_img_prefix = IMAGES + '/img_' + myms + '_datamask' corr_img_prefix = IMAGES + '/img_' + myms + '_pcalmask' # Initialise a list to hold all the job IDs id_list = [] # ------------------------------------------------------------------------------ # STEP 1: # Masked wsclean on DATA column id_wsclean1 = 'WSDMA' + code id_list.append(id_wsclean1) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=data_img_prefix, datacol='DATA', bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean1, infrastructure=INFRASTRUCTURE, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 2: # Predict MODEL_DATA id_predict1 = 'WSDPR' + code id_list.append(id_predict1) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict(msname=myms, imgbase=data_img_prefix) run_command = gen.job_handler(syscall=syscall, jobname=id_predict1, infrastructure=INFRASTRUCTURE, dependency=id_wsclean1, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 3: # Self-calibrate phases then amplitudes id_selfcal = 'CL2GC' + code id_list.append(id_selfcal) casalog = LOGS + '/casa_2GC_' + id_selfcal + '.log' syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' syscall += gen.generate_syscall_casa( casascript=OXKAT + '/2GC_casa_selfcal_target_amp_phases.py', casalogfile=casalog, extra_args='mslist=' + myms) run_command = gen.job_handler(syscall=syscall, jobname=id_selfcal, infrastructure=INFRASTRUCTURE, dependency=id_predict1) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 4: # Make gain table plots id_gainplots = 'PLTAB' + code id_list.append(id_gainplots) syscall = CONTAINER_RUNNER + RAGAVI_CONTAINER + ' ' syscall += 'python3 ' + OXKAT + '/PLOT_gaintables.py cal_2GC_*' + myms + '*' run_command = gen.job_handler(syscall=syscall, jobname=id_gainplots, infrastructure=INFRASTRUCTURE, dependency=id_selfcal) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 5: # Masked wsclean on CORRECTED_DATA column id_wsclean2 = 'WSCMA' + code id_list.append(id_wsclean2) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=corr_img_prefix, datacol='CORRECTED_DATA', bda=True, mask=mask) run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean2, infrastructure=INFRASTRUCTURE, dependency=id_selfcal, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 6: # Make a FITS mask syscall = CONTAINER_RUNNER + MAKEMASK_CONTAINER + ' ' syscall += gen.generate_syscall_makemask( restoredimage=corr_img_prefix + '-MFS-image.fits', outfile=corr_img_prefix + '-MFS-image.mask1.fits', thresh=5.5, zoompix=cfg.DDF_NPIX)[0] id_makemask = 'MASK1' + code id_list.append(id_makemask) run_command = gen.job_handler(syscall=syscall, jobname=id_makemask, infrastructure=INFRASTRUCTURE, dependency=id_wsclean2) f.write(run_command) # ------------------------------------------------------------------------------ # STEP 7: # Predict MODEL_DATA id_predict2 = 'WSCPR' + code id_list.append(id_predict2) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_predict(msname=myms, imgbase=corr_img_prefix) run_command = gen.job_handler(syscall=syscall, jobname=id_predict2, infrastructure=INFRASTRUCTURE, dependency=id_wsclean2, slurm_config=cfg.SLURM_WSCLEAN, pbs_config=cfg.PBS_WSCLEAN) f.write(run_command) # ------------------------------------------------------------------------------ if INFRASTRUCTURE == 'idia': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file)
def main(): USE_SINGULARITY = cfg.USE_SINGULARITY gen.preamble() print(gen.now() + 'Imaging secondary calibrators') # ------------------------------------------------------------------------------ # # Setup paths, required containers, infrastructure # # ------------------------------------------------------------------------------ OXKAT = cfg.OXKAT IMAGES = cfg.IMAGES SCRIPTS = cfg.SCRIPTS gen.setup_dir(IMAGES) gen.setup_dir(cfg.LOGS) gen.setup_dir(cfg.SCRIPTS) INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN, USE_SINGULARITY) # Get secondary calibrator information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') myms = project_info['master_ms'] pcal_names = project_info['secondary_names'] pcal_ids = project_info['secondary_ids'] # ------------------------------------------------------------------------------ # # Image calibrators -- recipe definition # # ------------------------------------------------------------------------------ cal_steps = [] codes = [] ii = 1 # Loop over secondaries for i in range(0, len(pcal_ids)): field = pcal_ids[i] calname = pcal_names[i] steps = [] filename_calname = gen.scrub_target_name(calname) code = gen.get_target_code(calname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) gen.print_spacer() print(gen.now() + 'Secondary | ' + calname) print(gen.now() + 'Code | ' + code) # Image prefix img_prefix = IMAGES + '/img_' + myms + '_' + filename_calname + '_corrblind' step = {} step['step'] = i step[ 'comment'] = 'Run wsclean, blind deconvolution of the CORRECTED_DATA for ' + calname if i == 0: step['dependency'] = None else: step['dependency'] = i - 1 step['id'] = 'WSCMA' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=img_prefix, field=field, datacol='CORRECTED_DATA', imsize=8192, niter=40000, gain=0.2, mgain=0.9, bda=True, mask='none') step['syscall'] = syscall steps.append(step) # ------------------------------------------------------------------------------ # # Write the run file and kill file based on the recipe # # ------------------------------------------------------------------------------ submit_file = 'submit_image_secondary_jobs.sh' kill_file = cfg.SCRIPTS + '/kill_image_secondary_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + cfg.BINDPATH + '\n') id_list = [] for step in steps: step_id = step['id'] id_list.append(step_id) if step['dependency'] is not None: dependency = steps[step['dependency']]['id'] else: dependency = None syscall = step['syscall'] if 'slurm_config' in step.keys(): slurm_config = step['slurm_config'] else: slurm_config = cfg.SLURM_DEFAULTS if 'pbs_config' in step.keys(): pbs_config = step['pbs_config'] else: pbs_config = cfg.PBS_DEFAULTS comment = step['comment'] run_command = gen.job_handler(syscall=syscall, jobname=step_id, infrastructure=INFRASTRUCTURE, dependency=dependency, slurm_config=slurm_config, pbs_config=pbs_config) f.write('\n# ' + comment + '\n') f.write(run_command) if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': kill = '\necho "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = '\necho "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file) gen.print_spacer() print(gen.now() + 'Created ' + submit_file) gen.print_spacer()
def main(): # ------------------------------------------------------------------------------ # Setup INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) # Get paths from config and setup folders CWD = cfg.CWD OXKAT = cfg.OXKAT PARSETS = cfg.PARSETS TOOLS = cfg.TOOLS GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES LOGS = cfg.LOGS SCRIPTS = cfg.SCRIPTS gen.setup_dir(LOGS) gen.setup_dir(SCRIPTS) gen.setup_dir(IMAGES) # Get container needed for this script WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN) # Bump up the walltimes SLURM_WSCLEAN_MOD = cfg.SLURM_WSCLEAN SLURM_WSCLEAN_MOD['TIME'] = '18:00:00' PBS_WSCLEAN_MOD = cfg.PBS_WSCLEAN PBS_WSCLEAN_MOD['WALLTIME'] = '18:00:00' # Set names of the run file, open for writing submit_file = 'submit_2GC_continue_clean_no_maskjobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') # Get target info from project_info.p project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # Loop over targets codes = [] ii = 1 for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) print('------------------------------------------------------') print(gen.now() + 'Target: ' + targetname) print(gen.now() + 'MS: ' + myms) corr_img_prefix = IMAGES + '/img_' + myms + '_pcalmask' # ------------------------------------------------------------------------------ # STEP 1: # Continue wsclean on CORRECTED_DATA column with additional scale id_wsclean = 'WSCN3' + code syscall = 'singularity exec ' + WSCLEAN_CONTAINER + ' ' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=corr_img_prefix, continueclean=True, datacol='CORRECTED_DATA', briggs=-1.5, multiscale=True, scales='0,3,9,18', niter=300000, bda=False, mask='none') run_command = gen.job_handler(syscall=syscall, jobname=id_wsclean, infrastructure=INFRASTRUCTURE, slurm_config=SLURM_WSCLEAN_MOD, pbs_config=PBS_WSCLEAN_MOD) f.write(run_command + '\n') f.close()
def main(): USE_SINGULARITY = cfg.USE_SINGULARITY gen.preamble() print(gen.col() + 'FLAG (target flagging & initial mask-making) setup') gen.print_spacer() # ------------------------------------------------------------------------------ # # Setup paths, required containers, infrastructure # # ------------------------------------------------------------------------------ OXKAT = cfg.OXKAT DATA = cfg.DATA IMAGES = cfg.IMAGES SCRIPTS = cfg.SCRIPTS TOOLS = cfg.TOOLS gen.setup_dir(IMAGES) gen.setup_dir(cfg.LOGS) gen.setup_dir(cfg.SCRIPTS) INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN, USE_SINGULARITY) CUBICAL_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CUBICAL_PATTERN, USE_SINGULARITY) MAKEMASK_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.MAKEMASK_PATTERN, USE_SINGULARITY) TRICOLOUR_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.TRICOLOUR_PATTERN, USE_SINGULARITY) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN, USE_SINGULARITY) # Get target information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # ------------------------------------------------------------------------------ # # FLAG recipe definition # # ------------------------------------------------------------------------------ target_steps = [] codes = [] ii = 1 # Loop over targets for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('MS') + 'not found, skipping') else: steps = [] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) # Image prefix img_prefix = IMAGES + '/img_' + myms + '_datablind' # Target-specific kill file kill_file = SCRIPTS + '/kill_flag_jobs_' + filename_targetname + '.sh' gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('Measurement Set') + myms) print(gen.col('Code') + code) step = {} step['step'] = 0 step['comment'] = 'Run Tricolour on ' + myms step['dependency'] = None step['id'] = 'TRIC0' + code step['slurm_config'] = cfg.SLURM_TRICOLOUR step['pbs_config'] = cfg.PBS_TRICOLOUR syscall = CONTAINER_RUNNER + TRICOLOUR_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_tricolour( myms=myms, config=DATA + '/tricolour/target_flagging_1_narrow.yaml', datacol='DATA', fields='0', strategy='polarisation') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 1 step['comment'] = 'Blind wsclean on DATA column of ' + myms step['dependency'] = 0 step['id'] = 'WSDBL' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN absmem = gen.absmem_helper(step, INFRASTRUCTURE, cfg.WSC_ABSMEM) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=img_prefix, datacol='DATA', bda=True, automask=False, autothreshold=False, localrms=False, mask=False, absmem=absmem) step['syscall'] = syscall steps.append(step) step = {} step['step'] = 2 step['comment'] = 'Make initial cleaning mask for ' + targetname step['dependency'] = 1 step['id'] = 'MASK0' + code syscall = CONTAINER_RUNNER + MAKEMASK_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_makemask( restoredimage=img_prefix + '-MFS-image.fits', outfile=img_prefix + '-MFS-image.mask0.fits', zoompix='')[0] step['syscall'] = syscall steps.append(step) step = {} step['step'] = 3 step['comment'] = 'Backup flag table for ' + myms step['dependency'] = 1 step['id'] = 'SAVFG' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += 'casa -c ' + OXKAT + '/FLAG_casa_backup_flag_table.py --nologger --log2term --nogui ' syscall += 'versionname=tricolour1 mslist=' + myms step['syscall'] = syscall steps.append(step) # step = {} # step['step'] = 4 # step['comment'] = 'Add CORRECTED_DATA column to '+myms # step['dependency'] = 3 # step['id'] = 'ADCOR'+code # syscall = CONTAINER_RUNNER+CUBICAL_CONTAINER+' ' if USE_SINGULARITY else '' # syscall += 'python '+TOOLS+'/add_MS_column.py ' # syscall += '--colname CORRECTED_DATA ' # syscall += myms # step['syscall'] = syscall # steps.append(step) # step = {} # step['step'] = 5 # step['comment'] = 'Copy DATA to CORRECTED_DATA for '+myms # step['dependency'] = 4 # step['id'] = 'CPDAT'+code # syscall = CONTAINER_RUNNER+CUBICAL_CONTAINER+' ' if USE_SINGULARITY else '' # syscall += 'python '+TOOLS+'/copy_MS_column.py ' # syscall += '--fromcol DATA ' # syscall += '--tocol CORRECTED_DATA ' # syscall += myms # step['syscall'] = syscall # steps.append(step) target_steps.append((steps, kill_file, targetname)) # ------------------------------------------------------------------------------ # # Write the run file and kill file based on the recipe # # ------------------------------------------------------------------------------ submit_file = 'submit_flag_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + cfg.BINDPATH + '\n') for content in target_steps: steps = content[0] kill_file = content[1] targetname = content[2] id_list = [] f.write('\n#---------------------------------------\n') f.write('# ' + targetname) f.write('\n#---------------------------------------\n') for step in steps: step_id = step['id'] id_list.append(step_id) if step['dependency'] is not None: dependency = steps[step['dependency']]['id'] else: dependency = None syscall = step['syscall'] if 'slurm_config' in step.keys(): slurm_config = step['slurm_config'] else: slurm_config = cfg.SLURM_DEFAULTS if 'pbs_config' in step.keys(): pbs_config = step['pbs_config'] else: pbs_config = cfg.PBS_DEFAULTS comment = step['comment'] run_command = gen.job_handler(syscall=syscall, jobname=step_id, infrastructure=INFRASTRUCTURE, dependency=dependency, slurm_config=slurm_config, pbs_config=pbs_config) f.write('\n# ' + comment + '\n') f.write(run_command) if INFRASTRUCTURE != 'node': f.write('\n# Generate kill script for ' + targetname + '\n') if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file) gen.print_spacer() print(gen.col('Run file') + submit_file) gen.print_spacer()
def main(): USE_SINGULARITY = cfg.USE_SINGULARITY gen.preamble() print(gen.col() + '2GC (direction independent selfcal) setup') gen.print_spacer() # ------------------------------------------------------------------------------ # # Setup paths, required containers, infrastructure # # ------------------------------------------------------------------------------ OXKAT = cfg.OXKAT DATA = cfg.DATA GAINTABLES = cfg.GAINTABLES IMAGES = cfg.IMAGES SCRIPTS = cfg.SCRIPTS gen.setup_dir(GAINTABLES) gen.setup_dir(IMAGES) gen.setup_dir(cfg.LOGS) gen.setup_dir(cfg.SCRIPTS) INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN, USE_SINGULARITY) MAKEMASK_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.MAKEMASK_PATTERN, USE_SINGULARITY) RAGAVI_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.RAGAVI_PATTERN, USE_SINGULARITY) WSCLEAN_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.WSCLEAN_PATTERN, USE_SINGULARITY) # Get target information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # ------------------------------------------------------------------------------ # # 2GC recipe definition # # ------------------------------------------------------------------------------ target_steps = [] codes = [] ii = 1 # Loop over targets for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('MS') + 'not found, skipping') else: steps = [] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) # Look for the FITS mask for this target mask0 = sorted( glob.glob(IMAGES + '/*' + filename_targetname + '*.mask0.fits')) if len(mask0) > 0: mask = mask0[0] else: mask = 'auto' gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('Measurement Set') + myms) print(gen.col('Code') + code) print(gen.col('Mask') + mask) # Image prefixes data_img_prefix = IMAGES + '/img_' + myms + '_datamask' corr_img_prefix = IMAGES + '/img_' + myms + '_pcalmask' # Target-specific kill file kill_file = SCRIPTS + '/kill_2GC_jobs_' + filename_targetname + '.sh' step = {} step['step'] = 0 step[ 'comment'] = 'Run wsclean, masked deconvolution of the DATA column of ' + myms step['dependency'] = None step['id'] = 'WSDMA' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN absmem = gen.absmem_helper(step, INFRASTRUCTURE, cfg.WSC_ABSMEM) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=data_img_prefix, datacol='DATA', bda=True, automask=False, autothreshold=False, localrms=False, mask=mask, absmem=absmem) step['syscall'] = syscall steps.append(step) step = {} step['step'] = 1 step[ 'comment'] = 'Predict model visibilities from imaging of the DATA column' step['dependency'] = 0 step['id'] = 'WSDPR' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN absmem = gen.absmem_helper(step, INFRASTRUCTURE, cfg.WSC_ABSMEM) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_predict(msname=myms, imgbase=data_img_prefix, absmem=absmem) step['syscall'] = syscall steps.append(step) step = {} step['step'] = 2 step['comment'] = 'Run CASA self-calibration script' step['dependency'] = 1 step['id'] = 'CL2GC' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_casa( casascript=OXKAT + '/2GC_casa_selfcal_target_amp_phases.py', extra_args='mslist=' + myms) step['syscall'] = syscall steps.append(step) step = {} step['step'] = 3 step['comment'] = 'Plot the self-calibration gain solutions' step['dependency'] = 2 step['id'] = 'PLTAB' + code syscall = CONTAINER_RUNNER + RAGAVI_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += 'python3 ' + OXKAT + '/PLOT_gaintables.py cal_2GC_*' + myms + '*' step['syscall'] = syscall steps.append(step) step = {} step['step'] = 4 step[ 'comment'] = 'Run wsclean, masked deconvolution of the CORRECTED_DATA column of ' + myms step['dependency'] = 2 step['id'] = 'WSCMA' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN absmem = gen.absmem_helper(step, INFRASTRUCTURE, cfg.WSC_ABSMEM) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_wsclean(mslist=[myms], imgname=corr_img_prefix, datacol='CORRECTED_DATA', bda=True, automask=False, autothreshold=False, localrms=False, mask=mask, absmem=absmem) step['syscall'] = syscall steps.append(step) step = {} step['step'] = 5 step[ 'comment'] = 'Refine the cleaning mask for ' + targetname + ', crop for use with DDFacet' step['dependency'] = 4 step['id'] = 'MASK1' + code syscall = CONTAINER_RUNNER + MAKEMASK_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_makemask( restoredimage=corr_img_prefix + '-MFS-image.fits', outfile=corr_img_prefix + '-MFS-image.mask1.fits', thresh=5.5, zoompix=cfg.DDF_NPIX)[0] step['syscall'] = syscall steps.append(step) step = {} step['step'] = 6 step[ 'comment'] = 'Predict model visibilities from imaging of the CORRECTED_DATA column' step['dependency'] = 4 step['id'] = 'WSCPR' + code step['slurm_config'] = cfg.SLURM_WSCLEAN step['pbs_config'] = cfg.PBS_WSCLEAN absmem = gen.absmem_helper(step, INFRASTRUCTURE, cfg.WSC_ABSMEM) syscall = CONTAINER_RUNNER + WSCLEAN_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_predict(msname=myms, imgbase=corr_img_prefix, absmem=absmem) step['syscall'] = syscall steps.append(step) target_steps.append((steps, kill_file, targetname)) # ------------------------------------------------------------------------------ # # Write the run file and kill file based on the recipe # # ------------------------------------------------------------------------------ submit_file = 'submit_2GC_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + cfg.BINDPATH + '\n') for content in target_steps: steps = content[0] kill_file = content[1] targetname = content[2] id_list = [] f.write('\n#---------------------------------------\n') f.write('# ' + targetname) f.write('\n#---------------------------------------\n') for step in steps: step_id = step['id'] id_list.append(step_id) if step['dependency'] is not None: dependency = steps[step['dependency']]['id'] else: dependency = None syscall = step['syscall'] if 'slurm_config' in step.keys(): slurm_config = step['slurm_config'] else: slurm_config = cfg.SLURM_DEFAULTS if 'pbs_config' in step.keys(): pbs_config = step['pbs_config'] else: pbs_config = cfg.PBS_DEFAULTS comment = step['comment'] run_command = gen.job_handler(syscall=syscall, jobname=step_id, infrastructure=INFRASTRUCTURE, dependency=dependency, slurm_config=slurm_config, pbs_config=pbs_config) f.write('\n# ' + comment + '\n') f.write(run_command) if INFRASTRUCTURE != 'node': f.write('\n# Generate kill script for ' + targetname + '\n') if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file) gen.print_spacer() print(gen.col('Run file') + submit_file) gen.print_spacer()
def main(): USE_SINGULARITY = cfg.USE_SINGULARITY gen.preamble() print(gen.col() + 'FLAG (target flagging) setup') gen.print_spacer() # ------------------------------------------------------------------------------ # # Setup paths, required containers, infrastructure # # ------------------------------------------------------------------------------ OXKAT = cfg.OXKAT DATA = cfg.DATA SCRIPTS = cfg.SCRIPTS TOOLS = cfg.TOOLS gen.setup_dir(cfg.LOGS) gen.setup_dir(cfg.SCRIPTS) INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv) if CONTAINER_PATH is not None: CONTAINER_RUNNER = 'singularity exec ' else: CONTAINER_RUNNER = '' CASA_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.CASA_PATTERN, USE_SINGULARITY) TRICOLOUR_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.TRICOLOUR_PATTERN, USE_SINGULARITY) # Get target information from project pickle project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1') target_ids = project_info['target_ids'] target_names = project_info['target_names'] target_ms = project_info['target_ms'] # ------------------------------------------------------------------------------ # # FLAG recipe definition # # ------------------------------------------------------------------------------ target_steps = [] codes = [] ii = 1 # Loop over targets for tt in range(0, len(target_ids)): targetname = target_names[tt] myms = target_ms[tt] if not o.isdir(myms): gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('MS') + 'not found, skipping') else: steps = [] filename_targetname = gen.scrub_target_name(targetname) code = gen.get_target_code(targetname) if code in codes: code += '_' + str(ii) ii += 1 codes.append(code) # Target-specific kill file kill_file = SCRIPTS + '/kill_flag_jobs_' + filename_targetname + '.sh' gen.print_spacer() print(gen.col('Target') + targetname) print(gen.col('Measurement Set') + myms) print(gen.col('Code') + code) step = {} step['step'] = 0 step['comment'] = 'Run Tricolour on ' + myms step['dependency'] = None step['id'] = 'TRIC0' + code step['slurm_config'] = cfg.SLURM_TRICOLOUR step['pbs_config'] = cfg.PBS_TRICOLOUR syscall = CONTAINER_RUNNER + TRICOLOUR_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += gen.generate_syscall_tricolour( myms=myms, config=DATA + '/tricolour/target_flagging_1_narrow.yaml', datacol='DATA', fields='0', strategy='polarisation') step['syscall'] = syscall steps.append(step) step = {} step['step'] = 1 step['comment'] = 'Backup flag table for ' + myms step['dependency'] = 0 step['id'] = 'SAVFG' + code syscall = CONTAINER_RUNNER + CASA_CONTAINER + ' ' if USE_SINGULARITY else '' syscall += 'casa -c ' + OXKAT + '/FLAG_casa_backup_flag_table.py --nologger --log2term --nogui ' syscall += 'versionname=tricolour1 mslist=' + myms step['syscall'] = syscall steps.append(step) target_steps.append((steps, kill_file, targetname)) # ------------------------------------------------------------------------------ # # Write the run file and kill file based on the recipe # # ------------------------------------------------------------------------------ submit_file = 'submit_flag_jobs.sh' f = open(submit_file, 'w') f.write('#!/usr/bin/env bash\n') f.write('export SINGULARITY_BINDPATH=' + cfg.BINDPATH + '\n') for content in target_steps: steps = content[0] kill_file = content[1] targetname = content[2] id_list = [] f.write('\n#---------------------------------------\n') f.write('# ' + targetname) f.write('\n#---------------------------------------\n') for step in steps: step_id = step['id'] id_list.append(step_id) if step['dependency'] is not None: dependency = steps[step['dependency']]['id'] else: dependency = None syscall = step['syscall'] if 'slurm_config' in step.keys(): slurm_config = step['slurm_config'] else: slurm_config = cfg.SLURM_DEFAULTS if 'pbs_config' in step.keys(): pbs_config = step['pbs_config'] else: pbs_config = cfg.PBS_DEFAULTS comment = step['comment'] run_command = gen.job_handler(syscall=syscall, jobname=step_id, infrastructure=INFRASTRUCTURE, dependency=dependency, slurm_config=slurm_config, pbs_config=pbs_config) f.write('\n# ' + comment + '\n') f.write(run_command) if INFRASTRUCTURE != 'node': f.write('\n# Generate kill script for ' + targetname + '\n') if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo': kill = 'echo "scancel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) elif INFRASTRUCTURE == 'chpc': kill = 'echo "qdel "$' + '" "$'.join( id_list) + ' > ' + kill_file + '\n' f.write(kill) f.close() gen.make_executable(submit_file) gen.print_spacer() print(gen.col('Run file') + submit_file) gen.print_spacer()