def main():
CWD = gen.CWD
OXKAT = gen.OXKAT
PARSETS = gen.PARSETS
SCRIPTS = gen.SCRIPTS
LOGS = gen.LOGS
TOOLS = gen.TOOLS
DDFACET_CONTAINER = gen.DDFACET_CONTAINER
SOURCEFINDER_CONTAINER = gen.SOURCEFINDER_CONTAINER
KILLMS_CONTAINER = gen.KILLMS_CONTAINER
CLUSTERCAT_CONTAINER = gen.CLUSTERCAT_CONTAINER
XDDFACET_CONTAINER = gen.XDDFACET_CONTAINER
XSOURCEFINDER_CONTAINER = gen.XSOURCEFINDER_CONTAINER
XKILLMS_CONTAINER = gen.XKILLMS_CONTAINER
XCLUSTERCAT_CONTAINER = gen.XCLUSTERCAT_CONTAINER
BEAM = '/users/ianh/Beams/hvfix/meerkat_pb_jones_cube_95channels_$(xy)_$(reim).fits'
submit_file = 'submit_3GC_jobs.sh'
kill_file = 'kill_3GC-beam_jobs.sh'
run_file = 'run_3GC_jobs.sh'
gen.setup_dir(SCRIPTS)
gen.setup_dir(LOGS)
f = open(submit_file,'w')
g = open(run_file,'w')
f.write('#!/usr/bin/env bash\n')
g.write('#!/usr/bin/env bash\n')
with open('project_info.p','rb') as f:
project_info = pickle.load(f,encoding='latin1')
targets = project_info['target_list']
f = open(submit_file,'w')
for target in targets:
MASK = glob.glob('zoom*'+target[0]+'*mask.fits')[0]
print('Using FITS mask: '+MASK)
code = target[0][-3:].replace('-','_').replace('.','p')
myms = target[2].rstrip('/')
mspat = '*'+target[0]+'*.ms'
ddf1_prefix = 'img_'+target[0]+'_DDF_corr_beam'
ddf2_prefix = 'img_'+target[0]+'_DDF_kMS_beam'
solnames = 'killms-cohjones'
# ------------------------------------------------------------------------------
# DDFacet 1
slurmfile = SCRIPTS+'/slurm_ddf_corr_'+code+'.sh'
logfile = LOGS+'/slurm_ddf_corr_'+code+'.log'
syscall = 'singularity exec '+DDFACET_CONTAINER+' '
syscall += gen.generate_syscall_ddfacet(mspattern=mspat,
imgname=ddf1_prefix,
chunkhours=1,
beam=BEAM,
mask=MASK)
syscall += ' ; singularity exec '+DDFACET_CONTAINER+' CleanSHM.py'
gen.write_slurm(opfile=slurmfile,
jobname=code+'_DDF1',
logfile=logfile,
syscall=syscall,
mem='480GB',
partition='HighMem')
syscall = syscall.replace(DDFACET_CONTAINER,XDDFACET_CONTAINER)
g.write(syscall+'\n')
job_id_ddf1 = 'DDF1_'+code
syscall = job_id_ddf1+"=`sbatch "+slurmfile+" | awk '{print $4}'`"
f.write(syscall+'\n')
# ------------------------------------------------------------------------------
# PyBDSF
slurmfile = SCRIPTS+'/slurm_pybdsf_'+code+'.sh'
logfile = LOGS+'/slurm_pybdsf_'+code+'.log'
syscall = 'singularity exec '+SOURCEFINDER_CONTAINER+' '
syscall2,bdsfcat = gen.generate_syscall_pybdsf(ddf1_prefix+'.app.restored.fits',
catalogtype='srl',
catalogformat='fits')
syscall += syscall2
gen.write_slurm(opfile=slurmfile,
jobname=code+'pbdsf',
logfile=logfile,
syscall=syscall)
syscall = syscall.replace(SOURCEFINDER_CONTAINER,XSOURCEFINDER_CONTAINER)
g.write(syscall+'\n')
job_id_bdsf = 'BDSF_'+code
syscall = job_id_bdsf+"=`sbatch -d afterok:${"+job_id_ddf1+"} "+slurmfile+" | awk '{print $4}'`"
f.write(syscall+'\n')
# ------------------------------------------------------------------------------
# Identify tessel centres
# slurmfile = SCRIPTS+'/slurm_dEid_'+code+'.sh'
# logfile = LOGS+'/slurm_dEid_'+code+'.log'
# syscall = 'singularity exec '+SOURCEFINDER_CONTAINER+' '
# syscall += 'python '+TOOLS+'/identify_dE_directions.py'
# gen.write_slurm(opfile=slurmfile,
# jobname=code+'dEid',
# logfile=logfile,
# syscall=syscall)
# job_id_deid = 'DEID_'+code
# syscall = job_id_deid+"=`sbatch -d afterok:${"+job_id_bdsf+"} "+slurmfile+" | awk '{print $4}'`"
# f.write(syscall+'\n')
# ------------------------------------------------------------------------------
# Run ClusterCat
slurmfile = SCRIPTS+'/slurm_cluster_'+code+'.sh'
logfile = LOGS+'/slurm_cluster_'+code+'.log'
syscall = 'singularity exec '+CLUSTERCAT_CONTAINER+' '
syscall1,clusterfile = gen.generate_syscall_clustercat(bdsfcat,ndir=7)
syscall += syscall1
gen.write_slurm(opfile=slurmfile,
jobname=code+'clstr',
logfile=logfile,
syscall=syscall)
syscall = syscall.replace(CLUSTERCAT_CONTAINER,XCLUSTERCAT_CONTAINER)
g.write(syscall+'\n')
job_id_cluster = 'CLSTR_'+code
syscall = job_id_cluster+"=`sbatch -d afterok:${"+job_id_bdsf+"} "+slurmfile+" | awk '{print $4}'`"
f.write(syscall+'\n')
# ------------------------------------------------------------------------------
# Run killMS
slurmfile = SCRIPTS+'/slurm_killMS_'+code+'.sh'
logfile = LOGS+'/slurm_killMS_'+code+'.log'
syscall = 'singularity exec '+KILLMS_CONTAINER+' '
syscall += gen.generate_syscall_killms(myms,
baseimg=ddf1_prefix,
outsols=solnames,
nodesfile=clusterfile,
dicomodel=ddf1_prefix+'.DicoModel',
beam=BEAM)
syscall += ' ; singularity exec '+KILLMS_CONTAINER+' CleanSHM.py'
gen.write_slurm(opfile=slurmfile,
jobname=code+'kilMS',
logfile=logfile,
syscall=syscall,
mem='480GB',
partition='HighMem')
syscall = syscall.replace(KILLMS_CONTAINER,XKILLMS_CONTAINER)
g.write(syscall+'\n')
job_id_killms = 'KILLMS_'+code
syscall = job_id_killms+"=`sbatch -d afterok:${"+job_id_cluster+"} "+slurmfile+" | awk '{print $4}'`"
f.write(syscall+'\n')
# ------------------------------------------------------------------------------
# DDFacet 2
slurmfile = SCRIPTS+'/slurm_ddf_kMS_'+code+'.sh'
logfile = LOGS+'/slurm_ddf_kMS_'+code+'.log'
syscall = 'singularity exec '+DDFACET_CONTAINER+' '
syscall += gen.generate_syscall_ddfacet(mspattern=mspat,
imgname=ddf2_prefix,
chunkhours=1,
beam=BEAM,
mask='auto',
masksigma=4.5,
maxmajoriter=1,
ddsols=solnames,
initdicomodel=ddf1_prefix+'.DicoModel')
syscall += ' ; singularity exec '+DDFACET_CONTAINER+' CleanSHM.py'
gen.write_slurm(opfile=slurmfile,
jobname=code+'_DDF2',
logfile=logfile,
syscall=syscall,
mem='480GB',
partition='HighMem')
syscall.replace(DDFACET_CONTAINER,XDDFACET_CONTAINER)
g.write(syscall+'\n')
job_id_ddf2 = 'DDF2_'+code
syscall = job_id_ddf2+"=`sbatch -d afterok:${"+job_id_killms+"} "+slurmfile+" | awk '{print $4}'`"
f.write(syscall+'\n')
# ------------------------------------------------------------------------------
kill = 'echo "scancel "$'+job_id_ddf1+'" "$'+job_id_bdsf+'" "$'+job_id_cluster+'" "$'+job_id_killms+'" "$'+job_id_ddf2+' >> '+kill_file
f.write(kill+'\n')
f.close()
def main():
# ------------------------------------------------------------------------------
# Setup
# Get paths from config and setup folders
CWD = cfg.CWD
OXKAT = cfg.OXKAT
IMAGES = cfg.IMAGES
PARSETS = cfg.PARSETS
TOOLS = cfg.TOOLS
LOGS = cfg.LOGS
SCRIPTS = cfg.SCRIPTS
gen.setup_dir(IMAGES)
gen.setup_dir(LOGS)
gen.setup_dir(SCRIPTS)
# Set infrastructure and container path
if len(sys.argv) == 1:
print('Please specify infrastructure (idia / chpc / node)')
sys.exit()
if sys.argv[1].lower() == 'idia':
infrastructure = 'idia'
CONTAINER_PATH = cfg.IDIA_CONTAINER_PATH
elif sys.argv[1].lower() == 'chpc':
infrastructure = 'chpc'
CONTAINER_PATH = cfg.CHPC_CONTAINER_PATH
elif sys.argv[1].lower() == 'node':
infrastructure = 'node'
CONTAINER_PATH = cfg.NODE_CONTAINER_PATH
# Get container and beam
DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, 'ddfacet-0.4.1')
# BEAM = cfg.BEAM_L
BEAM = ''
# Set names of the run files, open run file for writing
submit_file = 'submit_DDFacet_job.sh'
prefix = 'DDFpcal'
dependency = None
f = open(submit_file, 'w')
f.write('#!/usr/bin/env bash\n')
project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1')
targets = project_info['target_list']
for target in targets:
MASK = glob.glob('zoom*' + target[0] + '*mask.fits')[0]
print(gen.now() + 'Using FITS mask: ' + MASK)
code = target[0][-3:].replace('-', '_').replace('.', 'p')
myms = target[2].rstrip('/')
mspat = '*' + target[0] + '*.ms'
imgname = 'img_' + target[0] + '_' + prefix
# ------------------------------------------------------------------------------
# DDFacet
syscall = 'singularity exec ' + DDFACET_CONTAINER + ' '
syscall += gen.generate_syscall_ddfacet(mspattern=mspat,
imgname=imgname,
chunkhours=1,
beam=BEAM,
mask=MASK)
syscall += ' ; singularity exec ' + DDFACET_CONTAINER + ' CleanSHM.py'
id_ddf = 'DDFact' + code
run_command = gen.job_handler(syscall=syscall,
jobname=id_ddf,
infrastructure=infrastructure,
dependency=dependency,
slurm_config=cfg.SLURM_WSCLEAN,
pbs_config=cfg.PBS_WSCLEAN)
f.write(run_command + '\n')
f.close()
def main():
# ------------------------------------------------------------------------------
# Setup
INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv)
if INFRASTRUCTURE == 'idia':
myNCPU = 8
elif INFRASTRUCTURE == 'chpc':
myNCPU = 23
else:
myNCPU = 40
# Get paths from config and setup folders
CWD = cfg.CWD
OXKAT = cfg.OXKAT
PARSETS = cfg.PARSETS
TOOLS = cfg.TOOLS
GAINTABLES = cfg.GAINTABLES
IMAGES = cfg.IMAGES
LOGS = cfg.LOGS
SCRIPTS = cfg.SCRIPTS
gen.setup_dir(LOGS)
gen.setup_dir(SCRIPTS)
gen.setup_dir(IMAGES)
gen.setup_dir(GAINTABLES)
# Enable running without containers
if CONTAINER_PATH is not None:
CONTAINER_RUNNER = 'singularity exec '
else:
CONTAINER_RUNNER = ''
# Get containers needed for this script
DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.DDFACET_PATTERN)
KILLMS_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.KILLMS_PATTERN)
# Set names of the run and kill files, open run file for writing
submit_file = 'submit_3GC_jobs.sh'
f = open(submit_file, 'w')
f.write('#!/usr/bin/env bash\n')
# Get target info from project_info.p
project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1')
target_ids = project_info['target_ids']
target_names = project_info['target_names']
target_ms = project_info['target_ms']
# Loop over targets
codes = []
ii = 1
for tt in range(0, len(target_ids)):
targetname = target_names[tt]
myms = target_ms[tt]
if not o.isdir(myms):
print('------------------------------------------------------')
print(gen.now() + myms + ' not found, skipping ' + targetname)
else:
filename_targetname = gen.scrub_target_name(targetname)
code = gen.get_target_code(targetname)
if code in codes:
code += '_' + str(ii)
ii += 1
codes.append(code)
mask1 = sorted(
glob.glob(IMAGES + '/*' + filename_targetname +
'*.mask1.zoom*.fits'))
if len(mask1) > 0:
mask = mask1[0]
else:
mask = 'auto'
region = glob.glob('*' + targetname + '*.reg')
if len(region) == 0:
print(gen.now() + 'Please provide a region file')
print(
"Reg file not supplied - will crash or complain on step 2")
region = "noreg.reg"
#import IPython
#IPython.embed()
#sys.exit()
else:
region = region[0]
print('------------------------------------------------------')
print(gen.now() + 'Target: ' + targetname)
print(gen.now() + 'MS: ' + myms)
print(gen.now() + 'Using mask: ' + mask)
print(gen.now() + 'Using region: ' + region)
f.write('\n# ' + targetname + '\n')
kill_file = SCRIPTS + '/kill_3GC_jobs_' + filename_targetname + '.sh'
ddf_img_prefix = IMAGES + '/img_' + myms + '_DDFpcal'
kms_img_prefix = IMAGES + '/img_' + myms + '_DDFkMS'
# Initialise a list to hold all the job IDs
id_list = []
# ------------------------------------------------------------------------------
# STEP 1:
# DDFacet on CORRECTED_DATA column
id_ddfacet1 = 'DDCMA' + code
id_list.append(id_ddfacet1)
syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' '
syscall += gen.generate_syscall_ddfacet(
mspattern=myms,
imgname=ddf_img_prefix,
ncpu=myNCPU,
mask=mask,
sparsification='100,30,10,2')
run_command = gen.job_handler(syscall=syscall,
jobname=id_ddfacet1,
infrastructure=INFRASTRUCTURE,
slurm_config=cfg.SLURM_HIGHMEM,
pbs_config=cfg.PBS_WSCLEAN)
f.write(run_command)
# ------------------------------------------------------------------------------
# STEP 2:
# Run killMS
id_killms = 'KILMS' + code
id_list.append(id_killms)
syscall = CONTAINER_RUNNER + KILLMS_CONTAINER + ' '
syscall += 'python3 ' + TOOLS + '/reg2npy.py ' + region + '\n '
syscall += CONTAINER_RUNNER + KILLMS_CONTAINER + ' '
syscall += gen.generate_syscall_killms(myms=myms,
baseimg=ddf_img_prefix,
ncpu=myNCPU,
outsols='killms-cohjones',
nodesfile=region + '.npy')
run_command = gen.job_handler(syscall=syscall,
jobname=id_killms,
dependency=id_ddfacet1,
infrastructure=INFRASTRUCTURE,
slurm_config=cfg.SLURM_HIGHMEM,
pbs_config=cfg.PBS_WSCLEAN)
f.write(run_command)
# ------------------------------------------------------------------------------
# STEP 3:
# DDFacet on CORRECTED_DATA column, apply killMS solutions
id_ddfacet2 = 'DDKMA' + code
id_list.append(id_ddfacet1)
syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' '
syscall += gen.generate_syscall_ddfacet(
mspattern=myms,
imgname=kms_img_prefix,
chunkhours=1,
ncpu=myNCPU,
initdicomodel=ddf_img_prefix + '.DicoModel',
hogbom_maxmajoriter=0,
hogbom_maxminoriter=1000,
mask=mask,
ddsols='killms-cohjones')
run_command = gen.job_handler(syscall=syscall,
jobname=id_ddfacet2,
dependency=id_killms,
infrastructure=INFRASTRUCTURE,
slurm_config=cfg.SLURM_HIGHMEM,
pbs_config=cfg.PBS_WSCLEAN)
f.write(run_command)
# ------------------------------------------------------------------------------
if INFRASTRUCTURE == 'idia':
kill = 'echo "scancel "$' + '" "$'.join(
id_list) + ' > ' + kill_file + '\n'
f.write(kill)
elif INFRASTRUCTURE == 'chpc':
kill = 'echo "qdel "$' + '" "$'.join(
id_list) + ' > ' + kill_file + '\n'
f.write(kill)
f.close()
def main():
USE_SINGULARITY = cfg.USE_SINGULARITY
gen.preamble()
print(gen.col() + '3GC (facet-based corrections) setup')
gen.print_spacer()
# ------------------------------------------------------------------------------
#
# Setup paths, required containers, infrastructure
#
# ------------------------------------------------------------------------------
OXKAT = cfg.OXKAT
DATA = cfg.DATA
GAINTABLES = cfg.GAINTABLES
IMAGES = cfg.IMAGES
SCRIPTS = cfg.SCRIPTS
TOOLS = cfg.TOOLS
gen.setup_dir(GAINTABLES)
gen.setup_dir(IMAGES)
gen.setup_dir(cfg.LOGS)
gen.setup_dir(cfg.SCRIPTS)
INFRASTRUCTURE, CONTAINER_PATH = gen.set_infrastructure(sys.argv)
if INFRASTRUCTURE == 'idia':
myNCPU = 8 # Dial back the parallelism for IDIA nodes
elif INFRASTRUCTURE == 'chpc':
myNCPU = 23 # Kind of meaningless as this stuff probably won't ever run on CHPC
else:
myNCPU = 40 # Assumed NCPU for standalone nodes
if CONTAINER_PATH is not None:
CONTAINER_RUNNER = 'singularity exec '
else:
CONTAINER_RUNNER = ''
DDFACET_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.DDFACET_PATTERN,
USE_SINGULARITY)
KILLMS_CONTAINER = gen.get_container(CONTAINER_PATH, cfg.KILLMS_PATTERN,
USE_SINGULARITY)
# Get target information from project pickle
project_info = pickle.load(open('project_info.p', 'rb'), encoding='latin1')
target_ids = project_info['target_ids']
target_names = project_info['target_names']
target_ms = project_info['target_ms']
# ------------------------------------------------------------------------------
#
# 3GC peeling recipe definition
#
# ------------------------------------------------------------------------------
target_steps = []
codes = []
ii = 1
# Loop over targets
for tt in range(0, len(target_ids)):
targetname = target_names[tt]
myms = target_ms[tt]
if not o.isdir(myms):
gen.print_spacer()
print(gen.col('Target') + targetname)
print(gen.col('MS') + 'not found, skipping')
else:
steps = []
filename_targetname = gen.scrub_target_name(targetname)
code = gen.get_target_code(targetname)
if code in codes:
code += '_' + str(ii)
ii += 1
codes.append(code)
# Look for the zoomed FITS mask for this target
mask1 = sorted(
glob.glob(IMAGES + '/*' + filename_targetname +
'*.mask1.zoom*.fits'))
if len(mask1) > 0:
mask = mask1[0]
else:
mask = 'auto'
# Look for the DS9 region file that defines the tessel centres for this target
region = glob.glob('*' + targetname + '*.reg')
if len(region) == 0:
gen.print_spacer()
print(gen.col() + 'Please provide a region file of the form:')
print(gen.col() + ' *' + targetname + '*.reg')
print(gen.col() + 'for this field.')
gen.print_spacer()
sys.exit()
else:
region = region[0]
gen.print_spacer()
print(gen.col('Target') + targetname)
print(gen.col('Measurement Set') + myms)
print(gen.col('Code') + code)
print(gen.col('Mask') + mask)
print(gen.col('Region') + region)
# Image prefixes
ddf_img_prefix = IMAGES + '/img_' + myms + '_DDFpcal'
kms_img_prefix = IMAGES + '/img_' + myms + '_DDFkMS'
# Target-specific kill file
kill_file = SCRIPTS + '/kill_3GC_facet_jobs_' + filename_targetname + '.sh'
step = {}
step['step'] = 0
step[
'comment'] = 'Run DDFacet, masked deconvolution of CORRECTED_DATA column of ' + myms
step['dependency'] = None
step['id'] = 'DDCMA' + code
step['slurm_config'] = cfg.SLURM_WSCLEAN
step['pbs_config'] = cfg.PBS_WSCLEAN
syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' ' if USE_SINGULARITY else ''
syscall += gen.generate_syscall_ddfacet(mspattern=myms,
imgname=ddf_img_prefix,
ncpu=myNCPU,
mask=mask,
sparsification='50,20,5,2')
step['syscall'] = syscall
steps.append(step)
step = {}
step['step'] = 1
step[
'comment'] = 'Convert the DS9 region into a numpy file that killMS will recognise'
step['dependency'] = 0
step['id'] = 'RG2NP' + code
syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' ' if USE_SINGULARITY else ''
syscall += 'python3 ' + TOOLS + '/reg2npy.py ' + region
step['syscall'] = syscall
steps.append(step)
step = {}
step['step'] = 2
step['comment'] = 'Run killMS'
step['dependency'] = 1
step['id'] = 'KILMS' + code
syscall = CONTAINER_RUNNER + KILLMS_CONTAINER + ' ' if USE_SINGULARITY else ''
syscall += gen.generate_syscall_killms(myms=myms,
baseimg=ddf_img_prefix,
ncpu=myNCPU,
outsols='killms-cohjones',
nodesfile=region + '.npy')
step['syscall'] = syscall
steps.append(step)
step = {}
step['step'] = 3
step[
'comment'] = 'Run DDFacet on CORRECTED_DATA of ' + myms + ', applying killMS solutions'
step['dependency'] = 2
step['id'] = 'DDKMA' + code
syscall = CONTAINER_RUNNER + DDFACET_CONTAINER + ' ' if USE_SINGULARITY else ''
syscall += gen.generate_syscall_ddfacet(
mspattern=myms,
imgname=kms_img_prefix,
chunkhours=1,
ncpu=myNCPU,
initdicomodel=ddf_img_prefix + '.DicoModel',
hogbom_maxmajoriter=0,
hogbom_maxminoriter=1000,
mask=mask,
ddsols='killms-cohjones')
step['syscall'] = syscall
steps.append(step)
target_steps.append((steps, kill_file, targetname))
# ------------------------------------------------------------------------------
#
# Write the run file and kill file based on the recipe
#
# ------------------------------------------------------------------------------
submit_file = 'submit_3GC_facet_jobs.sh'
f = open(submit_file, 'w')
f.write('#!/usr/bin/env bash\n')
f.write('export SINGULARITY_BINDPATH=' + cfg.BINDPATH + '\n')
for content in target_steps:
steps = content[0]
kill_file = content[1]
targetname = content[2]
id_list = []
f.write('\n#---------------------------------------\n')
f.write('# ' + targetname)
f.write('\n#---------------------------------------\n')
for step in steps:
step_id = step['id']
id_list.append(step_id)
if step['dependency'] is not None:
dependency = steps[step['dependency']]['id']
else:
dependency = None
syscall = step['syscall']
if 'slurm_config' in step.keys():
slurm_config = step['slurm_config']
else:
slurm_config = cfg.SLURM_DEFAULTS
if 'pbs_config' in step.keys():
pbs_config = step['pbs_config']
else:
pbs_config = cfg.PBS_DEFAULTS
comment = step['comment']
run_command = gen.job_handler(syscall=syscall,
jobname=step_id,
infrastructure=INFRASTRUCTURE,
dependency=dependency,
slurm_config=slurm_config,
pbs_config=pbs_config)
f.write('\n# ' + comment + '\n')
f.write(run_command)
if INFRASTRUCTURE != 'node':
f.write('\n# Generate kill script for ' + targetname + '\n')
if INFRASTRUCTURE == 'idia' or INFRASTRUCTURE == 'hippo':
kill = 'echo "scancel "$' + '" "$'.join(
id_list) + ' > ' + kill_file + '\n'
f.write(kill)
elif INFRASTRUCTURE == 'chpc':
kill = 'echo "qdel "$' + '" "$'.join(
id_list) + ' > ' + kill_file + '\n'
f.write(kill)
f.close()
gen.make_executable(submit_file)
gen.print_spacer()
print(gen.col('Run file') + submit_file)
gen.print_spacer()