def main(args, taskvals):
visname = va(taskvals, 'data', 'vis', str)
calcrefant = va(taskvals, 'crosscal', 'calcrefant', bool, default=False)
refant = va(taskvals, 'crosscal', 'refant', str, default='m005')
spw = va(taskvals, 'crosscal', 'spw', str, default='')
tasks = va(taskvals, 'slurm', 'ntasks_per_node', int)
preavg = va(taskvals, 'crosscal', 'chanbin', int, default=1)
include_crosshand = va(taskvals, 'run', 'dopol', bool, default=False)
createmms = va(taskvals, 'crosscal', 'createmms', bool, default=True)
msmd.open(visname)
npol = msmd.ncorrforpol()[0]
if not include_crosshand and npol == 4:
npol = 2
CPUs = npol if tasks * npol <= processMeerKAT.CPUS_PER_NODE_LIMIT else 1 #hard-code for number of polarisations
mvis = do_partition(visname, spw, preavg, CPUs, include_crosshand,
createmms)
mvis = "'{0}'".format(mvis)
vis = "'{0}'".format(visname)
config_parser.overwrite_config(args['config'],
conf_sec='data',
conf_dict={'vis': mvis})
config_parser.overwrite_config(
args['config'],
conf_sec='run',
sec_comment='# Internal variables for pipeline execution',
conf_dict={'orig_vis': vis})
msmd.done()
def main():
# Get the name of the config file
args = config_parser.parse_args()
# Parse config file
taskvals, config = config_parser.parse_config(args['config'])
visname = va(taskvals, 'data', 'vis', str)
fields = bookkeeping.get_field_ids(taskvals['fields'])
calcrefant = va(taskvals, 'crosscal', 'calcrefant', bool, default=False)
# Calculate reference antenna
if calcrefant:
if len(fields.fluxfield.split(',')) > 1:
field = fields.fluxfield.split(',')[0]
else:
field = fields.fluxfield
refant, badants = get_ref_ant(visname, field)
# Overwrite config file with new refant
config_parser.overwrite_config(args['config'],
conf_sec='crosscal',
conf_dict={'refant': refant})
config_parser.overwrite_config(args['config'],
conf_sec='crosscal',
conf_dict={'badants': badants})
def main(args, taskvals):
visname = va(taskvals, 'data', 'vis', str)
calfiles, caldir = bookkeeping.bookkeeping(visname)
fields = bookkeeping.get_field_ids(taskvals['fields'])
spw = va(taskvals, 'crosscal', 'spw', str, default='')
specavg = va(taskvals, 'crosscal', 'width', int, default=1)
timeavg = va(taskvals, 'crosscal', 'timeavg', str, default='8s')
keepmms = va(taskvals, 'crosscal', 'keepmms', bool)
msmd.open(visname)
newvis = split_vis(visname, spw, fields, specavg, timeavg, keepmms)
config_parser.overwrite_config(args['config'],
conf_dict={'vis': "'{0}'".format(newvis)},
conf_sec='data')
config_parser.overwrite_config(
args['config'],
conf_dict={'crosscal_vis': "'{0}'".format(visname)},
conf_sec='run',
sec_comment='# Internal variables for pipeline execution')
msmd.done()
def main(args, taskvals):
visname = va(taskvals, 'data', 'vis', str)
calcrefant = va(taskvals, 'crosscal', 'calcrefant', bool, default=False)
refant = va(taskvals, 'crosscal', 'refant', str, default='m005')
spw = va(taskvals, 'crosscal', 'spw', str, default='')
nspw = va(taskvals, 'crosscal', 'nspw', int, default='')
tasks = va(taskvals, 'slurm', 'ntasks_per_node', int)
preavg = va(taskvals, 'crosscal', 'chanbin', int, default=1)
include_crosshand = va(taskvals, 'run', 'dopol', bool, default=False)
createmms = va(taskvals, 'crosscal', 'createmms', bool, default=True)
# HPC Specific Configuration
known_hpc_path = os.path.dirname(SCRIPT_DIR) + "/known_hpc.cfg"
KNOWN_HPCS, HPC_CONFIG = config_parser.parse_config(known_hpc_path)
HPC_NAME = taskvals["run"]["hpc"]
HPC_NAME = HPC_NAME if HPC_NAME in KNOWN_HPCS.keys() else "unknown"
CPUS_PER_NODE_LIMIT = va(KNOWN_HPCS,
HPC_NAME,
"CPUS_PER_NODE_LIMIT".lower(),
dtype=int)
if nspw > 1:
casalog.setlogfile(
'logs/{SLURM_JOB_NAME}-{SLURM_ARRAY_JOB_ID}_{SLURM_ARRAY_TASK_ID}.casa'
.format(**os.environ))
else:
logfile = casalog.logfile()
casalog.setlogfile(
'logs/{SLURM_JOB_NAME}-{SLURM_JOB_ID}.casa'.format(**os.environ))
if ',' in spw:
low, high, unit, dirs = config_parser.parse_spw(args['config'])
spwname = '{0:.0f}~{1:.0f}MHz'.format(min(low), max(high))
else:
spwname = spw.replace('0:', '')
msmd.open(visname)
npol = msmd.ncorrforpol()[0]
if not include_crosshand and npol == 4:
npol = 2
CPUs = npol if tasks * npol <= CPUS_PER_NODE_LIMIT else 1 #hard-code for number of polarisations
mvis = do_partition(visname, spw, preavg, CPUs, include_crosshand,
createmms, spwname)
mvis = "'{0}'".format(mvis)
vis = "'{0}'".format(visname)
config_parser.overwrite_config(args['config'],
conf_sec='data',
conf_dict={'vis': mvis})
config_parser.overwrite_config(
args['config'],
conf_sec='run',
sec_comment='# Internal variables for pipeline execution',
conf_dict={'orig_vis': vis})
msmd.done()
def main():
args = processMeerKAT.parse_args()
processMeerKAT.setup_logger(args.config,args.verbose)
msmd.open(args.MS)
dopol = args.dopol
refant = config_parser.parse_config(args.config)[0]['crosscal']['refant']
fields = get_fields(args.MS)
logger.info('[fields] section written to "{0}". Edit this section if you need to change field IDs (comma-seperated string for multiple IDs, not supported for calibrators).'.format(args.config))
npol = msmd.ncorrforpol()[0]
parang = 0
if 'phasecalfield' in fields:
parang = parang_coverage(args.MS, int(fields['phasecalfield'][1:-1])) #remove '' from field
if npol < 4:
logger.warn("Only {0} polarisations present in '{1}'. Any attempted polarisation calibration will fail, so setting dopol=False in [run] section of '{2}'.".format(npol,args.MS,args.config))
dopol = False
elif 0 < parang < 30:
logger.warn("Parallactic angle coverage is < 30 deg. Polarisation calibration will most likely fail, so setting dopol=False in [run] section of '{0}'.".format(args.config))
dopol = False
check_refant(args.MS, refant, args.config, warn=True)
threads = check_scans(args.MS,args.nodes,args.ntasks_per_node,dopol)
SPW = check_spw(args.config)
config_parser.overwrite_config(args.config, conf_dict={'dopol' : dopol}, conf_sec='run', sec_comment='# Internal variables for pipeline execution')
config_parser.overwrite_config(args.config, conf_dict=threads, conf_sec='slurm')
config_parser.overwrite_config(args.config, conf_dict=fields, conf_sec='fields')
config_parser.overwrite_config(args.config, conf_dict={'spw' : "'{0}'".format(SPW)}, conf_sec='crosscal')
msmd.done()
def run_script(func, logfile=''):
# Get the name of the config file
args = config_parser.parse_args()
# Parse config file
taskvals, config = config_parser.parse_config(args['config'])
continue_run = config_parser.validate_args(taskvals,
'run',
'continue',
bool,
default=True)
spw = config_parser.validate_args(taskvals, 'crosscal', 'spw', str)
nspw = config_parser.validate_args(taskvals, 'crosscal', 'nspw', int)
if continue_run:
try:
func(args, taskvals)
#rename_logs(logfile)
except Exception as err:
logger.error(
'Exception found in the pipeline of type {0}: {1}'.format(
type(err), err))
logger.error(traceback.format_exc())
config_parser.overwrite_config(
args['config'],
conf_dict={'continue': False},
conf_sec='run',
sec_comment='# Internal variables for pipeline execution')
if nspw > 1:
for SPW in spw.split(','):
spw_config = '{0}/{1}'.format(SPW.replace('0:', ''),
args['config'])
config_parser.overwrite_config(
spw_config,
conf_dict={'continue': False},
conf_sec='run',
sec_comment=
'# Internal variables for pipeline execution')
rename_logs(logfile)
sys.exit(1)
else:
logger.error(
'Exception found in previous pipeline job, which set "continue=False" in [run] section of "{0}". Skipping "{1}".'
.format(args['config'],
os.path.split(sys.argv[2])[1]))
#os.system('./killJobs.sh') # and cancelling remaining jobs (scancel not found since /opt overwritten)
rename_logs(logfile)
sys.exit(1)
def default_config(arg_dict):
"""Generate default config file in current directory, pointing to MS, with fields and SLURM parameters set.
Arguments:
----------
arg_dict : dict
Dictionary of arguments passed into this script, which is inserted into the config file under section [slurm]."""
filename = arg_dict['config']
MS = arg_dict['MS']
#Copy default config to current location
copyfile('{0}/{1}'.format(SCRIPT_DIR, CONFIG), filename)
#Add SLURM arguments to config file under section [slurm]
slurm_dict = get_slurm_dict(arg_dict, SLURM_CONFIG_KEYS)
for key in ['container', 'mpi_wrapper', 'partition', 'time', 'name']:
if key in slurm_dict.keys():
slurm_dict[key] = "'{0}'".format(slurm_dict[key])
#Overwrite parameters in config under section [slurm]
config_parser.overwrite_config(filename,
conf_dict=slurm_dict,
conf_sec='slurm')
#Add MS to config file under section [data]
config_parser.overwrite_config(filename,
conf_dict={'vis': "'{0}'".format(MS)},
conf_sec='data')
#Don't call srun if option --local used
mpi_wrapper = '' if arg_dict[
'local'] else 'srun --nodes=1 --ntasks=1 --time=10 --mem=4GB --partition={0}'.format(
arg_dict['partition'])
#Write and submit srun command to extract fields, and insert them into config file under section [fields]
params = '-B -M {MS} -C {config} -N {nodes} -t {ntasks_per_node}'.format(
**arg_dict)
command = write_command('get_fields.py',
params,
mpi_wrapper=mpi_wrapper,
container=arg_dict['container'],
logfile=False)
logger.info(
'Extracting field IDs from measurement set "{0}" using CASA.'.format(
MS))
logger.debug('Using the following command:\n\t{0}'.format(command))
os.system(command)
logger.info('Config "{0}" generated.'.format(filename))
def main():
args = processMeerKAT.parse_args()
msmd.open(args.MS)
refant = config_parser.parse_config(args.config)[0]['crosscal']['refant']
check_refant(args.MS, refant, warn=True)
threads = check_scans(args.MS,args.nodes,args.ntasks_per_node)
config_parser.overwrite_config(args.config, conf_dict=threads, conf_sec='slurm')
fields = get_fields(args.MS)
config_parser.overwrite_config(args.config, conf_dict=fields, conf_sec='fields')
logger.info('[fields] section written to "{0}". Edit this section to change field IDs (comma-seperated string for multiple IDs).'.format(args.config))
msmd.close()
msmd.done()
def main(args, taskvals):
visname = va(taskvals, 'data', 'vis', str)
fields = bookkeeping.get_field_ids(taskvals['fields'])
calcrefant = va(taskvals, 'crosscal', 'calcrefant', bool)
spw = va(taskvals, 'crosscal', 'spw', str)
nspw = va(taskvals, 'crosscal', 'nspw', int)
# Calculate reference antenna
if calcrefant:
if len(fields.fluxfield.split(',')) > 1:
field = fields.fluxfield.split(',')[0]
else:
field = fields.fluxfield
refant, badants = get_ref_ant(visname, field)
# Overwrite config file with new refant
config_parser.overwrite_config(
args['config'],
conf_sec='crosscal',
conf_dict={'refant': "'{0}'".format(refant)})
config_parser.overwrite_config(args['config'],
conf_sec='crosscal',
conf_dict={'badants': badants})
#Replace reference antenna in each SPW config
if nspw > 1:
for SPW in spw.split(','):
spw_config = '{0}/{1}'.format(SPW.replace('0:', ''),
args['config'])
# Overwrite config file with new refant
config_parser.overwrite_config(
spw_config,
conf_sec='crosscal',
conf_dict={'refant': "'{0}'".format(refant)})
config_parser.overwrite_config(spw_config,
conf_sec='crosscal',
conf_dict={'badants': badants})
config_parser.overwrite_config(spw_config,
conf_sec='crosscal',
conf_dict={'calcrefant': False})
else:
logger.info(
"Skipping calculation of reference antenna, as 'calcrefant=False' in '{0}'."
.format(args['config']))
def main(args, taskvals):
visname = va(taskvals, 'data', 'vis', str)
spw = va(taskvals, 'crosscal', 'spw', str, default='')
nspw = va(taskvals, 'crosscal', 'nspw', int, default='')
fields = bookkeeping.get_field_ids(taskvals['fields'])
dirs = config_parser.parse_spw(args['config'])[3]
if ',' in spw:
newvis = do_concat(visname, fields, dirs)
config_parser.overwrite_config(
args['config'],
conf_dict={'vis': "'{0}'".format(newvis)},
conf_sec='data')
config_parser.overwrite_config(
args['config'],
conf_dict={'crosscal_vis': "'{0}'".format(visname)},
conf_sec='run',
sec_comment='# Internal variables for pipeline execution')
else:
logger.error(
"Only found one SPW in '{0}', so will skip concat.".format(
args['config']))
def main():
# Get the name of the config file
args = config_parser.parse_args()
# Parse config file
taskvals, config = config_parser.parse_config(args['config'])
visname = va(taskvals, 'data', 'vis', str)
calcrefant = va(taskvals, 'crosscal', 'calcrefant', bool, default=False)
refant = va(taskvals, 'crosscal', 'refant', str, default='m005')
spw = va(taskvals, 'crosscal', 'spw', str, default='')
mvis = do_partition(visname, spw)
mvis = "'{0}'".format(mvis)
vis = "'{0}'".format(visname)
config_parser.overwrite_config(args['config'],
conf_sec='data',
conf_dict={'vis': mvis})
config_parser.overwrite_config(args['config'],
conf_sec='data',
conf_dict={'orig_vis': vis})
def spw_split_iris(spw,
nspw,
config,
badfreqranges,
MS,
partition,
createmms=True,
remove=True):
"""Split into N SPWs, placing an instance of the pipeline into N directories, each with 1 Nth of the bandwidth.
Arguments:
----------
spw : str
spw parameter from config.
nspw : int
Number of spectral windows to split into.
config : str
Path to config file.
mem : int
Memory in GB to use per instance.
badfreqranges : list
List of bad frequency ranges in MHz.
MS : str
Path to CASA Measurement Set.
partition : bool
Does this run include the partition step?
createmms : bool
Create MMS as output?
remove : bool, optional
Remove SPWs completely encompassed by bad frequency ranges?
Returns:
--------
nspw : int
New nspw, potentially a lower value than input (if any SPWs completely encompassed by badfreqranges)."""
if get_spw_bounds(spw) != None:
#Write nspw frequency ranges
low, high, unit, func = get_spw_bounds(spw)
interval = func((high - low) / float(nspw))
lo = linspace(low, high - interval, nspw)
hi = linspace(low + interval, high, nspw)
SPWs = []
#Remove SPWs entirely encompassed by bad frequency ranges (only for MHz unit)
for i in range(len(lo)):
SPWs.append('0:{0}~{1}{2}'.format(func(lo[i]), func(hi[i]), unit))
elif ',' in spw:
SPWs = spw.split(',')
unit = get_spw_bounds(SPWs[0])[2]
if len(SPWs) != nspw:
logger.logger.error(
"nspw ({0}) not equal to number of separate SPWs ({1} in '{2}') from '{3}'. Setting to nspw={1}."
.format(nspw, len(SPWs), spw, config))
nspw = len(SPWs)
else:
logger.logger.error(
"Can't split into {0} SPWs using SPW format '{1}'. Using nspw=1 in '{2}'."
.format(nspw, spw, config))
return 1
#Remove any SPWs completely encompassed by bad frequency ranges
i = 0
while i < nspw:
badfreq = False
low, high = get_spw_bounds(SPWs[i])[0:2]
if unit == 'MHz' and remove:
for freq in badfreqranges:
bad_low, bad_high = get_spw_bounds('0:{0}'.format(freq))[0:2]
if low >= bad_low and high <= bad_high:
logger.logger.info(
"Won't include spw '0:{0}~{1}{2}', since it's completely encompassed by bad frequency range '{3}'."
.format(low, high, unit, freq))
badfreq = True
break
if badfreq:
SPWs.pop(i)
i -= 1
nspw -= 1
i += 1
#Overwrite config with new SPWs
config_parser.overwrite_config(
config,
conf_dict={'spw': "'{0}'".format(','.join(SPWs))},
conf_sec='crosscal')
spw_config = '{0}_{1}'.format(config[:-4], "calib.txt")
copyfile(config, spw_config)
config_parser.overwrite_config(spw_config,
conf_dict={'nspw': nspw},
conf_sec='crosscal')
config_parser.overwrite_config(spw_config,
conf_dict={'calcrefant': False},
conf_sec='crosscal')
#config_parser.overwrite_config(spw_config, conf_dict={'precal_scripts' : []}, conf_sec='iris')
#config_parser.overwrite_config(spw_config, conf_dict={'postcal_scripts' : []}, conf_sec='iris')
basename, ext = os.path.splitext(MS.rstrip('/ '))
filebase = os.path.split(basename)[1].rstrip('.ms.tar')
extn = 'mms' if createmms else 'ms'
vis = '{0}.{1}.{2}.{3}'.format(filebase, spw.replace('0:', ''), extn,
'tar')
orig_vis = config_parser.get_key(spw_config, 'data', 'vis')
config_parser.overwrite_config(
spw_config,
conf_dict={'orig_vis': "'{0}'".format(orig_vis)},
conf_sec='run',
sec_comment='# Internal variables for pipeline execution')
config_parser.overwrite_config(spw_config,
conf_dict={'vis': "'{0}'".format(vis)},
conf_sec='data')
return nspw
if do_gaincal:
predict_model(vis, imagename, imsize, cell, gridder, wprojplanes,
deconvolver, robust, niter, multiscale, threshold,
nterms, pixmask, loop)
solnorm = 'a' in calmode[loop]
gaincal(vis=vis,
caltable=caltable,
selectdata=False,
refant=refant,
solint=solint[loop],
solnorm=solnorm,
normtype='median',
gaintable=prev_caltables,
calmode=calmode[loop],
append=False,
parang=False)
loop += 1
return loop
if __name__ == '__main__':
args, params = bookkeeping.get_selfcal_params()
loop = selfcal_part2(**params)
config_parser.overwrite_config(args['config'],
conf_dict={'loop': loop},
conf_sec='selfcal')
def default_config_iris(arg_dict):
"""Generate default config file in current directory, pointing to MS, with fields and SLURM parameters set.
Arguments:
----------
arg_dict : dict
Dictionary of arguments passed into this script, which is inserted into the config file under various sections."""
filename = arg_dict['config']
MS = arg_dict['MS']
#Copy default config to current location
copyfile('{0}/{1}'.format(globals.SCRIPT_DIR, globals.IRISCONFIG),
filename)
#Add MS to config file under section [data] and dopol under section [run]
config_parser.overwrite_config(filename,
conf_dict={'vis': "'{0}'".format(MS)},
conf_sec='data')
config_parser.overwrite_config(
filename,
conf_dict={'dopol': arg_dict['dopol']},
conf_sec='run',
sec_comment='# Internal variables for pipeline execution')
if not arg_dict['do2GC']:
config_parser.remove_section(filename, 'selfcal')
scripts = arg_dict['postcal_scripts']
i = 0
while i < len(scripts):
if 'selfcal' in scripts[i][0] or 'bdsf' in scripts[i][
0] or scripts[i][0] == 'make_pixmask.py':
scripts.pop(i)
i -= 1
i += 1
config_parser.overwrite_config(filename,
conf_dict={'postcal_scripts': scripts},
conf_sec='iris')
if not arg_dict['nofields']:
logger.logger.info('Field extraction not available for IRIS.')
#Skip extraction of field IDs and assume we're not processing multiple SPWs
logger.logger.info('Skipping extraction of field IDs')
#config_parser.overwrite_config(filename, conf_dict={'nspw' : 1}, conf_sec='crosscal')
#If dopol=True, replace second call of xx_yy_* scripts with xy_yx_* scripts
#Check in config (not CL args), in case read_ms.py forces dopol=False, and assume we only want to set this for 'scripts'
dopol = config_parser.get_key(filename, 'run', 'dopol')
if dopol:
count = 0
for ind, ss in enumerate(arg_dict['scripts']):
if ss[0] == 'xx_yy_solve.py' or ss[0] == 'xx_yy_apply.py':
count += 1
if count > 2:
if ss[0] == 'xx_yy_solve.py':
arg_dict['scripts'][ind] = ('xy_yx_solve.py',
arg_dict['scripts'][ind][1],
arg_dict['scripts'][ind][2])
if ss[0] == 'xx_yy_apply.py':
arg_dict['scripts'][ind] = ('xy_yx_apply.py',
arg_dict['scripts'][ind][1],
arg_dict['scripts'][ind][2])
config_parser.overwrite_config(filename,
conf_dict={'scripts': arg_dict['scripts']},
conf_sec='iris')
logger.logger.info('Config "{0}" generated.'.format(filename))
return
def default_config(arg_dict):
"""Generate default config file in current directory, pointing to MS, with fields and SLURM parameters set.
Arguments:
----------
arg_dict : dict
Dictionary of arguments passed into this script, which is inserted into the config file under various sections."""
filename = arg_dict['config']
MS = arg_dict['MS']
#Copy default config to current location
copyfile('{0}/{1}'.format(globals.SCRIPT_DIR, globals.CONFIG), filename)
#Add SLURM CL arguments to config file under section [slurm]
slurm_dict = get_slurm_dict(arg_dict, globals.SLURM_CONFIG_KEYS)
for key in globals.SLURM_CONFIG_STR_KEYS:
if key in slurm_dict.keys():
slurm_dict[key] = "'{0}'".format(slurm_dict[key])
#Overwrite CL parameters in config under section [slurm]
config_parser.overwrite_config(filename,
conf_dict=slurm_dict,
conf_sec='slurm')
#Add MS to config file under section [data] and dopol under section [run]
config_parser.overwrite_config(filename,
conf_dict={'vis': "'{0}'".format(MS)},
conf_sec='data')
config_parser.overwrite_config(
filename,
conf_dict={'dopol': arg_dict['dopol']},
conf_sec='run',
sec_comment='# Internal variables for pipeline execution')
if not arg_dict['do2GC']:
config_parser.remove_section(filename, 'selfcal')
scripts = arg_dict['postcal_scripts']
i = 0
while i < len(scripts):
if 'selfcal' in scripts[i][0] or 'bdsf' in scripts[i][
0] or scripts[i][0] == 'make_pixmask.py':
scripts.pop(i)
i -= 1
i += 1
config_parser.overwrite_config(filename,
conf_dict={'postcal_scripts': scripts},
conf_sec='slurm')
if not arg_dict['nofields']:
#Don't call srun if option --local used
if arg_dict['local']:
mpi_wrapper = ''
else:
mpi_wrapper = srun(arg_dict)
#Write and submit srun command to extract fields, and insert them into config file under section [fields]
params = '-B -M {MS} -C {config} -N {nodes} -t {ntasks_per_node}'.format(
**arg_dict)
if arg_dict['dopol']:
params += ' -P'
if arg_dict['verbose']:
params += ' -v'
command = write_command(globals.UTILS_DIR + '/read_ms.py',
params,
mpi_wrapper=mpi_wrapper,
container=arg_dict['container'],
logfile=False,
casa_script=False,
casacore=True)
logger.logger.info(
'Extracting field IDs from measurement set "{0}" using CASA.'.
format(MS))
logger.logger.debug(
'Using the following command:\n\t{0}'.format(command))
os.system(command)
else:
#Skip extraction of field IDs and assume we're not processing multiple SPWs
logger.logger.info(
'Skipping extraction of field IDs and assuming nspw=1.')
config_parser.overwrite_config(filename,
conf_dict={'nspw': 1},
conf_sec='crosscal')
#If dopol=True, replace second call of xx_yy_* scripts with xy_yx_* scripts
#Check in config (not CL args), in case read_ms.py forces dopol=False, and assume we only want to set this for 'scripts'
dopol = config_parser.get_key(filename, 'run', 'dopol')
if dopol:
count = 0
for ind, ss in enumerate(arg_dict['scripts']):
if ss[0] == 'xx_yy_solve.py' or ss[0] == 'xx_yy_apply.py':
count += 1
if count > 2:
if ss[0] == 'xx_yy_solve.py':
arg_dict['scripts'][ind] = ('xy_yx_solve.py',
arg_dict['scripts'][ind][1],
arg_dict['scripts'][ind][2])
if ss[0] == 'xx_yy_apply.py':
arg_dict['scripts'][ind] = ('xy_yx_apply.py',
arg_dict['scripts'][ind][1],
arg_dict['scripts'][ind][2])
config_parser.overwrite_config(
filename,
conf_dict={'scripts': arg_dict['scripts']},
conf_sec='slurm')
logger.logger.info('Config "{0}" generated.'.format(filename))
return
def main():
# Parse Arguments
args = processMeerKAT.parse_args()
processMeerKAT.setup_logger(args.config, args.verbose)
# Read in known_hpc and HPC_DEFAULTS from configuration file.
known_hpc_path = "{0}/{1}".format(os.path.dirname(__file__),
"known_hpc.cfg")
if os.path.isfile(known_hpc_path):
KNOWN_HPCS, _ = config_parser.parse_config(known_hpc_path)
else:
parser.error(
"Known HPC config file ({0}) not found.".format(known_hpc_path))
global HPC_DEFAULTS
HPC_DEFAULTS = KNOWN_HPCS[args.hpc if args.hpc in
KNOWN_HPCS.keys() else "unknown"]
# Open Measurement Set
msmd.open(args.MS)
dopol = args.dopol
refant = config_parser.parse_config(args.config)[0]['crosscal']['refant']
fields = get_fields(args.MS)
logger.info(
'[fields] section written to "{0}". Edit this section if you need to change field IDs (comma-seperated string for multiple IDs, not supported for calibrators).'
.format(args.config))
npol = msmd.ncorrforpol()[0]
parang = 0
if 'phasecalfield' in fields:
calfield = msmd.fieldsforname(fields['phasecalfield'][1:-1])[
0] #remove '' from field and convert to int
parang = parang_coverage(args.MS, calfield)
if npol < 4:
logger.warning(
"Only {0} polarisations present in '{1}'. Any attempted polarisation calibration will fail, so setting dopol=False in [run] section of '{2}'."
.format(npol, args.MS, args.config))
dopol = False
elif 0 < parang < 30:
logger.warning(
"Parallactic angle coverage is < 30 deg. Polarisation calibration will most likely fail, so setting dopol=False in [run] section of '{0}'."
.format(args.config))
dopol = False
check_refant(args.MS, refant, args.config, warn=True)
threads = check_scans(args.MS, args.nodes, args.ntasks_per_node, dopol)
SPW = check_spw(args.config)
config_parser.overwrite_config(
args.config,
conf_dict={'dopol': dopol},
conf_sec='run',
sec_comment='# Internal variables for pipeline execution')
config_parser.overwrite_config(args.config,
conf_dict=threads,
conf_sec='slurm')
config_parser.overwrite_config(args.config,
conf_dict=fields,
conf_sec='fields')
config_parser.overwrite_config(args.config,
conf_dict={'spw': "'{0}'".format(SPW)},
conf_sec='crosscal')
msmd.done()
def format_args_iris(config, submit, quiet, dependencies):
"""Format (and validate) arguments from config file, to be passed into write_jobs() function.
Arguments:
----------
config : str
Path to config file.
submit : bool
Allow user to force submitting to queue immediately.
quiet : bool
Activate quiet mode, with suppressed output?
dependencies : str
Comma-separated list of SLURM job dependencies.
selfcal : bool
Is selfcal being performed?
Returns:
--------
kwargs : dict
Keyword arguments extracted from [iris] section of config file, to be passed into write_jobs() function."""
#Ensure all keys exist in these sections
kwargs = get_config_kwargs(config, 'iris', globals.IRIS_CONFIG_KEYS)
data_kwargs = get_config_kwargs(config, 'data', ['vis'])
get_config_kwargs(config, 'fields', globals.FIELDS_CONFIG_KEYS)
crosscal_kwargs = get_config_kwargs(config, 'crosscal',
globals.CROSSCAL_CONFIG_KEYS)
#Check selfcal params
if config_parser.has_section(config, 'selfcal'):
selfcal_kwargs = get_config_kwargs(config, 'selfcal',
globals.SELFCAL_CONFIG_KEYS)
bookkeeping.get_selfcal_params()
#Force submit=True if user has requested it during [-R --run]
if submit:
kwargs['submit'] = True
#Ensure nspw is integer
if type(crosscal_kwargs['nspw']) is not int:
logger.logger.warn(
"Argument 'nspw'={0} in '{1}' is not an integer. Will set to integer ({2})."
.format(crosscal_kwargs['nspw']), config,
int(crosscal_kwargs['nspw']))
crosscal_kwargs['nspw'] = int(crosscal_kwargs['nspw'])
spw = crosscal_kwargs['spw']
nspw = crosscal_kwargs['nspw']
#mem = int(kwargs['mem'])
if nspw > 1 and len(kwargs['scripts']) == 0:
logger.logger.warn(
'Setting nspw=1, since no "scripts" parameter in "{0}" is empty, so there\'s nothing run inside SPW directories.'
.format(config))
config_parser.overwrite_config(config,
conf_dict={'nspw': 1},
conf_sec='crosscal')
nspw = 1
#If nspw = 1 and precal or postcal scripts present, overwrite config and reload
if nspw == 1:
if len(kwargs['precal_scripts']) > 0 or len(
kwargs['postcal_scripts']) > 0:
logger.logger.warn(
'Appending "precal_scripts" to beginning of "scripts", and "postcal_script" to end of "scripts", since nspw=1. Overwritting this in "{0}".'
.format(config))
#Drop first instance of calc_refant.py from precal scripts in preference for one in scripts (after flag_round_1.py)
if 'calc_refant.py' in [
i[0] for i in kwargs['precal_scripts']
] and 'calc_refant.py' in [i[0] for i in kwargs['scripts']]:
kwargs['precal_scripts'].pop([
i[0] for i in kwargs['precal_scripts']
].index('calc_refant.py'))
scripts = kwargs['precal_scripts'] + kwargs['scripts'] + kwargs[
'postcal_scripts']
config_parser.overwrite_config(config,
conf_dict={'scripts': scripts},
conf_sec='iris')
config_parser.overwrite_config(config,
conf_dict={'precal_scripts': []},
conf_sec='iris')
config_parser.overwrite_config(config,
conf_dict={'postcal_scripts': []},
conf_sec='iris')
kwargs = get_config_kwargs(config, 'iris',
globals.IRIS_CONFIG_KEYS)
else:
scripts = kwargs['scripts']
else:
scripts = kwargs['precal_scripts'] + kwargs['postcal_scripts']
kwargs['num_precal_scripts'] = len(kwargs['precal_scripts'])
# Validate kwargs along with MS
kwargs['MS'] = data_kwargs['vis']
validate_args_iris(kwargs, config)
#Reformat scripts tuple/list, to extract scripts, threadsafe, and containers as parallel lists
#Check that path to each script and container exists or is ''
kwargs['scripts'] = [check_path(i[0]) for i in scripts]
kwargs['threadsafe'] = [i[1] for i in scripts]
kwargs['containers'] = [check_path(i[2]) for i in scripts]
if not crosscal_kwargs['createmms']:
logger.logger.info(
"You've set 'createmms = False' in '{0}', so forcing 'keepmms = False'. Will use single CPU for every job other than 'quick_tclean.py', if present."
.format(config))
config_parser.overwrite_config(config,
conf_dict={'keepmms': False},
conf_sec='crosscal')
kwargs['threadsafe'] = [False] * len(scripts)
elif not crosscal_kwargs['keepmms']:
#Set threadsafe=False for split and postcal scripts (since working with MS not MMS).
if 'split.py' in kwargs['scripts']:
kwargs['threadsafe'][kwargs['scripts'].index('split.py')] = False
if nspw != 1:
kwargs['threadsafe'][kwargs['num_precal_scripts']:] = [
False
] * len(kwargs['postcal_scripts'])
#Set threadsafe=True for quick-tclean, as this uses MPI even for an MS
if 'quick_tclean.py' in kwargs['scripts']:
kwargs['threadsafe'][kwargs['scripts'].index('quick_tclean.py')] = True
#Only reduce the memory footprint if we're not using all CPUs on each node
# if kwargs['ntasks_per_node'] < NTASKS_PER_NODE_LIMIT:
# mem = mem // nspw
dopol = config_parser.get_key(config, 'run', 'dopol')
if not dopol and ('xy_yx_solve.py' in kwargs['scripts']
or 'xy_yx_apply.py' in kwargs['scripts']):
logger.logger.warn(
"Cross-hand calibration scripts 'xy_yx_*' found in scripts. Forcing dopol=True in '[run]' section of '{0}'."
.format(config))
config_parser.overwrite_config(
config,
conf_dict={'dopol': True},
conf_sec='run',
sec_comment='# Internal variables for pipeline execution')
includes_partition = any('partition' in script
for script in kwargs['scripts'])
#If single correctly formatted spw, split into nspw directories, and process each spw independently
if nspw > 1:
#Write timestamp to this pipeline run
kwargs['timestamp'] = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
config_parser.overwrite_config(
config,
conf_dict={'timestamp': "'{0}'".format(kwargs['timestamp'])},
conf_sec='run',
sec_comment='# Internal variables for pipeline execution')
nspw = spw_split_iris(spw,
nspw,
config,
crosscal_kwargs['badfreqranges'],
kwargs['MS'],
includes_partition,
createmms=crosscal_kwargs['createmms'])
config_parser.overwrite_config(config,
conf_dict={'nspw': "{0}".format(nspw)},
conf_sec='crosscal')
#Pop script to calculate reference antenna if calcrefant=False. Assume it won't be in postcal scripts
if not crosscal_kwargs['calcrefant']:
if pop_script(kwargs, 'calc_refant.py'):
kwargs['num_precal_scripts'] -= 1
#Replace empty containers with default container and remove unwanted kwargs
for i in range(len(kwargs['containers'])):
if kwargs['containers'][i] == '':
kwargs['containers'][i] = kwargs['container']
kwargs.pop('container')
kwargs.pop('MS')
kwargs.pop('precal_scripts')
kwargs.pop('postcal_scripts')
kwargs['quiet'] = quiet
#Force overwrite of dependencies
if dependencies != '':
kwargs['dependencies'] = dependencies
if len(kwargs['scripts']) == 0:
logger.logger.error(
'Nothing to do. Please insert scripts into "scripts" parameter in "{0}".'
.format(config))
sys.exit(1)
#If everything up until here has passed, we can copy config file to TMP_CONFIG (in case user runs sbatch manually) and inform user
logger.logger.debug(
"Copying '{0}' to '{1}', and using this to run pipeline.".format(
config, globals.TMP_CONFIG))
copyfile(config, globals.TMP_CONFIG)
if not quiet:
logger.logger.warn(
"Changing [iris] section in your config will have no effect until you submit your jdl"
)
return kwargs
def spw_split(spw,
nspw,
config,
mem,
badfreqranges,
MS,
partition,
createmms=True,
remove=True):
"""Split into N SPWs, placing an instance of the pipeline into N directories, each with 1 Nth of the bandwidth.
Arguments:
----------
spw : str
spw parameter from config.
nspw : int
Number of spectral windows to split into.
config : str
Path to config file.
mem : int
Memory in GB to use per instance.
badfreqranges : list
List of bad frequency ranges in MHz.
MS : str
Path to CASA Measurement Set.
partition : bool
Does this run include the partition step?
createmms : bool
Create MMS as output?
remove : bool, optional
Remove SPWs completely encompassed by bad frequency ranges?
Returns:
--------
nspw : int
New nspw, potentially a lower value than input (if any SPWs completely encompassed by badfreqranges)."""
if get_spw_bounds(spw) != None:
#Write nspw frequency ranges
low, high, unit, func = get_spw_bounds(spw)
interval = func((high - low) / float(nspw))
lo = linspace(low, high - interval, nspw)
hi = linspace(low + interval, high, nspw)
SPWs = []
#Remove SPWs entirely encompassed by bad frequency ranges (only for MHz unit)
for i in range(len(lo)):
SPWs.append('0:{0}~{1}{2}'.format(func(lo[i]), func(hi[i]), unit))
elif ',' in spw:
SPWs = spw.split(',')
unit = get_spw_bounds(SPWs[0])[2]
if len(SPWs) != nspw:
logger.logger.error(
"nspw ({0}) not equal to number of separate SPWs ({1} in '{2}') from '{3}'. Setting to nspw={1}."
.format(nspw, len(SPWs), spw, config))
nspw = len(SPWs)
else:
logger.logger.error(
"Can't split into {0} SPWs using SPW format '{1}'. Using nspw=1 in '{2}'."
.format(nspw, spw, config))
return 1
#Remove any SPWs completely encompassed by bad frequency ranges
i = 0
while i < nspw:
badfreq = False
low, high = get_spw_bounds(SPWs[i])[0:2]
if unit == 'MHz' and remove:
for freq in badfreqranges:
bad_low, bad_high = get_spw_bounds('0:{0}'.format(freq))[0:2]
if low >= bad_low and high <= bad_high:
logger.logger.info(
"Won't process spw '0:{0}~{1}{2}', since it's completely encompassed by bad frequency range '{3}'."
.format(low, high, unit, freq))
badfreq = True
break
if badfreq:
SPWs.pop(i)
i -= 1
nspw -= 1
i += 1
#Overwrite config with new SPWs
config_parser.overwrite_config(
config,
conf_dict={'spw': "'{0}'".format(','.join(SPWs))},
conf_sec='crosscal')
#Create each spw as directory and place config in there
logger.logger.info(
"Making {0} directories for SPWs ({1}) and copying '{2}' to each of them."
.format(nspw, SPWs, config))
for spw in SPWs:
spw_config = '{0}/{1}'.format(spw.replace('0:', ''), config)
if not os.path.exists(spw.replace('0:', '')):
os.mkdir(spw.replace('0:', ''))
copyfile(config, spw_config)
config_parser.overwrite_config(spw_config,
conf_dict={'spw': "'{0}'".format(spw)},
conf_sec='crosscal')
config_parser.overwrite_config(spw_config,
conf_dict={'nspw': 1},
conf_sec='crosscal')
config_parser.overwrite_config(spw_config,
conf_dict={'mem': mem},
conf_sec='slurm')
config_parser.overwrite_config(spw_config,
conf_dict={'calcrefant': False},
conf_sec='crosscal')
config_parser.overwrite_config(spw_config,
conf_dict={'precal_scripts': []},
conf_sec='slurm')
config_parser.overwrite_config(spw_config,
conf_dict={'postcal_scripts': []},
conf_sec='slurm')
#Look 1 directory up when using relative path
if MS[0] != '/':
config_parser.overwrite_config(
spw_config,
conf_dict={'vis': "'../{0}'".format(MS)},
conf_sec='data')
elif not partition:
basename, ext = os.path.splitext(MS.rstrip('/ '))
filebase = os.path.split(basename)[1]
extn = 'mms' if createmms else 'ms'
vis = '{0}.{1}.{2}'.format(filebase, spw.replace('0:', ''), extn)
logger.warn(
"Since script with 'partition' in its name isn't present in '{0}', assuming partition has already been done, and setting vis='{1}' in '{2}'. If '{1}' doesn't exist, please update '{2}', as the pipeline will not launch successfully."
.format(config, vis, spw_config))
orig_vis = config_parser.get_key(spw_config, 'data', 'vis')
config_parser.overwrite_config(
spw_config,
conf_dict={'orig_vis': "'{0}'".format(orig_vis)},
conf_sec='run',
sec_comment='# Internal variables for pipeline execution')
config_parser.overwrite_config(
spw_config,
conf_dict={'vis': "'{0}'".format(vis)},
conf_sec='data')
return nspw
if __name__ == '__main__':
args, params = bookkeeping.get_selfcal_params()
loop = params['loop']
selfcal_part2(**params)
rmsmap, outlierfile = find_outliers(**params, step='bdsf')
pixmask = mask_image(**params)
loop += 1
if config_parser.has_section(args['config'], 'image'):
config_parser.overwrite_config(
args['config'],
conf_dict={'mask': "'{0}'".format(pixmask)},
conf_sec='image')
config_parser.overwrite_config(
args['config'],
conf_dict={'rmsmap': "'{0}'".format(rmsmap)},
conf_sec='image')
config_parser.overwrite_config(
args['config'],
conf_dict={'outlierfile': "'{0}'".format(outlierfile)},
conf_sec='image')
config_parser.overwrite_config(args['config'],
conf_dict={'loop': loop},
conf_sec='selfcal')
bookkeeping.rename_logs(logfile)
def write_master(filename,config,scripts=[],submit=False,dir='jobScripts',pad_length=5,verbose=False, echo=True, dependencies='',slurm_kwargs={}):
"""Write master pipeline submission script, calling various sbatch files, and writing ancillary job scripts.
Arguments:
----------
filename : str
Name of master pipeline submission script.
config : str
Path to config file.
scripts : list, optional
List of sbatch scripts to call in order.
submit : bool, optional
Submit jobs to SLURM queue immediately?
dir : str, optional
Name of directory to output ancillary job scripts.
pad_length : int, optional
Length to pad the SLURM sacct output columns.
verbose : bool, optional
Verbose output (inserted into master script)?
echo : bool, optional
Echo the pupose of each job script for the user?
dependencies : str, optional
Comma-separated list of SLURM job dependencies.
slurm_kwargs : list, optional
Parameters parsed from [slurm] section of config."""
master = open(filename,'w')
master.write('#!/bin/bash\n')
timestamp = config_parser.get_key(config,'run','timestamp')
if timestamp == '':
timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
config_parser.overwrite_config(config, conf_dict={'timestamp' : "'{0}'".format(timestamp)}, conf_sec='run', sec_comment='# Internal variables for pipeline execution')
#Copy config file to TMP_CONFIG and inform user
if verbose:
master.write("\necho Copying \'{0}\' to \'{1}\', and using this to run pipeline.\n".format(config,globals.TMP_CONFIG))
master.write('cp {0} {1}\n'.format(config, TMP_CONFIG))
#Hack to perform correct number of selfcal loops
if 'selfcal_part1.sbatch' in scripts and 'selfcal_part2.sbatch' in scripts and 'run_bdsf.sbatch' in scripts and 'make_pixmask.sbatch' in scripts:
selfcal_loops = config_parser.parse_config(config)[0]['selfcal']['nloops']
scripts.extend(['selfcal_part1.sbatch','selfcal_part2.sbatch','run_bdsf.sbatch','make_pixmask.sbatch']*(selfcal_loops))
scripts.append('selfcal_part1.sbatch')
command = 'sbatch'
if dependencies != '':
master.write('\n#Run after these dependencies\nDep={0}\n'.format(dependencies))
command += ' -d afterok:$Dep --kill-on-invalid-dep=yes'
master.write('\n#{0}\n'.format(scripts[0]))
if verbose:
master.write('echo Submitting {0} SLURM queue with following command:\necho {1}\n'.format(scripts[0],command))
master.write("IDs=$({0} {1} | cut -d ' ' -f4)\n".format(command,scripts[0]))
scripts.pop(0)
#Submit each script with dependency on all previous scripts, and extract job IDs
for script in scripts:
command = 'sbatch -d afterok:$IDs --kill-on-invalid-dep=yes'
master.write('\n#{0}\n'.format(script))
if verbose:
master.write('echo Submitting {0} SLURM queue with following command\necho {1} {0}\n'.format(script,command))
master.write("IDs+=,$({0} {1} | cut -d ' ' -f4)\n".format(command,script))
master.write('\n#Output message and create {0} directory\n'.format(dir))
master.write('echo Submitted sbatch jobs with following IDs: $IDs\n')
master.write('mkdir -p {0}\n'.format(dir))
#Add time as extn to this pipeline run, to give unique filenames
master.write('\n#Add time as extn to this pipeline run, to give unique filenames')
master.write("\nDATE={0}".format(timestamp))
extn = '_$DATE.sh'
#Copy contents of config file to jobScripts directory
master.write('\n#Copy contents of config file to {0} directory\n'.format(dir))
master.write('cp {0} {1}/{2}_$DATE.txt\n'.format(config,dir,os.path.splitext(config)[0]))
#Write each job script - kill script, summary script, error script, and timing script
write_all_bash_jobs_scripts(master,extn,IDs='IDs',dir=dir,echo=echo,pad_length=pad_length,slurm_kwargs=slurm_kwargs)
#Close master submission script and make executable
master.close()
os.chmod(filename, 509)
#Submit script or output that it will not run
if submit:
if echo:
logger.logger.info('Running master script "{0}"'.format(filename))
os.system('./{0}'.format(filename))
else:
logger.logger.info('Master script "{0}" written, but will not run.'.format(filename))
