def main():
# Get the name of the config file
args = config_parser.parse_args()
# Parse config file
taskvals, config = config_parser.parse_config(args['config'])
visname = va(taskvals, 'data', 'vis', str)
fields = bookkeeping.get_field_ids(taskvals['fields'])
calcrefant = va(taskvals, 'crosscal', 'calcrefant', bool, default=False)
# Calculate reference antenna
if calcrefant:
if len(fields.fluxfield.split(',')) > 1:
field = fields.fluxfield.split(',')[0]
else:
field = fields.fluxfield
refant, badants = get_ref_ant(visname, field)
# Overwrite config file with new refant
config_parser.overwrite_config(args['config'],
conf_sec='crosscal',
conf_dict={'refant': refant})
config_parser.overwrite_config(args['config'],
conf_sec='crosscal',
conf_dict={'badants': badants})
def validateinput():
"""
Parse the input config file (command line argument) and validate that the
parameters look okay
"""
logger.info('This is version {0} of the pipeline'.format(processMeerKAT.__version__))
# Get the name of the config file
args = config_parser.parse_args()
# Parse config file
taskvals, config = config_parser.parse_config(args['config'])
visname = va(taskvals, 'data', 'vis', str)
calcrefant = va(taskvals, 'crosscal', 'calcrefant', bool)
refant = va(taskvals, 'crosscal', 'refant', str)
fields = bookkeeping.get_field_ids(taskvals['fields'])
# Check if the reference antenna exists, and complain and quit if it doesn't
if not calcrefant:
refant = va(taskvals, 'crosscal', 'refant', str)
msmd.open(visname)
get_fields.check_refant(MS=visname, refant=refant, warn=False)
msmd.close()
msmd.done()
if not os.path.exists(visname):
raise IOError("Path to MS %s not found" % (visname))
def get_imaging_params():
# Get the name of the config file
args = config_parser.parse_args()
# Parse config file
taskvals, config = config_parser.parse_config(args['config'])
params = taskvals['image']
params['vis'] = taskvals['data']['vis']
return args, params
def run_script(func, logfile=''):
# Get the name of the config file
args = config_parser.parse_args()
# Parse config file
taskvals, config = config_parser.parse_config(args['config'])
continue_run = config_parser.validate_args(taskvals,
'run',
'continue',
bool,
default=True)
spw = config_parser.validate_args(taskvals, 'crosscal', 'spw', str)
nspw = config_parser.validate_args(taskvals, 'crosscal', 'nspw', int)
if continue_run:
try:
func(args, taskvals)
#rename_logs(logfile)
except Exception as err:
logger.error(
'Exception found in the pipeline of type {0}: {1}'.format(
type(err), err))
logger.error(traceback.format_exc())
config_parser.overwrite_config(
args['config'],
conf_dict={'continue': False},
conf_sec='run',
sec_comment='# Internal variables for pipeline execution')
if nspw > 1:
for SPW in spw.split(','):
spw_config = '{0}/{1}'.format(SPW.replace('0:', ''),
args['config'])
config_parser.overwrite_config(
spw_config,
conf_dict={'continue': False},
conf_sec='run',
sec_comment=
'# Internal variables for pipeline execution')
rename_logs(logfile)
sys.exit(1)
else:
logger.error(
'Exception found in previous pipeline job, which set "continue=False" in [run] section of "{0}". Skipping "{1}".'
.format(args['config'],
os.path.split(sys.argv[2])[1]))
#os.system('./killJobs.sh') # and cancelling remaining jobs (scancel not found since /opt overwritten)
rename_logs(logfile)
sys.exit(1)
def main():
config = config_parser.parse_args()['config']
# Parse config file
taskvals, config = config_parser.parse_config(config)
visname = va(taskvals, 'data', 'vis', str)
keepmms = va(taskvals, 'crosscal', 'keepmms', bool)
calfiles, caldir = bookkeeping.bookkeeping(visname)
fields = bookkeeping.get_field_ids(taskvals['fields'])
msmd.open(visname)
if not os.path.exists(PLOT_DIR):
os.mkdir(PLOT_DIR)
#Plot solutions for bandpass calibrator
plotms(vis=calfiles.bpassfile, xaxis='Real', yaxis='Imag', coloraxis='corr', plotfile='{0}/bpass_real_imag.png'.format(PLOT_DIR),showgui=False)
plotms(vis=calfiles.bpassfile, xaxis='freq', yaxis='Amp', coloraxis='antenna1', plotfile='{0}/bpass_freq_amp.png'.format(PLOT_DIR),showgui=False)
plotms(vis=calfiles.bpassfile, xaxis='freq', yaxis='Phase', coloraxis='antenna1', plotfile='{0}/bpass_freq_phase.png'.format(PLOT_DIR),showgui=False)
#Plot solutions for phase calibrator
plotms(vis=calfiles.gainfile, xaxis='Real', yaxis='Imag', coloraxis='corr', plotfile='{0}/phasecal_real_imag.png'.format(PLOT_DIR),showgui=False)
plotms(vis=calfiles.gainfile, xaxis='Time', yaxis='Amp', coloraxis='antenna1', plotfile='{0}/phasecal_time_amp.png'.format(PLOT_DIR),showgui=False)
plotms(vis=calfiles.gainfile, xaxis='Time', yaxis='Phase', coloraxis='antenna1', plotfile='{0}/phasecal_time_phase.png'.format(PLOT_DIR),showgui=False)
#Plot solutions for individual antennas of bandpass and phase calibrator in 3x2 panels
plot_antennas('bpass',fields,calfiles,xaxis='freq',yaxis='amp')
plot_antennas('bpass',fields,calfiles,xaxis='freq',yaxis='phase')
plot_antennas('phasecal',fields,calfiles,xaxis='time',yaxis='amp')
plot_antennas('phasecal',fields,calfiles,xaxis='time',yaxis='phase')
extn = 'mms' if keepmms else 'ms'
for field in fields:
for subf in field.split(','):
fname = msmd.namesforfields(int(subf))[0]
inname = '%s.%s.%s' % (os.path.splitext(visname)[0], fname, extn)
if not os.path.exists('{0}/{1}_freq_amp.png'.format(PLOT_DIR,fname)):
plotms(vis=inname, xaxis='freq', yaxis='Amp', coloraxis='corr', plotfile='{0}/{1}_freq_amp.png'.format(PLOT_DIR,fname),showgui=False)
plotms(vis=inname, xaxis='Real', yaxis='Imag', coloraxis='corr', plotfile='{0}/{1}_real_imag.png'.format(PLOT_DIR,fname),showgui=False)
msmd.close()
msmd.done()
def main():
config = config_parser.parse_args()['config']
# Parse config file
taskvals, config = config_parser.parse_config(config)
visname = va(taskvals, 'data', 'vis', str)
calfiles, caldir = bookkeeping.bookkeeping(visname)
fields = bookkeeping.get_field_ids(taskvals['fields'])
#Plot solutions for bandpass calibrator
fastplot(calfiles.bpassfile,
col='CPARAM',
xaxis='Real',
yaxis='Imag',
fname='bpass_real_imag.png')
fastplot(calfiles.bpassfile,
col='CPARAM',
xaxis='chan',
yaxis='Amp',
logy=True,
fname='bpass_chan_amp.png')
fastplot(calfiles.bpassfile,
col='CPARAM',
xaxis='chan',
yaxis='Phase',
fname='bpass_chan_phase.png')
#Plot solutions for phase calibrator
fastplot(calfiles.gainfile,
col='CPARAM',
xaxis='Amp',
yaxis='Time',
fname='phasecal_time_amp.png',
markersize=2,
extent=1e-8)
fastplot(calfiles.gainfile,
col='CPARAM',
xaxis='Phase',
yaxis='Time',
fname='phasecal_time_phase.png',
markersize=2,
extent=1e-8)
def main():
# Get the name of the config file
args = config_parser.parse_args()
# Parse config file
taskvals, config = config_parser.parse_config(args['config'])
visname = va(taskvals, 'data', 'vis', str)
calcrefant = va(taskvals, 'crosscal', 'calcrefant', bool, default=False)
refant = va(taskvals, 'crosscal', 'refant', str, default='m005')
spw = va(taskvals, 'crosscal', 'spw', str, default='')
mvis = do_partition(visname, spw)
mvis = "'{0}'".format(mvis)
vis = "'{0}'".format(visname)
config_parser.overwrite_config(args['config'],
conf_sec='data',
conf_dict={'vis': mvis})
config_parser.overwrite_config(args['config'],
conf_sec='data',
conf_dict={'orig_vis': vis})
overwrite=True, writeflags=True)
# now flag using 'rflag' option
flagdata(vis=visname, mode="rflag", datacolumn="corrected",
field=fields.targetfield, timecutoff=5.0, freqcutoff=5.0, timefit="poly",
freqfit="poly", flagdimension="freqtime", extendflags=False,
timedevscale=5.0, freqdevscale=5.0, spectralmax=500.0,
extendpols=False, growaround=False, flagneartime=False,
flagnearfreq=False, action="apply", flagbackup=True, overwrite=True,
writeflags=True)
# Now summary
flagdata(vis=visname, mode="summary", datacolumn="corrected",
extendflags=True, name=visname + 'summary.split', action="apply",
flagbackup=True, overwrite=True, writeflags=True)
if __name__ == '__main__':
# Get the name of the config file
args = config_parser.parse_args()
# Parse config file
taskvals, config = config_parser.parse_config(args['config'])
visname = va(taskvals, 'data', 'vis', str)
calfiles, caldir = bookkeeping.bookkeeping(visname)
fields = bookkeeping.get_field_ids(taskvals['fields'])
do_pre_flag_2(visname, fields)
def get_selfcal_params():
#Flag for input errors
exit = False
# Get the name of the config file
args = config_parser.parse_args()
# Parse config file
taskvals, config = config_parser.parse_config(args['config'])
params = taskvals['selfcal']
check_params = params.keys()
check_params.pop(check_params.index('nloops'))
check_params.pop(check_params.index('restart_no'))
params['vis'] = taskvals['data']['vis']
params['refant'] = taskvals['crosscal']['refant']
if 'loop' not in params:
params['loop'] = 0
else:
check_params.pop(check_params.index('loop'))
for arg in check_params:
# Multiscale needs to be a list of lists (if specifying multiple scales)
# or a simple list (if specifying a single scale). So make sure these two
# cases are covered. Likewise for imsize.
if arg in ['multiscale', 'imsize']:
# Not a list of lists, so turn it into one of right length
if type(params[arg]) is list and (len(params[arg]) == 0 or type(
params[arg][0]) is not list):
params[arg] = [
params[arg],
] * (params['nloops'] + 1)
# Not a list at all, so put it into a list
elif type(params[arg]) is not list:
params[arg] = [
[
params[arg],
],
] * (params['nloops'] + 1)
# A list of lists of length 1, so put into list of lists of right length
elif type(params[arg]) is list and type(
params[arg][0]) is list and len(params[arg]) == 1:
params[arg] = [
params[arg][0],
] * (params['nloops'] + 1)
if len(params[arg]) != params['nloops'] + 1:
logger.error(
"Parameter '{0}' in '{1}' is the wrong length. It is {2} but must be a single value or equal to 'nloops' + 1 ({3})."
.format(arg, args['config'], len(params[arg]),
params['nloops'] + 1))
exit = True
else:
if type(params[arg]) is not list:
if arg in ['solint', 'calmode']:
params[arg] = [params[arg]] * (params['nloops'])
else:
params[arg] = [params[arg]] * (params['nloops'] + 1)
if arg in ['solint', 'calmode'
] and len(params[arg]) != params['nloops']:
logger.error(
"Parameter '{0}' in '{1}' is the wrong length. It is {2} long but must be 'nloops' ({3}) long or a single value (not a list)."
.format(arg, args['config'], len(params[arg]),
params['nloops']))
exit = True
elif arg not in ['solint', 'calmode'
] and len(params[arg]) != params['nloops'] + 1:
logger.error(
"Parameter '{0}' in '{1}' is the wrong length. It is {2} long but must 'nloops' + 1 ({3}) long or a single value (not a list)."
.format(arg, args['config'], len(params[arg]),
params['nloops'] + 1))
exit = True
if exit:
sys.exit(1)
return args, params
"""
Main module that controls the complete graph QA pipeline
"""
import math
from config_parser import parse_args
from trainer import Trainer
from evaluator import Evaluator
from data.data_loader import getMetaData, getDataLoader
if __name__ == "__main__":
args = parse_args()
args.gen_config()
# Calculate Meta Parameters from dataset
train_max_vertices, train_feature_size, train_data_size, train_max_total_vertices = getMetaData(
args, args.train_dir)
val_max_vertices, val_feature_size, val_data_size, val_max_total_vertices = getMetaData(
args, args.val_dir)
test_max_vertices, test_feature_size, test_data_size, test_max_total_vertices = getMetaData(
args, args.test_dir)
args.max_vertices = max(train_max_vertices, val_max_vertices,
test_max_vertices)
args.feature_size = max(train_feature_size, val_feature_size,
test_feature_size)
args.data_size = max(train_data_size, val_data_size, test_data_size)
args.max_total_vertices = max(train_max_total_vertices,
val_max_total_vertices,
test_max_total_vertices)
args.train_data_size = train_data_size
def get_selfcal_params():
#Flag for input errors
exit = False
# Get the name of the config file
args = config_parser.parse_args()
# Parse config file
taskvals, config = config_parser.parse_config(args['config'])
params = taskvals['selfcal']
other_params = list(params.keys())
params['vis'] = taskvals['data']['vis']
params['refant'] = taskvals['crosscal']['refant']
params['dopol'] = taskvals['run']['dopol']
if params['dopol'] and 'G' in params['gaintype']:
logger.warning(
"dopol is True, but gaintype includes 'G'. Use gaintype='T' for polarisation on linear feeds (e.g. MeerKAT)."
)
single_args = ['nloops', 'loop', 'discard_nloops',
'outlier_threshold'] #need to be 1 long (i.e. not a list)
gaincal_args = ['solint', 'calmode', 'gaintype',
'flag'] #need to be nloops long
list_args = ['imsize'] #allowed to be lists of lists
for arg in single_args:
if arg in other_params:
other_params.pop(other_params.index(arg))
for arg in single_args:
if type(params[arg]) is list or type(
params[arg]) is str and ',' in params[arg]:
logger.error(
"Parameter '{0}' in '{1}' cannot be a list. It must be a single value."
.format(arg, args['config']))
exit = True
for arg in other_params:
if type(params[arg]) is str and ',' in params[arg]:
logger.error(
"Parameter '{0}' in '{1}' cannot use comma-seprated values. It must be a list or values, or a single value."
.format(arg, args['config']))
exit = True
# These can be a list of lists or a simple list (if specifying a single value).
# So make sure these two cases are covered.
if arg in list_args:
# Not a list of lists, so turn it into one of right length
if type(params[arg]) is list and (len(params[arg]) == 0 or type(
params[arg][0]) is not list):
params[arg] = [
params[arg],
] * (params['nloops'] + 1)
# Not a list at all, so put it into a list
elif type(params[arg]) is not list:
params[arg] = [
[
params[arg],
],
] * (params['nloops'] + 1)
# A list of lists of length 1, so put into list of lists of right length
elif type(params[arg]) is list and type(
params[arg][0]) is list and len(params[arg]) == 1:
params[arg] = [
params[arg][0],
] * (params['nloops'] + 1)
elif type(params[arg]) is not list:
if arg in gaincal_args:
params[arg] = [params[arg]] * (params['nloops'])
else:
params[arg] = [params[arg]] * (params['nloops'] + 1)
for arg in other_params:
#By this point params[arg] will be a list
if arg in gaincal_args and len(params[arg]) != params['nloops']:
logger.error(
"Parameter '{0}' in '{1}' is the wrong length. It is {2} long but must be 'nloops' ({3}) long or a single value (not a list)."
.format(arg, args['config'], len(params[arg]),
params['nloops'] - 1))
exit = True
elif arg not in gaincal_args and len(
params[arg]) != params['nloops'] + 1:
logger.error(
"Parameter '{0}' in '{1}' is the wrong length. It is {2} long but must be 'nloops' + 1 ({3}) long or a single value (not a list)."
.format(arg, args['config'], len(params[arg]),
params['nloops'] + 2))
exit = True
if exit:
sys.exit(1)
return args, params
