def worker(pipeline, recipe, config):
uvrange = config['uvrange']
plotter = config['standard_plotter']
label_in = config['label_in']
nobs = pipeline.nobs
subdir = config['dirname']
output_dir = os.path.join(pipeline.diagnostic_plots, subdir) if subdir else pipeline.diagnostic_plots
if config['field'] == 'calibrators':
fields = ['bpcal', 'gcal', 'fcal']
elif config['field'] == 'target':
fields = ['target']
else:
fields = config['field'].split(',')
if set(fields).difference(['fcal', 'bpcal', 'gcal']):
raise ValueError("Eligible values for 'field': 'target', 'calibrators', 'fcal', 'bpcal' or 'gcal'. " \
"User selected {}".format(",".join(fields)))
log.info(f"plotting fields: {' '.join(fields)}")
for iobs in range(nobs):
label = config['label_plot']
mslist = pipeline.get_mslist(iobs, label_in, target=(config['field'] == 'target'))
for msname in mslist:
if not os.path.exists(os.path.join(pipeline.msdir, msname)):
raise IOError("MS {0:s} does not exist. Please check that is where it should be.".format(msname))
for msname in mslist:
log.info(f"plotting MS: {msname}")
msbase = os.path.splitext(msname)[0]
ms_info_dict = pipeline.get_msinfo(msname)
corr = config['correlation']
ms_corrs = ms_info_dict['CORR']['CORR_TYPE']
if corr == 'auto' or corr == 'all':
corr = ','.join(ms_corrs)
elif corr == 'diag' or corr == 'parallel':
corr = ','.join([c for c in ms_corrs if len(c) == 2 and c[0] == c[1]])
if not corr:
log.warning(f"No correlations found to plot for {msname}")
continue
log.info(f"plotting correlations: {corr}")
# new-school plots
if config['shadems']['enable']:
# make dict of substitutions
basesubst = OrderedDict(msbase=os.path.splitext(msname)[0])
# make a map: {(fields): field_type}, so we can loop over fields below, but only include unique fields
field_map = OrderedDict()
# make a reverse map: field_type -> "field_name,field_name"
field_map_names = OrderedDict()
# make set of all field names
all_fields = set()
for field_type in fields:
if (label_in != '') and (config['field'] == 'target'):
field_names = tuple(ms_info_dict['FIELD']['NAME'])
else:
field_names = tuple(getattr(pipeline, field_type)[iobs])
field_map.setdefault(field_names, field_type)
all_fields.update(field_names)
basesubst[field_type] = field_map_names[field_type] = ",".join(field_names)
basesubst["all_fields"] = ",".join(all_fields)
plot_args = []
def collect_plots(args, plotspecs, extra_args, subst=None):
"""Generic helper function to parse a list of shadems plot specs, and add them to plot_args"""
for iplot, plotspec in enumerate(plotspecs):
if plotspec:
plotspec = plotspec.format(**(subst or basesubst))
# add arguments from args, if not present in plotspec
plotspec = plotspec.split()
for arg, value in args.items():
arg = "--" + arg
if arg not in plotspec:
plotspec += [arg, value]
plotspec += extra_args
plot_args.append(" ".join(plotspec))
baseargs = OrderedDict(
png="{}-{}-{}".format(msbase, label,
"{field}{_Spw}{_Scan}{_Ant}-{label}{_alphalabel}{_colorlabel}{_suffix}.png"),
title="'{ms} {_field}{_Spw}{_Scan}{_Ant}{_title}{_Alphatitle}{_Colortitle}'",
col=config['shadems']['default_column'],
corr=corr.replace(' ', ''))
# collect generic plots
collect_plots(baseargs, config['shadems']['plots'], [])
# collect plots_by_corr
args = baseargs.copy()
args["field"] = ",".join(all_fields)
collect_plots(args, config['shadems']['plots_by_corr'], ["--iter-corr"])
# collect plots_by_field
for field_names, field_type in field_map.items():
args = baseargs.copy()
args["field"] = ",".join(field_names)
args["png"] = "{}-{}-{}-{}".format(msbase, label, field_type,
"{field}{_Spw}{_Scan}{_Ant}-{label}{_alphalabel}{_colorlabel}{_suffix}.png")
args["title"] = "'{ms} " + field_type + "{_field}{_Spw}{_Scan}{_Ant}{_title}{_Alphatitle}{_Colortitle}'"
subst = basesubst.copy()
subst["field"] = field_type
collect_plots(args, config['shadems']['plots_by_field'], ["--iter-field"])
# dispatch plots
if plot_args:
step = 'plot-shadems-ms{0:d}'.format(iobs)
recipe.add("cab/shadems_direct", step,
dict(ms=msname, args=plot_args,
ignore_errors=config["shadems"]["ignore_errors"]),
input=pipeline.input, output=output_dir,
label="{0:s}:: Plotting".format(step))
else:
log.warning("The shadems section is enabled, but doesn't specify any plot_by_field or plot_by_corr")
# old-school plots
# define plot attributes
diagnostic_plots = {}
diagnostic_plots["real_imag"] = dict(
plotms={"xaxis": "imag", "yaxis": "real",
"colouraxis": "baseline", "iteraxis": "corr"},
shadems={"xaxis": "real", "yaxis": "imag"},
ragavi_vis={"xaxis": "real", "yaxis": "imaginary",
"iter-axis": "scan", "canvas-width": 300,
"canvas-height": 300})
diagnostic_plots["amp_phase"] = dict(
plotms={"xaxis": "amp", "yaxis": "phase",
"colouraxis": "baseline", "iteraxis": "corr"},
shadems={"xaxis": "amp", "yaxis": "phase"},
ragavi_vis={"xaxis": "phase", "yaxis": "amplitude",
"iter-axis": "corr", "canvas-width": 1080,
"canvas-height": 720})
diagnostic_plots["amp_ant"] = dict(
plotms={"xaxis": "antenna", "yaxis": "amp",
"colouraxis": "baseline", "iteraxis": "corr"},
shadems={"xaxis": "ANTENNA1", "yaxis": "amp"},
ragavi_vis=None)
diagnostic_plots["amp_uvwave"] = dict(
plotms={"xaxis": "uvwave", "yaxis": "amp",
"colouraxis": "baseline", "iteraxis": "corr"},
shadems={"xaxis": "UV", "yaxis": "amp"},
ragavi_vis={"xaxis": "uvwave", "yaxis": "amplitude",
"iter-axis": "scan", "canvas-width": 300,
"canvas-height": 300})
diagnostic_plots["phase_uvwave"] = dict(
plotms={"xaxis": "uvwave", "yaxis": "phase",
"colouraxis": "baseline", "iteraxis": "corr"},
shadems={"xaxis": "UV", "yaxis": "phase"},
ragavi_vis={"xaxis": "uvwave", "yaxis": "phase",
"iter-axis": "scan", "canvas-width": 300,
"canvas-height": 300})
diagnostic_plots["amp_scan"] = dict(
plotms={"xaxis": "scan", "yaxis": "amp"},
shadems={"xaxis": "SCAN_NUMBER", "yaxis": "amp"},
ragavi_vis={"xaxis": "scan", "yaxis": "amplitude",
"iter-axis": None,
"canvas-width": 1080, "canvas-height": 720})
diagnostic_plots["amp_chan"] = dict(
plotms={"xaxis": "chan", "yaxis": "amp"},
shadems={"xaxis": "CHAN", "yaxis": "amp"},
ragavi_vis={"xaxis": "channel", "yaxis": "amplitude",
"iter-axis": "scan", "canvas-width": 300,
"canvas-height": 300})
diagnostic_plots["phase_chan"] = dict(
plotms={"xaxis": "chan", "yaxis": "phase"},
shadems={"xaxis": "CHAN", "yaxis": "phase"},
ragavi_vis={"xaxis": "channel", "yaxis": "phase",
"iter-axis": "scan", "canvas-width": 300,
"canvas-height": 300})
if plotter.lower() != "none":
for plotname in diagnostic_plots:
if not pipeline.enable_task(config, plotname):
continue
opts = diagnostic_plots[plotname][plotter]
if opts is None:
log.warn("The plotter '{0:s}' cannot make the plot '{1:s}'".format(
plotter, plotname))
continue
elif plotter == "ragavi_vis":
opts["num-cores"] = config["num_cores"]
opts["mem-limit"] = config["mem_limit"]
# make map from field name to field_type, field_id
field_map = OrderedDict()
for field_type in fields:
for field in getattr(pipeline, field_type)[iobs]:
if label_in != '' and field_type == 'target':
fid = 0
else:
fid = utils.get_field_id(ms_info_dict, field)[0]
field_map.setdefault(field, (field_type, fid))
if plotter == "shadems":
corr = corr.replace(" ", "").split(",")
for it, co in enumerate(corr):
if co in ms_corrs:
corr[it] = str(ms_corrs.index(co))
corr = ",".join(corr)
# for each corr
for co in corr.split(","):
opts["corr"] = co
for field, (field_type, fid) in field_map.items():
globals()[plotter](pipeline, recipe, config,
plotname, msname, field,
iobs, label, msbase, opts,
ftype=field_type, fid=fid, output_dir=output_dir,
corr_label=ms_corrs[int(co)])
elif plotter == "ragavi_vis" and not opts["iter-axis"] == "corr":
# change the labels to indices
corr = corr.replace(" ", "").split(",")
for it, co in enumerate(corr):
if co in ms_corrs:
corr[it] = str(ms_corrs.index(co))
corr = ",".join(corr)
# for each corr
for co in corr.split(","):
opts["corr"] = co
for field, (field_type, fid) in field_map.items():
globals()[plotter](pipeline, recipe, config,
plotname, msname, field,
iobs, label, msbase, opts,
ftype=field_type, fid=fid, output_dir=output_dir,
corr_label=ms_corrs[int(co)])
else:
opts["corr"] = corr
for field, (field_type, fid) in field_map.items():
globals()[plotter](pipeline, recipe, config,
plotname, msname, field, iobs, label,
msbase, opts, ftype=field_type,
fid=fid, output_dir=output_dir)
def worker(pipeline, recipe, config):
label = config['label_in']
wname = pipeline.CURRENT_WORKER
flags_before_worker = '{0:s}_{1:s}_before'.format(pipeline.prefix, wname)
flags_after_worker = '{0:s}_{1:s}_after'.format(pipeline.prefix, wname)
nobs = pipeline.nobs
msiter=0
for i in range(nobs):
prefix_msbase = pipeline.prefix_msbases[i]
mslist = pipeline.get_mslist(i, label, target=(config['field'] == "target"))
target_ls = pipeline.target[i] if config['field'] == "target" else []
for j, msname in enumerate(mslist):
msdict = pipeline.get_msinfo(msname)
prefix = os.path.splitext(msname)[0]
if not os.path.exists(os.path.join(pipeline.msdir, msname)):
raise IOError("MS file {0:s} does not exist. Please check that is where it should be.".format(msname))
# Write/rewind flag versions
available_flagversions = manflags.get_flags(pipeline, msname)
if config['rewind_flags']['enable']:
if config['rewind_flags']['mode'] == 'reset_worker':
version = flags_before_worker
stop_if_missing = False
elif config['rewind_flags']['mode'] == 'rewind_to_version':
version = config['rewind_flags']['version']
if version == 'auto':
version = flags_before_worker
stop_if_missing = True
if version in available_flagversions:
if flags_before_worker in available_flagversions and available_flagversions.index(flags_before_worker) < available_flagversions.index(version) and not config['overwrite_flagvers']:
manflags.conflict('rewind_too_little', pipeline, wname, msname, config, flags_before_worker, flags_after_worker)
substep = 'version-{0:s}-ms{1:d}'.format(version, msiter)
manflags.restore_cflags(pipeline, recipe, version, msname, cab_name=substep)
if version != available_flagversions[-1]:
substep = 'delete-flag_versions-after-{0:s}-ms{1:d}'.format(version, msiter)
manflags.delete_cflags(pipeline, recipe,
available_flagversions[available_flagversions.index(version)+1],
msname, cab_name=substep)
if version != flags_before_worker:
substep = 'save-{0:s}-ms{1:d}'.format(flags_before_worker, msiter)
manflags.add_cflags(pipeline, recipe, flags_before_worker,
msname, cab_name=substep, overwrite=config['overwrite_flagvers'])
elif stop_if_missing:
manflags.conflict('rewind_to_non_existing', pipeline, wname, msname, config, flags_before_worker, flags_after_worker)
else:
substep = 'save-{0:s}-ms{1:d}'.format(flags_before_worker, msiter)
manflags.add_cflags(pipeline, recipe, flags_before_worker,
msname, cab_name=substep, overwrite=config['overwrite_flagvers'])
else:
if flags_before_worker in available_flagversions and not config['overwrite_flagvers']:
manflags.conflict('would_overwrite_bw', pipeline, wname, msname, config, flags_before_worker, flags_after_worker)
else:
substep = 'save-{0:s}-ms{1:d}'.format(flags_before_worker, msiter)
manflags.add_cflags(pipeline, recipe, flags_before_worker,
msname, cab_name=substep, overwrite=config['overwrite_flagvers'])
# Define fields and field_ids to be used to only flag the fields selected with
# flagging:field (either 'target' or 'calibrators') and with
# flagging:calfields (for further selection among the calibrators)
if config['field'] == 'target':
fields = [target_ls[j]]
else:
fields = []
fld_string = config['calfields']
if fld_string == "auto":
iter_fields = "gcal bpcal xcal fcal".split()
else:
iter_fields = fld_string.split(",")
for item in iter_fields:
if hasattr(pipeline, item):
tfld = getattr(pipeline, item)[i]
else:
raise ValueError("Field given is invalid. Options are 'xcal bpcal gcal fcal'.")
if tfld:
fields += tfld
fields = list(set(fields))
field_ids = utils.get_field_id(msdict, fields)
fields = ",".join(fields)
if pipeline.enable_task(config, 'unflag'):
step = '{0:s}-unflag-ms{1:d}'.format(wname, msiter)
recipe.add('cab/casa_flagdata', step,
{
"vis": msname,
"mode": 'unflag',
"field": fields,
"flagbackup": False,
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}:: Unflag ms={1:s}'.format(step, msname))
# flag antennas automatically based on drifts in the scan average of the
# auto correlation spectra per field. This doesn't strictly require any calibration. It is also
# not field structure dependent, since it is just based on the DC of the field
# Compares scan to median power of scans per field per channel
# Also compares antenna to median of the array per scan per field per channel
# This should catch any antenna with severe temperature problems
if pipeline.enable_task(config, 'flag_autopowerspec'):
step = '{0:s}-autopowerspec-ms{1:d}'.format(wname, msiter)
recipe.add("cab/politsiyakat_autocorr_amp", step,
{
"msname": msname,
"field": ",".join([str(id) for id in field_ids]),
"cal_field": ",".join([str(id) for id in field_ids]),
"scan_to_scan_threshold": config["flag_autopowerspec"]["scan_thr"],
"antenna_to_group_threshold": config["flag_autopowerspec"]["ant_group_thr"],
"dpi": 300,
"plot_size": 6,
"nproc_threads": config['flag_autopowerspec']['threads'],
"data_column": config['flag_autopowerspec']['col']
},
input=pipeline.input, output=pipeline.output,
label="{0:s}:: Flag out antennas with drifts in autocorrelation powerspectra ms={1:s}".format(step,msname))
if pipeline.enable_task(config, 'flag_autocorr'):
step = '{0:s}-autocorr-ms{1:d}'.format(wname, msiter)
recipe.add('cab/casa_flagdata', step,
{
"vis": msname,
"mode": 'manual',
"autocorr": True,
"field": fields,
"flagbackup": False,
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}:: Flag auto-correlations ms={1:s}'.format(step, msname))
if pipeline.enable_task(config, 'flag_quack'):
step = '{0:s}-quack-ms{1:d}'.format(wname, msiter)
recipe.add('cab/casa_flagdata', step,
{
"vis": msname,
"mode": 'quack',
"quackinterval": config['flag_quack']['interval'],
"quackmode": config['flag_quack']['mode'],
"field": fields,
"flagbackup": False,
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}:: Quack flagging ms={1:s}'.format(step, msname))
if pipeline.enable_task(config, 'flag_elevation'):
step = '{0:s}-elevation-ms{1:d}'.format(wname, msiter)
recipe.add('cab/casa_flagdata', step,
{
"vis": msname,
"mode": 'elevation',
"lowerlimit": config['flag_elevation']['low'],
"upperlimit": config['flag_elevation']['high'],
"field": fields,
"flagbackup": False,
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}:: Flag elevation ms={1:s}'.format(step, msname))
if pipeline.enable_task(config, 'flag_shadow'):
if config['flag_shadow']['full_mk64']:
addantennafile = '{0:s}/mk64.txt'.format(pipeline.input)
subarray = msdict['ANT']['NAME']
idleants = open(addantennafile, 'r').readlines()
for aa in subarray:
for kk in range(len(idleants)):
if aa in idleants[kk]:
del(idleants[kk:kk+3])
break
addantennafile = 'idleants.txt'
with open('{0:s}/{1:s}'.format(pipeline.input, addantennafile), 'w') as ia:
for aa in idleants:
ia.write(aa)
addantennafile += ':input'
else:
addantennafile = None
step = '{0:s}-shadow-ms{1:d}'.format(wname, msiter)
recipe.add('cab/casa_flagdata', step,
{
"vis": msname,
"mode": 'shadow',
"tolerance": config['flag_shadow']['tol'],
"addantenna": addantennafile,
"flagbackup": False,
"field": fields,
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}:: Flag shadowed antennas ms={1:s}'.format(step, msname))
if pipeline.enable_task(config, 'flag_spw'):
step = '{0:s}-spw-ms{1:d}'.format(wname, msiter)
flagspwselection = config['flag_spw']['chans']
firsts = [min(ff) for ff in msdict['SPW']['CHAN_FREQ']]
lasts = [max(ff) for ff in msdict['SPW']['CHAN_FREQ']]
nrs = msdict['SPW']['NUM_CHAN']
nspws = len(nrs)
found_valid_data = 0
if config['flag_spw']['ensure_valid']:
scalefactor, scalefactor_dict = 1, {
'GHz': 1e+9, 'MHz': 1e+6, 'kHz': 1e+3 }
for ff in flagspwselection.split(','):
found_units = False
for dd in scalefactor_dict:
if dd.lower() in ff.lower():
ff, scalefactor = ff.lower().replace(
dd.lower(), ''), scalefactor_dict[dd]
found_units = True
if 'hz' in ff.lower():
ff = ff.lower().replace('hz', '')
found_units = True
ff = ff.split(':')
if len(ff) > 1:
spws = ff[0]
else:
spws = '*'
edges = [
ii*scalefactor for ii in map(float, ff[-1].split('~'))]
if '*' in spws:
spws = list(range(nspws))
elif '~' in spws:
spws = list(
range(int(spws.split('~')[0]), int(spws.split('~')[1])+1))
else:
spws = [int(spws), ]
edges = [edges for uu in range(len(spws))]
for ss in spws:
if found_units and ss < nspws and min(edges[ss][1], lasts[ss]) - max(edges[ss][0], firsts[ss]) > 0:
found_valid_data = 1
elif not found_units and ss < nspws and edges[ss][0]>=0 and edges[ss][1] < nrs[ss]:
found_valid_data = 1
if not found_valid_data:
caracal.log.warn(
'The following channel selection has been made in the flag_spw module of the flagging worker: "{1:s}". This selection would result in no valid data in {0:s}. This would lead to the FATAL error "No valid SPW & Chan combination found" in CASA/FLAGDATA. To avoid this error the corresponding cab {2:s} will not be added to the Stimela recipe of the flagging worker.'.format(msname, flagspwselection, step))
if found_valid_data or not config['flag_spw']['ensure_valid']:
recipe.add('cab/casa_flagdata', step,
{
"vis": msname,
"mode": 'manual',
"spw": flagspwselection,
"field": fields,
"flagbackup": False,
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}::Flag out channels ms={1:s}'.format(step, msname))
if pipeline.enable_task(config, 'flag_time'):
step = '{0:s}-time-ms{1:d}'.format(wname, msiter)
found_valid_data = 0
if config['flag_time']['ensure_valid']:
if pipeline.startdate[i]:
start_flagrange,end_flagrange=config['flag_time']['timerange'].split('~')
flag_start = float(''.join(re.split('/|:', start_flagrange)))
flag_end = float(''.join(re.split('/|:', end_flagrange)))
if (flag_start <= pipeline.enddate[i]) and (pipeline.startdate[i] <= flag_end):
found_valid_data = 1
else:
raise ValueError("You wanted to ensure a valid time range but we could not find a start and end time")
if not found_valid_data:
caracal.log.warn(
'The following time selection has been made in the flag_time module of the flagging worker: "{1:s}". This selection would result in no valid data in {0:s}. This would lead to the FATAL error " The selected table has zero rows" in CASA/FLAGDATA. To avoid this error the corresponding cab {2:s} will not be added to the Stimela recipe of the flagging worker.'.format(msname, config['flag_time']['timerange'], step))
if found_valid_data or not config['flag_time']['ensure_valid']:
recipe.add('cab/casa_flagdata', step,
{
"vis": msname,
"mode": 'manual',
"timerange": config['flag_time']['timerange'],
"flagbackup": False,
"field": fields,
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}::Flag out channels ms={1:s}'.format(step, msname))
if pipeline.enable_task(config, 'flag_scan'):
step = '{0:s}-scan-ms{1:d}'.format(wname, msiter)
recipe.add('cab/casa_flagdata', step,
{
"vis": msname,
"mode": 'manual',
"scan": config['flag_scan']['scans'],
"flagbackup": False,
"field": fields,
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}::Flag out channels ms={1:s}'.format(step, msname))
if pipeline.enable_task(config, 'flag_antennas'):
# step = '{0:s}-antennas-ms{1:d}'.format(wname, msiter)
antennas = [config['flag_antennas']['antennas']]
times = [config['flag_antennas']['timerange']]
found_valid_data = [0]
ensure = config['flag_antennas']['ensure_valid']
if times[0] == '':
ensure = False
if ensure:
if pipeline.startdate[i]:
antennas = config['flag_antennas']['antennas'].split(',')
times = config['flag_antennas']['timerange'].split(',')
while len(times) < len(antennas):
times.append(times[-1])
while len(found_valid_data) < len(antennas):
found_valid_data.append(0)
for nn,time_range in enumerate(times):
start_flagrange,end_flagrange=time_range.split('~')
flag_start = float(''.join(re.split('/|:', start_flagrange)))
flag_end = float(''.join(re.split('/|:', end_flagrange)))
if (flag_start <= pipeline.enddate[i]) and (pipeline.startdate[i] <= flag_end):
found_valid_data[nn] = 1
else:
raise ValueError("You wanted to ensure a valid time range but we could not find a start and end time")
for nn,antenna in enumerate(antennas):
antstep = 'ant-{0:s}-ms{1:d}-antsel{2:d}'.format(wname, i, nn)
if found_valid_data[nn] or not ensure:
recipe.add('cab/casa_flagdata', antstep,
{
"vis": msname,
"mode": 'manual',
"antenna": antenna,
"timerange": times[nn],
"field": fields,
"flagbackup": False,
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}:: Flagging bad antenna {2:s} ms={1:s}'.format(antstep, msname,antenna))
elif ensure and not found_valid_data[nn]:
caracal.log.warn(
'The following time selection has been made in the flag_antennas module of the flagging worker: "{1:s}". This selection would result in no valid data in {0:s}. This would lead to the FATAL error " The selected table has zero rows" in CASA/FLAGDATA. To avoid this error the corresponding cab {2:s} will not be added to the Stimela recipe of the flagging worker.'.format(msname, times[nn], antstep))
if pipeline.enable_task(config, 'flag_mask'):
step = '{0:s}-mask-ms{1:d}'.format(wname, msiter)
recipe.add('cab/rfimasker', step,
{
"msname": msname,
"mask": config['flag_mask']['mask'],
"accumulation_mode": 'or',
"uvrange": sdm.dismissable(config['flag_mask']['uvrange'] or None),
"memory": 4096,
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}:: Apply flag mask ms={1:s}'.format(step, msname))
if pipeline.enable_task(config, 'flag_manual'):
rules = config['flag_manual']['rules']
for irule, rule in enumerate(rules):
# a manual flagging rule has a pattern to match the MS name, followed by key:value pairs
rule_elements = rule.split()
if len(rule_elements) < 2 or not all(':' in el for el in rule_elements[1:]):
raise ValueError(f"invalid flag_manual rule '{rule}'")
pattern = rule_elements[0]
keywords = {tuple(elem.split(":", 1)) for elem in rule_elements[1:]}
# end of parsing block. Replace this with file if you like
if not fnmatch.fnmatch(msname, pattern):
continue
caracal.log.info(f"adding manual flagging rule for {pattern}")
step = f'{wname}-manual-ms{msiter}-{irule}'
args = {
"vis": msname,
"mode": 'manual',
"flagbackup": False,
"field": fields,
}
args.update(keywords)
recipe.add('cab/casa_flagdata', step, args,
input=pipeline.input,
output=pipeline.output,
label=f'{step}::Flag ms={msname} using {rule}')
if pipeline.enable_task(config, 'flag_rfi'):
step = '{0:s}-rfi-ms{1:d}'.format(wname, msiter)
if config['flag_rfi']["flagger"] == "aoflagger":
if config['flag_rfi']['aoflagger']['ensure_valid']:
ms_corr = msdict['CORR']['CORR_TYPE']
flag_corr=[]
with open('{0:s}/{1:s}'.format(pipeline.input,config['flag_rfi']['aoflagger']['strategy'])) as stdr:
for ss in stdr.readlines():
for pp in 'xx,xy,yx,yy,stokes-i,stokes-q,stokes-u,stokes-v'.split(','):
if '<on-{0:s}>1</on-{0:s}>'.format(pp) in ss: flag_corr.append(pp)
if ('stokes-u' in flag_corr and (('XY' not in ms_corr and 'RL' not in ms_corr) or ('YX' not in ms_corr and 'LR' not in ms_corr))) or\
('stokes-v' in flag_corr and (('XY' not in ms_corr and 'RR' not in ms_corr) or ('YX' not in ms_corr and 'LL' not in ms_corr))) or\
('stokes-i' in flag_corr and (('XX' not in ms_corr and 'RR' not in ms_corr) or ('YY' not in ms_corr and 'LL' not in ms_corr))) or\
('stokes-q' in flag_corr and (('XX' not in ms_corr and 'RL' not in ms_corr) or ('YY' not in ms_corr and 'LR' not in ms_corr))) or\
('xy' in flag_corr and ('XY' not in ms_corr and 'RL' not in ms_corr )) or\
('yx' in flag_corr and ('YX' not in ms_corr and 'LR' not in ms_corr)) or\
('xx' in flag_corr and ('XX' not in ms_corr and 'RR' not in ms_corr)) or\
('yy' in flag_corr and ('YY' not in ms_corr and 'LL' not in ms_corr)):
raise ValueError("The selected flagging strategy {0:s}/{1:s} will attempt to flag on {2:} but this is"\
" not compatible with the {3:} correlations available in {4:s}. To proceed you can edit the flagging"\
" strategy or, if you know what you are doing, disable aoflagger: ensure_valid.".format(
pipeline.input,config['flag_rfi']['aoflagger']['strategy'],flag_corr,ms_corr,msname))
recipe.add('cab/autoflagger', step,
{
"msname": msname,
"column": config['flag_rfi']['col'],
"fields": ",".join(map(str, field_ids)),
"strategy": config['flag_rfi']['aoflagger']['strategy'],
"indirect-read": True if config['flag_rfi']['aoflagger']['readmode'] == 'indirect' else False,
"memory-read": True if config['flag_rfi']['aoflagger']['readmode'] == 'memory' else False,
"auto-read-mode": True if config['flag_rfi']['aoflagger']['readmode'] == 'auto' else False,
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}:: AOFlagger auto-flagging flagging pass ms={1:s} fields={2:s}'.format(step, msname, fields))
elif config['flag_rfi']["flagger"] == "tricolour":
tricolour_strat=config['flag_rfi']['tricolour']['strategy']
if config['flag_rfi']['tricolour']['mode'] == 'auto':
bandwidth = msdict['SPW']['TOTAL_BANDWIDTH'][0]/10.0**6
caracal.log.info("Total Bandwidth = {0:} MHz".format(bandwidth))
if bandwidth <= 20.0:
caracal.log.info("Narrowband data detected, selecting appropriate flagging strategy")
tricolour_strat = config['flag_rfi']['tricolour']['strat_narrow']
caracal.log.info("Flagging strategy in use: {0:}".format(tricolour_strat))
recipe.add('cab/tricolour', step,
{
"ms": msname,
"data-column": config['flag_rfi']['col'],
"window-backend": config['flag_rfi']['tricolour']['backend'],
"field-names": fields,
"flagging-strategy": 'polarisation',
"config" : tricolour_strat,
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}:: Tricolour auto-flagging flagging pass ms={1:s} fields={2:s}'.format(step, msname, fields))
elif config['flag_rfi']["flagger"] == "tfcrop":
col = config['flag_rfi']['col'].split("_DATA")[0].lower()
recipe.add('cab/casa_flagdata', step,
{
"vis" : msname,
"datacolumn" : col,
"mode" : "tfcrop",
"field" : fields,
"usewindowstats" : config["flag_rfi"]["tfcrop"]["usewindowstats"],
"combinescans" : config["flag_rfi"]["tfcrop"]["combinescans"],
"flagdimension" : config["flag_rfi"]["tfcrop"]["flagdimension"],
"flagbackup" : False,
"timecutoff" : config["flag_rfi"]["tfcrop"]["timecutoff"],
"freqcutoff" : config["flag_rfi"]["tfcrop"]["freqcutoff"],
"correlation" : config["flag_rfi"]["tfcrop"]["correlation"],
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}:: Tfcrop auto-flagging flagging pass ms={1:s} fields={2:s}'.format(step, msname, fields))
else:
raise RuntimeError(
"Flagger, {0:s} is not available. Options are 'aoflagger, tricolour, tfcrop'.")
if pipeline.enable_task(config, 'inspect'):
step = '{0:s}-inspect-ms{1:d}'.format(wname,msiter)
if config['field'] == 'target':
field = '0'
else:
field = ",".join(map(str, utils.get_field_id(msdict, manfields.get_field(
pipeline, i, config['inspect']['field']).split(","))))
for f in field.split(','):
outlabel = '_{0:d}'.format(i) if len(field.split(',')) == 1 else '_{0:d}_{1:s}'.format(i,f)
recipe.add('cab/rfinder', step,
{
"msname": msname,
"field": int(f),
"plot_noise": "noise",
"RFInder_mode": "use_flags",
"outlabel": outlabel, # The output will be rfi_<pol>_<outlabel>
"polarization": config['inspect']['polarization'],
"spw_width": config['inspect']['spw_width'],
"time_step": config['inspect']['time_step'],
"time_enable": config['inspect']['time_enable'],
"spw_enable": config['inspect']['spw_enable'],
"1d_gif": config['inspect']['time_enable'],
"2d_gif": config['inspect']['time_enable'],
"altaz_gif": config['inspect']['spw_enable'],
"movies_in_report": config['inspect']['time_enable'] or config['spw_enable']
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}:: Investigate presence of rfi in ms={1:s}'.format(step, msname))
if pipeline.enable_task(config, 'summary'):
__label = config['label_in']
step = '{0:s}-summary-ms{1:d}'.format(wname,msiter)
recipe.add('cab/casa_flagdata', step,
{
"vis": msname,
"mode": 'summary',
"field": fields,
"flagbackup": False,
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}:: Flagging summary ms={1:s}'.format(step, msname))
recipe.run()
# Empty job que after execution
recipe.jobs = []
summary_log = glob.glob("{0:s}/log-flag-{1:s}-*.txt".format(pipeline.logs,
step))[0]
json_summary = manflags.get_json_flag_summary(pipeline, summary_log,
prefix_msbase, wname)
manflags.flag_summary_plots(pipeline, json_summary, prefix_msbase, wname, i)
substep = 'save-{0:s}-ms{1:d}'.format(flags_after_worker, msiter)
manflags.add_cflags(pipeline, recipe, flags_after_worker, msname, cab_name=substep, overwrite=config['overwrite_flagvers'])
msiter+=1
def solve(msname, msinfo, recipe, config, pipeline, iobs, prefix, label, ftype,
append_last_secondary=None, prev=None, prev_name=None, smodel=False):
"""
"""
gaintables = []
interps = []
fields = []
iters = {}
if prev and prev_name:
for item in config[ftype]['apply']:
gaintables.append("%s_%s.%s%d" % (prefix, prev_name, item, prev["iters"][item]))
ft = RULES[item]["field"]
fields.append(",".join(getattr(pipeline, ft)[iobs]))
interps.append(RULES[item]["interp"])
field = getattr(pipeline, CALS[ftype])[iobs]
order = config[ftype]["order"]
field_id = utils.get_field_id(msinfo, field)
def do_KGBF(i):
gtable_ = None
ftable_ = None
interp = RULES[term]["interp"]
params["refant"] = pipeline.refant[iobs] or '0'
params["solint"] = first_if_single(config[ftype]["solint"], i)
params["combine"] = first_if_single(config[ftype]["combine"], i).strip("'")
params["field"] = ",".join(field)
caltable = "%s_%s.%s%d" % (prefix, ftype, term, itern)
params["caltable"] = caltable + ":output"
my_term = term
did_I = 'I' in order[:i+1]
if not did_I and smodel and term in "KGF":
params["smodel"] = ["1", "0", "0", "0"]
# allow selection of band subset(s) for gaincal see #1204 on github issue tracker
if term in "KGF":
params["spw"] = config[ftype]["spw"]
if term == "B":
params["bandtype"] = term
params["solnorm"] = config[ftype]["b_solnorm"]
params["fillgaps"] = config[ftype]["b_fillgaps"]
params["uvrange"] = config["uvrange"]
elif term == "K":
params["gaintype"] = term
elif term in "FG":
my_term = ["F", "G"]
if term == "F":
# Never append to the original. Make a copy for each F that is needed
caltable_original = "%s_%s.G%d" % (prefix, prev_name, prev["iters"]["G"])
primary_G = "%s_%s_append-%d.G%d" % (prefix, prev_name, itern,prev["iters"]["G"])
caltable_path_original = os.path.join(pipeline.caltables, caltable_original)
caltable_path = os.path.join(pipeline.caltables, primary_G)
params["append"] = True
caltable = "%s_%s.F%d" % (prefix, ftype, itern)
params["caltable"] = primary_G + ":output"
params["gaintype"] = "G"
params["uvrange"] = config["uvrange"]
params["calmode"] = first_if_single(config[ftype]["calmode"], i).strip("'")
otf_apply = get_last_gain(gaintables, my_term=my_term)
if otf_apply:
params["gaintable"] = [gaintables[count]+":output" for count in otf_apply]
params["interp"] = [interps[count] for count in otf_apply]
params["gainfield"] = [fields[count] for count in otf_apply]
can_reuse = False
if config[ftype]["reuse_existing_gains"] and exists(pipeline.caltables, caltable):
# check if field is in gain table
fields_in_tab = set(table(os.path.join(pipeline.caltables, caltable), ack=False).getcol("FIELD_ID"))
if fields_in_tab.issubset(field_id):
can_reuse = True
if can_reuse:
caracal.log.info("Reusing existing gain table '%s' as requested" % caltable)
else:
if term == "F":
if os.path.exists(caltable_path):
shutil.rmtree(caltable_path)
cpstep = "copy_primary_gains_%s-%s-%d-%d-%s" % (name, label, itern, iobs, ftype)
recipe.add(shutil.copytree, cpstep, {
"src" : caltable_path_original,
"dst": caltable_path,
}, label="{0}:: Copy parimary gains".format(step))
recipe.add(RULES[term]["cab"], step,
copy.deepcopy(params),
input=pipeline.input, output=pipeline.caltables,
label="%s:: %s calibration" % (step, term))
if term == "F":
transfer_fluxscale(msname, recipe, primary_G+":output", caltable+":output", pipeline,
iobs, reference=pipeline.fluxscale_reference, label=label)
# Assume gains were plotted when they were created
if config[ftype]["plotgains"] and not can_reuse:
plotgains(recipe, pipeline, field_id if term != "F" else None, caltable, iobs, term=term)
fields.append(",".join(field))
interps.append(interp)
gaintables.append(caltable)
def do_IA(i):
if i==0:
raise RuntimeError("Have encountred an imaging/flagging request before any gains have been computed."\
"an I only makes sense after a G or K (usually both)."
"Please review your 'order' option in the self_cal:secondary section")
if not applied:
applycal(latest_KGBF_group, msname, recipe, gaintables,
interps, fields, CALS[ftype], pipeline, iobs,
calmode="calflag")
else:
caracal.log.info("Gains have already been applied using this exact set of gain tables and fields. Skipping unnecessary applycal step")
if term == "A":
if not set("KGBF").intersection(order[:i]):
raise RuntimeError("Have encountered a request to flag the secondary calibrator without any gain, bandpass or delay tables to apply first.")
step = "%s-%s-%d-%d-%s" % (name, label, itern, iobs, ftype)
params["mode"] = RULES[term]["mode"]
params["field"] = ",".join(field)
params["datacolumn"] = config[ftype]["flag"]["col"]
params["usewindowstats"] = config[ftype]["flag"]["usewindowstats"]
params["combinescans"] = config[ftype]["flag"]["combinescans"]
params["flagdimension"] = config[ftype]["flag"]["flagdimension"]
params["flagbackup"] = False
params["timecutoff"] = config[ftype]["flag"]["timecutoff"]
params["freqcutoff"] = config[ftype]["flag"]["freqcutoff"]
params["correlation"] = config[ftype]["flag"]["correlation"]
recipe.add(RULES[term]["cab"], step,
copy.deepcopy(params),
input=pipeline.input, output=pipeline.output,
label="%s::" % step)
else:
for fid in field_id:
step = "%s-%s-%d-%d-%s-field%d" % (name, label, itern, iobs, ftype, fid)
calimage = "%s-%s-I%d-%d-field%d:output" %(prefix, ftype, itern, iobs, fid)
recipe.add(RULES[term]["cab"], step, {
"msname" : msname,
"name" : calimage,
"size" : config[ftype]["image"]['npix'],
"scale" : config[ftype]["image"]['cell'],
"join-channels" : False if config[ftype]["image"]["nchans"]==1 else True,
"fit-spectral-pol" : config[ftype]["image"]["fit_spectral_pol"],
"channels-out" : config[ftype]["image"]['nchans'],
"auto-mask" : config[ftype]["image"]['auto_mask'],
"auto-threshold" : config[ftype]["image"]['auto_threshold'],
"local-rms-window" : config[ftype]["image"]['rms_window'],
"local-rms" : config[ftype]["image"]['local_rms'],
"padding" : config[ftype]["image"]['padding'],
"niter" : config[ftype]["image"]['niter'],
"weight" : config[ftype]["image"]["weight"],
"mgain" : config[ftype]["image"]['mgain'],
"field" : fid,
},
input=pipeline.input, output=pipeline.crosscal_continuum,
label="%s:: Image %s field" % (step, ftype))
nterms = len(order)
# terms that need an apply
groups_apply = list(filter(lambda g: g, re.findall("([AI]+)?", order)))
# terms that need a solve
groups_solve = list(filter(lambda g: g, re.findall("([KGBF]+)?", order)))
# Order has to start with solve group.
# TODO(sphe) in the philosophy of giving user enough roap to hang themselves
# Release II will allow both starting with I/A in case
# someone wants to apply primary gains to the secondary
n_apply = len(groups_apply)
n_solve = len(groups_solve)
groups = [None] * (n_apply + n_solve)
groups[::2] = groups_solve # even indices
groups[1::2] = groups_apply # odd indices
# no need to apply gains multiple when encountering consecutive terms that need to apply
applied = False
i = -1 #
for jj, group in enumerate(groups):
for g, term in enumerate(group):
i += 1
# if this is not the case, then something has gone horribly wrong
assert term == order[i]
if (jj % 2) == 0: # even counter is solve group
even = True
latest_KGBF_group = group
else:
latest_IA_group = group
even = False
if g == 0:
applied = False
else:
applied = True
name = RULES[term]["name"]
if term in iters:
iters[term] += 1
else:
iters[term] = 0
itern = iters[term]
params = {}
params["vis"] = msname
step = "%s-%s-%d-%d-%s" % (name, label, itern, iobs, ftype)
if even:
do_KGBF(i)
else:
do_IA(i)
return {
"gaintables" : gaintables,
"interps" : interps,
"iters" : iters,
"gainfield" : fields,
}
def worker(pipeline, recipe, config):
wname = pipeline.CURRENT_WORKER
flags_before_worker = '{0:s}_{1:s}_before'.format(pipeline.prefix, wname)
flags_after_worker = '{0:s}_{1:s}_after'.format(pipeline.prefix, wname)
label = config["label_cal"]
label_in = config["label_in"]
# loop over all MSs for this label
for i, msbase in enumerate(pipeline.msbasenames):
msname = pipeline.form_msname(msbase, label_in)
msinfo = pipeline.get_msinfo(msname)
prefix_msbase = f"{pipeline.prefix_msbases[i]}-{label}"
if {"gcal", "fcal", "target"}.intersection(config["apply_cal"]["applyto"]):
# Write/rewind flag versions
available_flagversions = manflags.get_flags(pipeline, msname)
if config['rewind_flags']['enable']:
if config['rewind_flags']['mode'] == 'reset_worker':
version = flags_before_worker
stop_if_missing = False
elif config['rewind_flags']['mode'] == 'rewind_to_version':
version = config['rewind_flags']['version']
if version == 'auto':
version = flags_before_worker
stop_if_missing = True
if version in available_flagversions:
if flags_before_worker in available_flagversions and available_flagversions.index(flags_before_worker) < available_flagversions.index(version) and not config['overwrite_flagvers']:
manflags.conflict('rewind_too_little', pipeline, wname, msname, config, flags_before_worker, flags_after_worker)
substep = 'version-{0:s}-ms{1:d}'.format(version, i)
manflags.restore_cflags(pipeline, recipe, version, msname, cab_name=substep)
if version != available_flagversions[-1]:
substep = 'delete-flag_versions-after-{0:s}-ms{1:d}'.format(version, i)
manflags.delete_cflags(pipeline, recipe,
available_flagversions[available_flagversions.index(version)+1],
msname, cab_name=substep)
if version != flags_before_worker:
substep = 'save-{0:s}-ms{1:d}'.format(flags_before_worker, i)
manflags.add_cflags(pipeline, recipe, flags_before_worker,
msname, cab_name=substep, overwrite=config['overwrite_flagvers'])
elif stop_if_missing:
manflags.conflict('rewind_to_non_existing', pipeline, wname, msname, config, flags_before_worker, flags_after_worker)
else:
substep = 'save-{0:s}-ms{1:d}'.format(flags_before_worker, i)
manflags.add_cflags(pipeline, recipe, flags_before_worker,
msname, cab_name=substep, overwrite=config['overwrite_flagvers'])
else:
if flags_before_worker in available_flagversions and not config['overwrite_flagvers']:
manflags.conflict('would_overwrite_bw', pipeline, wname, msname, config, flags_before_worker, flags_after_worker)
else:
substep = 'save-{0:s}-ms{1:d}'.format(flags_before_worker, i)
manflags.add_cflags(pipeline, recipe, flags_before_worker,
msname, cab_name=substep, overwrite=config['overwrite_flagvers'])
if len(pipeline.fcal[i]) > 1:
fluxscale_field = utils.observed_longest(msinfo, pipeline.fcal[i])
fluxscale_field_id = utils.get_field_id(msinfo, fluxscale_field)[0]
caracal.log.info("Found more than one flux calibrator."\
"Will use the one observed the logest (%s)." % fluxscale_field)
else:
fluxscale_field = pipeline.fcal[i][0]
fluxscale_field_id = utils.get_field_id(msinfo, fluxscale_field)[0]
pipeline.fluxscale_reference = fluxscale_field
if pipeline.enable_task(config, 'set_model'):
if config['set_model']['no_verify']:
opts = {
"vis": msname,
"field": fluxscale_field,
"scalebychan": True,
"usescratch": True,
}
else:
modelsky = utils.find_in_native_calibrators(msinfo, fluxscale_field, mode='sky')
modelpoint = utils.find_in_native_calibrators(msinfo, fluxscale_field, mode='mod')
standard = utils.find_in_casa_calibrators(msinfo, fluxscale_field)
if config['set_model']['meerkat_skymodel'] and modelsky:
# use local sky model of calibrator field if exists
opts = {
"skymodel": modelsky,
"msname": msname,
"field-id": utils.get_field_id(msinfo, fluxscale_field)[0],
"threads": config["set_model"]['threads'],
"mode": "simulate",
"tile-size": 128,
"column": "MODEL_DATA",
}
elif modelpoint: # spectral model if specified in our standard
opts = {
"vis": msname,
"field": fluxscale_field,
"standard": "manual",
"fluxdensity": modelpoint['I'],
"reffreq": '{0:f}GHz'.format(modelpoint['ref']/1e9),
"spix": [modelpoint[a] for a in 'abcd'],
"scalebychan": True,
"usescratch": True,
}
elif standard: # NRAO model otherwise
opts = {
"vis": msname,
"field": fluxscale_field,
"standard": standard,
"usescratch": True,
"scalebychan": True,
}
else:
raise RuntimeError('The flux calibrator field "{}" could not be '
'found in our database or in the CASA NRAO database'.format(fluxscale_field))
step = 'set_model_cal-{0:d}'.format(i)
cabtouse = 'cab/casa_setjy'
recipe.add(cabtouse if "skymodel" not in opts else 'cab/simulator', step,
opts,
input=pipeline.input,
output=pipeline.output,
label='{0:s}:: Set jansky ms={1:s}'.format(step, msname))
gcal_set = set(pipeline.gcal[i])
fcal_set = set(pipeline.fcal[i])
calmode = config["apply_cal"]["calmode"]
primary_order = config["primary"]["order"]
secondary_order = config["secondary"]["order"]
no_secondary = gcal_set == set() or len(gcal_set - fcal_set) == 0
if no_secondary:
primary_order = config["primary"]["order"]
primary = solve(msname, msinfo, recipe, config, pipeline, i,
prefix_msbase, label=label, ftype="primary")
caracal.log.info("Secondary calibrator is the same as the primary. Skipping fluxscale")
interps = primary["interps"]
gainfields = primary["gainfield"]
gaintables = primary["gaintables"]
if "bpcal" in config["apply_cal"]["applyto"] or "gcal" in config["apply_cal"]["applyto"]:
applycal(primary_order, msname, recipe, copy.deepcopy(gaintables), copy.deepcopy(interps),
"nearest", "bpcal", pipeline, i, calmode=calmode, label=label)
if "target" in config["apply_cal"]["applyto"]:
applycal(primary_order, msname, recipe, copy.deepcopy(gaintables), copy.deepcopy(interps),
"nearest", "target", pipeline, i, calmode=calmode, label=label)
else:
primary = solve(msname, msinfo, recipe, config, pipeline, i,
prefix_msbase, label=label, ftype="primary")
secondary = solve(msname, msinfo, recipe, config, pipeline, i,
prefix_msbase, label=label, ftype="secondary",
prev=primary, prev_name="primary", smodel=True)
interps = primary["interps"]
gaintables = primary["gaintables"]
if "bpcal" in config["apply_cal"]["applyto"]:
applycal(primary_order, msname, recipe, copy.deepcopy(gaintables), copy.deepcopy(interps),
"nearest", "bpcal", pipeline, i, calmode=calmode, label=label)
interps = secondary["interps"]
gainfields = secondary["gainfield"]
gaintables = secondary["gaintables"]
if "gcal" in config["apply_cal"]["applyto"]:
applycal(secondary_order, msname, recipe, copy.deepcopy(gaintables), interps,
gainfields, "gcal", pipeline, i, calmode=calmode, label=label)
if "target" in config["apply_cal"]["applyto"]:
applycal(secondary_order, msname, recipe, copy.deepcopy(gaintables), interps,
"nearest", "target", pipeline, i, calmode=calmode, label=label)
if {"gcal", "fcal", "target"}.intersection(config["apply_cal"]["applyto"]):
substep = 'save-{0:s}-ms{1:d}'.format(flags_after_worker, i)
manflags.add_cflags(pipeline, recipe, flags_after_worker, msname, cab_name=substep, overwrite=config['overwrite_flagvers'])
gt_final, itp_final, fd_final = get_caltab_final(primary_order if no_secondary else secondary_order,
copy.deepcopy(gaintables), interps, "nearest", "target")
applycal_recipes = []
calmodes = []
for ix,gt in enumerate(gt_final):
applycal_recipes.append(dict(zip(
['caltable', 'fldmap', 'interp'], [gt, fd_final[ix], itp_final[ix]])))
if '.K' in gt:
calmodes.append('delay_cal')
elif '.B' in gt:
calmodes.append('bp_cal')
elif '.F' in gt:
calmodes.append('transfer_fluxscale')
elif '.G' in gt:
calmodes.append('gain_cal')
callib_dir = "{}/callibs".format(
pipeline.caltables)
if not os.path.exists(callib_dir):
os.mkdir(callib_dir)
callib_dict = dict(zip(calmodes, applycal_recipes))
with open(os.path.join(callib_dir, f'callib_{prefix_msbase}.json'), 'w') as json_file:
json.dump(callib_dict, json_file)
if pipeline.enable_task(config, 'summary'):
step = 'summary-{0:s}-{1:d}'.format(label, i)
recipe.add('cab/casa_flagdata', step,
{
"vis" : msname,
"mode" : 'summary',
"field" : ",".join(set(pipeline.bpcal[i]+pipeline.fcal[i]+pipeline.gcal[i]))
},
input=pipeline.input,
output=pipeline.output,
label='{0:s}:: Flagging summary ms={1:s}'.format(step, msname))
recipe.run()
# Empty job que after execution
recipe.jobs = []
summary_log = glob.glob("{0:s}/log-{1:s}-{2:s}-*"
".txt".format(pipeline.logs, wname, step))[0]
json_summary = manflags.get_json_flag_summary(pipeline, summary_log, prefix_msbase, wname )
manflags.flag_summary_plots(pipeline, json_summary, prefix_msbase, wname, i)
def worker(pipeline, recipe, config):
recipe.msdir = pipeline.rawdatadir
recipe.output = pipeline.msdir
step = None
for i, (msname, msroot, prefix) in enumerate(zip(pipeline.msnames, pipeline.msbasenames, pipeline.prefix_msbases)):
# filenames generated
obsinfo = f'{msroot}-obsinfo.txt'
summary = f'{msroot}-summary.json'
elevplot = f'{msroot}-elevation-tracks.png'
if pipeline.enable_task(config, 'obsinfo'):
if config['obsinfo']['listobs']:
if os.path.exists(os.path.join(pipeline.msdir, obsinfo)):
caracal.log.info(f"obsinfo file {obsinfo} exists, not regenerating")
else:
step = f'listobs-ms{i}'
recipe.add('cab/casa_listobs', step,
{
"vis": msname,
"listfile": obsinfo,
"overwrite": True,
},
input=pipeline.input,
output=pipeline.msdir,
label='{0:s}:: Get observation information ms={1:s}'.format(step, msname))
if config['obsinfo']['summary_json']:
if os.path.exists(os.path.join(pipeline.msdir, summary)):
caracal.log.info(f"summary file {summary} exists, not regenerating")
else:
step = f'summary_json-ms{i}'
recipe.add('cab/msutils', step,
{
"msname": msname,
"command": 'summary',
"display": False,
"outfile": summary,
},
input=pipeline.input,
output=pipeline.msdir,
label='{0:s}:: Get observation information as a json file ms={1:s}'.format(step, msname))
if config['obsinfo']['vampirisms']:
step = 'vampirisms-ms{0:d}'.format(i)
recipe.add('cab/sunblocker', step,
{
"command": 'vampirisms',
"inset": msname,
"dryrun": True,
"nononsoleil": True,
"verb": True,
},
input=pipeline.input,
output=pipeline.msdir,
label='{0:s}:: Note sunrise and sunset'.format(step))
if pipeline.enable_task(config['obsinfo'], 'plotelev'):
if os.path.exists(os.path.join(pipeline.msdir, elevplot)):
caracal.log.info(f"elevation plot {elevplot} exists, not regenerating")
else:
step = "elevation-plots-ms{:d}".format(i)
if config['obsinfo']["plotelev"]["plotter"] in ["plotms"]:
recipe.add("cab/casa_plotms", step, {
"vis" : msname,
"xaxis" : "hourangle",
"yaxis" : "elevation",
"coloraxis" : "field",
"plotfile": elevplot,
"overwrite" : True,
},
input=pipeline.input,
output=pipeline.msdir,
label="{:s}:: Plotting elevation tracks".format(step))
elif config['obsinfo']["plotelev"]["plotter"] in ["owlcat"]:
recipe.add("cab/owlcat_plotelev", step, {
"msname" : msname,
"output-name" : elevplot
},
input=pipeline.input,
output=pipeline.msdir,
label="{:s}:: Plotting elevation tracks".format(step))
# if any steps at all were inserted, run the recipe
if step is not None:
recipe.run()
recipe.jobs = []
# initialse things
for item in 'xcal fcal bpcal gcal target refant'.split():
val = config[item]
for attr in ["", "_ra", "_dec", "_id"]:
setattr(pipeline, item+attr, repeat_val(val, pipeline.nobs))
setattr(pipeline, 'nchans', repeat_val(None,pipeline.nobs))
setattr(pipeline, 'firstchanfreq', repeat_val(None, pipeline.nobs))
setattr(pipeline, 'lastchanfreq', repeat_val(None, pipeline.nobs))
setattr(pipeline, 'chanwidth', repeat_val(None, pipeline.nobs))
setattr(pipeline, 'specframe', repeat_val(None, pipeline.nobs))
setattr(pipeline, 'startdate', repeat_val(None, pipeline.nobs))
setattr(pipeline, 'enddate', repeat_val(None, pipeline.nobs))
# Set antenna properties
#pipeline.Tsys_eta = config['Tsys_eta']
#pipeline.dish_diameter = config['dish_diameter']
for i, (msname, msroot, prefix) in enumerate(zip(pipeline.msnames, pipeline.msbasenames, pipeline.prefix_msbases)):
caracal.log.info(f"MS #{i}: {msname}")
msdict = pipeline.get_msinfo(msname)
obsinfo = f'{msroot}-obsinfo.txt'
summary = f'{msroot}-summary.json'
elevplot = f'{msroot}-elevation-tracks.png'
# copy these to obsinfo dir if needed
for filename in obsinfo, summary, elevplot:
src, dest = os.path.join(pipeline.msdir, filename), os.path.join(pipeline.obsinfo, filename)
if os.path.exists(src) and (not os.path.exists(dest) or os.path.getmtime(dest) < os.path.getmtime(src)):
caracal.log.info(f"generated new obsinfo/{filename}")
shutil.copy2(src, dest)
# get the actual date stamp for the start and end of the observations.
# !!!!!!! This info appears to not be present in the json file just the totals and start times (without slew times) so we'll get it from the txt file
with open(os.path.join(pipeline.msdir, obsinfo), 'r') as stdr:
content = stdr.readlines()
for line in content:
info_on_line = [x for x in line.split() if x != '']
if len(info_on_line) > 2:
if info_on_line[0].lower() == 'observed' and info_on_line[1].lower() == 'from':
calender_month_abbr = ['jan', 'feb', 'mar', 'apr', 'may','jun', 'jul', 'aug', 'sep', 'oct', 'nov',
'dec']
startdate,starttime =info_on_line[2].split('/')
hr,minute,sec = starttime.split(':')
day,month_abbr,year = startdate.split('-')
month_num = '{:02d}'.format(calender_month_abbr.index(month_abbr.lower())+1)
correct_date = ''.join([year,month_num,day,hr,minute,sec])
pipeline.startdate[i] = float(correct_date)
enddate,endtime =info_on_line[4].split('/')
hr,minute,sec = endtime.split(':')
day,month_abbr,year = enddate.split('-')
month_num = '{:02d}'.format(calender_month_abbr.index(month_abbr.lower())+1)
correct_date = ''.join([year,month_num,day,hr,minute,sec])
pipeline.enddate[i] = float(correct_date)
# get reference antenna LEAVING THIS LINE HERE
# FOR WHEN WE COME UP WITH A WAY TO AUTOSELECT
#if config.get('refant') == 'auto':
# pipeline.refant[i] = '0'
# Get channels in MS
spw = msdict['SPW']['NUM_CHAN']
pipeline.nchans[i] = spw
caracal.log.info(' {0:d} spectral windows, with NCHAN={1:s}'.format(
len(spw), ','.join(map(str, spw))))
# Get first chan, last chan, chan width
chfr = msdict['SPW']['CHAN_FREQ']
firstchanfreq = [ss[0] for ss in chfr]
lastchanfreq = [ss[-1] for ss in chfr]
chanwidth = [(ss[-1]-ss[0])/(len(ss)-1) for ss in chfr]
pipeline.firstchanfreq[i] = firstchanfreq
pipeline.lastchanfreq[i] = lastchanfreq
pipeline.chanwidth[i] = chanwidth
caracal.log.info(' CHAN_FREQ from {0:s} Hz to {1:s} Hz with average channel width of {2:s} Hz'.format(
','.join(map(str, firstchanfreq)), ','.join(map(str, lastchanfreq)), ','.join(map(str, chanwidth))))
if i == pipeline.nobs-1 and np.max(pipeline.chanwidth) > 0 and np.min(pipeline.chanwidth) < 0:
caracal.log.err('Some datasets have a positive channel increment, some negative. This will lead to errors. Exiting')
raise caracal.BadDataError("MSs with mixed channel ordering not supported")
# Get spectral frame
pipeline.specframe[i] = msdict['SPW']['MEAS_FREQ_REF']
targetinfo = msdict['FIELD']
intents = utils.categorize_fields(msdict)
# Save all fields in a list
all_fields = msdict["FIELD"]["NAME"]
# The order of fields here is important
for term in "target gcal fcal bpcal xcal".split():
conf_fields = getattr(pipeline, term)[i]
label, fields = intents[term]
label = ",".join(label)
# check if user set fields manually
if set(all_fields).intersection(conf_fields):
label = term
if term == 'target':
pipeline.target[i] = [value for value in getattr(pipeline, term)[i] if value in all_fields]
elif fields in [None, []]:
getattr(pipeline, term)[i] = []
continue
elif "all" in conf_fields:
getattr(pipeline, term)[i] = fields
elif "longest" in conf_fields:
f = utils.observed_longest(msdict, fields)
getattr(pipeline, term)[i] = [f]
elif "nearest" in conf_fields:
f = utils.select_gcal(msdict, fields, mode="nearest")
getattr(pipeline, term)[i] = [f]
else:
raise RuntimeError("Could not find field/selection {0}."\
" Please check the [observation_config.{1}] "\
"section of the config file".format(conf_fields, term))
# caracal.log.info(" ====================================")
caracal.log.info(f" {label}:")
# caracal.log.info(" ---------------------------------- ")
_ra = []
_dec = []
_fid = []
for f in getattr(pipeline, term)[i]:
fid = utils.get_field_id(msdict, f)[0]
targetpos = targetinfo['REFERENCE_DIR'][fid][0]
ra = targetpos[0]/np.pi*180
dec = targetpos[1]/np.pi*180
_ra.append(ra)
_dec.append(dec)
_fid.append(fid)
tobs = utils.field_observation_length(msdict, f)/60.0
caracal.log.info(
' {0:s} (ID={1:d}) : {2:.2f} minutes | RA={3:.2f} deg, Dec={4:.2f} deg'.format(f, fid, tobs, ra, dec))
getattr(pipeline, term+"_ra")[i] = _ra
getattr(pipeline, term+"_dec")[i] = _dec
getattr(pipeline, term+"_id")[i] = _fid