def _define_forest (ns):
# create Array object
num_antennas = 36 # for ASKAP simulation
xntd_list = [ str(i) for i in range(1,num_antennas+1) ];
array = Meow.IfrArray(ns,xntd_list,ms_uvw=True);
# create an Observation object
observation = Meow.Observation(ns);
# set global context
Meow.Context.set(array=array,observation=observation);
# create a source model and make list of corrupted sources
allsky = Meow.Patch(ns,'all',observation.phase_centre);
sources = sky_models.make_model(ns,"S");
for src in sources:
lm = src.direction.lm();
E = ns.E(src.name);
for p in array.stations():
pa= ns.ParAngle(p) << Meq.ParAngle(observation.phase_centre.radec(), array.xyz(p))
ns.CosPa(p) << Meq.Cos(pa)
ns.SinPa(p) << Meq.Sin(pa)
ns.rot_matrix(p) << Meq.Matrix22(ns.CosPa(p),-1.0 * ns.SinPa(p),ns.SinPa(p),ns.CosPa(p))
# compute "apparent" position of source per each antenna
lm_rot=ns.lm_rot(src.name,p) << Meq.MatrixMultiply(ns.rot_matrix(p),lm)
# compute E for apparent position
tdp_voltage_response(ns,src,p,E(p),lm_rot);
allsky.add(src.corrupt(E));
observed = allsky.visibilities();
# make some useful inspectors. Collect them into a list, since we need
# to give a list of 'post' nodes to make_sinks() below
pg = Bookmarks.Page("Inspectors",1,2);
inspectors = [];
inspectors.append(
Meow.StdTrees.vis_inspector(ns.inspect_observed,observed) );
pg.add(ns.inspect_observed,viewer="Collections Plotter");
Meow.StdTrees.make_sinks(ns,observed,spigots=False,post=inspectors);
def _define_forest(ns, **kw):
if parmtab is None:
raise RuntimeError, "please select a valid parmtable in compile-time options"
# make one MeqParm per each funklet
parmdef = Meq.Parm(table_name=parmtable_name)
parmnodes = dict([(name, (ns[name] << parmdef))
for name in parmtab.funklet_names()])
# reprocess flag is reset every time something changes in the parmnodes dict,
# so we simply repeat this loop until things stop changing
reprocess = True
while reprocess:
reprocess = False
# see if we have recipe for making compound objects based on last funklet name qualifier
nqlist = [
name.rsplit(':', 1) for name in parmnodes.iterkeys() if ':' in name
]
# set of basenames and set of all qualifiers
basenames = set([nq[0] for nq in nqlist])
quals = set([nq[1] for nq in nqlist])
for quallist, agg_func in aggregators:
# look for an aggregator whose qualifiers are a subset of the given set
if set(quallist) <= quals:
# for each basename, attempt to find funklets for every qualifier in the list
for basename in basenames:
fqnames = [basename + ':' + qq for qq in quallist]
funks = [parmnodes.get(name, None) for name in fqnames]
# if every basename:qq combination gives a valid funklet, call the aggregator
if None not in funks:
aggnode = ns[basename] << agg_func(*funks)
# remove aggregated funklets from the parmnodes dict
for name in fqnames:
del parmnodes[name]
# add new node to dict
parmnodes[basename] = aggnode
reprocess = True
# sort nodes by qualified name
nodes = list(parmnodes.iteritems())
nodes.sort(lambda a, b: ParmTables.cmp_qualified_names(a[0], b[0]))
nodes = [namenode[1] for namenode in nodes]
# make intermediate math node
pall = ns.all_parms_raw << Meq.Composer(
children=nodes, mt_polling=True, dims=[0])
pfm = ns.all_parms_fm << Meq.Mean(pall, reduction_axes="freq")
# ns.all_minus_fm << pall - pfm;
# ns.all_over_fm << pall/pfm;
# make root node
ns.root << Meq.ReqMux(
pfm
# ns.all_minus_fm,
# ns.all_over_fm,
# ns.fft_all_over_fm << Meq.FFTBrick(Meq.Selector(ns.all_over_fm,index=0,multi=True),axes_in=(hiid("time"),hiid("freq")),axes_out=(hiid("l"),hiid("m")))
)
Bookmarks.Page("Composite view").add(ns.all_parms_fm,
viewer="Collections Plotter")
# make bookmark folder
Bookmarks.make_node_folder("All parameters", nodes, sorted=True)
# close parmtable, else meqserver will be unable to get at it
parmtab.close()
# now make runtime options
global active_axes
active_axes = []
reg_domain_opts = []
for iaxis in range(mequtils.max_axis):
stats = parmtab.axis_stats(iaxis)
if not stats.empty():
active_axes.append((iaxis, stats.name, stats))
reg_domain_opts.append(
TDLOption("num_cells_%s" % stats.name,
"Number of grid points in %s" % stats.name,
[len(stats.cells)],
more=int))
TDLRuntimeMenu(
"View parameters over regularly gridded domain",
*(reg_domain_opts + [
TDLJob(_view_parameters_over_regularized_domain,
"View over gridded domain")
]))
def _define_forest(ns, parent=None, **kw):
if not mssel.msname:
raise RuntimeError("MS not set")
if run_purr:
Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.'])
# create Purr pipe
global purrpipe
purrpipe = Purr.Pipe.Pipe(mssel.msname)
# setup contexts from MS
mssel.setup_observation_context(ns)
array = Meow.Context.array
# make spigot nodes for data
mssel.enable_input_column(True)
spigots = array.spigots(corr=mssel.get_corr_index())
meqmaker.make_per_ifr_bookmarks(spigots, "Input visibilities")
# data tensor
ns.DT << Meq.Composer(
dims=[0], mt_polling=True, *[spigots(p, q) for p, q in array.ifrs()])
# list of model tensors
models = []
if read_ms_model:
mssel.enable_model_column(True)
model_spigots = array.spigots(column="PREDICT",
corr=mssel.get_corr_index())
meqmaker.make_per_ifr_bookmarks(model_spigots, "UV-model visibilities")
mtuv = ns.MT("uv") << Meq.Composer(
dims=[0],
mt_polling=True,
*[model_spigots(p, q) for p, q in array.ifrs()])
# predict tree using the MeqMaker
all_sources = meqmaker.get_source_list(ns)
dg_sources = deopts.enabled and dgsel.filter(all_sources)
if dg_sources:
# first group all sources without a diffgain on them
groups = [[src for src in all_sources if not src in dg_sources]]
# group diffgain-enabled sources by grouping tag
clusters = set(
[src.get_attr(diffgain_group, None) for src in dg_sources])
dg_groups = [(name, [
src for src in dg_sources if src.get_attr(diffgain_group) == name
]) for name in clusters if name]
# add sources without a grouping tag individually, as single-source groups
dg_groups += [(src.name, [src]) for src in dg_sources
if not src.get_attr(diffgain_group, None)]
# now sort by brightness
flux_dgg = [(sum(
[src.get_attr('Iapp', 0) or src.get_attr('I')
for src in dgg[1]]), dgg) for dgg in dg_groups]
from past.builtins import cmp
from functools import cmp_to_key
flux_dgg.sort(key=cmp_to_key(lambda a, b: cmp(b[0], a[0])))
diffgain_labels = [dgg[0] for flux, dgg in flux_dgg]
groups += [dgg[1] for flux, dgg in flux_dgg]
num_diffgains = len(flux_dgg)
# now make predict trees
for i, group in enumerate(groups):
MT = ns.MT(group[0].name if i else "all")
# first tensor is "MT", rest qualified by source names
predict = meqmaker.make_predict_tree(MT.Subscope(), sources=group)
MT << Meq.Composer(dims=[0],
mt_polling=True,
*[predict(p, q) for p, q in array.ifrs()])
# if reading an extra uv-model, add to first term
if not i and read_ms_model:
MT = ns.MT << Meq.Add(MT, mtuv)
models.append(MT)
print("Number of diffgain predict groups:", len(groups))
else:
diffgain_labels = []
num_diffgains = 0
predict = meqmaker.make_predict_tree(ns)
MT = ns.MT("all") << Meq.Composer(
dims=[0],
mt_polling=True,
*[predict(p, q) for p, q in array.ifrs()])
if read_ms_model:
MT = ns.MT << Meq.Add(MT, mtuv)
models.append(MT)
solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs()
downsample_subtiling = [
stefcal_downsample_timeint, stefcal_downsample_freqint
] if stefcal_downsample else [1, 1]
import Calico.OMS.StefCal.StefCal
kwopts = {}
gopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize)
bopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize)
deopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize)
ns.stefcal << Meq.PyNode(
class_name="StefCalNode",
module_name=Calico.OMS.StefCal.StefCal.__file__,
ifrs=["%s:%s" % (p, q) for p, q in array.ifrs()],
baselines=[
array.baseline(ip, iq) for (ip, p), (iq, q) in array.ifr_index()
],
solve_ifrs=["%s:%s" % (p, q) for p, q in solve_ifrs],
noise_per_chan=stefcal_noise_per_chan,
downsample_subtiling=downsample_subtiling,
num_major_loops=stefcal_nmajor,
regularization_factor=1e-6, #
rescale=stefcal_rescale,
init_from_previous=False,
critical_flag_threshold=critical_flag_threshold,
diffgain_labels=diffgain_labels,
# flagging options
output_flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT,
# IFR gain solution options
apply_ifr_gains=stefcal_ifr_gains,
solve_ifr_gains=(stefcal_ifr_gain_mode != MODE_SOLVE_APPLY),
reset_ifr_gains=stefcal_ifr_gain_reset,
save_ifr_gains=(stefcal_ifr_gain_mode == MODE_SOLVE_SAVE),
ifr_gain_table=stefcal_ifr_gain_table,
per_chan_ifr_gains=stefcal_per_chan_ifr_gains,
diag_ifr_gains=(stefcal_diagonal_ifr_gains == DIAGONLY),
residuals=(do_output == CORRECTED_RESIDUALS),
subtract_dgsrc=(do_output == CORRECTED_DATA_SUB),
verbose=stefcal_verbose,
children=[ns.DT] + models,
**kwopts)
inspectors = meqmaker.get_inspectors() or []
# make output bookmarks
nv = 0
for p, q in array.ifrs():
sel = ns.output_sel(p, q) << Meq.Selector(
ns.stefcal, index=list(range(nv, nv + 4)), multi=True)
ns.output(p, q) << Meq.Composer(sel, dims=[2, 2])
nv += 4
meqmaker.make_per_ifr_bookmarks(ns.output, "Output visibilities")
Bookmarks.Page("StefCal outputs").add(ns.stefcal, viewer="Record Browser")
if gopts.enabled and gopts.visualize and stefcal_visualize:
ns.stefcal_vis_G << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="G",
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
ns.stefcal_vis_G_avg << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="G",
freq_average=True,
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
Bookmarks.Page("StefCal G plotter").add(ns.stefcal_vis_G,
viewer="Result Plotter")
Bookmarks.Page("StefCal G inspector").add(ns.stefcal_vis_G_avg,
viewer="Collections Plotter")
inspectors += [ns.stefcal_vis_G, ns.stefcal_vis_G_avg]
if bopts.enabled and bopts.visualize and stefcal_visualize:
ns.stefcal_vis_B << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="B",
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
ns.stefcal_vis_B_avg << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="B",
freq_average=True,
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
Bookmarks.Page("StefCal B plotter").add(ns.stefcal_vis_B,
viewer="Result Plotter")
Bookmarks.Page("StefCal B inspector").add(ns.stefcal_vis_B_avg,
viewer="Collections Plotter")
inspectors += [ns.stefcal_vis_B, ns.stefcal_vis_B_avg]
if deopts.enabled and deopts.visualize and stefcal_visualize:
for i, label in enumerate(diffgain_labels):
vde = ns.stefcal_vis_dE(label) << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="dE:%s" % label,
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
vde_avg = ns.stefcal_vis_dE_avg(label) << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="dE:%s" % label,
freq_average=True,
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
Bookmarks.Page("StefCal dE:%s plotter" % label).add(
vde, viewer="Result Plotter")
Bookmarks.Page("StefCal dE:%s inspector" % label).add(
vde_avg, viewer="Collections Plotter")
inspectors += [vde, vde_avg]
# make sinks
StdTrees.make_sinks(ns,
ns.output,
spigots=spigots,
post=inspectors,
corr_index=mssel.get_corr_index())
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
# and close meqmaker -- this exports annotations, etc
meqmaker.close()
# add options to clear all solutions
from Calico.OMS.StefCal import StefCal
TDLRuntimeOption("stefcal_reset_all", "Remove all existing solutions",
False)
for opt in gopts, bopts, deopts:
if opt.enabled:
TDLRuntimeOption("reset",
"Remove existing %s solutions (%s)" %
(opt.label, os.path.basename(opt.table)),
False,
namespace=opt)
if stefcal_ifr_gains:
TDLRuntimeOption(
"stefcal_reset_ifr_gains",
"Remove existing interferometer errors (%s)" %
(os.path.basename(stefcal_ifr_gain_table)), False)
TDLRuntimeJob(_run_stefcal, "Run StefCal", job_id="stefcal")
def _define_forest(ns, parent=None, **kw):
if not mssel.msname:
raise RuntimeError('MS name not set')
mssel.setup_observation_context(ns)
array = Context.array
# Data and model input
if do_solve or output_type.need_data:
mssel.enable_input_column(True)
spigots = array.spigots(corr=mssel.get_corr_index())
meqmaker.make_per_ifr_bookmarks(spigots, 'Input visibilities')
else:
mssel.enable_input_column(False)
spigots = None
if do_solve or output_type.need_model:
predict = meqmaker.make_predict_tree(ns, uvdata=None)
else:
predict = None
# Data output
outputs = output_type.apply(ns, meqmaker, array.ifrs(), spigots, predict)
# Flagging
if flag_enable and output_type.flag_data:
flaggers = []
if flag_res is not None or flag_mean_res is not None:
for p, q in array.ifrs():
ns.absres(p, q) << Meq.Abs(outputs(p, q))
if flag_res is not None:
for p, q in array.ifrs():
ns.flagres(p, q) << Meq.ZeroFlagger(
ns.absres(p, q) - flag_res,
oper='gt',
flag_bit=MSUtils.FLAGMASK_OUTPUT)
flaggers.append(ns.flagres)
meqmaker.make_per_ifr_bookmarks(ns.flagres,
'Residual amplitude flags')
if flag_mean_res is not None:
ns.meanabsres << Meq.Mean(
*[ns.absres(p, q) for p, q in array.ifrs()])
ns.flagmeanres << Meq.ZeroFlagger(ns.meanabsres - flag_mean_res,
oper='gt',
flag_bit=MSUtils.FLAGMASK_OUTPUT)
Bookmarks.Page('Mean residual amplitude flags').add(
ns.flagmeanres, viewer='Result Plotter')
flaggers.append(lambda p, q: ns.flagmeanres)
if flaggers:
meqmaker.make_per_ifr_bookmarks(outputs,
output_type.desc + ' (unflagged)')
for p, q in array.ifrs():
ns.flagged(p, q) << Meq.MergeFlags(
outputs(p, q), *[f(p, q) for f in flaggers])
outputs = ns.flagged
meqmaker.make_per_ifr_bookmarks(outputs, output_type.desc)
# Solve trees
if do_solve:
# parse ifr specification
solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs()
if not solve_ifrs:
raise RuntimeError(
'No interferometers selected for calibration. '
'Check your ifr specification (under calibration options).')
lhs, rhs, weights, modulo = cal_quant.apply(solve_ifrs, predict,
spigots)
solve_tree = StdTrees.SolveTree(ns,
lhs,
solve_ifrs=solve_ifrs,
weights=weights,
modulo=modulo)
outputs = solve_tree.sequencers(inputs=rhs, outputs=outputs)
StdTrees.make_sinks(ns,
outputs,
spigots=spigots,
post=meqmaker.get_inspectors() or [],
corr_index=mssel.get_corr_index())
if not do_solve:
name = 'Generate ' + output_type.desc.lower()
comment = 'Generated ' + output_type.desc.lower()
def run_tree(mqs, parent, wait=False, **kw):
return mqs.execute(Context.vdm.name,
mssel.create_io_request(tile_size),
wait=wait)
doc = """Input data are sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
TDLRuntimeMenu(
name,
TDLOption('tile_size',
'Tile size, in timeslots', [10, 60, 120, 240],
more=int,
doc=doc),
TDLJob(run_tree, name, job_id='generate_visibilities'))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up
# during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu('Make an image', *imsel.option_list())
meqmaker.close()
