def init_nodes(self, ns, tags=None, label=""):
ifrs = Context.array.ifrs()
G = ns.gain
if not G(*ifrs[0]).initialized():
def1 = Meow.Parm(1, tags=tags)
def0 = Meow.Parm(0, tags=tags)
gains = []
for p, q in ifrs:
gg = []
for corr in Context.correlations:
if corr in Context.active_correlations:
cc = corr.lower()
gain_ri = [
resolve_parameter(cc, G(p, q, cc, "r"), def1, tags="ifr gain real"),
resolve_parameter(cc, G(p, q, cc, "i"), def0, tags="ifr gain imag"),
]
gg.append(G(p, q, cc) << Meq.ToComplex(*gain_ri))
gains += gain_ri
else:
gg.append(1)
G(p, q) << Meq.Matrix22(*gg)
pg_ifr_ampl = ParmGroup.ParmGroup(
label, gains, individual_toggles=False, table_name="%s.fmep" % label, bookmark=False
)
Bookmarks.make_node_folder(
"%s (interferometer-based gains)" % label, [G(p, q) for p, q in ifrs], sorted=True, nrow=2, ncol=2
)
ParmGroup.SolveJob("cal_%s" % label, "Calibrate %s (interferometer-based gains)" % label, pg_ifr_ampl)
return G
def compute_jones (self,jones,sources,stations=None,tags=None,label='',**kw):
stations = stations or Context.array.stations();
# figure out which sources to apply to
if self.subset:
srclist = self._subset_parser.parse_list(self.subset);
sources = [ sources[i] for i in srclist ];
# create parm definitions for each jones element
tags = NodeTags(tags) + "solvable";
rm_parm = Meq.Parm(0,tags=tags+"rm");
# loop over sources
for src in sources:
jj = jones(src);
jj("RM") << rm_parm;
fr = jj("fr") << jj("RM") / Meq.Sqr(Meq.Freq());
cos_fr = jj("cos") << Meq.Cos(fr);
sin_fr = jj("sin") << Meq.Sin(fr);
jj << Meq.Matrix22(cos_fr,-sin_fr,sin_fr,cos_fr);
for p in stations:
jj(p) << Meq.Identity(jj);
# make parmgroups for diagonal and off-diagonal terms
self.pg_fr = ParmGroup.ParmGroup(label+"_fr",
[ jones(src,"RM") for src in sources ],
table_name="%s_fr.fmep"%label,bookmark=False);
# make bookmarks
Bookmarks.make_node_folder("%s Faraday rotation"%label,
[ jones(src,"fr") for src in sources ],sorted=True);
# make solvejobs
ParmGroup.SolveJob("cal_"+label+"_fr","Calibrate %s"%label,self.pg_fr);
return jones;
def init_nodes (self,ns,tags=None,label=''):
ifrs = Context.array.ifrs();
G = ns.gain;
if not G(*ifrs[0]).initialized():
def1 = Meow.Parm(1,tags=tags);
def0 = Meow.Parm(0,tags=tags);
gains = [];
for p,q in ifrs:
gg = [];
for corr in Context.correlations:
if corr in Context.active_correlations:
cc = corr.lower();
gain_ri = [ resolve_parameter(cc,G(p,q,cc,'r'),def1,tags="ifr gain real"),
resolve_parameter(cc,G(p,q,cc,'i'),def0,tags="ifr gain imag") ];
gg.append(G(p,q,cc) << Meq.ToComplex(*gain_ri));
gains += gain_ri;
else:
gg.append(1);
G(p,q) << Meq.Matrix22(*gg);
pg_ifr_ampl = ParmGroup.ParmGroup(label,gains,
individual_toggles=False,
table_name="%s.fmep"%label,bookmark=False);
Bookmarks.make_node_folder("%s (interferometer-based gains)"%label,
[ G(p,q) for p,q in ifrs ],sorted=True,nrow=2,ncol=2);
ParmGroup.SolveJob("cal_%s"%label,
"Calibrate %s (interferometer-based gains)"%label,pg_ifr_ampl);
return G;
def _define_forest(ns):
# setup contexts from MS
mssel.setup_observation_context(ns);
array = Meow.Context.array;
observation = Meow.Context.observation;
stas = array.stations();
# make spigot nodes
spigots = spigots0 = array.spigots(corr=mssel.get_corr_index());
for p,q in array.ifrs():
spigots('abs',p,q) << Meq.Abs(spigots(p,q));
# ...and an inspector for them
Meow.StdTrees.vis_inspector(ns.inspector('input'),spigots,
bookmark="Inspect input visibilities");
Meow.StdTrees.vis_inspector(ns.inspector('ampl'),spigots('abs'),
bookmark="Inspect mean visibility amplitudes");
Bookmarks.make_node_folder("Input visibilities by baseline",
[ spigots(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2);
ns.inspectors << Meq.ReqMux(ns.inspector('input'),ns.inspector('ampl'));
ns.VisDataMux << Meq.VisDataMux(post=ns.inspectors);
# add imaging options
imsel = mssel.imaging_selector(npix=512,arcmin=120);
TDLRuntimeMenu("Make an image from this MS",*imsel.option_list());
def __init__ (self,label,members=[],name=None,
subgroups=None,
individual=False,individual_toggles=True,bookmark=True,
table_name="calibration.mep",table_in_ms=True,
**kw):
"""Creates a ParmGroup.
'label' is the label of the group
'members' is a list of MeqParm nodes
'name' is a descriptive name for the group. If not set, label is used.
'subgroups' is a list of Subgroup objects, if parms are to be subgrouped
'individual' is True if individual options are to be provided per parameter. Note that
this is incompatible with subgroups
'individual_toggles' is True if individual solvasbility toggles are to be provided in addition to
subgroups.
'bookmark' is True if bookmarks for the parameters are to be automatically provided
'table_name' is the name of a MEP table
'table_in_ms' is True if the table should reside inside the current MS (in which case the MS selector
from the global Meow.Context is invoked to get the name of the MS)
"""
self.label = label;
self.name = name or label;
# sort group members by name
self.nodes = sorted_nodes = self._sort_members(members);
# now collect list of subgroups
self.subgroups = subgroups or [];
self.individual_toggles = individual_toggles;
if subgroups and individual:
raise TypeError,"cannot combine subgroups and individual parameters";
self.parm_subgroups = dict();
for sg in self.subgroups:
for parm in sg.nodes:
self.parm_subgroups.setdefault(parm.name,[]).append(sg);
self.nodes = sorted_nodes;
# setup table names
self.table_name = table_name;
self.default_table_name = os.path.basename(table_name);
if table_in_ms and Context.mssel and Context.mssel.msname:
self.table_name = os.path.join(Context.mssel.msname,self.default_table_name);
self._table_in_ms = table_in_ms;
self._individual = individual;
# put table name into the parms
for node in self.nodes:
node.initrec().table_name = self.table_name;
# create bookmarks (if specified as a [W,H], it gives the number of parms to bookmark)
if bookmark:
if isinstance(bookmark,tuple) and len(bookmark) == 2:
ncol,nrow = bookmark;
else:
ncol,nrow = 2,3;
Bookmarks.make_node_folder("Parameters: %s"%label,
sorted_nodes,sorted=True,ncol=ncol,nrow=nrow);
# add ourselves to global list of parmgroups
global _all_parmgroups;
_all_parmgroups.append(self);
def __init__ (self,label,members=[],name=None,
subgroups=None,
individual=False,individual_toggles=True,bookmark=True,
table_name="calibration.mep",table_in_ms=True,
**kw):
"""Creates a ParmGroup.
'label' is the label of the group
'members' is a list of MeqParm nodes
'name' is a descriptive name for the group. If not set, label is used.
'subgroups' is a list of Subgroup objects, if parms are to be subgrouped
'individual' is True if individual options are to be provided per parameter. Note that
this is incompatible with subgroups
'individual_toggles' is True if individual solvasbility toggles are to be provided in addition to
subgroups.
'bookmark' is True if bookmarks for the parameters are to be automatically provided
'table_name' is the name of a MEP table
'table_in_ms' is True if the table should reside inside the current MS (in which case the MS selector
from the global Meow.Context is invoked to get the name of the MS)
"""
self.label = label;
self.name = name or label;
# sort group members by name
self.nodes = sorted_nodes = self._sort_members(members);
# now collect list of subgroups
self.subgroups = subgroups or [];
self.individual_toggles = individual_toggles;
if subgroups and individual:
raise TypeError("cannot combine subgroups and individual parameters");
self.parm_subgroups = dict();
for sg in self.subgroups:
for parm in sg.nodes:
self.parm_subgroups.setdefault(parm.name,[]).append(sg);
self.nodes = sorted_nodes;
# setup table names
self.table_name = table_name;
self.default_table_name = os.path.basename(table_name);
if table_in_ms and Context.mssel and Context.mssel.msname:
self.table_name = os.path.join(Context.mssel.msname,self.default_table_name);
self._table_in_ms = table_in_ms;
self._individual = individual;
# put table name into the parms
for node in self.nodes:
node.initrec().table_name = self.table_name;
# create bookmarks (if specified as a [W,H], it gives the number of parms to bookmark)
if bookmark:
if isinstance(bookmark,tuple) and len(bookmark) == 2:
ncol,nrow = bookmark;
else:
ncol,nrow = 2,3;
Bookmarks.make_node_folder("Parameters: %s"%label,
sorted_nodes,sorted=True,ncol=ncol,nrow=nrow);
# add ourselves to global list of parmgroups
global _all_parmgroups;
_all_parmgroups.append(self);
def _define_forest(ns, parent=None, **kw):
if run_purr:
Timba.TDL.GUI.log_message("starting purr")
Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.'])
mssel.setup_observation_context(ns)
array = Meow.Context.array
# make spigot nodes
outputs = spigots = array.spigots(corr=mssel.get_corr_index())
# ...and an inspector for them
Meow.StdTrees.vis_inspector(ns.inspector('input'),
spigots,
bookmark="Inspect input visibilities")
inspectors = [ns.inspector('input')]
Bookmarks.make_node_folder("Input visibilities by baseline",
[spigots(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
# add in-tree flaggers, if appropriate
if tree_flag_enable:
flaggers = []
# make nodes to take complex amplitude, if any flaggers are going to use them
if tree_flag_ampl is not None or tree_flag_mean_ampl is not None or tree_flag_mean_freq_ampl is not None:
# use matrix norm
if tree_flag_norm:
for p, q in array.ifrs():
node = outputs(p, q)
nsq = node('sq') << Meq.MatrixMultiply(
node,
node('conj') << Meq.ConjTranspose(node))
nsqtr = nsq(
'tr') << (nsq(11) << Meq.Selector(nsq, index=0)) + (
nsq(22) << Meq.Selector(nsq, index=3))
ns.ampl(p, q) << Meq.Sqrt(nsqtr('abs') << Meq.Abs(nsqtr))
# use per-element amplitudes
else:
for p, q in array.ifrs():
ns.ampl(p, q) << Meq.Abs(outputs(p, q))
# ...and an inspector for them
Meow.StdTrees.vis_inspector(
ns.inspector('ampl'),
ns.ampl,
bookmark="Inspect mean-freq visibility amplitudes")
inspectors.append(ns.inspector('ampl'))
# make flagger for amplitudes
if tree_flag_ampl is not None:
for p, q in array.ifrs():
ns.flagampl(p, q) << Meq.ZeroFlagger(
ns.ampl(p, q) - tree_flag_ampl,
oper='gt',
flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT)
flaggers.append(ns.flagampl)
# make flagger for mean-ifr amplitudes
if tree_flag_mean_ampl is not None:
ns.meanampl << Meq.Mean(*[ns.ampl(p, q) for p, q in array.ifrs()])
ns.flagmeanampl << Meq.ZeroFlagger(
ns.meanampl - tree_flag_mean_ampl,
oper='gt',
flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT)
Meow.Bookmarks.Page("Mean-ifr visibility amplitudes").add(
ns.meanampl, viewer="Result Plotter")
flaggers.append(lambda p, q: ns.flagmeanampl)
# make flagger for mean-fr-freq amplitudes
if tree_flag_mean_freq_ampl is not None:
ns.meanampl**Meq.Mean(*[ns.ampl(p, q) for p, q in array.ifrs()])
ns.meanfreqampl << Meq.Mean(ns.meanampl, reduction_axes="freq")
ns.flagmeanfreqampl << Meq.ZeroFlagger(
ns.meanfreqampl - tree_flag_mean_freq_ampl,
oper='gt',
flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT)
Meow.Bookmarks.Page("Mean-ifr-freq visibility amplitudes").add(
ns.meanfreqampl, viewer="Collections Plotter")
flaggers.append(lambda p, q: ns.flagmeanfreqampl)
# now merge all flaggers
if flaggers:
for p, q in array.ifrs():
ns.flagged(p, q) << Meq.MergeFlags(
outputs(p, q), *[fl(p, q) for fl in flaggers])
outputs = ns.flagged
Meow.StdTrees.vis_inspector(
ns.inspector('flagged'),
outputs,
bookmark="Inspect flagged visibilities")
inspectors.append(ns.inspector('flagged'))
Bookmarks.make_node_folder(
"Flagged visibilities by baseline",
[outputs(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
mssel.enable_write_flags(True)
treeflag_ms_opt.show(True)
if outputs is spigots:
ns.VisDataMux << Meq.VisDataMux(post=(
inspectors[0] if len(inspectors) < 2 else Meq.ReqMux(*inspectors)))
else:
Meow.StdTrees.make_sinks(ns, outputs, spigots=spigots, post=inspectors)
# add imaging options
imsel = mssel.imaging_selector(npix=512, arcmin=120)
TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
def compute_jones(self,
jones,
sources,
stations=None,
tags=None,
label='',
**kw):
stations = stations or Context.array.stations()
is_complex = self.matrix_type != "real"
# set up qualifier labels depending on polarization definition
if Context.observation.circular():
x, y, X, Y = 'r', 'l', 'R', 'L'
else:
x, y, X, Y = 'x', 'y', 'X', 'Y'
xx, xy, yx, yy = x + x, x + y, y + x, y + y
rxx, rxy, ryx, ryy = [q + ":r" for q in (xx, xy, yx, yy)]
ixx, ixy, iyx, iyy = [q + ":i" for q in (xx, xy, yx, yy)]
# prepare parm definitions for real and imag parts of diagonal and off-diagonal elements
tags = NodeTags(tags) + "solvable"
diag_pdefs = offdiag_pdefs = None
# loop over sources
parms_diag = []
parms_offdiag = []
subgroups_diag = []
subgroups_offdiag = []
# Define a function to put together a matrix element, depending on whether we're in real or complex mode.
# This is also responsible for appending parms to the appropriate parm and subgroup lists.
# Note that for the purely-real case we still create parms called 'J:xx:r' (and not 'J:xx' directly),
# this is to keep MEP tables mutually-compatible naming wise.
if self.matrix_type == "real":
def make_element(jj, parmdef, *parmlists):
parm = jj('r') << parmdef[0]
for plist in parmlists:
plist.append(parm)
return jj << Meq.Identity(parm)
else:
def make_element(jj, parmdef, *parmlists):
parms = [jj('r') << parmdef[0],
jj('i') << parmdef[1]]
for plist in parmlists:
plist += parms
return jj << Meq.ToComplex(*parms)
for src in sources:
sg = src.name if self.independent_solve else ''
# (re)create parm definitions.
if diag_pdefs is None or self.independent_solve:
diag_pdefs = (Meq.Parm(complex(self.init_diag).real,
tags=tags + "diag real",
solve_group=sg),
Meq.Parm(complex(self.init_diag).imag,
tags=tags + "diag imag",
solve_group=sg))
offdiag_pdfefs = (Meq.Parm(complex(self.init_offdiag).imag,
tags=tags + "offdiag real",
solve_group=sg),
Meq.Parm(complex(self.init_offdiag).imag,
tags=tags + "offdiag imag",
solve_group=sg))
# now loop to create nodes
sgdiag = []
sgoff = []
for p in stations:
jj = jones(src, p)
jj << Meq.Matrix22(
make_element(jj(xx), diag_pdefs, parms_diag, sgdiag),
make_element(jj(xy), offdiag_pdefs, parms_offdiag, sgoff)
if self._offdiag else 0,
make_element(jj(yx), offdiag_pdefs, parms_offdiag, sgoff)
if self._offdiag else 0,
make_element(jj(yy), diag_pdefs, parms_diag, sgdiag))
# add subgroup for this source
subgroups_diag.append(ParmGroup.Subgroup(src.name, sgdiag))
subgroups_offdiag.append(ParmGroup.Subgroup(src.name, sgoff))
# re-sort by name
from past.builtins import cmp
from functools import cmp_to_key
subgroups_diag.sort(key=cmp_to_key(lambda a, b: cmp(a.name, b.name)))
subgroups_offdiag.sort(
key=cmp_to_key(lambda a, b: cmp(a.name, b.name)))
# make parmgroups for diagonal and off-diagonal terms
self.pg_diag = ParmGroup.ParmGroup(label + "_diag",
parms_diag,
subgroups=subgroups_diag,
table_name="%s_diag.fmep" % label,
bookmark=False)
if self._offdiag:
self.pg_offdiag = ParmGroup.ParmGroup(
label + "_offdiag",
parms_offdiag,
subgroups=subgroups_offdiag,
table_name="%s_offdiag.fmep" % label,
bookmark=False)
# make bookmarks
Bookmarks.make_node_folder("%s diagonal terms" % label, [
jones(src, p, zz) for src in sources for p in stations
for zz in ["xx", "yy"]
],
sorted=True)
if self._offdiag:
Bookmarks.make_node_folder("%s off-diagonal terms" % label, [
jones(src, p, zz) for src in sources for p in stations
for zz in ["xy", "yx"]
],
sorted=True)
# make solvejobs
ParmGroup.SolveJob("cal_" + label + "_diag",
"Calibrate %s diagonal terms" % label, self.pg_diag)
if self._offdiag:
ParmGroup.SolveJob("cal_" + label + "_offdiag",
"Calibrate %s off-diagonal terms" % label,
self.pg_offdiag)
return jones
def _define_forest(ns, parent=None, **kw):
if run_purr:
Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.'])
# create Purr pipe
global purrpipe
purrpipe = Purr.Pipe.Pipe(mssel.msname)
# get antennas from MS
ANTENNAS = mssel.get_antenna_set(list(range(1, 15)))
array = Meow.IfrArray(ns, ANTENNAS, mirror_uvw=False)
stas = array.stations()
# get phase centre from MS, setup observation
observation = Meow.Observation(ns,
phase_centre=mssel.get_phase_dir(),
linear=mssel.is_linear_pol(),
circular=mssel.is_circular_pol())
Meow.Context.set(array, observation)
# get active correlations from MS
Meow.Context.active_correlations = mssel.get_correlations()
# make spigot nodes
spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index())
# ...and an inspector for them
StdTrees.vis_inspector(ns.inspector('input'),
spigots,
bookmark="Inspect input visibilities")
inspectors = [ns.inspector('input')]
Bookmarks.make_node_folder("Input visibilities by baseline",
[spigots(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
inspect_ifrs = array.ifrs()
if do_solve:
# filter solvable baselines by baseline length
solve_ifrs = []
antpos = mssel.ms_antenna_positions
if (min_baseline or max_baseline) and antpos is not None:
for (ip, p), (iq, q) in array.ifr_index():
baseline = math.sqrt(
((antpos[ip, :] - antpos[iq, :])**2).sum())
if (not min_baseline or baseline > min_baseline) and \
(not max_baseline or baseline < max_baseline):
solve_ifrs.append((p, q))
else:
solve_ifrs = array.ifrs()
inspect_ifrs = solve_ifrs
# make a predict tree using the MeqMaker
if do_solve or do_subtract:
predict = meqmaker.make_predict_tree(ns)
# make a ParmGroup and solve jobs for source parameters, if we have any
if do_solve:
parms = {}
for src in meqmaker.get_source_list(ns):
parms.update([(p.name, p) for p in src.get_solvables()])
if parms:
pg_src = ParmGroup.ParmGroup("source",
list(parms.values()),
table_name="sources.fmep",
individual=True,
bookmark=True)
# now make a solvejobs for the source
ParmGroup.SolveJob("cal_source", "Calibrate source model",
pg_src)
# make nodes to compute residuals
if do_subtract:
residuals = ns.residuals
for p, q in array.ifrs():
residuals(p, q) << spigots(p, q) - predict(p, q)
outputs = residuals
# and now we may need to correct the outputs
if do_correct:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns)
sky_correct = srcs and srcs[0]
else:
sky_correct = None
outputs = meqmaker.correct_uv_data(ns,
outputs,
sky_correct=sky_correct,
inspect_ifrs=inspect_ifrs)
# make solve trees
if do_solve:
# inputs to the solver are based on calibration type
# if calibrating visibilities, feed them to condeq directly
if cal_type == CAL.VIS:
observed = spigots
model = predict
# else take ampl/phase component
else:
model = ns.model
observed = ns.observed
if cal_type == CAL.AMPL:
for p, q in array.ifrs():
observed(p, q) << Meq.Abs(spigots(p, q))
model(p, q) << Meq.Abs(predict(p, q))
elif cal_type == CAL.LOGAMPL:
for p, q in array.ifrs():
observed(p, q) << Meq.Log(Meq.Abs(spigots(p, q)))
model(p, q) << Meq.Log(Meq.Abs(predict(p, q)))
elif cal_type == CAL.PHASE:
for p, q in array.ifrs():
observed(p, q) << 0
model(p, q) << Meq.Abs(predict(p, q)) * Meq.FMod(
Meq.Arg(spigots(p, q)) - Meq.Arg(predict(p, q)),
2 * math.pi)
else:
raise ValueError("unknown cal_type setting: " + str(cal_type))
# make a solve tree
solve_tree = StdTrees.SolveTree(ns, model, solve_ifrs=solve_ifrs)
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=observed, outputs=outputs)
# make sinks and vdm.
# The list of inspectors must be supplied here
inspectors += meqmaker.get_inspectors() or []
StdTrees.make_sinks(ns, outputs, spigots=spigots0, post=inspectors)
Bookmarks.make_node_folder("Corrected/residual visibilities by baseline",
[outputs(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
if not do_solve:
if do_subtract:
name = "Generate residuals"
comment = "Generated residual visibilities."
elif do_correct:
name = "Generate corrected data"
comment = "Generated corrected visibilities."
else:
name = None
if name:
# make a TDL job to runsthe tree
def run_tree(mqs, parent, **kw):
global tile_size
purrpipe.title("Calibrating").comment(comment)
mqs.execute(Meow.Context.vdm.name,
mssel.create_io_request(tile_size),
wait=False)
TDLRuntimeMenu(
name,
TDLOption(
'tile_size',
"Tile size, in timeslots", [10, 60, 120, 240],
more=int,
doc=
"""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
), TDLRuntimeJob(run_tree, name))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
# and close meqmaker -- this exports annotations, etc
meqmaker.close()
def _define_forest(ns, **kw):
if parmtab is None:
raise RuntimeError, "please select a valid parmtable in compile-time options"
# make one MeqParm per each funklet
parmdef = Meq.Parm(table_name=parmtable_name)
parmnodes = dict([(name, (ns[name] << parmdef))
for name in parmtab.funklet_names()])
# reprocess flag is reset every time something changes in the parmnodes dict,
# so we simply repeat this loop until things stop changing
reprocess = True
while reprocess:
reprocess = False
# see if we have recipe for making compound objects based on last funklet name qualifier
nqlist = [
name.rsplit(':', 1) for name in parmnodes.iterkeys() if ':' in name
]
# set of basenames and set of all qualifiers
basenames = set([nq[0] for nq in nqlist])
quals = set([nq[1] for nq in nqlist])
for quallist, agg_func in aggregators:
# look for an aggregator whose qualifiers are a subset of the given set
if set(quallist) <= quals:
# for each basename, attempt to find funklets for every qualifier in the list
for basename in basenames:
fqnames = [basename + ':' + qq for qq in quallist]
funks = [parmnodes.get(name, None) for name in fqnames]
# if every basename:qq combination gives a valid funklet, call the aggregator
if None not in funks:
aggnode = ns[basename] << agg_func(*funks)
# remove aggregated funklets from the parmnodes dict
for name in fqnames:
del parmnodes[name]
# add new node to dict
parmnodes[basename] = aggnode
reprocess = True
# sort nodes by qualified name
nodes = list(parmnodes.iteritems())
nodes.sort(lambda a, b: ParmTables.cmp_qualified_names(a[0], b[0]))
nodes = [namenode[1] for namenode in nodes]
# make intermediate math node
pall = ns.all_parms_raw << Meq.Composer(
children=nodes, mt_polling=True, dims=[0])
pfm = ns.all_parms_fm << Meq.Mean(pall, reduction_axes="freq")
# ns.all_minus_fm << pall - pfm;
# ns.all_over_fm << pall/pfm;
# make root node
ns.root << Meq.ReqMux(
pfm
# ns.all_minus_fm,
# ns.all_over_fm,
# ns.fft_all_over_fm << Meq.FFTBrick(Meq.Selector(ns.all_over_fm,index=0,multi=True),axes_in=(hiid("time"),hiid("freq")),axes_out=(hiid("l"),hiid("m")))
)
Bookmarks.Page("Composite view").add(ns.all_parms_fm,
viewer="Collections Plotter")
# make bookmark folder
Bookmarks.make_node_folder("All parameters", nodes, sorted=True)
# close parmtable, else meqserver will be unable to get at it
parmtab.close()
# now make runtime options
global active_axes
active_axes = []
reg_domain_opts = []
for iaxis in range(mequtils.max_axis):
stats = parmtab.axis_stats(iaxis)
if not stats.empty():
active_axes.append((iaxis, stats.name, stats))
reg_domain_opts.append(
TDLOption("num_cells_%s" % stats.name,
"Number of grid points in %s" % stats.name,
[len(stats.cells)],
more=int))
TDLRuntimeMenu(
"View parameters over regularly gridded domain",
*(reg_domain_opts + [
TDLJob(_view_parameters_over_regularized_domain,
"View over gridded domain")
]))
def _define_forest(ns,parent=None,**kw):
if run_purr:
Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']);
# create Purr pipe
global purrpipe;
purrpipe = Purr.Pipe.Pipe(mssel.msname);
# get antennas from MS
ANTENNAS = mssel.get_antenna_set(list(range(1,15)));
array = Meow.IfrArray(ns,ANTENNAS,mirror_uvw=False);
stas = array.stations();
# get phase centre from MS, setup observation
observation = Meow.Observation(ns,phase_centre=mssel.get_phase_dir(),
linear=mssel.is_linear_pol(),
circular=mssel.is_circular_pol());
Meow.Context.set(array,observation);
# get active correlations from MS
Meow.Context.active_correlations = mssel.get_correlations();
# make spigot nodes
spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index());
# ...and an inspector for them
StdTrees.vis_inspector(ns.inspector('input'),spigots,
bookmark="Inspect input visibilities");
inspectors = [ ns.inspector('input') ];
Bookmarks.make_node_folder("Input visibilities by baseline",
[ spigots(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2);
inspect_ifrs = array.ifrs();
if do_solve:
# filter solvable baselines by baseline length
solve_ifrs = [];
antpos = mssel.ms_antenna_positions;
if (min_baseline or max_baseline) and antpos is not None:
for (ip,p),(iq,q) in array.ifr_index():
baseline = math.sqrt(((antpos[ip,:]-antpos[iq,:])**2).sum());
if (not min_baseline or baseline > min_baseline) and \
(not max_baseline or baseline < max_baseline):
solve_ifrs.append((p,q));
else:
solve_ifrs = array.ifrs();
inspect_ifrs = solve_ifrs;
# make a predict tree using the MeqMaker
if do_solve or do_subtract:
predict = meqmaker.make_predict_tree(ns);
# make a ParmGroup and solve jobs for source parameters, if we have any
if do_solve:
parms = {};
for src in meqmaker.get_source_list(ns):
parms.update([(p.name,p) for p in src.get_solvables()]);
if parms:
pg_src = ParmGroup.ParmGroup("source",list(parms.values()),
table_name="sources.fmep",
individual=True,bookmark=True);
# now make a solvejobs for the source
ParmGroup.SolveJob("cal_source","Calibrate source model",pg_src);
# make nodes to compute residuals
if do_subtract:
residuals = ns.residuals;
for p,q in array.ifrs():
residuals(p,q) << spigots(p,q) - predict(p,q);
outputs = residuals;
# and now we may need to correct the outputs
if do_correct:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns);
sky_correct = srcs and srcs[0];
else:
sky_correct = None;
outputs = meqmaker.correct_uv_data(ns,outputs,sky_correct=sky_correct,inspect_ifrs=inspect_ifrs);
# make solve trees
if do_solve:
# inputs to the solver are based on calibration type
# if calibrating visibilities, feed them to condeq directly
if cal_type == CAL.VIS:
observed = spigots;
model = predict;
# else take ampl/phase component
else:
model = ns.model;
observed = ns.observed;
if cal_type == CAL.AMPL:
for p,q in array.ifrs():
observed(p,q) << Meq.Abs(spigots(p,q));
model(p,q) << Meq.Abs(predict(p,q));
elif cal_type == CAL.LOGAMPL:
for p,q in array.ifrs():
observed(p,q) << Meq.Log(Meq.Abs(spigots(p,q)));
model(p,q) << Meq.Log(Meq.Abs(predict(p,q)));
elif cal_type == CAL.PHASE:
for p,q in array.ifrs():
observed(p,q) << 0;
model(p,q) << Meq.Abs(predict(p,q))*Meq.FMod(Meq.Arg(spigots(p,q))-Meq.Arg(predict(p,q)),2*math.pi);
else:
raise ValueError("unknown cal_type setting: "+str(cal_type));
# make a solve tree
solve_tree = StdTrees.SolveTree(ns,model,solve_ifrs=solve_ifrs);
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=observed,outputs=outputs);
# make sinks and vdm.
# The list of inspectors must be supplied here
inspectors += meqmaker.get_inspectors() or [];
StdTrees.make_sinks(ns,outputs,spigots=spigots0,post=inspectors);
Bookmarks.make_node_folder("Corrected/residual visibilities by baseline",
[ outputs(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2);
if not do_solve:
if do_subtract:
name = "Generate residuals";
comment = "Generated residual visibilities.";
elif do_correct:
name = "Generate corrected data";
comment = "Generated corrected visibilities.";
else:
name = None;
if name:
# make a TDL job to runsthe tree
def run_tree (mqs,parent,**kw):
global tile_size;
purrpipe.title("Calibrating").comment(comment);
mqs.execute(Meow.Context.vdm.name,mssel.create_io_request(tile_size),wait=False);
TDLRuntimeMenu(name,
TDLOption('tile_size',"Tile size, in timeslots",[10,60,120,240],more=int,
doc="""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""),
TDLRuntimeJob(run_tree,name)
);
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False));
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options());
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size());
TDLRuntimeMenu("Make an image from this MS",*imsel.option_list());
# and close meqmaker -- this exports annotations, etc
meqmaker.close();
def compute_jones (self,jones,sources,stations=None,tags=None,label='',**kw):
stations = stations or Context.array.stations();
is_complex = self.matrix_type != "real";
# set up qualifier labels depending on polarization definition
if Context.observation.circular():
x,y,X,Y = 'r','l','R','L';
else:
x,y,X,Y = 'x','y','X','Y';
xx,xy,yx,yy = x+x,x+y,y+x,y+y;
rxx,rxy,ryx,ryy = [ q+":r" for q in xx,xy,yx,yy ];
ixx,ixy,iyx,iyy = [ q+":i" for q in xx,xy,yx,yy ];
# prepare parm definitions for real and imag parts of diagonal and off-diagonal elements
tags = NodeTags(tags) + "solvable";
diag_pdefs = offdiag_pdefs = None;
# loop over sources
parms_diag = [];
parms_offdiag = [];
subgroups_diag = [];
subgroups_offdiag = [];
# Define a function to put together a matrix element, depending on whether we're in real or complex mode.
# This is also responsible for appending parms to the appropriate parm and subgroup lists.
# Note that for the purely-real case we still create parms called 'J:xx:r' (and not 'J:xx' directly),
# this is to keep MEP tables mutually-compatible naming wise.
if self.matrix_type == "real":
def make_element (jj,parmdef,*parmlists):
parm = jj('r') << parmdef[0];
for plist in parmlists:
plist.append(parm);
return jj << Meq.Identity(parm);
else:
def make_element (jj,parmdef,*parmlists):
parms = [ jj('r') << parmdef[0],jj('i') << parmdef[1] ];
for plist in parmlists:
plist += parms;
return jj << Meq.ToComplex(*parms);
for src in sources:
sg = src.name if self.independent_solve else '';
# (re)create parm definitions.
if diag_pdefs is None or self.independent_solve:
diag_pdefs = ( Meq.Parm(complex(self.init_diag).real,tags=tags+"diag real",solve_group=sg),
Meq.Parm(complex(self.init_diag).imag,tags=tags+"diag imag",solve_group=sg) );
offdiag_pdfefs = ( Meq.Parm(complex(self.init_offdiag).imag,tags=tags+"offdiag real",solve_group=sg),
Meq.Parm(complex(self.init_offdiag).imag,tags=tags+"offdiag imag",solve_group=sg) );
# now loop to create nodes
sgdiag = [];
sgoff = [];
for p in stations:
jj = jones(src,p);
jj << Meq.Matrix22(
make_element(jj(xx),diag_pdefs,parms_diag,sgdiag),
make_element(jj(xy),offdiag_pdefs,parms_offdiag,sgoff) if self._offdiag else 0,
make_element(jj(yx),offdiag_pdefs,parms_offdiag,sgoff) if self._offdiag else 0,
make_element(jj(yy),diag_pdefs,parms_diag,sgdiag)
);
# add subgroup for this source
subgroups_diag.append(ParmGroup.Subgroup(src.name,sgdiag));
subgroups_offdiag.append(ParmGroup.Subgroup(src.name,sgoff));
# re-sort by name
subgroups_diag.sort(lambda a,b:cmp(a.name,b.name));
subgroups_offdiag.sort(lambda a,b:cmp(a.name,b.name));
# make parmgroups for diagonal and off-diagonal terms
self.pg_diag = ParmGroup.ParmGroup(label+"_diag",parms_diag,
subgroups=subgroups_diag,
table_name="%s_diag.fmep"%label,bookmark=False);
if self._offdiag:
self.pg_offdiag = ParmGroup.ParmGroup(label+"_offdiag",parms_offdiag,
subgroups=subgroups_offdiag,
table_name="%s_offdiag.fmep"%label,bookmark=False);
# make bookmarks
Bookmarks.make_node_folder("%s diagonal terms"%label,
[ jones(src,p,zz) for src in sources
for p in stations for zz in "xx","yy" ],sorted=True);
if self._offdiag:
Bookmarks.make_node_folder("%s off-diagonal terms"%label,
[ jones(src,p,zz) for src in sources
for p in stations for zz in "xy","yx" ],sorted=True);
# make solvejobs
ParmGroup.SolveJob("cal_"+label+"_diag","Calibrate %s diagonal terms"%label,self.pg_diag);
if self._offdiag:
ParmGroup.SolveJob("cal_"+label+"_offdiag","Calibrate %s off-diagonal terms"%label,self.pg_offdiag);
return jones;
def _define_forest(ns,parent=None,**kw):
if run_purr:
Timba.TDL.GUI.log_message("starting purr");
Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']);
mssel.setup_observation_context(ns);
array = Meow.Context.array;
# make spigot nodes
outputs = spigots = array.spigots(corr=mssel.get_corr_index());
# ...and an inspector for them
Meow.StdTrees.vis_inspector(ns.inspector('input'),spigots,
bookmark="Inspect input visibilities");
inspectors = [ ns.inspector('input') ];
Bookmarks.make_node_folder("Input visibilities by baseline",
[ spigots(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2);
# add in-tree flaggers, if appropriate
if tree_flag_enable:
flaggers = [];
# make nodes to take complex amplitude, if any flaggers are going to use them
if tree_flag_ampl is not None or tree_flag_mean_ampl is not None or tree_flag_mean_freq_ampl is not None:
# use matrix norm
if tree_flag_norm:
for p,q in array.ifrs():
node = outputs(p,q);
nsq = node('sq') << Meq.MatrixMultiply(node,node('conj') << Meq.ConjTranspose(node));
nsqtr = nsq('tr') << (nsq(11) << Meq.Selector(nsq,index=0)) + (nsq(22) << Meq.Selector(nsq,index=3));
ns.ampl(p,q) << Meq.Sqrt(nsqtr('abs') << Meq.Abs(nsqtr));
# use per-element amplitudes
else:
for p,q in array.ifrs():
ns.ampl(p,q) << Meq.Abs(outputs(p,q));
# ...and an inspector for them
Meow.StdTrees.vis_inspector(ns.inspector('ampl'),ns.ampl,bookmark="Inspect mean-freq visibility amplitudes");
inspectors.append(ns.inspector('ampl'));
# make flagger for amplitudes
if tree_flag_ampl is not None:
for p,q in array.ifrs():
ns.flagampl(p,q) << Meq.ZeroFlagger(ns.ampl(p,q)-tree_flag_ampl,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT);
flaggers.append(ns.flagampl);
# make flagger for mean-ifr amplitudes
if tree_flag_mean_ampl is not None:
ns.meanampl << Meq.Mean(*[ns.ampl(p,q) for p,q in array.ifrs()]);
ns.flagmeanampl << Meq.ZeroFlagger(ns.meanampl-tree_flag_mean_ampl,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT);
Meow.Bookmarks.Page("Mean-ifr visibility amplitudes").add(ns.meanampl,viewer="Result Plotter");
flaggers.append(lambda p,q:ns.flagmeanampl);
# make flagger for mean-fr-freq amplitudes
if tree_flag_mean_freq_ampl is not None:
ns.meanampl ** Meq.Mean(*[ns.ampl(p,q) for p,q in array.ifrs()]);
ns.meanfreqampl << Meq.Mean(ns.meanampl,reduction_axes="freq");
ns.flagmeanfreqampl << Meq.ZeroFlagger(ns.meanfreqampl-tree_flag_mean_freq_ampl,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT);
Meow.Bookmarks.Page("Mean-ifr-freq visibility amplitudes").add(ns.meanfreqampl,viewer="Collections Plotter");
flaggers.append(lambda p,q:ns.flagmeanfreqampl);
# now merge all flaggers
if flaggers:
for p,q in array.ifrs():
ns.flagged(p,q) << Meq.MergeFlags(outputs(p,q),*[fl(p,q) for fl in flaggers]);
outputs = ns.flagged;
Meow.StdTrees.vis_inspector(ns.inspector('flagged'),outputs,
bookmark="Inspect flagged visibilities");
inspectors.append(ns.inspector('flagged'));
Bookmarks.make_node_folder("Flagged visibilities by baseline",
[ outputs(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2);
mssel.enable_write_flags(True);
treeflag_ms_opt.show(True);
if outputs is spigots:
ns.VisDataMux << Meq.VisDataMux(post=(inspectors[0] if len(inspectors)<2 else Meq.ReqMux(*inspectors)));
else:
Meow.StdTrees.make_sinks(ns,outputs,spigots=spigots,post=inspectors);
# add imaging options
imsel = mssel.imaging_selector(npix=512,arcmin=120);
TDLRuntimeMenu("Make an image from this MS",*imsel.option_list());
subgroups = [
ParmGroup.Subgroup(X+X,[jones(p,zz) for zz in rxx,ixx for p in stations]),
ParmGroup.Subgroup(Y+Y,[jones(p,zz) for zz in ryy,iyy for p in stations]),
ParmGroup.Subgroup("real part",[jones(p,zz) for zz in rxx,ryy for p in stations]),
ParmGroup.Subgroup("imaginary part",[jones(p,zz) for zz in ixx,iyy for p in stations])
];
subgroups += [
ParmGroup.Subgroup("station %s"%p,[jones(p,zz) for zz in rxx,ixx,ryy,iyy ])
for p in stations
];
self.pg_diag = ParmGroup.ParmGroup(label+"_diag",
[ jones(p,zz) for zz in rxx,ixx,ryy,iyy for p in stations ],
subgroups = subgroups,
table_name="%s_diag.fmep"%label,bookmark=False);
# make bookmarks
Bookmarks.make_node_folder("%s diagonal terms"%label,
[ jones(p,zz) for p in stations for zz in xx,yy ],sorted=True);
# make solvejobs
ParmGroup.SolveJob("cal_"+label+"_diag","Calibrate %s diagonal terms"%label,self.pg_diag);
if self._offdiag:
subgroups = [
ParmGroup.Subgroup(X+Y,[jones(p,zz) for zz in rxy,ixy for p in stations]),
ParmGroup.Subgroup(Y+X,[jones(p,zz) for zz in ryx,iyx for p in stations]),
ParmGroup.Subgroup("real part",[jones(p,zz) for zz in rxy,ryx for p in stations]),
ParmGroup.Subgroup("imaginary part",[jones(p,zz) for zz in ixy,iyx for p in stations])
];
subgroups += [
ParmGroup.Subgroup("station %s"%p,[jones(p,zz) for zz in rxy,ixy,ryx,iyx ])
for p in stations
];
self.pg_offdiag = ParmGroup.ParmGroup(label+"_offdiag",
def _define_forest (ns,**kw):
if parmtab is None:
raise RuntimeError,"please select a valid parmtable in compile-time options";
# make one MeqParm per each funklet
parmdef = Meq.Parm(table_name=parmtable_name);
parmnodes = dict([(name,(ns[name] << parmdef)) for name in parmtab.funklet_names()]);
# reprocess flag is reset every time something changes in the parmnodes dict,
# so we simply repeat this loop until things stop changing
reprocess = True;
while reprocess:
reprocess = False;
# see if we have recipe for making compound objects based on last funklet name qualifier
nqlist = [ name.rsplit(':',1) for name in parmnodes.iterkeys() if ':' in name ];
# set of basenames and set of all qualifiers
basenames = set([nq[0] for nq in nqlist]);
quals = set([nq[1] for nq in nqlist]);
for quallist,agg_func in aggregators:
# look for an aggregator whose qualifiers are a subset of the given set
if set(quallist) <= quals:
# for each basename, attempt to find funklets for every qualifier in the list
for basename in basenames:
fqnames = [ basename+':'+qq for qq in quallist ];
funks = [ parmnodes.get(name,None) for name in fqnames ];
# if every basename:qq combination gives a valid funklet, call the aggregator
if None not in funks:
aggnode = ns[basename] << agg_func(*funks);
# remove aggregated funklets from the parmnodes dict
for name in fqnames:
del parmnodes[name];
# add new node to dict
parmnodes[basename] = aggnode;
reprocess = True;
# sort nodes by qualified name
nodes = list(parmnodes.iteritems());
nodes.sort(lambda a,b:ParmTables.cmp_qualified_names(a[0],b[0]));
nodes = [ namenode[1] for namenode in nodes ];
# make intermediate math node
pall = ns.all_parms_raw << Meq.Composer(children=nodes,mt_polling=True,dims=[0]);
pfm = ns.all_parms_fm << Meq.Mean(pall,reduction_axes="freq");
# ns.all_minus_fm << pall - pfm;
# ns.all_over_fm << pall/pfm;
# make root node
ns.root << Meq.ReqMux(
pfm
# ns.all_minus_fm,
# ns.all_over_fm,
# ns.fft_all_over_fm << Meq.FFTBrick(Meq.Selector(ns.all_over_fm,index=0,multi=True),axes_in=(hiid("time"),hiid("freq")),axes_out=(hiid("l"),hiid("m")))
);
Bookmarks.Page("Composite view").add(ns.all_parms_fm,viewer="Collections Plotter");
# make bookmark folder
Bookmarks.make_node_folder("All parameters",nodes,sorted=True);
# close parmtable, else meqserver will be unable to get at it
parmtab.close();
# now make runtime options
global active_axes;
active_axes = [];
reg_domain_opts = [];
for iaxis in range(mequtils.max_axis):
stats = parmtab.axis_stats(iaxis);
if not stats.empty():
active_axes.append((iaxis,stats.name,stats));
reg_domain_opts.append(
TDLOption("num_cells_%s"%stats.name,"Number of grid points in %s"%stats.name,[len(stats.cells)],more=int)
);
TDLRuntimeMenu("View parameters over regularly gridded domain",
*(reg_domain_opts + [TDLJob(_view_parameters_over_regularized_domain,"View over gridded domain")]));