def compute_jones (self,jones,sources,stations=None,tags=None,label='',**kw):
stations = stations or Context.array.stations();
# figure out which sources to apply to
if self.subset:
srclist = self._subset_parser.parse_list(self.subset);
sources = [ sources[i] for i in srclist ];
# create parm definitions for each jones element
tags = NodeTags(tags) + "solvable";
rm_parm = Meq.Parm(0,tags=tags+"rm");
# loop over sources
for src in sources:
jj = jones(src);
jj("RM") << rm_parm;
fr = jj("fr") << jj("RM") / Meq.Sqr(Meq.Freq());
cos_fr = jj("cos") << Meq.Cos(fr);
sin_fr = jj("sin") << Meq.Sin(fr);
jj << Meq.Matrix22(cos_fr,-sin_fr,sin_fr,cos_fr);
for p in stations:
jj(p) << Meq.Identity(jj);
# make parmgroups for diagonal and off-diagonal terms
self.pg_fr = ParmGroup.ParmGroup(label+"_fr",
[ jones(src,"RM") for src in sources ],
table_name="%s_fr.fmep"%label,bookmark=False);
# make bookmarks
Bookmarks.make_node_folder("%s Faraday rotation"%label,
[ jones(src,"fr") for src in sources ],sorted=True);
# make solvejobs
ParmGroup.SolveJob("cal_"+label+"_fr","Calibrate %s"%label,self.pg_fr);
return jones;
def init_nodes(self, ns, tags=None, label=""):
ifrs = Context.array.ifrs()
G = ns.gain
if not G(*ifrs[0]).initialized():
def1 = Meow.Parm(1, tags=tags)
def0 = Meow.Parm(0, tags=tags)
gains = []
for p, q in ifrs:
gg = []
for corr in Context.correlations:
if corr in Context.active_correlations:
cc = corr.lower()
gain_ri = [
resolve_parameter(cc, G(p, q, cc, "r"), def1, tags="ifr gain real"),
resolve_parameter(cc, G(p, q, cc, "i"), def0, tags="ifr gain imag"),
]
gg.append(G(p, q, cc) << Meq.ToComplex(*gain_ri))
gains += gain_ri
else:
gg.append(1)
G(p, q) << Meq.Matrix22(*gg)
pg_ifr_ampl = ParmGroup.ParmGroup(
label, gains, individual_toggles=False, table_name="%s.fmep" % label, bookmark=False
)
Bookmarks.make_node_folder(
"%s (interferometer-based gains)" % label, [G(p, q) for p, q in ifrs], sorted=True, nrow=2, ncol=2
)
ParmGroup.SolveJob("cal_%s" % label, "Calibrate %s (interferometer-based gains)" % label, pg_ifr_ampl)
return G
def init_nodes (self,ns,tags=None,label=''):
ifrs = Context.array.ifrs();
G = ns.gain;
if not G(*ifrs[0]).initialized():
def1 = Meow.Parm(1,tags=tags);
def0 = Meow.Parm(0,tags=tags);
gains = [];
for p,q in ifrs:
gg = [];
for corr in Context.correlations:
if corr in Context.active_correlations:
cc = corr.lower();
gain_ri = [ resolve_parameter(cc,G(p,q,cc,'r'),def1,tags="ifr gain real"),
resolve_parameter(cc,G(p,q,cc,'i'),def0,tags="ifr gain imag") ];
gg.append(G(p,q,cc) << Meq.ToComplex(*gain_ri));
gains += gain_ri;
else:
gg.append(1);
G(p,q) << Meq.Matrix22(*gg);
pg_ifr_ampl = ParmGroup.ParmGroup(label,gains,
individual_toggles=False,
table_name="%s.fmep"%label,bookmark=False);
Bookmarks.make_node_folder("%s (interferometer-based gains)"%label,
[ G(p,q) for p,q in ifrs ],sorted=True,nrow=2,ncol=2);
ParmGroup.SolveJob("cal_%s"%label,
"Calibrate %s (interferometer-based gains)"%label,pg_ifr_ampl);
return G;
def _define_forest(ns):
# setup contexts from MS
mssel.setup_observation_context(ns);
array = Meow.Context.array;
observation = Meow.Context.observation;
stas = array.stations();
# make spigot nodes
spigots = spigots0 = array.spigots(corr=mssel.get_corr_index());
for p,q in array.ifrs():
spigots('abs',p,q) << Meq.Abs(spigots(p,q));
# ...and an inspector for them
Meow.StdTrees.vis_inspector(ns.inspector('input'),spigots,
bookmark="Inspect input visibilities");
Meow.StdTrees.vis_inspector(ns.inspector('ampl'),spigots('abs'),
bookmark="Inspect mean visibility amplitudes");
Bookmarks.make_node_folder("Input visibilities by baseline",
[ spigots(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2);
ns.inspectors << Meq.ReqMux(ns.inspector('input'),ns.inspector('ampl'));
ns.VisDataMux << Meq.VisDataMux(post=ns.inspectors);
# add imaging options
imsel = mssel.imaging_selector(npix=512,arcmin=120);
TDLRuntimeMenu("Make an image from this MS",*imsel.option_list());
def __init__ (self,label,members=[],name=None,
subgroups=None,
individual=False,individual_toggles=True,bookmark=True,
table_name="calibration.mep",table_in_ms=True,
**kw):
"""Creates a ParmGroup.
'label' is the label of the group
'members' is a list of MeqParm nodes
'name' is a descriptive name for the group. If not set, label is used.
'subgroups' is a list of Subgroup objects, if parms are to be subgrouped
'individual' is True if individual options are to be provided per parameter. Note that
this is incompatible with subgroups
'individual_toggles' is True if individual solvasbility toggles are to be provided in addition to
subgroups.
'bookmark' is True if bookmarks for the parameters are to be automatically provided
'table_name' is the name of a MEP table
'table_in_ms' is True if the table should reside inside the current MS (in which case the MS selector
from the global Meow.Context is invoked to get the name of the MS)
"""
self.label = label;
self.name = name or label;
# sort group members by name
self.nodes = sorted_nodes = self._sort_members(members);
# now collect list of subgroups
self.subgroups = subgroups or [];
self.individual_toggles = individual_toggles;
if subgroups and individual:
raise TypeError,"cannot combine subgroups and individual parameters";
self.parm_subgroups = dict();
for sg in self.subgroups:
for parm in sg.nodes:
self.parm_subgroups.setdefault(parm.name,[]).append(sg);
self.nodes = sorted_nodes;
# setup table names
self.table_name = table_name;
self.default_table_name = os.path.basename(table_name);
if table_in_ms and Context.mssel and Context.mssel.msname:
self.table_name = os.path.join(Context.mssel.msname,self.default_table_name);
self._table_in_ms = table_in_ms;
self._individual = individual;
# put table name into the parms
for node in self.nodes:
node.initrec().table_name = self.table_name;
# create bookmarks (if specified as a [W,H], it gives the number of parms to bookmark)
if bookmark:
if isinstance(bookmark,tuple) and len(bookmark) == 2:
ncol,nrow = bookmark;
else:
ncol,nrow = 2,3;
Bookmarks.make_node_folder("Parameters: %s"%label,
sorted_nodes,sorted=True,ncol=ncol,nrow=nrow);
# add ourselves to global list of parmgroups
global _all_parmgroups;
_all_parmgroups.append(self);
def __init__ (self,label,members=[],name=None,
subgroups=None,
individual=False,individual_toggles=True,bookmark=True,
table_name="calibration.mep",table_in_ms=True,
**kw):
"""Creates a ParmGroup.
'label' is the label of the group
'members' is a list of MeqParm nodes
'name' is a descriptive name for the group. If not set, label is used.
'subgroups' is a list of Subgroup objects, if parms are to be subgrouped
'individual' is True if individual options are to be provided per parameter. Note that
this is incompatible with subgroups
'individual_toggles' is True if individual solvasbility toggles are to be provided in addition to
subgroups.
'bookmark' is True if bookmarks for the parameters are to be automatically provided
'table_name' is the name of a MEP table
'table_in_ms' is True if the table should reside inside the current MS (in which case the MS selector
from the global Meow.Context is invoked to get the name of the MS)
"""
self.label = label;
self.name = name or label;
# sort group members by name
self.nodes = sorted_nodes = self._sort_members(members);
# now collect list of subgroups
self.subgroups = subgroups or [];
self.individual_toggles = individual_toggles;
if subgroups and individual:
raise TypeError("cannot combine subgroups and individual parameters");
self.parm_subgroups = dict();
for sg in self.subgroups:
for parm in sg.nodes:
self.parm_subgroups.setdefault(parm.name,[]).append(sg);
self.nodes = sorted_nodes;
# setup table names
self.table_name = table_name;
self.default_table_name = os.path.basename(table_name);
if table_in_ms and Context.mssel and Context.mssel.msname:
self.table_name = os.path.join(Context.mssel.msname,self.default_table_name);
self._table_in_ms = table_in_ms;
self._individual = individual;
# put table name into the parms
for node in self.nodes:
node.initrec().table_name = self.table_name;
# create bookmarks (if specified as a [W,H], it gives the number of parms to bookmark)
if bookmark:
if isinstance(bookmark,tuple) and len(bookmark) == 2:
ncol,nrow = bookmark;
else:
ncol,nrow = 2,3;
Bookmarks.make_node_folder("Parameters: %s"%label,
sorted_nodes,sorted=True,ncol=ncol,nrow=nrow);
# add ourselves to global list of parmgroups
global _all_parmgroups;
_all_parmgroups.append(self);
def _define_forest (ns):
# create Array object
num_antennas = 36 # for ASKAP simulation
xntd_list = [ str(i) for i in range(1,num_antennas+1) ];
array = Meow.IfrArray(ns,xntd_list,ms_uvw=True);
# create an Observation object
observation = Meow.Observation(ns);
# set global context
Meow.Context.set(array=array,observation=observation);
# create a source model and make list of corrupted sources
allsky = Meow.Patch(ns,'all',observation.phase_centre);
sources = sky_models.make_model(ns,"S");
for src in sources:
lm = src.direction.lm();
E = ns.E(src.name);
for p in array.stations():
pa= ns.ParAngle(p) << Meq.ParAngle(observation.phase_centre.radec(), array.xyz(p))
ns.CosPa(p) << Meq.Cos(pa)
ns.SinPa(p) << Meq.Sin(pa)
ns.rot_matrix(p) << Meq.Matrix22(ns.CosPa(p),-1.0 * ns.SinPa(p),ns.SinPa(p),ns.CosPa(p))
# compute "apparent" position of source per each antenna
lm_rot=ns.lm_rot(src.name,p) << Meq.MatrixMultiply(ns.rot_matrix(p),lm)
# compute E for apparent position
tdp_voltage_response(ns,src,p,E(p),lm_rot);
allsky.add(src.corrupt(E));
observed = allsky.visibilities();
# make some useful inspectors. Collect them into a list, since we need
# to give a list of 'post' nodes to make_sinks() below
pg = Bookmarks.Page("Inspectors",1,2);
inspectors = [];
inspectors.append(
Meow.StdTrees.vis_inspector(ns.inspect_observed,observed) );
pg.add(ns.inspect_observed,viewer="Collections Plotter");
Meow.StdTrees.make_sinks(ns,observed,spigots=False,post=inspectors);
def compute_jones(self,
jones,
sources,
stations=None,
tags=None,
label='',
**kw):
stations = stations or Context.array.stations()
is_complex = self.matrix_type != "real"
# set up qualifier labels depending on polarization definition
if Context.observation.circular():
x, y, X, Y = 'r', 'l', 'R', 'L'
else:
x, y, X, Y = 'x', 'y', 'X', 'Y'
xx, xy, yx, yy = x + x, x + y, y + x, y + y
rxx, rxy, ryx, ryy = [q + ":r" for q in (xx, xy, yx, yy)]
ixx, ixy, iyx, iyy = [q + ":i" for q in (xx, xy, yx, yy)]
# prepare parm definitions for real and imag parts of diagonal and off-diagonal elements
tags = NodeTags(tags) + "solvable"
diag_pdefs = offdiag_pdefs = None
# loop over sources
parms_diag = []
parms_offdiag = []
subgroups_diag = []
subgroups_offdiag = []
# Define a function to put together a matrix element, depending on whether we're in real or complex mode.
# This is also responsible for appending parms to the appropriate parm and subgroup lists.
# Note that for the purely-real case we still create parms called 'J:xx:r' (and not 'J:xx' directly),
# this is to keep MEP tables mutually-compatible naming wise.
if self.matrix_type == "real":
def make_element(jj, parmdef, *parmlists):
parm = jj('r') << parmdef[0]
for plist in parmlists:
plist.append(parm)
return jj << Meq.Identity(parm)
else:
def make_element(jj, parmdef, *parmlists):
parms = [jj('r') << parmdef[0],
jj('i') << parmdef[1]]
for plist in parmlists:
plist += parms
return jj << Meq.ToComplex(*parms)
for src in sources:
sg = src.name if self.independent_solve else ''
# (re)create parm definitions.
if diag_pdefs is None or self.independent_solve:
diag_pdefs = (Meq.Parm(complex(self.init_diag).real,
tags=tags + "diag real",
solve_group=sg),
Meq.Parm(complex(self.init_diag).imag,
tags=tags + "diag imag",
solve_group=sg))
offdiag_pdfefs = (Meq.Parm(complex(self.init_offdiag).imag,
tags=tags + "offdiag real",
solve_group=sg),
Meq.Parm(complex(self.init_offdiag).imag,
tags=tags + "offdiag imag",
solve_group=sg))
# now loop to create nodes
sgdiag = []
sgoff = []
for p in stations:
jj = jones(src, p)
jj << Meq.Matrix22(
make_element(jj(xx), diag_pdefs, parms_diag, sgdiag),
make_element(jj(xy), offdiag_pdefs, parms_offdiag, sgoff)
if self._offdiag else 0,
make_element(jj(yx), offdiag_pdefs, parms_offdiag, sgoff)
if self._offdiag else 0,
make_element(jj(yy), diag_pdefs, parms_diag, sgdiag))
# add subgroup for this source
subgroups_diag.append(ParmGroup.Subgroup(src.name, sgdiag))
subgroups_offdiag.append(ParmGroup.Subgroup(src.name, sgoff))
# re-sort by name
from past.builtins import cmp
from functools import cmp_to_key
subgroups_diag.sort(key=cmp_to_key(lambda a, b: cmp(a.name, b.name)))
subgroups_offdiag.sort(
key=cmp_to_key(lambda a, b: cmp(a.name, b.name)))
# make parmgroups for diagonal and off-diagonal terms
self.pg_diag = ParmGroup.ParmGroup(label + "_diag",
parms_diag,
subgroups=subgroups_diag,
table_name="%s_diag.fmep" % label,
bookmark=False)
if self._offdiag:
self.pg_offdiag = ParmGroup.ParmGroup(
label + "_offdiag",
parms_offdiag,
subgroups=subgroups_offdiag,
table_name="%s_offdiag.fmep" % label,
bookmark=False)
# make bookmarks
Bookmarks.make_node_folder("%s diagonal terms" % label, [
jones(src, p, zz) for src in sources for p in stations
for zz in ["xx", "yy"]
],
sorted=True)
if self._offdiag:
Bookmarks.make_node_folder("%s off-diagonal terms" % label, [
jones(src, p, zz) for src in sources for p in stations
for zz in ["xy", "yx"]
],
sorted=True)
# make solvejobs
ParmGroup.SolveJob("cal_" + label + "_diag",
"Calibrate %s diagonal terms" % label, self.pg_diag)
if self._offdiag:
ParmGroup.SolveJob("cal_" + label + "_offdiag",
"Calibrate %s off-diagonal terms" % label,
self.pg_offdiag)
return jones
def _define_forest(ns, parent=None, **kw):
if run_purr:
Timba.TDL.GUI.log_message("starting purr")
Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.'])
mssel.setup_observation_context(ns)
array = Meow.Context.array
# make spigot nodes
outputs = spigots = array.spigots(corr=mssel.get_corr_index())
# ...and an inspector for them
Meow.StdTrees.vis_inspector(ns.inspector('input'),
spigots,
bookmark="Inspect input visibilities")
inspectors = [ns.inspector('input')]
Bookmarks.make_node_folder("Input visibilities by baseline",
[spigots(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
# add in-tree flaggers, if appropriate
if tree_flag_enable:
flaggers = []
# make nodes to take complex amplitude, if any flaggers are going to use them
if tree_flag_ampl is not None or tree_flag_mean_ampl is not None or tree_flag_mean_freq_ampl is not None:
# use matrix norm
if tree_flag_norm:
for p, q in array.ifrs():
node = outputs(p, q)
nsq = node('sq') << Meq.MatrixMultiply(
node,
node('conj') << Meq.ConjTranspose(node))
nsqtr = nsq(
'tr') << (nsq(11) << Meq.Selector(nsq, index=0)) + (
nsq(22) << Meq.Selector(nsq, index=3))
ns.ampl(p, q) << Meq.Sqrt(nsqtr('abs') << Meq.Abs(nsqtr))
# use per-element amplitudes
else:
for p, q in array.ifrs():
ns.ampl(p, q) << Meq.Abs(outputs(p, q))
# ...and an inspector for them
Meow.StdTrees.vis_inspector(
ns.inspector('ampl'),
ns.ampl,
bookmark="Inspect mean-freq visibility amplitudes")
inspectors.append(ns.inspector('ampl'))
# make flagger for amplitudes
if tree_flag_ampl is not None:
for p, q in array.ifrs():
ns.flagampl(p, q) << Meq.ZeroFlagger(
ns.ampl(p, q) - tree_flag_ampl,
oper='gt',
flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT)
flaggers.append(ns.flagampl)
# make flagger for mean-ifr amplitudes
if tree_flag_mean_ampl is not None:
ns.meanampl << Meq.Mean(*[ns.ampl(p, q) for p, q in array.ifrs()])
ns.flagmeanampl << Meq.ZeroFlagger(
ns.meanampl - tree_flag_mean_ampl,
oper='gt',
flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT)
Meow.Bookmarks.Page("Mean-ifr visibility amplitudes").add(
ns.meanampl, viewer="Result Plotter")
flaggers.append(lambda p, q: ns.flagmeanampl)
# make flagger for mean-fr-freq amplitudes
if tree_flag_mean_freq_ampl is not None:
ns.meanampl**Meq.Mean(*[ns.ampl(p, q) for p, q in array.ifrs()])
ns.meanfreqampl << Meq.Mean(ns.meanampl, reduction_axes="freq")
ns.flagmeanfreqampl << Meq.ZeroFlagger(
ns.meanfreqampl - tree_flag_mean_freq_ampl,
oper='gt',
flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT)
Meow.Bookmarks.Page("Mean-ifr-freq visibility amplitudes").add(
ns.meanfreqampl, viewer="Collections Plotter")
flaggers.append(lambda p, q: ns.flagmeanfreqampl)
# now merge all flaggers
if flaggers:
for p, q in array.ifrs():
ns.flagged(p, q) << Meq.MergeFlags(
outputs(p, q), *[fl(p, q) for fl in flaggers])
outputs = ns.flagged
Meow.StdTrees.vis_inspector(
ns.inspector('flagged'),
outputs,
bookmark="Inspect flagged visibilities")
inspectors.append(ns.inspector('flagged'))
Bookmarks.make_node_folder(
"Flagged visibilities by baseline",
[outputs(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
mssel.enable_write_flags(True)
treeflag_ms_opt.show(True)
if outputs is spigots:
ns.VisDataMux << Meq.VisDataMux(post=(
inspectors[0] if len(inspectors) < 2 else Meq.ReqMux(*inspectors)))
else:
Meow.StdTrees.make_sinks(ns, outputs, spigots=spigots, post=inspectors)
# add imaging options
imsel = mssel.imaging_selector(npix=512, arcmin=120)
TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
def _define_forest(ns, parent=None, **kw):
if run_purr:
Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.'])
# create Purr pipe
global purrpipe
purrpipe = Purr.Pipe.Pipe(mssel.msname)
# get antennas from MS
ANTENNAS = mssel.get_antenna_set(list(range(1, 15)))
array = Meow.IfrArray(ns, ANTENNAS, mirror_uvw=False)
stas = array.stations()
# get phase centre from MS, setup observation
observation = Meow.Observation(ns,
phase_centre=mssel.get_phase_dir(),
linear=mssel.is_linear_pol(),
circular=mssel.is_circular_pol())
Meow.Context.set(array, observation)
# get active correlations from MS
Meow.Context.active_correlations = mssel.get_correlations()
# make spigot nodes
spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index())
# ...and an inspector for them
StdTrees.vis_inspector(ns.inspector('input'),
spigots,
bookmark="Inspect input visibilities")
inspectors = [ns.inspector('input')]
Bookmarks.make_node_folder("Input visibilities by baseline",
[spigots(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
inspect_ifrs = array.ifrs()
if do_solve:
# filter solvable baselines by baseline length
solve_ifrs = []
antpos = mssel.ms_antenna_positions
if (min_baseline or max_baseline) and antpos is not None:
for (ip, p), (iq, q) in array.ifr_index():
baseline = math.sqrt(
((antpos[ip, :] - antpos[iq, :])**2).sum())
if (not min_baseline or baseline > min_baseline) and \
(not max_baseline or baseline < max_baseline):
solve_ifrs.append((p, q))
else:
solve_ifrs = array.ifrs()
inspect_ifrs = solve_ifrs
# make a predict tree using the MeqMaker
if do_solve or do_subtract:
predict = meqmaker.make_predict_tree(ns)
# make a ParmGroup and solve jobs for source parameters, if we have any
if do_solve:
parms = {}
for src in meqmaker.get_source_list(ns):
parms.update([(p.name, p) for p in src.get_solvables()])
if parms:
pg_src = ParmGroup.ParmGroup("source",
list(parms.values()),
table_name="sources.fmep",
individual=True,
bookmark=True)
# now make a solvejobs for the source
ParmGroup.SolveJob("cal_source", "Calibrate source model",
pg_src)
# make nodes to compute residuals
if do_subtract:
residuals = ns.residuals
for p, q in array.ifrs():
residuals(p, q) << spigots(p, q) - predict(p, q)
outputs = residuals
# and now we may need to correct the outputs
if do_correct:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns)
sky_correct = srcs and srcs[0]
else:
sky_correct = None
outputs = meqmaker.correct_uv_data(ns,
outputs,
sky_correct=sky_correct,
inspect_ifrs=inspect_ifrs)
# make solve trees
if do_solve:
# inputs to the solver are based on calibration type
# if calibrating visibilities, feed them to condeq directly
if cal_type == CAL.VIS:
observed = spigots
model = predict
# else take ampl/phase component
else:
model = ns.model
observed = ns.observed
if cal_type == CAL.AMPL:
for p, q in array.ifrs():
observed(p, q) << Meq.Abs(spigots(p, q))
model(p, q) << Meq.Abs(predict(p, q))
elif cal_type == CAL.LOGAMPL:
for p, q in array.ifrs():
observed(p, q) << Meq.Log(Meq.Abs(spigots(p, q)))
model(p, q) << Meq.Log(Meq.Abs(predict(p, q)))
elif cal_type == CAL.PHASE:
for p, q in array.ifrs():
observed(p, q) << 0
model(p, q) << Meq.Abs(predict(p, q)) * Meq.FMod(
Meq.Arg(spigots(p, q)) - Meq.Arg(predict(p, q)),
2 * math.pi)
else:
raise ValueError("unknown cal_type setting: " + str(cal_type))
# make a solve tree
solve_tree = StdTrees.SolveTree(ns, model, solve_ifrs=solve_ifrs)
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=observed, outputs=outputs)
# make sinks and vdm.
# The list of inspectors must be supplied here
inspectors += meqmaker.get_inspectors() or []
StdTrees.make_sinks(ns, outputs, spigots=spigots0, post=inspectors)
Bookmarks.make_node_folder("Corrected/residual visibilities by baseline",
[outputs(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
if not do_solve:
if do_subtract:
name = "Generate residuals"
comment = "Generated residual visibilities."
elif do_correct:
name = "Generate corrected data"
comment = "Generated corrected visibilities."
else:
name = None
if name:
# make a TDL job to runsthe tree
def run_tree(mqs, parent, **kw):
global tile_size
purrpipe.title("Calibrating").comment(comment)
mqs.execute(Meow.Context.vdm.name,
mssel.create_io_request(tile_size),
wait=False)
TDLRuntimeMenu(
name,
TDLOption(
'tile_size',
"Tile size, in timeslots", [10, 60, 120, 240],
more=int,
doc=
"""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
), TDLRuntimeJob(run_tree, name))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
# and close meqmaker -- this exports annotations, etc
meqmaker.close()
def _define_forest(ns, parent=None, **kw):
if not mssel.msname:
raise RuntimeError('MS name not set')
mssel.setup_observation_context(ns)
array = Context.array
# Data and model input
if do_solve or output_type.need_data:
mssel.enable_input_column(True)
spigots = array.spigots(corr=mssel.get_corr_index())
meqmaker.make_per_ifr_bookmarks(spigots, 'Input visibilities')
else:
mssel.enable_input_column(False)
spigots = None
if do_solve or output_type.need_model:
predict = meqmaker.make_predict_tree(ns, uvdata=None)
else:
predict = None
# Data output
outputs = output_type.apply(ns, meqmaker, array.ifrs(), spigots, predict)
# Flagging
if flag_enable and output_type.flag_data:
flaggers = []
if flag_res is not None or flag_mean_res is not None:
for p, q in array.ifrs():
ns.absres(p, q) << Meq.Abs(outputs(p, q))
if flag_res is not None:
for p, q in array.ifrs():
ns.flagres(p, q) << Meq.ZeroFlagger(
ns.absres(p, q) - flag_res,
oper='gt',
flag_bit=MSUtils.FLAGMASK_OUTPUT)
flaggers.append(ns.flagres)
meqmaker.make_per_ifr_bookmarks(ns.flagres,
'Residual amplitude flags')
if flag_mean_res is not None:
ns.meanabsres << Meq.Mean(
*[ns.absres(p, q) for p, q in array.ifrs()])
ns.flagmeanres << Meq.ZeroFlagger(ns.meanabsres - flag_mean_res,
oper='gt',
flag_bit=MSUtils.FLAGMASK_OUTPUT)
Bookmarks.Page('Mean residual amplitude flags').add(
ns.flagmeanres, viewer='Result Plotter')
flaggers.append(lambda p, q: ns.flagmeanres)
if flaggers:
meqmaker.make_per_ifr_bookmarks(outputs,
output_type.desc + ' (unflagged)')
for p, q in array.ifrs():
ns.flagged(p, q) << Meq.MergeFlags(
outputs(p, q), *[f(p, q) for f in flaggers])
outputs = ns.flagged
meqmaker.make_per_ifr_bookmarks(outputs, output_type.desc)
# Solve trees
if do_solve:
# parse ifr specification
solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs()
if not solve_ifrs:
raise RuntimeError(
'No interferometers selected for calibration. '
'Check your ifr specification (under calibration options).')
lhs, rhs, weights, modulo = cal_quant.apply(solve_ifrs, predict,
spigots)
solve_tree = StdTrees.SolveTree(ns,
lhs,
solve_ifrs=solve_ifrs,
weights=weights,
modulo=modulo)
outputs = solve_tree.sequencers(inputs=rhs, outputs=outputs)
StdTrees.make_sinks(ns,
outputs,
spigots=spigots,
post=meqmaker.get_inspectors() or [],
corr_index=mssel.get_corr_index())
if not do_solve:
name = 'Generate ' + output_type.desc.lower()
comment = 'Generated ' + output_type.desc.lower()
def run_tree(mqs, parent, wait=False, **kw):
return mqs.execute(Context.vdm.name,
mssel.create_io_request(tile_size),
wait=wait)
doc = """Input data are sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
TDLRuntimeMenu(
name,
TDLOption('tile_size',
'Tile size, in timeslots', [10, 60, 120, 240],
more=int,
doc=doc),
TDLJob(run_tree, name, job_id='generate_visibilities'))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up
# during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu('Make an image', *imsel.option_list())
meqmaker.close()
def _define_forest(ns,parent=None,**kw):
if run_purr:
Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']);
# create Purr pipe
global purrpipe;
purrpipe = Purr.Pipe.Pipe(mssel.msname);
# get antennas from MS
ANTENNAS = mssel.get_antenna_set(list(range(1,15)));
array = Meow.IfrArray(ns,ANTENNAS,mirror_uvw=False);
stas = array.stations();
# get phase centre from MS, setup observation
observation = Meow.Observation(ns,phase_centre=mssel.get_phase_dir(),
linear=mssel.is_linear_pol(),
circular=mssel.is_circular_pol());
Meow.Context.set(array,observation);
# get active correlations from MS
Meow.Context.active_correlations = mssel.get_correlations();
# make spigot nodes
spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index());
# ...and an inspector for them
StdTrees.vis_inspector(ns.inspector('input'),spigots,
bookmark="Inspect input visibilities");
inspectors = [ ns.inspector('input') ];
Bookmarks.make_node_folder("Input visibilities by baseline",
[ spigots(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2);
inspect_ifrs = array.ifrs();
if do_solve:
# filter solvable baselines by baseline length
solve_ifrs = [];
antpos = mssel.ms_antenna_positions;
if (min_baseline or max_baseline) and antpos is not None:
for (ip,p),(iq,q) in array.ifr_index():
baseline = math.sqrt(((antpos[ip,:]-antpos[iq,:])**2).sum());
if (not min_baseline or baseline > min_baseline) and \
(not max_baseline or baseline < max_baseline):
solve_ifrs.append((p,q));
else:
solve_ifrs = array.ifrs();
inspect_ifrs = solve_ifrs;
# make a predict tree using the MeqMaker
if do_solve or do_subtract:
predict = meqmaker.make_predict_tree(ns);
# make a ParmGroup and solve jobs for source parameters, if we have any
if do_solve:
parms = {};
for src in meqmaker.get_source_list(ns):
parms.update([(p.name,p) for p in src.get_solvables()]);
if parms:
pg_src = ParmGroup.ParmGroup("source",list(parms.values()),
table_name="sources.fmep",
individual=True,bookmark=True);
# now make a solvejobs for the source
ParmGroup.SolveJob("cal_source","Calibrate source model",pg_src);
# make nodes to compute residuals
if do_subtract:
residuals = ns.residuals;
for p,q in array.ifrs():
residuals(p,q) << spigots(p,q) - predict(p,q);
outputs = residuals;
# and now we may need to correct the outputs
if do_correct:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns);
sky_correct = srcs and srcs[0];
else:
sky_correct = None;
outputs = meqmaker.correct_uv_data(ns,outputs,sky_correct=sky_correct,inspect_ifrs=inspect_ifrs);
# make solve trees
if do_solve:
# inputs to the solver are based on calibration type
# if calibrating visibilities, feed them to condeq directly
if cal_type == CAL.VIS:
observed = spigots;
model = predict;
# else take ampl/phase component
else:
model = ns.model;
observed = ns.observed;
if cal_type == CAL.AMPL:
for p,q in array.ifrs():
observed(p,q) << Meq.Abs(spigots(p,q));
model(p,q) << Meq.Abs(predict(p,q));
elif cal_type == CAL.LOGAMPL:
for p,q in array.ifrs():
observed(p,q) << Meq.Log(Meq.Abs(spigots(p,q)));
model(p,q) << Meq.Log(Meq.Abs(predict(p,q)));
elif cal_type == CAL.PHASE:
for p,q in array.ifrs():
observed(p,q) << 0;
model(p,q) << Meq.Abs(predict(p,q))*Meq.FMod(Meq.Arg(spigots(p,q))-Meq.Arg(predict(p,q)),2*math.pi);
else:
raise ValueError("unknown cal_type setting: "+str(cal_type));
# make a solve tree
solve_tree = StdTrees.SolveTree(ns,model,solve_ifrs=solve_ifrs);
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=observed,outputs=outputs);
# make sinks and vdm.
# The list of inspectors must be supplied here
inspectors += meqmaker.get_inspectors() or [];
StdTrees.make_sinks(ns,outputs,spigots=spigots0,post=inspectors);
Bookmarks.make_node_folder("Corrected/residual visibilities by baseline",
[ outputs(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2);
if not do_solve:
if do_subtract:
name = "Generate residuals";
comment = "Generated residual visibilities.";
elif do_correct:
name = "Generate corrected data";
comment = "Generated corrected visibilities.";
else:
name = None;
if name:
# make a TDL job to runsthe tree
def run_tree (mqs,parent,**kw):
global tile_size;
purrpipe.title("Calibrating").comment(comment);
mqs.execute(Meow.Context.vdm.name,mssel.create_io_request(tile_size),wait=False);
TDLRuntimeMenu(name,
TDLOption('tile_size',"Tile size, in timeslots",[10,60,120,240],more=int,
doc="""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""),
TDLRuntimeJob(run_tree,name)
);
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False));
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options());
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size());
TDLRuntimeMenu("Make an image from this MS",*imsel.option_list());
# and close meqmaker -- this exports annotations, etc
meqmaker.close();
def _define_forest(ns, **kw):
if parmtab is None:
raise RuntimeError, "please select a valid parmtable in compile-time options"
# make one MeqParm per each funklet
parmdef = Meq.Parm(table_name=parmtable_name)
parmnodes = dict([(name, (ns[name] << parmdef))
for name in parmtab.funklet_names()])
# reprocess flag is reset every time something changes in the parmnodes dict,
# so we simply repeat this loop until things stop changing
reprocess = True
while reprocess:
reprocess = False
# see if we have recipe for making compound objects based on last funklet name qualifier
nqlist = [
name.rsplit(':', 1) for name in parmnodes.iterkeys() if ':' in name
]
# set of basenames and set of all qualifiers
basenames = set([nq[0] for nq in nqlist])
quals = set([nq[1] for nq in nqlist])
for quallist, agg_func in aggregators:
# look for an aggregator whose qualifiers are a subset of the given set
if set(quallist) <= quals:
# for each basename, attempt to find funklets for every qualifier in the list
for basename in basenames:
fqnames = [basename + ':' + qq for qq in quallist]
funks = [parmnodes.get(name, None) for name in fqnames]
# if every basename:qq combination gives a valid funklet, call the aggregator
if None not in funks:
aggnode = ns[basename] << agg_func(*funks)
# remove aggregated funklets from the parmnodes dict
for name in fqnames:
del parmnodes[name]
# add new node to dict
parmnodes[basename] = aggnode
reprocess = True
# sort nodes by qualified name
nodes = list(parmnodes.iteritems())
nodes.sort(lambda a, b: ParmTables.cmp_qualified_names(a[0], b[0]))
nodes = [namenode[1] for namenode in nodes]
# make intermediate math node
pall = ns.all_parms_raw << Meq.Composer(
children=nodes, mt_polling=True, dims=[0])
pfm = ns.all_parms_fm << Meq.Mean(pall, reduction_axes="freq")
# ns.all_minus_fm << pall - pfm;
# ns.all_over_fm << pall/pfm;
# make root node
ns.root << Meq.ReqMux(
pfm
# ns.all_minus_fm,
# ns.all_over_fm,
# ns.fft_all_over_fm << Meq.FFTBrick(Meq.Selector(ns.all_over_fm,index=0,multi=True),axes_in=(hiid("time"),hiid("freq")),axes_out=(hiid("l"),hiid("m")))
)
Bookmarks.Page("Composite view").add(ns.all_parms_fm,
viewer="Collections Plotter")
# make bookmark folder
Bookmarks.make_node_folder("All parameters", nodes, sorted=True)
# close parmtable, else meqserver will be unable to get at it
parmtab.close()
# now make runtime options
global active_axes
active_axes = []
reg_domain_opts = []
for iaxis in range(mequtils.max_axis):
stats = parmtab.axis_stats(iaxis)
if not stats.empty():
active_axes.append((iaxis, stats.name, stats))
reg_domain_opts.append(
TDLOption("num_cells_%s" % stats.name,
"Number of grid points in %s" % stats.name,
[len(stats.cells)],
more=int))
TDLRuntimeMenu(
"View parameters over regularly gridded domain",
*(reg_domain_opts + [
TDLJob(_view_parameters_over_regularized_domain,
"View over gridded domain")
]))
def _define_forest (ns,**kw):
if parmtab is None:
raise RuntimeError,"please select a valid parmtable in compile-time options";
# make one MeqParm per each funklet
parmdef = Meq.Parm(table_name=parmtable_name);
parmnodes = dict([(name,(ns[name] << parmdef)) for name in parmtab.funklet_names()]);
# reprocess flag is reset every time something changes in the parmnodes dict,
# so we simply repeat this loop until things stop changing
reprocess = True;
while reprocess:
reprocess = False;
# see if we have recipe for making compound objects based on last funklet name qualifier
nqlist = [ name.rsplit(':',1) for name in parmnodes.iterkeys() if ':' in name ];
# set of basenames and set of all qualifiers
basenames = set([nq[0] for nq in nqlist]);
quals = set([nq[1] for nq in nqlist]);
for quallist,agg_func in aggregators:
# look for an aggregator whose qualifiers are a subset of the given set
if set(quallist) <= quals:
# for each basename, attempt to find funklets for every qualifier in the list
for basename in basenames:
fqnames = [ basename+':'+qq for qq in quallist ];
funks = [ parmnodes.get(name,None) for name in fqnames ];
# if every basename:qq combination gives a valid funklet, call the aggregator
if None not in funks:
aggnode = ns[basename] << agg_func(*funks);
# remove aggregated funklets from the parmnodes dict
for name in fqnames:
del parmnodes[name];
# add new node to dict
parmnodes[basename] = aggnode;
reprocess = True;
# sort nodes by qualified name
nodes = list(parmnodes.iteritems());
nodes.sort(lambda a,b:ParmTables.cmp_qualified_names(a[0],b[0]));
nodes = [ namenode[1] for namenode in nodes ];
# make intermediate math node
pall = ns.all_parms_raw << Meq.Composer(children=nodes,mt_polling=True,dims=[0]);
pfm = ns.all_parms_fm << Meq.Mean(pall,reduction_axes="freq");
# ns.all_minus_fm << pall - pfm;
# ns.all_over_fm << pall/pfm;
# make root node
ns.root << Meq.ReqMux(
pfm
# ns.all_minus_fm,
# ns.all_over_fm,
# ns.fft_all_over_fm << Meq.FFTBrick(Meq.Selector(ns.all_over_fm,index=0,multi=True),axes_in=(hiid("time"),hiid("freq")),axes_out=(hiid("l"),hiid("m")))
);
Bookmarks.Page("Composite view").add(ns.all_parms_fm,viewer="Collections Plotter");
# make bookmark folder
Bookmarks.make_node_folder("All parameters",nodes,sorted=True);
# close parmtable, else meqserver will be unable to get at it
parmtab.close();
# now make runtime options
global active_axes;
active_axes = [];
reg_domain_opts = [];
for iaxis in range(mequtils.max_axis):
stats = parmtab.axis_stats(iaxis);
if not stats.empty():
active_axes.append((iaxis,stats.name,stats));
reg_domain_opts.append(
TDLOption("num_cells_%s"%stats.name,"Number of grid points in %s"%stats.name,[len(stats.cells)],more=int)
);
TDLRuntimeMenu("View parameters over regularly gridded domain",
*(reg_domain_opts + [TDLJob(_view_parameters_over_regularized_domain,"View over gridded domain")]));
def compute_jones (self,jones,sources,stations=None,tags=None,label='',**kw):
stations = stations or Context.array.stations();
is_complex = self.matrix_type != "real";
# set up qualifier labels depending on polarization definition
if Context.observation.circular():
x,y,X,Y = 'r','l','R','L';
else:
x,y,X,Y = 'x','y','X','Y';
xx,xy,yx,yy = x+x,x+y,y+x,y+y;
rxx,rxy,ryx,ryy = [ q+":r" for q in xx,xy,yx,yy ];
ixx,ixy,iyx,iyy = [ q+":i" for q in xx,xy,yx,yy ];
# prepare parm definitions for real and imag parts of diagonal and off-diagonal elements
tags = NodeTags(tags) + "solvable";
diag_pdefs = offdiag_pdefs = None;
# loop over sources
parms_diag = [];
parms_offdiag = [];
subgroups_diag = [];
subgroups_offdiag = [];
# Define a function to put together a matrix element, depending on whether we're in real or complex mode.
# This is also responsible for appending parms to the appropriate parm and subgroup lists.
# Note that for the purely-real case we still create parms called 'J:xx:r' (and not 'J:xx' directly),
# this is to keep MEP tables mutually-compatible naming wise.
if self.matrix_type == "real":
def make_element (jj,parmdef,*parmlists):
parm = jj('r') << parmdef[0];
for plist in parmlists:
plist.append(parm);
return jj << Meq.Identity(parm);
else:
def make_element (jj,parmdef,*parmlists):
parms = [ jj('r') << parmdef[0],jj('i') << parmdef[1] ];
for plist in parmlists:
plist += parms;
return jj << Meq.ToComplex(*parms);
for src in sources:
sg = src.name if self.independent_solve else '';
# (re)create parm definitions.
if diag_pdefs is None or self.independent_solve:
diag_pdefs = ( Meq.Parm(complex(self.init_diag).real,tags=tags+"diag real",solve_group=sg),
Meq.Parm(complex(self.init_diag).imag,tags=tags+"diag imag",solve_group=sg) );
offdiag_pdfefs = ( Meq.Parm(complex(self.init_offdiag).imag,tags=tags+"offdiag real",solve_group=sg),
Meq.Parm(complex(self.init_offdiag).imag,tags=tags+"offdiag imag",solve_group=sg) );
# now loop to create nodes
sgdiag = [];
sgoff = [];
for p in stations:
jj = jones(src,p);
jj << Meq.Matrix22(
make_element(jj(xx),diag_pdefs,parms_diag,sgdiag),
make_element(jj(xy),offdiag_pdefs,parms_offdiag,sgoff) if self._offdiag else 0,
make_element(jj(yx),offdiag_pdefs,parms_offdiag,sgoff) if self._offdiag else 0,
make_element(jj(yy),diag_pdefs,parms_diag,sgdiag)
);
# add subgroup for this source
subgroups_diag.append(ParmGroup.Subgroup(src.name,sgdiag));
subgroups_offdiag.append(ParmGroup.Subgroup(src.name,sgoff));
# re-sort by name
subgroups_diag.sort(lambda a,b:cmp(a.name,b.name));
subgroups_offdiag.sort(lambda a,b:cmp(a.name,b.name));
# make parmgroups for diagonal and off-diagonal terms
self.pg_diag = ParmGroup.ParmGroup(label+"_diag",parms_diag,
subgroups=subgroups_diag,
table_name="%s_diag.fmep"%label,bookmark=False);
if self._offdiag:
self.pg_offdiag = ParmGroup.ParmGroup(label+"_offdiag",parms_offdiag,
subgroups=subgroups_offdiag,
table_name="%s_offdiag.fmep"%label,bookmark=False);
# make bookmarks
Bookmarks.make_node_folder("%s diagonal terms"%label,
[ jones(src,p,zz) for src in sources
for p in stations for zz in "xx","yy" ],sorted=True);
if self._offdiag:
Bookmarks.make_node_folder("%s off-diagonal terms"%label,
[ jones(src,p,zz) for src in sources
for p in stations for zz in "xy","yx" ],sorted=True);
# make solvejobs
ParmGroup.SolveJob("cal_"+label+"_diag","Calibrate %s diagonal terms"%label,self.pg_diag);
if self._offdiag:
ParmGroup.SolveJob("cal_"+label+"_offdiag","Calibrate %s off-diagonal terms"%label,self.pg_offdiag);
return jones;
subgroups = [
ParmGroup.Subgroup(X+X,[jones(p,zz) for zz in rxx,ixx for p in stations]),
ParmGroup.Subgroup(Y+Y,[jones(p,zz) for zz in ryy,iyy for p in stations]),
ParmGroup.Subgroup("real part",[jones(p,zz) for zz in rxx,ryy for p in stations]),
ParmGroup.Subgroup("imaginary part",[jones(p,zz) for zz in ixx,iyy for p in stations])
];
subgroups += [
ParmGroup.Subgroup("station %s"%p,[jones(p,zz) for zz in rxx,ixx,ryy,iyy ])
for p in stations
];
self.pg_diag = ParmGroup.ParmGroup(label+"_diag",
[ jones(p,zz) for zz in rxx,ixx,ryy,iyy for p in stations ],
subgroups = subgroups,
table_name="%s_diag.fmep"%label,bookmark=False);
# make bookmarks
Bookmarks.make_node_folder("%s diagonal terms"%label,
[ jones(p,zz) for p in stations for zz in xx,yy ],sorted=True);
# make solvejobs
ParmGroup.SolveJob("cal_"+label+"_diag","Calibrate %s diagonal terms"%label,self.pg_diag);
if self._offdiag:
subgroups = [
ParmGroup.Subgroup(X+Y,[jones(p,zz) for zz in rxy,ixy for p in stations]),
ParmGroup.Subgroup(Y+X,[jones(p,zz) for zz in ryx,iyx for p in stations]),
ParmGroup.Subgroup("real part",[jones(p,zz) for zz in rxy,ryx for p in stations]),
ParmGroup.Subgroup("imaginary part",[jones(p,zz) for zz in ixy,iyx for p in stations])
];
subgroups += [
ParmGroup.Subgroup("station %s"%p,[jones(p,zz) for zz in rxy,ixy,ryx,iyx ])
for p in stations
];
self.pg_offdiag = ParmGroup.ParmGroup(label+"_offdiag",
def _define_forest(ns,parent=None,**kw):
if run_purr:
Timba.TDL.GUI.log_message("starting purr");
Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']);
mssel.setup_observation_context(ns);
array = Meow.Context.array;
# make spigot nodes
outputs = spigots = array.spigots(corr=mssel.get_corr_index());
# ...and an inspector for them
Meow.StdTrees.vis_inspector(ns.inspector('input'),spigots,
bookmark="Inspect input visibilities");
inspectors = [ ns.inspector('input') ];
Bookmarks.make_node_folder("Input visibilities by baseline",
[ spigots(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2);
# add in-tree flaggers, if appropriate
if tree_flag_enable:
flaggers = [];
# make nodes to take complex amplitude, if any flaggers are going to use them
if tree_flag_ampl is not None or tree_flag_mean_ampl is not None or tree_flag_mean_freq_ampl is not None:
# use matrix norm
if tree_flag_norm:
for p,q in array.ifrs():
node = outputs(p,q);
nsq = node('sq') << Meq.MatrixMultiply(node,node('conj') << Meq.ConjTranspose(node));
nsqtr = nsq('tr') << (nsq(11) << Meq.Selector(nsq,index=0)) + (nsq(22) << Meq.Selector(nsq,index=3));
ns.ampl(p,q) << Meq.Sqrt(nsqtr('abs') << Meq.Abs(nsqtr));
# use per-element amplitudes
else:
for p,q in array.ifrs():
ns.ampl(p,q) << Meq.Abs(outputs(p,q));
# ...and an inspector for them
Meow.StdTrees.vis_inspector(ns.inspector('ampl'),ns.ampl,bookmark="Inspect mean-freq visibility amplitudes");
inspectors.append(ns.inspector('ampl'));
# make flagger for amplitudes
if tree_flag_ampl is not None:
for p,q in array.ifrs():
ns.flagampl(p,q) << Meq.ZeroFlagger(ns.ampl(p,q)-tree_flag_ampl,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT);
flaggers.append(ns.flagampl);
# make flagger for mean-ifr amplitudes
if tree_flag_mean_ampl is not None:
ns.meanampl << Meq.Mean(*[ns.ampl(p,q) for p,q in array.ifrs()]);
ns.flagmeanampl << Meq.ZeroFlagger(ns.meanampl-tree_flag_mean_ampl,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT);
Meow.Bookmarks.Page("Mean-ifr visibility amplitudes").add(ns.meanampl,viewer="Result Plotter");
flaggers.append(lambda p,q:ns.flagmeanampl);
# make flagger for mean-fr-freq amplitudes
if tree_flag_mean_freq_ampl is not None:
ns.meanampl ** Meq.Mean(*[ns.ampl(p,q) for p,q in array.ifrs()]);
ns.meanfreqampl << Meq.Mean(ns.meanampl,reduction_axes="freq");
ns.flagmeanfreqampl << Meq.ZeroFlagger(ns.meanfreqampl-tree_flag_mean_freq_ampl,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT);
Meow.Bookmarks.Page("Mean-ifr-freq visibility amplitudes").add(ns.meanfreqampl,viewer="Collections Plotter");
flaggers.append(lambda p,q:ns.flagmeanfreqampl);
# now merge all flaggers
if flaggers:
for p,q in array.ifrs():
ns.flagged(p,q) << Meq.MergeFlags(outputs(p,q),*[fl(p,q) for fl in flaggers]);
outputs = ns.flagged;
Meow.StdTrees.vis_inspector(ns.inspector('flagged'),outputs,
bookmark="Inspect flagged visibilities");
inspectors.append(ns.inspector('flagged'));
Bookmarks.make_node_folder("Flagged visibilities by baseline",
[ outputs(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2);
mssel.enable_write_flags(True);
treeflag_ms_opt.show(True);
if outputs is spigots:
ns.VisDataMux << Meq.VisDataMux(post=(inspectors[0] if len(inspectors)<2 else Meq.ReqMux(*inspectors)));
else:
Meow.StdTrees.make_sinks(ns,outputs,spigots=spigots,post=inspectors);
# add imaging options
imsel = mssel.imaging_selector(npix=512,arcmin=120);
TDLRuntimeMenu("Make an image from this MS",*imsel.option_list());
def _define_forest(ns, parent=None, **kw):
if not mssel.msname:
raise RuntimeError("MS not set")
if run_purr:
Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.'])
# create Purr pipe
global purrpipe
purrpipe = Purr.Pipe.Pipe(mssel.msname)
# setup contexts from MS
mssel.setup_observation_context(ns)
array = Meow.Context.array
# make spigot nodes for data
mssel.enable_input_column(True)
spigots = array.spigots(corr=mssel.get_corr_index())
meqmaker.make_per_ifr_bookmarks(spigots, "Input visibilities")
# data tensor
ns.DT << Meq.Composer(
dims=[0], mt_polling=True, *[spigots(p, q) for p, q in array.ifrs()])
# list of model tensors
models = []
if read_ms_model:
mssel.enable_model_column(True)
model_spigots = array.spigots(column="PREDICT",
corr=mssel.get_corr_index())
meqmaker.make_per_ifr_bookmarks(model_spigots, "UV-model visibilities")
mtuv = ns.MT("uv") << Meq.Composer(
dims=[0],
mt_polling=True,
*[model_spigots(p, q) for p, q in array.ifrs()])
# predict tree using the MeqMaker
all_sources = meqmaker.get_source_list(ns)
dg_sources = deopts.enabled and dgsel.filter(all_sources)
if dg_sources:
# first group all sources without a diffgain on them
groups = [[src for src in all_sources if not src in dg_sources]]
# group diffgain-enabled sources by grouping tag
clusters = set(
[src.get_attr(diffgain_group, None) for src in dg_sources])
dg_groups = [(name, [
src for src in dg_sources if src.get_attr(diffgain_group) == name
]) for name in clusters if name]
# add sources without a grouping tag individually, as single-source groups
dg_groups += [(src.name, [src]) for src in dg_sources
if not src.get_attr(diffgain_group, None)]
# now sort by brightness
flux_dgg = [(sum(
[src.get_attr('Iapp', 0) or src.get_attr('I')
for src in dgg[1]]), dgg) for dgg in dg_groups]
from past.builtins import cmp
from functools import cmp_to_key
flux_dgg.sort(key=cmp_to_key(lambda a, b: cmp(b[0], a[0])))
diffgain_labels = [dgg[0] for flux, dgg in flux_dgg]
groups += [dgg[1] for flux, dgg in flux_dgg]
num_diffgains = len(flux_dgg)
# now make predict trees
for i, group in enumerate(groups):
MT = ns.MT(group[0].name if i else "all")
# first tensor is "MT", rest qualified by source names
predict = meqmaker.make_predict_tree(MT.Subscope(), sources=group)
MT << Meq.Composer(dims=[0],
mt_polling=True,
*[predict(p, q) for p, q in array.ifrs()])
# if reading an extra uv-model, add to first term
if not i and read_ms_model:
MT = ns.MT << Meq.Add(MT, mtuv)
models.append(MT)
print("Number of diffgain predict groups:", len(groups))
else:
diffgain_labels = []
num_diffgains = 0
predict = meqmaker.make_predict_tree(ns)
MT = ns.MT("all") << Meq.Composer(
dims=[0],
mt_polling=True,
*[predict(p, q) for p, q in array.ifrs()])
if read_ms_model:
MT = ns.MT << Meq.Add(MT, mtuv)
models.append(MT)
solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs()
downsample_subtiling = [
stefcal_downsample_timeint, stefcal_downsample_freqint
] if stefcal_downsample else [1, 1]
import Calico.OMS.StefCal.StefCal
kwopts = {}
gopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize)
bopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize)
deopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize)
ns.stefcal << Meq.PyNode(
class_name="StefCalNode",
module_name=Calico.OMS.StefCal.StefCal.__file__,
ifrs=["%s:%s" % (p, q) for p, q in array.ifrs()],
baselines=[
array.baseline(ip, iq) for (ip, p), (iq, q) in array.ifr_index()
],
solve_ifrs=["%s:%s" % (p, q) for p, q in solve_ifrs],
noise_per_chan=stefcal_noise_per_chan,
downsample_subtiling=downsample_subtiling,
num_major_loops=stefcal_nmajor,
regularization_factor=1e-6, #
rescale=stefcal_rescale,
init_from_previous=False,
critical_flag_threshold=critical_flag_threshold,
diffgain_labels=diffgain_labels,
# flagging options
output_flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT,
# IFR gain solution options
apply_ifr_gains=stefcal_ifr_gains,
solve_ifr_gains=(stefcal_ifr_gain_mode != MODE_SOLVE_APPLY),
reset_ifr_gains=stefcal_ifr_gain_reset,
save_ifr_gains=(stefcal_ifr_gain_mode == MODE_SOLVE_SAVE),
ifr_gain_table=stefcal_ifr_gain_table,
per_chan_ifr_gains=stefcal_per_chan_ifr_gains,
diag_ifr_gains=(stefcal_diagonal_ifr_gains == DIAGONLY),
residuals=(do_output == CORRECTED_RESIDUALS),
subtract_dgsrc=(do_output == CORRECTED_DATA_SUB),
verbose=stefcal_verbose,
children=[ns.DT] + models,
**kwopts)
inspectors = meqmaker.get_inspectors() or []
# make output bookmarks
nv = 0
for p, q in array.ifrs():
sel = ns.output_sel(p, q) << Meq.Selector(
ns.stefcal, index=list(range(nv, nv + 4)), multi=True)
ns.output(p, q) << Meq.Composer(sel, dims=[2, 2])
nv += 4
meqmaker.make_per_ifr_bookmarks(ns.output, "Output visibilities")
Bookmarks.Page("StefCal outputs").add(ns.stefcal, viewer="Record Browser")
if gopts.enabled and gopts.visualize and stefcal_visualize:
ns.stefcal_vis_G << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="G",
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
ns.stefcal_vis_G_avg << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="G",
freq_average=True,
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
Bookmarks.Page("StefCal G plotter").add(ns.stefcal_vis_G,
viewer="Result Plotter")
Bookmarks.Page("StefCal G inspector").add(ns.stefcal_vis_G_avg,
viewer="Collections Plotter")
inspectors += [ns.stefcal_vis_G, ns.stefcal_vis_G_avg]
if bopts.enabled and bopts.visualize and stefcal_visualize:
ns.stefcal_vis_B << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="B",
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
ns.stefcal_vis_B_avg << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="B",
freq_average=True,
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
Bookmarks.Page("StefCal B plotter").add(ns.stefcal_vis_B,
viewer="Result Plotter")
Bookmarks.Page("StefCal B inspector").add(ns.stefcal_vis_B_avg,
viewer="Collections Plotter")
inspectors += [ns.stefcal_vis_B, ns.stefcal_vis_B_avg]
if deopts.enabled and deopts.visualize and stefcal_visualize:
for i, label in enumerate(diffgain_labels):
vde = ns.stefcal_vis_dE(label) << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="dE:%s" % label,
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
vde_avg = ns.stefcal_vis_dE_avg(label) << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="dE:%s" % label,
freq_average=True,
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
Bookmarks.Page("StefCal dE:%s plotter" % label).add(
vde, viewer="Result Plotter")
Bookmarks.Page("StefCal dE:%s inspector" % label).add(
vde_avg, viewer="Collections Plotter")
inspectors += [vde, vde_avg]
# make sinks
StdTrees.make_sinks(ns,
ns.output,
spigots=spigots,
post=inspectors,
corr_index=mssel.get_corr_index())
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
# and close meqmaker -- this exports annotations, etc
meqmaker.close()
# add options to clear all solutions
from Calico.OMS.StefCal import StefCal
TDLRuntimeOption("stefcal_reset_all", "Remove all existing solutions",
False)
for opt in gopts, bopts, deopts:
if opt.enabled:
TDLRuntimeOption("reset",
"Remove existing %s solutions (%s)" %
(opt.label, os.path.basename(opt.table)),
False,
namespace=opt)
if stefcal_ifr_gains:
TDLRuntimeOption(
"stefcal_reset_ifr_gains",
"Remove existing interferometer errors (%s)" %
(os.path.basename(stefcal_ifr_gain_table)), False)
TDLRuntimeJob(_run_stefcal, "Run StefCal", job_id="stefcal")
