def compute_jones(self, nodes, stations=None, tags=None, label='', **kw):
stations = stations or Context.array.stations()
g_ampl_def = Meow.Parm(1)
g_phase_def = Meow.Parm(0)
nodes = Jones.gain_ap_matrix(nodes,
g_ampl_def,
g_phase_def,
tags=tags,
series=stations)
# make parmgroups for phases and gains
self.pg_phase = ParmGroup.ParmGroup(
label + "_phase",
nodes.search(tags="solvable phase"),
table_name="%s_phase.fmep" % label,
bookmark=4)
self.pg_ampl = ParmGroup.ParmGroup(label + "_ampl",
nodes.search(tags="solvable ampl"),
table_name="%s_ampl.fmep" % label,
bookmark=4)
# make solvejobs
ParmGroup.SolveJob("cal_" + label + "_phase",
"Calibrate %s phases" % label, self.pg_phase)
ParmGroup.SolveJob("cal_" + label + "_ampl",
"Calibrate %s amplitudes" % label, self.pg_ampl)
return nodes
def compute_jones_tensor (Jones,sources,stations=None,label="Z",lmn=None,meqmaker=None,inspectors=[],**kw):
stations = stations or Context.array.stations;
ns = Jones.Subscope();
# if lmn tensor is not set for us, create a composer
if lmn is None:
lmn = ns.lmnT << Meq.Composer(dims=[0],*[ src.direction.lmn() for src in sources ]);
parmlist = make_mim_parms(ns,stations,dims=[3]);
# choose frequency scaling so that a value of "1" produces 1 radian of phase
# delay at 1 GHz.
ns.freqscale = 1e+9/Meq.Freq();
xyz = Context.array.xyz;
xyz0 = Context.array.xyz0();
for p in stations:
dp = ns.dphase(p) << Meq.MatrixMultiply(lmn,ns.ab(p));
Jones(p) << Meq.Polar(1,dp*ns.freqscale);
# make solvejobs
global pg;
pg = ParmGroup.ParmGroup(label,parmlist,table_name="%s.fmep"%label,bookmark=True);
ParmGroup.SolveJob("cal_"+label,"Calibrate %s (gradient MIM)"%label,pg);
# make an inspector for the a and b parms
meqmaker.make_per_station_bookmarks(ns.ab,"MIM gradients",freqmean=False);
return Jones;
def compute_jones (Jones,sources,stations=None,inspectors=[],meqmaker=None,label='Z',**kw):
"""Creates the Z Jones for ionospheric phase, given a TEC gradient.""";
stations = stations or Context.array.stations;
ns = Jones.Subscope();
parmlist = make_mim_parms(ns,stations,dims=[1,2]);
# choose frequency scaling so that a value of "1" produces 1 radian of phase
# delay at 1 GHz.
ns.freqscale = 1e+9/Meq.Freq();
for p in stations:
for isrc,src in enumerate(sources):
dp = ns.dphase(src,p) << Meq.MatrixMultiply(ns.ab(p),src.direction.lm());
Jones(src,p) << Meq.Polar(1,dp*ns.freqscale);
# make bookmarks
meqmaker.make_per_station_bookmarks(ns.ab,"MIM gradients",freqmean=False);
meqmaker.make_per_source_per_station_bookmarks(ns.dphase,"MIM phase screen",
sources,freqmean=False);
# make solvejobs
global pg;
pg = ParmGroup.ParmGroup(label,parmlist,table_name="%s.fmep"%label,bookmark=True);
ParmGroup.SolveJob("cal_"+label,"Calibrate %s (gradient MIM)"%label,pg);
return Jones;
def compute_pointings(nodes,
stations=None,
label="pnt",
return_parms=None,
**kw):
"""Computes pointing errors for a list of stations."""
# if the error generator is set to "no error", return None
ns = nodes.Subscope()
# create parms for pointing errors per antenna
pps = []
for p in stations or Context.array.stations():
dl = ns.dl(p) << Meq.Parm(0, tags="pointing solvable")
dm = ns.dm(p) << Meq.Parm(0, tags="pointing solvable")
nodes(p) << Meq.Composer(dl, dm)
pps += [dl, dm]
if return_parms is None:
# add parmgroup
global pg_pointing
pg_pointing = ParmGroup.ParmGroup(label,
pps,
table_name="%s.fmep" % label,
bookmark=True)
ParmGroup.SolveJob("cal_" + label, "Calibrate pointing errors",
pg_pointing)
else:
return_parms += pps
return nodes
def init_nodes (self,ns,tags=None,label=''):
ifrs = Context.array.ifrs();
G = ns.gain;
if not G(*ifrs[0]).initialized():
def1 = Meow.Parm(1,tags=tags);
def0 = Meow.Parm(0,tags=tags);
gains = [];
for p,q in ifrs:
gg = [];
for corr in Context.correlations:
if corr in Context.active_correlations:
cc = corr.lower();
gain_ri = [ resolve_parameter(cc,G(p,q,cc,'r'),def1,tags="ifr gain real"),
resolve_parameter(cc,G(p,q,cc,'i'),def0,tags="ifr gain imag") ];
gg.append(G(p,q,cc) << Meq.ToComplex(*gain_ri));
gains += gain_ri;
else:
gg.append(1);
G(p,q) << Meq.Matrix22(*gg);
pg_ifr_ampl = ParmGroup.ParmGroup(label,gains,
individual_toggles=False,
table_name="%s.fmep"%label,bookmark=False);
Bookmarks.make_node_folder("%s (interferometer-based gains)"%label,
[ G(p,q) for p,q in ifrs ],sorted=True,nrow=2,ncol=2);
ParmGroup.SolveJob("cal_%s"%label,
"Calibrate %s (interferometer-based gains)"%label,pg_ifr_ampl);
return G;
def compute_jones (self,jones,sources,stations=None,tags=None,label='',**kw):
stations = stations or Context.array.stations();
# figure out which sources to apply to
if self.subset:
srclist = self._subset_parser.parse_list(self.subset);
sources = [ sources[i] for i in srclist ];
# create parm definitions for each jones element
tags = NodeTags(tags) + "solvable";
rm_parm = Meq.Parm(0,tags=tags+"rm");
# loop over sources
for src in sources:
jj = jones(src);
jj("RM") << rm_parm;
fr = jj("fr") << jj("RM") / Meq.Sqr(Meq.Freq());
cos_fr = jj("cos") << Meq.Cos(fr);
sin_fr = jj("sin") << Meq.Sin(fr);
jj << Meq.Matrix22(cos_fr,-sin_fr,sin_fr,cos_fr);
for p in stations:
jj(p) << Meq.Identity(jj);
# make parmgroups for diagonal and off-diagonal terms
self.pg_fr = ParmGroup.ParmGroup(label+"_fr",
[ jones(src,"RM") for src in sources ],
table_name="%s_fr.fmep"%label,bookmark=False);
# make bookmarks
Bookmarks.make_node_folder("%s Faraday rotation"%label,
[ jones(src,"fr") for src in sources ],sorted=True);
# make solvejobs
ParmGroup.SolveJob("cal_"+label+"_fr","Calibrate %s"%label,self.pg_fr);
return jones;
def source_list(ns):
# figure out spectral index parameters
if spectral_index is not None:
spi_def = Meow.Parm(spectral_index)
freq0_def = ref_freq
else:
spi_def = freq0_def = None
if spectral_index1 is not None:
spi_def1 = Meow.Parm(spectral_index1)
freq0_def1 = ref_freq
else:
spi_def1 = freq0_def1 = None
# and flux parameters
i_def = Meow.Parm(1)
quv_def = Meow.Parm(0)
dir1 = Meow.Direction(ns, "3C343.1", 4.356645791155902, 1.092208429052697)
dir0 = Meow.Direction(ns, "3C343", 4.3396003966265599, 1.0953677174056471)
src1 = Meow.PointSource(ns,
"3C343.1",
dir1,
I=Meow.Parm(6.02061051),
Q=Meow.Parm(0.0179716185),
U=quv_def,
V=quv_def,
spi=spi_def1,
freq0=freq0_def1)
src0 = Meow.PointSource(ns,
"3C343",
dir0,
I=Meow.Parm(1.83336309),
Q=Meow.Parm(0.0241450607),
U=quv_def,
V=quv_def,
spi=spi_def1,
freq0=freq0_def1)
## define a parmgroup for source parameters
pg_src = ParmGroup.ParmGroup("source",
src1.coherency().search(tags="solvable") +
src0.coherency().search(tags="solvable"),
table_name="sources.mep")
ParmGroup.SolveJob("cal_sources", "Calibrate sources", pg_src)
return [src1, src0]
def compute_jones(Jones,
sources,
stations=None,
inspectors=[],
meqmaker=None,
label='R',
**kw):
"""Creates the Z Jones for ionospheric phase, given TECs (per source,
per station)."""
stations = stations or Context.array.stations
ns = Jones.Subscope()
parmdef = Meq.Parm(0, tags="pos_offset")
parms = []
uvw = Context.array.uvw()
# now loop over sources
for isrc, src in enumerate(sources):
parms += [ns.dl(src) << parmdef,
ns.dm(src) << parmdef]
dlmn = ns.dlmn(src) << Meq.Composer(ns.dl(src), ns.dm(src), 0)
for p in stations:
Jones(src, p) << Meq.VisPhaseShift(lmn=dlmn, uvw=uvw(p))
# make bookmarks
srcnames = [src.name for src in sources]
meqmaker.make_bookmark_set(Jones, [(src, p) for src in srcnames
for p in stations],
"%s: inspector plot" % label,
"%s: by source-station" % label,
freqmean=True)
inspectors.append(ns.inspector(label,'dlmn') << \
StdTrees.define_inspector(ns.dlmn,srcnames,label=label))
# make parmgroups and solvejobs
global pg
pg = ParmGroup.ParmGroup(label,
parms,
table_name="%s.fmep" % label,
bookmark=False)
# make solvejobs
ParmGroup.SolveJob("cal_" + label,
"Calibrate %s (position shifts)" % label, pg)
return Jones
def compute_jones(self,
nodes,
stations=None,
tags=None,
label='',
inspectors=[],
**kw):
stations = stations or Context.array.stations()
rot_def = Meow.Parm(0)
nr = Jones.rotation_matrix(nodes("R"),
rot_def,
tags=tags,
series=stations)
ne = Jones.ellipticity_matrix(nodes("E"),
rot_def,
tags=tags,
series=stations)
for p in stations:
nodes(p) << Meq.MatrixMultiply(nr(p), ne(p))
# make parmgroups for phases and gains
self.pg_rot = ParmGroup.ParmGroup(label + "_leakage",
nodes.search(tags="solvable"),
table_name="%s_leakage.mep" % label,
bookmark=True)
# make solvejobs
ParmGroup.SolveJob("cal_" + label + "_leakage",
"Calibrate %s (leakage)" % label, self.pg_rot)
# make inspector for parameters
StdTrees.inspector(nodes('inspector'),
self.pg_rot.nodes,
bookmark=False)
inspectors.append(nodes('inspector'))
return nodes
def compute_jones (self,nodes,sources,stations=None,tags=None,label='',**kw):
stations = stations or Context.array.stations();
# set up qualifier labels depending on polarization definition
if Context.observation.circular():
x,y,X,Y = 'r','l','R','L';
else:
x,y,X,Y = 'x','y','X','Y';
xx,xy,yx,yy = x+x,x+y,y+x,y+y;
axx,axy,ayx,ayy = [ q+":a" for q in xx,xy,yx,yy ];
pxx,pxy,pyx,pyy = [ q+":p" for q in xx,xy,yx,yy ];
ampl_def = phase_def = None;
parms_phase = [];
parms_ampl = [];
subgroups_phase = [];
subgroups_ampl = [];
for src in sources:
sg = src.name if self.independent_solve else '';
# (re)create parm definitions.
if ampl_def is None or self.independent_solve:
ampl_def = Meq.Parm(self.init_ampl,tags=tags+"solvable diag ampl",solve_group=sg);
phase_def = Meq.Parm(self.init_phase,tags=tags+"solvable diag phase",solve_group=sg);
sga = [];
sgp = [];
# now loop to create nodes
for p in stations:
jj = nodes(src,p);
jj << Meq.Matrix22( Meq.Polar(jj(axx) << ampl_def,jj(pxx) << phase_def),0,0,
Meq.Polar(jj(ayy) << ampl_def,jj(pyy) << phase_def));
sga += [jj(axx),jj(ayy)];
sgp += [jj(pxx),jj(pyy)];
parms_ampl += [jj(axx),jj(ayy)];
parms_phase += [jj(pxx),jj(pyy)];
# add subgroups for this source
subgroups_ampl.append(ParmGroup.Subgroup(src.name,sga));
subgroups_phase.append(ParmGroup.Subgroup(src.name,sgp));
# make parmgroups for phases and gains
self.pg_phase = ParmGroup.ParmGroup(label+"_phase",
parms_phase,
subgroups = subgroups_phase,
table_name="%s_phase.fmep"%label,bookmark=4);
self.pg_ampl = ParmGroup.ParmGroup(label+"_ampl",
parms_ampl,
subgroups = subgroups_ampl,
table_name="%s_ampl.fmep"%label,bookmark=4);
ParmGroup.SolveJob("cal_"+label+"_phase","Calibrate %s phases"%label,self.pg_phase);
ParmGroup.SolveJob("cal_"+label+"_ampl","Calibrate %s amplitudes"%label,self.pg_ampl);
return nodes;
[jones(p, zz) for zz in rxx, ixx, ryy, iyy])
for p in stations
]
self.pg_diag = ParmGroup.ParmGroup(
label + "_diag",
[jones(p, zz) for zz in rxx, ixx, ryy, iyy for p in stations],
subgroups=subgroups,
table_name="%s_diag.fmep" % label,
bookmark=False)
# make bookmarks
Bookmarks.make_node_folder(
"%s diagonal terms" % label,
[jones(p, zz) for p in stations for zz in xx, yy],
sorted=True)
# make solvejobs
ParmGroup.SolveJob("cal_" + label + "_diag",
"Calibrate %s diagonal terms" % label, self.pg_diag)
if self._offdiag:
subgroups = [
ParmGroup.Subgroup(
X + Y,
[jones(p, zz) for zz in rxy, ixy for p in stations]),
ParmGroup.Subgroup(
Y + X,
[jones(p, zz) for zz in ryx, iyx for p in stations]),
ParmGroup.Subgroup(
"real part",
[jones(p, zz) for zz in rxy, ryx for p in stations]),
ParmGroup.Subgroup(
"imaginary part",
[jones(p, zz) for zz in ixy, iyx for p in stations])
def compute_jones(self,
jones,
sources,
stations=None,
tags=None,
label='',
**kw):
stations = stations or Context.array.stations()
is_complex = self.matrix_type != "real"
# set up qualifier labels depending on polarization definition
if Context.observation.circular():
x, y, X, Y = 'r', 'l', 'R', 'L'
else:
x, y, X, Y = 'x', 'y', 'X', 'Y'
xx, xy, yx, yy = x + x, x + y, y + x, y + y
rxx, rxy, ryx, ryy = [q + ":r" for q in (xx, xy, yx, yy)]
ixx, ixy, iyx, iyy = [q + ":i" for q in (xx, xy, yx, yy)]
# prepare parm definitions for real and imag parts of diagonal and off-diagonal elements
tags = NodeTags(tags) + "solvable"
diag_pdefs = offdiag_pdefs = None
# loop over sources
parms_diag = []
parms_offdiag = []
subgroups_diag = []
subgroups_offdiag = []
# Define a function to put together a matrix element, depending on whether we're in real or complex mode.
# This is also responsible for appending parms to the appropriate parm and subgroup lists.
# Note that for the purely-real case we still create parms called 'J:xx:r' (and not 'J:xx' directly),
# this is to keep MEP tables mutually-compatible naming wise.
if self.matrix_type == "real":
def make_element(jj, parmdef, *parmlists):
parm = jj('r') << parmdef[0]
for plist in parmlists:
plist.append(parm)
return jj << Meq.Identity(parm)
else:
def make_element(jj, parmdef, *parmlists):
parms = [jj('r') << parmdef[0],
jj('i') << parmdef[1]]
for plist in parmlists:
plist += parms
return jj << Meq.ToComplex(*parms)
for src in sources:
sg = src.name if self.independent_solve else ''
# (re)create parm definitions.
if diag_pdefs is None or self.independent_solve:
diag_pdefs = (Meq.Parm(complex(self.init_diag).real,
tags=tags + "diag real",
solve_group=sg),
Meq.Parm(complex(self.init_diag).imag,
tags=tags + "diag imag",
solve_group=sg))
offdiag_pdfefs = (Meq.Parm(complex(self.init_offdiag).imag,
tags=tags + "offdiag real",
solve_group=sg),
Meq.Parm(complex(self.init_offdiag).imag,
tags=tags + "offdiag imag",
solve_group=sg))
# now loop to create nodes
sgdiag = []
sgoff = []
for p in stations:
jj = jones(src, p)
jj << Meq.Matrix22(
make_element(jj(xx), diag_pdefs, parms_diag, sgdiag),
make_element(jj(xy), offdiag_pdefs, parms_offdiag, sgoff)
if self._offdiag else 0,
make_element(jj(yx), offdiag_pdefs, parms_offdiag, sgoff)
if self._offdiag else 0,
make_element(jj(yy), diag_pdefs, parms_diag, sgdiag))
# add subgroup for this source
subgroups_diag.append(ParmGroup.Subgroup(src.name, sgdiag))
subgroups_offdiag.append(ParmGroup.Subgroup(src.name, sgoff))
# re-sort by name
from past.builtins import cmp
from functools import cmp_to_key
subgroups_diag.sort(key=cmp_to_key(lambda a, b: cmp(a.name, b.name)))
subgroups_offdiag.sort(
key=cmp_to_key(lambda a, b: cmp(a.name, b.name)))
# make parmgroups for diagonal and off-diagonal terms
self.pg_diag = ParmGroup.ParmGroup(label + "_diag",
parms_diag,
subgroups=subgroups_diag,
table_name="%s_diag.fmep" % label,
bookmark=False)
if self._offdiag:
self.pg_offdiag = ParmGroup.ParmGroup(
label + "_offdiag",
parms_offdiag,
subgroups=subgroups_offdiag,
table_name="%s_offdiag.fmep" % label,
bookmark=False)
# make bookmarks
Bookmarks.make_node_folder("%s diagonal terms" % label, [
jones(src, p, zz) for src in sources for p in stations
for zz in ["xx", "yy"]
],
sorted=True)
if self._offdiag:
Bookmarks.make_node_folder("%s off-diagonal terms" % label, [
jones(src, p, zz) for src in sources for p in stations
for zz in ["xy", "yx"]
],
sorted=True)
# make solvejobs
ParmGroup.SolveJob("cal_" + label + "_diag",
"Calibrate %s diagonal terms" % label, self.pg_diag)
if self._offdiag:
ParmGroup.SolveJob("cal_" + label + "_offdiag",
"Calibrate %s off-diagonal terms" % label,
self.pg_offdiag)
return jones
def _define_forest(ns, parent=None, **kw):
if run_purr:
Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.'])
# create Purr pipe
global purrpipe
purrpipe = Purr.Pipe.Pipe(mssel.msname)
# get antennas from MS
ANTENNAS = mssel.get_antenna_set(list(range(1, 15)))
array = Meow.IfrArray(ns, ANTENNAS, mirror_uvw=False)
stas = array.stations()
# get phase centre from MS, setup observation
observation = Meow.Observation(ns,
phase_centre=mssel.get_phase_dir(),
linear=mssel.is_linear_pol(),
circular=mssel.is_circular_pol())
Meow.Context.set(array, observation)
# get active correlations from MS
Meow.Context.active_correlations = mssel.get_correlations()
# make spigot nodes
spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index())
# ...and an inspector for them
StdTrees.vis_inspector(ns.inspector('input'),
spigots,
bookmark="Inspect input visibilities")
inspectors = [ns.inspector('input')]
Bookmarks.make_node_folder("Input visibilities by baseline",
[spigots(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
inspect_ifrs = array.ifrs()
if do_solve:
# filter solvable baselines by baseline length
solve_ifrs = []
antpos = mssel.ms_antenna_positions
if (min_baseline or max_baseline) and antpos is not None:
for (ip, p), (iq, q) in array.ifr_index():
baseline = math.sqrt(
((antpos[ip, :] - antpos[iq, :])**2).sum())
if (not min_baseline or baseline > min_baseline) and \
(not max_baseline or baseline < max_baseline):
solve_ifrs.append((p, q))
else:
solve_ifrs = array.ifrs()
inspect_ifrs = solve_ifrs
# make a predict tree using the MeqMaker
if do_solve or do_subtract:
predict = meqmaker.make_predict_tree(ns)
# make a ParmGroup and solve jobs for source parameters, if we have any
if do_solve:
parms = {}
for src in meqmaker.get_source_list(ns):
parms.update([(p.name, p) for p in src.get_solvables()])
if parms:
pg_src = ParmGroup.ParmGroup("source",
list(parms.values()),
table_name="sources.fmep",
individual=True,
bookmark=True)
# now make a solvejobs for the source
ParmGroup.SolveJob("cal_source", "Calibrate source model",
pg_src)
# make nodes to compute residuals
if do_subtract:
residuals = ns.residuals
for p, q in array.ifrs():
residuals(p, q) << spigots(p, q) - predict(p, q)
outputs = residuals
# and now we may need to correct the outputs
if do_correct:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns)
sky_correct = srcs and srcs[0]
else:
sky_correct = None
outputs = meqmaker.correct_uv_data(ns,
outputs,
sky_correct=sky_correct,
inspect_ifrs=inspect_ifrs)
# make solve trees
if do_solve:
# inputs to the solver are based on calibration type
# if calibrating visibilities, feed them to condeq directly
if cal_type == CAL.VIS:
observed = spigots
model = predict
# else take ampl/phase component
else:
model = ns.model
observed = ns.observed
if cal_type == CAL.AMPL:
for p, q in array.ifrs():
observed(p, q) << Meq.Abs(spigots(p, q))
model(p, q) << Meq.Abs(predict(p, q))
elif cal_type == CAL.LOGAMPL:
for p, q in array.ifrs():
observed(p, q) << Meq.Log(Meq.Abs(spigots(p, q)))
model(p, q) << Meq.Log(Meq.Abs(predict(p, q)))
elif cal_type == CAL.PHASE:
for p, q in array.ifrs():
observed(p, q) << 0
model(p, q) << Meq.Abs(predict(p, q)) * Meq.FMod(
Meq.Arg(spigots(p, q)) - Meq.Arg(predict(p, q)),
2 * math.pi)
else:
raise ValueError("unknown cal_type setting: " + str(cal_type))
# make a solve tree
solve_tree = StdTrees.SolveTree(ns, model, solve_ifrs=solve_ifrs)
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=observed, outputs=outputs)
# make sinks and vdm.
# The list of inspectors must be supplied here
inspectors += meqmaker.get_inspectors() or []
StdTrees.make_sinks(ns, outputs, spigots=spigots0, post=inspectors)
Bookmarks.make_node_folder("Corrected/residual visibilities by baseline",
[outputs(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
if not do_solve:
if do_subtract:
name = "Generate residuals"
comment = "Generated residual visibilities."
elif do_correct:
name = "Generate corrected data"
comment = "Generated corrected visibilities."
else:
name = None
if name:
# make a TDL job to runsthe tree
def run_tree(mqs, parent, **kw):
global tile_size
purrpipe.title("Calibrating").comment(comment)
mqs.execute(Meow.Context.vdm.name,
mssel.create_io_request(tile_size),
wait=False)
TDLRuntimeMenu(
name,
TDLOption(
'tile_size',
"Tile size, in timeslots", [10, 60, 120, 240],
more=int,
doc=
"""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
), TDLRuntimeJob(run_tree, name))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
# and close meqmaker -- this exports annotations, etc
meqmaker.close()
def _define_forest(ns):
# make pynodes, xyzcomponent for sources
ANTENNAS = mssel.get_antenna_set(list(range(1, 15)))
array = Meow.IfrArray(ns, ANTENNAS, mirror_uvw=False)
observation = Meow.Observation(ns)
Meow.Context.set(array, observation)
# make a predict tree using the MeqMaker
if do_solve or do_subtract or not do_not_simulate:
outputs = predict = meqmaker.make_tree(ns)
# make a list of selected corrs
selected_corrs = cal_corr.split(" ")
# make spigot nodes
if not do_not_simulate and do_add:
spigots = spigots0 = outputs = array.spigots()
sums = ns.sums
for p, q in array.ifrs():
sums(p, q) << spigots(p, q) + predict(p, q)
outputs = sums
# make spigot nodes
if do_not_simulate:
spigots = spigots0 = outputs = array.spigots()
# make nodes to compute residuals
# make nodes to compute residuals
if do_subtract:
residuals = ns.residuals
for p, q in array.ifrs():
residuals(p, q) << spigots(p, q) - predict(p, q)
outputs = residuals
# and now we may need to correct the outputs
if do_correct:
if do_correct_sky:
if src_name:
sky_correct = src_name
else:
srcs = meqmaker.get_source_list(ns)
sky_correct = srcs and srcs[0]
else:
sky_correct = None
outputs = meqmaker.correct_uv_data(ns,
outputs,
sky_correct=sky_correct)
# make solve trees
if do_solve:
# extract selected correlations
if cal_corr != ALL_CORRS:
index = [CORR_INDICES[c] for c in selected_corrs]
for p, q in array.ifrs():
ns.sel_predict(p, q) << Meq.Selector(
predict(p, q), index=index, multi=True)
ns.sel_spigot(p, q) << Meq.Selector(
spigots(p, q), index=index, multi=True)
spigots = ns.sel_spigot
predict = ns.sel_predict
model = predict
observed = spigots
# make a solve tree
solve_tree = Meow.StdTrees.SolveTree(ns, model)
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=observed, outputs=outputs)
# throw in a bit of noise
if not do_not_simulate and noise_stddev:
# make two complex noise terms per station (x/y)
noisedef = Meq.GaussNoise(stddev=noise_stddev)
noise_x = ns.sta_noise('x')
noise_y = ns.sta_noise('y')
for p in array.stations():
noise_x(p) << Meq.ToComplex(noisedef, noisedef)
noise_y(p) << Meq.ToComplex(noisedef, noisedef)
# now combine them into per-baseline noise matrices
for p, q in array.ifrs():
noise = ns.noise(p, q) << Meq.Matrix22(
noise_x(p) + noise_x(q),
noise_x(p) + noise_y(q),
noise_y(p) + noise_x(q),
noise_y(p) + noise_y(q))
ns.noisy_predict(p, q) << outputs(p, q) + noise
outputs = ns.noisy_predict
# make sinks and vdm.
# The list of inspectors comes in handy here
Meow.StdTrees.make_sinks(ns,
outputs,
spigots=None,
post=meqmaker.get_inspectors())
if not do_not_simulate:
# add simulate job
TDLRuntimeJob(job_simulate, "Simulate")
if do_not_simulate and not do_solve:
# add subtract or correct job
TDLRuntimeJob(job_subtract, "Subtract or Correct the data")
if do_not_simulate and do_solve:
pg_iono = ParmGroup.ParmGroup("Z_iono",
outputs.search(tags="solvable Z"),
table_name="iono.mep",
bookmark=4)
ParmGroup.SolveJob("cal_iono", "Calibrate Ionosphere parameters ",
pg_iono)
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
# TDLRuntimeOptions(*meqmaker.runtime_options(nest=False));
# and insert all solvejobs
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu("Imaging options", *imsel.option_list())
def init_parameters (self,ns,sources,stations,inspectors=[]):
if self.beamshape is not None:
return;
# create solvables if enabled
parms = [];
parmgroups = [];
parmdef_scale = Meq.Parm(self.bf*1e-9,tags="beam solvable");
parmdef_ell = Meq.Parm(self.ellipticity,tags="beam solvable");
self.per_station = False;
# solvable beam scale
if self.solve_scale is PER_STATION:
self.beamshape = ns.beamshape;
for p in stations:
parms.append(beamshape(p) << parmdef_scale);
inspectors.append(ns.inspector("scale") <<
StdTrees.define_inspector(ns.beamshape,stations,label=self.label));
self.per_station = True;
elif self.solve_scale is PER_ARRAY:
parms.append(ns.beamshape << parmdef_scale);
self.beamshape = lambda p:ns.beamshape;
else:
ns.beamshape ** (self.bf*1e-9);
self.beamshape = lambda p:ns.beamshape;
# solvable ellipticities
if self.solve_ell is PER_STATION:
self.ell = ns.ell;
for p in stations:
ell_xy = ns.ell_xy(p) << parmdef_ell;
parms.append(ell_xy);
self.ell(p) << Meq.Composer(ell_xy,-ell_xy);
inspectors.append(ns.inspector("ellipticity") <<
StdTrees.define_inspector(ns.ell_xy,stations,label=self.label));
self.per_station = True;
elif self.solve_ell is PER_ARRAY:
ell_xy = ns.ell_xy << parmdef_ell;
parms.append(ell_xy);
ns.ell << Meq.Composer(ell_xy,-ell_xy);
self.ell = lambda p:ns.ell;
elif self.ellipticity != 0:
ns.ell ** Meq.Constant([self.ellipticity,-self.ellipticity]);
self.ell = lambda p:ns.ell;
else:
self.ell = lambda p:None;
# make parm group, if any solvables have been created
if parms:
parmgroups.append(ParmGroup.Subgroup("beam shape",list(parms)));
# solvable pointings
if self.solve_pointings:
self.dlm = ns.dlm;
parmdef_0 = Meq.Parm(0,tags="beam solvable");
pparms = [];
for p in stations:
dl = ns.dl(p) << parmdef_0;
dm = ns.dm(p) << parmdef_0;
ns.dlm(p) << Meq.Composer(dl,dm);
pparms += [dl,dm];
parmgroups.append(ParmGroup.Subgroup("pointing offsets",pparms));
parms += pparms;
inspectors.append(ns.inspector('dlm') <<
StdTrees.define_inspector(ns.dlm,stations,label=self.label));
self.per_station = True;
else:
ns.dlm_null ** Meq.Constant([0,0]);
self.dlm = lambda p:ns.dlm_null;
# add solve jobs
self.solvable = bool(parms);
if self.solvable:
self.pg_beam = ParmGroup.ParmGroup(self.label,parms,subgroups=parmgroups,table_name="%s.fmep"%self.label,bookmark=True);
ParmGroup.SolveJob("cal_"+self.label,"Calibrate beam parameters",self.pg_beam);
def _define_forest(ns,parent=None,**kw):
if run_purr:
Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']);
# create Purr pipe
global purrpipe;
purrpipe = Purr.Pipe.Pipe(mssel.msname);
# get antennas from MS
ANTENNAS = mssel.get_antenna_set(list(range(1,15)));
array = Meow.IfrArray(ns,ANTENNAS,mirror_uvw=False);
stas = array.stations();
# get phase centre from MS, setup observation
observation = Meow.Observation(ns,phase_centre=mssel.get_phase_dir(),
linear=mssel.is_linear_pol(),
circular=mssel.is_circular_pol());
Meow.Context.set(array,observation);
# get active correlations from MS
Meow.Context.active_correlations = mssel.get_correlations();
# make spigot nodes
spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index());
# ...and an inspector for them
StdTrees.vis_inspector(ns.inspector('input'),spigots,
bookmark="Inspect input visibilities");
inspectors = [ ns.inspector('input') ];
Bookmarks.make_node_folder("Input visibilities by baseline",
[ spigots(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2);
inspect_ifrs = array.ifrs();
if do_solve:
# filter solvable baselines by baseline length
solve_ifrs = [];
antpos = mssel.ms_antenna_positions;
if (min_baseline or max_baseline) and antpos is not None:
for (ip,p),(iq,q) in array.ifr_index():
baseline = math.sqrt(((antpos[ip,:]-antpos[iq,:])**2).sum());
if (not min_baseline or baseline > min_baseline) and \
(not max_baseline or baseline < max_baseline):
solve_ifrs.append((p,q));
else:
solve_ifrs = array.ifrs();
inspect_ifrs = solve_ifrs;
# make a predict tree using the MeqMaker
if do_solve or do_subtract:
predict = meqmaker.make_predict_tree(ns);
# make a ParmGroup and solve jobs for source parameters, if we have any
if do_solve:
parms = {};
for src in meqmaker.get_source_list(ns):
parms.update([(p.name,p) for p in src.get_solvables()]);
if parms:
pg_src = ParmGroup.ParmGroup("source",list(parms.values()),
table_name="sources.fmep",
individual=True,bookmark=True);
# now make a solvejobs for the source
ParmGroup.SolveJob("cal_source","Calibrate source model",pg_src);
# make nodes to compute residuals
if do_subtract:
residuals = ns.residuals;
for p,q in array.ifrs():
residuals(p,q) << spigots(p,q) - predict(p,q);
outputs = residuals;
# and now we may need to correct the outputs
if do_correct:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns);
sky_correct = srcs and srcs[0];
else:
sky_correct = None;
outputs = meqmaker.correct_uv_data(ns,outputs,sky_correct=sky_correct,inspect_ifrs=inspect_ifrs);
# make solve trees
if do_solve:
# inputs to the solver are based on calibration type
# if calibrating visibilities, feed them to condeq directly
if cal_type == CAL.VIS:
observed = spigots;
model = predict;
# else take ampl/phase component
else:
model = ns.model;
observed = ns.observed;
if cal_type == CAL.AMPL:
for p,q in array.ifrs():
observed(p,q) << Meq.Abs(spigots(p,q));
model(p,q) << Meq.Abs(predict(p,q));
elif cal_type == CAL.LOGAMPL:
for p,q in array.ifrs():
observed(p,q) << Meq.Log(Meq.Abs(spigots(p,q)));
model(p,q) << Meq.Log(Meq.Abs(predict(p,q)));
elif cal_type == CAL.PHASE:
for p,q in array.ifrs():
observed(p,q) << 0;
model(p,q) << Meq.Abs(predict(p,q))*Meq.FMod(Meq.Arg(spigots(p,q))-Meq.Arg(predict(p,q)),2*math.pi);
else:
raise ValueError("unknown cal_type setting: "+str(cal_type));
# make a solve tree
solve_tree = StdTrees.SolveTree(ns,model,solve_ifrs=solve_ifrs);
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=observed,outputs=outputs);
# make sinks and vdm.
# The list of inspectors must be supplied here
inspectors += meqmaker.get_inspectors() or [];
StdTrees.make_sinks(ns,outputs,spigots=spigots0,post=inspectors);
Bookmarks.make_node_folder("Corrected/residual visibilities by baseline",
[ outputs(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2);
if not do_solve:
if do_subtract:
name = "Generate residuals";
comment = "Generated residual visibilities.";
elif do_correct:
name = "Generate corrected data";
comment = "Generated corrected visibilities.";
else:
name = None;
if name:
# make a TDL job to runsthe tree
def run_tree (mqs,parent,**kw):
global tile_size;
purrpipe.title("Calibrating").comment(comment);
mqs.execute(Meow.Context.vdm.name,mssel.create_io_request(tile_size),wait=False);
TDLRuntimeMenu(name,
TDLOption('tile_size',"Tile size, in timeslots",[10,60,120,240],more=int,
doc="""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""),
TDLRuntimeJob(run_tree,name)
);
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False));
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options());
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size());
TDLRuntimeMenu("Make an image from this MS",*imsel.option_list());
# and close meqmaker -- this exports annotations, etc
meqmaker.close();
def _define_forest(ns):
#make pynodes, xyzcomponent for sources
ANTENNAS = mssel.get_antenna_set(range(1,15));
array = Meow.IfrArray(ns,ANTENNAS,mirror_uvw=False);
observation = Meow.Observation(ns);
Meow.Context.set(array,observation);
# make a predict tree using the MeqMaker
if do_solve or do_subtract or not do_not_simulate:
outputs=predict = meqmaker.make_tree(ns);
#make a list of selected corrs
selected_corrs = cal_corr.split(" ");
# make spigot nodes
if not do_not_simulate and do_add:
spigots = spigots0 = outputs = array.spigots();
sums = ns.sums;
for p,q in array.ifrs():
sums(p,q) << spigots(p,q) + predict(p,q);
outputs = sums;
# make spigot nodes
if do_not_simulate:
spigots = spigots0 = outputs = array.spigots();
# make nodes to compute residuals
# make nodes to compute residuals
if do_subtract:
residuals = ns.residuals;
for p,q in array.ifrs():
residuals(p,q) << spigots(p,q) - predict(p,q);
outputs = residuals;
# and now we may need to correct the outputs
if do_correct:
if do_correct_sky:
if src_name:
sky_correct = src_name;
else:
srcs = meqmaker.get_source_list(ns);
sky_correct = srcs and srcs[0];
else:
sky_correct = None;
outputs = meqmaker.correct_uv_data(ns,outputs,sky_correct=sky_correct);
# make solve trees
if do_solve:
# extract selected correlations
if cal_corr != ALL_CORRS:
index = [ CORR_INDICES[c] for c in selected_corrs ];
for p,q in array.ifrs():
ns.sel_predict(p,q) << Meq.Selector(predict(p,q),index=index,multi=True);
ns.sel_spigot(p,q) << Meq.Selector(spigots(p,q),index=index,multi=True);
spigots = ns.sel_spigot;
predict = ns.sel_predict;
model = predict;
observed = spigots;
# make a solve tree
solve_tree = Meow.StdTrees.SolveTree(ns,model);
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=observed,outputs=outputs);
# throw in a bit of noise
if not do_not_simulate and noise_stddev:
# make two complex noise terms per station (x/y)
noisedef = Meq.GaussNoise(stddev=noise_stddev)
noise_x = ns.sta_noise('x');
noise_y = ns.sta_noise('y');
for p in array.stations():
noise_x(p) << Meq.ToComplex(noisedef,noisedef);
noise_y(p) << Meq.ToComplex(noisedef,noisedef);
# now combine them into per-baseline noise matrices
for p,q in array.ifrs():
noise = ns.noise(p,q) << Meq.Matrix22(
noise_x(p)+noise_x(q),noise_x(p)+noise_y(q),
noise_y(p)+noise_x(q),noise_y(p)+noise_y(q)
);
ns.noisy_predict(p,q) << outputs(p,q) + noise;
outputs = ns.noisy_predict;
# make sinks and vdm.
# The list of inspectors comes in handy here
Meow.StdTrees.make_sinks(ns,outputs,spigots=None,post=meqmaker.get_inspectors());
if not do_not_simulate:
#add simulate job
TDLRuntimeJob(job_simulate,"Simulate");
if do_not_simulate and not do_solve:
#add subtract or correct job
TDLRuntimeJob(job_subtract,"Subtract or Correct the data");
if do_not_simulate and do_solve:
pg_iono = ParmGroup.ParmGroup("Z_iono",
outputs.search(tags="solvable Z"),
table_name="iono.mep",bookmark=4);
ParmGroup.SolveJob("cal_iono","Calibrate Ionosphere parameters ",pg_iono);
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
#TDLRuntimeOptions(*meqmaker.runtime_options(nest=False));
# and insert all solvejobs
TDLRuntimeOptions(*ParmGroup.get_solvejob_options());
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size());
TDLRuntimeMenu("Imaging options",*imsel.option_list());
def _define_forest (ns, parent=None, **kw):
if not mssel.msname:
raise RuntimeError ('MS name not set')
mssel.setup_observation_context (ns)
array = Context.array
# Data and model input
if do_solve or output_type.need_data:
mssel.enable_input_column (True)
spigots = array.spigots (corr=mssel.get_corr_index ())
meqmaker.make_per_ifr_bookmarks (spigots, 'Input visibilities')
else:
mssel.enable_input_column (False)
spigots = None
if do_solve or output_type.need_model:
predict = meqmaker.make_predict_tree (ns, uvdata=None)
else:
predict = None
# Data output
outputs = output_type.apply (ns, meqmaker, array.ifrs (), spigots, predict)
# Flagging
if flag_enable and output_type.flag_data:
flaggers = []
if flag_res is not None or flag_mean_res is not None:
for p, q in array.ifrs ():
ns.absres(p,q) << Meq.Abs (outputs(p,q))
if flag_res is not None:
for p, q in array.ifrs ():
ns.flagres(p,q) << Meq.ZeroFlagger (ns.absres(p,q) - flag_res,
oper='gt',
flag_bit=MSUtils.FLAGMASK_OUTPUT)
flaggers.append (ns.flagres)
meqmaker.make_per_ifr_bookmarks (ns.flagres, 'Residual amplitude flags')
if flag_mean_res is not None:
ns.meanabsres << Meq.Mean (*[ns.absres(p,q) for p, q in array.ifrs()])
ns.flagmeanres << Meq.ZeroFlagger (ns.meanabsres - flag_mean_res,
oper='gt', flag_bit=MSUtils.FLAGMASK_OUTPUT)
Bookmarks.Page ('Mean residual amplitude flags').add (ns.flagmeanres,
viewer='Result Plotter')
flaggers.append (lambda p, q: ns.flagmeanres)
if flaggers:
meqmaker.make_per_ifr_bookmarks (outputs, output_type.desc + ' (unflagged)')
for p, q in array.ifrs ():
ns.flagged(p,q) << Meq.MergeFlags (outputs(p,q), *[f(p,q) for f in flaggers])
outputs = ns.flagged
meqmaker.make_per_ifr_bookmarks (outputs, output_type.desc)
# Solve trees
if do_solve:
# parse ifr specification
solve_ifrs = array.subset (calibrate_ifrs, strict=False).ifrs()
if not solve_ifrs:
raise RuntimeError ('No interferometers selected for calibration. '
'Check your ifr specification (under calibration options).')
lhs, rhs, weights, modulo = cal_quant.apply (solve_ifrs, predict, spigots)
solve_tree = StdTrees.SolveTree (ns, lhs, solve_ifrs=solve_ifrs,
weights=weights, modulo=modulo)
outputs = solve_tree.sequencers (inputs=rhs, outputs=outputs)
StdTrees.make_sinks (ns, outputs, spigots=spigots,
post=meqmaker.get_inspectors () or [],
corr_index=mssel.get_corr_index ())
if not do_solve:
name = 'Generate ' + output_type.desc.lower ()
comment = 'Generated ' + output_type.desc.lower ()
def run_tree (mqs, parent, wait=False, **kw):
return mqs.execute (Context.vdm.name, mssel.create_io_request (tile_size),
wait=wait)
doc = """Input data are sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
TDLRuntimeMenu(name, TDLOption ('tile_size', 'Tile size, in timeslots',
[10, 60, 120, 240], more=int, doc=doc),
TDLJob (run_tree, name, job_id='generate_visibilities'))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up
# during compilation.
TDLRuntimeOptions (*meqmaker.runtime_options (nest=False))
if do_solve:
TDLRuntimeOptions (*ParmGroup.get_solvejob_options ())
imsel = mssel.imaging_selector (npix=512, arcmin=meqmaker.estimate_image_size ())
TDLRuntimeMenu ('Make an image', *imsel.option_list ())
meqmaker.close()
def _define_forest(ns, parent=None, **kw):
if not mssel.msname:
raise RuntimeError('MS name not set')
mssel.setup_observation_context(ns)
array = Context.array
# Data and model input
if do_solve or output_type.need_data:
mssel.enable_input_column(True)
spigots = array.spigots(corr=mssel.get_corr_index())
meqmaker.make_per_ifr_bookmarks(spigots, 'Input visibilities')
else:
mssel.enable_input_column(False)
spigots = None
if do_solve or output_type.need_model:
predict = meqmaker.make_predict_tree(ns, uvdata=None)
else:
predict = None
# Data output
outputs = output_type.apply(ns, meqmaker, array.ifrs(), spigots, predict)
# Flagging
if flag_enable and output_type.flag_data:
flaggers = []
if flag_res is not None or flag_mean_res is not None:
for p, q in array.ifrs():
ns.absres(p, q) << Meq.Abs(outputs(p, q))
if flag_res is not None:
for p, q in array.ifrs():
ns.flagres(p, q) << Meq.ZeroFlagger(
ns.absres(p, q) - flag_res,
oper='gt',
flag_bit=MSUtils.FLAGMASK_OUTPUT)
flaggers.append(ns.flagres)
meqmaker.make_per_ifr_bookmarks(ns.flagres,
'Residual amplitude flags')
if flag_mean_res is not None:
ns.meanabsres << Meq.Mean(
*[ns.absres(p, q) for p, q in array.ifrs()])
ns.flagmeanres << Meq.ZeroFlagger(ns.meanabsres - flag_mean_res,
oper='gt',
flag_bit=MSUtils.FLAGMASK_OUTPUT)
Bookmarks.Page('Mean residual amplitude flags').add(
ns.flagmeanres, viewer='Result Plotter')
flaggers.append(lambda p, q: ns.flagmeanres)
if flaggers:
meqmaker.make_per_ifr_bookmarks(outputs,
output_type.desc + ' (unflagged)')
for p, q in array.ifrs():
ns.flagged(p, q) << Meq.MergeFlags(
outputs(p, q), *[f(p, q) for f in flaggers])
outputs = ns.flagged
meqmaker.make_per_ifr_bookmarks(outputs, output_type.desc)
# Solve trees
if do_solve:
# parse ifr specification
solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs()
if not solve_ifrs:
raise RuntimeError(
'No interferometers selected for calibration. '
'Check your ifr specification (under calibration options).')
lhs, rhs, weights, modulo = cal_quant.apply(solve_ifrs, predict,
spigots)
solve_tree = StdTrees.SolveTree(ns,
lhs,
solve_ifrs=solve_ifrs,
weights=weights,
modulo=modulo)
outputs = solve_tree.sequencers(inputs=rhs, outputs=outputs)
StdTrees.make_sinks(ns,
outputs,
spigots=spigots,
post=meqmaker.get_inspectors() or [],
corr_index=mssel.get_corr_index())
if not do_solve:
name = 'Generate ' + output_type.desc.lower()
comment = 'Generated ' + output_type.desc.lower()
def run_tree(mqs, parent, wait=False, **kw):
return mqs.execute(Context.vdm.name,
mssel.create_io_request(tile_size),
wait=wait)
doc = """Input data are sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
TDLRuntimeMenu(
name,
TDLOption('tile_size',
'Tile size, in timeslots', [10, 60, 120, 240],
more=int,
doc=doc),
TDLJob(run_tree, name, job_id='generate_visibilities'))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up
# during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu('Make an image', *imsel.option_list())
meqmaker.close()
def _define_forest(ns,parent=None,**kw):
if not mssel.msname:
raise RuntimeError,"MS not set";
if run_purr:
Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']);
# create Purr pipe
global purrpipe;
purrpipe = Purr.Pipe.Pipe(mssel.msname);
# setup contexts from MS
mssel.setup_observation_context(ns,prefer_baseline_uvw=True);
array = Meow.Context.array;
# make spigot nodes for data
if do_solve or do_output not in [CORRUPTED_MODEL]:
mssel.enable_input_column(True);
spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index());
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(spigots,"Input visibilities");
# add IFR-based errors, if any
spigots = meqmaker.apply_visibility_processing(ns,spigots);
else:
mssel.enable_input_column(False);
spigots = spigots0 = None;
# make spigot nodes for model
corrupt_uvdata = model_spigots = None;
if read_ms_model:
mssel.enable_model_column(True);
model_spigots = array.spigots(column="PREDICT",corr=mssel.get_corr_index());
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(model_spigots,"UV-model visibilities");
# if calibrating on (input-corrupt model), make corrupt model
if do_solve and cal_type == CAL.DIFF:
corrupt_uvdata = meqmaker.corrupt_uv_data(ns,model_spigots);
# if needed, then make a predict tree using the MeqMaker
if do_solve or do_output != CORRECTED_DATA:
if model_spigots and not corrupt_uvdata:
uvdata = model_spigots;
else:
uvdata = None;
predict = meqmaker.make_predict_tree(ns,uvdata=uvdata);
else:
predict = None;
output_title = "Uncorrected residuals";
# make nodes to compute residuals
if do_output in [CORRECTED_RESIDUALS,RESIDUALS]:
residuals = ns.residuals;
for p,q in array.ifrs():
if corrupt_uvdata:
residuals(p,q) << Meq.Subtract(spigots(p,q),corrupt_uvdata(p,q),predict(p,q));
else:
residuals(p,q) << spigots(p,q) - predict(p,q);
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(residuals,"Uncorrected residuals");
outputs = residuals;
# and now we may need to correct the outputs
if do_output in [CORRECTED_DATA,CORRECTED_RESIDUALS]:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns);
if do_correct_sky is FIRST_SOURCE:
sky_correct = srcs and srcs[0];
else:
srcs = [ src for src in srcs if fnmatch.fnmatchcase(src.name,do_correct_sky) ];
sky_correct = srcs and srcs[0];
else:
sky_correct = None;
outputs = meqmaker.correct_uv_data(ns,outputs,sky_correct=sky_correct,
flag_jones=flag_jones);
output_title = "Corrected data" if do_output is CORRECTED_DATA else "Corrected residuals";
elif do_output == CORRUPTED_MODEL:
outputs = predict;
output_title = "Predict";
elif do_output == CORRUPTED_MODEL_ADD:
outputs = ns.output;
for p,q in array.ifrs():
outputs(p,q) << spigots(p,q) + predict(p,q);
output_title = "Data+predict";
# make flaggers
if flag_enable and do_output in [ CORRECTED_DATA,RESIDUALS,CORRECTED_RESIDUALS ]:
flaggers = [];
if flag_res is not None or flag_mean_res is not None:
for p,q in array.ifrs():
ns.absres(p,q) << Meq.Abs(outputs(p,q));
# make flagger for residuals
if flag_res is not None:
for p,q in array.ifrs():
ns.flagres(p,q) << Meq.ZeroFlagger(ns.absres(p,q)-flag_res,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT);
flaggers.append(ns.flagres);
# ...and an inspector for them
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(ns.flagres,"Residual amplitude flags");
# make flagger for mean residuals
if flag_mean_res is not None:
ns.meanabsres << Meq.Mean(*[ns.absres(p,q) for p,q in array.ifrs()]);
ns.flagmeanres << Meq.ZeroFlagger(ns.meanabsres-flag_mean_res,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT);
Meow.Bookmarks.Page("Mean residual amplitude flags").add(ns.flagmeanres,viewer="Result Plotter");
flaggers.append(lambda p,q:ns.flagmeanres);
# merge flags into output
if flaggers:
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(outputs,output_title+" (unflagged)");
for p,q in array.ifrs():
ns.flagged(p,q) << Meq.MergeFlags(outputs(p,q),*[f(p,q) for f in flaggers]);
outputs = ns.flagged;
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(outputs,output_title);
abs_outputs = outputs('abs');
for p,q in array.ifrs():
abs_outputs(p,q) << Meq.Abs(outputs(p,q));
meqmaker.make_per_ifr_bookmarks(abs_outputs,output_title+" (mean amplitudes)");
# make solve trees
if do_solve:
# parse ifr specification
solve_ifrs = array.subset(calibrate_ifrs,strict=False).ifrs();
if not solve_ifrs:
raise RuntimeError,"No interferometers selected for calibration. Check your ifr specification (under calibration options).";
# inputs to the solver are based on calibration type
if corrupt_uvdata:
[ ns.diff(p,q) << spigots(p,q) - corrupt_uvdata(p,q) for p,q in solve_ifrs ];
rhs = ns.diff;
else:
rhs = spigots;
lhs = predict;
weights = modulo = None;
# if calibrating visibilities, feed them to condeq directly, else take ampl/phase
if cal_what == CAL.VIS:
pass;
elif cal_what == CAL.AMPL:
[ x('ampl',p,q) << Meq.Abs(x(p,q)) for p,q in ifrs for x in rhs,lhs ];
lhs = lhs('ampl');
rhs = rhs('ampl');
elif cal_what == CAL.LOGAMPL:
[ x('logampl',p,q) << Meq.Log(Meq.Abs(x(p,q))) for p,q in ifrs for x in rhs,lhs ];
lhs = lhs('logampl');
rhs = rhs('logampl');
elif cal_what == CAL.PHASE:
[ x('phase',p,q) << Meq.Arg(x(p,q)) for p,q in ifrs for x in rhs,lhs ];
[ rhs('ampl',p,q) << Meq.Abs(rhs(p,q)) for p,q in ifrs ];
lhs = lhs('phase');
rhs = rhs('phase');
weights = rhs('ampl');
modulo = 2*math.pi;
else:
raise ValueError,"unknown cal_what setting: "+str(cal_what);
# make a solve tree
solve_tree = StdTrees.SolveTree(ns,lhs,solve_ifrs=solve_ifrs,weights=weights,modulo=modulo);
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=rhs,outputs=outputs);
post = ( ( enable_inspectors and meqmaker.get_inspectors() ) or [] );
StdTrees.make_sinks(ns,outputs,spigots=spigots0,post=post,corr_index=mssel.get_corr_index());
if not do_solve:
name = "Generate "+output_title.lower();
comment = "Generated "+output_title.lower();
if name:
# make a TDL job to run the tree
def run_tree (mqs,parent,wait=False,**kw):
global tile_size;
purrpipe.title("Calibrating").comment(comment);
return mqs.execute(Meow.Context.vdm.name,mssel.create_io_request(tile_size),wait=wait);
TDLRuntimeMenu(name,
TDLOption('tile_size',"Tile size, in timeslots",[10,60,120,240],more=int,
doc="""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""),
TDLJob(run_tree,name,job_id='generate_visibilities')
);
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False));
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options());
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size());
TDLRuntimeMenu("Make an image from this MS",*imsel.option_list());
# and close meqmaker -- this exports annotations, etc
meqmaker.close();
def _define_forest(ns, parent=None, **kw):
if not mssel.msname:
raise RuntimeError("MS not set")
if run_purr:
Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.'])
# create Purr pipe
global purrpipe
purrpipe = Purr.Pipe.Pipe(mssel.msname)
# setup contexts from MS
mssel.setup_observation_context(ns, prefer_baseline_uvw=True)
array = Meow.Context.array
# make spigot nodes for data
if do_solve or do_output not in [CORRUPTED_MODEL]:
mssel.enable_input_column(True)
spigots = spigots0 = outputs = array.spigots(
corr=mssel.get_corr_index())
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(spigots, "Input visibilities")
# add IFR-based errors, if any
spigots = meqmaker.apply_visibility_processing(ns, spigots)
else:
mssel.enable_input_column(False)
spigots = spigots0 = None
# make spigot nodes for model
corrupt_uvdata = model_spigots = None
if read_ms_model:
mssel.enable_model_column(True)
model_spigots = array.spigots(column="PREDICT",
corr=mssel.get_corr_index())
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(model_spigots,
"UV-model visibilities")
# if calibrating on (input-corrupt model), make corrupt model
if do_solve and cal_type == CAL.DIFF:
corrupt_uvdata = meqmaker.corrupt_uv_data(ns, model_spigots)
# if needed, then make a predict tree using the MeqMaker
if do_solve or do_output != CORRECTED_DATA:
if model_spigots and not corrupt_uvdata:
uvdata = model_spigots
else:
uvdata = None
predict = meqmaker.make_predict_tree(ns, uvdata=uvdata)
else:
predict = None
output_title = "Uncorrected residuals"
# make nodes to compute residuals
if do_output in [CORRECTED_RESIDUALS, RESIDUALS]:
residuals = ns.residuals
for p, q in array.ifrs():
if corrupt_uvdata:
residuals(p, q) << Meq.Subtract(spigots(
p, q), corrupt_uvdata(p, q), predict(p, q))
else:
residuals(p, q) << spigots(p, q) - predict(p, q)
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(residuals, "Uncorrected residuals")
outputs = residuals
# and now we may need to correct the outputs
if do_output in [CORRECTED_DATA, CORRECTED_RESIDUALS]:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns)
if do_correct_sky is FIRST_SOURCE:
sky_correct = srcs and srcs[0]
else:
srcs = [
src for src in srcs
if fnmatch.fnmatchcase(src.name, do_correct_sky)
]
sky_correct = srcs and srcs[0]
else:
sky_correct = None
outputs = meqmaker.correct_uv_data(ns,
outputs,
sky_correct=sky_correct,
flag_jones=flag_jones)
output_title = "Corrected data" if do_output is CORRECTED_DATA else "Corrected residuals"
elif do_output == CORRUPTED_MODEL:
outputs = predict
output_title = "Predict"
elif do_output == CORRUPTED_MODEL_ADD:
outputs = ns.output
for p, q in array.ifrs():
outputs(p, q) << spigots(p, q) + predict(p, q)
output_title = "Data+predict"
# make flaggers
if flag_enable and do_output in [
CORRECTED_DATA, RESIDUALS, CORRECTED_RESIDUALS
]:
flaggers = []
if flag_res is not None or flag_mean_res is not None:
for p, q in array.ifrs():
ns.absres(p, q) << Meq.Abs(outputs(p, q))
# make flagger for residuals
if flag_res is not None:
for p, q in array.ifrs():
ns.flagres(p, q) << Meq.ZeroFlagger(
ns.absres(p, q) - flag_res,
oper='gt',
flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT)
flaggers.append(ns.flagres)
# ...and an inspector for them
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(ns.flagres,
"Residual amplitude flags")
# make flagger for mean residuals
if flag_mean_res is not None:
ns.meanabsres << Meq.Mean(
*[ns.absres(p, q) for p, q in array.ifrs()])
ns.flagmeanres << Meq.ZeroFlagger(
ns.meanabsres - flag_mean_res,
oper='gt',
flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT)
Meow.Bookmarks.Page("Mean residual amplitude flags").add(
ns.flagmeanres, viewer="Result Plotter")
flaggers.append(lambda p, q: ns.flagmeanres)
# merge flags into output
if flaggers:
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(outputs,
output_title + " (unflagged)")
for p, q in array.ifrs():
ns.flagged(p, q) << Meq.MergeFlags(
outputs(p, q), *[f(p, q) for f in flaggers])
outputs = ns.flagged
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(outputs, output_title)
abs_outputs = outputs('abs')
for p, q in array.ifrs():
abs_outputs(p, q) << Meq.Abs(outputs(p, q))
meqmaker.make_per_ifr_bookmarks(abs_outputs,
output_title + " (mean amplitudes)")
# make solve trees
if do_solve:
# parse ifr specification
solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs()
if not solve_ifrs:
raise RuntimeError(
"No interferometers selected for calibration. Check your ifr specification (under calibration options)."
)
# inputs to the solver are based on calibration type
if corrupt_uvdata:
[
ns.diff(p, q) << spigots(p, q) - corrupt_uvdata(p, q)
for p, q in solve_ifrs
]
rhs = ns.diff
else:
rhs = spigots
lhs = predict
weights = modulo = None
# if calibrating visibilities, feed them to condeq directly, else take ampl/phase
if cal_what == CAL.VIS:
pass
elif cal_what == CAL.AMPL:
[
x('ampl', p, q) << Meq.Abs(x(p, q)) for p, q in ifrs
for x in [rhs, lhs]
]
lhs = lhs('ampl')
rhs = rhs('ampl')
elif cal_what == CAL.LOGAMPL:
[
x('logampl', p, q) << Meq.Log(Meq.Abs(x(p, q)))
for p, q in ifrs for x in [rhs, lhs]
]
lhs = lhs('logampl')
rhs = rhs('logampl')
elif cal_what == CAL.PHASE:
[
x('phase', p, q) << Meq.Arg(x(p, q)) for p, q in ifrs
for x in [rhs, lhs]
]
[rhs('ampl', p, q) << Meq.Abs(rhs(p, q)) for p, q in ifrs]
lhs = lhs('phase')
rhs = rhs('phase')
weights = rhs('ampl')
modulo = 2 * math.pi
else:
raise ValueError("unknown cal_what setting: " + str(cal_what))
# make a solve tree
solve_tree = StdTrees.SolveTree(ns,
lhs,
solve_ifrs=solve_ifrs,
weights=weights,
modulo=modulo)
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=rhs, outputs=outputs)
post = ((enable_inspectors and meqmaker.get_inspectors()) or [])
StdTrees.make_sinks(ns,
outputs,
spigots=spigots0,
post=post,
corr_index=mssel.get_corr_index())
if not do_solve:
name = "Generate " + output_title.lower()
comment = "Generated " + output_title.lower()
if name:
# make a TDL job to run the tree
def run_tree(mqs, parent, wait=False, **kw):
global tile_size
purrpipe.title("Calibrating").comment(comment)
return mqs.execute(Meow.Context.vdm.name,
mssel.create_io_request(tile_size),
wait=wait)
TDLRuntimeMenu(
name,
TDLOption(
'tile_size',
"Tile size, in timeslots", [10, 60, 120, 240],
more=int,
doc=
"""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
), TDLJob(run_tree, name, job_id='generate_visibilities'))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
# and close meqmaker -- this exports annotations, etc
meqmaker.close()
def compute_jones(Jones,
sources,
stations=None,
inspectors=[],
meqmaker=None,
label='R',
**kw):
"""Creates the Z Jones for ionospheric phase, given TECs (per source,
per station)."""
stations = stations or Context.array.stations
ns = Jones.Subscope()
# get reference source
if ref_source:
# treat as index first
dir0 = None
try:
dir0 = sources[int(ref_source)].direction
except:
pass
# else treat as name, find in list
if not dir0:
for src0 in sources:
if src0.name == ref_source:
dir0 = src0.direction
break
# else treat as direction string
if not dir0:
ff = list(ref_source.split())
if len(ff) < 2 or len(ff) > 3:
raise RuntimeError(
"invalid reference dir '%s' specified for %s-Jones" %
(ref_source, label))
global dm
if not dm:
raise RuntimeError(
"pyrap measures module not available, cannot use direction strings for %s-Jones"
% label)
if len(ff) == 2:
ff = ['J2000'] + ff
# treat as direction measure
try:
dmdir = dm.direction(*ff)
except:
raise RuntimeError(
"invalid reference dir '%s' specified for %s-Jones" %
(ref_source, label))
# convert to J2000 and make direction object
dmdir = dm.measure(dmdir, 'J2000')
ra, dec = dm.getvalue(dmdir)[0].get_value(), dm.getvalue(
dmdir)[1].get_value()
dir0 = Meow.Direction(ns, "refdir", ra, dec, static=True)
else:
dir0 = Context.observation.phase_centre
# make refraction scale node
scale = ns.scale(0) << Meq.Parm(0, tags="refraction")
xyz0 = Context.array.xyz0()
if coord_approx:
# get PA, and assume it's the same over the whole field
pa = ns.pa0 << Meq.ParAngle(dir0.radec(), xyz0)
# second column of the Rot(-PA) matrix. Multiply this by del to get a rotation of (0,del) into the lm plane.
# The third component (0) is for convenience, as it immediately gives us dl,dm,dn, since we assume dn~0
rot_pa = ns.rotpa0 << Meq.Composer(Meq.Sin(pa), Meq.Cos(pa), 0)
# el0: elevation of field centre
el0 = dir0.el()
if do_extinction:
ns.inv_ext0 << Meq.Sin(el0)
# inverse of extinction towards el0
# station UVWs
uvw = Context.array.uvw()
# now loop over sources
for isrc, src in enumerate(sources):
# reference direction: no refraction at all
if src.direction is dir0:
for p in stations:
Jones(src, p) << 1
continue
# dEl is source elevation minus el0
# ddEl = scale*dEl: amount by which source refracts (negative means field is compressed)
el = src.direction.el()
ns.dEl(src) << el - el0
ddel = ns.ddEl(src) << ns.dEl(src) * scale
# get el1: refracted elevation angle
if not coord_approx or do_extinction:
el1 = ns.el1(src) << el + ddel
# compute extinction component
if do_extinction:
# compute inverse of extinction towards the refracted direction el1
iext = ns.inv_ext(src) << Meq.Sin(el1)
#
# and differential extinction is then ext1/ext0
ext = ns.dext(src) << ns.inv_ext0 / iext
# Compute dlmn offset in lm plane.
if coord_approx:
# Approximate mode: ddel is added to elevation, so to get the lm offset, we need
# to apply Rot(PA) to the column vector (0,ddel), and then take the sine of the result.
dlmn = ns.dlmn(src) << Meq.Sin(ddel * rot_pa)
else:
ns.azel1(src) << Meq.Composer(src.direction.az(), el1)
ns.radec1(src) << Meq.RADec(ns.azel1(src), xyz0)
ns.lmn1(src) << Meq.LMN(Context.observation.radec0(),
ns.radec1(src))
dlmn = ns.dlmn(src) << ns.lmn1(src) - src.lmn()
# get per-station phases
for p in stations:
if do_extinction:
Jones(src, p) << ext * (ns.phase(src, p) << Meq.VisPhaseShift(
lmn=dlmn, uvw=uvw(p)))
else:
Jones(src, p) << Meq.VisPhaseShift(lmn=dlmn, uvw=uvw(p))
# make bookmarks
srcnames = [src.name for src in sources]
meqmaker.make_bookmark_set(Jones, [(src, p) for src in srcnames
for p in stations],
"%s: inspector plot" % label,
"%s: by source-station" % label,
freqmean=True)
inspectors.append(ns.inspector(label,'scale') << \
StdTrees.define_inspector(ns.scale,[0],label=label))
inspectors.append(ns.inspector(label,'delta-el') << \
StdTrees.define_inspector(ns.ddEl,srcnames,label=label))
inspectors.append(ns.inspector(label,'delta-el') << \
StdTrees.define_inspector(ns.ddEl,srcnames,label=label))
inspectors.append(ns.inspector(label,'dlmn') << \
StdTrees.define_inspector(ns.dlmn,srcnames,label=label))
if do_extinction:
inspectors.append(ns.inspector(label,'inv-ext') << \
StdTrees.define_inspector(ns.inv_ext,srcnames,label=label))
inspectors.append(ns.inspector(label,'diff-ext') << \
StdTrees.define_inspector(ns.dext,srcnames,label=label))
# make parmgroups and solvejobs
global pg
pg = ParmGroup.ParmGroup(label, [scale],
table_name="%s.fmep" % label,
bookmark=False)
# make solvejobs
ParmGroup.SolveJob("cal_" + label,
"Calibrate %s (differential refraction)" % label, pg)
return Jones
