def _define_forest (ns):
# create Array object
num_antennas = 36 # for ASKAP simulation
xntd_list = [ str(i) for i in range(1,num_antennas+1) ];
array = Meow.IfrArray(ns,xntd_list,ms_uvw=True);
# create an Observation object
observation = Meow.Observation(ns);
# set global context
Meow.Context.set(array=array,observation=observation);
# create a source model and make list of corrupted sources
allsky = Meow.Patch(ns,'all',observation.phase_centre);
sources = sky_models.make_model(ns,"S");
for src in sources:
lm = src.direction.lm();
E = ns.E(src.name);
for p in array.stations():
pa= ns.ParAngle(p) << Meq.ParAngle(observation.phase_centre.radec(), array.xyz(p))
ns.CosPa(p) << Meq.Cos(pa)
ns.SinPa(p) << Meq.Sin(pa)
ns.rot_matrix(p) << Meq.Matrix22(ns.CosPa(p),-1.0 * ns.SinPa(p),ns.SinPa(p),ns.CosPa(p))
# compute "apparent" position of source per each antenna
lm_rot=ns.lm_rot(src.name,p) << Meq.MatrixMultiply(ns.rot_matrix(p),lm)
# compute E for apparent position
tdp_voltage_response(ns,src,p,E(p),lm_rot);
allsky.add(src.corrupt(E));
observed = allsky.visibilities();
# make some useful inspectors. Collect them into a list, since we need
# to give a list of 'post' nodes to make_sinks() below
pg = Bookmarks.Page("Inspectors",1,2);
inspectors = [];
inspectors.append(
Meow.StdTrees.vis_inspector(ns.inspect_observed,observed) );
pg.add(ns.inspect_observed,viewer="Collections Plotter");
Meow.StdTrees.make_sinks(ns,observed,spigots=False,post=inspectors);
def _define_forest(ns, parent=None, **kw):
if run_purr:
Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.'])
# create Purr pipe
global purrpipe
purrpipe = Purr.Pipe.Pipe(mssel.msname)
# get antennas from MS
ANTENNAS = mssel.get_antenna_set(list(range(1, 15)))
array = Meow.IfrArray(ns, ANTENNAS, mirror_uvw=False)
stas = array.stations()
# get phase centre from MS, setup observation
observation = Meow.Observation(ns,
phase_centre=mssel.get_phase_dir(),
linear=mssel.is_linear_pol(),
circular=mssel.is_circular_pol())
Meow.Context.set(array, observation)
# get active correlations from MS
Meow.Context.active_correlations = mssel.get_correlations()
# make spigot nodes
spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index())
# ...and an inspector for them
StdTrees.vis_inspector(ns.inspector('input'),
spigots,
bookmark="Inspect input visibilities")
inspectors = [ns.inspector('input')]
Bookmarks.make_node_folder("Input visibilities by baseline",
[spigots(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
inspect_ifrs = array.ifrs()
if do_solve:
# filter solvable baselines by baseline length
solve_ifrs = []
antpos = mssel.ms_antenna_positions
if (min_baseline or max_baseline) and antpos is not None:
for (ip, p), (iq, q) in array.ifr_index():
baseline = math.sqrt(
((antpos[ip, :] - antpos[iq, :])**2).sum())
if (not min_baseline or baseline > min_baseline) and \
(not max_baseline or baseline < max_baseline):
solve_ifrs.append((p, q))
else:
solve_ifrs = array.ifrs()
inspect_ifrs = solve_ifrs
# make a predict tree using the MeqMaker
if do_solve or do_subtract:
predict = meqmaker.make_predict_tree(ns)
# make a ParmGroup and solve jobs for source parameters, if we have any
if do_solve:
parms = {}
for src in meqmaker.get_source_list(ns):
parms.update([(p.name, p) for p in src.get_solvables()])
if parms:
pg_src = ParmGroup.ParmGroup("source",
list(parms.values()),
table_name="sources.fmep",
individual=True,
bookmark=True)
# now make a solvejobs for the source
ParmGroup.SolveJob("cal_source", "Calibrate source model",
pg_src)
# make nodes to compute residuals
if do_subtract:
residuals = ns.residuals
for p, q in array.ifrs():
residuals(p, q) << spigots(p, q) - predict(p, q)
outputs = residuals
# and now we may need to correct the outputs
if do_correct:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns)
sky_correct = srcs and srcs[0]
else:
sky_correct = None
outputs = meqmaker.correct_uv_data(ns,
outputs,
sky_correct=sky_correct,
inspect_ifrs=inspect_ifrs)
# make solve trees
if do_solve:
# inputs to the solver are based on calibration type
# if calibrating visibilities, feed them to condeq directly
if cal_type == CAL.VIS:
observed = spigots
model = predict
# else take ampl/phase component
else:
model = ns.model
observed = ns.observed
if cal_type == CAL.AMPL:
for p, q in array.ifrs():
observed(p, q) << Meq.Abs(spigots(p, q))
model(p, q) << Meq.Abs(predict(p, q))
elif cal_type == CAL.LOGAMPL:
for p, q in array.ifrs():
observed(p, q) << Meq.Log(Meq.Abs(spigots(p, q)))
model(p, q) << Meq.Log(Meq.Abs(predict(p, q)))
elif cal_type == CAL.PHASE:
for p, q in array.ifrs():
observed(p, q) << 0
model(p, q) << Meq.Abs(predict(p, q)) * Meq.FMod(
Meq.Arg(spigots(p, q)) - Meq.Arg(predict(p, q)),
2 * math.pi)
else:
raise ValueError("unknown cal_type setting: " + str(cal_type))
# make a solve tree
solve_tree = StdTrees.SolveTree(ns, model, solve_ifrs=solve_ifrs)
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=observed, outputs=outputs)
# make sinks and vdm.
# The list of inspectors must be supplied here
inspectors += meqmaker.get_inspectors() or []
StdTrees.make_sinks(ns, outputs, spigots=spigots0, post=inspectors)
Bookmarks.make_node_folder("Corrected/residual visibilities by baseline",
[outputs(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
if not do_solve:
if do_subtract:
name = "Generate residuals"
comment = "Generated residual visibilities."
elif do_correct:
name = "Generate corrected data"
comment = "Generated corrected visibilities."
else:
name = None
if name:
# make a TDL job to runsthe tree
def run_tree(mqs, parent, **kw):
global tile_size
purrpipe.title("Calibrating").comment(comment)
mqs.execute(Meow.Context.vdm.name,
mssel.create_io_request(tile_size),
wait=False)
TDLRuntimeMenu(
name,
TDLOption(
'tile_size',
"Tile size, in timeslots", [10, 60, 120, 240],
more=int,
doc=
"""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
), TDLRuntimeJob(run_tree, name))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
# and close meqmaker -- this exports annotations, etc
meqmaker.close()
def _define_forest(ns):
# make pynodes, xyzcomponent for sources
ANTENNAS = mssel.get_antenna_set(list(range(1, 15)))
array = Meow.IfrArray(ns, ANTENNAS, mirror_uvw=False)
observation = Meow.Observation(ns)
Meow.Context.set(array, observation)
# make a predict tree using the MeqMaker
if do_solve or do_subtract or not do_not_simulate:
outputs = predict = meqmaker.make_tree(ns)
# make a list of selected corrs
selected_corrs = cal_corr.split(" ")
# make spigot nodes
if not do_not_simulate and do_add:
spigots = spigots0 = outputs = array.spigots()
sums = ns.sums
for p, q in array.ifrs():
sums(p, q) << spigots(p, q) + predict(p, q)
outputs = sums
# make spigot nodes
if do_not_simulate:
spigots = spigots0 = outputs = array.spigots()
# make nodes to compute residuals
# make nodes to compute residuals
if do_subtract:
residuals = ns.residuals
for p, q in array.ifrs():
residuals(p, q) << spigots(p, q) - predict(p, q)
outputs = residuals
# and now we may need to correct the outputs
if do_correct:
if do_correct_sky:
if src_name:
sky_correct = src_name
else:
srcs = meqmaker.get_source_list(ns)
sky_correct = srcs and srcs[0]
else:
sky_correct = None
outputs = meqmaker.correct_uv_data(ns,
outputs,
sky_correct=sky_correct)
# make solve trees
if do_solve:
# extract selected correlations
if cal_corr != ALL_CORRS:
index = [CORR_INDICES[c] for c in selected_corrs]
for p, q in array.ifrs():
ns.sel_predict(p, q) << Meq.Selector(
predict(p, q), index=index, multi=True)
ns.sel_spigot(p, q) << Meq.Selector(
spigots(p, q), index=index, multi=True)
spigots = ns.sel_spigot
predict = ns.sel_predict
model = predict
observed = spigots
# make a solve tree
solve_tree = Meow.StdTrees.SolveTree(ns, model)
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=observed, outputs=outputs)
# throw in a bit of noise
if not do_not_simulate and noise_stddev:
# make two complex noise terms per station (x/y)
noisedef = Meq.GaussNoise(stddev=noise_stddev)
noise_x = ns.sta_noise('x')
noise_y = ns.sta_noise('y')
for p in array.stations():
noise_x(p) << Meq.ToComplex(noisedef, noisedef)
noise_y(p) << Meq.ToComplex(noisedef, noisedef)
# now combine them into per-baseline noise matrices
for p, q in array.ifrs():
noise = ns.noise(p, q) << Meq.Matrix22(
noise_x(p) + noise_x(q),
noise_x(p) + noise_y(q),
noise_y(p) + noise_x(q),
noise_y(p) + noise_y(q))
ns.noisy_predict(p, q) << outputs(p, q) + noise
outputs = ns.noisy_predict
# make sinks and vdm.
# The list of inspectors comes in handy here
Meow.StdTrees.make_sinks(ns,
outputs,
spigots=None,
post=meqmaker.get_inspectors())
if not do_not_simulate:
# add simulate job
TDLRuntimeJob(job_simulate, "Simulate")
if do_not_simulate and not do_solve:
# add subtract or correct job
TDLRuntimeJob(job_subtract, "Subtract or Correct the data")
if do_not_simulate and do_solve:
pg_iono = ParmGroup.ParmGroup("Z_iono",
outputs.search(tags="solvable Z"),
table_name="iono.mep",
bookmark=4)
ParmGroup.SolveJob("cal_iono", "Calibrate Ionosphere parameters ",
pg_iono)
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
# TDLRuntimeOptions(*meqmaker.runtime_options(nest=False));
# and insert all solvejobs
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu("Imaging options", *imsel.option_list())
def _define_forest(ns):
ANTENNAS = mssel.get_antenna_set(list(range(1, 15)))
array = Meow.IfrArray(ns, ANTENNAS, mirror_uvw=False)
observation = Meow.Observation(ns)
Meow.Context.set(array, observation)
outputs = spigots = array.spigots(corr=mssel.get_corr_index(), flag_bit=1)
Meow.Bookmarks.make_node_folder("Input visibilities by baseline",
[spigots(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
# extract xx/yy if asked
if flag_xx_yy:
outputs = ns.xxyy
for p, q in array.ifrs():
outputs(p, q) << Meq.Selector(
spigots(p, q), index=[0, 3], multi=True)
# add freq averaging if needed
if avg_freq:
for p, q in array.ifrs():
ns.freqavg(p, q) << Meq.Mean(outputs(p, q),
reduction_axes=['freq'])
outputs = ns.freqavg
# make an inspector for spigots, we'll add more to this list
inspectors = [
Meow.StdTrees.vis_inspector(ns.inspect('spigots'),
spigots,
bookmark=False)
]
# flag on absolute value first
if flag_absmax is not None or flag_absmin is not None:
outputs = abs_clip(outputs, flag_absmax, flag_absmin)
inspectors.append(
Meow.StdTrees.vis_inspector(ns.inspect('abs'),
outputs,
bookmark=False))
# then flag on rms
if flag_rms is not None:
outputs = rms_clip(outputs, flag_rms)
inspectors.append(
Meow.StdTrees.vis_inspector(ns.inspect('rms'),
outputs,
bookmark=False))
# recreate 2x2 result (if only flagging on xx/yy)
if flag_xx_yy:
for p, q in array.ifrs():
out = outputs(p, q)
xx = ns.xx_out(p, q) << Meq.Selector(out, index=0)
yy = ns.yy_out(p, q) << Meq.Selector(out, index=1)
ns.make4corr(p, q) << Meq.Matrix22(xx, 0, 0, yy)
outputs = ns.make4corr
# merge flags across correlations if asked
if flag_all_corrs:
for p, q in array.ifrs():
ns.mergecorrflags(p, q) << Meq.MergeFlags(outputs(p, q))
outputs = ns.mergecorrflags
# finally, merge flags with spigots (so that we can inspect output flags with original data)
for p, q in array.ifrs():
ns.output(p, q) << Meq.MergeFlags(spigots(p, q), outputs(p, q))
outputs = ns.output
inspectors.append(
Meow.StdTrees.vis_inspector(ns.inspect('output'),
outputs,
bookmark=False))
# make sinks and vdm
Meow.StdTrees.make_sinks(ns, outputs, post=inspectors, output_col='')
Meow.Bookmarks.make_node_folder("Output visibilities by baseline",
[outputs(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
# put all inspectors into bookmarks
pg = Meow.Bookmarks.Page("Vis Inspectors", 2, 2)
for node in inspectors:
pg.add(node, viewer="Collections Plotter")
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512)
TDLRuntimeMenu("Imaging options", *imsel.option_list())