def _define_forest(ns, parent=None, **kw):
if not mssel.msname:
raise RuntimeError("MS not set")
if run_purr:
Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.'])
# create Purr pipe
global purrpipe
purrpipe = Purr.Pipe.Pipe(mssel.msname)
# setup contexts from MS
mssel.setup_observation_context(ns, prefer_baseline_uvw=True)
array = Meow.Context.array
# make spigot nodes for data
if do_solve or do_output not in [CORRUPTED_MODEL]:
mssel.enable_input_column(True)
spigots = spigots0 = outputs = array.spigots(
corr=mssel.get_corr_index())
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(spigots, "Input visibilities")
# add IFR-based errors, if any
spigots = meqmaker.apply_visibility_processing(ns, spigots)
else:
mssel.enable_input_column(False)
spigots = spigots0 = None
# make spigot nodes for model
corrupt_uvdata = model_spigots = None
if read_ms_model:
mssel.enable_model_column(True)
model_spigots = array.spigots(column="PREDICT",
corr=mssel.get_corr_index())
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(model_spigots,
"UV-model visibilities")
# if calibrating on (input-corrupt model), make corrupt model
if do_solve and cal_type == CAL.DIFF:
corrupt_uvdata = meqmaker.corrupt_uv_data(ns, model_spigots)
# if needed, then make a predict tree using the MeqMaker
if do_solve or do_output != CORRECTED_DATA:
if model_spigots and not corrupt_uvdata:
uvdata = model_spigots
else:
uvdata = None
predict = meqmaker.make_predict_tree(ns, uvdata=uvdata)
else:
predict = None
output_title = "Uncorrected residuals"
# make nodes to compute residuals
if do_output in [CORRECTED_RESIDUALS, RESIDUALS]:
residuals = ns.residuals
for p, q in array.ifrs():
if corrupt_uvdata:
residuals(p, q) << Meq.Subtract(spigots(
p, q), corrupt_uvdata(p, q), predict(p, q))
else:
residuals(p, q) << spigots(p, q) - predict(p, q)
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(residuals, "Uncorrected residuals")
outputs = residuals
# and now we may need to correct the outputs
if do_output in [CORRECTED_DATA, CORRECTED_RESIDUALS]:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns)
if do_correct_sky is FIRST_SOURCE:
sky_correct = srcs and srcs[0]
else:
srcs = [
src for src in srcs
if fnmatch.fnmatchcase(src.name, do_correct_sky)
]
sky_correct = srcs and srcs[0]
else:
sky_correct = None
outputs = meqmaker.correct_uv_data(ns,
outputs,
sky_correct=sky_correct,
flag_jones=flag_jones)
output_title = "Corrected data" if do_output is CORRECTED_DATA else "Corrected residuals"
elif do_output == CORRUPTED_MODEL:
outputs = predict
output_title = "Predict"
elif do_output == CORRUPTED_MODEL_ADD:
outputs = ns.output
for p, q in array.ifrs():
outputs(p, q) << spigots(p, q) + predict(p, q)
output_title = "Data+predict"
# make flaggers
if flag_enable and do_output in [
CORRECTED_DATA, RESIDUALS, CORRECTED_RESIDUALS
]:
flaggers = []
if flag_res is not None or flag_mean_res is not None:
for p, q in array.ifrs():
ns.absres(p, q) << Meq.Abs(outputs(p, q))
# make flagger for residuals
if flag_res is not None:
for p, q in array.ifrs():
ns.flagres(p, q) << Meq.ZeroFlagger(
ns.absres(p, q) - flag_res,
oper='gt',
flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT)
flaggers.append(ns.flagres)
# ...and an inspector for them
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(ns.flagres,
"Residual amplitude flags")
# make flagger for mean residuals
if flag_mean_res is not None:
ns.meanabsres << Meq.Mean(
*[ns.absres(p, q) for p, q in array.ifrs()])
ns.flagmeanres << Meq.ZeroFlagger(
ns.meanabsres - flag_mean_res,
oper='gt',
flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT)
Meow.Bookmarks.Page("Mean residual amplitude flags").add(
ns.flagmeanres, viewer="Result Plotter")
flaggers.append(lambda p, q: ns.flagmeanres)
# merge flags into output
if flaggers:
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(outputs,
output_title + " (unflagged)")
for p, q in array.ifrs():
ns.flagged(p, q) << Meq.MergeFlags(
outputs(p, q), *[f(p, q) for f in flaggers])
outputs = ns.flagged
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(outputs, output_title)
abs_outputs = outputs('abs')
for p, q in array.ifrs():
abs_outputs(p, q) << Meq.Abs(outputs(p, q))
meqmaker.make_per_ifr_bookmarks(abs_outputs,
output_title + " (mean amplitudes)")
# make solve trees
if do_solve:
# parse ifr specification
solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs()
if not solve_ifrs:
raise RuntimeError(
"No interferometers selected for calibration. Check your ifr specification (under calibration options)."
)
# inputs to the solver are based on calibration type
if corrupt_uvdata:
[
ns.diff(p, q) << spigots(p, q) - corrupt_uvdata(p, q)
for p, q in solve_ifrs
]
rhs = ns.diff
else:
rhs = spigots
lhs = predict
weights = modulo = None
# if calibrating visibilities, feed them to condeq directly, else take ampl/phase
if cal_what == CAL.VIS:
pass
elif cal_what == CAL.AMPL:
[
x('ampl', p, q) << Meq.Abs(x(p, q)) for p, q in ifrs
for x in [rhs, lhs]
]
lhs = lhs('ampl')
rhs = rhs('ampl')
elif cal_what == CAL.LOGAMPL:
[
x('logampl', p, q) << Meq.Log(Meq.Abs(x(p, q)))
for p, q in ifrs for x in [rhs, lhs]
]
lhs = lhs('logampl')
rhs = rhs('logampl')
elif cal_what == CAL.PHASE:
[
x('phase', p, q) << Meq.Arg(x(p, q)) for p, q in ifrs
for x in [rhs, lhs]
]
[rhs('ampl', p, q) << Meq.Abs(rhs(p, q)) for p, q in ifrs]
lhs = lhs('phase')
rhs = rhs('phase')
weights = rhs('ampl')
modulo = 2 * math.pi
else:
raise ValueError("unknown cal_what setting: " + str(cal_what))
# make a solve tree
solve_tree = StdTrees.SolveTree(ns,
lhs,
solve_ifrs=solve_ifrs,
weights=weights,
modulo=modulo)
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=rhs, outputs=outputs)
post = ((enable_inspectors and meqmaker.get_inspectors()) or [])
StdTrees.make_sinks(ns,
outputs,
spigots=spigots0,
post=post,
corr_index=mssel.get_corr_index())
if not do_solve:
name = "Generate " + output_title.lower()
comment = "Generated " + output_title.lower()
if name:
# make a TDL job to run the tree
def run_tree(mqs, parent, wait=False, **kw):
global tile_size
purrpipe.title("Calibrating").comment(comment)
return mqs.execute(Meow.Context.vdm.name,
mssel.create_io_request(tile_size),
wait=wait)
TDLRuntimeMenu(
name,
TDLOption(
'tile_size',
"Tile size, in timeslots", [10, 60, 120, 240],
more=int,
doc=
"""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
), TDLJob(run_tree, name, job_id='generate_visibilities'))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
# and close meqmaker -- this exports annotations, etc
meqmaker.close()
def _define_forest(ns, parent=None, **kw):
if run_purr:
Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.'])
# create Purr pipe
global purrpipe
purrpipe = Purr.Pipe.Pipe(mssel.msname)
# get antennas from MS
ANTENNAS = mssel.get_antenna_set(list(range(1, 15)))
array = Meow.IfrArray(ns, ANTENNAS, mirror_uvw=False)
stas = array.stations()
# get phase centre from MS, setup observation
observation = Meow.Observation(ns,
phase_centre=mssel.get_phase_dir(),
linear=mssel.is_linear_pol(),
circular=mssel.is_circular_pol())
Meow.Context.set(array, observation)
# get active correlations from MS
Meow.Context.active_correlations = mssel.get_correlations()
# make spigot nodes
spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index())
# ...and an inspector for them
StdTrees.vis_inspector(ns.inspector('input'),
spigots,
bookmark="Inspect input visibilities")
inspectors = [ns.inspector('input')]
Bookmarks.make_node_folder("Input visibilities by baseline",
[spigots(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
inspect_ifrs = array.ifrs()
if do_solve:
# filter solvable baselines by baseline length
solve_ifrs = []
antpos = mssel.ms_antenna_positions
if (min_baseline or max_baseline) and antpos is not None:
for (ip, p), (iq, q) in array.ifr_index():
baseline = math.sqrt(
((antpos[ip, :] - antpos[iq, :])**2).sum())
if (not min_baseline or baseline > min_baseline) and \
(not max_baseline or baseline < max_baseline):
solve_ifrs.append((p, q))
else:
solve_ifrs = array.ifrs()
inspect_ifrs = solve_ifrs
# make a predict tree using the MeqMaker
if do_solve or do_subtract:
predict = meqmaker.make_predict_tree(ns)
# make a ParmGroup and solve jobs for source parameters, if we have any
if do_solve:
parms = {}
for src in meqmaker.get_source_list(ns):
parms.update([(p.name, p) for p in src.get_solvables()])
if parms:
pg_src = ParmGroup.ParmGroup("source",
list(parms.values()),
table_name="sources.fmep",
individual=True,
bookmark=True)
# now make a solvejobs for the source
ParmGroup.SolveJob("cal_source", "Calibrate source model",
pg_src)
# make nodes to compute residuals
if do_subtract:
residuals = ns.residuals
for p, q in array.ifrs():
residuals(p, q) << spigots(p, q) - predict(p, q)
outputs = residuals
# and now we may need to correct the outputs
if do_correct:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns)
sky_correct = srcs and srcs[0]
else:
sky_correct = None
outputs = meqmaker.correct_uv_data(ns,
outputs,
sky_correct=sky_correct,
inspect_ifrs=inspect_ifrs)
# make solve trees
if do_solve:
# inputs to the solver are based on calibration type
# if calibrating visibilities, feed them to condeq directly
if cal_type == CAL.VIS:
observed = spigots
model = predict
# else take ampl/phase component
else:
model = ns.model
observed = ns.observed
if cal_type == CAL.AMPL:
for p, q in array.ifrs():
observed(p, q) << Meq.Abs(spigots(p, q))
model(p, q) << Meq.Abs(predict(p, q))
elif cal_type == CAL.LOGAMPL:
for p, q in array.ifrs():
observed(p, q) << Meq.Log(Meq.Abs(spigots(p, q)))
model(p, q) << Meq.Log(Meq.Abs(predict(p, q)))
elif cal_type == CAL.PHASE:
for p, q in array.ifrs():
observed(p, q) << 0
model(p, q) << Meq.Abs(predict(p, q)) * Meq.FMod(
Meq.Arg(spigots(p, q)) - Meq.Arg(predict(p, q)),
2 * math.pi)
else:
raise ValueError("unknown cal_type setting: " + str(cal_type))
# make a solve tree
solve_tree = StdTrees.SolveTree(ns, model, solve_ifrs=solve_ifrs)
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=observed, outputs=outputs)
# make sinks and vdm.
# The list of inspectors must be supplied here
inspectors += meqmaker.get_inspectors() or []
StdTrees.make_sinks(ns, outputs, spigots=spigots0, post=inspectors)
Bookmarks.make_node_folder("Corrected/residual visibilities by baseline",
[outputs(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
if not do_solve:
if do_subtract:
name = "Generate residuals"
comment = "Generated residual visibilities."
elif do_correct:
name = "Generate corrected data"
comment = "Generated corrected visibilities."
else:
name = None
if name:
# make a TDL job to runsthe tree
def run_tree(mqs, parent, **kw):
global tile_size
purrpipe.title("Calibrating").comment(comment)
mqs.execute(Meow.Context.vdm.name,
mssel.create_io_request(tile_size),
wait=False)
TDLRuntimeMenu(
name,
TDLOption(
'tile_size',
"Tile size, in timeslots", [10, 60, 120, 240],
more=int,
doc=
"""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
), TDLRuntimeJob(run_tree, name))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
# and close meqmaker -- this exports annotations, etc
meqmaker.close()
def _define_forest(ns, parent=None, **kw):
if not mssel.msname:
raise RuntimeError('MS name not set')
mssel.setup_observation_context(ns)
array = Context.array
# Data and model input
if do_solve or output_type.need_data:
mssel.enable_input_column(True)
spigots = array.spigots(corr=mssel.get_corr_index())
meqmaker.make_per_ifr_bookmarks(spigots, 'Input visibilities')
else:
mssel.enable_input_column(False)
spigots = None
if do_solve or output_type.need_model:
predict = meqmaker.make_predict_tree(ns, uvdata=None)
else:
predict = None
# Data output
outputs = output_type.apply(ns, meqmaker, array.ifrs(), spigots, predict)
# Flagging
if flag_enable and output_type.flag_data:
flaggers = []
if flag_res is not None or flag_mean_res is not None:
for p, q in array.ifrs():
ns.absres(p, q) << Meq.Abs(outputs(p, q))
if flag_res is not None:
for p, q in array.ifrs():
ns.flagres(p, q) << Meq.ZeroFlagger(
ns.absres(p, q) - flag_res,
oper='gt',
flag_bit=MSUtils.FLAGMASK_OUTPUT)
flaggers.append(ns.flagres)
meqmaker.make_per_ifr_bookmarks(ns.flagres,
'Residual amplitude flags')
if flag_mean_res is not None:
ns.meanabsres << Meq.Mean(
*[ns.absres(p, q) for p, q in array.ifrs()])
ns.flagmeanres << Meq.ZeroFlagger(ns.meanabsres - flag_mean_res,
oper='gt',
flag_bit=MSUtils.FLAGMASK_OUTPUT)
Bookmarks.Page('Mean residual amplitude flags').add(
ns.flagmeanres, viewer='Result Plotter')
flaggers.append(lambda p, q: ns.flagmeanres)
if flaggers:
meqmaker.make_per_ifr_bookmarks(outputs,
output_type.desc + ' (unflagged)')
for p, q in array.ifrs():
ns.flagged(p, q) << Meq.MergeFlags(
outputs(p, q), *[f(p, q) for f in flaggers])
outputs = ns.flagged
meqmaker.make_per_ifr_bookmarks(outputs, output_type.desc)
# Solve trees
if do_solve:
# parse ifr specification
solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs()
if not solve_ifrs:
raise RuntimeError(
'No interferometers selected for calibration. '
'Check your ifr specification (under calibration options).')
lhs, rhs, weights, modulo = cal_quant.apply(solve_ifrs, predict,
spigots)
solve_tree = StdTrees.SolveTree(ns,
lhs,
solve_ifrs=solve_ifrs,
weights=weights,
modulo=modulo)
outputs = solve_tree.sequencers(inputs=rhs, outputs=outputs)
StdTrees.make_sinks(ns,
outputs,
spigots=spigots,
post=meqmaker.get_inspectors() or [],
corr_index=mssel.get_corr_index())
if not do_solve:
name = 'Generate ' + output_type.desc.lower()
comment = 'Generated ' + output_type.desc.lower()
def run_tree(mqs, parent, wait=False, **kw):
return mqs.execute(Context.vdm.name,
mssel.create_io_request(tile_size),
wait=wait)
doc = """Input data are sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
TDLRuntimeMenu(
name,
TDLOption('tile_size',
'Tile size, in timeslots', [10, 60, 120, 240],
more=int,
doc=doc),
TDLJob(run_tree, name, job_id='generate_visibilities'))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up
# during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu('Make an image', *imsel.option_list())
meqmaker.close()