def compute_jones (Jones,sources,stations=None,inspectors=[],**kw):
"""Creates the Z Jones for ionospheric phase, given TECs (per source,
per station).""";
stations = stations or Context.array.stations;
ns = Jones.Subscope();
piercings = iono_geometry.compute_piercings(ns,sources,stations);
za_cos = iono_geometry.compute_za_cosines(ns,sources,stations);
tecs = iono_model(ns,piercings,za_cos,sources,stations);
# make inspector for TECs
inspectors.append(
Jones.scope.inspector('TEC') << StdTrees.define_inspector(tecs,sources,stations,
label='tec',freqavg=False)
);
if diff_mode:
absJones = Jones('abs');
iono_geometry.compute_zeta_jones_from_tecs(absJones,tecs,sources,stations);
for src in sources:
for p in stations:
Jones(src,p) << absJones(src,p)/absJones(sources[0],p);
else:
iono_geometry.compute_zeta_jones_from_tecs(Jones,tecs,sources,stations);
# make inspector for ionospheric phases
Zphase = ns.Zphase;
for src in sources:
for p in stations:
Zphase(src,p) << Meq.Arg(Jones(src,p));
inspectors += [
Jones.scope.inspector('iono_phase') << \
StdTrees.define_inspector(Zphase,sources,stations,label='z'),
Jones.scope.inspector('iono_piercings') << \
StdTrees.define_inspector(ns.dxy,sources,stations,label='dxy'),
]
return Jones;
def compute_jones (Jones,sources,stations=None,inspectors=[],label='L',**kw):
"""Creates the dipole projection matrix.""";
stations = stations or Context.array.stations;
ns = Jones.Subscope();
insp = Jones.scope.inspector(label);
for src in sources:
for p in stations:
Jones(src,p) << proj_matrix(src,Context.array.xyz(p));
insp << StdTrees.define_inspector(Jones,sources,stations,label=label);
# add inspectors
StdTrees.inspector(Jones.scope.inspector(label,'AzEl') ,[src.direction.azel() for src in sources],bookmark=False);
inspectors += [ insp,Jones.scope.inspector(label,'AzEl') ];
return Jones;
def compute_jones (self,nodes,stations=None,tags=None,label='',inspectors=[],**kw):
stations = stations or Context.array.stations();
rot_def = Meow.Parm(0);
nr = Jones.rotation_matrix(nodes("R"),rot_def,tags=tags,series=stations);
ne = Jones.ellipticity_matrix(nodes("E"),rot_def,tags=tags,series=stations);
for p in stations:
nodes(p) << Meq.MatrixMultiply(nr(p),ne(p));
# make parmgroups for phases and gains
self.pg_rot = ParmGroup.ParmGroup(label+"_leakage",
nodes.search(tags="solvable"),
table_name="%s_leakage.mep"%label,bookmark=True);
# make solvejobs
ParmGroup.SolveJob("cal_"+label+"_leakage","Calibrate %s (leakage)"%label,self.pg_rot);
# make inspector for parameters
StdTrees.inspector(nodes('inspector'),self.pg_rot.nodes,bookmark=False);
inspectors.append(nodes('inspector'));
return nodes;
def compute_jones (Jones,sources,stations=None,pointing_offsets=None,inspectors=[],label='E',**kw):
stations = stations or Context.array.stations;
ns = Jones.Subscope();
# declare an inspector node for the Jones matrix -- will be defined below
insp = Jones.scope.inspector(label);
inspectors += [ insp ];
# loop over sources
for src in sources:
# If sky rotation and/or pointing offsets are in effect, we have a per-station beam.
# Otherwise the beam is the same for all stations.
if sky_rotation or pointing_offsets:
for p in stations:
lm = src.direction.lm();
# apply rotation to put sources into the antenna frame
if sky_rotation:
xyz = Context.array.xyz(p);
pa_rot = Context.observation.phase_centre.pa_rot(xyz);
lm = ns.lmrot(src,p) << Meq.MatrixMultiply(pa_rot,src.direction.lm());
# apply offset in the node (so pointing offsets are interpreted in the azel frame, if rotating)
make_beam_node(Jones(src,p),filename_pattern,lm,pointing_offsets and pointing_offsets(p));
else:
make_beam_node(Jones(src),filename_pattern,src.direction.lm());
for p in stations:
Jones(src,p) << Meq.Identity(Jones(src));
# define an inspector
if sky_rotation or pointing_offsets:
# Jones inspector is per-source, per-station
insp << StdTrees.define_inspector(Jones,sources,stations,label=label);
else:
# Jones inspector is per-source
insp << StdTrees.define_inspector(Jones,sources,label=label);
return Jones;
def compute_jones (Jones,sources,stations=None,pointing_offsets=None,inspectors=[],label='E',**kw):
stations = stations or Context.array.stations;
ns = Jones.Subscope();
# declare an inspector node for the Jones matrix -- will be defined below
insp = Jones.scope.inspector(label);
inspectors += [ insp ];
radec0 = Context.observation.phase_centre.radec() if interpol_coord is COORD_THETAPHI else None;
# loop over sources
for src in sources:
xy = src.direction.radec() if interpol_coord is COORD_THETAPHI else src.direction.lm();
# If sky rotation and/or horizon masking and/or pointing offsets are in effect, we have a per-station beam.
# Otherwise the beam is the same for all stations.
if sky_rotation or pointing_offsets or horizon_masking:
for p in stations:
azel = src.direction.azel(Context.array.xyz(p)) if horizon_masking else None;
pa = Context.observation.phase_centre.pa(Context.array.xyz(p)) if sky_rotation else None;
# apply offset in the node (so pointing offsets are interpreted in the azel frame, if rotating)
make_beam_node(Jones(src,p),filename_pattern,xy,radec0=radec0,
dlm=pointing_offsets and pointing_offsets(p),azel=azel,pa=pa);
else:
make_beam_node(Jones(src),filename_pattern,xy,radec0=radec0);
for p in stations:
Jones(src,p) << Meq.Identity(Jones(src));
# define an inspector
if sky_rotation or pointing_offsets or horizon_masking:
# Jones inspector is per-source, per-station
insp << StdTrees.define_inspector(Jones,sources,stations,label=label);
else:
# Jones inspector is per-source
insp << StdTrees.define_inspector(Jones,sources,label=label);
return Jones;
def compute_jones (self,Jones,sources,stations=None,inspectors=[],**kw):
"""Computes beam gain for a list of sources.
The output node, will be qualified with either a source only, or a source/station pair
""";
stations = stations or Context.array.stations();
ns = Jones.Subscope();
# init solvables etc.
self.init_parameters(ns,sources,stations,inspectors);
# this dict will hold LM tuples (or nodes) for each source.
lmsrc = {};
# see if sources have a "beam_lm" attribute, use that for beam offsets
for src in sources:
lm = src.get_attr("beam_lm",None) or src.get_attr("_lm_ncp",None);
if lm:
src.set_attr(self.label+'r',math.sqrt(lm[0]**2+lm[1]**2)/math.pi*(180*60));
lmsrc[src.name] = ns.lm(src) << Meq.Constant(value=Timba.array.array(lm));
# else try to use static lm coordinates
else:
# else try to use static lm coordinates
lmnst = src.direction.lmn_static();
if lmnst:
lm = lmnst[0:2];
src.set_attr(self.label+'r',math.sqrt(lm[0]**2+lm[1]**2)/math.pi*(180*60));
lmsrc[src.name] = ns.lm(src) << Meq.Constant(value=Timba.array.array(lm));
# else use lmn node
else:
lmsrc[src.name] = src.direction.lm();
if self.per_station:
for src in sources:
for p in stations:
self.make_beam_nodes(Jones(src,p),self.beamshape(p),lmsrc[src.name],self.ell(p),self.dlm(p));
else:
p0 = stations[0];
for src in sources:
self.make_beam_nodes(Jones(src,p0),self.beamshape(p0),lmsrc[src.name],self.ell(p0),self.dlm(p0));
for p in stations[1:]:
Jones(src,p) << Meq.Identity(Jones(src,p0));
# make inspectors
inspectors.append(ns.inspector << StdTrees.define_inspector(
Jones,sources,stations,label=self.label));
return Jones;
def _define_forest(ns,parent=None,**kw):
if not mssel.msname:
raise RuntimeError,"MS not set";
if run_purr:
Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']);
# create Purr pipe
global purrpipe;
purrpipe = Purr.Pipe.Pipe(mssel.msname);
# setup contexts from MS
mssel.setup_observation_context(ns,prefer_baseline_uvw=True);
array = Meow.Context.array;
# make spigot nodes for data
if do_solve or do_output not in [CORRUPTED_MODEL]:
mssel.enable_input_column(True);
spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index());
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(spigots,"Input visibilities");
# add IFR-based errors, if any
spigots = meqmaker.apply_visibility_processing(ns,spigots);
else:
mssel.enable_input_column(False);
spigots = spigots0 = None;
# make spigot nodes for model
corrupt_uvdata = model_spigots = None;
if read_ms_model:
mssel.enable_model_column(True);
model_spigots = array.spigots(column="PREDICT",corr=mssel.get_corr_index());
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(model_spigots,"UV-model visibilities");
# if calibrating on (input-corrupt model), make corrupt model
if do_solve and cal_type == CAL.DIFF:
corrupt_uvdata = meqmaker.corrupt_uv_data(ns,model_spigots);
# if needed, then make a predict tree using the MeqMaker
if do_solve or do_output != CORRECTED_DATA:
if model_spigots and not corrupt_uvdata:
uvdata = model_spigots;
else:
uvdata = None;
predict = meqmaker.make_predict_tree(ns,uvdata=uvdata);
else:
predict = None;
output_title = "Uncorrected residuals";
# make nodes to compute residuals
if do_output in [CORRECTED_RESIDUALS,RESIDUALS]:
residuals = ns.residuals;
for p,q in array.ifrs():
if corrupt_uvdata:
residuals(p,q) << Meq.Subtract(spigots(p,q),corrupt_uvdata(p,q),predict(p,q));
else:
residuals(p,q) << spigots(p,q) - predict(p,q);
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(residuals,"Uncorrected residuals");
outputs = residuals;
# and now we may need to correct the outputs
if do_output in [CORRECTED_DATA,CORRECTED_RESIDUALS]:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns);
if do_correct_sky is FIRST_SOURCE:
sky_correct = srcs and srcs[0];
else:
srcs = [ src for src in srcs if fnmatch.fnmatchcase(src.name,do_correct_sky) ];
sky_correct = srcs and srcs[0];
else:
sky_correct = None;
outputs = meqmaker.correct_uv_data(ns,outputs,sky_correct=sky_correct,
flag_jones=flag_jones);
output_title = "Corrected data" if do_output is CORRECTED_DATA else "Corrected residuals";
elif do_output == CORRUPTED_MODEL:
outputs = predict;
output_title = "Predict";
elif do_output == CORRUPTED_MODEL_ADD:
outputs = ns.output;
for p,q in array.ifrs():
outputs(p,q) << spigots(p,q) + predict(p,q);
output_title = "Data+predict";
# make flaggers
if flag_enable and do_output in [ CORRECTED_DATA,RESIDUALS,CORRECTED_RESIDUALS ]:
flaggers = [];
if flag_res is not None or flag_mean_res is not None:
for p,q in array.ifrs():
ns.absres(p,q) << Meq.Abs(outputs(p,q));
# make flagger for residuals
if flag_res is not None:
for p,q in array.ifrs():
ns.flagres(p,q) << Meq.ZeroFlagger(ns.absres(p,q)-flag_res,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT);
flaggers.append(ns.flagres);
# ...and an inspector for them
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(ns.flagres,"Residual amplitude flags");
# make flagger for mean residuals
if flag_mean_res is not None:
ns.meanabsres << Meq.Mean(*[ns.absres(p,q) for p,q in array.ifrs()]);
ns.flagmeanres << Meq.ZeroFlagger(ns.meanabsres-flag_mean_res,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT);
Meow.Bookmarks.Page("Mean residual amplitude flags").add(ns.flagmeanres,viewer="Result Plotter");
flaggers.append(lambda p,q:ns.flagmeanres);
# merge flags into output
if flaggers:
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(outputs,output_title+" (unflagged)");
for p,q in array.ifrs():
ns.flagged(p,q) << Meq.MergeFlags(outputs(p,q),*[f(p,q) for f in flaggers]);
outputs = ns.flagged;
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(outputs,output_title);
abs_outputs = outputs('abs');
for p,q in array.ifrs():
abs_outputs(p,q) << Meq.Abs(outputs(p,q));
meqmaker.make_per_ifr_bookmarks(abs_outputs,output_title+" (mean amplitudes)");
# make solve trees
if do_solve:
# parse ifr specification
solve_ifrs = array.subset(calibrate_ifrs,strict=False).ifrs();
if not solve_ifrs:
raise RuntimeError,"No interferometers selected for calibration. Check your ifr specification (under calibration options).";
# inputs to the solver are based on calibration type
if corrupt_uvdata:
[ ns.diff(p,q) << spigots(p,q) - corrupt_uvdata(p,q) for p,q in solve_ifrs ];
rhs = ns.diff;
else:
rhs = spigots;
lhs = predict;
weights = modulo = None;
# if calibrating visibilities, feed them to condeq directly, else take ampl/phase
if cal_what == CAL.VIS:
pass;
elif cal_what == CAL.AMPL:
[ x('ampl',p,q) << Meq.Abs(x(p,q)) for p,q in ifrs for x in rhs,lhs ];
lhs = lhs('ampl');
rhs = rhs('ampl');
elif cal_what == CAL.LOGAMPL:
[ x('logampl',p,q) << Meq.Log(Meq.Abs(x(p,q))) for p,q in ifrs for x in rhs,lhs ];
lhs = lhs('logampl');
rhs = rhs('logampl');
elif cal_what == CAL.PHASE:
[ x('phase',p,q) << Meq.Arg(x(p,q)) for p,q in ifrs for x in rhs,lhs ];
[ rhs('ampl',p,q) << Meq.Abs(rhs(p,q)) for p,q in ifrs ];
lhs = lhs('phase');
rhs = rhs('phase');
weights = rhs('ampl');
modulo = 2*math.pi;
else:
raise ValueError,"unknown cal_what setting: "+str(cal_what);
# make a solve tree
solve_tree = StdTrees.SolveTree(ns,lhs,solve_ifrs=solve_ifrs,weights=weights,modulo=modulo);
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=rhs,outputs=outputs);
post = ( ( enable_inspectors and meqmaker.get_inspectors() ) or [] );
StdTrees.make_sinks(ns,outputs,spigots=spigots0,post=post,corr_index=mssel.get_corr_index());
if not do_solve:
name = "Generate "+output_title.lower();
comment = "Generated "+output_title.lower();
if name:
# make a TDL job to run the tree
def run_tree (mqs,parent,wait=False,**kw):
global tile_size;
purrpipe.title("Calibrating").comment(comment);
return mqs.execute(Meow.Context.vdm.name,mssel.create_io_request(tile_size),wait=wait);
TDLRuntimeMenu(name,
TDLOption('tile_size',"Tile size, in timeslots",[10,60,120,240],more=int,
doc="""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""),
TDLJob(run_tree,name,job_id='generate_visibilities')
);
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False));
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options());
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size());
TDLRuntimeMenu("Make an image from this MS",*imsel.option_list());
# and close meqmaker -- this exports annotations, etc
meqmaker.close();
def _define_forest(ns, parent=None, **kw):
if not mssel.msname:
raise RuntimeError("MS not set")
if run_purr:
Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.'])
# create Purr pipe
global purrpipe
purrpipe = Purr.Pipe.Pipe(mssel.msname)
# setup contexts from MS
mssel.setup_observation_context(ns)
array = Meow.Context.array
# make spigot nodes for data
if do_solve or do_output not in [CORRUPTED_MODEL]:
mssel.enable_input_column(True)
spigots = spigots0 = outputs = array.spigots(
corr=mssel.get_corr_index())
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(spigots, "Input visibilities")
else:
mssel.enable_input_column(False)
spigots = spigots0 = None
# make spigot nodes for model
corrupt_uvdata = model_spigots = None
if read_ms_model:
mssel.enable_model_column(True)
model_spigots = array.spigots(column="PREDICT",
corr=mssel.get_corr_index())
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(model_spigots,
"UV-model visibilities")
# if calibrating on (input-corrupt model), make corrupt model
if do_solve and cal_type == CAL.DIFF:
corrupt_uvdata = meqmaker.corrupt_uv_data(ns, model_spigots)
# if needed, then make a predict tree using the MeqMaker
if do_solve or do_output != CORRECTED_DATA:
if model_spigots and not corrupt_uvdata:
uvdata = model_spigots
else:
uvdata = None
predict = meqmaker.make_predict_tree(ns, uvdata=uvdata)
else:
predict = None
output_title = "Uncorrected residuals"
# make nodes to compute residuals
if do_output in [CORRECTED_RESIDUALS, RESIDUALS]:
residuals = ns.residuals
for p, q in array.ifrs():
if corrupt_uvdata:
residuals(p, q) << Meq.Subtract(spigots(
p, q), corrupt_uvdata(p, q), predict(p, q))
else:
residuals(p, q) << spigots(p, q) - predict(p, q)
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(residuals, "Uncorrected residuals")
outputs = residuals
# and now we may need to correct the outputs
if do_output in [CORRECTED_DATA, CORRECTED_RESIDUALS]:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns)
sky_correct = srcs and srcs[0]
else:
sky_correct = None
global flag_jones
if flag_enable and flag_jones and not (flag_jones_min is None
and flag_jones_max is None):
flag_jones_minmax = (flag_jones_min, flag_jones_max)
else:
flag_jones_minmax = None
outputs = meqmaker.correct_uv_data(ns,
outputs,
sky_correct=sky_correct,
flag_jones_minmax=flag_jones and
(flag_jones_min, flag_jones_max))
output_title = "Corrected data" if do_output is CORRECTED_DATA else "Corrected residuals"
elif do_output == CORRUPTED_MODEL:
outputs = predict
output_title = "Predict"
elif do_output == CORRUPTED_MODEL_ADD:
outputs = ns.output
for p, q in array.ifrs():
outputs(p, q) << spigots(p, q) + predict(p, q)
output_title = "Data+predict"
# make flaggers
if flag_enable and do_output in [
CORRECTED_DATA, RESIDUALS, CORRECTED_RESIDUALS
]:
flaggers = []
if flag_res is not None or flag_mean_res is not None:
for p, q in array.ifrs():
ns.absres(p, q) << Meq.Abs(outputs(p, q))
# make flagger for residuals
if flag_res is not None:
for p, q in array.ifrs():
ns.flagres(p, q) << Meq.ZeroFlagger(
ns.absres(p, q) - flag_res,
oper='gt',
flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT)
flaggers.append(ns.flagres)
# ...and an inspector for them
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(ns.flagres,
"Residual amplitude flags")
# make flagger for mean residuals
if flag_mean_res is not None:
ns.meanabsres << Meq.Mean(
*[ns.absres(p, q) for p, q in array.ifrs()])
ns.flagmeanres << Meq.ZeroFlagger(
ns.meanabsres - flag_mean_res,
oper='gt',
flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT)
Meow.Bookmarks.Page("Mean residual amplitude flags").add(
ns.flagmeanres, viewer="Result Plotter")
flaggers.append(lambda p, q: ns.flagmeanres)
# merge flags into output
if flaggers:
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(outputs,
output_title + " (unflagged)")
for p, q in array.ifrs():
ns.flagged(p, q) << Meq.MergeFlags(
outputs(p, q), *[f(p, q) for f in flaggers])
outputs = ns.flagged
if enable_inspectors:
meqmaker.make_per_ifr_bookmarks(outputs, output_title)
abs_outputs = outputs('abs')
for p, q in array.ifrs():
abs_outputs(p, q) << Meq.Abs(outputs(p, q))
meqmaker.make_per_ifr_bookmarks(abs_outputs,
output_title + " (mean amplitudes)")
# make solve trees
if do_solve:
# parse ifr specification
solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs()
if not solve_ifrs:
raise RuntimeError(
"No interferometers selected for calibration. Check your ifr specification (under calibration options)."
)
# inputs to the solver are based on calibration type
if corrupt_uvdata:
[
ns.diff(p, q) << spigots(p, q) - corrupt_uvdata(p, q)
for p, q in solve_ifrs
]
rhs = ns.diff
else:
rhs = spigots
lhs = predict
weights = modulo = None
# if calibrating visibilities, feed them to condeq directly, else take ampl/phase
if cal_what == CAL.VIS:
pass
elif cal_what == CAL.AMPL:
[
x('ampl', p, q) << Meq.Abs(x(p, q)) for p, q in ifrs
for x in (rhs, lhs)
]
lhs = lhs('ampl')
rhs = rhs('ampl')
elif cal_what == CAL.LOGAMPL:
[
x('logampl', p, q) << Meq.Log(Meq.Abs(x(p, q)))
for p, q in ifrs for x in (rhs, lhs)
]
lhs = lhs('logampl')
rhs = rhs('logampl')
elif cal_what == CAL.PHASE:
[
x('phase', p, q) << Meq.Arg(x(p, q)) for p, q in ifrs
for x in (rhs, lhs)
]
[rhs('ampl', p, q) << Meq.Abs(rhs(p, q)) for p, q in ifrs]
lhs = lhs('phase')
rhs = rhs('phase')
weights = rhs('ampl')
modulo = 2 * math.pi
else:
raise ValueError("unknown cal_what setting: " + str(cal_what))
# make a solve tree
solve_tree = StdTrees.SolveTree(ns,
lhs,
solve_ifrs=solve_ifrs,
weights=weights,
modulo=modulo)
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=rhs, outputs=outputs)
post = (enable_inspectors and meqmaker.get_inspectors()) or []
StdTrees.make_sinks(ns,
outputs,
spigots=spigots0,
post=post,
corr_index=mssel.get_corr_index())
if not do_solve:
name = "Generate " + output_title.lower()
comment = "Generated " + output_title.lower()
if name:
# make a TDL job to run the tree
def run_tree(mqs, parent, wait=False, **kw):
global tile_size
purrpipe.title("Calibrating").comment(comment)
return mqs.execute(Meow.Context.vdm.name,
mssel.create_io_request(tile_size),
wait=wait)
TDLRuntimeMenu(
name,
TDLOption(
'tile_size',
"Tile size, in timeslots", [10, 60, 120, 240],
more=int,
doc=
"""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
), TDLJob(run_tree, name, job_id='generate_visibilities'))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
# and close meqmaker -- this exports annotations, etc
meqmaker.close()
def compute_jones (Jones,sources,stations=None,inspectors=[],meqmaker=None,label='R',**kw):
"""Creates the Z Jones for ionospheric phase, given TECs (per source,
per station).""";
stations = stations or Context.array.stations;
ns = Jones.Subscope();
# get reference source
if ref_source:
# treat as index first
dir0 = None;
try:
dir0 = sources[int(ref_source)].direction;
except:
pass;
# else treat as name, find in list
if not dir0:
for src0 in sources:
if src0.name == ref_source:
dir0 = src0.direction;
break;
# else treat as direction string
if not dir0:
ff = list(ref_source.split());
if len(ff) < 2 or len(ff) > 3:
raise RuntimeError,"invalid reference dir '%s' specified for %s-Jones"%(ref_source,label);
global dm;
if not dm:
raise RuntimeError,"pyrap measures module not available, cannot use direction strings for %s-Jones"%label;
if len(ff) == 2:
ff = [ 'J2000' ] +ff;
# treat as direction measure
try:
dmdir = dm.direction(*ff);
except:
raise RuntimeError,"invalid reference dir '%s' specified for %s-Jones"%(ref_source,label);
# convert to J2000 and make direction object
dmdir = dm.measure(dmdir,'J2000');
ra,dec = dm.getvalue(dmdir)[0].get_value(),dm.getvalue(dmdir)[1].get_value();
dir0 = Meow.Direction(ns,"refdir",ra,dec,static=True);
else:
dir0 = Context.observation.phase_centre;
# make refraction scale node
scale = ns.scale(0) << Meq.Parm(0,tags="refraction");
xyz0 = Context.array.xyz0();
if coord_approx:
# get PA, and assume it's the same over the whole field
pa = ns.pa0 << Meq.ParAngle(dir0.radec(),xyz0);
# second column of the Rot(-PA) matrix. Multiply this by del to get a rotation of (0,del) into the lm plane.
# The third component (0) is for convenience, as it immediately gives us dl,dm,dn, since we assume dn~0
rot_pa = ns.rotpa0 << Meq.Composer(Meq.Sin(pa),Meq.Cos(pa),0);
# el0: elevation of field centre
el0 = dir0.el();
if do_extinction:
ns.inv_ext0 << Meq.Sin(el0); # inverse of extinction towards el0
# station UVWs
uvw = Context.array.uvw();
# now loop over sources
for isrc,src in enumerate(sources):
# reference direction: no refraction at all
if src.direction is dir0:
for p in stations:
Jones(src,p) << 1;
continue;
# dEl is source elevation minus el0
# ddEl = scale*dEl: amount by which source refracts (negative means field is compressed)
el = src.direction.el()
ns.dEl(src) << el - el0;
ddel = ns.ddEl(src) << ns.dEl(src)*scale;
# get el1: refracted elevation angle
if not coord_approx or do_extinction:
el1 = ns.el1(src) << el + ddel;
# compute extinction component
if do_extinction:
# compute inverse of extinction towards the refracted direction el1
iext = ns.inv_ext(src) << Meq.Sin(el1); #
# and differential extinction is then ext1/ext0
ext = ns.dext(src) << ns.inv_ext0/iext;
# Compute dlmn offset in lm plane.
if coord_approx:
# Approximate mode: ddel is added to elevation, so to get the lm offset, we need
# to apply Rot(PA) to the column vector (0,ddel), and then take the sine of the result.
dlmn = ns.dlmn(src) << Meq.Sin(ddel*rot_pa);
else:
ns.azel1(src) << Meq.Composer(src.direction.az(),el1);
ns.radec1(src) << Meq.RADec(ns.azel1(src),xyz0);
ns.lmn1(src) << Meq.LMN(Context.observation.radec0(),ns.radec1(src));
dlmn = ns.dlmn(src) << ns.lmn1(src) - src.lmn();
# get per-station phases
for p in stations:
if do_extinction:
Jones(src,p) << ext*(ns.phase(src,p) << Meq.VisPhaseShift(lmn=dlmn,uvw=uvw(p)));
else:
Jones(src,p) << Meq.VisPhaseShift(lmn=dlmn,uvw=uvw(p));
# make bookmarks
srcnames = [ src.name for src in sources ];
meqmaker.make_bookmark_set(Jones,[ (src,p) for src in srcnames for p in stations ],
"%s: inspector plot"%label,"%s: by source-station"%label,freqmean=True);
inspectors.append(ns.inspector(label,'scale') << \
StdTrees.define_inspector(ns.scale,[0],label=label));
inspectors.append(ns.inspector(label,'delta-el') << \
StdTrees.define_inspector(ns.ddEl,srcnames,label=label));
inspectors.append(ns.inspector(label,'delta-el') << \
StdTrees.define_inspector(ns.ddEl,srcnames,label=label));
inspectors.append(ns.inspector(label,'dlmn') << \
StdTrees.define_inspector(ns.dlmn,srcnames,label=label));
if do_extinction:
inspectors.append(ns.inspector(label,'inv-ext') << \
StdTrees.define_inspector(ns.inv_ext,srcnames,label=label));
inspectors.append(ns.inspector(label,'diff-ext') << \
StdTrees.define_inspector(ns.dext,srcnames,label=label));
# make parmgroups and solvejobs
global pg;
pg = ParmGroup.ParmGroup(label,[scale],table_name="%s.fmep"%label,bookmark=False);
# make solvejobs
ParmGroup.SolveJob("cal_"+label,"Calibrate %s (differential refraction)"%label,pg);
return Jones;
def _define_forest(ns, parent=None, **kw):
if run_purr:
Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.'])
# create Purr pipe
global purrpipe
purrpipe = Purr.Pipe.Pipe(mssel.msname)
# get antennas from MS
ANTENNAS = mssel.get_antenna_set(range(1, 15))
array = Meow.IfrArray(ns, ANTENNAS, mirror_uvw=False)
stas = array.stations()
# get phase centre from MS, setup observation
observation = Meow.Observation(ns,
phase_centre=mssel.get_phase_dir(),
linear=mssel.is_linear_pol(),
circular=mssel.is_circular_pol())
Meow.Context.set(array, observation)
# get active correlations from MS
Meow.Context.active_correlations = mssel.get_correlations()
# make spigot nodes
spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index())
# ...and an inspector for them
StdTrees.vis_inspector(ns.inspector('input'),
spigots,
bookmark="Inspect input visibilities")
inspectors = [ns.inspector('input')]
Bookmarks.make_node_folder("Input visibilities by baseline",
[spigots(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
inspect_ifrs = array.ifrs()
if do_solve:
# filter solvable baselines by baseline length
solve_ifrs = []
antpos = mssel.ms_antenna_positions
if (min_baseline or max_baseline) and antpos is not None:
for (ip, p), (iq, q) in array.ifr_index():
baseline = math.sqrt(
((antpos[ip, :] - antpos[iq, :])**2).sum())
if (not min_baseline or baseline > min_baseline) and \
(not max_baseline or baseline < max_baseline):
solve_ifrs.append((p, q))
else:
solve_ifrs = array.ifrs()
inspect_ifrs = solve_ifrs
# make a predict tree using the MeqMaker
if do_solve or do_subtract:
predict = meqmaker.make_predict_tree(ns)
# make a ParmGroup and solve jobs for source parameters, if we have any
if do_solve:
parms = {}
for src in meqmaker.get_source_list(ns):
parms.update([(p.name, p) for p in src.get_solvables()])
if parms:
pg_src = ParmGroup.ParmGroup("source",
parms.values(),
table_name="sources.fmep",
individual=True,
bookmark=True)
# now make a solvejobs for the source
ParmGroup.SolveJob("cal_source", "Calibrate source model",
pg_src)
# make nodes to compute residuals
if do_subtract:
residuals = ns.residuals
for p, q in array.ifrs():
residuals(p, q) << spigots(p, q) - predict(p, q)
outputs = residuals
# and now we may need to correct the outputs
if do_correct:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns)
sky_correct = srcs and srcs[0]
else:
sky_correct = None
outputs = meqmaker.correct_uv_data(ns,
outputs,
sky_correct=sky_correct,
inspect_ifrs=inspect_ifrs)
# make solve trees
if do_solve:
# inputs to the solver are based on calibration type
# if calibrating visibilities, feed them to condeq directly
if cal_type == CAL.VIS:
observed = spigots
model = predict
# else take ampl/phase component
else:
model = ns.model
observed = ns.observed
if cal_type == CAL.AMPL:
for p, q in array.ifrs():
observed(p, q) << Meq.Abs(spigots(p, q))
model(p, q) << Meq.Abs(predict(p, q))
elif cal_type == CAL.LOGAMPL:
for p, q in array.ifrs():
observed(p, q) << Meq.Log(Meq.Abs(spigots(p, q)))
model(p, q) << Meq.Log(Meq.Abs(predict(p, q)))
elif cal_type == CAL.PHASE:
for p, q in array.ifrs():
observed(p, q) << 0
model(p, q) << Meq.Abs(predict(p, q)) * Meq.FMod(
Meq.Arg(spigots(p, q)) - Meq.Arg(predict(p, q)),
2 * math.pi)
else:
raise ValueError, "unknown cal_type setting: " + str(cal_type)
# make a solve tree
solve_tree = StdTrees.SolveTree(ns, model, solve_ifrs=solve_ifrs)
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=observed, outputs=outputs)
# make sinks and vdm.
# The list of inspectors must be supplied here
inspectors += meqmaker.get_inspectors() or []
StdTrees.make_sinks(ns, outputs, spigots=spigots0, post=inspectors)
Bookmarks.make_node_folder("Corrected/residual visibilities by baseline",
[outputs(p, q) for p, q in array.ifrs()],
sorted=True,
ncol=2,
nrow=2)
if not do_solve:
if do_subtract:
name = "Generate residuals"
comment = "Generated residual visibilities."
elif do_correct:
name = "Generate corrected data"
comment = "Generated corrected visibilities."
else:
name = None
if name:
# make a TDL job to runsthe tree
def run_tree(mqs, parent, **kw):
global tile_size
purrpipe.title("Calibrating").comment(comment)
mqs.execute(Meow.Context.vdm.name,
mssel.create_io_request(tile_size),
wait=False)
TDLRuntimeMenu(
name,
TDLOption(
'tile_size',
"Tile size, in timeslots", [10, 60, 120, 240],
more=int,
doc=
"""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
), TDLRuntimeJob(run_tree, name))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options())
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
# and close meqmaker -- this exports annotations, etc
meqmaker.close()
def compute_jones(self,
Jones,
sources,
stations=None,
inspectors=[],
**kw):
"""Computes beam gain for a list of sources.
The output node, will be qualified with either a source only, or a source/station pair
"""
stations = stations or Context.array.stations()
ns = Jones.Subscope()
# init solvables etc.
self.init_parameters(ns, sources, stations, inspectors)
# this dict will hold LM tuples (or nodes) for each source.
lmsrc = {}
# see if sources have a "beam_lm" attribute, use that for beam offsets
for src in sources:
lm = src.get_attr("beam_lm", None) or src.get_attr(
"_lm_ncp", None)
if lm:
src.set_attr(
self.label + 'r',
math.sqrt(lm[0]**2 + lm[1]**2) / math.pi * (180 * 60))
lmsrc[src.name] = ns.lm(src) << Meq.Constant(
value=Timba.array.array(lm))
# else try to use static lm coordinates
else:
# else try to use static lm coordinates
lmnst = src.direction.lmn_static()
if lmnst:
lm = lmnst[0:2]
src.set_attr(
self.label + 'r',
math.sqrt(lm[0]**2 + lm[1]**2) / math.pi * (180 * 60))
lmsrc[src.name] = ns.lm(src) << Meq.Constant(
value=Timba.array.array(lm))
# else use lmn node
else:
lmsrc[src.name] = src.direction.lm()
if self.per_station:
for src in sources:
for p in stations:
self.make_beam_nodes(Jones(src, p), self.beamshape(p),
lmsrc[src.name], self.ell(p),
self.dlm(p))
else:
p0 = stations[0]
for src in sources:
self.make_beam_nodes(Jones(src, p0),
self.beamshape(p0), lmsrc[src.name],
self.ell(p0), self.dlm(p0))
for p in stations[1:]:
Jones(src, p) << Meq.Identity(Jones(src, p0))
# make inspectors
inspectors.append(ns.inspector << StdTrees.define_inspector(
Jones, sources, stations, label=self.label))
return Jones
def init_parameters(self, ns, sources, stations, inspectors=[]):
if self.beamshape is not None:
return
# create solvables if enabled
parms = []
parmgroups = []
parmdef_scale = Meq.Parm(self.bf * 1e-9, tags="beam solvable")
parmdef_ell = Meq.Parm(self.ellipticity, tags="beam solvable")
self.per_station = False
# solvable beam scale
if self.solve_scale is PER_STATION:
self.beamshape = ns.beamshape
for p in stations:
parms.append(beamshape(p) << parmdef_scale)
inspectors.append(
ns.inspector("scale") << StdTrees.define_inspector(
ns.beamshape, stations, label=self.label))
self.per_station = True
elif self.solve_scale is PER_ARRAY:
parms.append(ns.beamshape << parmdef_scale)
self.beamshape = lambda p: ns.beamshape
else:
ns.beamshape**(self.bf * 1e-9)
self.beamshape = lambda p: ns.beamshape
# solvable ellipticities
if self.solve_ell is PER_STATION:
self.ell = ns.ell
for p in stations:
ell_xy = ns.ell_xy(p) << parmdef_ell
parms.append(ell_xy)
self.ell(p) << Meq.Composer(ell_xy, -ell_xy)
inspectors.append(
ns.inspector("ellipticity") << StdTrees.define_inspector(
ns.ell_xy, stations, label=self.label))
self.per_station = True
elif self.solve_ell is PER_ARRAY:
ell_xy = ns.ell_xy << parmdef_ell
parms.append(ell_xy)
ns.ell << Meq.Composer(ell_xy, -ell_xy)
self.ell = lambda p: ns.ell
elif self.ellipticity != 0:
ns.ell**Meq.Constant([self.ellipticity, -self.ellipticity])
self.ell = lambda p: ns.ell
else:
self.ell = lambda p: None
# make parm group, if any solvables have been created
if parms:
parmgroups.append(ParmGroup.Subgroup("beam shape", list(parms)))
# solvable pointings
if self.solve_pointings:
self.dlm = ns.dlm
parmdef_0 = Meq.Parm(0, tags="beam solvable")
pparms = []
for p in stations:
dl = ns.dl(p) << parmdef_0
dm = ns.dm(p) << parmdef_0
ns.dlm(p) << Meq.Composer(dl, dm)
pparms += [dl, dm]
parmgroups.append(ParmGroup.Subgroup("pointing offsets", pparms))
parms += pparms
inspectors.append(
ns.inspector('dlm') << StdTrees.define_inspector(
ns.dlm, stations, label=self.label))
self.per_station = True
else:
ns.dlm_null**Meq.Constant([0, 0])
self.dlm = lambda p: ns.dlm_null
# add solve jobs
self.solvable = bool(parms)
if self.solvable:
self.pg_beam = ParmGroup.ParmGroup(self.label,
parms,
subgroups=parmgroups,
table_name="%s.fmep" %
self.label,
bookmark=True)
ParmGroup.SolveJob("cal_" + self.label,
"Calibrate beam parameters", self.pg_beam)
def compute_jones (Jones,sources,stations=None,pointing_offsets=None,inspectors=[],label='E',**kw):
stations = stations or Context.array.stations;
ns = Jones.Subscope();
JE = Jones("elem");
per_station = sky_rotation or pointing_offsets;
# read offsets file
if read_offsets_file:
offsets = list(map(float,open(offsets_file).read().split()));
if len(offsets) < (beam_number+1)*2:
raise ValueError("beam number %d not found in offsets file"%beam_number);
l_offset,m_offset = offsets[beam_number*2:(beam_number+1)*2];
if invert_l:
l_offset = -l_offset;
else:
l_offset,m_offset = 0,0;
# loop over sources to create per-element beamgains
for src in sources:
# If sky rotation and/or pointing offsets are in effect, we have a per-station beam.
# Otherwise the beam is the same for all stations.
if per_station:
for p in stations:
lm = src.direction.lm();
# apply rotation to put sources into the antenna frame
if sky_rotation:
xyz = Context.array.xyz(p);
pa_rot = Context.observation.phase_centre.pa_rot(xyz);
lm = ns.lmrot(src,p) << Meq.MatrixMultiply(pa_rot,src.direction.lm());
# apply offset (so pointing offsets are interpreted in the azel frame, if rotating)
if pointing_offsets:
lm = ns.lmoff(src,p) << lm + pointing_offsets(p);
# now make the beam node
make_beam_node(JE(src,p),filename_pattern,l_offset,m_offset,lm);
else:
make_beam_node(JE(src),filename_pattern,l_offset,m_offset,src.direction.lm());
# now load weights
wx = pickle.load(open(weight_filename_x, "rb"));
wy = pickle.load(open(weight_filename_y, "rb"));
if wx.shape[1] != num_elements or wy.shape[1] != num_elements:
raise ValueError("""weights files contain weights for %d (X) and %d (Y) complex
elements, %d expected"""%(wx.shape[1],wy.shape[1],num_elements));
if beam_number > wx.shape[0] or beam_number > wy.shape[0]:
raise ValueError("beam number %d not found in weights files"%beam_number);
# w0:x and w0:y are the nominal weight vectors
ns.w0('x') << Meq.Constant(value=wx[beam_number,:]);
ns.w0('y') << Meq.Constant(value=wy[beam_number,:]);
if sim_element_errors:
# create perturbed weights
a0 = 10**(min_ampl_var/20)-1;
a1 = 10**(max_ampl_var/20)-1;
for p in stations:
for xy in 'x','y':
werr = ns.werr(xy,p);
# amplitude and phase period and offset
for ap in 'ampl','phase':
p0 = werr("period",ap) << Meq.Constant(value=[random.uniform(min_period_var*3600,max_period_var*3600)/(2*math.pi) for i in range(num_elements)]);
if start_phased_up:
werr("sin",ap) << Meq.Sin(Meq.Time()/p0);
else:
t0 = werr("offset",ap) << Meq.Constant(value=[random.uniform(0,2*math.pi) for i in range(num_elements)]);
werr("sin",ap) << Meq.Sin((Meq.Time()/p0)+t0);
# amplitude excursion
e0 = werr("maxampl") << Meq.Constant(value=[random.uniform(a0,a1) for i in range(num_elements)]);
ep = werr("maxphase") << Meq.Constant(value=[random.uniform(min_phase_var*DEG,max_phase_var*DEG) for i in range(num_elements)]);
# weight errors
werr << Meq.Polar(1+e0*werr("sin","ampl"),ep*werr("sin","phase"));
ns.weight(xy,p) << ns.w0(xy)*werr;
# compute matrix norms based on nominal matrices
if per_station:
quallist = [ [src,p] for src in sources for p in stations ];
else:
quallist = [ [src] for src in sources ];
for qq in quallist:
ex0 = Jones(*(qq+["x0"])) << Meq.MatrixMultiply(JE(*qq),ns.w0("x"));
ey0 = Jones(*(qq+["y0"])) << Meq.MatrixMultiply(JE(*qq),ns.w0("y"));
J0 = Jones(*(qq+["nominal"])) << Meq.Composer(ex0,ey0,dims=[2,2]);
if do_normalize or (qq[0] is sources[0] and not do_correct):
make_norm(J0,Jones(*(qq+["norm"])));
if do_normalize:
# norm towards source is average per-station norm
if per_station:
for src in sources:
Jones(src,"norm") << Meq.Add(*[Jones(src,p,"norm") for p in stations])/len(stations);
elif not do_correct:
if per_station:
for src in sources:
Jones(sources[0],"norm") << Meq.Add(*[Jones(sources[0],p,"norm") for p in stations])/len(stations);
for src in sources[1:]:
Jones(src,"norm") << Meq.Identity(Jones(sources[0],"norm"));
else:
for src in sources:
Jones(src,"norm") << 1;
# put these together into Jones matrices
for src in sources:
# jesrc/JJ will eventually point to the unqualified element beam node
# of the unqualified jones node. Depending on whether we're per-station or
# not, this is qualified with src,p or just src.
jesrc = JE(src);
jnorm = Jones(src,"norm");
JJ = Jones(src);
if sim_element_errors:
for p in stations:
if per_station:
jesrc = JE(src,p);
JJ = JJ(p);
# jesrc returns a (2,N) matrix of element gains towards this source.
# Multiply this by the weights to get the "x" and "y" beams
ex = Jones(src,p,"x") << Meq.MatrixMultiply(jesrc,ns.weight("x",p));
ey = Jones(src,p,"y") << Meq.MatrixMultiply(jesrc,ns.weight("y",p));
if do_correct:
J0 = Jones(src,p,"ref") << Meq.Composer(ex,ey,dims=[2,2])/jnorm;
if src is sources[0]:
Jones(src,p) << Meq.Constant(value=[1,0,0,1],dims=[2,2]);
Jones(src,p,"inv") << Meq.MatrixInvert22(J0);
else:
Jones(src,p) << Meq.MatrixMultiply(Jones(sources[0],p,"inv"),J0);
else:
Jones(src,p) << Meq.Composer(ex,ey,dims=[2,2])/jnorm;
# no element errors, use nominal beam
else:
if per_station:
for p in stations:
if do_correct:
J0 = Jones(src,p,"ref") << Jones(src,p,"nominal")/jnorm;
if src is sources[0]:
Jones(src,p) << Meq.Constant(value=[1,0,0,1],dims=[2,2]);
Jones(src,p,"inv") << Meq.MatrixInvert22(J0);
else:
Jones(src,p) << Meq.MatrixMultiply(Jones(sources[0],p,"inv"),J0);
else:
Jones(src,p) << Jones(src,p,"nominal")/jnorm;
else:
if do_correct:
J0 = Jones(src,"ref") << Jones(src,"nominal")/jnorm;
if src is sources[0]:
Jones(src) << Meq.Constant(value=[1,0,0,1],dims=[2,2]);
Jones(src,"inv") << Meq.MatrixInvert22(J0);
else:
Jones(src) << Meq.MatrixMultiply(Jones(sources[0],"inv"),J0);
else:
Jones(src) << Jones(src,"nominal")/jnorm;
for p in stations:
Jones(src,p) << Meq.Identity(Jones(src));
# declare an inspector node for the Jones matrix -- will be defined below
insp = Jones.scope.inspector(label);
# define inspectors
insp << StdTrees.define_inspector(Jones,sources,stations,label=label);
inspectors += [ insp ];
if sim_element_errors:
insp1 = insp("werr") << StdTrees.define_inspector(ns.werr,("x","y"),stations,label="%s weight drifts"%label);
insp2 = insp("weights") << StdTrees.define_inspector(ns.weight,("x","y"),stations,label="%s weights"%label);
inspectors += [ insp1,insp2 ];
return Jones;
def compute_jones (Jones,sources,stations=None,pointing_offsets=None,inspectors=[],label='E',**kw):
stations = stations or Context.array.stations;
ns = Jones.Subscope();
JE = Jones("elem");
per_station = sky_rotation or pointing_offsets;
# read offsets file
if read_offsets_file:
offsets = map(float,open(offsets_file).read().split());
if len(offsets) < (beam_number+1)*2:
raise ValueError,"beam number %d not found in offsets file"%beam_number;
l_offset,m_offset = offsets[beam_number*2:(beam_number+1)*2];
if invert_l:
l_offset = -l_offset;
else:
l_offset,m_offset = 0,0;
# loop over sources to create per-element beamgains
for src in sources:
# If sky rotation and/or pointing offsets are in effect, we have a per-station beam.
# Otherwise the beam is the same for all stations.
if per_station:
for p in stations:
lm = src.direction.lm();
# apply rotation to put sources into the antenna frame
if sky_rotation:
xyz = Context.array.xyz(p);
pa_rot = Context.observation.phase_centre.pa_rot(xyz);
lm = ns.lmrot(src,p) << Meq.MatrixMultiply(pa_rot,src.direction.lm());
# apply offset (so pointing offsets are interpreted in the azel frame, if rotating)
if pointing_offsets:
lm = ns.lmoff(src,p) << lm + pointing_offsets(p);
# now make the beam node
make_beam_node(JE(src,p),filename_pattern,l_offset,m_offset,lm);
else:
make_beam_node(JE(src),filename_pattern,l_offset,m_offset,src.direction.lm());
# now load weights
wx = cPickle.load(open(weight_filename_x));
wy = cPickle.load(open(weight_filename_y));
if wx.shape[1] != num_elements or wy.shape[1] != num_elements:
raise ValueError,"""weights files contain weights for %d (X) and %d (Y) complex
elements, %d expected"""%(wx.shape[1],wy.shape[1],num_elements);
if beam_number > wx.shape[0] or beam_number > wy.shape[0]:
raise ValueError,"beam number %d not found in weights files"%beam_number;
# w0:x and w0:y are the nominal weight vectors
ns.w0('x') << Meq.Constant(value=wx[beam_number,:]);
ns.w0('y') << Meq.Constant(value=wy[beam_number,:]);
if sim_element_errors:
# create perturbed weights
a0 = 10**(min_ampl_var/20)-1;
a1 = 10**(max_ampl_var/20)-1;
for p in stations:
for xy in 'x','y':
werr = ns.werr(xy,p);
# amplitude and phase period and offset
for ap in 'ampl','phase':
p0 = werr("period",ap) << Meq.Constant(value=[random.uniform(min_period_var*3600,max_period_var*3600)/(2*math.pi) for i in range(num_elements)]);
if start_phased_up:
werr("sin",ap) << Meq.Sin(Meq.Time()/p0);
else:
t0 = werr("offset",ap) << Meq.Constant(value=[random.uniform(0,2*math.pi) for i in range(num_elements)]);
werr("sin",ap) << Meq.Sin((Meq.Time()/p0)+t0);
# amplitude excursion
e0 = werr("maxampl") << Meq.Constant(value=[random.uniform(a0,a1) for i in range(num_elements)]);
ep = werr("maxphase") << Meq.Constant(value=[random.uniform(min_phase_var*DEG,max_phase_var*DEG) for i in range(num_elements)]);
# weight errors
werr << Meq.Polar(1+e0*werr("sin","ampl"),ep*werr("sin","phase"));
ns.weight(xy,p) << ns.w0(xy)*werr;
# compute matrix norms based on nominal matrices
if per_station:
quallist = [ [src,p] for src in sources for p in stations ];
else:
quallist = [ [src] for src in sources ];
for qq in quallist:
ex0 = Jones(*(qq+["x0"])) << Meq.MatrixMultiply(JE(*qq),ns.w0("x"));
ey0 = Jones(*(qq+["y0"])) << Meq.MatrixMultiply(JE(*qq),ns.w0("y"));
J0 = Jones(*(qq+["nominal"])) << Meq.Composer(ex0,ey0,dims=[2,2]);
if do_normalize or (qq[0] is sources[0] and do_correct):
make_norm(J0,Jones(*(qq+["norm"])));
if do_normalize:
# norm towards source is average per-station norm
if per_station:
for src in sources:
Jones(src,"norm") << Meq.Add(*[Jones(src,p,"norm") for p in stations])/len(stations);
elif do_correct:
if per_station:
for src in sources:
Jones(sources[0],"norm") << Meq.Add(*[Jones(sources[0],p,"norm") for p in stations])/len(stations);
for src in sources[1:]:
Jones(src,"norm") << Meq.Identity(Jones(sources[0],"norm"));
else:
for src in sources:
Jones(src,"norm") << 1;
# put these together into Jones matrices
for src in sources:
# jesrc/JJ will eventually point to the unqualified element beam node
# of the unqualified jones node. Depending on whether we're per-station or
# not, this is qualified with src,p or just src.
jesrc = JE(src);
jnorm = Jones(src,"norm");
JJ = Jones(src);
if sim_element_errors:
for p in stations:
if per_station:
jesrc = JE(src,p);
JJ = JJ(p);
# jesrc returns a (2,N) matrix of element gains towards this source.
# Multiply this by the weights to get the "x" and "y" beams
ex = Jones(src,p,"x") << Meq.MatrixMultiply(jesrc,ns.weight("x",p));
ey = Jones(src,p,"y") << Meq.MatrixMultiply(jesrc,ns.weight("y",p));
if do_correct:
J0 = Jones(src,p,"ref") << Meq.Composer(ex,ey,dims=[2,2])/jnorm;
if src is sources[0]:
Jones(src,p) << Meq.Constant(value=[1,0,0,1],dims=[2,2]);
Jones(src,p,"inv") << Meq.MatrixInvert22(J0);
else:
Jones(src,p) << Meq.MatrixMultiply(Jones(sources[0],p,"inv"),J0);
else:
Jones(src,p) << Meq.Composer(ex,ey,dims=[2,2])/jnorm;
# no element errors, use nominal beam
else:
if per_station:
for p in stations:
if do_correct:
J0 = Jones(src,p,"ref") << Jones(src,p,"nominal")/jnorm;
if src is sources[0]:
Jones(src,p) << Meq.Constant(value=[1,0,0,1],dims=[2,2]);
Jones(src,p,"inv") << Meq.MatrixInvert22(J0);
else:
Jones(src,p) << Meq.MatrixMultiply(Jones(sources[0],p,"inv"),J0);
else:
Jones(src,p) << Jones(src,p,"nominal")/jnorm;
else:
if do_correct:
J0 = Jones(src,"ref") << Jones(src,"nominal")/jnorm;
if src is sources[0]:
Jones(src) << Meq.Constant(value=[1,0,0,1],dims=[2,2]);
Jones(src,"inv") << Meq.MatrixInvert22(J0);
else:
Jones(src) << Meq.MatrixMultiply(Jones(sources[0],"inv"),J0);
else:
Jones(src) << Jones(src,"nominal")/jnorm;
for p in stations:
Jones(src,p) << Meq.Identity(Jones(src));
# declare an inspector node for the Jones matrix -- will be defined below
insp = Jones.scope.inspector(label);
# define inspectors
insp << StdTrees.define_inspector(Jones,sources,stations,label=label);
inspectors += [ insp ];
if sim_element_errors:
insp1 = insp("werr") << StdTrees.define_inspector(ns.werr,("x","y"),stations,label="%s weight drifts"%label);
insp2 = insp("weights") << StdTrees.define_inspector(ns.weight,("x","y"),stations,label="%s weights"%label);
inspectors += [ insp1,insp2 ];
return Jones;
def init_parameters (self,ns,sources,stations,inspectors=[]):
if self.beamshape is not None:
return;
# create solvables if enabled
parms = [];
parmgroups = [];
parmdef_scale = Meq.Parm(self.bf*1e-9,tags="beam solvable");
parmdef_ell = Meq.Parm(self.ellipticity,tags="beam solvable");
self.per_station = False;
# solvable beam scale
if self.solve_scale is PER_STATION:
self.beamshape = ns.beamshape;
for p in stations:
parms.append(beamshape(p) << parmdef_scale);
inspectors.append(ns.inspector("scale") <<
StdTrees.define_inspector(ns.beamshape,stations,label=self.label));
self.per_station = True;
elif self.solve_scale is PER_ARRAY:
parms.append(ns.beamshape << parmdef_scale);
self.beamshape = lambda p:ns.beamshape;
else:
ns.beamshape ** (self.bf*1e-9);
self.beamshape = lambda p:ns.beamshape;
# solvable ellipticities
if self.solve_ell is PER_STATION:
self.ell = ns.ell;
for p in stations:
ell_xy = ns.ell_xy(p) << parmdef_ell;
parms.append(ell_xy);
self.ell(p) << Meq.Composer(ell_xy,-ell_xy);
inspectors.append(ns.inspector("ellipticity") <<
StdTrees.define_inspector(ns.ell_xy,stations,label=self.label));
self.per_station = True;
elif self.solve_ell is PER_ARRAY:
ell_xy = ns.ell_xy << parmdef_ell;
parms.append(ell_xy);
ns.ell << Meq.Composer(ell_xy,-ell_xy);
self.ell = lambda p:ns.ell;
elif self.ellipticity != 0:
ns.ell ** Meq.Constant([self.ellipticity,-self.ellipticity]);
self.ell = lambda p:ns.ell;
else:
self.ell = lambda p:None;
# make parm group, if any solvables have been created
if parms:
parmgroups.append(ParmGroup.Subgroup("beam shape",list(parms)));
# solvable pointings
if self.solve_pointings:
self.dlm = ns.dlm;
parmdef_0 = Meq.Parm(0,tags="beam solvable");
pparms = [];
for p in stations:
dl = ns.dl(p) << parmdef_0;
dm = ns.dm(p) << parmdef_0;
ns.dlm(p) << Meq.Composer(dl,dm);
pparms += [dl,dm];
parmgroups.append(ParmGroup.Subgroup("pointing offsets",pparms));
parms += pparms;
inspectors.append(ns.inspector('dlm') <<
StdTrees.define_inspector(ns.dlm,stations,label=self.label));
self.per_station = True;
else:
ns.dlm_null ** Meq.Constant([0,0]);
self.dlm = lambda p:ns.dlm_null;
# add solve jobs
self.solvable = bool(parms);
if self.solvable:
self.pg_beam = ParmGroup.ParmGroup(self.label,parms,subgroups=parmgroups,table_name="%s.fmep"%self.label,bookmark=True);
ParmGroup.SolveJob("cal_"+self.label,"Calibrate beam parameters",self.pg_beam);
def _define_forest(ns,parent=None,**kw):
if not mssel.msname:
raise RuntimeError,"MS not set";
if run_purr:
Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']);
# create Purr pipe
global purrpipe;
purrpipe = Purr.Pipe.Pipe(mssel.msname);
# setup contexts from MS
mssel.setup_observation_context(ns);
array = Meow.Context.array;
# make spigot nodes for data
mssel.enable_input_column(True);
spigots = array.spigots(corr=mssel.get_corr_index());
meqmaker.make_per_ifr_bookmarks(spigots,"Input visibilities");
# data tensor
ns.DT << Meq.Composer(dims=[0],mt_polling=True,*[ spigots(p,q) for p,q in array.ifrs() ]);
# predict tree using the MeqMaker
all_sources = meqmaker.get_source_list(ns);
dg_sources = deopts.enabled and dgsel.filter(all_sources);
if dg_sources:
# first group all sources without a diffgain on them
groups = [ [ src for src in all_sources if not src in dg_sources ] ];
# group diffgain-enabled sources by grouping tag
clusters = set([src.get_attr(diffgain_group,None) for src in dg_sources]);
dg_groups = [ (name,[ src for src in dg_sources if src.get_attr(diffgain_group) == name ]) for name in clusters if name ];
# add sources without a grouping tag individually, as single-source groups
dg_groups += [ (src.name,[src]) for src in dg_sources if not src.get_attr(diffgain_group,None) ];
# now sort by brightness
flux_dgg = [ (sum([src.get_attr('Iapp',0) or src.get_attr('I') for src in dgg[1]]),dgg) for dgg in dg_groups ];
flux_dgg.sort(lambda a,b:cmp(b[0],a[0]));
diffgain_labels = [ dgg[0] for flux,dgg in flux_dgg ];
groups += [ dgg[1] for flux,dgg in flux_dgg ];
num_diffgains = len(flux_dgg);
# now make predict trees
models = [];
for i,group in enumerate(groups):
MT = ns.MT(i);
predict = meqmaker.make_predict_tree(MT.Subscope(),sources=group);
ns.MT(i) << Meq.Composer(dims=[0],mt_polling=True,*[ predict(p,q) for p,q in array.ifrs() ]);
models.append(ns.MT(i));
print "Number of diffgain predict groups:",len(groups);
else:
diffgain_labels = [];
num_diffgains = 0;
predict = meqmaker.make_predict_tree(ns);
ns.MT << Meq.Composer(dims=[0],mt_polling=True,*[ predict(p,q) for p,q in array.ifrs() ]);
models = [ ns.MT ];
solve_ifrs = array.subset(calibrate_ifrs,strict=False).ifrs();
downsample_subtiling = [ stefcal_downsample_timeint,stefcal_downsample_freqint ] if stefcal_downsample else [1,1];
import Calico.OMS.StefCal.StefCal
kwopts = {}
gopts.set_stefcal_node_options(kwopts,visualize=stefcal_visualize);
bopts.set_stefcal_node_options(kwopts,visualize=stefcal_visualize);
deopts.set_stefcal_node_options(kwopts,visualize=stefcal_visualize);
ns.stefcal << Meq.PyNode(class_name="StefCalNode",module_name=Calico.OMS.StefCal.StefCal.__file__,
ifrs=[ "%s:%s"%(p,q) for p,q in array.ifrs() ],
baselines=[ array.baseline(ip,iq) for (ip,p),(iq,q) in array.ifr_index() ],
solve_ifrs=[ "%s:%s"%(p,q) for p,q in solve_ifrs ],
noise_per_chan=stefcal_noise_per_chan,
downsample_subtiling=downsample_subtiling,
num_major_loops=stefcal_nmajor,
regularization_factor=1e-6,#
rescale=stefcal_rescale,
init_from_previous=False,
critical_flag_threshold=critical_flag_threshold,
diffgain_labels=diffgain_labels,
# flagging options
output_flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT,
# IFR gain solution options
apply_ifr_gains=stefcal_ifr_gains,
solve_ifr_gains=(stefcal_ifr_gain_mode != MODE_SOLVE_APPLY),
reset_ifr_gains=stefcal_ifr_gain_reset,
save_ifr_gains=(stefcal_ifr_gain_mode == MODE_SOLVE_SAVE),
ifr_gain_table=stefcal_ifr_gain_table,
per_chan_ifr_gains=stefcal_per_chan_ifr_gains,
diag_ifr_gains=(stefcal_diagonal_ifr_gains == DIAGONLY),
residuals=(do_output == CORRECTED_RESIDUALS),
subtract_dgsrc=(do_output == CORRECTED_DATA_SUB),
verbose=stefcal_verbose,
children=[ns.DT]+models,**kwopts);
inspectors = meqmaker.get_inspectors() or [];
# make output bookmarks
nv = 0;
for p,q in array.ifrs():
sel = ns.output_sel(p,q) << Meq.Selector(ns.stefcal,index=range(nv,nv+4),multi=True);
ns.output(p,q) << Meq.Composer(sel,dims=[2,2]);
nv += 4;
meqmaker.make_per_ifr_bookmarks(ns.output,"Output visibilities");
Bookmarks.Page("StefCal outputs").add(ns.stefcal,viewer="Record Browser");
if gopts.enabled and gopts.visualize and stefcal_visualize:
ns.stefcal_vis_G << Meq.PyNode(class_name="StefCalVisualizer",module_name=Calico.OMS.StefCal.StefCal.__file__,
label="G",flag_unity=visualize_flag_unity,norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations);
ns.stefcal_vis_G_avg << Meq.PyNode(class_name="StefCalVisualizer",module_name=Calico.OMS.StefCal.StefCal.__file__,
label="G",freq_average=True,flag_unity=visualize_flag_unity,norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations);
Bookmarks.Page("StefCal G plotter").add(ns.stefcal_vis_G,viewer="Result Plotter");
Bookmarks.Page("StefCal G inspector").add(ns.stefcal_vis_G_avg,viewer="Collections Plotter");
inspectors += [ ns.stefcal_vis_G,ns.stefcal_vis_G_avg ];
if bopts.enabled and bopts.visualize and stefcal_visualize:
ns.stefcal_vis_B << Meq.PyNode(class_name="StefCalVisualizer",module_name=Calico.OMS.StefCal.StefCal.__file__,
label="B",flag_unity=visualize_flag_unity,norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations);
ns.stefcal_vis_B_avg << Meq.PyNode(class_name="StefCalVisualizer",module_name=Calico.OMS.StefCal.StefCal.__file__,
label="B",freq_average=True,flag_unity=visualize_flag_unity,norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations);
Bookmarks.Page("StefCal B plotter").add(ns.stefcal_vis_B,viewer="Result Plotter");
Bookmarks.Page("StefCal B inspector").add(ns.stefcal_vis_B_avg,viewer="Collections Plotter");
inspectors += [ ns.stefcal_vis_B,ns.stefcal_vis_B_avg ];
if deopts.enabled and deopts.visualize and stefcal_visualize:
for i,label in enumerate(diffgain_labels):
vde = ns.stefcal_vis_dE(label) << Meq.PyNode(class_name="StefCalVisualizer",module_name=Calico.OMS.StefCal.StefCal.__file__,
label="dE:%s"%label,flag_unity=visualize_flag_unity,norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations);
vde_avg = ns.stefcal_vis_dE_avg(label) << Meq.PyNode(class_name="StefCalVisualizer",module_name=Calico.OMS.StefCal.StefCal.__file__,
label="dE:%s"%label,freq_average=True,flag_unity=visualize_flag_unity,norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations);
Bookmarks.Page("StefCal dE:%s plotter"%label).add(vde,viewer="Result Plotter");
Bookmarks.Page("StefCal dE:%s inspector"%label).add(vde_avg,viewer="Collections Plotter");
inspectors += [ vde,vde_avg ];
# make sinks
StdTrees.make_sinks(ns,ns.output,spigots=spigots,post=inspectors,
corr_index=mssel.get_corr_index());
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False));
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size());
TDLRuntimeMenu("Make an image from this MS",*imsel.option_list());
# and close meqmaker -- this exports annotations, etc
meqmaker.close();
# add options to clear all solutions
from Calico.OMS.StefCal import StefCal
TDLRuntimeOption("stefcal_reset_all","Remove all existing solutions",False);
for opt in gopts,bopts,deopts:
if opt.enabled:
TDLRuntimeOption("reset","Remove existing %s solutions (%s)"%(opt.label,os.path.basename(opt.table)),False,namespace=opt);
if stefcal_ifr_gains:
TDLRuntimeOption("stefcal_reset_ifr_gains","Remove existing interferometer errors (%s)"%(
os.path.basename(stefcal_ifr_gain_table)),False);
TDLRuntimeJob(_run_stefcal,"Run StefCal",job_id="stefcal");
def _define_forest (ns, parent=None, **kw):
if not mssel.msname:
raise RuntimeError ('MS name not set')
mssel.setup_observation_context (ns)
array = Context.array
# Data and model input
if do_solve or output_type.need_data:
mssel.enable_input_column (True)
spigots = array.spigots (corr=mssel.get_corr_index ())
meqmaker.make_per_ifr_bookmarks (spigots, 'Input visibilities')
else:
mssel.enable_input_column (False)
spigots = None
if do_solve or output_type.need_model:
predict = meqmaker.make_predict_tree (ns, uvdata=None)
else:
predict = None
# Data output
outputs = output_type.apply (ns, meqmaker, array.ifrs (), spigots, predict)
# Flagging
if flag_enable and output_type.flag_data:
flaggers = []
if flag_res is not None or flag_mean_res is not None:
for p, q in array.ifrs ():
ns.absres(p,q) << Meq.Abs (outputs(p,q))
if flag_res is not None:
for p, q in array.ifrs ():
ns.flagres(p,q) << Meq.ZeroFlagger (ns.absres(p,q) - flag_res,
oper='gt',
flag_bit=MSUtils.FLAGMASK_OUTPUT)
flaggers.append (ns.flagres)
meqmaker.make_per_ifr_bookmarks (ns.flagres, 'Residual amplitude flags')
if flag_mean_res is not None:
ns.meanabsres << Meq.Mean (*[ns.absres(p,q) for p, q in array.ifrs()])
ns.flagmeanres << Meq.ZeroFlagger (ns.meanabsres - flag_mean_res,
oper='gt', flag_bit=MSUtils.FLAGMASK_OUTPUT)
Bookmarks.Page ('Mean residual amplitude flags').add (ns.flagmeanres,
viewer='Result Plotter')
flaggers.append (lambda p, q: ns.flagmeanres)
if flaggers:
meqmaker.make_per_ifr_bookmarks (outputs, output_type.desc + ' (unflagged)')
for p, q in array.ifrs ():
ns.flagged(p,q) << Meq.MergeFlags (outputs(p,q), *[f(p,q) for f in flaggers])
outputs = ns.flagged
meqmaker.make_per_ifr_bookmarks (outputs, output_type.desc)
# Solve trees
if do_solve:
# parse ifr specification
solve_ifrs = array.subset (calibrate_ifrs, strict=False).ifrs()
if not solve_ifrs:
raise RuntimeError ('No interferometers selected for calibration. '
'Check your ifr specification (under calibration options).')
lhs, rhs, weights, modulo = cal_quant.apply (solve_ifrs, predict, spigots)
solve_tree = StdTrees.SolveTree (ns, lhs, solve_ifrs=solve_ifrs,
weights=weights, modulo=modulo)
outputs = solve_tree.sequencers (inputs=rhs, outputs=outputs)
StdTrees.make_sinks (ns, outputs, spigots=spigots,
post=meqmaker.get_inspectors () or [],
corr_index=mssel.get_corr_index ())
if not do_solve:
name = 'Generate ' + output_type.desc.lower ()
comment = 'Generated ' + output_type.desc.lower ()
def run_tree (mqs, parent, wait=False, **kw):
return mqs.execute (Context.vdm.name, mssel.create_io_request (tile_size),
wait=wait)
doc = """Input data are sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""
TDLRuntimeMenu(name, TDLOption ('tile_size', 'Tile size, in timeslots',
[10, 60, 120, 240], more=int, doc=doc),
TDLJob (run_tree, name, job_id='generate_visibilities'))
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up
# during compilation.
TDLRuntimeOptions (*meqmaker.runtime_options (nest=False))
if do_solve:
TDLRuntimeOptions (*ParmGroup.get_solvejob_options ())
imsel = mssel.imaging_selector (npix=512, arcmin=meqmaker.estimate_image_size ())
TDLRuntimeMenu ('Make an image', *imsel.option_list ())
meqmaker.close()
def _define_forest(ns,parent=None,**kw):
if run_purr:
Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']);
# create Purr pipe
global purrpipe;
purrpipe = Purr.Pipe.Pipe(mssel.msname);
# get antennas from MS
ANTENNAS = mssel.get_antenna_set(list(range(1,15)));
array = Meow.IfrArray(ns,ANTENNAS,mirror_uvw=False);
stas = array.stations();
# get phase centre from MS, setup observation
observation = Meow.Observation(ns,phase_centre=mssel.get_phase_dir(),
linear=mssel.is_linear_pol(),
circular=mssel.is_circular_pol());
Meow.Context.set(array,observation);
# get active correlations from MS
Meow.Context.active_correlations = mssel.get_correlations();
# make spigot nodes
spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index());
# ...and an inspector for them
StdTrees.vis_inspector(ns.inspector('input'),spigots,
bookmark="Inspect input visibilities");
inspectors = [ ns.inspector('input') ];
Bookmarks.make_node_folder("Input visibilities by baseline",
[ spigots(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2);
inspect_ifrs = array.ifrs();
if do_solve:
# filter solvable baselines by baseline length
solve_ifrs = [];
antpos = mssel.ms_antenna_positions;
if (min_baseline or max_baseline) and antpos is not None:
for (ip,p),(iq,q) in array.ifr_index():
baseline = math.sqrt(((antpos[ip,:]-antpos[iq,:])**2).sum());
if (not min_baseline or baseline > min_baseline) and \
(not max_baseline or baseline < max_baseline):
solve_ifrs.append((p,q));
else:
solve_ifrs = array.ifrs();
inspect_ifrs = solve_ifrs;
# make a predict tree using the MeqMaker
if do_solve or do_subtract:
predict = meqmaker.make_predict_tree(ns);
# make a ParmGroup and solve jobs for source parameters, if we have any
if do_solve:
parms = {};
for src in meqmaker.get_source_list(ns):
parms.update([(p.name,p) for p in src.get_solvables()]);
if parms:
pg_src = ParmGroup.ParmGroup("source",list(parms.values()),
table_name="sources.fmep",
individual=True,bookmark=True);
# now make a solvejobs for the source
ParmGroup.SolveJob("cal_source","Calibrate source model",pg_src);
# make nodes to compute residuals
if do_subtract:
residuals = ns.residuals;
for p,q in array.ifrs():
residuals(p,q) << spigots(p,q) - predict(p,q);
outputs = residuals;
# and now we may need to correct the outputs
if do_correct:
if do_correct_sky:
srcs = meqmaker.get_source_list(ns);
sky_correct = srcs and srcs[0];
else:
sky_correct = None;
outputs = meqmaker.correct_uv_data(ns,outputs,sky_correct=sky_correct,inspect_ifrs=inspect_ifrs);
# make solve trees
if do_solve:
# inputs to the solver are based on calibration type
# if calibrating visibilities, feed them to condeq directly
if cal_type == CAL.VIS:
observed = spigots;
model = predict;
# else take ampl/phase component
else:
model = ns.model;
observed = ns.observed;
if cal_type == CAL.AMPL:
for p,q in array.ifrs():
observed(p,q) << Meq.Abs(spigots(p,q));
model(p,q) << Meq.Abs(predict(p,q));
elif cal_type == CAL.LOGAMPL:
for p,q in array.ifrs():
observed(p,q) << Meq.Log(Meq.Abs(spigots(p,q)));
model(p,q) << Meq.Log(Meq.Abs(predict(p,q)));
elif cal_type == CAL.PHASE:
for p,q in array.ifrs():
observed(p,q) << 0;
model(p,q) << Meq.Abs(predict(p,q))*Meq.FMod(Meq.Arg(spigots(p,q))-Meq.Arg(predict(p,q)),2*math.pi);
else:
raise ValueError("unknown cal_type setting: "+str(cal_type));
# make a solve tree
solve_tree = StdTrees.SolveTree(ns,model,solve_ifrs=solve_ifrs);
# the output of the sequencer is either the residuals or the spigots,
# according to what has been set above
outputs = solve_tree.sequencers(inputs=observed,outputs=outputs);
# make sinks and vdm.
# The list of inspectors must be supplied here
inspectors += meqmaker.get_inspectors() or [];
StdTrees.make_sinks(ns,outputs,spigots=spigots0,post=inspectors);
Bookmarks.make_node_folder("Corrected/residual visibilities by baseline",
[ outputs(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2);
if not do_solve:
if do_subtract:
name = "Generate residuals";
comment = "Generated residual visibilities.";
elif do_correct:
name = "Generate corrected data";
comment = "Generated corrected visibilities.";
else:
name = None;
if name:
# make a TDL job to runsthe tree
def run_tree (mqs,parent,**kw):
global tile_size;
purrpipe.title("Calibrating").comment(comment);
mqs.execute(Meow.Context.vdm.name,mssel.create_io_request(tile_size),wait=False);
TDLRuntimeMenu(name,
TDLOption('tile_size',"Tile size, in timeslots",[10,60,120,240],more=int,
doc="""Input data is sliced by time, and processed in chunks (tiles) of
the indicated size. Larger tiles are faster, but use more memory."""),
TDLRuntimeJob(run_tree,name)
);
# very important -- insert meqmaker's runtime options properly
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False));
# insert solvejobs
if do_solve:
TDLRuntimeOptions(*ParmGroup.get_solvejob_options());
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size());
TDLRuntimeMenu("Make an image from this MS",*imsel.option_list());
# and close meqmaker -- this exports annotations, etc
meqmaker.close();
def _define_forest(ns, parent=None, **kw):
if not mssel.msname:
raise RuntimeError, "MS not set"
if run_purr:
Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.'])
# create Purr pipe
global purrpipe
purrpipe = Purr.Pipe.Pipe(mssel.msname)
# setup contexts from MS
mssel.setup_observation_context(ns)
array = Meow.Context.array
# make spigot nodes for data
mssel.enable_input_column(True)
spigots = array.spigots(corr=mssel.get_corr_index())
meqmaker.make_per_ifr_bookmarks(spigots, "Input visibilities")
# data tensor
ns.DT << Meq.Composer(
dims=[0], mt_polling=True, *[spigots(p, q) for p, q in array.ifrs()])
# predict tree using the MeqMaker
all_sources = meqmaker.get_source_list(ns)
dg_sources = deopts.enabled and dgsel.filter(all_sources)
if dg_sources:
# first group all sources without a diffgain on them
groups = [[src for src in all_sources if not src in dg_sources]]
# group diffgain-enabled sources by grouping tag
clusters = set(
[src.get_attr(diffgain_group, None) for src in dg_sources])
dg_groups = [(name, [
src for src in dg_sources if src.get_attr(diffgain_group) == name
]) for name in clusters if name]
# add sources without a grouping tag individually, as single-source groups
dg_groups += [(src.name, [src]) for src in dg_sources
if not src.get_attr(diffgain_group, None)]
# now sort by brightness
flux_dgg = [(sum(
[src.get_attr('Iapp', 0) or src.get_attr('I')
for src in dgg[1]]), dgg) for dgg in dg_groups]
flux_dgg.sort(lambda a, b: cmp(b[0], a[0]))
diffgain_labels = [dgg[0] for flux, dgg in flux_dgg]
groups += [dgg[1] for flux, dgg in flux_dgg]
num_diffgains = len(flux_dgg)
# now make predict trees
models = []
for i, group in enumerate(groups):
MT = ns.MT(i)
predict = meqmaker.make_predict_tree(MT.Subscope(), sources=group)
ns.MT(i) << Meq.Composer(dims=[0],
mt_polling=True,
*[predict(p, q) for p, q in array.ifrs()])
models.append(ns.MT(i))
print "Number of diffgain predict groups:", len(groups)
else:
diffgain_labels = []
num_diffgains = 0
predict = meqmaker.make_predict_tree(ns)
ns.MT << Meq.Composer(dims=[0],
mt_polling=True,
*[predict(p, q) for p, q in array.ifrs()])
models = [ns.MT]
solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs()
downsample_subtiling = [
stefcal_downsample_timeint, stefcal_downsample_freqint
] if stefcal_downsample else [1, 1]
import Calico.OMS.StefCal.StefCal
kwopts = {}
gopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize)
bopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize)
deopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize)
ns.stefcal << Meq.PyNode(
class_name="StefCalNode",
module_name=Calico.OMS.StefCal.StefCal.__file__,
ifrs=["%s:%s" % (p, q) for p, q in array.ifrs()],
baselines=[
array.baseline(ip, iq) for (ip, p), (iq, q) in array.ifr_index()
],
solve_ifrs=["%s:%s" % (p, q) for p, q in solve_ifrs],
noise_per_chan=stefcal_noise_per_chan,
downsample_subtiling=downsample_subtiling,
num_major_loops=stefcal_nmajor,
regularization_factor=1e-6, #
rescale=stefcal_rescale,
init_from_previous=False,
critical_flag_threshold=critical_flag_threshold,
diffgain_labels=diffgain_labels,
# flagging options
output_flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT,
# IFR gain solution options
apply_ifr_gains=stefcal_ifr_gains,
solve_ifr_gains=(stefcal_ifr_gain_mode != MODE_SOLVE_APPLY),
reset_ifr_gains=stefcal_ifr_gain_reset,
save_ifr_gains=(stefcal_ifr_gain_mode == MODE_SOLVE_SAVE),
ifr_gain_table=stefcal_ifr_gain_table,
per_chan_ifr_gains=stefcal_per_chan_ifr_gains,
diag_ifr_gains=(stefcal_diagonal_ifr_gains == DIAGONLY),
residuals=(do_output == CORRECTED_RESIDUALS),
subtract_dgsrc=(do_output == CORRECTED_DATA_SUB),
verbose=stefcal_verbose,
children=[ns.DT] + models,
**kwopts)
inspectors = meqmaker.get_inspectors() or []
# make output bookmarks
nv = 0
for p, q in array.ifrs():
sel = ns.output_sel(p, q) << Meq.Selector(
ns.stefcal, index=range(nv, nv + 4), multi=True)
ns.output(p, q) << Meq.Composer(sel, dims=[2, 2])
nv += 4
meqmaker.make_per_ifr_bookmarks(ns.output, "Output visibilities")
Bookmarks.Page("StefCal outputs").add(ns.stefcal, viewer="Record Browser")
if gopts.enabled and gopts.visualize and stefcal_visualize:
ns.stefcal_vis_G << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="G",
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
ns.stefcal_vis_G_avg << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="G",
freq_average=True,
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
Bookmarks.Page("StefCal G plotter").add(ns.stefcal_vis_G,
viewer="Result Plotter")
Bookmarks.Page("StefCal G inspector").add(ns.stefcal_vis_G_avg,
viewer="Collections Plotter")
inspectors += [ns.stefcal_vis_G, ns.stefcal_vis_G_avg]
if bopts.enabled and bopts.visualize and stefcal_visualize:
ns.stefcal_vis_B << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="B",
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
ns.stefcal_vis_B_avg << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="B",
freq_average=True,
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
Bookmarks.Page("StefCal B plotter").add(ns.stefcal_vis_B,
viewer="Result Plotter")
Bookmarks.Page("StefCal B inspector").add(ns.stefcal_vis_B_avg,
viewer="Collections Plotter")
inspectors += [ns.stefcal_vis_B, ns.stefcal_vis_B_avg]
if deopts.enabled and deopts.visualize and stefcal_visualize:
for i, label in enumerate(diffgain_labels):
vde = ns.stefcal_vis_dE(label) << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="dE:%s" % label,
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
vde_avg = ns.stefcal_vis_dE_avg(label) << Meq.PyNode(
class_name="StefCalVisualizer",
module_name=Calico.OMS.StefCal.StefCal.__file__,
label="dE:%s" % label,
freq_average=True,
flag_unity=visualize_flag_unity,
norm_offdiag=visualize_norm_offdiag,
vells_label=Context.correlations)
Bookmarks.Page("StefCal dE:%s plotter" % label).add(
vde, viewer="Result Plotter")
Bookmarks.Page("StefCal dE:%s inspector" % label).add(
vde_avg, viewer="Collections Plotter")
inspectors += [vde, vde_avg]
# make sinks
StdTrees.make_sinks(ns,
ns.output,
spigots=spigots,
post=inspectors,
corr_index=mssel.get_corr_index())
# this should come last, since runtime options may be built up during compilation.
TDLRuntimeOptions(*meqmaker.runtime_options(nest=False))
# finally, setup imaging options
imsel = mssel.imaging_selector(npix=512,
arcmin=meqmaker.estimate_image_size())
TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
# and close meqmaker -- this exports annotations, etc
meqmaker.close()
# add options to clear all solutions
from Calico.OMS.StefCal import StefCal
TDLRuntimeOption("stefcal_reset_all", "Remove all existing solutions",
False)
for opt in gopts, bopts, deopts:
if opt.enabled:
TDLRuntimeOption("reset",
"Remove existing %s solutions (%s)" %
(opt.label, os.path.basename(opt.table)),
False,
namespace=opt)
if stefcal_ifr_gains:
TDLRuntimeOption(
"stefcal_reset_ifr_gains",
"Remove existing interferometer errors (%s)" %
(os.path.basename(stefcal_ifr_gain_table)), False)
TDLRuntimeJob(_run_stefcal, "Run StefCal", job_id="stefcal")
