def compute_jones (self,jones,sources,stations=None,tags=None,label='',**kw): stations = stations or Context.array.stations(); # figure out which sources to apply to if self.subset: srclist = self._subset_parser.parse_list(self.subset); sources = [ sources[i] for i in srclist ]; # create parm definitions for each jones element tags = NodeTags(tags) + "solvable"; rm_parm = Meq.Parm(0,tags=tags+"rm"); # loop over sources for src in sources: jj = jones(src); jj("RM") << rm_parm; fr = jj("fr") << jj("RM") / Meq.Sqr(Meq.Freq()); cos_fr = jj("cos") << Meq.Cos(fr); sin_fr = jj("sin") << Meq.Sin(fr); jj << Meq.Matrix22(cos_fr,-sin_fr,sin_fr,cos_fr); for p in stations: jj(p) << Meq.Identity(jj); # make parmgroups for diagonal and off-diagonal terms self.pg_fr = ParmGroup.ParmGroup(label+"_fr", [ jones(src,"RM") for src in sources ], table_name="%s_fr.fmep"%label,bookmark=False); # make bookmarks Bookmarks.make_node_folder("%s Faraday rotation"%label, [ jones(src,"fr") for src in sources ],sorted=True); # make solvejobs ParmGroup.SolveJob("cal_"+label+"_fr","Calibrate %s"%label,self.pg_fr); return jones;
def init_nodes(self, ns, tags=None, label=""): ifrs = Context.array.ifrs() G = ns.gain if not G(*ifrs[0]).initialized(): def1 = Meow.Parm(1, tags=tags) def0 = Meow.Parm(0, tags=tags) gains = [] for p, q in ifrs: gg = [] for corr in Context.correlations: if corr in Context.active_correlations: cc = corr.lower() gain_ri = [ resolve_parameter(cc, G(p, q, cc, "r"), def1, tags="ifr gain real"), resolve_parameter(cc, G(p, q, cc, "i"), def0, tags="ifr gain imag"), ] gg.append(G(p, q, cc) << Meq.ToComplex(*gain_ri)) gains += gain_ri else: gg.append(1) G(p, q) << Meq.Matrix22(*gg) pg_ifr_ampl = ParmGroup.ParmGroup( label, gains, individual_toggles=False, table_name="%s.fmep" % label, bookmark=False ) Bookmarks.make_node_folder( "%s (interferometer-based gains)" % label, [G(p, q) for p, q in ifrs], sorted=True, nrow=2, ncol=2 ) ParmGroup.SolveJob("cal_%s" % label, "Calibrate %s (interferometer-based gains)" % label, pg_ifr_ampl) return G
def init_nodes (self,ns,tags=None,label=''): ifrs = Context.array.ifrs(); G = ns.gain; if not G(*ifrs[0]).initialized(): def1 = Meow.Parm(1,tags=tags); def0 = Meow.Parm(0,tags=tags); gains = []; for p,q in ifrs: gg = []; for corr in Context.correlations: if corr in Context.active_correlations: cc = corr.lower(); gain_ri = [ resolve_parameter(cc,G(p,q,cc,'r'),def1,tags="ifr gain real"), resolve_parameter(cc,G(p,q,cc,'i'),def0,tags="ifr gain imag") ]; gg.append(G(p,q,cc) << Meq.ToComplex(*gain_ri)); gains += gain_ri; else: gg.append(1); G(p,q) << Meq.Matrix22(*gg); pg_ifr_ampl = ParmGroup.ParmGroup(label,gains, individual_toggles=False, table_name="%s.fmep"%label,bookmark=False); Bookmarks.make_node_folder("%s (interferometer-based gains)"%label, [ G(p,q) for p,q in ifrs ],sorted=True,nrow=2,ncol=2); ParmGroup.SolveJob("cal_%s"%label, "Calibrate %s (interferometer-based gains)"%label,pg_ifr_ampl); return G;
def _define_forest(ns): # setup contexts from MS mssel.setup_observation_context(ns); array = Meow.Context.array; observation = Meow.Context.observation; stas = array.stations(); # make spigot nodes spigots = spigots0 = array.spigots(corr=mssel.get_corr_index()); for p,q in array.ifrs(): spigots('abs',p,q) << Meq.Abs(spigots(p,q)); # ...and an inspector for them Meow.StdTrees.vis_inspector(ns.inspector('input'),spigots, bookmark="Inspect input visibilities"); Meow.StdTrees.vis_inspector(ns.inspector('ampl'),spigots('abs'), bookmark="Inspect mean visibility amplitudes"); Bookmarks.make_node_folder("Input visibilities by baseline", [ spigots(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2); ns.inspectors << Meq.ReqMux(ns.inspector('input'),ns.inspector('ampl')); ns.VisDataMux << Meq.VisDataMux(post=ns.inspectors); # add imaging options imsel = mssel.imaging_selector(npix=512,arcmin=120); TDLRuntimeMenu("Make an image from this MS",*imsel.option_list());
def __init__ (self,label,members=[],name=None, subgroups=None, individual=False,individual_toggles=True,bookmark=True, table_name="calibration.mep",table_in_ms=True, **kw): """Creates a ParmGroup. 'label' is the label of the group 'members' is a list of MeqParm nodes 'name' is a descriptive name for the group. If not set, label is used. 'subgroups' is a list of Subgroup objects, if parms are to be subgrouped 'individual' is True if individual options are to be provided per parameter. Note that this is incompatible with subgroups 'individual_toggles' is True if individual solvasbility toggles are to be provided in addition to subgroups. 'bookmark' is True if bookmarks for the parameters are to be automatically provided 'table_name' is the name of a MEP table 'table_in_ms' is True if the table should reside inside the current MS (in which case the MS selector from the global Meow.Context is invoked to get the name of the MS) """ self.label = label; self.name = name or label; # sort group members by name self.nodes = sorted_nodes = self._sort_members(members); # now collect list of subgroups self.subgroups = subgroups or []; self.individual_toggles = individual_toggles; if subgroups and individual: raise TypeError,"cannot combine subgroups and individual parameters"; self.parm_subgroups = dict(); for sg in self.subgroups: for parm in sg.nodes: self.parm_subgroups.setdefault(parm.name,[]).append(sg); self.nodes = sorted_nodes; # setup table names self.table_name = table_name; self.default_table_name = os.path.basename(table_name); if table_in_ms and Context.mssel and Context.mssel.msname: self.table_name = os.path.join(Context.mssel.msname,self.default_table_name); self._table_in_ms = table_in_ms; self._individual = individual; # put table name into the parms for node in self.nodes: node.initrec().table_name = self.table_name; # create bookmarks (if specified as a [W,H], it gives the number of parms to bookmark) if bookmark: if isinstance(bookmark,tuple) and len(bookmark) == 2: ncol,nrow = bookmark; else: ncol,nrow = 2,3; Bookmarks.make_node_folder("Parameters: %s"%label, sorted_nodes,sorted=True,ncol=ncol,nrow=nrow); # add ourselves to global list of parmgroups global _all_parmgroups; _all_parmgroups.append(self);
def __init__ (self,label,members=[],name=None, subgroups=None, individual=False,individual_toggles=True,bookmark=True, table_name="calibration.mep",table_in_ms=True, **kw): """Creates a ParmGroup. 'label' is the label of the group 'members' is a list of MeqParm nodes 'name' is a descriptive name for the group. If not set, label is used. 'subgroups' is a list of Subgroup objects, if parms are to be subgrouped 'individual' is True if individual options are to be provided per parameter. Note that this is incompatible with subgroups 'individual_toggles' is True if individual solvasbility toggles are to be provided in addition to subgroups. 'bookmark' is True if bookmarks for the parameters are to be automatically provided 'table_name' is the name of a MEP table 'table_in_ms' is True if the table should reside inside the current MS (in which case the MS selector from the global Meow.Context is invoked to get the name of the MS) """ self.label = label; self.name = name or label; # sort group members by name self.nodes = sorted_nodes = self._sort_members(members); # now collect list of subgroups self.subgroups = subgroups or []; self.individual_toggles = individual_toggles; if subgroups and individual: raise TypeError("cannot combine subgroups and individual parameters"); self.parm_subgroups = dict(); for sg in self.subgroups: for parm in sg.nodes: self.parm_subgroups.setdefault(parm.name,[]).append(sg); self.nodes = sorted_nodes; # setup table names self.table_name = table_name; self.default_table_name = os.path.basename(table_name); if table_in_ms and Context.mssel and Context.mssel.msname: self.table_name = os.path.join(Context.mssel.msname,self.default_table_name); self._table_in_ms = table_in_ms; self._individual = individual; # put table name into the parms for node in self.nodes: node.initrec().table_name = self.table_name; # create bookmarks (if specified as a [W,H], it gives the number of parms to bookmark) if bookmark: if isinstance(bookmark,tuple) and len(bookmark) == 2: ncol,nrow = bookmark; else: ncol,nrow = 2,3; Bookmarks.make_node_folder("Parameters: %s"%label, sorted_nodes,sorted=True,ncol=ncol,nrow=nrow); # add ourselves to global list of parmgroups global _all_parmgroups; _all_parmgroups.append(self);
def _define_forest (ns): # create Array object num_antennas = 36 # for ASKAP simulation xntd_list = [ str(i) for i in range(1,num_antennas+1) ]; array = Meow.IfrArray(ns,xntd_list,ms_uvw=True); # create an Observation object observation = Meow.Observation(ns); # set global context Meow.Context.set(array=array,observation=observation); # create a source model and make list of corrupted sources allsky = Meow.Patch(ns,'all',observation.phase_centre); sources = sky_models.make_model(ns,"S"); for src in sources: lm = src.direction.lm(); E = ns.E(src.name); for p in array.stations(): pa= ns.ParAngle(p) << Meq.ParAngle(observation.phase_centre.radec(), array.xyz(p)) ns.CosPa(p) << Meq.Cos(pa) ns.SinPa(p) << Meq.Sin(pa) ns.rot_matrix(p) << Meq.Matrix22(ns.CosPa(p),-1.0 * ns.SinPa(p),ns.SinPa(p),ns.CosPa(p)) # compute "apparent" position of source per each antenna lm_rot=ns.lm_rot(src.name,p) << Meq.MatrixMultiply(ns.rot_matrix(p),lm) # compute E for apparent position tdp_voltage_response(ns,src,p,E(p),lm_rot); allsky.add(src.corrupt(E)); observed = allsky.visibilities(); # make some useful inspectors. Collect them into a list, since we need # to give a list of 'post' nodes to make_sinks() below pg = Bookmarks.Page("Inspectors",1,2); inspectors = []; inspectors.append( Meow.StdTrees.vis_inspector(ns.inspect_observed,observed) ); pg.add(ns.inspect_observed,viewer="Collections Plotter"); Meow.StdTrees.make_sinks(ns,observed,spigots=False,post=inspectors);
def compute_jones(self, jones, sources, stations=None, tags=None, label='', **kw): stations = stations or Context.array.stations() is_complex = self.matrix_type != "real" # set up qualifier labels depending on polarization definition if Context.observation.circular(): x, y, X, Y = 'r', 'l', 'R', 'L' else: x, y, X, Y = 'x', 'y', 'X', 'Y' xx, xy, yx, yy = x + x, x + y, y + x, y + y rxx, rxy, ryx, ryy = [q + ":r" for q in (xx, xy, yx, yy)] ixx, ixy, iyx, iyy = [q + ":i" for q in (xx, xy, yx, yy)] # prepare parm definitions for real and imag parts of diagonal and off-diagonal elements tags = NodeTags(tags) + "solvable" diag_pdefs = offdiag_pdefs = None # loop over sources parms_diag = [] parms_offdiag = [] subgroups_diag = [] subgroups_offdiag = [] # Define a function to put together a matrix element, depending on whether we're in real or complex mode. # This is also responsible for appending parms to the appropriate parm and subgroup lists. # Note that for the purely-real case we still create parms called 'J:xx:r' (and not 'J:xx' directly), # this is to keep MEP tables mutually-compatible naming wise. if self.matrix_type == "real": def make_element(jj, parmdef, *parmlists): parm = jj('r') << parmdef[0] for plist in parmlists: plist.append(parm) return jj << Meq.Identity(parm) else: def make_element(jj, parmdef, *parmlists): parms = [jj('r') << parmdef[0], jj('i') << parmdef[1]] for plist in parmlists: plist += parms return jj << Meq.ToComplex(*parms) for src in sources: sg = src.name if self.independent_solve else '' # (re)create parm definitions. if diag_pdefs is None or self.independent_solve: diag_pdefs = (Meq.Parm(complex(self.init_diag).real, tags=tags + "diag real", solve_group=sg), Meq.Parm(complex(self.init_diag).imag, tags=tags + "diag imag", solve_group=sg)) offdiag_pdfefs = (Meq.Parm(complex(self.init_offdiag).imag, tags=tags + "offdiag real", solve_group=sg), Meq.Parm(complex(self.init_offdiag).imag, tags=tags + "offdiag imag", solve_group=sg)) # now loop to create nodes sgdiag = [] sgoff = [] for p in stations: jj = jones(src, p) jj << Meq.Matrix22( make_element(jj(xx), diag_pdefs, parms_diag, sgdiag), make_element(jj(xy), offdiag_pdefs, parms_offdiag, sgoff) if self._offdiag else 0, make_element(jj(yx), offdiag_pdefs, parms_offdiag, sgoff) if self._offdiag else 0, make_element(jj(yy), diag_pdefs, parms_diag, sgdiag)) # add subgroup for this source subgroups_diag.append(ParmGroup.Subgroup(src.name, sgdiag)) subgroups_offdiag.append(ParmGroup.Subgroup(src.name, sgoff)) # re-sort by name from past.builtins import cmp from functools import cmp_to_key subgroups_diag.sort(key=cmp_to_key(lambda a, b: cmp(a.name, b.name))) subgroups_offdiag.sort( key=cmp_to_key(lambda a, b: cmp(a.name, b.name))) # make parmgroups for diagonal and off-diagonal terms self.pg_diag = ParmGroup.ParmGroup(label + "_diag", parms_diag, subgroups=subgroups_diag, table_name="%s_diag.fmep" % label, bookmark=False) if self._offdiag: self.pg_offdiag = ParmGroup.ParmGroup( label + "_offdiag", parms_offdiag, subgroups=subgroups_offdiag, table_name="%s_offdiag.fmep" % label, bookmark=False) # make bookmarks Bookmarks.make_node_folder("%s diagonal terms" % label, [ jones(src, p, zz) for src in sources for p in stations for zz in ["xx", "yy"] ], sorted=True) if self._offdiag: Bookmarks.make_node_folder("%s off-diagonal terms" % label, [ jones(src, p, zz) for src in sources for p in stations for zz in ["xy", "yx"] ], sorted=True) # make solvejobs ParmGroup.SolveJob("cal_" + label + "_diag", "Calibrate %s diagonal terms" % label, self.pg_diag) if self._offdiag: ParmGroup.SolveJob("cal_" + label + "_offdiag", "Calibrate %s off-diagonal terms" % label, self.pg_offdiag) return jones
def _define_forest(ns, parent=None, **kw): if run_purr: Timba.TDL.GUI.log_message("starting purr") Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.']) mssel.setup_observation_context(ns) array = Meow.Context.array # make spigot nodes outputs = spigots = array.spigots(corr=mssel.get_corr_index()) # ...and an inspector for them Meow.StdTrees.vis_inspector(ns.inspector('input'), spigots, bookmark="Inspect input visibilities") inspectors = [ns.inspector('input')] Bookmarks.make_node_folder("Input visibilities by baseline", [spigots(p, q) for p, q in array.ifrs()], sorted=True, ncol=2, nrow=2) # add in-tree flaggers, if appropriate if tree_flag_enable: flaggers = [] # make nodes to take complex amplitude, if any flaggers are going to use them if tree_flag_ampl is not None or tree_flag_mean_ampl is not None or tree_flag_mean_freq_ampl is not None: # use matrix norm if tree_flag_norm: for p, q in array.ifrs(): node = outputs(p, q) nsq = node('sq') << Meq.MatrixMultiply( node, node('conj') << Meq.ConjTranspose(node)) nsqtr = nsq( 'tr') << (nsq(11) << Meq.Selector(nsq, index=0)) + ( nsq(22) << Meq.Selector(nsq, index=3)) ns.ampl(p, q) << Meq.Sqrt(nsqtr('abs') << Meq.Abs(nsqtr)) # use per-element amplitudes else: for p, q in array.ifrs(): ns.ampl(p, q) << Meq.Abs(outputs(p, q)) # ...and an inspector for them Meow.StdTrees.vis_inspector( ns.inspector('ampl'), ns.ampl, bookmark="Inspect mean-freq visibility amplitudes") inspectors.append(ns.inspector('ampl')) # make flagger for amplitudes if tree_flag_ampl is not None: for p, q in array.ifrs(): ns.flagampl(p, q) << Meq.ZeroFlagger( ns.ampl(p, q) - tree_flag_ampl, oper='gt', flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT) flaggers.append(ns.flagampl) # make flagger for mean-ifr amplitudes if tree_flag_mean_ampl is not None: ns.meanampl << Meq.Mean(*[ns.ampl(p, q) for p, q in array.ifrs()]) ns.flagmeanampl << Meq.ZeroFlagger( ns.meanampl - tree_flag_mean_ampl, oper='gt', flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT) Meow.Bookmarks.Page("Mean-ifr visibility amplitudes").add( ns.meanampl, viewer="Result Plotter") flaggers.append(lambda p, q: ns.flagmeanampl) # make flagger for mean-fr-freq amplitudes if tree_flag_mean_freq_ampl is not None: ns.meanampl**Meq.Mean(*[ns.ampl(p, q) for p, q in array.ifrs()]) ns.meanfreqampl << Meq.Mean(ns.meanampl, reduction_axes="freq") ns.flagmeanfreqampl << Meq.ZeroFlagger( ns.meanfreqampl - tree_flag_mean_freq_ampl, oper='gt', flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT) Meow.Bookmarks.Page("Mean-ifr-freq visibility amplitudes").add( ns.meanfreqampl, viewer="Collections Plotter") flaggers.append(lambda p, q: ns.flagmeanfreqampl) # now merge all flaggers if flaggers: for p, q in array.ifrs(): ns.flagged(p, q) << Meq.MergeFlags( outputs(p, q), *[fl(p, q) for fl in flaggers]) outputs = ns.flagged Meow.StdTrees.vis_inspector( ns.inspector('flagged'), outputs, bookmark="Inspect flagged visibilities") inspectors.append(ns.inspector('flagged')) Bookmarks.make_node_folder( "Flagged visibilities by baseline", [outputs(p, q) for p, q in array.ifrs()], sorted=True, ncol=2, nrow=2) mssel.enable_write_flags(True) treeflag_ms_opt.show(True) if outputs is spigots: ns.VisDataMux << Meq.VisDataMux(post=( inspectors[0] if len(inspectors) < 2 else Meq.ReqMux(*inspectors))) else: Meow.StdTrees.make_sinks(ns, outputs, spigots=spigots, post=inspectors) # add imaging options imsel = mssel.imaging_selector(npix=512, arcmin=120) TDLRuntimeMenu("Make an image from this MS", *imsel.option_list())
def _define_forest(ns, parent=None, **kw): if run_purr: Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.']) # create Purr pipe global purrpipe purrpipe = Purr.Pipe.Pipe(mssel.msname) # get antennas from MS ANTENNAS = mssel.get_antenna_set(list(range(1, 15))) array = Meow.IfrArray(ns, ANTENNAS, mirror_uvw=False) stas = array.stations() # get phase centre from MS, setup observation observation = Meow.Observation(ns, phase_centre=mssel.get_phase_dir(), linear=mssel.is_linear_pol(), circular=mssel.is_circular_pol()) Meow.Context.set(array, observation) # get active correlations from MS Meow.Context.active_correlations = mssel.get_correlations() # make spigot nodes spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index()) # ...and an inspector for them StdTrees.vis_inspector(ns.inspector('input'), spigots, bookmark="Inspect input visibilities") inspectors = [ns.inspector('input')] Bookmarks.make_node_folder("Input visibilities by baseline", [spigots(p, q) for p, q in array.ifrs()], sorted=True, ncol=2, nrow=2) inspect_ifrs = array.ifrs() if do_solve: # filter solvable baselines by baseline length solve_ifrs = [] antpos = mssel.ms_antenna_positions if (min_baseline or max_baseline) and antpos is not None: for (ip, p), (iq, q) in array.ifr_index(): baseline = math.sqrt( ((antpos[ip, :] - antpos[iq, :])**2).sum()) if (not min_baseline or baseline > min_baseline) and \ (not max_baseline or baseline < max_baseline): solve_ifrs.append((p, q)) else: solve_ifrs = array.ifrs() inspect_ifrs = solve_ifrs # make a predict tree using the MeqMaker if do_solve or do_subtract: predict = meqmaker.make_predict_tree(ns) # make a ParmGroup and solve jobs for source parameters, if we have any if do_solve: parms = {} for src in meqmaker.get_source_list(ns): parms.update([(p.name, p) for p in src.get_solvables()]) if parms: pg_src = ParmGroup.ParmGroup("source", list(parms.values()), table_name="sources.fmep", individual=True, bookmark=True) # now make a solvejobs for the source ParmGroup.SolveJob("cal_source", "Calibrate source model", pg_src) # make nodes to compute residuals if do_subtract: residuals = ns.residuals for p, q in array.ifrs(): residuals(p, q) << spigots(p, q) - predict(p, q) outputs = residuals # and now we may need to correct the outputs if do_correct: if do_correct_sky: srcs = meqmaker.get_source_list(ns) sky_correct = srcs and srcs[0] else: sky_correct = None outputs = meqmaker.correct_uv_data(ns, outputs, sky_correct=sky_correct, inspect_ifrs=inspect_ifrs) # make solve trees if do_solve: # inputs to the solver are based on calibration type # if calibrating visibilities, feed them to condeq directly if cal_type == CAL.VIS: observed = spigots model = predict # else take ampl/phase component else: model = ns.model observed = ns.observed if cal_type == CAL.AMPL: for p, q in array.ifrs(): observed(p, q) << Meq.Abs(spigots(p, q)) model(p, q) << Meq.Abs(predict(p, q)) elif cal_type == CAL.LOGAMPL: for p, q in array.ifrs(): observed(p, q) << Meq.Log(Meq.Abs(spigots(p, q))) model(p, q) << Meq.Log(Meq.Abs(predict(p, q))) elif cal_type == CAL.PHASE: for p, q in array.ifrs(): observed(p, q) << 0 model(p, q) << Meq.Abs(predict(p, q)) * Meq.FMod( Meq.Arg(spigots(p, q)) - Meq.Arg(predict(p, q)), 2 * math.pi) else: raise ValueError("unknown cal_type setting: " + str(cal_type)) # make a solve tree solve_tree = StdTrees.SolveTree(ns, model, solve_ifrs=solve_ifrs) # the output of the sequencer is either the residuals or the spigots, # according to what has been set above outputs = solve_tree.sequencers(inputs=observed, outputs=outputs) # make sinks and vdm. # The list of inspectors must be supplied here inspectors += meqmaker.get_inspectors() or [] StdTrees.make_sinks(ns, outputs, spigots=spigots0, post=inspectors) Bookmarks.make_node_folder("Corrected/residual visibilities by baseline", [outputs(p, q) for p, q in array.ifrs()], sorted=True, ncol=2, nrow=2) if not do_solve: if do_subtract: name = "Generate residuals" comment = "Generated residual visibilities." elif do_correct: name = "Generate corrected data" comment = "Generated corrected visibilities." else: name = None if name: # make a TDL job to runsthe tree def run_tree(mqs, parent, **kw): global tile_size purrpipe.title("Calibrating").comment(comment) mqs.execute(Meow.Context.vdm.name, mssel.create_io_request(tile_size), wait=False) TDLRuntimeMenu( name, TDLOption( 'tile_size', "Tile size, in timeslots", [10, 60, 120, 240], more=int, doc= """Input data is sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory.""" ), TDLRuntimeJob(run_tree, name)) # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)) # insert solvejobs if do_solve: TDLRuntimeOptions(*ParmGroup.get_solvejob_options()) # finally, setup imaging options imsel = mssel.imaging_selector(npix=512, arcmin=meqmaker.estimate_image_size()) TDLRuntimeMenu("Make an image from this MS", *imsel.option_list()) # and close meqmaker -- this exports annotations, etc meqmaker.close()
def _define_forest(ns, parent=None, **kw): if not mssel.msname: raise RuntimeError('MS name not set') mssel.setup_observation_context(ns) array = Context.array # Data and model input if do_solve or output_type.need_data: mssel.enable_input_column(True) spigots = array.spigots(corr=mssel.get_corr_index()) meqmaker.make_per_ifr_bookmarks(spigots, 'Input visibilities') else: mssel.enable_input_column(False) spigots = None if do_solve or output_type.need_model: predict = meqmaker.make_predict_tree(ns, uvdata=None) else: predict = None # Data output outputs = output_type.apply(ns, meqmaker, array.ifrs(), spigots, predict) # Flagging if flag_enable and output_type.flag_data: flaggers = [] if flag_res is not None or flag_mean_res is not None: for p, q in array.ifrs(): ns.absres(p, q) << Meq.Abs(outputs(p, q)) if flag_res is not None: for p, q in array.ifrs(): ns.flagres(p, q) << Meq.ZeroFlagger( ns.absres(p, q) - flag_res, oper='gt', flag_bit=MSUtils.FLAGMASK_OUTPUT) flaggers.append(ns.flagres) meqmaker.make_per_ifr_bookmarks(ns.flagres, 'Residual amplitude flags') if flag_mean_res is not None: ns.meanabsres << Meq.Mean( *[ns.absres(p, q) for p, q in array.ifrs()]) ns.flagmeanres << Meq.ZeroFlagger(ns.meanabsres - flag_mean_res, oper='gt', flag_bit=MSUtils.FLAGMASK_OUTPUT) Bookmarks.Page('Mean residual amplitude flags').add( ns.flagmeanres, viewer='Result Plotter') flaggers.append(lambda p, q: ns.flagmeanres) if flaggers: meqmaker.make_per_ifr_bookmarks(outputs, output_type.desc + ' (unflagged)') for p, q in array.ifrs(): ns.flagged(p, q) << Meq.MergeFlags( outputs(p, q), *[f(p, q) for f in flaggers]) outputs = ns.flagged meqmaker.make_per_ifr_bookmarks(outputs, output_type.desc) # Solve trees if do_solve: # parse ifr specification solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs() if not solve_ifrs: raise RuntimeError( 'No interferometers selected for calibration. ' 'Check your ifr specification (under calibration options).') lhs, rhs, weights, modulo = cal_quant.apply(solve_ifrs, predict, spigots) solve_tree = StdTrees.SolveTree(ns, lhs, solve_ifrs=solve_ifrs, weights=weights, modulo=modulo) outputs = solve_tree.sequencers(inputs=rhs, outputs=outputs) StdTrees.make_sinks(ns, outputs, spigots=spigots, post=meqmaker.get_inspectors() or [], corr_index=mssel.get_corr_index()) if not do_solve: name = 'Generate ' + output_type.desc.lower() comment = 'Generated ' + output_type.desc.lower() def run_tree(mqs, parent, wait=False, **kw): return mqs.execute(Context.vdm.name, mssel.create_io_request(tile_size), wait=wait) doc = """Input data are sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory.""" TDLRuntimeMenu( name, TDLOption('tile_size', 'Tile size, in timeslots', [10, 60, 120, 240], more=int, doc=doc), TDLJob(run_tree, name, job_id='generate_visibilities')) # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up # during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)) if do_solve: TDLRuntimeOptions(*ParmGroup.get_solvejob_options()) imsel = mssel.imaging_selector(npix=512, arcmin=meqmaker.estimate_image_size()) TDLRuntimeMenu('Make an image', *imsel.option_list()) meqmaker.close()
def _define_forest(ns,parent=None,**kw): if run_purr: Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']); # create Purr pipe global purrpipe; purrpipe = Purr.Pipe.Pipe(mssel.msname); # get antennas from MS ANTENNAS = mssel.get_antenna_set(list(range(1,15))); array = Meow.IfrArray(ns,ANTENNAS,mirror_uvw=False); stas = array.stations(); # get phase centre from MS, setup observation observation = Meow.Observation(ns,phase_centre=mssel.get_phase_dir(), linear=mssel.is_linear_pol(), circular=mssel.is_circular_pol()); Meow.Context.set(array,observation); # get active correlations from MS Meow.Context.active_correlations = mssel.get_correlations(); # make spigot nodes spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index()); # ...and an inspector for them StdTrees.vis_inspector(ns.inspector('input'),spigots, bookmark="Inspect input visibilities"); inspectors = [ ns.inspector('input') ]; Bookmarks.make_node_folder("Input visibilities by baseline", [ spigots(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2); inspect_ifrs = array.ifrs(); if do_solve: # filter solvable baselines by baseline length solve_ifrs = []; antpos = mssel.ms_antenna_positions; if (min_baseline or max_baseline) and antpos is not None: for (ip,p),(iq,q) in array.ifr_index(): baseline = math.sqrt(((antpos[ip,:]-antpos[iq,:])**2).sum()); if (not min_baseline or baseline > min_baseline) and \ (not max_baseline or baseline < max_baseline): solve_ifrs.append((p,q)); else: solve_ifrs = array.ifrs(); inspect_ifrs = solve_ifrs; # make a predict tree using the MeqMaker if do_solve or do_subtract: predict = meqmaker.make_predict_tree(ns); # make a ParmGroup and solve jobs for source parameters, if we have any if do_solve: parms = {}; for src in meqmaker.get_source_list(ns): parms.update([(p.name,p) for p in src.get_solvables()]); if parms: pg_src = ParmGroup.ParmGroup("source",list(parms.values()), table_name="sources.fmep", individual=True,bookmark=True); # now make a solvejobs for the source ParmGroup.SolveJob("cal_source","Calibrate source model",pg_src); # make nodes to compute residuals if do_subtract: residuals = ns.residuals; for p,q in array.ifrs(): residuals(p,q) << spigots(p,q) - predict(p,q); outputs = residuals; # and now we may need to correct the outputs if do_correct: if do_correct_sky: srcs = meqmaker.get_source_list(ns); sky_correct = srcs and srcs[0]; else: sky_correct = None; outputs = meqmaker.correct_uv_data(ns,outputs,sky_correct=sky_correct,inspect_ifrs=inspect_ifrs); # make solve trees if do_solve: # inputs to the solver are based on calibration type # if calibrating visibilities, feed them to condeq directly if cal_type == CAL.VIS: observed = spigots; model = predict; # else take ampl/phase component else: model = ns.model; observed = ns.observed; if cal_type == CAL.AMPL: for p,q in array.ifrs(): observed(p,q) << Meq.Abs(spigots(p,q)); model(p,q) << Meq.Abs(predict(p,q)); elif cal_type == CAL.LOGAMPL: for p,q in array.ifrs(): observed(p,q) << Meq.Log(Meq.Abs(spigots(p,q))); model(p,q) << Meq.Log(Meq.Abs(predict(p,q))); elif cal_type == CAL.PHASE: for p,q in array.ifrs(): observed(p,q) << 0; model(p,q) << Meq.Abs(predict(p,q))*Meq.FMod(Meq.Arg(spigots(p,q))-Meq.Arg(predict(p,q)),2*math.pi); else: raise ValueError("unknown cal_type setting: "+str(cal_type)); # make a solve tree solve_tree = StdTrees.SolveTree(ns,model,solve_ifrs=solve_ifrs); # the output of the sequencer is either the residuals or the spigots, # according to what has been set above outputs = solve_tree.sequencers(inputs=observed,outputs=outputs); # make sinks and vdm. # The list of inspectors must be supplied here inspectors += meqmaker.get_inspectors() or []; StdTrees.make_sinks(ns,outputs,spigots=spigots0,post=inspectors); Bookmarks.make_node_folder("Corrected/residual visibilities by baseline", [ outputs(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2); if not do_solve: if do_subtract: name = "Generate residuals"; comment = "Generated residual visibilities."; elif do_correct: name = "Generate corrected data"; comment = "Generated corrected visibilities."; else: name = None; if name: # make a TDL job to runsthe tree def run_tree (mqs,parent,**kw): global tile_size; purrpipe.title("Calibrating").comment(comment); mqs.execute(Meow.Context.vdm.name,mssel.create_io_request(tile_size),wait=False); TDLRuntimeMenu(name, TDLOption('tile_size',"Tile size, in timeslots",[10,60,120,240],more=int, doc="""Input data is sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory."""), TDLRuntimeJob(run_tree,name) ); # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)); # insert solvejobs if do_solve: TDLRuntimeOptions(*ParmGroup.get_solvejob_options()); # finally, setup imaging options imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size()); TDLRuntimeMenu("Make an image from this MS",*imsel.option_list()); # and close meqmaker -- this exports annotations, etc meqmaker.close();
def _define_forest(ns, **kw): if parmtab is None: raise RuntimeError, "please select a valid parmtable in compile-time options" # make one MeqParm per each funklet parmdef = Meq.Parm(table_name=parmtable_name) parmnodes = dict([(name, (ns[name] << parmdef)) for name in parmtab.funklet_names()]) # reprocess flag is reset every time something changes in the parmnodes dict, # so we simply repeat this loop until things stop changing reprocess = True while reprocess: reprocess = False # see if we have recipe for making compound objects based on last funklet name qualifier nqlist = [ name.rsplit(':', 1) for name in parmnodes.iterkeys() if ':' in name ] # set of basenames and set of all qualifiers basenames = set([nq[0] for nq in nqlist]) quals = set([nq[1] for nq in nqlist]) for quallist, agg_func in aggregators: # look for an aggregator whose qualifiers are a subset of the given set if set(quallist) <= quals: # for each basename, attempt to find funklets for every qualifier in the list for basename in basenames: fqnames = [basename + ':' + qq for qq in quallist] funks = [parmnodes.get(name, None) for name in fqnames] # if every basename:qq combination gives a valid funklet, call the aggregator if None not in funks: aggnode = ns[basename] << agg_func(*funks) # remove aggregated funklets from the parmnodes dict for name in fqnames: del parmnodes[name] # add new node to dict parmnodes[basename] = aggnode reprocess = True # sort nodes by qualified name nodes = list(parmnodes.iteritems()) nodes.sort(lambda a, b: ParmTables.cmp_qualified_names(a[0], b[0])) nodes = [namenode[1] for namenode in nodes] # make intermediate math node pall = ns.all_parms_raw << Meq.Composer( children=nodes, mt_polling=True, dims=[0]) pfm = ns.all_parms_fm << Meq.Mean(pall, reduction_axes="freq") # ns.all_minus_fm << pall - pfm; # ns.all_over_fm << pall/pfm; # make root node ns.root << Meq.ReqMux( pfm # ns.all_minus_fm, # ns.all_over_fm, # ns.fft_all_over_fm << Meq.FFTBrick(Meq.Selector(ns.all_over_fm,index=0,multi=True),axes_in=(hiid("time"),hiid("freq")),axes_out=(hiid("l"),hiid("m"))) ) Bookmarks.Page("Composite view").add(ns.all_parms_fm, viewer="Collections Plotter") # make bookmark folder Bookmarks.make_node_folder("All parameters", nodes, sorted=True) # close parmtable, else meqserver will be unable to get at it parmtab.close() # now make runtime options global active_axes active_axes = [] reg_domain_opts = [] for iaxis in range(mequtils.max_axis): stats = parmtab.axis_stats(iaxis) if not stats.empty(): active_axes.append((iaxis, stats.name, stats)) reg_domain_opts.append( TDLOption("num_cells_%s" % stats.name, "Number of grid points in %s" % stats.name, [len(stats.cells)], more=int)) TDLRuntimeMenu( "View parameters over regularly gridded domain", *(reg_domain_opts + [ TDLJob(_view_parameters_over_regularized_domain, "View over gridded domain") ]))
def _define_forest (ns,**kw): if parmtab is None: raise RuntimeError,"please select a valid parmtable in compile-time options"; # make one MeqParm per each funklet parmdef = Meq.Parm(table_name=parmtable_name); parmnodes = dict([(name,(ns[name] << parmdef)) for name in parmtab.funklet_names()]); # reprocess flag is reset every time something changes in the parmnodes dict, # so we simply repeat this loop until things stop changing reprocess = True; while reprocess: reprocess = False; # see if we have recipe for making compound objects based on last funklet name qualifier nqlist = [ name.rsplit(':',1) for name in parmnodes.iterkeys() if ':' in name ]; # set of basenames and set of all qualifiers basenames = set([nq[0] for nq in nqlist]); quals = set([nq[1] for nq in nqlist]); for quallist,agg_func in aggregators: # look for an aggregator whose qualifiers are a subset of the given set if set(quallist) <= quals: # for each basename, attempt to find funklets for every qualifier in the list for basename in basenames: fqnames = [ basename+':'+qq for qq in quallist ]; funks = [ parmnodes.get(name,None) for name in fqnames ]; # if every basename:qq combination gives a valid funklet, call the aggregator if None not in funks: aggnode = ns[basename] << agg_func(*funks); # remove aggregated funklets from the parmnodes dict for name in fqnames: del parmnodes[name]; # add new node to dict parmnodes[basename] = aggnode; reprocess = True; # sort nodes by qualified name nodes = list(parmnodes.iteritems()); nodes.sort(lambda a,b:ParmTables.cmp_qualified_names(a[0],b[0])); nodes = [ namenode[1] for namenode in nodes ]; # make intermediate math node pall = ns.all_parms_raw << Meq.Composer(children=nodes,mt_polling=True,dims=[0]); pfm = ns.all_parms_fm << Meq.Mean(pall,reduction_axes="freq"); # ns.all_minus_fm << pall - pfm; # ns.all_over_fm << pall/pfm; # make root node ns.root << Meq.ReqMux( pfm # ns.all_minus_fm, # ns.all_over_fm, # ns.fft_all_over_fm << Meq.FFTBrick(Meq.Selector(ns.all_over_fm,index=0,multi=True),axes_in=(hiid("time"),hiid("freq")),axes_out=(hiid("l"),hiid("m"))) ); Bookmarks.Page("Composite view").add(ns.all_parms_fm,viewer="Collections Plotter"); # make bookmark folder Bookmarks.make_node_folder("All parameters",nodes,sorted=True); # close parmtable, else meqserver will be unable to get at it parmtab.close(); # now make runtime options global active_axes; active_axes = []; reg_domain_opts = []; for iaxis in range(mequtils.max_axis): stats = parmtab.axis_stats(iaxis); if not stats.empty(): active_axes.append((iaxis,stats.name,stats)); reg_domain_opts.append( TDLOption("num_cells_%s"%stats.name,"Number of grid points in %s"%stats.name,[len(stats.cells)],more=int) ); TDLRuntimeMenu("View parameters over regularly gridded domain", *(reg_domain_opts + [TDLJob(_view_parameters_over_regularized_domain,"View over gridded domain")]));
def compute_jones (self,jones,sources,stations=None,tags=None,label='',**kw): stations = stations or Context.array.stations(); is_complex = self.matrix_type != "real"; # set up qualifier labels depending on polarization definition if Context.observation.circular(): x,y,X,Y = 'r','l','R','L'; else: x,y,X,Y = 'x','y','X','Y'; xx,xy,yx,yy = x+x,x+y,y+x,y+y; rxx,rxy,ryx,ryy = [ q+":r" for q in xx,xy,yx,yy ]; ixx,ixy,iyx,iyy = [ q+":i" for q in xx,xy,yx,yy ]; # prepare parm definitions for real and imag parts of diagonal and off-diagonal elements tags = NodeTags(tags) + "solvable"; diag_pdefs = offdiag_pdefs = None; # loop over sources parms_diag = []; parms_offdiag = []; subgroups_diag = []; subgroups_offdiag = []; # Define a function to put together a matrix element, depending on whether we're in real or complex mode. # This is also responsible for appending parms to the appropriate parm and subgroup lists. # Note that for the purely-real case we still create parms called 'J:xx:r' (and not 'J:xx' directly), # this is to keep MEP tables mutually-compatible naming wise. if self.matrix_type == "real": def make_element (jj,parmdef,*parmlists): parm = jj('r') << parmdef[0]; for plist in parmlists: plist.append(parm); return jj << Meq.Identity(parm); else: def make_element (jj,parmdef,*parmlists): parms = [ jj('r') << parmdef[0],jj('i') << parmdef[1] ]; for plist in parmlists: plist += parms; return jj << Meq.ToComplex(*parms); for src in sources: sg = src.name if self.independent_solve else ''; # (re)create parm definitions. if diag_pdefs is None or self.independent_solve: diag_pdefs = ( Meq.Parm(complex(self.init_diag).real,tags=tags+"diag real",solve_group=sg), Meq.Parm(complex(self.init_diag).imag,tags=tags+"diag imag",solve_group=sg) ); offdiag_pdfefs = ( Meq.Parm(complex(self.init_offdiag).imag,tags=tags+"offdiag real",solve_group=sg), Meq.Parm(complex(self.init_offdiag).imag,tags=tags+"offdiag imag",solve_group=sg) ); # now loop to create nodes sgdiag = []; sgoff = []; for p in stations: jj = jones(src,p); jj << Meq.Matrix22( make_element(jj(xx),diag_pdefs,parms_diag,sgdiag), make_element(jj(xy),offdiag_pdefs,parms_offdiag,sgoff) if self._offdiag else 0, make_element(jj(yx),offdiag_pdefs,parms_offdiag,sgoff) if self._offdiag else 0, make_element(jj(yy),diag_pdefs,parms_diag,sgdiag) ); # add subgroup for this source subgroups_diag.append(ParmGroup.Subgroup(src.name,sgdiag)); subgroups_offdiag.append(ParmGroup.Subgroup(src.name,sgoff)); # re-sort by name subgroups_diag.sort(lambda a,b:cmp(a.name,b.name)); subgroups_offdiag.sort(lambda a,b:cmp(a.name,b.name)); # make parmgroups for diagonal and off-diagonal terms self.pg_diag = ParmGroup.ParmGroup(label+"_diag",parms_diag, subgroups=subgroups_diag, table_name="%s_diag.fmep"%label,bookmark=False); if self._offdiag: self.pg_offdiag = ParmGroup.ParmGroup(label+"_offdiag",parms_offdiag, subgroups=subgroups_offdiag, table_name="%s_offdiag.fmep"%label,bookmark=False); # make bookmarks Bookmarks.make_node_folder("%s diagonal terms"%label, [ jones(src,p,zz) for src in sources for p in stations for zz in "xx","yy" ],sorted=True); if self._offdiag: Bookmarks.make_node_folder("%s off-diagonal terms"%label, [ jones(src,p,zz) for src in sources for p in stations for zz in "xy","yx" ],sorted=True); # make solvejobs ParmGroup.SolveJob("cal_"+label+"_diag","Calibrate %s diagonal terms"%label,self.pg_diag); if self._offdiag: ParmGroup.SolveJob("cal_"+label+"_offdiag","Calibrate %s off-diagonal terms"%label,self.pg_offdiag); return jones;
subgroups = [ ParmGroup.Subgroup(X+X,[jones(p,zz) for zz in rxx,ixx for p in stations]), ParmGroup.Subgroup(Y+Y,[jones(p,zz) for zz in ryy,iyy for p in stations]), ParmGroup.Subgroup("real part",[jones(p,zz) for zz in rxx,ryy for p in stations]), ParmGroup.Subgroup("imaginary part",[jones(p,zz) for zz in ixx,iyy for p in stations]) ]; subgroups += [ ParmGroup.Subgroup("station %s"%p,[jones(p,zz) for zz in rxx,ixx,ryy,iyy ]) for p in stations ]; self.pg_diag = ParmGroup.ParmGroup(label+"_diag", [ jones(p,zz) for zz in rxx,ixx,ryy,iyy for p in stations ], subgroups = subgroups, table_name="%s_diag.fmep"%label,bookmark=False); # make bookmarks Bookmarks.make_node_folder("%s diagonal terms"%label, [ jones(p,zz) for p in stations for zz in xx,yy ],sorted=True); # make solvejobs ParmGroup.SolveJob("cal_"+label+"_diag","Calibrate %s diagonal terms"%label,self.pg_diag); if self._offdiag: subgroups = [ ParmGroup.Subgroup(X+Y,[jones(p,zz) for zz in rxy,ixy for p in stations]), ParmGroup.Subgroup(Y+X,[jones(p,zz) for zz in ryx,iyx for p in stations]), ParmGroup.Subgroup("real part",[jones(p,zz) for zz in rxy,ryx for p in stations]), ParmGroup.Subgroup("imaginary part",[jones(p,zz) for zz in ixy,iyx for p in stations]) ]; subgroups += [ ParmGroup.Subgroup("station %s"%p,[jones(p,zz) for zz in rxy,ixy,ryx,iyx ]) for p in stations ]; self.pg_offdiag = ParmGroup.ParmGroup(label+"_offdiag",
def _define_forest(ns,parent=None,**kw): if run_purr: Timba.TDL.GUI.log_message("starting purr"); Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']); mssel.setup_observation_context(ns); array = Meow.Context.array; # make spigot nodes outputs = spigots = array.spigots(corr=mssel.get_corr_index()); # ...and an inspector for them Meow.StdTrees.vis_inspector(ns.inspector('input'),spigots, bookmark="Inspect input visibilities"); inspectors = [ ns.inspector('input') ]; Bookmarks.make_node_folder("Input visibilities by baseline", [ spigots(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2); # add in-tree flaggers, if appropriate if tree_flag_enable: flaggers = []; # make nodes to take complex amplitude, if any flaggers are going to use them if tree_flag_ampl is not None or tree_flag_mean_ampl is not None or tree_flag_mean_freq_ampl is not None: # use matrix norm if tree_flag_norm: for p,q in array.ifrs(): node = outputs(p,q); nsq = node('sq') << Meq.MatrixMultiply(node,node('conj') << Meq.ConjTranspose(node)); nsqtr = nsq('tr') << (nsq(11) << Meq.Selector(nsq,index=0)) + (nsq(22) << Meq.Selector(nsq,index=3)); ns.ampl(p,q) << Meq.Sqrt(nsqtr('abs') << Meq.Abs(nsqtr)); # use per-element amplitudes else: for p,q in array.ifrs(): ns.ampl(p,q) << Meq.Abs(outputs(p,q)); # ...and an inspector for them Meow.StdTrees.vis_inspector(ns.inspector('ampl'),ns.ampl,bookmark="Inspect mean-freq visibility amplitudes"); inspectors.append(ns.inspector('ampl')); # make flagger for amplitudes if tree_flag_ampl is not None: for p,q in array.ifrs(): ns.flagampl(p,q) << Meq.ZeroFlagger(ns.ampl(p,q)-tree_flag_ampl,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT); flaggers.append(ns.flagampl); # make flagger for mean-ifr amplitudes if tree_flag_mean_ampl is not None: ns.meanampl << Meq.Mean(*[ns.ampl(p,q) for p,q in array.ifrs()]); ns.flagmeanampl << Meq.ZeroFlagger(ns.meanampl-tree_flag_mean_ampl,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT); Meow.Bookmarks.Page("Mean-ifr visibility amplitudes").add(ns.meanampl,viewer="Result Plotter"); flaggers.append(lambda p,q:ns.flagmeanampl); # make flagger for mean-fr-freq amplitudes if tree_flag_mean_freq_ampl is not None: ns.meanampl ** Meq.Mean(*[ns.ampl(p,q) for p,q in array.ifrs()]); ns.meanfreqampl << Meq.Mean(ns.meanampl,reduction_axes="freq"); ns.flagmeanfreqampl << Meq.ZeroFlagger(ns.meanfreqampl-tree_flag_mean_freq_ampl,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT); Meow.Bookmarks.Page("Mean-ifr-freq visibility amplitudes").add(ns.meanfreqampl,viewer="Collections Plotter"); flaggers.append(lambda p,q:ns.flagmeanfreqampl); # now merge all flaggers if flaggers: for p,q in array.ifrs(): ns.flagged(p,q) << Meq.MergeFlags(outputs(p,q),*[fl(p,q) for fl in flaggers]); outputs = ns.flagged; Meow.StdTrees.vis_inspector(ns.inspector('flagged'),outputs, bookmark="Inspect flagged visibilities"); inspectors.append(ns.inspector('flagged')); Bookmarks.make_node_folder("Flagged visibilities by baseline", [ outputs(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2); mssel.enable_write_flags(True); treeflag_ms_opt.show(True); if outputs is spigots: ns.VisDataMux << Meq.VisDataMux(post=(inspectors[0] if len(inspectors)<2 else Meq.ReqMux(*inspectors))); else: Meow.StdTrees.make_sinks(ns,outputs,spigots=spigots,post=inspectors); # add imaging options imsel = mssel.imaging_selector(npix=512,arcmin=120); TDLRuntimeMenu("Make an image from this MS",*imsel.option_list());
def _define_forest(ns, parent=None, **kw): if not mssel.msname: raise RuntimeError("MS not set") if run_purr: Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.']) # create Purr pipe global purrpipe purrpipe = Purr.Pipe.Pipe(mssel.msname) # setup contexts from MS mssel.setup_observation_context(ns) array = Meow.Context.array # make spigot nodes for data mssel.enable_input_column(True) spigots = array.spigots(corr=mssel.get_corr_index()) meqmaker.make_per_ifr_bookmarks(spigots, "Input visibilities") # data tensor ns.DT << Meq.Composer( dims=[0], mt_polling=True, *[spigots(p, q) for p, q in array.ifrs()]) # list of model tensors models = [] if read_ms_model: mssel.enable_model_column(True) model_spigots = array.spigots(column="PREDICT", corr=mssel.get_corr_index()) meqmaker.make_per_ifr_bookmarks(model_spigots, "UV-model visibilities") mtuv = ns.MT("uv") << Meq.Composer( dims=[0], mt_polling=True, *[model_spigots(p, q) for p, q in array.ifrs()]) # predict tree using the MeqMaker all_sources = meqmaker.get_source_list(ns) dg_sources = deopts.enabled and dgsel.filter(all_sources) if dg_sources: # first group all sources without a diffgain on them groups = [[src for src in all_sources if not src in dg_sources]] # group diffgain-enabled sources by grouping tag clusters = set( [src.get_attr(diffgain_group, None) for src in dg_sources]) dg_groups = [(name, [ src for src in dg_sources if src.get_attr(diffgain_group) == name ]) for name in clusters if name] # add sources without a grouping tag individually, as single-source groups dg_groups += [(src.name, [src]) for src in dg_sources if not src.get_attr(diffgain_group, None)] # now sort by brightness flux_dgg = [(sum( [src.get_attr('Iapp', 0) or src.get_attr('I') for src in dgg[1]]), dgg) for dgg in dg_groups] from past.builtins import cmp from functools import cmp_to_key flux_dgg.sort(key=cmp_to_key(lambda a, b: cmp(b[0], a[0]))) diffgain_labels = [dgg[0] for flux, dgg in flux_dgg] groups += [dgg[1] for flux, dgg in flux_dgg] num_diffgains = len(flux_dgg) # now make predict trees for i, group in enumerate(groups): MT = ns.MT(group[0].name if i else "all") # first tensor is "MT", rest qualified by source names predict = meqmaker.make_predict_tree(MT.Subscope(), sources=group) MT << Meq.Composer(dims=[0], mt_polling=True, *[predict(p, q) for p, q in array.ifrs()]) # if reading an extra uv-model, add to first term if not i and read_ms_model: MT = ns.MT << Meq.Add(MT, mtuv) models.append(MT) print("Number of diffgain predict groups:", len(groups)) else: diffgain_labels = [] num_diffgains = 0 predict = meqmaker.make_predict_tree(ns) MT = ns.MT("all") << Meq.Composer( dims=[0], mt_polling=True, *[predict(p, q) for p, q in array.ifrs()]) if read_ms_model: MT = ns.MT << Meq.Add(MT, mtuv) models.append(MT) solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs() downsample_subtiling = [ stefcal_downsample_timeint, stefcal_downsample_freqint ] if stefcal_downsample else [1, 1] import Calico.OMS.StefCal.StefCal kwopts = {} gopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize) bopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize) deopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize) ns.stefcal << Meq.PyNode( class_name="StefCalNode", module_name=Calico.OMS.StefCal.StefCal.__file__, ifrs=["%s:%s" % (p, q) for p, q in array.ifrs()], baselines=[ array.baseline(ip, iq) for (ip, p), (iq, q) in array.ifr_index() ], solve_ifrs=["%s:%s" % (p, q) for p, q in solve_ifrs], noise_per_chan=stefcal_noise_per_chan, downsample_subtiling=downsample_subtiling, num_major_loops=stefcal_nmajor, regularization_factor=1e-6, # rescale=stefcal_rescale, init_from_previous=False, critical_flag_threshold=critical_flag_threshold, diffgain_labels=diffgain_labels, # flagging options output_flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT, # IFR gain solution options apply_ifr_gains=stefcal_ifr_gains, solve_ifr_gains=(stefcal_ifr_gain_mode != MODE_SOLVE_APPLY), reset_ifr_gains=stefcal_ifr_gain_reset, save_ifr_gains=(stefcal_ifr_gain_mode == MODE_SOLVE_SAVE), ifr_gain_table=stefcal_ifr_gain_table, per_chan_ifr_gains=stefcal_per_chan_ifr_gains, diag_ifr_gains=(stefcal_diagonal_ifr_gains == DIAGONLY), residuals=(do_output == CORRECTED_RESIDUALS), subtract_dgsrc=(do_output == CORRECTED_DATA_SUB), verbose=stefcal_verbose, children=[ns.DT] + models, **kwopts) inspectors = meqmaker.get_inspectors() or [] # make output bookmarks nv = 0 for p, q in array.ifrs(): sel = ns.output_sel(p, q) << Meq.Selector( ns.stefcal, index=list(range(nv, nv + 4)), multi=True) ns.output(p, q) << Meq.Composer(sel, dims=[2, 2]) nv += 4 meqmaker.make_per_ifr_bookmarks(ns.output, "Output visibilities") Bookmarks.Page("StefCal outputs").add(ns.stefcal, viewer="Record Browser") if gopts.enabled and gopts.visualize and stefcal_visualize: ns.stefcal_vis_G << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="G", flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) ns.stefcal_vis_G_avg << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="G", freq_average=True, flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) Bookmarks.Page("StefCal G plotter").add(ns.stefcal_vis_G, viewer="Result Plotter") Bookmarks.Page("StefCal G inspector").add(ns.stefcal_vis_G_avg, viewer="Collections Plotter") inspectors += [ns.stefcal_vis_G, ns.stefcal_vis_G_avg] if bopts.enabled and bopts.visualize and stefcal_visualize: ns.stefcal_vis_B << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="B", flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) ns.stefcal_vis_B_avg << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="B", freq_average=True, flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) Bookmarks.Page("StefCal B plotter").add(ns.stefcal_vis_B, viewer="Result Plotter") Bookmarks.Page("StefCal B inspector").add(ns.stefcal_vis_B_avg, viewer="Collections Plotter") inspectors += [ns.stefcal_vis_B, ns.stefcal_vis_B_avg] if deopts.enabled and deopts.visualize and stefcal_visualize: for i, label in enumerate(diffgain_labels): vde = ns.stefcal_vis_dE(label) << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="dE:%s" % label, flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) vde_avg = ns.stefcal_vis_dE_avg(label) << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="dE:%s" % label, freq_average=True, flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) Bookmarks.Page("StefCal dE:%s plotter" % label).add( vde, viewer="Result Plotter") Bookmarks.Page("StefCal dE:%s inspector" % label).add( vde_avg, viewer="Collections Plotter") inspectors += [vde, vde_avg] # make sinks StdTrees.make_sinks(ns, ns.output, spigots=spigots, post=inspectors, corr_index=mssel.get_corr_index()) # this should come last, since runtime options may be built up during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)) # finally, setup imaging options imsel = mssel.imaging_selector(npix=512, arcmin=meqmaker.estimate_image_size()) TDLRuntimeMenu("Make an image from this MS", *imsel.option_list()) # and close meqmaker -- this exports annotations, etc meqmaker.close() # add options to clear all solutions from Calico.OMS.StefCal import StefCal TDLRuntimeOption("stefcal_reset_all", "Remove all existing solutions", False) for opt in gopts, bopts, deopts: if opt.enabled: TDLRuntimeOption("reset", "Remove existing %s solutions (%s)" % (opt.label, os.path.basename(opt.table)), False, namespace=opt) if stefcal_ifr_gains: TDLRuntimeOption( "stefcal_reset_ifr_gains", "Remove existing interferometer errors (%s)" % (os.path.basename(stefcal_ifr_gain_table)), False) TDLRuntimeJob(_run_stefcal, "Run StefCal", job_id="stefcal")