def _define_forest (ns): # create Array object num_antennas = 36 # for ASKAP simulation xntd_list = [ str(i) for i in range(1,num_antennas+1) ]; array = Meow.IfrArray(ns,xntd_list,ms_uvw=True); # create an Observation object observation = Meow.Observation(ns); # set global context Meow.Context.set(array=array,observation=observation); # create a source model and make list of corrupted sources allsky = Meow.Patch(ns,'all',observation.phase_centre); sources = sky_models.make_model(ns,"S"); for src in sources: lm = src.direction.lm(); E = ns.E(src.name); for p in array.stations(): pa= ns.ParAngle(p) << Meq.ParAngle(observation.phase_centre.radec(), array.xyz(p)) ns.CosPa(p) << Meq.Cos(pa) ns.SinPa(p) << Meq.Sin(pa) ns.rot_matrix(p) << Meq.Matrix22(ns.CosPa(p),-1.0 * ns.SinPa(p),ns.SinPa(p),ns.CosPa(p)) # compute "apparent" position of source per each antenna lm_rot=ns.lm_rot(src.name,p) << Meq.MatrixMultiply(ns.rot_matrix(p),lm) # compute E for apparent position tdp_voltage_response(ns,src,p,E(p),lm_rot); allsky.add(src.corrupt(E)); observed = allsky.visibilities(); # make some useful inspectors. Collect them into a list, since we need # to give a list of 'post' nodes to make_sinks() below pg = Bookmarks.Page("Inspectors",1,2); inspectors = []; inspectors.append( Meow.StdTrees.vis_inspector(ns.inspect_observed,observed) ); pg.add(ns.inspect_observed,viewer="Collections Plotter"); Meow.StdTrees.make_sinks(ns,observed,spigots=False,post=inspectors);
def _define_forest(ns, **kw): if parmtab is None: raise RuntimeError, "please select a valid parmtable in compile-time options" # make one MeqParm per each funklet parmdef = Meq.Parm(table_name=parmtable_name) parmnodes = dict([(name, (ns[name] << parmdef)) for name in parmtab.funklet_names()]) # reprocess flag is reset every time something changes in the parmnodes dict, # so we simply repeat this loop until things stop changing reprocess = True while reprocess: reprocess = False # see if we have recipe for making compound objects based on last funklet name qualifier nqlist = [ name.rsplit(':', 1) for name in parmnodes.iterkeys() if ':' in name ] # set of basenames and set of all qualifiers basenames = set([nq[0] for nq in nqlist]) quals = set([nq[1] for nq in nqlist]) for quallist, agg_func in aggregators: # look for an aggregator whose qualifiers are a subset of the given set if set(quallist) <= quals: # for each basename, attempt to find funklets for every qualifier in the list for basename in basenames: fqnames = [basename + ':' + qq for qq in quallist] funks = [parmnodes.get(name, None) for name in fqnames] # if every basename:qq combination gives a valid funklet, call the aggregator if None not in funks: aggnode = ns[basename] << agg_func(*funks) # remove aggregated funklets from the parmnodes dict for name in fqnames: del parmnodes[name] # add new node to dict parmnodes[basename] = aggnode reprocess = True # sort nodes by qualified name nodes = list(parmnodes.iteritems()) nodes.sort(lambda a, b: ParmTables.cmp_qualified_names(a[0], b[0])) nodes = [namenode[1] for namenode in nodes] # make intermediate math node pall = ns.all_parms_raw << Meq.Composer( children=nodes, mt_polling=True, dims=[0]) pfm = ns.all_parms_fm << Meq.Mean(pall, reduction_axes="freq") # ns.all_minus_fm << pall - pfm; # ns.all_over_fm << pall/pfm; # make root node ns.root << Meq.ReqMux( pfm # ns.all_minus_fm, # ns.all_over_fm, # ns.fft_all_over_fm << Meq.FFTBrick(Meq.Selector(ns.all_over_fm,index=0,multi=True),axes_in=(hiid("time"),hiid("freq")),axes_out=(hiid("l"),hiid("m"))) ) Bookmarks.Page("Composite view").add(ns.all_parms_fm, viewer="Collections Plotter") # make bookmark folder Bookmarks.make_node_folder("All parameters", nodes, sorted=True) # close parmtable, else meqserver will be unable to get at it parmtab.close() # now make runtime options global active_axes active_axes = [] reg_domain_opts = [] for iaxis in range(mequtils.max_axis): stats = parmtab.axis_stats(iaxis) if not stats.empty(): active_axes.append((iaxis, stats.name, stats)) reg_domain_opts.append( TDLOption("num_cells_%s" % stats.name, "Number of grid points in %s" % stats.name, [len(stats.cells)], more=int)) TDLRuntimeMenu( "View parameters over regularly gridded domain", *(reg_domain_opts + [ TDLJob(_view_parameters_over_regularized_domain, "View over gridded domain") ]))
def _define_forest(ns, parent=None, **kw): if not mssel.msname: raise RuntimeError("MS not set") if run_purr: Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.']) # create Purr pipe global purrpipe purrpipe = Purr.Pipe.Pipe(mssel.msname) # setup contexts from MS mssel.setup_observation_context(ns) array = Meow.Context.array # make spigot nodes for data mssel.enable_input_column(True) spigots = array.spigots(corr=mssel.get_corr_index()) meqmaker.make_per_ifr_bookmarks(spigots, "Input visibilities") # data tensor ns.DT << Meq.Composer( dims=[0], mt_polling=True, *[spigots(p, q) for p, q in array.ifrs()]) # list of model tensors models = [] if read_ms_model: mssel.enable_model_column(True) model_spigots = array.spigots(column="PREDICT", corr=mssel.get_corr_index()) meqmaker.make_per_ifr_bookmarks(model_spigots, "UV-model visibilities") mtuv = ns.MT("uv") << Meq.Composer( dims=[0], mt_polling=True, *[model_spigots(p, q) for p, q in array.ifrs()]) # predict tree using the MeqMaker all_sources = meqmaker.get_source_list(ns) dg_sources = deopts.enabled and dgsel.filter(all_sources) if dg_sources: # first group all sources without a diffgain on them groups = [[src for src in all_sources if not src in dg_sources]] # group diffgain-enabled sources by grouping tag clusters = set( [src.get_attr(diffgain_group, None) for src in dg_sources]) dg_groups = [(name, [ src for src in dg_sources if src.get_attr(diffgain_group) == name ]) for name in clusters if name] # add sources without a grouping tag individually, as single-source groups dg_groups += [(src.name, [src]) for src in dg_sources if not src.get_attr(diffgain_group, None)] # now sort by brightness flux_dgg = [(sum( [src.get_attr('Iapp', 0) or src.get_attr('I') for src in dgg[1]]), dgg) for dgg in dg_groups] from past.builtins import cmp from functools import cmp_to_key flux_dgg.sort(key=cmp_to_key(lambda a, b: cmp(b[0], a[0]))) diffgain_labels = [dgg[0] for flux, dgg in flux_dgg] groups += [dgg[1] for flux, dgg in flux_dgg] num_diffgains = len(flux_dgg) # now make predict trees for i, group in enumerate(groups): MT = ns.MT(group[0].name if i else "all") # first tensor is "MT", rest qualified by source names predict = meqmaker.make_predict_tree(MT.Subscope(), sources=group) MT << Meq.Composer(dims=[0], mt_polling=True, *[predict(p, q) for p, q in array.ifrs()]) # if reading an extra uv-model, add to first term if not i and read_ms_model: MT = ns.MT << Meq.Add(MT, mtuv) models.append(MT) print("Number of diffgain predict groups:", len(groups)) else: diffgain_labels = [] num_diffgains = 0 predict = meqmaker.make_predict_tree(ns) MT = ns.MT("all") << Meq.Composer( dims=[0], mt_polling=True, *[predict(p, q) for p, q in array.ifrs()]) if read_ms_model: MT = ns.MT << Meq.Add(MT, mtuv) models.append(MT) solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs() downsample_subtiling = [ stefcal_downsample_timeint, stefcal_downsample_freqint ] if stefcal_downsample else [1, 1] import Calico.OMS.StefCal.StefCal kwopts = {} gopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize) bopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize) deopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize) ns.stefcal << Meq.PyNode( class_name="StefCalNode", module_name=Calico.OMS.StefCal.StefCal.__file__, ifrs=["%s:%s" % (p, q) for p, q in array.ifrs()], baselines=[ array.baseline(ip, iq) for (ip, p), (iq, q) in array.ifr_index() ], solve_ifrs=["%s:%s" % (p, q) for p, q in solve_ifrs], noise_per_chan=stefcal_noise_per_chan, downsample_subtiling=downsample_subtiling, num_major_loops=stefcal_nmajor, regularization_factor=1e-6, # rescale=stefcal_rescale, init_from_previous=False, critical_flag_threshold=critical_flag_threshold, diffgain_labels=diffgain_labels, # flagging options output_flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT, # IFR gain solution options apply_ifr_gains=stefcal_ifr_gains, solve_ifr_gains=(stefcal_ifr_gain_mode != MODE_SOLVE_APPLY), reset_ifr_gains=stefcal_ifr_gain_reset, save_ifr_gains=(stefcal_ifr_gain_mode == MODE_SOLVE_SAVE), ifr_gain_table=stefcal_ifr_gain_table, per_chan_ifr_gains=stefcal_per_chan_ifr_gains, diag_ifr_gains=(stefcal_diagonal_ifr_gains == DIAGONLY), residuals=(do_output == CORRECTED_RESIDUALS), subtract_dgsrc=(do_output == CORRECTED_DATA_SUB), verbose=stefcal_verbose, children=[ns.DT] + models, **kwopts) inspectors = meqmaker.get_inspectors() or [] # make output bookmarks nv = 0 for p, q in array.ifrs(): sel = ns.output_sel(p, q) << Meq.Selector( ns.stefcal, index=list(range(nv, nv + 4)), multi=True) ns.output(p, q) << Meq.Composer(sel, dims=[2, 2]) nv += 4 meqmaker.make_per_ifr_bookmarks(ns.output, "Output visibilities") Bookmarks.Page("StefCal outputs").add(ns.stefcal, viewer="Record Browser") if gopts.enabled and gopts.visualize and stefcal_visualize: ns.stefcal_vis_G << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="G", flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) ns.stefcal_vis_G_avg << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="G", freq_average=True, flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) Bookmarks.Page("StefCal G plotter").add(ns.stefcal_vis_G, viewer="Result Plotter") Bookmarks.Page("StefCal G inspector").add(ns.stefcal_vis_G_avg, viewer="Collections Plotter") inspectors += [ns.stefcal_vis_G, ns.stefcal_vis_G_avg] if bopts.enabled and bopts.visualize and stefcal_visualize: ns.stefcal_vis_B << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="B", flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) ns.stefcal_vis_B_avg << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="B", freq_average=True, flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) Bookmarks.Page("StefCal B plotter").add(ns.stefcal_vis_B, viewer="Result Plotter") Bookmarks.Page("StefCal B inspector").add(ns.stefcal_vis_B_avg, viewer="Collections Plotter") inspectors += [ns.stefcal_vis_B, ns.stefcal_vis_B_avg] if deopts.enabled and deopts.visualize and stefcal_visualize: for i, label in enumerate(diffgain_labels): vde = ns.stefcal_vis_dE(label) << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="dE:%s" % label, flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) vde_avg = ns.stefcal_vis_dE_avg(label) << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="dE:%s" % label, freq_average=True, flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) Bookmarks.Page("StefCal dE:%s plotter" % label).add( vde, viewer="Result Plotter") Bookmarks.Page("StefCal dE:%s inspector" % label).add( vde_avg, viewer="Collections Plotter") inspectors += [vde, vde_avg] # make sinks StdTrees.make_sinks(ns, ns.output, spigots=spigots, post=inspectors, corr_index=mssel.get_corr_index()) # this should come last, since runtime options may be built up during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)) # finally, setup imaging options imsel = mssel.imaging_selector(npix=512, arcmin=meqmaker.estimate_image_size()) TDLRuntimeMenu("Make an image from this MS", *imsel.option_list()) # and close meqmaker -- this exports annotations, etc meqmaker.close() # add options to clear all solutions from Calico.OMS.StefCal import StefCal TDLRuntimeOption("stefcal_reset_all", "Remove all existing solutions", False) for opt in gopts, bopts, deopts: if opt.enabled: TDLRuntimeOption("reset", "Remove existing %s solutions (%s)" % (opt.label, os.path.basename(opt.table)), False, namespace=opt) if stefcal_ifr_gains: TDLRuntimeOption( "stefcal_reset_ifr_gains", "Remove existing interferometer errors (%s)" % (os.path.basename(stefcal_ifr_gain_table)), False) TDLRuntimeJob(_run_stefcal, "Run StefCal", job_id="stefcal")
def _define_forest(ns, parent=None, **kw): if not mssel.msname: raise RuntimeError('MS name not set') mssel.setup_observation_context(ns) array = Context.array # Data and model input if do_solve or output_type.need_data: mssel.enable_input_column(True) spigots = array.spigots(corr=mssel.get_corr_index()) meqmaker.make_per_ifr_bookmarks(spigots, 'Input visibilities') else: mssel.enable_input_column(False) spigots = None if do_solve or output_type.need_model: predict = meqmaker.make_predict_tree(ns, uvdata=None) else: predict = None # Data output outputs = output_type.apply(ns, meqmaker, array.ifrs(), spigots, predict) # Flagging if flag_enable and output_type.flag_data: flaggers = [] if flag_res is not None or flag_mean_res is not None: for p, q in array.ifrs(): ns.absres(p, q) << Meq.Abs(outputs(p, q)) if flag_res is not None: for p, q in array.ifrs(): ns.flagres(p, q) << Meq.ZeroFlagger( ns.absres(p, q) - flag_res, oper='gt', flag_bit=MSUtils.FLAGMASK_OUTPUT) flaggers.append(ns.flagres) meqmaker.make_per_ifr_bookmarks(ns.flagres, 'Residual amplitude flags') if flag_mean_res is not None: ns.meanabsres << Meq.Mean( *[ns.absres(p, q) for p, q in array.ifrs()]) ns.flagmeanres << Meq.ZeroFlagger(ns.meanabsres - flag_mean_res, oper='gt', flag_bit=MSUtils.FLAGMASK_OUTPUT) Bookmarks.Page('Mean residual amplitude flags').add( ns.flagmeanres, viewer='Result Plotter') flaggers.append(lambda p, q: ns.flagmeanres) if flaggers: meqmaker.make_per_ifr_bookmarks(outputs, output_type.desc + ' (unflagged)') for p, q in array.ifrs(): ns.flagged(p, q) << Meq.MergeFlags( outputs(p, q), *[f(p, q) for f in flaggers]) outputs = ns.flagged meqmaker.make_per_ifr_bookmarks(outputs, output_type.desc) # Solve trees if do_solve: # parse ifr specification solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs() if not solve_ifrs: raise RuntimeError( 'No interferometers selected for calibration. ' 'Check your ifr specification (under calibration options).') lhs, rhs, weights, modulo = cal_quant.apply(solve_ifrs, predict, spigots) solve_tree = StdTrees.SolveTree(ns, lhs, solve_ifrs=solve_ifrs, weights=weights, modulo=modulo) outputs = solve_tree.sequencers(inputs=rhs, outputs=outputs) StdTrees.make_sinks(ns, outputs, spigots=spigots, post=meqmaker.get_inspectors() or [], corr_index=mssel.get_corr_index()) if not do_solve: name = 'Generate ' + output_type.desc.lower() comment = 'Generated ' + output_type.desc.lower() def run_tree(mqs, parent, wait=False, **kw): return mqs.execute(Context.vdm.name, mssel.create_io_request(tile_size), wait=wait) doc = """Input data are sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory.""" TDLRuntimeMenu( name, TDLOption('tile_size', 'Tile size, in timeslots', [10, 60, 120, 240], more=int, doc=doc), TDLJob(run_tree, name, job_id='generate_visibilities')) # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up # during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)) if do_solve: TDLRuntimeOptions(*ParmGroup.get_solvejob_options()) imsel = mssel.imaging_selector(npix=512, arcmin=meqmaker.estimate_image_size()) TDLRuntimeMenu('Make an image', *imsel.option_list()) meqmaker.close()