def _define_forest(ns, parent=None, **kw): if run_purr: Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.']) # create Purr pipe global purrpipe purrpipe = Purr.Pipe.Pipe(mssel.msname) # get antennas from MS ANTENNAS = mssel.get_antenna_set(list(range(1, 15))) array = Meow.IfrArray(ns, ANTENNAS, mirror_uvw=False) stas = array.stations() # get phase centre from MS, setup observation observation = Meow.Observation(ns, phase_centre=mssel.get_phase_dir(), linear=mssel.is_linear_pol(), circular=mssel.is_circular_pol()) Meow.Context.set(array, observation) # get active correlations from MS Meow.Context.active_correlations = mssel.get_correlations() # make spigot nodes spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index()) # ...and an inspector for them StdTrees.vis_inspector(ns.inspector('input'), spigots, bookmark="Inspect input visibilities") inspectors = [ns.inspector('input')] Bookmarks.make_node_folder("Input visibilities by baseline", [spigots(p, q) for p, q in array.ifrs()], sorted=True, ncol=2, nrow=2) inspect_ifrs = array.ifrs() if do_solve: # filter solvable baselines by baseline length solve_ifrs = [] antpos = mssel.ms_antenna_positions if (min_baseline or max_baseline) and antpos is not None: for (ip, p), (iq, q) in array.ifr_index(): baseline = math.sqrt( ((antpos[ip, :] - antpos[iq, :])**2).sum()) if (not min_baseline or baseline > min_baseline) and \ (not max_baseline or baseline < max_baseline): solve_ifrs.append((p, q)) else: solve_ifrs = array.ifrs() inspect_ifrs = solve_ifrs # make a predict tree using the MeqMaker if do_solve or do_subtract: predict = meqmaker.make_predict_tree(ns) # make a ParmGroup and solve jobs for source parameters, if we have any if do_solve: parms = {} for src in meqmaker.get_source_list(ns): parms.update([(p.name, p) for p in src.get_solvables()]) if parms: pg_src = ParmGroup.ParmGroup("source", list(parms.values()), table_name="sources.fmep", individual=True, bookmark=True) # now make a solvejobs for the source ParmGroup.SolveJob("cal_source", "Calibrate source model", pg_src) # make nodes to compute residuals if do_subtract: residuals = ns.residuals for p, q in array.ifrs(): residuals(p, q) << spigots(p, q) - predict(p, q) outputs = residuals # and now we may need to correct the outputs if do_correct: if do_correct_sky: srcs = meqmaker.get_source_list(ns) sky_correct = srcs and srcs[0] else: sky_correct = None outputs = meqmaker.correct_uv_data(ns, outputs, sky_correct=sky_correct, inspect_ifrs=inspect_ifrs) # make solve trees if do_solve: # inputs to the solver are based on calibration type # if calibrating visibilities, feed them to condeq directly if cal_type == CAL.VIS: observed = spigots model = predict # else take ampl/phase component else: model = ns.model observed = ns.observed if cal_type == CAL.AMPL: for p, q in array.ifrs(): observed(p, q) << Meq.Abs(spigots(p, q)) model(p, q) << Meq.Abs(predict(p, q)) elif cal_type == CAL.LOGAMPL: for p, q in array.ifrs(): observed(p, q) << Meq.Log(Meq.Abs(spigots(p, q))) model(p, q) << Meq.Log(Meq.Abs(predict(p, q))) elif cal_type == CAL.PHASE: for p, q in array.ifrs(): observed(p, q) << 0 model(p, q) << Meq.Abs(predict(p, q)) * Meq.FMod( Meq.Arg(spigots(p, q)) - Meq.Arg(predict(p, q)), 2 * math.pi) else: raise ValueError("unknown cal_type setting: " + str(cal_type)) # make a solve tree solve_tree = StdTrees.SolveTree(ns, model, solve_ifrs=solve_ifrs) # the output of the sequencer is either the residuals or the spigots, # according to what has been set above outputs = solve_tree.sequencers(inputs=observed, outputs=outputs) # make sinks and vdm. # The list of inspectors must be supplied here inspectors += meqmaker.get_inspectors() or [] StdTrees.make_sinks(ns, outputs, spigots=spigots0, post=inspectors) Bookmarks.make_node_folder("Corrected/residual visibilities by baseline", [outputs(p, q) for p, q in array.ifrs()], sorted=True, ncol=2, nrow=2) if not do_solve: if do_subtract: name = "Generate residuals" comment = "Generated residual visibilities." elif do_correct: name = "Generate corrected data" comment = "Generated corrected visibilities." else: name = None if name: # make a TDL job to runsthe tree def run_tree(mqs, parent, **kw): global tile_size purrpipe.title("Calibrating").comment(comment) mqs.execute(Meow.Context.vdm.name, mssel.create_io_request(tile_size), wait=False) TDLRuntimeMenu( name, TDLOption( 'tile_size', "Tile size, in timeslots", [10, 60, 120, 240], more=int, doc= """Input data is sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory.""" ), TDLRuntimeJob(run_tree, name)) # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)) # insert solvejobs if do_solve: TDLRuntimeOptions(*ParmGroup.get_solvejob_options()) # finally, setup imaging options imsel = mssel.imaging_selector(npix=512, arcmin=meqmaker.estimate_image_size()) TDLRuntimeMenu("Make an image from this MS", *imsel.option_list()) # and close meqmaker -- this exports annotations, etc meqmaker.close()
def _define_forest(ns,parent=None,**kw): if run_purr: Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']); # create Purr pipe global purrpipe; purrpipe = Purr.Pipe.Pipe(mssel.msname); # get antennas from MS ANTENNAS = mssel.get_antenna_set(list(range(1,15))); array = Meow.IfrArray(ns,ANTENNAS,mirror_uvw=False); stas = array.stations(); # get phase centre from MS, setup observation observation = Meow.Observation(ns,phase_centre=mssel.get_phase_dir(), linear=mssel.is_linear_pol(), circular=mssel.is_circular_pol()); Meow.Context.set(array,observation); # get active correlations from MS Meow.Context.active_correlations = mssel.get_correlations(); # make spigot nodes spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index()); # ...and an inspector for them StdTrees.vis_inspector(ns.inspector('input'),spigots, bookmark="Inspect input visibilities"); inspectors = [ ns.inspector('input') ]; Bookmarks.make_node_folder("Input visibilities by baseline", [ spigots(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2); inspect_ifrs = array.ifrs(); if do_solve: # filter solvable baselines by baseline length solve_ifrs = []; antpos = mssel.ms_antenna_positions; if (min_baseline or max_baseline) and antpos is not None: for (ip,p),(iq,q) in array.ifr_index(): baseline = math.sqrt(((antpos[ip,:]-antpos[iq,:])**2).sum()); if (not min_baseline or baseline > min_baseline) and \ (not max_baseline or baseline < max_baseline): solve_ifrs.append((p,q)); else: solve_ifrs = array.ifrs(); inspect_ifrs = solve_ifrs; # make a predict tree using the MeqMaker if do_solve or do_subtract: predict = meqmaker.make_predict_tree(ns); # make a ParmGroup and solve jobs for source parameters, if we have any if do_solve: parms = {}; for src in meqmaker.get_source_list(ns): parms.update([(p.name,p) for p in src.get_solvables()]); if parms: pg_src = ParmGroup.ParmGroup("source",list(parms.values()), table_name="sources.fmep", individual=True,bookmark=True); # now make a solvejobs for the source ParmGroup.SolveJob("cal_source","Calibrate source model",pg_src); # make nodes to compute residuals if do_subtract: residuals = ns.residuals; for p,q in array.ifrs(): residuals(p,q) << spigots(p,q) - predict(p,q); outputs = residuals; # and now we may need to correct the outputs if do_correct: if do_correct_sky: srcs = meqmaker.get_source_list(ns); sky_correct = srcs and srcs[0]; else: sky_correct = None; outputs = meqmaker.correct_uv_data(ns,outputs,sky_correct=sky_correct,inspect_ifrs=inspect_ifrs); # make solve trees if do_solve: # inputs to the solver are based on calibration type # if calibrating visibilities, feed them to condeq directly if cal_type == CAL.VIS: observed = spigots; model = predict; # else take ampl/phase component else: model = ns.model; observed = ns.observed; if cal_type == CAL.AMPL: for p,q in array.ifrs(): observed(p,q) << Meq.Abs(spigots(p,q)); model(p,q) << Meq.Abs(predict(p,q)); elif cal_type == CAL.LOGAMPL: for p,q in array.ifrs(): observed(p,q) << Meq.Log(Meq.Abs(spigots(p,q))); model(p,q) << Meq.Log(Meq.Abs(predict(p,q))); elif cal_type == CAL.PHASE: for p,q in array.ifrs(): observed(p,q) << 0; model(p,q) << Meq.Abs(predict(p,q))*Meq.FMod(Meq.Arg(spigots(p,q))-Meq.Arg(predict(p,q)),2*math.pi); else: raise ValueError("unknown cal_type setting: "+str(cal_type)); # make a solve tree solve_tree = StdTrees.SolveTree(ns,model,solve_ifrs=solve_ifrs); # the output of the sequencer is either the residuals or the spigots, # according to what has been set above outputs = solve_tree.sequencers(inputs=observed,outputs=outputs); # make sinks and vdm. # The list of inspectors must be supplied here inspectors += meqmaker.get_inspectors() or []; StdTrees.make_sinks(ns,outputs,spigots=spigots0,post=inspectors); Bookmarks.make_node_folder("Corrected/residual visibilities by baseline", [ outputs(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2); if not do_solve: if do_subtract: name = "Generate residuals"; comment = "Generated residual visibilities."; elif do_correct: name = "Generate corrected data"; comment = "Generated corrected visibilities."; else: name = None; if name: # make a TDL job to runsthe tree def run_tree (mqs,parent,**kw): global tile_size; purrpipe.title("Calibrating").comment(comment); mqs.execute(Meow.Context.vdm.name,mssel.create_io_request(tile_size),wait=False); TDLRuntimeMenu(name, TDLOption('tile_size',"Tile size, in timeslots",[10,60,120,240],more=int, doc="""Input data is sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory."""), TDLRuntimeJob(run_tree,name) ); # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)); # insert solvejobs if do_solve: TDLRuntimeOptions(*ParmGroup.get_solvejob_options()); # finally, setup imaging options imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size()); TDLRuntimeMenu("Make an image from this MS",*imsel.option_list()); # and close meqmaker -- this exports annotations, etc meqmaker.close();
def _define_forest(ns): # make pynodes, xyzcomponent for sources ANTENNAS = mssel.get_antenna_set(list(range(1, 15))) array = Meow.IfrArray(ns, ANTENNAS, mirror_uvw=False) observation = Meow.Observation(ns) Meow.Context.set(array, observation) # make a predict tree using the MeqMaker if do_solve or do_subtract or not do_not_simulate: outputs = predict = meqmaker.make_tree(ns) # make a list of selected corrs selected_corrs = cal_corr.split(" ") # make spigot nodes if not do_not_simulate and do_add: spigots = spigots0 = outputs = array.spigots() sums = ns.sums for p, q in array.ifrs(): sums(p, q) << spigots(p, q) + predict(p, q) outputs = sums # make spigot nodes if do_not_simulate: spigots = spigots0 = outputs = array.spigots() # make nodes to compute residuals # make nodes to compute residuals if do_subtract: residuals = ns.residuals for p, q in array.ifrs(): residuals(p, q) << spigots(p, q) - predict(p, q) outputs = residuals # and now we may need to correct the outputs if do_correct: if do_correct_sky: if src_name: sky_correct = src_name else: srcs = meqmaker.get_source_list(ns) sky_correct = srcs and srcs[0] else: sky_correct = None outputs = meqmaker.correct_uv_data(ns, outputs, sky_correct=sky_correct) # make solve trees if do_solve: # extract selected correlations if cal_corr != ALL_CORRS: index = [CORR_INDICES[c] for c in selected_corrs] for p, q in array.ifrs(): ns.sel_predict(p, q) << Meq.Selector( predict(p, q), index=index, multi=True) ns.sel_spigot(p, q) << Meq.Selector( spigots(p, q), index=index, multi=True) spigots = ns.sel_spigot predict = ns.sel_predict model = predict observed = spigots # make a solve tree solve_tree = Meow.StdTrees.SolveTree(ns, model) # the output of the sequencer is either the residuals or the spigots, # according to what has been set above outputs = solve_tree.sequencers(inputs=observed, outputs=outputs) # throw in a bit of noise if not do_not_simulate and noise_stddev: # make two complex noise terms per station (x/y) noisedef = Meq.GaussNoise(stddev=noise_stddev) noise_x = ns.sta_noise('x') noise_y = ns.sta_noise('y') for p in array.stations(): noise_x(p) << Meq.ToComplex(noisedef, noisedef) noise_y(p) << Meq.ToComplex(noisedef, noisedef) # now combine them into per-baseline noise matrices for p, q in array.ifrs(): noise = ns.noise(p, q) << Meq.Matrix22( noise_x(p) + noise_x(q), noise_x(p) + noise_y(q), noise_y(p) + noise_x(q), noise_y(p) + noise_y(q)) ns.noisy_predict(p, q) << outputs(p, q) + noise outputs = ns.noisy_predict # make sinks and vdm. # The list of inspectors comes in handy here Meow.StdTrees.make_sinks(ns, outputs, spigots=None, post=meqmaker.get_inspectors()) if not do_not_simulate: # add simulate job TDLRuntimeJob(job_simulate, "Simulate") if do_not_simulate and not do_solve: # add subtract or correct job TDLRuntimeJob(job_subtract, "Subtract or Correct the data") if do_not_simulate and do_solve: pg_iono = ParmGroup.ParmGroup("Z_iono", outputs.search(tags="solvable Z"), table_name="iono.mep", bookmark=4) ParmGroup.SolveJob("cal_iono", "Calibrate Ionosphere parameters ", pg_iono) # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up during compilation. # TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)); # and insert all solvejobs TDLRuntimeOptions(*ParmGroup.get_solvejob_options()) # finally, setup imaging options imsel = mssel.imaging_selector(npix=512, arcmin=meqmaker.estimate_image_size()) TDLRuntimeMenu("Imaging options", *imsel.option_list())
def _define_forest(ns): #make pynodes, xyzcomponent for sources ANTENNAS = mssel.get_antenna_set(range(1,15)); array = Meow.IfrArray(ns,ANTENNAS,mirror_uvw=False); observation = Meow.Observation(ns); Meow.Context.set(array,observation); # make a predict tree using the MeqMaker if do_solve or do_subtract or not do_not_simulate: outputs=predict = meqmaker.make_tree(ns); #make a list of selected corrs selected_corrs = cal_corr.split(" "); # make spigot nodes if not do_not_simulate and do_add: spigots = spigots0 = outputs = array.spigots(); sums = ns.sums; for p,q in array.ifrs(): sums(p,q) << spigots(p,q) + predict(p,q); outputs = sums; # make spigot nodes if do_not_simulate: spigots = spigots0 = outputs = array.spigots(); # make nodes to compute residuals # make nodes to compute residuals if do_subtract: residuals = ns.residuals; for p,q in array.ifrs(): residuals(p,q) << spigots(p,q) - predict(p,q); outputs = residuals; # and now we may need to correct the outputs if do_correct: if do_correct_sky: if src_name: sky_correct = src_name; else: srcs = meqmaker.get_source_list(ns); sky_correct = srcs and srcs[0]; else: sky_correct = None; outputs = meqmaker.correct_uv_data(ns,outputs,sky_correct=sky_correct); # make solve trees if do_solve: # extract selected correlations if cal_corr != ALL_CORRS: index = [ CORR_INDICES[c] for c in selected_corrs ]; for p,q in array.ifrs(): ns.sel_predict(p,q) << Meq.Selector(predict(p,q),index=index,multi=True); ns.sel_spigot(p,q) << Meq.Selector(spigots(p,q),index=index,multi=True); spigots = ns.sel_spigot; predict = ns.sel_predict; model = predict; observed = spigots; # make a solve tree solve_tree = Meow.StdTrees.SolveTree(ns,model); # the output of the sequencer is either the residuals or the spigots, # according to what has been set above outputs = solve_tree.sequencers(inputs=observed,outputs=outputs); # throw in a bit of noise if not do_not_simulate and noise_stddev: # make two complex noise terms per station (x/y) noisedef = Meq.GaussNoise(stddev=noise_stddev) noise_x = ns.sta_noise('x'); noise_y = ns.sta_noise('y'); for p in array.stations(): noise_x(p) << Meq.ToComplex(noisedef,noisedef); noise_y(p) << Meq.ToComplex(noisedef,noisedef); # now combine them into per-baseline noise matrices for p,q in array.ifrs(): noise = ns.noise(p,q) << Meq.Matrix22( noise_x(p)+noise_x(q),noise_x(p)+noise_y(q), noise_y(p)+noise_x(q),noise_y(p)+noise_y(q) ); ns.noisy_predict(p,q) << outputs(p,q) + noise; outputs = ns.noisy_predict; # make sinks and vdm. # The list of inspectors comes in handy here Meow.StdTrees.make_sinks(ns,outputs,spigots=None,post=meqmaker.get_inspectors()); if not do_not_simulate: #add simulate job TDLRuntimeJob(job_simulate,"Simulate"); if do_not_simulate and not do_solve: #add subtract or correct job TDLRuntimeJob(job_subtract,"Subtract or Correct the data"); if do_not_simulate and do_solve: pg_iono = ParmGroup.ParmGroup("Z_iono", outputs.search(tags="solvable Z"), table_name="iono.mep",bookmark=4); ParmGroup.SolveJob("cal_iono","Calibrate Ionosphere parameters ",pg_iono); # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up during compilation. #TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)); # and insert all solvejobs TDLRuntimeOptions(*ParmGroup.get_solvejob_options()); # finally, setup imaging options imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size()); TDLRuntimeMenu("Imaging options",*imsel.option_list());
def _define_forest(ns, parent=None, **kw): if not mssel.msname: raise RuntimeError("MS not set") if run_purr: Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.']) # create Purr pipe global purrpipe purrpipe = Purr.Pipe.Pipe(mssel.msname) # setup contexts from MS mssel.setup_observation_context(ns, prefer_baseline_uvw=True) array = Meow.Context.array # make spigot nodes for data if do_solve or do_output not in [CORRUPTED_MODEL]: mssel.enable_input_column(True) spigots = spigots0 = outputs = array.spigots( corr=mssel.get_corr_index()) if enable_inspectors: meqmaker.make_per_ifr_bookmarks(spigots, "Input visibilities") # add IFR-based errors, if any spigots = meqmaker.apply_visibility_processing(ns, spigots) else: mssel.enable_input_column(False) spigots = spigots0 = None # make spigot nodes for model corrupt_uvdata = model_spigots = None if read_ms_model: mssel.enable_model_column(True) model_spigots = array.spigots(column="PREDICT", corr=mssel.get_corr_index()) if enable_inspectors: meqmaker.make_per_ifr_bookmarks(model_spigots, "UV-model visibilities") # if calibrating on (input-corrupt model), make corrupt model if do_solve and cal_type == CAL.DIFF: corrupt_uvdata = meqmaker.corrupt_uv_data(ns, model_spigots) # if needed, then make a predict tree using the MeqMaker if do_solve or do_output != CORRECTED_DATA: if model_spigots and not corrupt_uvdata: uvdata = model_spigots else: uvdata = None predict = meqmaker.make_predict_tree(ns, uvdata=uvdata) else: predict = None output_title = "Uncorrected residuals" # make nodes to compute residuals if do_output in [CORRECTED_RESIDUALS, RESIDUALS]: residuals = ns.residuals for p, q in array.ifrs(): if corrupt_uvdata: residuals(p, q) << Meq.Subtract(spigots( p, q), corrupt_uvdata(p, q), predict(p, q)) else: residuals(p, q) << spigots(p, q) - predict(p, q) if enable_inspectors: meqmaker.make_per_ifr_bookmarks(residuals, "Uncorrected residuals") outputs = residuals # and now we may need to correct the outputs if do_output in [CORRECTED_DATA, CORRECTED_RESIDUALS]: if do_correct_sky: srcs = meqmaker.get_source_list(ns) if do_correct_sky is FIRST_SOURCE: sky_correct = srcs and srcs[0] else: srcs = [ src for src in srcs if fnmatch.fnmatchcase(src.name, do_correct_sky) ] sky_correct = srcs and srcs[0] else: sky_correct = None outputs = meqmaker.correct_uv_data(ns, outputs, sky_correct=sky_correct, flag_jones=flag_jones) output_title = "Corrected data" if do_output is CORRECTED_DATA else "Corrected residuals" elif do_output == CORRUPTED_MODEL: outputs = predict output_title = "Predict" elif do_output == CORRUPTED_MODEL_ADD: outputs = ns.output for p, q in array.ifrs(): outputs(p, q) << spigots(p, q) + predict(p, q) output_title = "Data+predict" # make flaggers if flag_enable and do_output in [ CORRECTED_DATA, RESIDUALS, CORRECTED_RESIDUALS ]: flaggers = [] if flag_res is not None or flag_mean_res is not None: for p, q in array.ifrs(): ns.absres(p, q) << Meq.Abs(outputs(p, q)) # make flagger for residuals if flag_res is not None: for p, q in array.ifrs(): ns.flagres(p, q) << Meq.ZeroFlagger( ns.absres(p, q) - flag_res, oper='gt', flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT) flaggers.append(ns.flagres) # ...and an inspector for them if enable_inspectors: meqmaker.make_per_ifr_bookmarks(ns.flagres, "Residual amplitude flags") # make flagger for mean residuals if flag_mean_res is not None: ns.meanabsres << Meq.Mean( *[ns.absres(p, q) for p, q in array.ifrs()]) ns.flagmeanres << Meq.ZeroFlagger( ns.meanabsres - flag_mean_res, oper='gt', flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT) Meow.Bookmarks.Page("Mean residual amplitude flags").add( ns.flagmeanres, viewer="Result Plotter") flaggers.append(lambda p, q: ns.flagmeanres) # merge flags into output if flaggers: if enable_inspectors: meqmaker.make_per_ifr_bookmarks(outputs, output_title + " (unflagged)") for p, q in array.ifrs(): ns.flagged(p, q) << Meq.MergeFlags( outputs(p, q), *[f(p, q) for f in flaggers]) outputs = ns.flagged if enable_inspectors: meqmaker.make_per_ifr_bookmarks(outputs, output_title) abs_outputs = outputs('abs') for p, q in array.ifrs(): abs_outputs(p, q) << Meq.Abs(outputs(p, q)) meqmaker.make_per_ifr_bookmarks(abs_outputs, output_title + " (mean amplitudes)") # make solve trees if do_solve: # parse ifr specification solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs() if not solve_ifrs: raise RuntimeError( "No interferometers selected for calibration. Check your ifr specification (under calibration options)." ) # inputs to the solver are based on calibration type if corrupt_uvdata: [ ns.diff(p, q) << spigots(p, q) - corrupt_uvdata(p, q) for p, q in solve_ifrs ] rhs = ns.diff else: rhs = spigots lhs = predict weights = modulo = None # if calibrating visibilities, feed them to condeq directly, else take ampl/phase if cal_what == CAL.VIS: pass elif cal_what == CAL.AMPL: [ x('ampl', p, q) << Meq.Abs(x(p, q)) for p, q in ifrs for x in [rhs, lhs] ] lhs = lhs('ampl') rhs = rhs('ampl') elif cal_what == CAL.LOGAMPL: [ x('logampl', p, q) << Meq.Log(Meq.Abs(x(p, q))) for p, q in ifrs for x in [rhs, lhs] ] lhs = lhs('logampl') rhs = rhs('logampl') elif cal_what == CAL.PHASE: [ x('phase', p, q) << Meq.Arg(x(p, q)) for p, q in ifrs for x in [rhs, lhs] ] [rhs('ampl', p, q) << Meq.Abs(rhs(p, q)) for p, q in ifrs] lhs = lhs('phase') rhs = rhs('phase') weights = rhs('ampl') modulo = 2 * math.pi else: raise ValueError("unknown cal_what setting: " + str(cal_what)) # make a solve tree solve_tree = StdTrees.SolveTree(ns, lhs, solve_ifrs=solve_ifrs, weights=weights, modulo=modulo) # the output of the sequencer is either the residuals or the spigots, # according to what has been set above outputs = solve_tree.sequencers(inputs=rhs, outputs=outputs) post = ((enable_inspectors and meqmaker.get_inspectors()) or []) StdTrees.make_sinks(ns, outputs, spigots=spigots0, post=post, corr_index=mssel.get_corr_index()) if not do_solve: name = "Generate " + output_title.lower() comment = "Generated " + output_title.lower() if name: # make a TDL job to run the tree def run_tree(mqs, parent, wait=False, **kw): global tile_size purrpipe.title("Calibrating").comment(comment) return mqs.execute(Meow.Context.vdm.name, mssel.create_io_request(tile_size), wait=wait) TDLRuntimeMenu( name, TDLOption( 'tile_size', "Tile size, in timeslots", [10, 60, 120, 240], more=int, doc= """Input data is sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory.""" ), TDLJob(run_tree, name, job_id='generate_visibilities')) # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)) # insert solvejobs if do_solve: TDLRuntimeOptions(*ParmGroup.get_solvejob_options()) # finally, setup imaging options imsel = mssel.imaging_selector(npix=512, arcmin=meqmaker.estimate_image_size()) TDLRuntimeMenu("Make an image from this MS", *imsel.option_list()) # and close meqmaker -- this exports annotations, etc meqmaker.close()
def _define_forest(ns,parent=None,**kw): if not mssel.msname: raise RuntimeError,"MS not set"; if run_purr: Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']); # create Purr pipe global purrpipe; purrpipe = Purr.Pipe.Pipe(mssel.msname); # setup contexts from MS mssel.setup_observation_context(ns,prefer_baseline_uvw=True); array = Meow.Context.array; # make spigot nodes for data if do_solve or do_output not in [CORRUPTED_MODEL]: mssel.enable_input_column(True); spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index()); if enable_inspectors: meqmaker.make_per_ifr_bookmarks(spigots,"Input visibilities"); # add IFR-based errors, if any spigots = meqmaker.apply_visibility_processing(ns,spigots); else: mssel.enable_input_column(False); spigots = spigots0 = None; # make spigot nodes for model corrupt_uvdata = model_spigots = None; if read_ms_model: mssel.enable_model_column(True); model_spigots = array.spigots(column="PREDICT",corr=mssel.get_corr_index()); if enable_inspectors: meqmaker.make_per_ifr_bookmarks(model_spigots,"UV-model visibilities"); # if calibrating on (input-corrupt model), make corrupt model if do_solve and cal_type == CAL.DIFF: corrupt_uvdata = meqmaker.corrupt_uv_data(ns,model_spigots); # if needed, then make a predict tree using the MeqMaker if do_solve or do_output != CORRECTED_DATA: if model_spigots and not corrupt_uvdata: uvdata = model_spigots; else: uvdata = None; predict = meqmaker.make_predict_tree(ns,uvdata=uvdata); else: predict = None; output_title = "Uncorrected residuals"; # make nodes to compute residuals if do_output in [CORRECTED_RESIDUALS,RESIDUALS]: residuals = ns.residuals; for p,q in array.ifrs(): if corrupt_uvdata: residuals(p,q) << Meq.Subtract(spigots(p,q),corrupt_uvdata(p,q),predict(p,q)); else: residuals(p,q) << spigots(p,q) - predict(p,q); if enable_inspectors: meqmaker.make_per_ifr_bookmarks(residuals,"Uncorrected residuals"); outputs = residuals; # and now we may need to correct the outputs if do_output in [CORRECTED_DATA,CORRECTED_RESIDUALS]: if do_correct_sky: srcs = meqmaker.get_source_list(ns); if do_correct_sky is FIRST_SOURCE: sky_correct = srcs and srcs[0]; else: srcs = [ src for src in srcs if fnmatch.fnmatchcase(src.name,do_correct_sky) ]; sky_correct = srcs and srcs[0]; else: sky_correct = None; outputs = meqmaker.correct_uv_data(ns,outputs,sky_correct=sky_correct, flag_jones=flag_jones); output_title = "Corrected data" if do_output is CORRECTED_DATA else "Corrected residuals"; elif do_output == CORRUPTED_MODEL: outputs = predict; output_title = "Predict"; elif do_output == CORRUPTED_MODEL_ADD: outputs = ns.output; for p,q in array.ifrs(): outputs(p,q) << spigots(p,q) + predict(p,q); output_title = "Data+predict"; # make flaggers if flag_enable and do_output in [ CORRECTED_DATA,RESIDUALS,CORRECTED_RESIDUALS ]: flaggers = []; if flag_res is not None or flag_mean_res is not None: for p,q in array.ifrs(): ns.absres(p,q) << Meq.Abs(outputs(p,q)); # make flagger for residuals if flag_res is not None: for p,q in array.ifrs(): ns.flagres(p,q) << Meq.ZeroFlagger(ns.absres(p,q)-flag_res,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT); flaggers.append(ns.flagres); # ...and an inspector for them if enable_inspectors: meqmaker.make_per_ifr_bookmarks(ns.flagres,"Residual amplitude flags"); # make flagger for mean residuals if flag_mean_res is not None: ns.meanabsres << Meq.Mean(*[ns.absres(p,q) for p,q in array.ifrs()]); ns.flagmeanres << Meq.ZeroFlagger(ns.meanabsres-flag_mean_res,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT); Meow.Bookmarks.Page("Mean residual amplitude flags").add(ns.flagmeanres,viewer="Result Plotter"); flaggers.append(lambda p,q:ns.flagmeanres); # merge flags into output if flaggers: if enable_inspectors: meqmaker.make_per_ifr_bookmarks(outputs,output_title+" (unflagged)"); for p,q in array.ifrs(): ns.flagged(p,q) << Meq.MergeFlags(outputs(p,q),*[f(p,q) for f in flaggers]); outputs = ns.flagged; if enable_inspectors: meqmaker.make_per_ifr_bookmarks(outputs,output_title); abs_outputs = outputs('abs'); for p,q in array.ifrs(): abs_outputs(p,q) << Meq.Abs(outputs(p,q)); meqmaker.make_per_ifr_bookmarks(abs_outputs,output_title+" (mean amplitudes)"); # make solve trees if do_solve: # parse ifr specification solve_ifrs = array.subset(calibrate_ifrs,strict=False).ifrs(); if not solve_ifrs: raise RuntimeError,"No interferometers selected for calibration. Check your ifr specification (under calibration options)."; # inputs to the solver are based on calibration type if corrupt_uvdata: [ ns.diff(p,q) << spigots(p,q) - corrupt_uvdata(p,q) for p,q in solve_ifrs ]; rhs = ns.diff; else: rhs = spigots; lhs = predict; weights = modulo = None; # if calibrating visibilities, feed them to condeq directly, else take ampl/phase if cal_what == CAL.VIS: pass; elif cal_what == CAL.AMPL: [ x('ampl',p,q) << Meq.Abs(x(p,q)) for p,q in ifrs for x in rhs,lhs ]; lhs = lhs('ampl'); rhs = rhs('ampl'); elif cal_what == CAL.LOGAMPL: [ x('logampl',p,q) << Meq.Log(Meq.Abs(x(p,q))) for p,q in ifrs for x in rhs,lhs ]; lhs = lhs('logampl'); rhs = rhs('logampl'); elif cal_what == CAL.PHASE: [ x('phase',p,q) << Meq.Arg(x(p,q)) for p,q in ifrs for x in rhs,lhs ]; [ rhs('ampl',p,q) << Meq.Abs(rhs(p,q)) for p,q in ifrs ]; lhs = lhs('phase'); rhs = rhs('phase'); weights = rhs('ampl'); modulo = 2*math.pi; else: raise ValueError,"unknown cal_what setting: "+str(cal_what); # make a solve tree solve_tree = StdTrees.SolveTree(ns,lhs,solve_ifrs=solve_ifrs,weights=weights,modulo=modulo); # the output of the sequencer is either the residuals or the spigots, # according to what has been set above outputs = solve_tree.sequencers(inputs=rhs,outputs=outputs); post = ( ( enable_inspectors and meqmaker.get_inspectors() ) or [] ); StdTrees.make_sinks(ns,outputs,spigots=spigots0,post=post,corr_index=mssel.get_corr_index()); if not do_solve: name = "Generate "+output_title.lower(); comment = "Generated "+output_title.lower(); if name: # make a TDL job to run the tree def run_tree (mqs,parent,wait=False,**kw): global tile_size; purrpipe.title("Calibrating").comment(comment); return mqs.execute(Meow.Context.vdm.name,mssel.create_io_request(tile_size),wait=wait); TDLRuntimeMenu(name, TDLOption('tile_size',"Tile size, in timeslots",[10,60,120,240],more=int, doc="""Input data is sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory."""), TDLJob(run_tree,name,job_id='generate_visibilities') ); # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)); # insert solvejobs if do_solve: TDLRuntimeOptions(*ParmGroup.get_solvejob_options()); # finally, setup imaging options imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size()); TDLRuntimeMenu("Make an image from this MS",*imsel.option_list()); # and close meqmaker -- this exports annotations, etc meqmaker.close();
def _define_forest(ns, parent=None, **kw): if not mssel.msname: raise RuntimeError('MS name not set') mssel.setup_observation_context(ns) array = Context.array # Data and model input if do_solve or output_type.need_data: mssel.enable_input_column(True) spigots = array.spigots(corr=mssel.get_corr_index()) meqmaker.make_per_ifr_bookmarks(spigots, 'Input visibilities') else: mssel.enable_input_column(False) spigots = None if do_solve or output_type.need_model: predict = meqmaker.make_predict_tree(ns, uvdata=None) else: predict = None # Data output outputs = output_type.apply(ns, meqmaker, array.ifrs(), spigots, predict) # Flagging if flag_enable and output_type.flag_data: flaggers = [] if flag_res is not None or flag_mean_res is not None: for p, q in array.ifrs(): ns.absres(p, q) << Meq.Abs(outputs(p, q)) if flag_res is not None: for p, q in array.ifrs(): ns.flagres(p, q) << Meq.ZeroFlagger( ns.absres(p, q) - flag_res, oper='gt', flag_bit=MSUtils.FLAGMASK_OUTPUT) flaggers.append(ns.flagres) meqmaker.make_per_ifr_bookmarks(ns.flagres, 'Residual amplitude flags') if flag_mean_res is not None: ns.meanabsres << Meq.Mean( *[ns.absres(p, q) for p, q in array.ifrs()]) ns.flagmeanres << Meq.ZeroFlagger(ns.meanabsres - flag_mean_res, oper='gt', flag_bit=MSUtils.FLAGMASK_OUTPUT) Bookmarks.Page('Mean residual amplitude flags').add( ns.flagmeanres, viewer='Result Plotter') flaggers.append(lambda p, q: ns.flagmeanres) if flaggers: meqmaker.make_per_ifr_bookmarks(outputs, output_type.desc + ' (unflagged)') for p, q in array.ifrs(): ns.flagged(p, q) << Meq.MergeFlags( outputs(p, q), *[f(p, q) for f in flaggers]) outputs = ns.flagged meqmaker.make_per_ifr_bookmarks(outputs, output_type.desc) # Solve trees if do_solve: # parse ifr specification solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs() if not solve_ifrs: raise RuntimeError( 'No interferometers selected for calibration. ' 'Check your ifr specification (under calibration options).') lhs, rhs, weights, modulo = cal_quant.apply(solve_ifrs, predict, spigots) solve_tree = StdTrees.SolveTree(ns, lhs, solve_ifrs=solve_ifrs, weights=weights, modulo=modulo) outputs = solve_tree.sequencers(inputs=rhs, outputs=outputs) StdTrees.make_sinks(ns, outputs, spigots=spigots, post=meqmaker.get_inspectors() or [], corr_index=mssel.get_corr_index()) if not do_solve: name = 'Generate ' + output_type.desc.lower() comment = 'Generated ' + output_type.desc.lower() def run_tree(mqs, parent, wait=False, **kw): return mqs.execute(Context.vdm.name, mssel.create_io_request(tile_size), wait=wait) doc = """Input data are sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory.""" TDLRuntimeMenu( name, TDLOption('tile_size', 'Tile size, in timeslots', [10, 60, 120, 240], more=int, doc=doc), TDLJob(run_tree, name, job_id='generate_visibilities')) # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up # during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)) if do_solve: TDLRuntimeOptions(*ParmGroup.get_solvejob_options()) imsel = mssel.imaging_selector(npix=512, arcmin=meqmaker.estimate_image_size()) TDLRuntimeMenu('Make an image', *imsel.option_list()) meqmaker.close()
def _define_forest (ns, parent=None, **kw): if not mssel.msname: raise RuntimeError ('MS name not set') mssel.setup_observation_context (ns) array = Context.array # Data and model input if do_solve or output_type.need_data: mssel.enable_input_column (True) spigots = array.spigots (corr=mssel.get_corr_index ()) meqmaker.make_per_ifr_bookmarks (spigots, 'Input visibilities') else: mssel.enable_input_column (False) spigots = None if do_solve or output_type.need_model: predict = meqmaker.make_predict_tree (ns, uvdata=None) else: predict = None # Data output outputs = output_type.apply (ns, meqmaker, array.ifrs (), spigots, predict) # Flagging if flag_enable and output_type.flag_data: flaggers = [] if flag_res is not None or flag_mean_res is not None: for p, q in array.ifrs (): ns.absres(p,q) << Meq.Abs (outputs(p,q)) if flag_res is not None: for p, q in array.ifrs (): ns.flagres(p,q) << Meq.ZeroFlagger (ns.absres(p,q) - flag_res, oper='gt', flag_bit=MSUtils.FLAGMASK_OUTPUT) flaggers.append (ns.flagres) meqmaker.make_per_ifr_bookmarks (ns.flagres, 'Residual amplitude flags') if flag_mean_res is not None: ns.meanabsres << Meq.Mean (*[ns.absres(p,q) for p, q in array.ifrs()]) ns.flagmeanres << Meq.ZeroFlagger (ns.meanabsres - flag_mean_res, oper='gt', flag_bit=MSUtils.FLAGMASK_OUTPUT) Bookmarks.Page ('Mean residual amplitude flags').add (ns.flagmeanres, viewer='Result Plotter') flaggers.append (lambda p, q: ns.flagmeanres) if flaggers: meqmaker.make_per_ifr_bookmarks (outputs, output_type.desc + ' (unflagged)') for p, q in array.ifrs (): ns.flagged(p,q) << Meq.MergeFlags (outputs(p,q), *[f(p,q) for f in flaggers]) outputs = ns.flagged meqmaker.make_per_ifr_bookmarks (outputs, output_type.desc) # Solve trees if do_solve: # parse ifr specification solve_ifrs = array.subset (calibrate_ifrs, strict=False).ifrs() if not solve_ifrs: raise RuntimeError ('No interferometers selected for calibration. ' 'Check your ifr specification (under calibration options).') lhs, rhs, weights, modulo = cal_quant.apply (solve_ifrs, predict, spigots) solve_tree = StdTrees.SolveTree (ns, lhs, solve_ifrs=solve_ifrs, weights=weights, modulo=modulo) outputs = solve_tree.sequencers (inputs=rhs, outputs=outputs) StdTrees.make_sinks (ns, outputs, spigots=spigots, post=meqmaker.get_inspectors () or [], corr_index=mssel.get_corr_index ()) if not do_solve: name = 'Generate ' + output_type.desc.lower () comment = 'Generated ' + output_type.desc.lower () def run_tree (mqs, parent, wait=False, **kw): return mqs.execute (Context.vdm.name, mssel.create_io_request (tile_size), wait=wait) doc = """Input data are sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory.""" TDLRuntimeMenu(name, TDLOption ('tile_size', 'Tile size, in timeslots', [10, 60, 120, 240], more=int, doc=doc), TDLJob (run_tree, name, job_id='generate_visibilities')) # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up # during compilation. TDLRuntimeOptions (*meqmaker.runtime_options (nest=False)) if do_solve: TDLRuntimeOptions (*ParmGroup.get_solvejob_options ()) imsel = mssel.imaging_selector (npix=512, arcmin=meqmaker.estimate_image_size ()) TDLRuntimeMenu ('Make an image', *imsel.option_list ()) meqmaker.close()