def compute_jones (Jones,sources,stations=None,inspectors=[],**kw): """Creates the Z Jones for ionospheric phase, given TECs (per source, per station)."""; stations = stations or Context.array.stations; ns = Jones.Subscope(); piercings = iono_geometry.compute_piercings(ns,sources,stations); za_cos = iono_geometry.compute_za_cosines(ns,sources,stations); tecs = iono_model(ns,piercings,za_cos,sources,stations); # make inspector for TECs inspectors.append( Jones.scope.inspector('TEC') << StdTrees.define_inspector(tecs,sources,stations, label='tec',freqavg=False) ); if diff_mode: absJones = Jones('abs'); iono_geometry.compute_zeta_jones_from_tecs(absJones,tecs,sources,stations); for src in sources: for p in stations: Jones(src,p) << absJones(src,p)/absJones(sources[0],p); else: iono_geometry.compute_zeta_jones_from_tecs(Jones,tecs,sources,stations); # make inspector for ionospheric phases Zphase = ns.Zphase; for src in sources: for p in stations: Zphase(src,p) << Meq.Arg(Jones(src,p)); inspectors += [ Jones.scope.inspector('iono_phase') << \ StdTrees.define_inspector(Zphase,sources,stations,label='z'), Jones.scope.inspector('iono_piercings') << \ StdTrees.define_inspector(ns.dxy,sources,stations,label='dxy'), ] return Jones;
def compute_jones (Jones,sources,stations=None,inspectors=[],label='L',**kw): """Creates the dipole projection matrix."""; stations = stations or Context.array.stations; ns = Jones.Subscope(); insp = Jones.scope.inspector(label); for src in sources: for p in stations: Jones(src,p) << proj_matrix(src,Context.array.xyz(p)); insp << StdTrees.define_inspector(Jones,sources,stations,label=label); # add inspectors StdTrees.inspector(Jones.scope.inspector(label,'AzEl') ,[src.direction.azel() for src in sources],bookmark=False); inspectors += [ insp,Jones.scope.inspector(label,'AzEl') ]; return Jones;
def compute_jones (self,nodes,stations=None,tags=None,label='',inspectors=[],**kw): stations = stations or Context.array.stations(); rot_def = Meow.Parm(0); nr = Jones.rotation_matrix(nodes("R"),rot_def,tags=tags,series=stations); ne = Jones.ellipticity_matrix(nodes("E"),rot_def,tags=tags,series=stations); for p in stations: nodes(p) << Meq.MatrixMultiply(nr(p),ne(p)); # make parmgroups for phases and gains self.pg_rot = ParmGroup.ParmGroup(label+"_leakage", nodes.search(tags="solvable"), table_name="%s_leakage.mep"%label,bookmark=True); # make solvejobs ParmGroup.SolveJob("cal_"+label+"_leakage","Calibrate %s (leakage)"%label,self.pg_rot); # make inspector for parameters StdTrees.inspector(nodes('inspector'),self.pg_rot.nodes,bookmark=False); inspectors.append(nodes('inspector')); return nodes;
def compute_jones (Jones,sources,stations=None,pointing_offsets=None,inspectors=[],label='E',**kw): stations = stations or Context.array.stations; ns = Jones.Subscope(); # declare an inspector node for the Jones matrix -- will be defined below insp = Jones.scope.inspector(label); inspectors += [ insp ]; # loop over sources for src in sources: # If sky rotation and/or pointing offsets are in effect, we have a per-station beam. # Otherwise the beam is the same for all stations. if sky_rotation or pointing_offsets: for p in stations: lm = src.direction.lm(); # apply rotation to put sources into the antenna frame if sky_rotation: xyz = Context.array.xyz(p); pa_rot = Context.observation.phase_centre.pa_rot(xyz); lm = ns.lmrot(src,p) << Meq.MatrixMultiply(pa_rot,src.direction.lm()); # apply offset in the node (so pointing offsets are interpreted in the azel frame, if rotating) make_beam_node(Jones(src,p),filename_pattern,lm,pointing_offsets and pointing_offsets(p)); else: make_beam_node(Jones(src),filename_pattern,src.direction.lm()); for p in stations: Jones(src,p) << Meq.Identity(Jones(src)); # define an inspector if sky_rotation or pointing_offsets: # Jones inspector is per-source, per-station insp << StdTrees.define_inspector(Jones,sources,stations,label=label); else: # Jones inspector is per-source insp << StdTrees.define_inspector(Jones,sources,label=label); return Jones;
def compute_jones (Jones,sources,stations=None,pointing_offsets=None,inspectors=[],label='E',**kw): stations = stations or Context.array.stations; ns = Jones.Subscope(); # declare an inspector node for the Jones matrix -- will be defined below insp = Jones.scope.inspector(label); inspectors += [ insp ]; radec0 = Context.observation.phase_centre.radec() if interpol_coord is COORD_THETAPHI else None; # loop over sources for src in sources: xy = src.direction.radec() if interpol_coord is COORD_THETAPHI else src.direction.lm(); # If sky rotation and/or horizon masking and/or pointing offsets are in effect, we have a per-station beam. # Otherwise the beam is the same for all stations. if sky_rotation or pointing_offsets or horizon_masking: for p in stations: azel = src.direction.azel(Context.array.xyz(p)) if horizon_masking else None; pa = Context.observation.phase_centre.pa(Context.array.xyz(p)) if sky_rotation else None; # apply offset in the node (so pointing offsets are interpreted in the azel frame, if rotating) make_beam_node(Jones(src,p),filename_pattern,xy,radec0=radec0, dlm=pointing_offsets and pointing_offsets(p),azel=azel,pa=pa); else: make_beam_node(Jones(src),filename_pattern,xy,radec0=radec0); for p in stations: Jones(src,p) << Meq.Identity(Jones(src)); # define an inspector if sky_rotation or pointing_offsets or horizon_masking: # Jones inspector is per-source, per-station insp << StdTrees.define_inspector(Jones,sources,stations,label=label); else: # Jones inspector is per-source insp << StdTrees.define_inspector(Jones,sources,label=label); return Jones;
def compute_jones (self,Jones,sources,stations=None,inspectors=[],**kw): """Computes beam gain for a list of sources. The output node, will be qualified with either a source only, or a source/station pair """; stations = stations or Context.array.stations(); ns = Jones.Subscope(); # init solvables etc. self.init_parameters(ns,sources,stations,inspectors); # this dict will hold LM tuples (or nodes) for each source. lmsrc = {}; # see if sources have a "beam_lm" attribute, use that for beam offsets for src in sources: lm = src.get_attr("beam_lm",None) or src.get_attr("_lm_ncp",None); if lm: src.set_attr(self.label+'r',math.sqrt(lm[0]**2+lm[1]**2)/math.pi*(180*60)); lmsrc[src.name] = ns.lm(src) << Meq.Constant(value=Timba.array.array(lm)); # else try to use static lm coordinates else: # else try to use static lm coordinates lmnst = src.direction.lmn_static(); if lmnst: lm = lmnst[0:2]; src.set_attr(self.label+'r',math.sqrt(lm[0]**2+lm[1]**2)/math.pi*(180*60)); lmsrc[src.name] = ns.lm(src) << Meq.Constant(value=Timba.array.array(lm)); # else use lmn node else: lmsrc[src.name] = src.direction.lm(); if self.per_station: for src in sources: for p in stations: self.make_beam_nodes(Jones(src,p),self.beamshape(p),lmsrc[src.name],self.ell(p),self.dlm(p)); else: p0 = stations[0]; for src in sources: self.make_beam_nodes(Jones(src,p0),self.beamshape(p0),lmsrc[src.name],self.ell(p0),self.dlm(p0)); for p in stations[1:]: Jones(src,p) << Meq.Identity(Jones(src,p0)); # make inspectors inspectors.append(ns.inspector << StdTrees.define_inspector( Jones,sources,stations,label=self.label)); return Jones;
def _define_forest(ns,parent=None,**kw): if not mssel.msname: raise RuntimeError,"MS not set"; if run_purr: Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']); # create Purr pipe global purrpipe; purrpipe = Purr.Pipe.Pipe(mssel.msname); # setup contexts from MS mssel.setup_observation_context(ns,prefer_baseline_uvw=True); array = Meow.Context.array; # make spigot nodes for data if do_solve or do_output not in [CORRUPTED_MODEL]: mssel.enable_input_column(True); spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index()); if enable_inspectors: meqmaker.make_per_ifr_bookmarks(spigots,"Input visibilities"); # add IFR-based errors, if any spigots = meqmaker.apply_visibility_processing(ns,spigots); else: mssel.enable_input_column(False); spigots = spigots0 = None; # make spigot nodes for model corrupt_uvdata = model_spigots = None; if read_ms_model: mssel.enable_model_column(True); model_spigots = array.spigots(column="PREDICT",corr=mssel.get_corr_index()); if enable_inspectors: meqmaker.make_per_ifr_bookmarks(model_spigots,"UV-model visibilities"); # if calibrating on (input-corrupt model), make corrupt model if do_solve and cal_type == CAL.DIFF: corrupt_uvdata = meqmaker.corrupt_uv_data(ns,model_spigots); # if needed, then make a predict tree using the MeqMaker if do_solve or do_output != CORRECTED_DATA: if model_spigots and not corrupt_uvdata: uvdata = model_spigots; else: uvdata = None; predict = meqmaker.make_predict_tree(ns,uvdata=uvdata); else: predict = None; output_title = "Uncorrected residuals"; # make nodes to compute residuals if do_output in [CORRECTED_RESIDUALS,RESIDUALS]: residuals = ns.residuals; for p,q in array.ifrs(): if corrupt_uvdata: residuals(p,q) << Meq.Subtract(spigots(p,q),corrupt_uvdata(p,q),predict(p,q)); else: residuals(p,q) << spigots(p,q) - predict(p,q); if enable_inspectors: meqmaker.make_per_ifr_bookmarks(residuals,"Uncorrected residuals"); outputs = residuals; # and now we may need to correct the outputs if do_output in [CORRECTED_DATA,CORRECTED_RESIDUALS]: if do_correct_sky: srcs = meqmaker.get_source_list(ns); if do_correct_sky is FIRST_SOURCE: sky_correct = srcs and srcs[0]; else: srcs = [ src for src in srcs if fnmatch.fnmatchcase(src.name,do_correct_sky) ]; sky_correct = srcs and srcs[0]; else: sky_correct = None; outputs = meqmaker.correct_uv_data(ns,outputs,sky_correct=sky_correct, flag_jones=flag_jones); output_title = "Corrected data" if do_output is CORRECTED_DATA else "Corrected residuals"; elif do_output == CORRUPTED_MODEL: outputs = predict; output_title = "Predict"; elif do_output == CORRUPTED_MODEL_ADD: outputs = ns.output; for p,q in array.ifrs(): outputs(p,q) << spigots(p,q) + predict(p,q); output_title = "Data+predict"; # make flaggers if flag_enable and do_output in [ CORRECTED_DATA,RESIDUALS,CORRECTED_RESIDUALS ]: flaggers = []; if flag_res is not None or flag_mean_res is not None: for p,q in array.ifrs(): ns.absres(p,q) << Meq.Abs(outputs(p,q)); # make flagger for residuals if flag_res is not None: for p,q in array.ifrs(): ns.flagres(p,q) << Meq.ZeroFlagger(ns.absres(p,q)-flag_res,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT); flaggers.append(ns.flagres); # ...and an inspector for them if enable_inspectors: meqmaker.make_per_ifr_bookmarks(ns.flagres,"Residual amplitude flags"); # make flagger for mean residuals if flag_mean_res is not None: ns.meanabsres << Meq.Mean(*[ns.absres(p,q) for p,q in array.ifrs()]); ns.flagmeanres << Meq.ZeroFlagger(ns.meanabsres-flag_mean_res,oper='gt',flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT); Meow.Bookmarks.Page("Mean residual amplitude flags").add(ns.flagmeanres,viewer="Result Plotter"); flaggers.append(lambda p,q:ns.flagmeanres); # merge flags into output if flaggers: if enable_inspectors: meqmaker.make_per_ifr_bookmarks(outputs,output_title+" (unflagged)"); for p,q in array.ifrs(): ns.flagged(p,q) << Meq.MergeFlags(outputs(p,q),*[f(p,q) for f in flaggers]); outputs = ns.flagged; if enable_inspectors: meqmaker.make_per_ifr_bookmarks(outputs,output_title); abs_outputs = outputs('abs'); for p,q in array.ifrs(): abs_outputs(p,q) << Meq.Abs(outputs(p,q)); meqmaker.make_per_ifr_bookmarks(abs_outputs,output_title+" (mean amplitudes)"); # make solve trees if do_solve: # parse ifr specification solve_ifrs = array.subset(calibrate_ifrs,strict=False).ifrs(); if not solve_ifrs: raise RuntimeError,"No interferometers selected for calibration. Check your ifr specification (under calibration options)."; # inputs to the solver are based on calibration type if corrupt_uvdata: [ ns.diff(p,q) << spigots(p,q) - corrupt_uvdata(p,q) for p,q in solve_ifrs ]; rhs = ns.diff; else: rhs = spigots; lhs = predict; weights = modulo = None; # if calibrating visibilities, feed them to condeq directly, else take ampl/phase if cal_what == CAL.VIS: pass; elif cal_what == CAL.AMPL: [ x('ampl',p,q) << Meq.Abs(x(p,q)) for p,q in ifrs for x in rhs,lhs ]; lhs = lhs('ampl'); rhs = rhs('ampl'); elif cal_what == CAL.LOGAMPL: [ x('logampl',p,q) << Meq.Log(Meq.Abs(x(p,q))) for p,q in ifrs for x in rhs,lhs ]; lhs = lhs('logampl'); rhs = rhs('logampl'); elif cal_what == CAL.PHASE: [ x('phase',p,q) << Meq.Arg(x(p,q)) for p,q in ifrs for x in rhs,lhs ]; [ rhs('ampl',p,q) << Meq.Abs(rhs(p,q)) for p,q in ifrs ]; lhs = lhs('phase'); rhs = rhs('phase'); weights = rhs('ampl'); modulo = 2*math.pi; else: raise ValueError,"unknown cal_what setting: "+str(cal_what); # make a solve tree solve_tree = StdTrees.SolveTree(ns,lhs,solve_ifrs=solve_ifrs,weights=weights,modulo=modulo); # the output of the sequencer is either the residuals or the spigots, # according to what has been set above outputs = solve_tree.sequencers(inputs=rhs,outputs=outputs); post = ( ( enable_inspectors and meqmaker.get_inspectors() ) or [] ); StdTrees.make_sinks(ns,outputs,spigots=spigots0,post=post,corr_index=mssel.get_corr_index()); if not do_solve: name = "Generate "+output_title.lower(); comment = "Generated "+output_title.lower(); if name: # make a TDL job to run the tree def run_tree (mqs,parent,wait=False,**kw): global tile_size; purrpipe.title("Calibrating").comment(comment); return mqs.execute(Meow.Context.vdm.name,mssel.create_io_request(tile_size),wait=wait); TDLRuntimeMenu(name, TDLOption('tile_size',"Tile size, in timeslots",[10,60,120,240],more=int, doc="""Input data is sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory."""), TDLJob(run_tree,name,job_id='generate_visibilities') ); # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)); # insert solvejobs if do_solve: TDLRuntimeOptions(*ParmGroup.get_solvejob_options()); # finally, setup imaging options imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size()); TDLRuntimeMenu("Make an image from this MS",*imsel.option_list()); # and close meqmaker -- this exports annotations, etc meqmaker.close();
def _define_forest(ns, parent=None, **kw): if not mssel.msname: raise RuntimeError("MS not set") if run_purr: Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.']) # create Purr pipe global purrpipe purrpipe = Purr.Pipe.Pipe(mssel.msname) # setup contexts from MS mssel.setup_observation_context(ns) array = Meow.Context.array # make spigot nodes for data if do_solve or do_output not in [CORRUPTED_MODEL]: mssel.enable_input_column(True) spigots = spigots0 = outputs = array.spigots( corr=mssel.get_corr_index()) if enable_inspectors: meqmaker.make_per_ifr_bookmarks(spigots, "Input visibilities") else: mssel.enable_input_column(False) spigots = spigots0 = None # make spigot nodes for model corrupt_uvdata = model_spigots = None if read_ms_model: mssel.enable_model_column(True) model_spigots = array.spigots(column="PREDICT", corr=mssel.get_corr_index()) if enable_inspectors: meqmaker.make_per_ifr_bookmarks(model_spigots, "UV-model visibilities") # if calibrating on (input-corrupt model), make corrupt model if do_solve and cal_type == CAL.DIFF: corrupt_uvdata = meqmaker.corrupt_uv_data(ns, model_spigots) # if needed, then make a predict tree using the MeqMaker if do_solve or do_output != CORRECTED_DATA: if model_spigots and not corrupt_uvdata: uvdata = model_spigots else: uvdata = None predict = meqmaker.make_predict_tree(ns, uvdata=uvdata) else: predict = None output_title = "Uncorrected residuals" # make nodes to compute residuals if do_output in [CORRECTED_RESIDUALS, RESIDUALS]: residuals = ns.residuals for p, q in array.ifrs(): if corrupt_uvdata: residuals(p, q) << Meq.Subtract(spigots( p, q), corrupt_uvdata(p, q), predict(p, q)) else: residuals(p, q) << spigots(p, q) - predict(p, q) if enable_inspectors: meqmaker.make_per_ifr_bookmarks(residuals, "Uncorrected residuals") outputs = residuals # and now we may need to correct the outputs if do_output in [CORRECTED_DATA, CORRECTED_RESIDUALS]: if do_correct_sky: srcs = meqmaker.get_source_list(ns) sky_correct = srcs and srcs[0] else: sky_correct = None global flag_jones if flag_enable and flag_jones and not (flag_jones_min is None and flag_jones_max is None): flag_jones_minmax = (flag_jones_min, flag_jones_max) else: flag_jones_minmax = None outputs = meqmaker.correct_uv_data(ns, outputs, sky_correct=sky_correct, flag_jones_minmax=flag_jones and (flag_jones_min, flag_jones_max)) output_title = "Corrected data" if do_output is CORRECTED_DATA else "Corrected residuals" elif do_output == CORRUPTED_MODEL: outputs = predict output_title = "Predict" elif do_output == CORRUPTED_MODEL_ADD: outputs = ns.output for p, q in array.ifrs(): outputs(p, q) << spigots(p, q) + predict(p, q) output_title = "Data+predict" # make flaggers if flag_enable and do_output in [ CORRECTED_DATA, RESIDUALS, CORRECTED_RESIDUALS ]: flaggers = [] if flag_res is not None or flag_mean_res is not None: for p, q in array.ifrs(): ns.absres(p, q) << Meq.Abs(outputs(p, q)) # make flagger for residuals if flag_res is not None: for p, q in array.ifrs(): ns.flagres(p, q) << Meq.ZeroFlagger( ns.absres(p, q) - flag_res, oper='gt', flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT) flaggers.append(ns.flagres) # ...and an inspector for them if enable_inspectors: meqmaker.make_per_ifr_bookmarks(ns.flagres, "Residual amplitude flags") # make flagger for mean residuals if flag_mean_res is not None: ns.meanabsres << Meq.Mean( *[ns.absres(p, q) for p, q in array.ifrs()]) ns.flagmeanres << Meq.ZeroFlagger( ns.meanabsres - flag_mean_res, oper='gt', flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT) Meow.Bookmarks.Page("Mean residual amplitude flags").add( ns.flagmeanres, viewer="Result Plotter") flaggers.append(lambda p, q: ns.flagmeanres) # merge flags into output if flaggers: if enable_inspectors: meqmaker.make_per_ifr_bookmarks(outputs, output_title + " (unflagged)") for p, q in array.ifrs(): ns.flagged(p, q) << Meq.MergeFlags( outputs(p, q), *[f(p, q) for f in flaggers]) outputs = ns.flagged if enable_inspectors: meqmaker.make_per_ifr_bookmarks(outputs, output_title) abs_outputs = outputs('abs') for p, q in array.ifrs(): abs_outputs(p, q) << Meq.Abs(outputs(p, q)) meqmaker.make_per_ifr_bookmarks(abs_outputs, output_title + " (mean amplitudes)") # make solve trees if do_solve: # parse ifr specification solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs() if not solve_ifrs: raise RuntimeError( "No interferometers selected for calibration. Check your ifr specification (under calibration options)." ) # inputs to the solver are based on calibration type if corrupt_uvdata: [ ns.diff(p, q) << spigots(p, q) - corrupt_uvdata(p, q) for p, q in solve_ifrs ] rhs = ns.diff else: rhs = spigots lhs = predict weights = modulo = None # if calibrating visibilities, feed them to condeq directly, else take ampl/phase if cal_what == CAL.VIS: pass elif cal_what == CAL.AMPL: [ x('ampl', p, q) << Meq.Abs(x(p, q)) for p, q in ifrs for x in (rhs, lhs) ] lhs = lhs('ampl') rhs = rhs('ampl') elif cal_what == CAL.LOGAMPL: [ x('logampl', p, q) << Meq.Log(Meq.Abs(x(p, q))) for p, q in ifrs for x in (rhs, lhs) ] lhs = lhs('logampl') rhs = rhs('logampl') elif cal_what == CAL.PHASE: [ x('phase', p, q) << Meq.Arg(x(p, q)) for p, q in ifrs for x in (rhs, lhs) ] [rhs('ampl', p, q) << Meq.Abs(rhs(p, q)) for p, q in ifrs] lhs = lhs('phase') rhs = rhs('phase') weights = rhs('ampl') modulo = 2 * math.pi else: raise ValueError("unknown cal_what setting: " + str(cal_what)) # make a solve tree solve_tree = StdTrees.SolveTree(ns, lhs, solve_ifrs=solve_ifrs, weights=weights, modulo=modulo) # the output of the sequencer is either the residuals or the spigots, # according to what has been set above outputs = solve_tree.sequencers(inputs=rhs, outputs=outputs) post = (enable_inspectors and meqmaker.get_inspectors()) or [] StdTrees.make_sinks(ns, outputs, spigots=spigots0, post=post, corr_index=mssel.get_corr_index()) if not do_solve: name = "Generate " + output_title.lower() comment = "Generated " + output_title.lower() if name: # make a TDL job to run the tree def run_tree(mqs, parent, wait=False, **kw): global tile_size purrpipe.title("Calibrating").comment(comment) return mqs.execute(Meow.Context.vdm.name, mssel.create_io_request(tile_size), wait=wait) TDLRuntimeMenu( name, TDLOption( 'tile_size', "Tile size, in timeslots", [10, 60, 120, 240], more=int, doc= """Input data is sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory.""" ), TDLJob(run_tree, name, job_id='generate_visibilities')) # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)) # insert solvejobs if do_solve: TDLRuntimeOptions(*ParmGroup.get_solvejob_options()) # finally, setup imaging options imsel = mssel.imaging_selector(npix=512, arcmin=meqmaker.estimate_image_size()) TDLRuntimeMenu("Make an image from this MS", *imsel.option_list()) # and close meqmaker -- this exports annotations, etc meqmaker.close()
def compute_jones (Jones,sources,stations=None,inspectors=[],meqmaker=None,label='R',**kw): """Creates the Z Jones for ionospheric phase, given TECs (per source, per station)."""; stations = stations or Context.array.stations; ns = Jones.Subscope(); # get reference source if ref_source: # treat as index first dir0 = None; try: dir0 = sources[int(ref_source)].direction; except: pass; # else treat as name, find in list if not dir0: for src0 in sources: if src0.name == ref_source: dir0 = src0.direction; break; # else treat as direction string if not dir0: ff = list(ref_source.split()); if len(ff) < 2 or len(ff) > 3: raise RuntimeError,"invalid reference dir '%s' specified for %s-Jones"%(ref_source,label); global dm; if not dm: raise RuntimeError,"pyrap measures module not available, cannot use direction strings for %s-Jones"%label; if len(ff) == 2: ff = [ 'J2000' ] +ff; # treat as direction measure try: dmdir = dm.direction(*ff); except: raise RuntimeError,"invalid reference dir '%s' specified for %s-Jones"%(ref_source,label); # convert to J2000 and make direction object dmdir = dm.measure(dmdir,'J2000'); ra,dec = dm.getvalue(dmdir)[0].get_value(),dm.getvalue(dmdir)[1].get_value(); dir0 = Meow.Direction(ns,"refdir",ra,dec,static=True); else: dir0 = Context.observation.phase_centre; # make refraction scale node scale = ns.scale(0) << Meq.Parm(0,tags="refraction"); xyz0 = Context.array.xyz0(); if coord_approx: # get PA, and assume it's the same over the whole field pa = ns.pa0 << Meq.ParAngle(dir0.radec(),xyz0); # second column of the Rot(-PA) matrix. Multiply this by del to get a rotation of (0,del) into the lm plane. # The third component (0) is for convenience, as it immediately gives us dl,dm,dn, since we assume dn~0 rot_pa = ns.rotpa0 << Meq.Composer(Meq.Sin(pa),Meq.Cos(pa),0); # el0: elevation of field centre el0 = dir0.el(); if do_extinction: ns.inv_ext0 << Meq.Sin(el0); # inverse of extinction towards el0 # station UVWs uvw = Context.array.uvw(); # now loop over sources for isrc,src in enumerate(sources): # reference direction: no refraction at all if src.direction is dir0: for p in stations: Jones(src,p) << 1; continue; # dEl is source elevation minus el0 # ddEl = scale*dEl: amount by which source refracts (negative means field is compressed) el = src.direction.el() ns.dEl(src) << el - el0; ddel = ns.ddEl(src) << ns.dEl(src)*scale; # get el1: refracted elevation angle if not coord_approx or do_extinction: el1 = ns.el1(src) << el + ddel; # compute extinction component if do_extinction: # compute inverse of extinction towards the refracted direction el1 iext = ns.inv_ext(src) << Meq.Sin(el1); # # and differential extinction is then ext1/ext0 ext = ns.dext(src) << ns.inv_ext0/iext; # Compute dlmn offset in lm plane. if coord_approx: # Approximate mode: ddel is added to elevation, so to get the lm offset, we need # to apply Rot(PA) to the column vector (0,ddel), and then take the sine of the result. dlmn = ns.dlmn(src) << Meq.Sin(ddel*rot_pa); else: ns.azel1(src) << Meq.Composer(src.direction.az(),el1); ns.radec1(src) << Meq.RADec(ns.azel1(src),xyz0); ns.lmn1(src) << Meq.LMN(Context.observation.radec0(),ns.radec1(src)); dlmn = ns.dlmn(src) << ns.lmn1(src) - src.lmn(); # get per-station phases for p in stations: if do_extinction: Jones(src,p) << ext*(ns.phase(src,p) << Meq.VisPhaseShift(lmn=dlmn,uvw=uvw(p))); else: Jones(src,p) << Meq.VisPhaseShift(lmn=dlmn,uvw=uvw(p)); # make bookmarks srcnames = [ src.name for src in sources ]; meqmaker.make_bookmark_set(Jones,[ (src,p) for src in srcnames for p in stations ], "%s: inspector plot"%label,"%s: by source-station"%label,freqmean=True); inspectors.append(ns.inspector(label,'scale') << \ StdTrees.define_inspector(ns.scale,[0],label=label)); inspectors.append(ns.inspector(label,'delta-el') << \ StdTrees.define_inspector(ns.ddEl,srcnames,label=label)); inspectors.append(ns.inspector(label,'delta-el') << \ StdTrees.define_inspector(ns.ddEl,srcnames,label=label)); inspectors.append(ns.inspector(label,'dlmn') << \ StdTrees.define_inspector(ns.dlmn,srcnames,label=label)); if do_extinction: inspectors.append(ns.inspector(label,'inv-ext') << \ StdTrees.define_inspector(ns.inv_ext,srcnames,label=label)); inspectors.append(ns.inspector(label,'diff-ext') << \ StdTrees.define_inspector(ns.dext,srcnames,label=label)); # make parmgroups and solvejobs global pg; pg = ParmGroup.ParmGroup(label,[scale],table_name="%s.fmep"%label,bookmark=False); # make solvejobs ParmGroup.SolveJob("cal_"+label,"Calibrate %s (differential refraction)"%label,pg); return Jones;
def _define_forest(ns, parent=None, **kw): if run_purr: Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.']) # create Purr pipe global purrpipe purrpipe = Purr.Pipe.Pipe(mssel.msname) # get antennas from MS ANTENNAS = mssel.get_antenna_set(range(1, 15)) array = Meow.IfrArray(ns, ANTENNAS, mirror_uvw=False) stas = array.stations() # get phase centre from MS, setup observation observation = Meow.Observation(ns, phase_centre=mssel.get_phase_dir(), linear=mssel.is_linear_pol(), circular=mssel.is_circular_pol()) Meow.Context.set(array, observation) # get active correlations from MS Meow.Context.active_correlations = mssel.get_correlations() # make spigot nodes spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index()) # ...and an inspector for them StdTrees.vis_inspector(ns.inspector('input'), spigots, bookmark="Inspect input visibilities") inspectors = [ns.inspector('input')] Bookmarks.make_node_folder("Input visibilities by baseline", [spigots(p, q) for p, q in array.ifrs()], sorted=True, ncol=2, nrow=2) inspect_ifrs = array.ifrs() if do_solve: # filter solvable baselines by baseline length solve_ifrs = [] antpos = mssel.ms_antenna_positions if (min_baseline or max_baseline) and antpos is not None: for (ip, p), (iq, q) in array.ifr_index(): baseline = math.sqrt( ((antpos[ip, :] - antpos[iq, :])**2).sum()) if (not min_baseline or baseline > min_baseline) and \ (not max_baseline or baseline < max_baseline): solve_ifrs.append((p, q)) else: solve_ifrs = array.ifrs() inspect_ifrs = solve_ifrs # make a predict tree using the MeqMaker if do_solve or do_subtract: predict = meqmaker.make_predict_tree(ns) # make a ParmGroup and solve jobs for source parameters, if we have any if do_solve: parms = {} for src in meqmaker.get_source_list(ns): parms.update([(p.name, p) for p in src.get_solvables()]) if parms: pg_src = ParmGroup.ParmGroup("source", parms.values(), table_name="sources.fmep", individual=True, bookmark=True) # now make a solvejobs for the source ParmGroup.SolveJob("cal_source", "Calibrate source model", pg_src) # make nodes to compute residuals if do_subtract: residuals = ns.residuals for p, q in array.ifrs(): residuals(p, q) << spigots(p, q) - predict(p, q) outputs = residuals # and now we may need to correct the outputs if do_correct: if do_correct_sky: srcs = meqmaker.get_source_list(ns) sky_correct = srcs and srcs[0] else: sky_correct = None outputs = meqmaker.correct_uv_data(ns, outputs, sky_correct=sky_correct, inspect_ifrs=inspect_ifrs) # make solve trees if do_solve: # inputs to the solver are based on calibration type # if calibrating visibilities, feed them to condeq directly if cal_type == CAL.VIS: observed = spigots model = predict # else take ampl/phase component else: model = ns.model observed = ns.observed if cal_type == CAL.AMPL: for p, q in array.ifrs(): observed(p, q) << Meq.Abs(spigots(p, q)) model(p, q) << Meq.Abs(predict(p, q)) elif cal_type == CAL.LOGAMPL: for p, q in array.ifrs(): observed(p, q) << Meq.Log(Meq.Abs(spigots(p, q))) model(p, q) << Meq.Log(Meq.Abs(predict(p, q))) elif cal_type == CAL.PHASE: for p, q in array.ifrs(): observed(p, q) << 0 model(p, q) << Meq.Abs(predict(p, q)) * Meq.FMod( Meq.Arg(spigots(p, q)) - Meq.Arg(predict(p, q)), 2 * math.pi) else: raise ValueError, "unknown cal_type setting: " + str(cal_type) # make a solve tree solve_tree = StdTrees.SolveTree(ns, model, solve_ifrs=solve_ifrs) # the output of the sequencer is either the residuals or the spigots, # according to what has been set above outputs = solve_tree.sequencers(inputs=observed, outputs=outputs) # make sinks and vdm. # The list of inspectors must be supplied here inspectors += meqmaker.get_inspectors() or [] StdTrees.make_sinks(ns, outputs, spigots=spigots0, post=inspectors) Bookmarks.make_node_folder("Corrected/residual visibilities by baseline", [outputs(p, q) for p, q in array.ifrs()], sorted=True, ncol=2, nrow=2) if not do_solve: if do_subtract: name = "Generate residuals" comment = "Generated residual visibilities." elif do_correct: name = "Generate corrected data" comment = "Generated corrected visibilities." else: name = None if name: # make a TDL job to runsthe tree def run_tree(mqs, parent, **kw): global tile_size purrpipe.title("Calibrating").comment(comment) mqs.execute(Meow.Context.vdm.name, mssel.create_io_request(tile_size), wait=False) TDLRuntimeMenu( name, TDLOption( 'tile_size', "Tile size, in timeslots", [10, 60, 120, 240], more=int, doc= """Input data is sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory.""" ), TDLRuntimeJob(run_tree, name)) # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)) # insert solvejobs if do_solve: TDLRuntimeOptions(*ParmGroup.get_solvejob_options()) # finally, setup imaging options imsel = mssel.imaging_selector(npix=512, arcmin=meqmaker.estimate_image_size()) TDLRuntimeMenu("Make an image from this MS", *imsel.option_list()) # and close meqmaker -- this exports annotations, etc meqmaker.close()
def compute_jones(self, Jones, sources, stations=None, inspectors=[], **kw): """Computes beam gain for a list of sources. The output node, will be qualified with either a source only, or a source/station pair """ stations = stations or Context.array.stations() ns = Jones.Subscope() # init solvables etc. self.init_parameters(ns, sources, stations, inspectors) # this dict will hold LM tuples (or nodes) for each source. lmsrc = {} # see if sources have a "beam_lm" attribute, use that for beam offsets for src in sources: lm = src.get_attr("beam_lm", None) or src.get_attr( "_lm_ncp", None) if lm: src.set_attr( self.label + 'r', math.sqrt(lm[0]**2 + lm[1]**2) / math.pi * (180 * 60)) lmsrc[src.name] = ns.lm(src) << Meq.Constant( value=Timba.array.array(lm)) # else try to use static lm coordinates else: # else try to use static lm coordinates lmnst = src.direction.lmn_static() if lmnst: lm = lmnst[0:2] src.set_attr( self.label + 'r', math.sqrt(lm[0]**2 + lm[1]**2) / math.pi * (180 * 60)) lmsrc[src.name] = ns.lm(src) << Meq.Constant( value=Timba.array.array(lm)) # else use lmn node else: lmsrc[src.name] = src.direction.lm() if self.per_station: for src in sources: for p in stations: self.make_beam_nodes(Jones(src, p), self.beamshape(p), lmsrc[src.name], self.ell(p), self.dlm(p)) else: p0 = stations[0] for src in sources: self.make_beam_nodes(Jones(src, p0), self.beamshape(p0), lmsrc[src.name], self.ell(p0), self.dlm(p0)) for p in stations[1:]: Jones(src, p) << Meq.Identity(Jones(src, p0)) # make inspectors inspectors.append(ns.inspector << StdTrees.define_inspector( Jones, sources, stations, label=self.label)) return Jones
def init_parameters(self, ns, sources, stations, inspectors=[]): if self.beamshape is not None: return # create solvables if enabled parms = [] parmgroups = [] parmdef_scale = Meq.Parm(self.bf * 1e-9, tags="beam solvable") parmdef_ell = Meq.Parm(self.ellipticity, tags="beam solvable") self.per_station = False # solvable beam scale if self.solve_scale is PER_STATION: self.beamshape = ns.beamshape for p in stations: parms.append(beamshape(p) << parmdef_scale) inspectors.append( ns.inspector("scale") << StdTrees.define_inspector( ns.beamshape, stations, label=self.label)) self.per_station = True elif self.solve_scale is PER_ARRAY: parms.append(ns.beamshape << parmdef_scale) self.beamshape = lambda p: ns.beamshape else: ns.beamshape**(self.bf * 1e-9) self.beamshape = lambda p: ns.beamshape # solvable ellipticities if self.solve_ell is PER_STATION: self.ell = ns.ell for p in stations: ell_xy = ns.ell_xy(p) << parmdef_ell parms.append(ell_xy) self.ell(p) << Meq.Composer(ell_xy, -ell_xy) inspectors.append( ns.inspector("ellipticity") << StdTrees.define_inspector( ns.ell_xy, stations, label=self.label)) self.per_station = True elif self.solve_ell is PER_ARRAY: ell_xy = ns.ell_xy << parmdef_ell parms.append(ell_xy) ns.ell << Meq.Composer(ell_xy, -ell_xy) self.ell = lambda p: ns.ell elif self.ellipticity != 0: ns.ell**Meq.Constant([self.ellipticity, -self.ellipticity]) self.ell = lambda p: ns.ell else: self.ell = lambda p: None # make parm group, if any solvables have been created if parms: parmgroups.append(ParmGroup.Subgroup("beam shape", list(parms))) # solvable pointings if self.solve_pointings: self.dlm = ns.dlm parmdef_0 = Meq.Parm(0, tags="beam solvable") pparms = [] for p in stations: dl = ns.dl(p) << parmdef_0 dm = ns.dm(p) << parmdef_0 ns.dlm(p) << Meq.Composer(dl, dm) pparms += [dl, dm] parmgroups.append(ParmGroup.Subgroup("pointing offsets", pparms)) parms += pparms inspectors.append( ns.inspector('dlm') << StdTrees.define_inspector( ns.dlm, stations, label=self.label)) self.per_station = True else: ns.dlm_null**Meq.Constant([0, 0]) self.dlm = lambda p: ns.dlm_null # add solve jobs self.solvable = bool(parms) if self.solvable: self.pg_beam = ParmGroup.ParmGroup(self.label, parms, subgroups=parmgroups, table_name="%s.fmep" % self.label, bookmark=True) ParmGroup.SolveJob("cal_" + self.label, "Calibrate beam parameters", self.pg_beam)
def compute_jones (Jones,sources,stations=None,pointing_offsets=None,inspectors=[],label='E',**kw): stations = stations or Context.array.stations; ns = Jones.Subscope(); JE = Jones("elem"); per_station = sky_rotation or pointing_offsets; # read offsets file if read_offsets_file: offsets = list(map(float,open(offsets_file).read().split())); if len(offsets) < (beam_number+1)*2: raise ValueError("beam number %d not found in offsets file"%beam_number); l_offset,m_offset = offsets[beam_number*2:(beam_number+1)*2]; if invert_l: l_offset = -l_offset; else: l_offset,m_offset = 0,0; # loop over sources to create per-element beamgains for src in sources: # If sky rotation and/or pointing offsets are in effect, we have a per-station beam. # Otherwise the beam is the same for all stations. if per_station: for p in stations: lm = src.direction.lm(); # apply rotation to put sources into the antenna frame if sky_rotation: xyz = Context.array.xyz(p); pa_rot = Context.observation.phase_centre.pa_rot(xyz); lm = ns.lmrot(src,p) << Meq.MatrixMultiply(pa_rot,src.direction.lm()); # apply offset (so pointing offsets are interpreted in the azel frame, if rotating) if pointing_offsets: lm = ns.lmoff(src,p) << lm + pointing_offsets(p); # now make the beam node make_beam_node(JE(src,p),filename_pattern,l_offset,m_offset,lm); else: make_beam_node(JE(src),filename_pattern,l_offset,m_offset,src.direction.lm()); # now load weights wx = pickle.load(open(weight_filename_x, "rb")); wy = pickle.load(open(weight_filename_y, "rb")); if wx.shape[1] != num_elements or wy.shape[1] != num_elements: raise ValueError("""weights files contain weights for %d (X) and %d (Y) complex elements, %d expected"""%(wx.shape[1],wy.shape[1],num_elements)); if beam_number > wx.shape[0] or beam_number > wy.shape[0]: raise ValueError("beam number %d not found in weights files"%beam_number); # w0:x and w0:y are the nominal weight vectors ns.w0('x') << Meq.Constant(value=wx[beam_number,:]); ns.w0('y') << Meq.Constant(value=wy[beam_number,:]); if sim_element_errors: # create perturbed weights a0 = 10**(min_ampl_var/20)-1; a1 = 10**(max_ampl_var/20)-1; for p in stations: for xy in 'x','y': werr = ns.werr(xy,p); # amplitude and phase period and offset for ap in 'ampl','phase': p0 = werr("period",ap) << Meq.Constant(value=[random.uniform(min_period_var*3600,max_period_var*3600)/(2*math.pi) for i in range(num_elements)]); if start_phased_up: werr("sin",ap) << Meq.Sin(Meq.Time()/p0); else: t0 = werr("offset",ap) << Meq.Constant(value=[random.uniform(0,2*math.pi) for i in range(num_elements)]); werr("sin",ap) << Meq.Sin((Meq.Time()/p0)+t0); # amplitude excursion e0 = werr("maxampl") << Meq.Constant(value=[random.uniform(a0,a1) for i in range(num_elements)]); ep = werr("maxphase") << Meq.Constant(value=[random.uniform(min_phase_var*DEG,max_phase_var*DEG) for i in range(num_elements)]); # weight errors werr << Meq.Polar(1+e0*werr("sin","ampl"),ep*werr("sin","phase")); ns.weight(xy,p) << ns.w0(xy)*werr; # compute matrix norms based on nominal matrices if per_station: quallist = [ [src,p] for src in sources for p in stations ]; else: quallist = [ [src] for src in sources ]; for qq in quallist: ex0 = Jones(*(qq+["x0"])) << Meq.MatrixMultiply(JE(*qq),ns.w0("x")); ey0 = Jones(*(qq+["y0"])) << Meq.MatrixMultiply(JE(*qq),ns.w0("y")); J0 = Jones(*(qq+["nominal"])) << Meq.Composer(ex0,ey0,dims=[2,2]); if do_normalize or (qq[0] is sources[0] and not do_correct): make_norm(J0,Jones(*(qq+["norm"]))); if do_normalize: # norm towards source is average per-station norm if per_station: for src in sources: Jones(src,"norm") << Meq.Add(*[Jones(src,p,"norm") for p in stations])/len(stations); elif not do_correct: if per_station: for src in sources: Jones(sources[0],"norm") << Meq.Add(*[Jones(sources[0],p,"norm") for p in stations])/len(stations); for src in sources[1:]: Jones(src,"norm") << Meq.Identity(Jones(sources[0],"norm")); else: for src in sources: Jones(src,"norm") << 1; # put these together into Jones matrices for src in sources: # jesrc/JJ will eventually point to the unqualified element beam node # of the unqualified jones node. Depending on whether we're per-station or # not, this is qualified with src,p or just src. jesrc = JE(src); jnorm = Jones(src,"norm"); JJ = Jones(src); if sim_element_errors: for p in stations: if per_station: jesrc = JE(src,p); JJ = JJ(p); # jesrc returns a (2,N) matrix of element gains towards this source. # Multiply this by the weights to get the "x" and "y" beams ex = Jones(src,p,"x") << Meq.MatrixMultiply(jesrc,ns.weight("x",p)); ey = Jones(src,p,"y") << Meq.MatrixMultiply(jesrc,ns.weight("y",p)); if do_correct: J0 = Jones(src,p,"ref") << Meq.Composer(ex,ey,dims=[2,2])/jnorm; if src is sources[0]: Jones(src,p) << Meq.Constant(value=[1,0,0,1],dims=[2,2]); Jones(src,p,"inv") << Meq.MatrixInvert22(J0); else: Jones(src,p) << Meq.MatrixMultiply(Jones(sources[0],p,"inv"),J0); else: Jones(src,p) << Meq.Composer(ex,ey,dims=[2,2])/jnorm; # no element errors, use nominal beam else: if per_station: for p in stations: if do_correct: J0 = Jones(src,p,"ref") << Jones(src,p,"nominal")/jnorm; if src is sources[0]: Jones(src,p) << Meq.Constant(value=[1,0,0,1],dims=[2,2]); Jones(src,p,"inv") << Meq.MatrixInvert22(J0); else: Jones(src,p) << Meq.MatrixMultiply(Jones(sources[0],p,"inv"),J0); else: Jones(src,p) << Jones(src,p,"nominal")/jnorm; else: if do_correct: J0 = Jones(src,"ref") << Jones(src,"nominal")/jnorm; if src is sources[0]: Jones(src) << Meq.Constant(value=[1,0,0,1],dims=[2,2]); Jones(src,"inv") << Meq.MatrixInvert22(J0); else: Jones(src) << Meq.MatrixMultiply(Jones(sources[0],"inv"),J0); else: Jones(src) << Jones(src,"nominal")/jnorm; for p in stations: Jones(src,p) << Meq.Identity(Jones(src)); # declare an inspector node for the Jones matrix -- will be defined below insp = Jones.scope.inspector(label); # define inspectors insp << StdTrees.define_inspector(Jones,sources,stations,label=label); inspectors += [ insp ]; if sim_element_errors: insp1 = insp("werr") << StdTrees.define_inspector(ns.werr,("x","y"),stations,label="%s weight drifts"%label); insp2 = insp("weights") << StdTrees.define_inspector(ns.weight,("x","y"),stations,label="%s weights"%label); inspectors += [ insp1,insp2 ]; return Jones;
def compute_jones (Jones,sources,stations=None,pointing_offsets=None,inspectors=[],label='E',**kw): stations = stations or Context.array.stations; ns = Jones.Subscope(); JE = Jones("elem"); per_station = sky_rotation or pointing_offsets; # read offsets file if read_offsets_file: offsets = map(float,open(offsets_file).read().split()); if len(offsets) < (beam_number+1)*2: raise ValueError,"beam number %d not found in offsets file"%beam_number; l_offset,m_offset = offsets[beam_number*2:(beam_number+1)*2]; if invert_l: l_offset = -l_offset; else: l_offset,m_offset = 0,0; # loop over sources to create per-element beamgains for src in sources: # If sky rotation and/or pointing offsets are in effect, we have a per-station beam. # Otherwise the beam is the same for all stations. if per_station: for p in stations: lm = src.direction.lm(); # apply rotation to put sources into the antenna frame if sky_rotation: xyz = Context.array.xyz(p); pa_rot = Context.observation.phase_centre.pa_rot(xyz); lm = ns.lmrot(src,p) << Meq.MatrixMultiply(pa_rot,src.direction.lm()); # apply offset (so pointing offsets are interpreted in the azel frame, if rotating) if pointing_offsets: lm = ns.lmoff(src,p) << lm + pointing_offsets(p); # now make the beam node make_beam_node(JE(src,p),filename_pattern,l_offset,m_offset,lm); else: make_beam_node(JE(src),filename_pattern,l_offset,m_offset,src.direction.lm()); # now load weights wx = cPickle.load(open(weight_filename_x)); wy = cPickle.load(open(weight_filename_y)); if wx.shape[1] != num_elements or wy.shape[1] != num_elements: raise ValueError,"""weights files contain weights for %d (X) and %d (Y) complex elements, %d expected"""%(wx.shape[1],wy.shape[1],num_elements); if beam_number > wx.shape[0] or beam_number > wy.shape[0]: raise ValueError,"beam number %d not found in weights files"%beam_number; # w0:x and w0:y are the nominal weight vectors ns.w0('x') << Meq.Constant(value=wx[beam_number,:]); ns.w0('y') << Meq.Constant(value=wy[beam_number,:]); if sim_element_errors: # create perturbed weights a0 = 10**(min_ampl_var/20)-1; a1 = 10**(max_ampl_var/20)-1; for p in stations: for xy in 'x','y': werr = ns.werr(xy,p); # amplitude and phase period and offset for ap in 'ampl','phase': p0 = werr("period",ap) << Meq.Constant(value=[random.uniform(min_period_var*3600,max_period_var*3600)/(2*math.pi) for i in range(num_elements)]); if start_phased_up: werr("sin",ap) << Meq.Sin(Meq.Time()/p0); else: t0 = werr("offset",ap) << Meq.Constant(value=[random.uniform(0,2*math.pi) for i in range(num_elements)]); werr("sin",ap) << Meq.Sin((Meq.Time()/p0)+t0); # amplitude excursion e0 = werr("maxampl") << Meq.Constant(value=[random.uniform(a0,a1) for i in range(num_elements)]); ep = werr("maxphase") << Meq.Constant(value=[random.uniform(min_phase_var*DEG,max_phase_var*DEG) for i in range(num_elements)]); # weight errors werr << Meq.Polar(1+e0*werr("sin","ampl"),ep*werr("sin","phase")); ns.weight(xy,p) << ns.w0(xy)*werr; # compute matrix norms based on nominal matrices if per_station: quallist = [ [src,p] for src in sources for p in stations ]; else: quallist = [ [src] for src in sources ]; for qq in quallist: ex0 = Jones(*(qq+["x0"])) << Meq.MatrixMultiply(JE(*qq),ns.w0("x")); ey0 = Jones(*(qq+["y0"])) << Meq.MatrixMultiply(JE(*qq),ns.w0("y")); J0 = Jones(*(qq+["nominal"])) << Meq.Composer(ex0,ey0,dims=[2,2]); if do_normalize or (qq[0] is sources[0] and do_correct): make_norm(J0,Jones(*(qq+["norm"]))); if do_normalize: # norm towards source is average per-station norm if per_station: for src in sources: Jones(src,"norm") << Meq.Add(*[Jones(src,p,"norm") for p in stations])/len(stations); elif do_correct: if per_station: for src in sources: Jones(sources[0],"norm") << Meq.Add(*[Jones(sources[0],p,"norm") for p in stations])/len(stations); for src in sources[1:]: Jones(src,"norm") << Meq.Identity(Jones(sources[0],"norm")); else: for src in sources: Jones(src,"norm") << 1; # put these together into Jones matrices for src in sources: # jesrc/JJ will eventually point to the unqualified element beam node # of the unqualified jones node. Depending on whether we're per-station or # not, this is qualified with src,p or just src. jesrc = JE(src); jnorm = Jones(src,"norm"); JJ = Jones(src); if sim_element_errors: for p in stations: if per_station: jesrc = JE(src,p); JJ = JJ(p); # jesrc returns a (2,N) matrix of element gains towards this source. # Multiply this by the weights to get the "x" and "y" beams ex = Jones(src,p,"x") << Meq.MatrixMultiply(jesrc,ns.weight("x",p)); ey = Jones(src,p,"y") << Meq.MatrixMultiply(jesrc,ns.weight("y",p)); if do_correct: J0 = Jones(src,p,"ref") << Meq.Composer(ex,ey,dims=[2,2])/jnorm; if src is sources[0]: Jones(src,p) << Meq.Constant(value=[1,0,0,1],dims=[2,2]); Jones(src,p,"inv") << Meq.MatrixInvert22(J0); else: Jones(src,p) << Meq.MatrixMultiply(Jones(sources[0],p,"inv"),J0); else: Jones(src,p) << Meq.Composer(ex,ey,dims=[2,2])/jnorm; # no element errors, use nominal beam else: if per_station: for p in stations: if do_correct: J0 = Jones(src,p,"ref") << Jones(src,p,"nominal")/jnorm; if src is sources[0]: Jones(src,p) << Meq.Constant(value=[1,0,0,1],dims=[2,2]); Jones(src,p,"inv") << Meq.MatrixInvert22(J0); else: Jones(src,p) << Meq.MatrixMultiply(Jones(sources[0],p,"inv"),J0); else: Jones(src,p) << Jones(src,p,"nominal")/jnorm; else: if do_correct: J0 = Jones(src,"ref") << Jones(src,"nominal")/jnorm; if src is sources[0]: Jones(src) << Meq.Constant(value=[1,0,0,1],dims=[2,2]); Jones(src,"inv") << Meq.MatrixInvert22(J0); else: Jones(src) << Meq.MatrixMultiply(Jones(sources[0],"inv"),J0); else: Jones(src) << Jones(src,"nominal")/jnorm; for p in stations: Jones(src,p) << Meq.Identity(Jones(src)); # declare an inspector node for the Jones matrix -- will be defined below insp = Jones.scope.inspector(label); # define inspectors insp << StdTrees.define_inspector(Jones,sources,stations,label=label); inspectors += [ insp ]; if sim_element_errors: insp1 = insp("werr") << StdTrees.define_inspector(ns.werr,("x","y"),stations,label="%s weight drifts"%label); insp2 = insp("weights") << StdTrees.define_inspector(ns.weight,("x","y"),stations,label="%s weights"%label); inspectors += [ insp1,insp2 ]; return Jones;
def init_parameters (self,ns,sources,stations,inspectors=[]): if self.beamshape is not None: return; # create solvables if enabled parms = []; parmgroups = []; parmdef_scale = Meq.Parm(self.bf*1e-9,tags="beam solvable"); parmdef_ell = Meq.Parm(self.ellipticity,tags="beam solvable"); self.per_station = False; # solvable beam scale if self.solve_scale is PER_STATION: self.beamshape = ns.beamshape; for p in stations: parms.append(beamshape(p) << parmdef_scale); inspectors.append(ns.inspector("scale") << StdTrees.define_inspector(ns.beamshape,stations,label=self.label)); self.per_station = True; elif self.solve_scale is PER_ARRAY: parms.append(ns.beamshape << parmdef_scale); self.beamshape = lambda p:ns.beamshape; else: ns.beamshape ** (self.bf*1e-9); self.beamshape = lambda p:ns.beamshape; # solvable ellipticities if self.solve_ell is PER_STATION: self.ell = ns.ell; for p in stations: ell_xy = ns.ell_xy(p) << parmdef_ell; parms.append(ell_xy); self.ell(p) << Meq.Composer(ell_xy,-ell_xy); inspectors.append(ns.inspector("ellipticity") << StdTrees.define_inspector(ns.ell_xy,stations,label=self.label)); self.per_station = True; elif self.solve_ell is PER_ARRAY: ell_xy = ns.ell_xy << parmdef_ell; parms.append(ell_xy); ns.ell << Meq.Composer(ell_xy,-ell_xy); self.ell = lambda p:ns.ell; elif self.ellipticity != 0: ns.ell ** Meq.Constant([self.ellipticity,-self.ellipticity]); self.ell = lambda p:ns.ell; else: self.ell = lambda p:None; # make parm group, if any solvables have been created if parms: parmgroups.append(ParmGroup.Subgroup("beam shape",list(parms))); # solvable pointings if self.solve_pointings: self.dlm = ns.dlm; parmdef_0 = Meq.Parm(0,tags="beam solvable"); pparms = []; for p in stations: dl = ns.dl(p) << parmdef_0; dm = ns.dm(p) << parmdef_0; ns.dlm(p) << Meq.Composer(dl,dm); pparms += [dl,dm]; parmgroups.append(ParmGroup.Subgroup("pointing offsets",pparms)); parms += pparms; inspectors.append(ns.inspector('dlm') << StdTrees.define_inspector(ns.dlm,stations,label=self.label)); self.per_station = True; else: ns.dlm_null ** Meq.Constant([0,0]); self.dlm = lambda p:ns.dlm_null; # add solve jobs self.solvable = bool(parms); if self.solvable: self.pg_beam = ParmGroup.ParmGroup(self.label,parms,subgroups=parmgroups,table_name="%s.fmep"%self.label,bookmark=True); ParmGroup.SolveJob("cal_"+self.label,"Calibrate beam parameters",self.pg_beam);
def _define_forest(ns,parent=None,**kw): if not mssel.msname: raise RuntimeError,"MS not set"; if run_purr: Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']); # create Purr pipe global purrpipe; purrpipe = Purr.Pipe.Pipe(mssel.msname); # setup contexts from MS mssel.setup_observation_context(ns); array = Meow.Context.array; # make spigot nodes for data mssel.enable_input_column(True); spigots = array.spigots(corr=mssel.get_corr_index()); meqmaker.make_per_ifr_bookmarks(spigots,"Input visibilities"); # data tensor ns.DT << Meq.Composer(dims=[0],mt_polling=True,*[ spigots(p,q) for p,q in array.ifrs() ]); # predict tree using the MeqMaker all_sources = meqmaker.get_source_list(ns); dg_sources = deopts.enabled and dgsel.filter(all_sources); if dg_sources: # first group all sources without a diffgain on them groups = [ [ src for src in all_sources if not src in dg_sources ] ]; # group diffgain-enabled sources by grouping tag clusters = set([src.get_attr(diffgain_group,None) for src in dg_sources]); dg_groups = [ (name,[ src for src in dg_sources if src.get_attr(diffgain_group) == name ]) for name in clusters if name ]; # add sources without a grouping tag individually, as single-source groups dg_groups += [ (src.name,[src]) for src in dg_sources if not src.get_attr(diffgain_group,None) ]; # now sort by brightness flux_dgg = [ (sum([src.get_attr('Iapp',0) or src.get_attr('I') for src in dgg[1]]),dgg) for dgg in dg_groups ]; flux_dgg.sort(lambda a,b:cmp(b[0],a[0])); diffgain_labels = [ dgg[0] for flux,dgg in flux_dgg ]; groups += [ dgg[1] for flux,dgg in flux_dgg ]; num_diffgains = len(flux_dgg); # now make predict trees models = []; for i,group in enumerate(groups): MT = ns.MT(i); predict = meqmaker.make_predict_tree(MT.Subscope(),sources=group); ns.MT(i) << Meq.Composer(dims=[0],mt_polling=True,*[ predict(p,q) for p,q in array.ifrs() ]); models.append(ns.MT(i)); print "Number of diffgain predict groups:",len(groups); else: diffgain_labels = []; num_diffgains = 0; predict = meqmaker.make_predict_tree(ns); ns.MT << Meq.Composer(dims=[0],mt_polling=True,*[ predict(p,q) for p,q in array.ifrs() ]); models = [ ns.MT ]; solve_ifrs = array.subset(calibrate_ifrs,strict=False).ifrs(); downsample_subtiling = [ stefcal_downsample_timeint,stefcal_downsample_freqint ] if stefcal_downsample else [1,1]; import Calico.OMS.StefCal.StefCal kwopts = {} gopts.set_stefcal_node_options(kwopts,visualize=stefcal_visualize); bopts.set_stefcal_node_options(kwopts,visualize=stefcal_visualize); deopts.set_stefcal_node_options(kwopts,visualize=stefcal_visualize); ns.stefcal << Meq.PyNode(class_name="StefCalNode",module_name=Calico.OMS.StefCal.StefCal.__file__, ifrs=[ "%s:%s"%(p,q) for p,q in array.ifrs() ], baselines=[ array.baseline(ip,iq) for (ip,p),(iq,q) in array.ifr_index() ], solve_ifrs=[ "%s:%s"%(p,q) for p,q in solve_ifrs ], noise_per_chan=stefcal_noise_per_chan, downsample_subtiling=downsample_subtiling, num_major_loops=stefcal_nmajor, regularization_factor=1e-6,# rescale=stefcal_rescale, init_from_previous=False, critical_flag_threshold=critical_flag_threshold, diffgain_labels=diffgain_labels, # flagging options output_flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT, # IFR gain solution options apply_ifr_gains=stefcal_ifr_gains, solve_ifr_gains=(stefcal_ifr_gain_mode != MODE_SOLVE_APPLY), reset_ifr_gains=stefcal_ifr_gain_reset, save_ifr_gains=(stefcal_ifr_gain_mode == MODE_SOLVE_SAVE), ifr_gain_table=stefcal_ifr_gain_table, per_chan_ifr_gains=stefcal_per_chan_ifr_gains, diag_ifr_gains=(stefcal_diagonal_ifr_gains == DIAGONLY), residuals=(do_output == CORRECTED_RESIDUALS), subtract_dgsrc=(do_output == CORRECTED_DATA_SUB), verbose=stefcal_verbose, children=[ns.DT]+models,**kwopts); inspectors = meqmaker.get_inspectors() or []; # make output bookmarks nv = 0; for p,q in array.ifrs(): sel = ns.output_sel(p,q) << Meq.Selector(ns.stefcal,index=range(nv,nv+4),multi=True); ns.output(p,q) << Meq.Composer(sel,dims=[2,2]); nv += 4; meqmaker.make_per_ifr_bookmarks(ns.output,"Output visibilities"); Bookmarks.Page("StefCal outputs").add(ns.stefcal,viewer="Record Browser"); if gopts.enabled and gopts.visualize and stefcal_visualize: ns.stefcal_vis_G << Meq.PyNode(class_name="StefCalVisualizer",module_name=Calico.OMS.StefCal.StefCal.__file__, label="G",flag_unity=visualize_flag_unity,norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations); ns.stefcal_vis_G_avg << Meq.PyNode(class_name="StefCalVisualizer",module_name=Calico.OMS.StefCal.StefCal.__file__, label="G",freq_average=True,flag_unity=visualize_flag_unity,norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations); Bookmarks.Page("StefCal G plotter").add(ns.stefcal_vis_G,viewer="Result Plotter"); Bookmarks.Page("StefCal G inspector").add(ns.stefcal_vis_G_avg,viewer="Collections Plotter"); inspectors += [ ns.stefcal_vis_G,ns.stefcal_vis_G_avg ]; if bopts.enabled and bopts.visualize and stefcal_visualize: ns.stefcal_vis_B << Meq.PyNode(class_name="StefCalVisualizer",module_name=Calico.OMS.StefCal.StefCal.__file__, label="B",flag_unity=visualize_flag_unity,norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations); ns.stefcal_vis_B_avg << Meq.PyNode(class_name="StefCalVisualizer",module_name=Calico.OMS.StefCal.StefCal.__file__, label="B",freq_average=True,flag_unity=visualize_flag_unity,norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations); Bookmarks.Page("StefCal B plotter").add(ns.stefcal_vis_B,viewer="Result Plotter"); Bookmarks.Page("StefCal B inspector").add(ns.stefcal_vis_B_avg,viewer="Collections Plotter"); inspectors += [ ns.stefcal_vis_B,ns.stefcal_vis_B_avg ]; if deopts.enabled and deopts.visualize and stefcal_visualize: for i,label in enumerate(diffgain_labels): vde = ns.stefcal_vis_dE(label) << Meq.PyNode(class_name="StefCalVisualizer",module_name=Calico.OMS.StefCal.StefCal.__file__, label="dE:%s"%label,flag_unity=visualize_flag_unity,norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations); vde_avg = ns.stefcal_vis_dE_avg(label) << Meq.PyNode(class_name="StefCalVisualizer",module_name=Calico.OMS.StefCal.StefCal.__file__, label="dE:%s"%label,freq_average=True,flag_unity=visualize_flag_unity,norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations); Bookmarks.Page("StefCal dE:%s plotter"%label).add(vde,viewer="Result Plotter"); Bookmarks.Page("StefCal dE:%s inspector"%label).add(vde_avg,viewer="Collections Plotter"); inspectors += [ vde,vde_avg ]; # make sinks StdTrees.make_sinks(ns,ns.output,spigots=spigots,post=inspectors, corr_index=mssel.get_corr_index()); # this should come last, since runtime options may be built up during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)); # finally, setup imaging options imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size()); TDLRuntimeMenu("Make an image from this MS",*imsel.option_list()); # and close meqmaker -- this exports annotations, etc meqmaker.close(); # add options to clear all solutions from Calico.OMS.StefCal import StefCal TDLRuntimeOption("stefcal_reset_all","Remove all existing solutions",False); for opt in gopts,bopts,deopts: if opt.enabled: TDLRuntimeOption("reset","Remove existing %s solutions (%s)"%(opt.label,os.path.basename(opt.table)),False,namespace=opt); if stefcal_ifr_gains: TDLRuntimeOption("stefcal_reset_ifr_gains","Remove existing interferometer errors (%s)"%( os.path.basename(stefcal_ifr_gain_table)),False); TDLRuntimeJob(_run_stefcal,"Run StefCal",job_id="stefcal");
def _define_forest (ns, parent=None, **kw): if not mssel.msname: raise RuntimeError ('MS name not set') mssel.setup_observation_context (ns) array = Context.array # Data and model input if do_solve or output_type.need_data: mssel.enable_input_column (True) spigots = array.spigots (corr=mssel.get_corr_index ()) meqmaker.make_per_ifr_bookmarks (spigots, 'Input visibilities') else: mssel.enable_input_column (False) spigots = None if do_solve or output_type.need_model: predict = meqmaker.make_predict_tree (ns, uvdata=None) else: predict = None # Data output outputs = output_type.apply (ns, meqmaker, array.ifrs (), spigots, predict) # Flagging if flag_enable and output_type.flag_data: flaggers = [] if flag_res is not None or flag_mean_res is not None: for p, q in array.ifrs (): ns.absres(p,q) << Meq.Abs (outputs(p,q)) if flag_res is not None: for p, q in array.ifrs (): ns.flagres(p,q) << Meq.ZeroFlagger (ns.absres(p,q) - flag_res, oper='gt', flag_bit=MSUtils.FLAGMASK_OUTPUT) flaggers.append (ns.flagres) meqmaker.make_per_ifr_bookmarks (ns.flagres, 'Residual amplitude flags') if flag_mean_res is not None: ns.meanabsres << Meq.Mean (*[ns.absres(p,q) for p, q in array.ifrs()]) ns.flagmeanres << Meq.ZeroFlagger (ns.meanabsres - flag_mean_res, oper='gt', flag_bit=MSUtils.FLAGMASK_OUTPUT) Bookmarks.Page ('Mean residual amplitude flags').add (ns.flagmeanres, viewer='Result Plotter') flaggers.append (lambda p, q: ns.flagmeanres) if flaggers: meqmaker.make_per_ifr_bookmarks (outputs, output_type.desc + ' (unflagged)') for p, q in array.ifrs (): ns.flagged(p,q) << Meq.MergeFlags (outputs(p,q), *[f(p,q) for f in flaggers]) outputs = ns.flagged meqmaker.make_per_ifr_bookmarks (outputs, output_type.desc) # Solve trees if do_solve: # parse ifr specification solve_ifrs = array.subset (calibrate_ifrs, strict=False).ifrs() if not solve_ifrs: raise RuntimeError ('No interferometers selected for calibration. ' 'Check your ifr specification (under calibration options).') lhs, rhs, weights, modulo = cal_quant.apply (solve_ifrs, predict, spigots) solve_tree = StdTrees.SolveTree (ns, lhs, solve_ifrs=solve_ifrs, weights=weights, modulo=modulo) outputs = solve_tree.sequencers (inputs=rhs, outputs=outputs) StdTrees.make_sinks (ns, outputs, spigots=spigots, post=meqmaker.get_inspectors () or [], corr_index=mssel.get_corr_index ()) if not do_solve: name = 'Generate ' + output_type.desc.lower () comment = 'Generated ' + output_type.desc.lower () def run_tree (mqs, parent, wait=False, **kw): return mqs.execute (Context.vdm.name, mssel.create_io_request (tile_size), wait=wait) doc = """Input data are sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory.""" TDLRuntimeMenu(name, TDLOption ('tile_size', 'Tile size, in timeslots', [10, 60, 120, 240], more=int, doc=doc), TDLJob (run_tree, name, job_id='generate_visibilities')) # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up # during compilation. TDLRuntimeOptions (*meqmaker.runtime_options (nest=False)) if do_solve: TDLRuntimeOptions (*ParmGroup.get_solvejob_options ()) imsel = mssel.imaging_selector (npix=512, arcmin=meqmaker.estimate_image_size ()) TDLRuntimeMenu ('Make an image', *imsel.option_list ()) meqmaker.close()
def _define_forest(ns,parent=None,**kw): if run_purr: Timba.TDL.GUI.purr(mssel.msname+".purrlog",[mssel.msname,'.']); # create Purr pipe global purrpipe; purrpipe = Purr.Pipe.Pipe(mssel.msname); # get antennas from MS ANTENNAS = mssel.get_antenna_set(list(range(1,15))); array = Meow.IfrArray(ns,ANTENNAS,mirror_uvw=False); stas = array.stations(); # get phase centre from MS, setup observation observation = Meow.Observation(ns,phase_centre=mssel.get_phase_dir(), linear=mssel.is_linear_pol(), circular=mssel.is_circular_pol()); Meow.Context.set(array,observation); # get active correlations from MS Meow.Context.active_correlations = mssel.get_correlations(); # make spigot nodes spigots = spigots0 = outputs = array.spigots(corr=mssel.get_corr_index()); # ...and an inspector for them StdTrees.vis_inspector(ns.inspector('input'),spigots, bookmark="Inspect input visibilities"); inspectors = [ ns.inspector('input') ]; Bookmarks.make_node_folder("Input visibilities by baseline", [ spigots(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2); inspect_ifrs = array.ifrs(); if do_solve: # filter solvable baselines by baseline length solve_ifrs = []; antpos = mssel.ms_antenna_positions; if (min_baseline or max_baseline) and antpos is not None: for (ip,p),(iq,q) in array.ifr_index(): baseline = math.sqrt(((antpos[ip,:]-antpos[iq,:])**2).sum()); if (not min_baseline or baseline > min_baseline) and \ (not max_baseline or baseline < max_baseline): solve_ifrs.append((p,q)); else: solve_ifrs = array.ifrs(); inspect_ifrs = solve_ifrs; # make a predict tree using the MeqMaker if do_solve or do_subtract: predict = meqmaker.make_predict_tree(ns); # make a ParmGroup and solve jobs for source parameters, if we have any if do_solve: parms = {}; for src in meqmaker.get_source_list(ns): parms.update([(p.name,p) for p in src.get_solvables()]); if parms: pg_src = ParmGroup.ParmGroup("source",list(parms.values()), table_name="sources.fmep", individual=True,bookmark=True); # now make a solvejobs for the source ParmGroup.SolveJob("cal_source","Calibrate source model",pg_src); # make nodes to compute residuals if do_subtract: residuals = ns.residuals; for p,q in array.ifrs(): residuals(p,q) << spigots(p,q) - predict(p,q); outputs = residuals; # and now we may need to correct the outputs if do_correct: if do_correct_sky: srcs = meqmaker.get_source_list(ns); sky_correct = srcs and srcs[0]; else: sky_correct = None; outputs = meqmaker.correct_uv_data(ns,outputs,sky_correct=sky_correct,inspect_ifrs=inspect_ifrs); # make solve trees if do_solve: # inputs to the solver are based on calibration type # if calibrating visibilities, feed them to condeq directly if cal_type == CAL.VIS: observed = spigots; model = predict; # else take ampl/phase component else: model = ns.model; observed = ns.observed; if cal_type == CAL.AMPL: for p,q in array.ifrs(): observed(p,q) << Meq.Abs(spigots(p,q)); model(p,q) << Meq.Abs(predict(p,q)); elif cal_type == CAL.LOGAMPL: for p,q in array.ifrs(): observed(p,q) << Meq.Log(Meq.Abs(spigots(p,q))); model(p,q) << Meq.Log(Meq.Abs(predict(p,q))); elif cal_type == CAL.PHASE: for p,q in array.ifrs(): observed(p,q) << 0; model(p,q) << Meq.Abs(predict(p,q))*Meq.FMod(Meq.Arg(spigots(p,q))-Meq.Arg(predict(p,q)),2*math.pi); else: raise ValueError("unknown cal_type setting: "+str(cal_type)); # make a solve tree solve_tree = StdTrees.SolveTree(ns,model,solve_ifrs=solve_ifrs); # the output of the sequencer is either the residuals or the spigots, # according to what has been set above outputs = solve_tree.sequencers(inputs=observed,outputs=outputs); # make sinks and vdm. # The list of inspectors must be supplied here inspectors += meqmaker.get_inspectors() or []; StdTrees.make_sinks(ns,outputs,spigots=spigots0,post=inspectors); Bookmarks.make_node_folder("Corrected/residual visibilities by baseline", [ outputs(p,q) for p,q in array.ifrs() ],sorted=True,ncol=2,nrow=2); if not do_solve: if do_subtract: name = "Generate residuals"; comment = "Generated residual visibilities."; elif do_correct: name = "Generate corrected data"; comment = "Generated corrected visibilities."; else: name = None; if name: # make a TDL job to runsthe tree def run_tree (mqs,parent,**kw): global tile_size; purrpipe.title("Calibrating").comment(comment); mqs.execute(Meow.Context.vdm.name,mssel.create_io_request(tile_size),wait=False); TDLRuntimeMenu(name, TDLOption('tile_size',"Tile size, in timeslots",[10,60,120,240],more=int, doc="""Input data is sliced by time, and processed in chunks (tiles) of the indicated size. Larger tiles are faster, but use more memory."""), TDLRuntimeJob(run_tree,name) ); # very important -- insert meqmaker's runtime options properly # this should come last, since runtime options may be built up during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)); # insert solvejobs if do_solve: TDLRuntimeOptions(*ParmGroup.get_solvejob_options()); # finally, setup imaging options imsel = mssel.imaging_selector(npix=512,arcmin=meqmaker.estimate_image_size()); TDLRuntimeMenu("Make an image from this MS",*imsel.option_list()); # and close meqmaker -- this exports annotations, etc meqmaker.close();
def _define_forest(ns, parent=None, **kw): if not mssel.msname: raise RuntimeError, "MS not set" if run_purr: Timba.TDL.GUI.purr(mssel.msname + ".purrlog", [mssel.msname, '.']) # create Purr pipe global purrpipe purrpipe = Purr.Pipe.Pipe(mssel.msname) # setup contexts from MS mssel.setup_observation_context(ns) array = Meow.Context.array # make spigot nodes for data mssel.enable_input_column(True) spigots = array.spigots(corr=mssel.get_corr_index()) meqmaker.make_per_ifr_bookmarks(spigots, "Input visibilities") # data tensor ns.DT << Meq.Composer( dims=[0], mt_polling=True, *[spigots(p, q) for p, q in array.ifrs()]) # predict tree using the MeqMaker all_sources = meqmaker.get_source_list(ns) dg_sources = deopts.enabled and dgsel.filter(all_sources) if dg_sources: # first group all sources without a diffgain on them groups = [[src for src in all_sources if not src in dg_sources]] # group diffgain-enabled sources by grouping tag clusters = set( [src.get_attr(diffgain_group, None) for src in dg_sources]) dg_groups = [(name, [ src for src in dg_sources if src.get_attr(diffgain_group) == name ]) for name in clusters if name] # add sources without a grouping tag individually, as single-source groups dg_groups += [(src.name, [src]) for src in dg_sources if not src.get_attr(diffgain_group, None)] # now sort by brightness flux_dgg = [(sum( [src.get_attr('Iapp', 0) or src.get_attr('I') for src in dgg[1]]), dgg) for dgg in dg_groups] flux_dgg.sort(lambda a, b: cmp(b[0], a[0])) diffgain_labels = [dgg[0] for flux, dgg in flux_dgg] groups += [dgg[1] for flux, dgg in flux_dgg] num_diffgains = len(flux_dgg) # now make predict trees models = [] for i, group in enumerate(groups): MT = ns.MT(i) predict = meqmaker.make_predict_tree(MT.Subscope(), sources=group) ns.MT(i) << Meq.Composer(dims=[0], mt_polling=True, *[predict(p, q) for p, q in array.ifrs()]) models.append(ns.MT(i)) print "Number of diffgain predict groups:", len(groups) else: diffgain_labels = [] num_diffgains = 0 predict = meqmaker.make_predict_tree(ns) ns.MT << Meq.Composer(dims=[0], mt_polling=True, *[predict(p, q) for p, q in array.ifrs()]) models = [ns.MT] solve_ifrs = array.subset(calibrate_ifrs, strict=False).ifrs() downsample_subtiling = [ stefcal_downsample_timeint, stefcal_downsample_freqint ] if stefcal_downsample else [1, 1] import Calico.OMS.StefCal.StefCal kwopts = {} gopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize) bopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize) deopts.set_stefcal_node_options(kwopts, visualize=stefcal_visualize) ns.stefcal << Meq.PyNode( class_name="StefCalNode", module_name=Calico.OMS.StefCal.StefCal.__file__, ifrs=["%s:%s" % (p, q) for p, q in array.ifrs()], baselines=[ array.baseline(ip, iq) for (ip, p), (iq, q) in array.ifr_index() ], solve_ifrs=["%s:%s" % (p, q) for p, q in solve_ifrs], noise_per_chan=stefcal_noise_per_chan, downsample_subtiling=downsample_subtiling, num_major_loops=stefcal_nmajor, regularization_factor=1e-6, # rescale=stefcal_rescale, init_from_previous=False, critical_flag_threshold=critical_flag_threshold, diffgain_labels=diffgain_labels, # flagging options output_flag_bit=Meow.MSUtils.FLAGMASK_OUTPUT, # IFR gain solution options apply_ifr_gains=stefcal_ifr_gains, solve_ifr_gains=(stefcal_ifr_gain_mode != MODE_SOLVE_APPLY), reset_ifr_gains=stefcal_ifr_gain_reset, save_ifr_gains=(stefcal_ifr_gain_mode == MODE_SOLVE_SAVE), ifr_gain_table=stefcal_ifr_gain_table, per_chan_ifr_gains=stefcal_per_chan_ifr_gains, diag_ifr_gains=(stefcal_diagonal_ifr_gains == DIAGONLY), residuals=(do_output == CORRECTED_RESIDUALS), subtract_dgsrc=(do_output == CORRECTED_DATA_SUB), verbose=stefcal_verbose, children=[ns.DT] + models, **kwopts) inspectors = meqmaker.get_inspectors() or [] # make output bookmarks nv = 0 for p, q in array.ifrs(): sel = ns.output_sel(p, q) << Meq.Selector( ns.stefcal, index=range(nv, nv + 4), multi=True) ns.output(p, q) << Meq.Composer(sel, dims=[2, 2]) nv += 4 meqmaker.make_per_ifr_bookmarks(ns.output, "Output visibilities") Bookmarks.Page("StefCal outputs").add(ns.stefcal, viewer="Record Browser") if gopts.enabled and gopts.visualize and stefcal_visualize: ns.stefcal_vis_G << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="G", flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) ns.stefcal_vis_G_avg << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="G", freq_average=True, flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) Bookmarks.Page("StefCal G plotter").add(ns.stefcal_vis_G, viewer="Result Plotter") Bookmarks.Page("StefCal G inspector").add(ns.stefcal_vis_G_avg, viewer="Collections Plotter") inspectors += [ns.stefcal_vis_G, ns.stefcal_vis_G_avg] if bopts.enabled and bopts.visualize and stefcal_visualize: ns.stefcal_vis_B << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="B", flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) ns.stefcal_vis_B_avg << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="B", freq_average=True, flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) Bookmarks.Page("StefCal B plotter").add(ns.stefcal_vis_B, viewer="Result Plotter") Bookmarks.Page("StefCal B inspector").add(ns.stefcal_vis_B_avg, viewer="Collections Plotter") inspectors += [ns.stefcal_vis_B, ns.stefcal_vis_B_avg] if deopts.enabled and deopts.visualize and stefcal_visualize: for i, label in enumerate(diffgain_labels): vde = ns.stefcal_vis_dE(label) << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="dE:%s" % label, flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) vde_avg = ns.stefcal_vis_dE_avg(label) << Meq.PyNode( class_name="StefCalVisualizer", module_name=Calico.OMS.StefCal.StefCal.__file__, label="dE:%s" % label, freq_average=True, flag_unity=visualize_flag_unity, norm_offdiag=visualize_norm_offdiag, vells_label=Context.correlations) Bookmarks.Page("StefCal dE:%s plotter" % label).add( vde, viewer="Result Plotter") Bookmarks.Page("StefCal dE:%s inspector" % label).add( vde_avg, viewer="Collections Plotter") inspectors += [vde, vde_avg] # make sinks StdTrees.make_sinks(ns, ns.output, spigots=spigots, post=inspectors, corr_index=mssel.get_corr_index()) # this should come last, since runtime options may be built up during compilation. TDLRuntimeOptions(*meqmaker.runtime_options(nest=False)) # finally, setup imaging options imsel = mssel.imaging_selector(npix=512, arcmin=meqmaker.estimate_image_size()) TDLRuntimeMenu("Make an image from this MS", *imsel.option_list()) # and close meqmaker -- this exports annotations, etc meqmaker.close() # add options to clear all solutions from Calico.OMS.StefCal import StefCal TDLRuntimeOption("stefcal_reset_all", "Remove all existing solutions", False) for opt in gopts, bopts, deopts: if opt.enabled: TDLRuntimeOption("reset", "Remove existing %s solutions (%s)" % (opt.label, os.path.basename(opt.table)), False, namespace=opt) if stefcal_ifr_gains: TDLRuntimeOption( "stefcal_reset_ifr_gains", "Remove existing interferometer errors (%s)" % (os.path.basename(stefcal_ifr_gain_table)), False) TDLRuntimeJob(_run_stefcal, "Run StefCal", job_id="stefcal")