def compute_vis_noise(sefd): """Computes nominal per-visibility noise""" sefd = sefd or SEFD tab = ms.ms() spwtab = ms.ms(subtable="SPECTRAL_WINDOW") freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID, 0] wavelength = 300e+6 / freq0 bw = spwtab.getcol("CHAN_WIDTH")[ms.SPWID, 0] dt = tab.getcol("EXPOSURE", 0, 1)[0] dtf = (tab.getcol("TIME", tab.nrows() - 1, 1) - tab.getcol("TIME", 0, 1))[0] # close tables properly, else the calls below will hang waiting for a lock... tab.close() spwtab.close() info( ">>> $MS freq %.2f MHz (lambda=%.2fm), bandwidth %.2g kHz, %.2fs integrations, %.2fh synthesis" % (freq0 * 1e-6, wavelength, bw * 1e-3, dt, dtf / 3600)) noise = sefd / math.sqrt(2 * bw * dt) info(">>> SEFD of %.2f Jy gives per-visibility noise of %.2f mJy" % (sefd, noise * 1000)) return noise
def compute_vis_noise (noise=0,sefd=SEFD): """Computes nominal per-visibility noise""" tab = ms.ms(); spwtab = ms.ms(subtable="SPECTRAL_WINDOW"); global BASEFREQ BASEFREQ = freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0]; global WAVELENGTH WAVELENGTH = 300e+6/freq0 bw = BANDWIDTH or spwtab.getcol("CHAN_WIDTH")[ms.SPWID,0]; dt = INTEGRATION or tab.getcol("EXPOSURE",0,1)[0]; dtf = (tab.getcol("TIME",tab.nrows()-1,1)-tab.getcol("TIME",0,1))[0] # close tables properly, else the calls below will hang waiting for a lock... tab.close(); spwtab.close(); info(">>> $MS freq %.2f MHz (lambda=%.2fm), bandwidth %.2g kHz, %.2fs integrations, %.2fh synthesis"%(freq0*1e-6,WAVELENGTH,bw*1e-3,dt,dtf/3600)); _writestat("freq0",freq0); _writestat("wavelength",WAVELENGTH); _writestat("bw",bw); _writestat("synthesis_time",dtf); _writestat("integration",dt); if not noise: noise = sefd/math.sqrt(2*bw*dt); info(">>> SEFD of %.2f Jy gives per-visibility noise of %.2f mJy"%(sefd,noise*1000)); else: info(">>> using per-visibility noise of %.2f mJy"%(noise*1000)); _writestat("vis_noise",noise); return noise;
def image_noise(sigma): """ Estimates RMS in noise in naturally weighted image """ tab = ms.ms() data = tab.getcol('DATA') sigma_vis = sigma N_vis = np.product(data.shape[:2]) return sigma_vis/math.sqrt(N_vis)
def image_noise(sigma): """ Estimates RMS in noise in naturally weighted image """ tab = ms.ms() data = tab.getcol('DATA') sigma_vis = sigma N_vis = np.product(data.shape[:2]) return sigma_vis / math.sqrt(N_vis)
def get_time_slots(): t = ms.ms() A1 = t.getcol("ANTENNA1") A2 = t.getcol("ANTENNA2") uvw=t.getcol("UVW") temp = uvw[(A1==0)&(A2==1),0] return(len(temp))
def compute_vis_noise (noise=0): tab = ms.ms().query("FIELD_ID==%d"%ms.FIELD); spwtab = ms.ms(subtable="SPECTRAL_WINDOW"); freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0]; global WAVELENGTH WAVELENGTH = 300e+6/freq0 bw = spwtab.getcol("CHAN_WIDTH")[ms.SPWID,0]; dt = INTEGRATION or tab.getcol("EXPOSURE",0,1)[0]; dtf = (tab.getcol("TIME",tab.nrows()-1,1)-tab.getcol("TIME",0,1))[0] # close tables properly, else the calls below will hang waiting for a lock... tab.close(); spwtab.close(); info(">>> $MS freq %.2f MHz (lambda=%.2fm), bandwidth %.2g kHz, %.2fs integrations, %.2fh synthesis"%(freq0*1e-6,WAVELENGTH,bw*1e-3,dt,dtf/3600)); if not noise: noise = SEFD/math.sqrt(2*bw*dt); info(">>> SEFD of %.2f Jy gives per-visibility noise of %.2f mJy"%(SEFD,noise*1000)); else: info(">>> using per-visibility noise of %.2f mJy"%(noise*1000)); return noise;
def set_defaults(msname='$MS', psf=False): """ Extract some basic information about the data""" tab = ms.ms() uvmax = max(tab.getcol("UVW")[:2].sum(0)**2) res = 1.22 * ((2.998e8 / freq) / maxuv) / 2. im.cellsize = "%farcsec" % (numpy.rad2deg(res / 6) * 3600) im.stokes = 'I' # make a brightness map by default im.npix = 2048 im.mode = 'channel' im.IMAGE_CHANNELIZE = 0
def set_defaults(msname='$MS',psf=False): """ Extract some basic information about the data""" tab = ms.ms() uvmax = max( tab.getcol("UVW")[:2].sum(0)**2 ) res = 1.22*((2.998e8/freq)/maxuv)/2. im.cellsize = "%farcsec"%(numpy.rad2deg(res/6)*3600) im.stokes = 'I' # make a brightness map by default im.npix = 2048 im.mode = 'channel' im.IMAGE_CHANNELIZE = 0
def flag_stepped_timeslot (step=3): """Flags every Nth timeslot""" nant = ms.ms(subtable="ANTENNA").nrows(); tab = ms.msw(); nb = nant*(nant+1)/2 frow = tab.getcol("FLAG_ROW"); nr = len(frow); nt = len(frow)/nb; info("$MS has $nr rows, $nant antennas, $nb baselines and $nt timeslots, flagging every $step timeslots"); frow = frow.reshape([nt,nb]); frow[::step,:] = True; tab.putcol("FLAG_ROW",frow.reshape((nr,)));
def compute_vis_noise (sefd): """Computes nominal per-visibility noise""" #sefd = sefd or SEFD tab = ms.ms() spwtab = ms.ms(subtable="SPECTRAL_WINDOW") freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0] wavelength = 300e+6/freq0 bw = spwtab.getcol("CHAN_WIDTH")[ms.SPWID,0] dt = tab.getcol("EXPOSURE",0,1)[0] dtf = (tab.getcol("TIME",tab.nrows()-1,1)-tab.getcol("TIME",0,1))[0] # close tables properly, else the calls below will hang waiting for a lock... tab.close() spwtab.close() info(">>> $MS freq %.2f MHz (lambda=%.2fm), bandwidth %.2g kHz, %.2fs integrations, %.2fh synthesis"%(freq0*1e-6,wavelength,bw*1e-3,dt,dtf/3600)) noise = sefd/math.sqrt(2*bw*dt) info(">>> SEFD of %.2f Jy gives per-visibility noise of %.2f mJy"%(sefd,noise*1000)) return noise
def compute_vis_noise(noise=0): tab = ms.ms().query("FIELD_ID==%d" % ms.FIELD) spwtab = ms.ms(subtable="SPECTRAL_WINDOW") freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID, 0] global WAVELENGTH WAVELENGTH = 300e+6 / freq0 bw = spwtab.getcol("CHAN_WIDTH")[ms.SPWID, 0] dt = INTEGRATION or tab.getcol("EXPOSURE", 0, 1)[0] dtf = (tab.getcol("TIME", tab.nrows() - 1, 1) - tab.getcol("TIME", 0, 1))[0] # close tables properly, else the calls below will hang waiting for a lock... tab.close() spwtab.close() info( ">>> $MS freq %.2f MHz (lambda=%.2fm), bandwidth %.2g kHz, %.2fs integrations, %.2fh synthesis" % (freq0 * 1e-6, WAVELENGTH, bw * 1e-3, dt, dtf / 3600)) if not noise: noise = SEFD / math.sqrt(2 * bw * dt) info(">>> SEFD of %.2f Jy gives per-visibility noise of %.2f mJy" % (SEFD, noise * 1000)) else: info(">>> using per-visibility noise of %.2f mJy" % (noise * 1000)) return noise
def simnoise (noise=0,rowchunk=100000,skipnoise=False,addToCol=None,scale_noise=1.0,column='MODEL_DATA'): conf = MS.split('_')[0] spwtab = ms.ms(subtable="SPECTRAL_WINDOW") freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0]/1e6 tab = ms.msw() dshape = list(tab.getcol('DATA').shape) nrows = dshape[0] noise = noise or compute_vis_noise() if addToCol: colData = tab.getcol(addToCol) for row0 in range(0,nrows,rowchunk): nr = min(rowchunk,nrows-row0) dshape[0] = nr data = noise*(numpy.random.randn(*dshape) + 1j*numpy.random.randn(*dshape)) * scale_noise if addToCol: data+=colData[row0:(row0+nr)] info(" $addToCol + noise --> CORRECTED_DATA (rows $row0 to %d)"%(row0+nr-1)) column = 'CORRECTED_DATA' else : info("Adding noise to $column (rows $row0 to %d)"%(row0+nr-1)) tab.putcol(column,data,row0,nr); tab.close()
def simcube (cube,nchan=None,npix=4096,cellsize=".5arcsec",niter=100000,padding=1.5,threshold=".2mJy",predict=True,dirty=False,restore=False,noise=0,resume=False,label=None,column='DATA',channelize=1,wprojplanes=0): if label: v.LABEL = label tab = ms.ms(subtable='SPECTRAL_WINDOW') nchan = nchan or tab.getcol('NUM_CHAN')[0] ms.CHANRANGE = 0,nchan-1; imager.wprojplanes = wprojplanes; imager.IMAGE_CHANNELIZE = 1; if predict: imager.predict_vis(image=cube,padding=padding,copy=False,column=column); if noise > 0: simnoise(addToCol=column,noise=noise) if dirty: info('Weights are%s'%WEIGHTS) for weight in WEIGHTS.split(':'): opts,weight_txt,quals = get_weight_opts(weight) restore.update(opts) dirty_image = II("${OUTFILE}-$weight.dirty.fits") model_image = II("${OUTFILE}-$weight.model.fits") residual_image = II("${OUTFILE}-$weight.residual.fits") restored_image = II("${OUTFILE}-$weight.restored.fits") imager.make_image(dirty=dirty,restore=restore,channelize=channelize,column=column if restore==False else "CORRECTED_DATA",dirty_image=dirty_image,model_image=model_image,residual_image=residual_image,restored_image=restored_image,**opts);
def azishe(config='$CFG'): # Get parameters from json file with open(II(config)) as jsn_std: jparams = json.load(jsn_std) # Remove empty strings and convert unicode characters to strings params = {} for key, val in jparams.iteritems(): # Make sure all keys are strings _key = str(key) # ignore empty strings and comments if val == "" or _key == "#": pass # convert unicode values to strings elif isinstance(val, unicode): params[_key] = str(val).lower() else: params[_key] = val get_opts = lambda prefix: filter(lambda a: a[0].startswith(prefix), params.items()) # Retrieve MS and imager options ms_dict = dict([(key.split('ms_')[-1], val) for (key, val) in get_opts('ms_')]) im_dict = dict([(key.split('im_')[-1], val) for (key, val) in get_opts('im_')]) # Seperate deconvolution settings _deconv = {} for dcv in 'lwimager wsclean moresane casa'.split(): if params[dcv]: _deconv.update({ dcv: dict([(key.split(dcv + '_')[-1], val) for (key, val) in get_opts(dcv + '_')]) }) # Set imaging options im.IMAGER = imager.IMAGER = params['imager'].lower() for opt in 'npix weight robust mode stokes'.split(): if opt in im_dict.keys(): setattr(im, opt, im_dict.pop(opt)) im.cellsize = '%farcsec' % (im_dict.pop('cellsize')) im.stokes = im.stokes.upper() weight_fov = im_dict.pop('weight_fov') if weight_fov: im.weight_fov = '%farcmin' % weight_fov # Create empty MS synthesis = ms_dict.pop('synthesis') scalenoise = 1 if synthesis > 12: scalenoise = math.sqrt(12.0 / synthesis) msname = II('rodrigues%f.MS' % (time.time())) obs = params['observatory'].lower() antennas = _OBS[obs] freq0 = ms_dict.pop('freq0') * 1e6 dfreq = ms_dict.pop('dfreq') * 1e3 direction = "J2000,%s,%s" % (ms_dict.pop('ra'), ms_dict.pop('dec')) ms.create_empty_ms(msname=msname, synthesis=synthesis, freq0=freq0, dfreq=dfreq, tel=obs, pos='%s/%s' % (OBSDIR, antennas), direction=direction, **ms_dict) if exists(msname): v.MS = msname else: abort( "Something went wrong while creating the empty MS. Please check logs for details" ) makedir(DESTDIR) ms.plot_uvcov(ms=.1, width=10, height=10, dpi=150, save="$OUTFILE-uvcov.png") # Set noise for simulation spwtab = ms.ms(subtable="SPECTRAL_WINDOW") freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID, 0] spwtab.close() if params['add_noise']: noise = compute_vis_noise(sefd=get_sefd(freq0)) * scalenoise else: noise = 0 # Simulate Visibilities _katalog = params['katalog_id'] if _katalog: katalog = '%s/%s' % (KATDIR, _KATALOG[params['katalog_id']]) lsmname = II('${OUTDIR>/}${MS:BASE}.lsm.html') radius = float(params['radius']) fluxrange = params['fluxrange'].split('-') if len(fluxrange) > 1: fluxrange = map(float, fluxrange) elif len(fluxrange) == 1: fluxrange = [0, float(fluxrange[0])] select = '' fits = verify_sky(katalog) == 'FITS' if radius or fluxrange: if radius: select += '--select="r<%fdeg" ' % radius if fluxrange: select += '--select="I<%f" ' % fluxrange[1] select += '--select="I>%f" ' % fluxrange[0] if not fits: x.sh( 'tigger-convert $select --recenter=$direction $katalog $lsmname -f' ) else: from pyrap.measures import measures dm = measures() direction = dm.direction('J2000', ms_opts[ra], ms_opts[dec]) ra = np.rad2deg(direction['m0']['value']) dec = np.rad2deg(direction['m1']['value']) hdu = pyfits.open(temp_file) hdu[0].hdr['CRVAL1'] = ra hdu[0].hdr['CRVAL2'] = dec hdu.writeto(tmp_file, clobber=True) simsky(lsmname=lsmname, tdlsec='turbo-sim:custom', noise=noise, column='CORRECTED_DATA') skymodel = params['sky_model'] if skymodel: simsky(lsmname=skymodel, noise=0 if katalog else noise, addToCol='CORRECTED_DATA') ## Finally Lets image # make dirty map im.make_image(psf=True, **im_dict) # Deconvolve for deconv in _deconv: deconv = deconv.lower() if deconv in STAND_ALONE_DECONV: im.make_image(algorithm=deconv, restore=_deconv[deconv], restore_lsm=False, **im_dict) else: im.IMAGER = deconv im.make_image(dirty=False, restore=_deconv[deconv], restore_lsm=False, **im_dict) xo.sh('tar -czvf ${OUTDIR>/}${MS:BASE}.tar.gz $msname')
#!/usr/bin/env python from rlist import randList from ms import ms from array_printer import print_array with open('../size.txt','r') as f: size = int(f.readline().strip()) pre_sort = randList(size) post_sort = ms(pre_sort) print_array(pre_sort) print_array(post_sort)
def make_ifrgain_plots(filename="$STEFCAL_DIFFGAIN_SAVE", prefix="IG", feed="$IFRGAIN_PLOT_FEED", msname="$MS"): """Makes a set of ifrgain plots from the specified saved file.""" import pylab from Timba.Meq import meq filename, prefix, msname = interpolate_locals("filename prefix msname") global __IFRGAIN_PREFIX, _IFRGAIN_TYPE, _IFRGAIN_COMP _IFRGAIN_PREFIX = prefix dir = os.path.dirname(IFRGAIN_PLOT) if dir: makedir(dir) # load baseline info, if MS is available baseline = None if msname: try: anttab = ms.ms(msname, "ANTENNA") antpos = anttab.getcol("POSITION") antname = anttab.getcol("NAME") antpos = [(name, antpos[i]) for i, name in enumerate(antname)] # make dict of p,q to baseline length baseline = dict([("%s-%s" % (p, q), math.sqrt(((ppos - qpos)**2).sum())) for p, ppos in antpos for q, qpos in antpos]) except: traceback.print_exc() warn( "can't access $msname antenna table, so IFR gain versus baseline length plots will not be made" ) else: warn( "MS not specified, thus no antenna info. IFR gain versus baseline length plots will not be made" ) # feed labels feeds = ("RR", "LL", "RL", "LR") if FEED_TYPE.upper() == "RL" else ("XX", "YY", "XY", "YX") ig = pickle.load(open(filename, "rb")) def plot_xy(content, title): """Plots x vs y""" pylab.errorbar([x for l, (x, xe), (y, ye) in content], [y for l, (x, xe), (y, ye) in content], [ye for l, (x, xe), (y, ye) in content], [xe for l, (x, xe), (y, ye) in content], fmt=None, ecolor="lightgrey") # plot labels for label, (x, xe), (y, ye) in content: pylab.text(x, y, label, horizontalalignment='center', verticalalignment='center', size=8) pylab.title(title) def plot_baseline(content, baseline, title, feeds): """Plots offset versus baseline""" bl = [] xx = [] xxe = [] lab = [] col = [] for l, (x, xe), (y, ye) in content: b = baseline.get(l, None) if b is None: warn("baseline $l not found in MS ANTENNA table") else: lab += ["%s:%s" % (l, feeds[0]), "%s:%s" % (l, feeds[1])] col += ["blue", "red"] xx += [x, y] xxe += [xe, ye] bl += [b, b] pylab.axhline(1, color='lightgrey') pylab.errorbar(bl, xx, yerr=xxe, fmt=None, ecolor="lightgrey") # plot labels for l, x, y, c in zip(lab, bl, xx, col): pylab.text(x, y, l, horizontalalignment='center', verticalalignment='center', size=8, color=c) pylab.xlabel("Baseline, m.") pylab.title(title) def plot_hist(content, title): """Plots histogram""" values = [x for l, (x, xe), (y, ye) in content ] + [y for l, (x, xe), (y, ye) in content] hist = pylab.hist(values) pylab.xlim(min(values), max(values)) pylab.title(title) def plot_complex(content, title): """Plots x vs y""" # plot errors bars, if available pylab.axhline(0, color='lightgrey') pylab.axvline(1, color='lightgrey') pylab.errorbar([x.real for l1, l2, (x, xe), (y, ye) in content], [x.imag for l1, l2, (x, xe), (y, ye) in content], [xe for l1, l2, (x, xe), (y, ye) in content], [xe for l1, l2, (x, xe), (y, ye) in content], fmt=None, ecolor="lightgrey") pylab.errorbar([y.real for l1, l2, (x, xe), (y, ye) in content], [y.imag for l1, l2, (x, xe), (y, ye) in content], [ye for l1, l2, (x, xe), (y, ye) in content], [ye for l1, l2, (x, xe), (y, ye) in content], fmt=None, ecolor="lightgrey") # max plot amplitude -- max point plus 1/4th of the error bar maxa = max( [max(abs(x), abs(y)) for l1, l2, (x, xe), (y, ye) in content]) # plotlim = max([ abs(numpy.array([ # getattr(v,attr)+sign*e/4 for v,e in (x,xe),(y,ye) for attr in 'real','imag' for sign in 1,-1 # ])).max() # for l1,l2,(x,xe),(y,ye) in content ]); minre, maxre, minim, maxim = 2, -2, 2, -2 for l1, l2, (x, xe), (y, ye) in content: offs = numpy.array([ getattr(v, attr) + sign * e / 4 for v, e in (x, xe), (y, ye) for attr in 'real', 'imag' for sign in 1, -1 ]) minre, maxre = min(x.real - xe / 4, y.real - ye / 4, minre), max(x.real + xe / 4, y.real + ye / 4, maxre) minim, maxim = min(x.imag - xe / 4, y.imag - ye / 4, minim), max(x.imag + xe / 4, y.imag + ye / 4, maxim)
def get_data(msfile='$MS',src_name='$SRC',out_data_set_tag='$TAG'): dataSetFolder = II('%s'%out_data_set_tag) msname = II('%s'%msfile) srcFolder = II('%s'%src_name) x.sh("mkdir -p ./%s"%srcFolder) x.sh("mkdir -p ./%s/%s"%(srcFolder,dataSetFolder)) tab = ms.ms(msname,write=False); info("columns are",*(tab.colnames())); uvw = tab.getcol("UVW") spwtab = ms.ms(subtable="SPECTRAL_WINDOW") freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0]#get the first freq wavelength = LightSpeed/freq0 freqStepVect = spwtab.getcol("CHAN_WIDTH") freqsVect=spwtab.getcol("CHAN_FREQ") info(freqsVect) #Specs nchan = len(freqsVect[0,:]); bandwidth = spwtab.getcol("CHAN_WIDTH")[ms.SPWID,0] spwtab.close() field = tab.getcol("FIELD_ID"); ant1=tab.getcol("ANTENNA1"); ant2=tab.getcol("ANTENNA2"); time = tab.getcol("TIME"); scantab = tab.getcol("SCAN_NUMBER") integrationTimeVect = tab.getcol("EXPOSURE"); dt = tab.getcol("EXPOSURE",0,1)[0] dtf = (tab.getcol("TIME",tab.nrows()-1,1)-tab.getcol("TIME",0,1))[0] info(">>>> Freq %.2f MHz (lambda=%.2fm), bandwidth %.2g kHz, %.2fs integrations, %.2fh synthesis"%(freq0*1e-6,wavelength,bandwidth*1e-3,dt,dtf/3600)) sio.savemat("./%s/%s/msSpecs.mat"%(srcFolder,dataSetFolder),{'field':field,'ant1':ant1.astype(float),'ant2':ant2.astype(float),'freqsVect':freqsVect,'time':time,'uvw':uvw,'scans':scantab}) #Data weightfull = tab.getcol("WEIGHT_SPECTRUM"); flag_row = tab.getcol("FLAG_ROW") flagAll= tab.getcol("FLAG") data_full = tab.getcol("DATA"); for ifreq in range(0, nchan): data_selected = data_full[:,ifreq,:]; weight_selected = weightfull[:,ifreq,:]; weightI= (weight_selected[:,0]+weight_selected[:,3]) dataI_w = (weight_selected[:,0]*data_selected[:,0]+weight_selected[:,3]*data_selected[:,3])/weightI ; flag_bis = numpy.logical_or(flagAll[:,ifreq,0],flagAll[:,ifreq,3]); flag = numpy.logical_or(flag_bis,flag_row) flag = flag.astype(float) sio.savemat("./%s/%s/data.mat"%(srcFolder,dataSetFolder),{'y_I':dataI_w, 'weights':weightI.astype(float), 'flag':flag}); info("Read data .. writing file..Freq %s"%(ifreq)) tab.close()
def get_field_center(): t = ms.ms(subtable="FIELD") phase_centre = (t.getcol("PHASE_DIR"))[0,0,:] t.close() return phase_centre[0], phase_centre[1] #ra0,dec0 in radians
def azishe(): jdict = readJson(CONFIG) add = jdict.get("add_to_column", False) addnoise = jdict.get("addnoise", False) sefd = jdict.get("sefd", 500) v.MS = "{:s}/{:s}".format(MSDIR, jdict["msname"]) v.LOG = II("${OUTDIR>/}log-${MS:BASE}-simulation.txt") column = jdict.get("column", "DATA") copy = jdict.get("copy_to_CORRECTED_DATA", False) newcol = jdict.get("custom_data_column", None) if newcol: im.argo.addcol(newcol) if jdict["skymodel"] in [None, False] and addnoise: noise = compute_vis_noise(sefd) simnoise(noise, column=column) if copy and column!="CORRECTED_DATA": ms.copycol(fromcol=column, tocol="CORRECTED_DATA") return for item in ["/input/", "/data/skymodels/"]: lsmname = "{:s}/{:s}".format(item, jdict["skymodel"]) if os.path.exists(lsmname): break v.LSM = II("${lsmname:FILE}") std.copy(lsmname, LSM) if jdict.get("recenter", False): direction = jdict.get("direction", False) if direction: ftab = ms.ms(subtable="FIELD") ra,dec = ftab.getcol("PHASE_DIR")[jdict.get("field_id",0)][0] direction = "J2000,%frad,%frad"%(ra,dec) x.sh("tigger-convert --recenter=$direction $LSM -f") options = {} if addnoise: noise = compute_vis_noise(sefd) options["noise_stddev"] = noise beam = jdict.get("Ejones", False) if beam: options["me.e_enable"] = 1 options["me.p_enable"] = 1 options["me.e_module"] = "Siamese_OMS_pybeams_fits" options["me.e_advanced"] = 1 options["me.e_all_stations"] = 1 options["pybeams_fits.l_axis"] = jdict.get("beam_l_axis", "L") options["pybeams_fits.m_axis"] = jdict.get("beam_m_axis", "M") options["pybeams_fits.filename_pattern"] = "%s/%s"%(INDIR, jdict["beam_files_pattern"]) rms_perr = jdict.get("pointing_accuracy", 0) # Include pointing errors if needed if rms_perr: anttab = ms.ms(subtable="ANTENNA") NANT = anttab.nrows() options["me.epe_enable"] = 1 perr = numpy.random.randn(NANT)*rms_perr, numpy.random.randn(NANT)*rms_perr ll, mm = " ".join( map(str, perr[0]) ), " ".join( map(str, perr[-1]) ) options['oms_pointing_errors.pe_l.values_str'] = "'%s'"%ll options['oms_pointing_errors.pe_m.values_str'] = "'%s'"%mm _section = dict(sim = "sim", add_G = "sim:G") if jdict.get("gjones", False): section = "add_G" else: section = "sim" mode = '"add to MS"' if add else '"sim only"' options["sim_mode"] = mode options["ms_sel.input_column"] = add options["ms_sel.output_column"] = column mqt.msrun(II("${mqt.CATTERY}/Siamese/turbo-sim.py"), job = '_tdl_job_1_simulate_MS', section = _section[section], options = options, args = ["${lsm.LSM_TDL}"]) if copy and column!="CORRECTED_DATA": ms.copycol(fromcol=column, tocol="CORRECTED_DATA")
def azishe(config='$CFG'): # Get parameters from json file with open(II(config)) as jsn_std: jparams = json.load(jsn_std) # Remove empty strings and convert unicode characters to strings params = {} for key, val in jparams.iteritems(): # Make sure all keys are strings _key = str(key) # ignore empty strings and comments if val=="" or _key=="#": pass # convert unicode values to strings elif isinstance(val,unicode): params[_key] = str(val).lower() else: params[_key] = val get_opts = lambda prefix: filter(lambda a: a[0].startswith(prefix), params.items()) # Retrieve MS and imager options ms_dict = dict([(key.split('ms_')[-1],val) for (key,val) in get_opts('ms_') ] ) im_dict = dict([(key.split('im_')[-1],val) for (key,val) in get_opts('im_') ] ) # Seperate deconvolution settings _deconv = {} for dcv in 'lwimager wsclean moresane casa'.split(): if params[dcv]: _deconv.update( {dcv:dict([(key.split(dcv+'_')[-1],val) for (key,val) in get_opts(dcv+'_') ] )} ) # Set imaging options im.IMAGER = imager.IMAGER = params['imager'].lower() for opt in 'npix weight robust mode stokes'.split(): if opt in im_dict.keys(): setattr(im,opt,im_dict.pop(opt)) im.cellsize = '%farcsec'%(im_dict.pop('cellsize')) im.stokes = im.stokes.upper() weight_fov = im_dict.pop('weight_fov') if weight_fov: im.weight_fov = '%farcmin'%weight_fov # Create empty MS synthesis = ms_dict.pop('synthesis') scalenoise = 1 if synthesis > 12: scalenoise = math.sqrt(12.0/synthesis) msname = II('rodrigues%f.MS'%(time.time())) obs = params['observatory'].lower() antennas = _OBS[obs] freq0 = ms_dict.pop('freq0')*1e6 dfreq = ms_dict.pop('dfreq')*1e3 direction = "J2000,%s,%s"%(ms_dict.pop('ra'),ms_dict.pop('dec')) ms.create_empty_ms(msname=msname,synthesis=synthesis,freq0=freq0, dfreq=dfreq,tel=obs,pos='%s/%s'%(OBSDIR,antennas), direction=direction,**ms_dict) if exists(msname): v.MS = msname else: abort("Something went wrong while creating the empty MS. Please check logs for details") makedir(DESTDIR) ms.plot_uvcov(ms=.1,width=10,height=10,dpi=150,save="$OUTFILE-uvcov.png") # Set noise for simulation spwtab = ms.ms(subtable="SPECTRAL_WINDOW") freq0 = spwtab.getcol("CHAN_FREQ")[ms.SPWID,0] spwtab.close() if params['add_noise']: noise = compute_vis_noise(sefd=get_sefd(freq0)) * scalenoise else: noise = 0 # Simulate Visibilities _katalog = params['katalog_id'] if _katalog: katalog = '%s/%s'%(KATDIR,_KATALOG[params['katalog_id']]) lsmname = II('${OUTDIR>/}${MS:BASE}.lsm.html') radius = float(params['radius']) fluxrange = params['fluxrange'].split('-') if len(fluxrange)>1: fluxrange = map(float,fluxrange) elif len(fluxrange)==1: fluxrange = [0,float(fluxrange[0])] select = '' fits = verify_sky(katalog) == 'FITS' if radius or fluxrange: if radius: select += '--select="r<%fdeg" '%radius if fluxrange: select += '--select="I<%f" '%fluxrange[1] select += '--select="I>%f" '%fluxrange[0] if not fits: x.sh('tigger-convert $select --recenter=$direction $katalog $lsmname -f') else: from pyrap.measures import measures dm = measures() direction = dm.direction('J2000',ms_opts[ra],ms_opts[dec]) ra = np.rad2deg(direction['m0']['value']) dec = np.rad2deg(direction['m1']['value']) hdu = pyfits.open(temp_file) hdu[0].hdr['CRVAL1'] = ra hdu[0].hdr['CRVAL2'] = dec hdu.writeto(tmp_file,clobber=True) simsky(lsmname=lsmname,tdlsec='turbo-sim:custom',noise=noise,column='CORRECTED_DATA') skymodel = params['sky_model'] if skymodel: simsky(lsmname=skymodel,noise=0 if katalog else noise,addToCol='CORRECTED_DATA') ## Finally Lets image # make dirty map im.make_image(psf=True,**im_dict) # Deconvolve for deconv in _deconv: deconv = deconv.lower() if deconv in STAND_ALONE_DECONV: im.make_image(algorithm=deconv,restore=_deconv[deconv],restore_lsm=False,**im_dict) else: im.IMAGER = deconv im.make_image(dirty=False,restore=_deconv[deconv],restore_lsm=False,**im_dict) xo.sh('tar -czvf ${OUTDIR>/}${MS:BASE}.tar.gz $msname')
def make_ifrgain_plots (filename="$STEFCAL_DIFFGAIN_SAVE",prefix="IG",feed="$IFRGAIN_PLOT_FEED",msname="$MS"): """Makes a set of ifrgain plots from the specified saved file.""" import pylab from Timba.Meq import meq filename,prefix,msname = interpolate_locals("filename prefix msname"); global __IFRGAIN_PREFIX,_IFRGAIN_TYPE,_IFRGAIN_COMP; _IFRGAIN_PREFIX = prefix; dir = os.path.dirname(IFRGAIN_PLOT); if dir: makedir(dir) # load baseline info, if MS is available baseline = None; if msname: try: anttab = ms.ms(msname,"ANTENNA"); antpos = anttab.getcol("POSITION"); antname = anttab.getcol("NAME"); antpos = [ (name,antpos[i]) for i,name in enumerate(antname) ]; # make dict of p,q to baseline length baseline = dict([ ("%s-%s"%(p,q),math.sqrt(((ppos-qpos)**2).sum())) for p,ppos in antpos for q,qpos in antpos ]) except: traceback.print_exc(); warn("can't access $msname antenna table, so IFR gain versus baseline length plots will not be made"); else: warn("MS not specified, thus no antenna info. IFR gain versus baseline length plots will not be made"); # feed labels feeds = ("RR","LL","RL","LR") if FEED_TYPE.upper() == "RL" else ("XX","YY","XY","YX"); ig = cPickle.load(file(filename)) def plot_xy (content,title): """Plots x vs y"""; pylab.errorbar( [x for l,(x,xe),(y,ye) in content],[y for l,(x,xe),(y,ye) in content], [ye for l,(x,xe),(y,ye) in content],[xe for l,(x,xe),(y,ye) in content], fmt=None,ecolor="lightgrey" ); # plot labels for label,(x,xe),(y,ye) in content: pylab.text(x,y,label,horizontalalignment='center',verticalalignment='center',size=8) pylab.title(title) def plot_baseline (content,baseline,title,feeds): """Plots offset versus baseline"""; bl = []; xx = []; xxe = []; lab = []; col = []; for l,(x,xe),(y,ye) in content: b = baseline.get(l,None); if b is None: warn("baseline $l not found in MS ANTENNA table") else: lab += [ "%s:%s"%(l,feeds[0]),"%s:%s"%(l,feeds[1]) ]; col += [ "blue","red" ] xx += [x,y]; xxe += [xe,ye]; bl += [b,b]; pylab.axhline(1,color='lightgrey') pylab.errorbar(bl,xx,yerr=xxe,fmt=None,ecolor="lightgrey"); # plot labels for l,x,y,c in zip(lab,bl,xx,col): pylab.text(x,y,l,horizontalalignment='center',verticalalignment='center',size=8,color=c) pylab.xlabel("Baseline, m."); pylab.title(title) def plot_hist (content,title): """Plots histogram"""; values = [x for l,(x,xe),(y,ye) in content] + [y for l,(x,xe),(y,ye) in content]; hist = pylab.hist(values); pylab.xlim(min(values),max(values)); pylab.title(title) def plot_complex (content,title): """Plots x vs y"""; # plot errors bars, if available pylab.axhline(0,color='lightgrey') pylab.axvline(1,color='lightgrey') pylab.errorbar( [x.real for l1,l2,(x,xe),(y,ye) in content],[x.imag for l1,l2,(x,xe),(y,ye) in content], [xe for l1,l2,(x,xe),(y,ye) in content],[xe for l1,l2,(x,xe),(y,ye) in content], fmt=None,ecolor="lightgrey" ); pylab.errorbar( [y.real for l1,l2,(x,xe),(y,ye) in content],[y.imag for l1,l2,(x,xe),(y,ye) in content], [ye for l1,l2,(x,xe),(y,ye) in content],[ye for l1,l2,(x,xe),(y,ye) in content], fmt=None,ecolor="lightgrey" ); # max plot amplitude -- max point plus 1/4th of the error bar maxa = max([ max(abs(x),abs(y)) for l1,l2,(x,xe),(y,ye) in content ]); # plotlim = max([ abs(numpy.array([ # getattr(v,attr)+sign*e/4 for v,e in (x,xe),(y,ye) for attr in 'real','imag' for sign in 1,-1 # ])).max() # for l1,l2,(x,xe),(y,ye) in content ]); minre, maxre, minim, maxim = 2, -2, 2, -2 for l1,l2,(x,xe),(y,ye) in content: offs = numpy.array([ getattr(v,attr)+sign*e/4 for v,e in (x,xe),(y,ye) for attr in 'real','imag' for sign in 1,-1 ]) minre, maxre = min(x.real-xe/4, y.real-ye/4, minre), max(x.real+xe/4, y.real+ye/4, maxre) minim, maxim = min(x.imag-xe/4, y.imag-ye/4, minim), max(x.imag+xe/4, y.imag+ye/4, maxim)
def residual(): t = ms.ms(write=True) corr_data = t.getcol("CORRECTED_DATA") data = t.getcol("DATA") t.putcol("CORRECTED_DATA",corr_data-data) t.close()