def copy_column_from_bands(mslist, ms_to, inputcol, outputcol): """ Copies one column from multiple MS files (bands) to a single MS file Note: the bands are assumed to be ordered by frequency, with a nonexisting file (e.g., 'dummy.ms') denoting missing bands Parameters ---------- mslist : list MS files to copy from ms_to : str MS file receiving copy inputcol : str Column name to copy from outputcol : str Column name to copy to """ dataout = pt.table(ms_to, readonly=False) data = dataout.getcol(outputcol, nrow=1) numberofchans = numpy.int(numpy.shape(data)[1]) chanperms = numberofchans/numpy.int(len(mslist)) for ms_id, ms in enumerate(mslist): if os.path.isdir(ms): datain = pt.table(ms, readonly=True) data = datain.getcol(inputcol) dataout.putcolslice(outputcol, data, [chanperms*ms_id,0], [(chanperms*(ms_id+1))-1,3]) datain.close() dataout.flush() dataout.close()
def copy_column_to_bands(mslist, ms_from, inputcol, outputcol): """ Copies one column from an MS file to multiple MS files (bands) Parameters ---------- mslist : list MS files receiving copy ms_from : str MS file to copy from inputcol : str Column name to copy from outputcol : str Column name to copy to """ datain = pt.table(ms_from) data = datain.getcol(inputcol, nrow=1) numberofchans = numpy.int(numpy.shape(data)[1]) chanperms = numberofchans/numpy.int(len(mslist)) for ms_id, ms in enumerate(mslist): if os.path.isdir(ms): data = datain.getcolslice(inputcol, [chanperms*ms_id,0], [(chanperms*(ms_id+1))-1,3]) dataout = pt.table(ms, readonly=False) dataout.putcol(outputcol, data) dataout.flush() dataout.close()
def addextraweights(msfiles): ''' Adds the column WEIGHT_SPECTRUM_FROM_IMAGING_WEIGHT from IMAGING_WEIGHT from DR2 Input msfiles (list of ms) ''' for ms in msfiles: ts = pt.table(ms, readonly=False) colnames = ts.colnames() if 'WEIGHT_SPECTRUM_FROM_IMAGING_WEIGHT' not in colnames: desc = ts.getcoldesc('WEIGHT_SPECTRUM') desc['name']='WEIGHT_SPECTRUM_FROM_IMAGING_WEIGHT' ts.addcols(desc) ts.close() # to write results else: print 'WEIGHT_SPECTRUM_FROM_IMAGING_WEIGHT already exists' ts.close() ts = pt.table(ms, readonly=False) if 'IMAGING_WEIGHT' in colnames: iw = ts.getcol('IMAGING_WEIGHT') ws_tmp = ts.getcol('WEIGHT_SPECTRUM_FROM_IMAGING_WEIGHT') n, nfreq, npol = np.shape(ws_tmp) for i in range(npol): print 'Copying over correlation ', i, ms ws_tmp[:,:,i] = iw ts.putcol('WEIGHT_SPECTRUM_FROM_IMAGING_WEIGHT',ws_tmp) else: print 'IMAGING_WEIGHT column is not present in:', ms ts.close() return
def get_baseline_lengths(ms): """ Returns dict of baseline lengths in km for all baselines in input dataset """ anttab = pt.table(ms+'::ANTENNA', ack=False) antnames = anttab.getcol('NAME') anttab.close() t = pt.table(ms, ack=False) ant1 = t.getcol('ANTENNA1') ant2 = t.getcol('ANTENNA2') all_uvw = t.getcol('UVW') t.close() baseline_dict = {} for ant in itertools.product(set(ant1), set(ant2)): if ant[0] >= ant[1]: continue sel1 = np.where(ant1 == ant[0])[0] sel2 = np.where(ant2 == ant[1])[0] sel = sorted(list(frozenset(sel1).intersection(sel2))) uvw = all_uvw[sel, :] uvw_dist = np.sqrt(uvw[:, 0]**2 + uvw[:, 1]**2 + uvw[:, 2]**2) baseline_dict['{0}'.format(ant[0])] = antnames[ant[0]] baseline_dict['{0}'.format(ant[1])] = antnames[ant[1]] baseline_dict['{0}-{1}'.format(ant[0], ant[1])] = np.mean(uvw_dist) / 1.e3 return baseline_dict
def copy_column_to_ms(ms, inputcol, outputcol, ms_from=None): """ Copies one column to another, within an MS file or between two MS files Parameters ---------- ms : str MS file receiving copy inputcol : str Column name to copy from outputcol : str Column name to copy to ms_from : str, optional MS file to copy from. If None, the column is copied internally """ t = pt.table(ms, readonly=False, ack=False) if ms_from is not None: tf = pt.table(ms_from, readonly=False, ack=False) data = tf.getcol(inputcol) desc = tf.getcoldesc(inputcol) else: data = t.getcol(inputcol) desc = t.getcoldesc(inputcol) # Add the output column if needed if outputcol not in t.colnames(): desc['name'] = outputcol t.addcols(desc) t.putcol(outputcol, data) t.flush() t.close()
def copy_column_to_ms(ms, inputcol, outputcol, ms_from=None, use_compression=False): """ Copies one column to another, within an MS file or between two MS files Parameters ---------- ms : str MS file receiving copy inputcol : str Column name to copy from outputcol : str Column name to copy to ms_from : str, optional MS file to copy from. If None, the column is copied internally """ t = pt.table(ms, readonly=False, ack=False) if ms_from is not None: tf = pt.table(ms_from, readonly=False, ack=False) data = tf.getcol(inputcol) desc = tf.getcoldesc(inputcol) else: data = t.getcol(inputcol) desc = t.getcoldesc(inputcol) # Add the output column if needed if outputcol not in t.colnames(): if use_compression: # Set DyscoStMan to be storage manager for DATA and WEIGHT_SPECTRUM # We use a visibility bit rate of 16 and truncation of 1.5 sigma to keep the # compression noise below ~ 0.01 mJy, as estimated from Fig 4 of # Offringa (2016). For the weights, we use a bit rate of 12, as # recommended in Sec 4.4 of Offringa (2016) desc['name'] = outputcol dmi = { 'SPEC': { 'dataBitCount': numpy.uint32(16), 'distribution': 'TruncatedGaussian', 'distributionTruncation': 1.5, 'normalization': 'RF', 'weightBitCount': numpy.uint32(12)}, 'NAME': '{}_dm'.format(outputcol), 'SEQNR': 1, 'TYPE': 'DyscoStMan'} desc['option'] = 1 # make a Direct column t.addcols(desc, dmi) else: desc['name'] = outputcol t.addcols(desc) if use_compression: # Replace flagged values with NaNs before compression flags = t.getcol('FLAG') flagged = numpy.where(flags) data[flagged] = numpy.NaN t.putcol(outputcol, data) t.flush() t.close()
def readMS(fn, sbs, column='DATA'): """Return the visibilites and UVW coordinates from a SE607 LOFAR XST format file fn: XST filename column: string, data column sbs: 1-D array of subband IDs returns: vis: visibilities [4, Nsamples, Nsubbands] uvw: UVW coordinates [Nsamples, 3, Nsubbands] freqs: frequencies [Nsubbands] obsdata: [latitude, longitude, LST] """ try: import casacore.tables as tbls except ImportError: print 'ERROR: could not import casacore.tables, cannot read measurement sets' exit(1) MS = tbls.table(fn, readonly=True) data_column = column.upper() uvw = MS.col('UVW').getcol() # [vis id, (u,v,w)] vis = MS.col(data_column).getcol() #[vis id, freq id, stokes id] vis = vis[:,sbs,:] #select subbands MS.close() # lat/long/lst information ANTS = tbls.table(fn + '/ANTENNA') positions = ANTS.col('POSITION').getcol() ant0Lat, ant0Long, ant0hgt = ecef.ecef2geodetic(positions[0,0], positions[0,1], positions[0,2], degrees=False) # use the first antenna in the table to get the array lat/long ANTS.close() SRC = tbls.table(fn + '/SOURCE') direction = SRC.col('DIRECTION').getcol() obsLat = direction[0,1] obsLong = ant0Long LSTangle = direction[0,0] SRC.close() # freq information, convert uvw coordinates SW = tbls.table(fn + '/SPECTRAL_WINDOW') freqs = SW.col('CHAN_FREQ').getcol()[0, sbs] # [nchan] print 'SUBBANDS:', sbs, '(', freqs/1e6, 'MHz)' SW.close() # TODO: check rotation reference is the same as with LOFAR data, north pole is dec=+90, ra=0 # in order to accommodate multiple observations at different times/sidereal times all the positions need to be rotated relative to sidereal time 0 print 'LST:', LSTangle rotAngle = float(LSTangle) - obsLong # adjust LST to that of the Observatory longitutude to make the LST that at Greenwich # to be honest, the next two lines change the LST to make the images come out but i haven't worked out the coordinate transforms, so for now these work without justification rotAngle += np.pi rotAngle *= -1 # Rotation matrix for antenna positions rotMatrix = np.array([[np.cos(rotAngle), -1.*np.sin(rotAngle), 0.], [np.sin(rotAngle), np.cos(rotAngle), 0.], [0., 0., 1.]]) #rotate about the z-axis uvwRot = np.dot(uvw, rotMatrix).reshape(uvw.shape[0], uvw.shape[1], 1) uvwRotRepeat = np.repeat(uvwRot, len(sbs), axis=2) return np.transpose(vis, (2,0,1)), uvwRotRepeat, freqs, [obsLat, obsLong, LSTangle]
def test_numpy_unicode(self): table_path = join(self.workdir, 'blah.ms') col1 = makescacoldesc('mycol1', 'test', valuetype='string') col2 = makescacoldesc('mycol2', 'test', valuetype='string') t = table(table_path, maketabdesc([col1, col2]), ack=False) t.addrows(2) t.putcol('mycol1', np.array([unicode_string, unicode_string])) t.putcol('mycol2', [unicode_string, unicode_string]) t.close() t = table(table_path) self.assertEqual(t.getcol('mycol1'), t.getcol('mycol2'))
def RotateMS(self,radec): import ModRotate ModRotate.Rotate(self,radec) ta=table(self.MSName+'/FIELD/',ack=False,readonly=False) ra,dec=radec radec=np.array([[[ra,dec]]]) ta.putcol("DELAY_DIR",radec) ta.putcol("PHASE_DIR",radec) ta.putcol("REFERENCE_DIR",radec) ta.close() t=table(self.MSName,ack=False,readonly=False) t.putcol(self.ColName,self.data) t.putcol("UVW",self.uvw) t.close()
def test_subtables(self): """Testing subtables.""" c1 = makescacoldesc("coli", 0) c2 = makescacoldesc("cold", 0.) c3 = makescacoldesc("cols", "") c4 = makescacoldesc("colb", True) c5 = makescacoldesc("colc", 0. + 0j) c6 = makearrcoldesc("colarr", 0.) t = table("ttable.py_tmp.tab1", maketabdesc((c1, c2, c3, c4, c5, c6)), ack=False) sub = table("sub", maketabdesc((c1, c2, c3))) t.putkeyword("subtablename", sub, makesubrecord=True) print(t.getsubtables()) t.close() tabledelete("ttable.py_tmp.tab1")
def test_tableinfo(self): """Test table info.""" c1 = makescacoldesc("coli", 0) c2 = makescacoldesc("cold", 0.) c3 = makescacoldesc("cols", "") c4 = makescacoldesc("colb", True) c5 = makescacoldesc("colc", 0. + 0j) c6 = makearrcoldesc("colarr", 0.) t = table("ttable.py_tmp.tab1", maketabdesc((c1, c2, c3, c4, c5, c6)), ack=False) self.assertTrue(tableexists("ttable.py_tmp.tab1")) self.assertTrue(tableiswritable("ttable.py_tmp.tab1")) self.assertEqual(t.nrows(), 0) self.assertEqual(t.ncols(), 6) self.assertTrue(compare(t.colnames(), ['cols', 'colc', 'coli', 'cold', 'colb', 'colarr'])) self.assertEqual(tableinfo("ttable.py_tmp.tab1"), {'readme': '', 'subType': '', 'type': ''}) t.addreadmeline("test table run") t.putinfo({'type': 'test', 'subType': 'test1'}) self.assertEqual(t.info()['readme'], 'test table run\n') self.assertEqual(t.info()['subType'], 'test1') self.assertEqual(t.info()['type'], 'test') self.assertEqual(len(t), 0) print(str(t)) self.assertEqual(t.endianformat(), 'little') t.close() tabledelete("ttable.py_tmp.tab1")
def PutBackupCol(self,back="CORRECTED_DATA"): backname="%s_BACKUP"%back backnameFlag="FLAG_BACKUP" self.PutCasaCols() t=table(self.MSName,readonly=False,ack=False) JustAdded=False if not(backname in t.colnames()): print " Putting column ",backname," in MS" desc=t.getcoldesc("CORRECTED_DATA") desc["name"]=backname desc['comment']=desc['comment'].replace(" ","_") t.addcols(desc) print " Copying CORRECTED_DATA in CORRECTED_DATA_BACKUP" self.CopyCol("CORRECTED_DATA",backname) #t.putcol(backname,t.getcol("CORRECTED_DATA")) if not(backnameFlag in t.colnames()): desc=t.getcoldesc("FLAG") desc["name"]=backnameFlag desc['comment']=desc['comment'].replace(" ","_") t.addcols(desc) self.CopyCol("FLAG",backnameFlag) #t.putcol(backnameFlag,t.getcol("FLAG")) JustAdded=True #else: #print " Column %s already there..."%backname t.close() return JustAdded
def main(ms_file, column1, column2): """ Switch the names of two columns Parameters ---------- ms_file : str Name of MS file column1 : str Name of column 1 column2 : str Name of column 2 """ t = pt.table(ms_file, readonly=False, ack=False) if column1 == column2: return if column1 not in t.colnames() and column2 not in t.colnames(): print('Both columns must be present in MS') sys.exit(1) # Rename column1 to temp col t.renamecol(column1, column1+'_TEMP') # Rename column2 to column1 t.renamecol(column2, column1) # Rename temp col to column2 t.renamecol(column1+'_TEMP', column2) t.flush() t.close()
def __init__(self, ms_file): """ ms_file: a MeasurementSet file """ logging.info('Reading: %s' % ms_file) self.ms_file = ms_file self.ms = table(ms_files[0], readonly=False, ack=False)
def main(ms_file): """ Switch the names of normal and baseline-averaged columns Parameters ---------- ms_file : str Name of MS file """ t = pt.table(ms_file, readonly=False, ack=False) for (column1, column2) in zip(['DATA', 'WEIGHT_SPECTRUM'], ['BLAVG_DATA', 'BLAVG_WEIGHT_SPECTRUM']): if column1 not in t.colnames() and column2 not in t.colnames(): print('Both columns must be present in MS') sys.exit(1) # Rename column1 to temp col t.renamecol(column1, column1+'_TEMP') # Rename column2 to column1 t.renamecol(column2, column1) # Rename temp col to column2 t.renamecol(column1+'_TEMP', column2) t.flush() t.close()
def main(fitsimage, outfilename, force_stokes_i=False): """ Convert a fits image to a CASA image Parameters ---------- fitsimage : str Name of FITS image outfilename : str Name of output CASA image force_stokes_i : bool, optional If True, force Stokes axis to be 'I' """ casaimage = pim.image(fitsimage) casaimage.saveas(outfilename, overwrite=True) if type(force_stokes_i) is str: if force_stokes_i.lower() == 'true': force_stokes_i = True else: force_stokes_i = False if force_stokes_i: coords = casaimage.coordinates().dict() coords['stokes1']['stokes'] = ['I'] freq = coords['spectral2']['wcs']['crval'] coords['spectral2']['restfreqs'] = np.array([freq]) outtable = pt.table(outfilename, readonly=False, ack=False) outtable.putkeywords({'coords': coords}) outtable.done()
def removecolumn(msfile,colname): t = pt.table(msfile,readonly=False) colnames =t.colnames() if colname in colnames: print 'Removing ', colname, 'from ', msfile t.removecols(colname) t.close() return
def columnchecker(mslist, colname): for ms in mslist: t = pt.table(ms, ack=False) if colname not in t.colnames(): print colname, ' not present in ', ms sys.exit() t.close()
def PutNewCol(self,Name,LikeCol="CORRECTED_DATA"): if not(Name in self.ColNames): print " Putting column %s in MS, with format of %s"%(Name,LikeCol) t=table(self.MSName,readonly=False,ack=False) desc=t.getcoldesc(LikeCol) desc["name"]=Name t.addcols(desc) t.close()
def ZeroFlagSave(self,spw=0): self.flag_all.fill(0) if self.swapped: flagout=np.swapaxes(self.flag_all[spw*self.Nchan:(spw+1)*self.Nchan],0,1) else: flagout=self.flag_all t=table(self.MSName,readonly=False,ack=False) t.putcol("FLAG",flagout) t.close()
def SaveVis(self,vis=None,Col="CORRECTED_DATA",spw=0,DoPrint=True): if vis==None: vis=self.data if DoPrint: print " Writting data in column %s"%ModColor.Str(Col,col="green") table_all=table(self.MSName,ack=False,readonly=False) if self.swapped: visout=np.swapaxes(vis[spw*self.Nchan:(spw+1)*self.Nchan],0,1) else: visout=vis table_all.putcol(Col,visout,self.ROW0,self.nRowRead) table_all.close()
def SaveAllDataStruct(self): t=table(self.MSName,ack=False,readonly=False) t.putcol('ANTENNA1',self.A0) t.putcol('ANTENNA2',self.A1) t.putcol("TIME",self.times_all) t.putcol("TIME_CENTROID",self.times_all) t.putcol("UVW",self.uvw) t.putcol("FLAG",self.flag_all) for icol in range(len(self.ColName)): t.putcol(self.ColName[icol],self.data[icol]) t.close()
def LoadSR(self): if self.SR!=None: return import lofar.stationresponse as lsr f=self.ChanFreq.flatten() if f.shape[0]>1: t=table(self.MSName+"/SPECTRAL_WINDOW/",readonly=False) c=t.getcol("CHAN_WIDTH") c.fill(np.abs((f[0:-1]-f[1::])[0])) t.putcol("CHAN_WIDTH",c) t.close() self.SR = lsr.stationresponse(self.MSName) self.SR.setDirection(self.rarad,self.decrad)
def get_uvw(vis_ms_path): """ Extract uvw data as squeezed numpy array. Units of metres. Args: vis_ms_path (str): Path to visibility MeasurementSet Returns (numpy.array): uvw data. Array of floats, shape: (n_baseline_samples, 3) """ with closing(casatables.table(vis_ms_path)) as tbl: uvw_metres = tbl.getcol('UVW') return uvw_metres
def get_stokes_i_vis(vis_ms_path): """ Export 'CORRECTED_DATA' Stokes I complex visibilities to numpy-array Args: vis_ms_path (str): Path to visibility MeasurementSet Returns (numpy.array): visibility data. Array of complex, shape: (n_baseline_samples,) """ with closing(casatables.table(vis_ms_path)) as tbl: stokes_i = tbl.getcol('CORRECTED_DATA').squeeze()[:, 0] return stokes_i
def Restore(self): backname="CORRECTED_DATA_BACKUP" backnameFlag="FLAG_BACKUP" t=table(self.MSName,readonly=False,ack=False) if backname in t.colnames(): print " Copying ",backname," to CORRECTED_DATA" #t.putcol("CORRECTED_DATA",t.getcol(backname)) self.CopyCol(backname,"CORRECTED_DATA") print " Copying ",backnameFlag," to FLAG" self.CopyCol(backnameFlag,"FLAG") #t.putcol(,t.getcol(backnameFlag)) t.close()
def test_tableascii(self): """Testing ASCII table.""" c1 = makescacoldesc("coli", 0) c2 = makescacoldesc("cold", 0.) c3 = makescacoldesc("cols", "") c4 = makescacoldesc("colb", True) c5 = makescacoldesc("colc", 0. + 0j) t = table("ttable.py_tmp.tab1", maketabdesc((c1, c2, c3, c4, c5)), ack=False) tcol = t.colnames() t.addrows(5) t.toascii('asciitemp1', columnnames=tcol) tablefromascii(tablename='tablefromascii', asciifile='asciitemp1') ta = table("tablefromascii", readonly=False) tacol = ta.colnames() self.assertEqual(tcol, tacol) ta.close() t.close() tabledelete('tablefromascii') tabledelete("ttable.py_tmp.tab1")
def test_hypercolumn(self): """Test hypercolumns.""" scd1 = makescacoldesc("col2", "aa") scd2 = makescacoldesc("col1", 1, "IncrementalStMan") scd3 = makescacoldesc("colrec1", {}) acd1 = makearrcoldesc("arr1", 1, 0, [2, 3, 4]) acd2 = makearrcoldesc("arr2", 0. + 0j) td = maketabdesc([scd1, scd2, scd3, acd1, acd2]) tabledefinehypercolumn(td, "TiledArray", 4, ["arr1"]) tab = table("mytable", tabledesc=td, nrow=100) tab.done() tabledelete("mytable")
def main(options): ms = options.ms if ms == '': logging.error('You have to specify an input MS, use -h for help') return cols = options.cols incol = options.incol t = pt.table(ms, readonly=False, ack=False) for col in cols.split(','): if col not in t.colnames(): logging.info('Adding the output column '+col+' to '+ms+'.') if incol == '': # prepare col metadata cd = t.getcoldesc('DATA') coldmi = t.getdminfo('DATA') if options.dysco: cd['dataManagerType'] = 'DyscoStMan' cd['dataManagerGroup'] = 'DyscoData' coldmi = {'NAME': col,'SEQNR': 3,'SPEC': {'dataBitCount': 10,'distribution': 'TruncatedGaussian','distributionTruncation': 2.5,'normalization': 'AF','studentTNu': 0.0,'weightBitCount': 12},'TYPE': 'DyscoStMan'} # not as performing as standard DATA else: coldmi["NAME"] = col # cd['dataManagerType'] = 'StandardStMan' # cd['dataManagerGroup'] = 'SSMVar' # coldmi = {'NAME': col,'SEQNR': 0,'SPEC': {'ActualCacheSize': 2,'BUCKETSIZE': 32768,'IndexLength': 799832,'PERSCACHESIZE': 2},'TYPE': 'StandardStMan'} cd['comment'] = 'Added by addcol2ms' t.addcols(pt.makecoldesc(col, cd), coldmi) # if non dysco is done by default if options.dysco: logging.warning('Setting '+col+' = 0') pt.taql("update $t set "+col+"=0") else: # prepare col metadata coldmi = t.getdminfo(incol) coldmi['NAME'] = col cd = t.getcoldesc(incol) cd['comment'] = 'Added by addcol2ms' t.addcols(pt.makecoldesc(col, cd), coldmi) logging.warning('Setting '+col+' = '+incol) pt.taql("update $t set "+col+"="+incol) else: logging.warning('Column '+col+' already exists.') t.close()
def mscolexist(ms, colname): """ Check if a colname exists in the measurement set ms, returns either True or False """ if os.path.isdir(ms): t = pt.table(ms,readonly=True) colnames =t.colnames() if colname in colnames: # check if the column is in the list exist = True else: exist = False t.close() else: exist = False # ms does not exist return exist
def measureflux_ms( src_dir, imagems, fitms, catname, ATCA_band, sourcepar, n_spw, timerange="", field="", ): try: split( vis=imagems, datacolumn="data", outputvis=fitms) listobs(vis=fitms, filename=f"listobs_{fitms}.dat", overwrite=True) except: print("Not splitting") int_flux_c = [] uvrange = "" for i in range(n_spw): spw = str(i) # If things look like theyre not working, then check the source position! Chances are it can't find the source too far away from the phase centre outfile = f"{src_dir}/casa_files/{catname}_{spw}.cl" os.system(f"rm -r {outfile}") uvmodelfit( vis=fitms, niter=15, comptype="P", spw=spw, sourcepar=sourcepar, outfile=outfile, uvrange=uvrange, field=field, selectdata=True, timerange=timerange, ) tbl = table(outfile) flux = tbl.getcell("Flux", 0)[0].astype("float64") int_flux_c.append(flux) print(flux) if ATCA_band == "C": np.savetxt( f"{src_dir}/{catname}.csv", int_flux_c, delimiter=",", header="S_Cband", ) print(int_flux_c) elif ATCA_band == "X": np.savetxt( f"{src_dir}/{catname}.csv", int_flux_c, delimiter=",", header="S_Xband", ) print(int_flux_c) elif ATCA_band == "L": # int_flux_l = np.array(int_flux_c[::-1]) int_flux_l = int_flux_c np.savetxt( f"{src_dir}/{catname}.csv", int_flux_l, header="S_Lband", delimiter=",", ) print(int_flux_l) return
def set_beam_calibration(self, caltable, verbose=True): """ Given a a CASA measurement set containing a bandpass calibration, load the bandpass calibration into the appropriate pipeline(s). """ # Validate assert (os.path.exists(caltable)) assert (os.path.isdir(caltable)) assert (os.path.exists(os.path.join(caltable, 'SPECTRAL_WINDOW'))) assert (os.path.isdir(os.path.join(caltable, 'SPECTRAL_WINDOW'))) # Load in the calibration data and normalize it tab = tables.table(caltable, ack=False) caldata = tab.getcol('CPARAM')[...] caldata /= numpy.abs(caldata) # Load in the flagging data for the calibration flgdata = tab.getcol('FLAG')[...] tab.close() # Load in the frequency information for the calibration tab = tables.table(os.path.join(caltable, 'SPECTRAL_WINDOW'), ack=False) calfreq = tab.getcol('CHAN_FREQ')[...] calfreq = calfreq.ravel() tab.close() if verbose: print( f"Loaded {caldata.shape[0]} by {caldata.shape[1]} by {caldata.shape[2]} complex gains covering {calfreq[0]/1e6:.3f} to {calfreq[-1]/1e6:.3f} MHz" ) # Validate the calibration data structure assert (caldata.shape[0] == NSTAND) assert (caldata.shape[1] == NCHAN_PIPELINE * NPIPELINE_SUBBAND) assert (caldata.shape[2] == NPOL) # Find the pipelines that should correspond to the specified subband # TODO: Use the freuqency information to figure this out for the user subband_pipelines = [] for i in range(NPIPELINE_SUBBAND): ## Get the frequency range for the pipeline in the subband and pull ## out the middle center_freq = calfreq[i * NCHAN_PIPELINE:(i + 1) * NCHAN_PIPELINE] center_freq = center_freq[center_freq.size // 2] ## Try to map that frequency to a pipeline. If it works, save the ## pipeline to subband_pipelines. try: j = self._freq_to_pipeline(center_freq) subband_pipelines.append(self.pipelines[j]) if verbose: print( f"Found pipeline {j} covering {self.freqs[j][0]/1e6:.3f} to {self.freqs[j][-1]/1e6:.3f} MHz" ) except ValueError: pass # Issue a warning if we don't seem to have the right number of pipelines # for the subband if len(subband_pipelines) != NPIPELINE_SUBBAND: warnings.warn( f"Found {len(subband_pipelines)} pipelines associated with these data instead of the expected {NPIPELINE_SUBBAND}" ) # Set the coefficients - this is slow pb = progressbar.ProgressBar(redirect_stdout=True) pb.start(max_value=len(subband_pipelines) * NSTAND) for i, p in enumerate(subband_pipelines): for j in range(NSTAND): for pol in range(NPOL): cal = 1. / caldata[j, i * NCHAN_PIPELINE:(i + 1) * NCHAN_PIPELINE, pol].ravel() cal = numpy.where(numpy.isfinite(cal), cal, 0) flg = flgdata[j, i * NCHAN_PIPELINE:(i + 1) * NCHAN_PIPELINE, pol].ravel() cal *= (1 - flg) with AllowedPipelineFailure(p): p.beamform.update_calibration_gains( 2 * (self.beam - 1) + pol, NPOL * j + pol, cal) time.sleep(0.005) pb += 1 self._cal_set[i] = True pb.finish()
def create_visibility_from_ms(msname, channum=0): """ Minimal MS to Visibility converter The MS format is much more general than the ARL Visibility so we cut many corners. This requires casacore to be installed. If not an exception ModuleNotFoundError is raised. Creates a list of Visibilities, one per phasecentre """ try: from casacore.tables import table # pylint: disable=import-error except ModuleNotFoundError: raise ModuleNotFoundError("casacore is not installed") tab = table(msname) print(tab.info()) fields = numpy.unique(tab.getcol('FIELD_ID')) print("Found unique field ids %s" % fields) vis_list = list() for field in fields: # First get the main table information ms = tab.query("FIELD_ID==%d" % field) print("Found %d rows for field %d" % (ms.nrows(), field)) time = ms.getcol('TIME') channels = len(numpy.transpose(ms.getcol('DATA'))[0]) print("Found %d channels" % (channels)) try: vis = ms.getcol('DATA')[:, channum, :] except IndexError: raise IndexError("channel number exceeds max. within ms") weight = ms.getcol('WEIGHT') uvw = -1 * ms.getcol('UVW') antenna1 = ms.getcol('ANTENNA1') antenna2 = ms.getcol('ANTENNA2') integration_time = ms.getcol('INTERVAL') ddid = ms.getcol('DATA_DESC_ID') # Now get info from the subtables spwtab = table('%s/SPECTRAL_WINDOW' % msname, ack=False) cfrequency = spwtab.getcol('CHAN_FREQ') frequency = numpy.array([cfrequency[dd] for dd in ddid])[:, channum] cchannel_bandwidth = spwtab.getcol('CHAN_WIDTH') channel_bandwidth = numpy.array( [cchannel_bandwidth[dd] for dd in ddid])[:, 0] uvw *= frequency[:, numpy.newaxis] / constants.c.to('m/s').value # Get polarisation info # poltab = table('%s/POLARIZATION' % msname, ack=False) # corr_type = poltab.getcol('CORR_TYPE') # TODO: Do interpretation correctly polarisation_frame = PolarisationFrame('stokesIQUV') # Get configuration anttab = table('%s/ANTENNA' % msname, ack=False) mount = anttab.getcol('MOUNT') names = anttab.getcol('NAME') diameter = anttab.getcol('DISH_DIAMETER') xyz = anttab.getcol('POSITION') configuration = Configuration(name='', data=None, location=None, names=names, xyz=xyz, mount=mount, frame=None, receptor_frame=ReceptorFrame("linear"), diameter=diameter) # Get phasecentres fieldtab = table('%s/FIELD' % msname, ack=False) pc = fieldtab.getcol('PHASE_DIR')[field, 0, :] phasecentre = SkyCoord(ra=[pc[0]] * u.rad, dec=pc[1] * u.rad, frame='icrs', equinox='J2000') vis_list.append( Visibility(uvw=uvw, time=time, antenna1=antenna1, antenna2=antenna2, frequency=frequency, vis=vis, weight=weight, imaging_weight=weight, integration_time=integration_time, channel_bandwidth=channel_bandwidth, configuration=configuration, phasecentre=phasecentre, polarisation_frame=polarisation_frame)) return vis_list
def makeMAIN2(tablename,specdata,time,state_id,texp=0.2,tBW=2.5e9): # modules import casacore.tables as tb # params nrow = specdata.shape[0] nspw = specdata.shape[1] npol = specdata.shape[2] nchan = specdata.shape[3] weight = tBW/float(nchan) * texp sigma = (tBW/float(nchan) * texp)**-0.5 ind_spw = (np.linspace(0,2*nrow-1,2*nrow,dtype='int32') % 2) # tables colnames = ['UVW', 'FLAG', 'FLAG_CATEGORY', 'WEIGHT', 'SIGMA', 'ANTENNA1', 'ANTENNA2', 'ARRAY_ID', 'DATA_DESC_ID', 'EXPOSURE', 'FEED1', 'FEED2', 'FIELD_ID', 'FLAG_ROW', 'INTERVAL', 'OBSERVATION_ID', 'PROCESSOR_ID', 'SCAN_NUMBER', 'STATE_ID', 'TIME', 'TIME_CENTROID', 'FLOAT_DATA' ] colkeywords = [ {'MEASINFO': {'Ref': 'ITRF', 'type': 'uvw'},'QuantumUnits': np.array(['m', 'm', 'm'],dtype='|S2')}, {}, {'CATEGORY': np.array([],dtype='|S1')}, {},{},{},{},{},{}, {'QuantumUnits': np.array(['s'],dtype='|S2')}, {},{},{},{}, {'QuantumUnits': np.array(['s'],dtype='|S2')}, {},{},{},{}, {'MEASINFO': {'Ref': 'UTC', 'type': 'epoch'}, 'QuantumUnits': np.array(['s'],dtype='|S2')}, {'MEASINFO': {'Ref': 'UTC', 'type': 'epoch'}, 'QuantumUnits': np.array(['s'],dtype='|S2')}, {'UNIT': 'K'} ] ndims = [1,2,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2] isarrays = [True,True,True,True,True,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,False,True] valuetypes = ['double','bool','bool','float','float','int','int','int','int','double','int','int','int','bool','double','int','int','int','int','double','double','float'] descs = [] for colname,colkeyword,ndim,valuetype,isarray in zip(colnames,colkeywords,ndims,valuetypes,isarrays): if colname=='UVW': descs.append(tb.makearrcoldesc(colname,0.0,datamanagertype='StandardStMan',datamanagergroup='StandardStMan',ndim=ndim,keywords=colkeyword,valuetype=valuetype,options=5,shape=np.array([3], dtype='int32'))) elif isarray: descs.append(tb.makearrcoldesc(colname,0.0,datamanagertype='StandardStMan',datamanagergroup='StandardStMan',ndim=ndim,keywords=colkeyword,valuetype=valuetype)) else: descs.append(tb.makescacoldesc(colname,0.0,datamanagertype='StandardStMan',datamanagergroup='StandardStMan',keywords=colkeyword,valuetype=valuetype)) td = tb.maketabdesc(descs=descs) returned_table = tb.table(tablename,tabledesc=td,nrow=2*nrow,readonly=False) # put values value = np.zeros([2*nrow,3],dtype='float64') returned_table.putcol('UVW',value) value = np.full([2*nrow,2],sigma) returned_table.putcol('SIGMA',value) value = np.full([2*nrow,2],weight) returned_table.putcol('WEIGHT',value) value = np.zeros([2*nrow],dtype='int32') returned_table.putcol('ANTENNA1',value) returned_table.putcol('ANTENNA2',value) returned_table.putcol('ARRAY_ID',value) returned_table.putcol('FEED1',value) returned_table.putcol('FEED2',value) returned_table.putcol('FIELD_ID',value) returned_table.putcol('OBSERVATION_ID',value) returned_table.putcol('PROCESSOR_ID',value) value = np.zeros([2*nrow],dtype='bool') returned_table.putcol('FLAG_ROW',value) value = np.full([2*nrow],texp,dtype='float64') returned_table.putcol('EXPOSURE',value) returned_table.putcol('INTERVAL',value) value = np.zeros_like(np.concatenate([specdata[:,0],specdata[:,1]],axis=0),dtype='bool') value = value.transpose(0,2,1) returned_table.putcol('FLAG',value) value = np.zeros_like(np.concatenate([specdata[:,0],specdata[:,1]],axis=0),dtype='float64') for i in range(2): value[ind_spw==i] = specdata[:,i].copy() value = value.transpose(0,2,1) returned_table.putcol('FLOAT_DATA',value) value = np.zeros(2*nrow,dtype='int32') for i in range(2): value[ind_spw==i] = state_id value = np.zeros(2*nrow,dtype='int32') for i in range(2): value[ind_spw==i] = i returned_table.putcol('DATA_DESC_ID',value) value = np.zeros(2*nrow,dtype='int32') for i in range(2): value[ind_spw==i] = state_id returned_table.putcol('STATE_ID',value) value = np.zeros(2*nrow,dtype='int32') for i in range(2): value[ind_spw==i] = np.linspace(0,nrow-1,nrow,dtype='int32') returned_table.putcol('SCAN_NUMBER',value) value = np.zeros(2*nrow,dtype='float64') for i in range(2): value[ind_spw==i] = time.copy() returned_table.putcol('TIME',value) returned_table.putcol('TIME_CENTROID',value) returned_table.flush() returned_table.close()
def main(ms_files, outfile, clobber=True): """ Performs a virtual concatenation with possible frequency gaps Parameters ---------- ms_files : list List of files to merge, ordered by frequency. Files that do not exist are identified as gaps and are filled with flagged dummy data outfile : str Output file clobber : bool, optional If True, existing files are overwritten """ if type(ms_files) is str: ms_files = [f.strip(' \'\"') for f in ms_files.strip('[]').split(',')] if type(clobber) is str: if clobber.lower() == 'true': clobber = True else: clobber = False if os.path.exists(outfile): if clobber: pt.tabledelete(outfile) else: return # Find at least one existing ms ms_exists = None for ms in ms_files: if os.path.exists(ms): ms_exists = ms sw = pt.table('{}::SPECTRAL_WINDOW'.format(ms)) ms_exists_ref_freq = sw.getcol('REF_FREQUENCY')[0] sw.close() break if ms_exists is None: print('ERROR: no files exist') sys.exit(1) # Identify gaps ms_files_to_concat = [] for i, ms in enumerate(ms_files): if not os.path.exists(ms): # Missing file means gap, so create an appropriate dummy dataset with # a random name ms_new = '{0}_{1}.ms'.format( os.path.splitext(ms)[0], uuid.uuid4().urn.split('-')[-1]) pt.tableutil.tablecopy(ms_exists, ms_new) # Alter SPECTRAL_WINDOW subtable as appropriate to fill gap sw = pt.table('{}::SPECTRAL_WINDOW'.format(ms_new), readonly=False) tot_bandwidth = sw.getcol('TOTAL_BANDWIDTH')[0] if i > 0: sw_low = pt.table('{}::SPECTRAL_WINDOW'.format(ms_files[i - 1])) ref_freq = sw_low.getcol('REF_FREQUENCY') + tot_bandwidth sw_low.close() else: for j in range(1, len(ms_files) - 1): if os.path.exists(ms_files[j]): sw_high = pt.table('{}::SPECTRAL_WINDOW'.format( ms_files[j])) ref_freq = sw_high.getcol( 'REF_FREQUENCY') - tot_bandwidth * j sw_high.close() break chan_freq = sw.getcol('CHAN_FREQ') - ms_exists_ref_freq + ref_freq sw.putcol('REF_FREQUENCY', ref_freq) sw.putcol('CHAN_FREQ', chan_freq) sw.close() # Flag all data t = pt.table(ms_new, readonly=False) t.putcol('FLAG_ROW', np.ones(len(t), dtype=bool)) t.close() ms_files_to_concat.append(ms_new) else: ms_files_to_concat.append(ms) # Concat pt.msutil.msconcat(ms_files_to_concat, outfile)
def plugin_main(args, **kwargs): """ Makes a mapfile with the MSs spread across the full bandwidth Parameters ---------- mapfile_in : str Filename of datamap containing MS files mapfile_dir : str Directory for output mapfile filename: str Name of output mapfile num: int, optional Number of frequencies in output mapfile Returns ------- result : dict New parmdb datamap filename """ mapfile_in = kwargs['mapfile_in'] mapfile_dir = kwargs['mapfile_dir'] filename = kwargs['filename'] if 'num' in kwargs: num = int(kwargs['num']) else: num = 6 fileid = os.path.join(mapfile_dir, filename) map_in = DataMap.load(mapfile_in) map_in.iterator = DataMap.SkipIterator map_out = DataMap() map_out.data = [] map_out._data = [] # do not re-run if we already ran, and input files are deleted. if os.path.exists(fileid) and not os.path.exists(map_in[0].file): print 'PipelineStep_selectDistFreqs: Not re-running because output file exists, but input files don\'t!' return {'mapfile': fileid} #sort into frequency groups freq_groups = {} hosts = [] for item in map_in: # Get the frequency info from the MS file sw = pt.table(item.file + '::SPECTRAL_WINDOW', ack=False) freq = int(sw.col('REF_FREQUENCY')[0]) sw.close() if freq in freq_groups: freq_groups[freq].append(item.file) else: freq_groups[freq] = [item.file] if not item.host in hosts: hosts.append(item.host) # select frequencies freqs = freq_groups.keys() freqs.sort() num_freqs = len(freqs) if num > num_freqs: print 'PipelineStep_selectDistFreqs: fewer than %d frequency groups found, continuig with %d groups.' % ( num, num_freqs) num = num_freqs dist_ind = get_distributed_indices(0, num_freqs - 1, num) selfreqs = [freqs[ind] for ind in dist_ind] if len(selfreqs) < 1: print "PipelineStep_selectDistFreqs: Selected fewer than one frequency band." raise ValueError("Selected fewer than one frequency band.") all_files = [] for selfreq in selfreqs: all_files.extend(freq_groups[selfreq]) # extend the hosts-list for i in range(len(all_files) - len(hosts)): hosts.append(hosts[i]) # fill the output-map for (host, fname) in zip(hosts, all_files): map_out.append(DataProduct(host, fname, False)) map_out.save(fileid) del (map_in) del (map_out) result = {'mapfile': fileid} return result
def main(options): ms = options.ms if ms == '': logging.error('You have to specify an input MS, use -h for help') return cols = options.cols incol = options.incol t = pt.table(ms, readonly=False, ack=False) for col in cols.split(','): if col not in t.colnames(): logging.info('Adding the output column ' + col + ' to ' + ms + '.') if incol == '': # prepare col metadata cd = t.getcoldesc('DATA') coldmi = t.getdminfo('DATA') if options.dysco: cd['dataManagerType'] = 'DyscoStMan' cd['dataManagerGroup'] = 'DyscoData' coldmi = { 'NAME': col, 'SEQNR': 3, 'SPEC': { 'dataBitCount': 10, 'distribution': 'TruncatedGaussian', 'distributionTruncation': 2.5, 'normalization': 'AF', 'studentTNu': 0.0, 'weightBitCount': 12 }, 'TYPE': 'DyscoStMan' } # not as performing as standard DATA else: coldmi["NAME"] = col # cd['dataManagerType'] = 'StandardStMan' # cd['dataManagerGroup'] = 'SSMVar' # coldmi = {'NAME': col,'SEQNR': 0,'SPEC': {'ActualCacheSize': 2,'BUCKETSIZE': 32768,'IndexLength': 799832,'PERSCACHESIZE': 2},'TYPE': 'StandardStMan'} cd['comment'] = 'Added by addcol2ms' t.addcols(pt.makecoldesc(col, cd), coldmi) # if non dysco is done by default if options.dysco: logging.warning('Setting ' + col + ' = 0') pt.taql("update $t set " + col + "=0") else: # prepare col metadata coldmi = t.getdminfo(incol) coldmi['NAME'] = col cd = t.getcoldesc(incol) cd['comment'] = 'Added by addcol2ms' t.addcols(pt.makecoldesc(col, cd), coldmi) logging.warning('Setting ' + col + ' = ' + incol) pt.taql("update $t set " + col + "=" + incol) else: logging.warning('Column ' + col + ' already exists.') t.close()
def read_ms(self, filepath, run_check=True, check_extra=True, run_check_acceptability=True, data_column='DATA', pol_order='AIPS'): ''' read in a casa measurement set Args: filepath: name of the measurement set folder run_check: Option to check for the existence and proper shapes of parameters after reading in the file. Default is True. check_extra: Option to check optional parameters as well as required ones. Default is True. run_check_acceptability: Option to check the values of parameters after reading in the file. Default is True. data_column: specify which CASA measurement set data column to read from (can be 'DATA','CORRECTED', or 'MODEL') pol_order: use 'AIPS' or 'CASA' ordering of polarizations? ''' # make sure user requests a valid data_column if data_column != 'DATA' and data_column != 'CORRECTED_DATA' and data_column != 'MODEL': raise ValueError( 'Invalid data_column value supplied. Use \'Data\',\'MODEL\' or \'CORRECTED_DATA\'' ) if not os.path.exists(filepath): raise (IOError, filepath + ' not found') # set visibility units if (data_column == 'DATA'): self.vis_units = "UNCALIB" elif (data_column == 'CORRECTED_DATA'): self.vis_units = "JY" elif (data_column == 'MODEL'): self.vis_units = "JY" # limit length of extra_keywords keys to 8 characters to match uvfits & miriad self.extra_keywords['DATA_COL'] = data_column # get frequency information from spectral window table tb_spws = tables.table(filepath + '/SPECTRAL_WINDOW') freqs = tb_spws.getcol('CHAN_FREQ') self.freq_array = freqs self.Nfreqs = int(freqs.shape[1]) self.channel_width = float(tb_spws.getcol('CHAN_WIDTH')[0, 0]) self.Nspws = int(freqs.shape[0]) if self.Nspws > 1: raise ValueError('Sorry. Files with more than one spectral' + 'window (spw) are not yet supported. A ' + 'great project for the interested student!') self.spw_array = np.arange(self.Nspws) tb_spws.close() # now get the data tb = tables.table(filepath) # check for multiple subarrays. importuvfits does not appear to preserve subarray information! subarray = np.unique(np.int32(tb.getcol('ARRAY_ID')) - 1) if len(set(subarray)) > 1: raise ValueError('This file appears to have multiple subarray ' 'values; only files with one subarray are ' 'supported.') times_unique = time.Time(np.unique(tb.getcol('TIME') / (3600. * 24.)), format='mjd').jd self.Ntimes = int(len(times_unique)) data_array = tb.getcol(data_column) self.Nblts = int(data_array.shape[0]) flag_array = tb.getcol('FLAG') # CASA stores data in complex array with dimension NbltsxNfreqsxNpols if (len(data_array.shape) == 3): data_array = np.expand_dims(data_array, axis=1) flag_array = np.expand_dims(flag_array, axis=1) self.data_array = data_array self.flag_array = flag_array self.Npols = int(data_array.shape[-1]) self.uvw_array = tb.getcol('UVW') self.ant_1_array = tb.getcol('ANTENNA1').astype(np.int32) self.ant_2_array = tb.getcol('ANTENNA2').astype(np.int32) self.Nants_data = len( np.unique( np.concatenate((np.unique(self.ant_1_array), np.unique(self.ant_2_array))))) self.baseline_array = self.antnums_to_baseline(self.ant_1_array, self.ant_2_array) self.Nbls = len(np.unique(self.baseline_array)) # Get times. MS from cotter are modified Julian dates in seconds (thanks to Danny Jacobs for figuring out the proper conversion) self.time_array = time.Time(tb.getcol('TIME') / (3600. * 24.), format='mjd').jd # Polarization array tbPol = tables.table(filepath + '/POLARIZATION') # list of lists, probably with each list corresponding to SPW. polList = tbPol.getcol('CORR_TYPE')[0] self.polarization_array = np.zeros(len(polList), dtype=np.int32) for polnum in range(len(polList)): self.polarization_array[polnum] = int(polDict[polList[polnum]]) tbPol.close() # Integration time # use first interval and assume rest are constant (though measurement set has all integration times for each Nblt ) # self.integration_time=tb.getcol('INTERVAL')[0] # for some reason, interval ends up larger than the difference between times... self.integration_time = float(times_unique[1] - times_unique[0]) * 3600. * 24. # open table with antenna location information tbAnt = tables.table(filepath + '/ANTENNA') tbObs = tables.table(filepath + '/OBSERVATION') self.telescope_name = tbObs.getcol('TELESCOPE_NAME')[0] self.instrument = tbObs.getcol('TELESCOPE_NAME')[0] tbObs.close() # Use Telescopes.py dictionary to set array position full_antenna_positions = tbAnt.getcol('POSITION') xyz_telescope_frame = tbAnt.getcolkeyword('POSITION', 'MEASINFO')['Ref'] antFlags = np.empty(len(full_antenna_positions), dtype=bool) antFlags[:] = False for antnum in range(len(antFlags)): antFlags[antnum] = np.all(full_antenna_positions[antnum, :] == 0) if (xyz_telescope_frame == 'ITRF'): self.telescope_location = np.array( np.mean(full_antenna_positions[np.invert(antFlags), :], axis=0)) if self.telescope_location is None: try: self.set_telescope_params() except ValueError: warnings.warn( 'Telescope frame is not ITRF and telescope is not ' 'in known_telescopes, so telescope_location is not set.') # antenna names ant_names = tbAnt.getcol('STATION') ant_diams = tbAnt.getcol('DISH_DIAMETER') self.antenna_diameters = ant_diams[ant_diams > 0] self.Nants_telescope = len(antFlags[np.invert(antFlags)]) test_name = ant_names[0] names_same = True for antnum in range(len(ant_names)): if (not (ant_names[antnum] == test_name)): names_same = False if (not (names_same)): # cotter measurement sets store antenna names in the NAMES column. self.antenna_names = ant_names else: # importuvfits measurement sets store antenna names in the STATION column. self.antenna_names = tbAnt.getcol('NAME') self.antenna_numbers = np.arange(len(self.antenna_names)).astype(int) nAntOrig = len(self.antenna_names) ant_names = [] for antNum in range(len(self.antenna_names)): if not (antFlags[antNum]): ant_names.append(self.antenna_names[antNum]) self.antenna_names = ant_names self.antenna_numbers = self.antenna_numbers[np.invert(antFlags)] relative_positions = np.zeros_like(full_antenna_positions) relative_positions = full_antenna_positions - self.telescope_location.reshape( 1, 3) self.antenna_positions = relative_positions[np.invert(antFlags), :] tbAnt.close() tbField = tables.table(filepath + '/FIELD') if (tbField.getcol('PHASE_DIR').shape[1] == 2): self.phase_type = 'drift' self.set_drift() elif (tbField.getcol('PHASE_DIR').shape[1] == 1): self.phase_type = 'phased' # MSv2.0 appears to assume J2000. Not sure how to specifiy otherwise self.phase_center_epoch = float( tb.getcolkeyword('UVW', 'MEASINFO')['Ref'][1:]) self.phase_center_ra = float(tbField.getcol('PHASE_DIR')[0][0][0]) self.phase_center_dec = float(tbField.getcol('PHASE_DIR')[0][0][1]) self.set_phased() # set LST array from times and itrf self.set_lsts_from_time_array() # set the history parameter _, self.history = self._ms_hist_to_string( tables.table(filepath + '/HISTORY')) # CASA weights column keeps track of number of data points averaged. if not uvutils.check_history_version(self.history, self.pyuvdata_version_str): self.history += self.pyuvdata_version_str self.nsample_array = tb.getcol('WEIGHT_SPECTRUM') if (len(self.nsample_array.shape) == 3): self.nsample_array = np.expand_dims(self.nsample_array, axis=1) self.object_name = tbField.getcol('NAME')[0] tbField.close() tb.close() # order polarizations self.order_pols(pol_order) if run_check: self.check(check_extra=check_extra, run_check_acceptability=run_check_acceptability)
def create_blockvisibility_from_ms(msname, channum=None, ack=False): """ Minimal MS to BlockVisibility converter The MS format is much more general than the ARL BlockVisibility so we cut many corners. This requires casacore to be installed. If not an exception ModuleNotFoundError is raised. Creates a list of BlockVisibility's, split by field and spectral window :param msname: File name of MS :param channum: range of channels e.g. range(17,32), default is None meaning all :return: """ try: from casacore.tables import table # pylint: disable=import-error except ModuleNotFoundError: raise ModuleNotFoundError("casacore is not installed") tab = table(msname, ack=ack) log.debug("create_blockvisibility_from_ms: %s" % str(tab.info())) fields = numpy.unique(tab.getcol('FIELD_ID')) dds = numpy.unique(tab.getcol('DATA_DESC_ID')) log.debug( "create_blockvisibility_from_ms: Found unique fields %s, unique data descriptions %s" % (str(fields), str(dds))) vis_list = list() for dd in dds: dtab = table(msname, ack=ack).query('DATA_DESC_ID==%d' % dd, style='') for field in fields: ms = dtab.query('FIELD_ID==%d' % field, style='') assert ms.nrows( ) > 0, "Empty selection for FIELD_ID=%d and DATA_DESC_ID=%d" % ( field, dd) log.debug("create_blockvisibility_from_ms: Found %d rows" % (ms.nrows())) time = ms.getcol('TIME') channels = ms.getcol('DATA').shape[-2] log.debug("create_visibility_from_ms: Found %d channels" % (channels)) if channum is None: channum = range(channels) try: ms_vis = ms.getcol('DATA')[:, channum, :] ms_weight = ms.getcol('WEIGHT')[:, :] except IndexError: raise IndexError("channel number exceeds max. within ms") uvw = -1 * ms.getcol('UVW') antenna1 = ms.getcol('ANTENNA1') antenna2 = ms.getcol('ANTENNA2') integration_time = ms.getcol('INTERVAL') # Now get info from the subtables spwtab = table('%s/SPECTRAL_WINDOW' % msname, ack=False) cfrequency = spwtab.getcol('CHAN_FREQ')[dd][channum] cchannel_bandwidth = spwtab.getcol('CHAN_WIDTH')[dd][channum] nchan = cfrequency.shape[0] # Get polarisation info poltab = table('%s/POLARIZATION' % msname, ack=False) corr_type = poltab.getcol('CORR_TYPE') # These correspond to the CASA Stokes enumerations if numpy.array_equal(corr_type[0], [1, 2, 3, 4]): polarisation_frame = PolarisationFrame('stokesIQUV') elif numpy.array_equal(corr_type[0], [5, 6, 7, 8]): polarisation_frame = PolarisationFrame('circular') elif numpy.array_equal(corr_type[0], [9, 10, 11, 12]): polarisation_frame = PolarisationFrame('linear') else: raise KeyError("Polarisation not understood: %s" % str(corr_type)) npol = 4 # Get configuration anttab = table('%s/ANTENNA' % msname, ack=False) nants = anttab.nrows() mount = anttab.getcol('MOUNT') names = anttab.getcol('NAME') diameter = anttab.getcol('DISH_DIAMETER') xyz = anttab.getcol('POSITION') configuration = Configuration( name='', data=None, location=None, names=names, xyz=xyz, mount=mount, frame=None, receptor_frame=ReceptorFrame("linear"), diameter=diameter) # Get phasecentres fieldtab = table('%s/FIELD' % msname, ack=False) pc = fieldtab.getcol('PHASE_DIR')[field, 0, :] phasecentre = SkyCoord(ra=[pc[0]] * u.rad, dec=pc[1] * u.rad, frame='icrs', equinox='J2000') bv_times = numpy.unique(time) ntimes = len(bv_times) bv_vis = numpy.zeros([ntimes, nants, nants, nchan, npol]).astype('complex') bv_weight = numpy.zeros([ntimes, nants, nants, nchan, npol]) bv_uvw = numpy.zeros([ntimes, nants, nants, 3]) time_last = time[0] time_index = 0 for row, _ in enumerate(ms_vis): # MS has shape [row, npol, nchan] # BV has shape [ntimes, nants, nants, nchan, npol] if time[row] != time_last: assert time[ row] > time_last, "MS is not time-sorted - cannot convert" time_index += 1 time_last = time[row] bv_vis[time_index, antenna2[row], antenna1[row], ...] = ms_vis[row, ...] bv_weight[time_index, antenna2[row], antenna1[row], :, ...] = ms_weight[row, numpy.newaxis, ...] bv_uvw[time_index, antenna2[row], antenna1[row], :] = uvw[row, :] vis_list.append( BlockVisibility(uvw=bv_uvw, time=bv_times, frequency=cfrequency, channel_bandwidth=cchannel_bandwidth, vis=bv_vis, weight=bv_weight, configuration=configuration, phasecentre=phasecentre, polarisation_frame=polarisation_frame)) tab.close() return vis_list
if opts.save and opts.rewrite: raise Exception('Only one mode at a time can be used: choose -s OR -r') if not opts.fname and opts.rewrite: raise Exception('-f/--fname must be specified in rewriting mode') fversion = 0 if opts.fname == None: fname_pattern = ms_name + '.flag_v' opts.fname = fname_pattern + '%02d' % fversion fname_check = os.path.exists(opts.fname + '.npy') while fname_check == True and opts.save: fversion += 1 opts.fname = fname_pattern + '%02d' % fversion fname_check = os.path.exists(opts.fname + '.npy') t = tb.table(filename, readonly=opts.save) if opts.save: print('Saving FLAG column in ' + opts.fname + '.npy') flag = t.getcol('FLAG') flagfile = open(opts.fname + '.npy', 'wb') np.save(flagfile, flag) if opts.rewrite: print('Overwriting FLAG column with ' + opts.fname + '.npy') flagfile = open(opts.fname + '.npy', 'rb') flag = np.load(flagfile) t.putcol('FLAG', flag) flagfile.close() t.close()
def add_dummyms(msfiles): ''' Add dummy ms to create a regular freuqency grid when doing a concat with DPPP ''' if len(msfiles) == 1: # there is nothing to do return msfiles keyname = 'REF_FREQUENCY' freqaxis = [] newmslist = [] # Check for wrong REF_FREQUENCY which happens after a DPPP split in frequency for ms in msfiles: t = pt.table(ms + '/SPECTRAL_WINDOW', readonly=True) freq = t.getcol('REF_FREQUENCY')[0] t.close() freqaxis.append(freq) freqaxis = np.sort(np.array(freqaxis)) minfreqspacing = np.min(np.diff(freqaxis)) if minfreqspacing == 0.0: keyname = 'CHAN_FREQ' freqaxis = [] for ms in msfiles: t = pt.table(ms + '/SPECTRAL_WINDOW', readonly=True) if keyname == 'CHAN_FREQ': freq = t.getcol(keyname)[0][0] else: freq = t.getcol(keyname)[0] t.close() freqaxis.append(freq) # put everything in order of increasing frequency freqaxis = np.array(freqaxis) idx = np.argsort(freqaxis) freqaxis = freqaxis[np.array(tuple(idx))] sortedmslist = list(msfiles[i] for i in idx) freqspacing = np.diff(freqaxis) minfreqspacing = np.min(np.diff(freqaxis)) # insert dummies in the ms list if needed count = 0 newmslist.append( sortedmslist[0]) # always start with the first ms the list for msnumber, ms in enumerate(sortedmslist[1::]): if int(round(old_div(freqspacing[msnumber], minfreqspacing))) > 1: ndummy = int(round(old_div(freqspacing[msnumber], minfreqspacing))) - 1 for dummy in range(ndummy): newmslist.append('dummy' + str(count) + '.ms') print('Added dummy:', 'dummy' + str(count) + '.ms') count = count + 1 newmslist.append(ms) print( 'Updated ms list with dummies inserted to create a regular frequency grid' ) print(newmslist) return newmslist
uvsel = "[0.100000,1000.000000]" #print doflagafter, takeoutbeam, aoflagger, dysco, split filechecker(clustercat, fullmask, indico, args['h5sols']) if args['h5sols'] == None: fixsymlinks() solsfile = glob.glob('DDS3_full*smoothed.npz') if len(solsfile) < 1: print('Cannot find the correct solution file') sys.exit() msfiles = ascii.read(args['mslist'], data_start=0) msfiles = list( msfiles[:][msfiles.colnames[0]]) # convert to normal list of strings t = pt.table(msfiles[0] + '/OBSERVATION') fieldname = t.getcol('LOFAR_TARGET')['array'][0] t.close() msoutconcat = fieldname + '_' + obsid + '.dysco.sub.shift.avg.weights.ms.archive' if boxfile != 'fullfield': composite = False r = pyregion.open(boxfile) if len(r[:]) > 1: composite = True else: print(boxfile) phasecenter = '[' + getregionboxcenter(boxfile) + ']' print(phasecenter) else:
def test_table_unicode(self): t = table(join(self.workdir, unicode_string), maketabdesc(), ack=False)
def test_msutil(self): """Testing msutil.""" datacoldesc = makearrcoldesc("DATA", 0., ndim=2, shape=[20, 4]) ms = default_ms("tabtemp", maketabdesc((datacoldesc))) ms.close() spw = table("tabtemp/SPECTRAL_WINDOW", readonly=False) spw.addrows() spw.putcell('NUM_CHAN', 0, 20) t = table("tabtemp", readonly=False) print(t.colnames()) addImagingColumns("tabtemp") self.assertIn('MODEL_DATA', t.colnames()) self.assertIn('CORRECTED_DATA', t.colnames()) self.assertIn('IMAGING_WEIGHT', t.colnames()) removeImagingColumns("tabtemp") self.assertNotIn('MODEL_DATA', t.colnames()) self.assertNotIn('CORRECTED_DATA', t.colnames()) self.assertNotIn('IMAGING_WEIGHT', t.colnames()) addDerivedMSCal("tabtemp") self.assertIn('PA1', t.colnames()) self.assertIn('PA2', t.colnames()) self.assertIn('LAST', t.colnames()) self.assertIn('AZEL2', t.colnames()) self.assertIn('AZEL1', t.colnames()) self.assertIn('UVW_J2000', t.colnames()) self.assertIn('LAST1', t.colnames()) self.assertIn('LAST2', t.colnames()) self.assertIn('HA1', t.colnames()) self.assertIn('HA2', t.colnames()) self.assertIn('HA', t.colnames()) removeDerivedMSCal("tabtemp") self.assertNotIn('PA1', t.colnames()) self.assertNotIn('PA2', t.colnames()) self.assertNotIn('LAST', t.colnames()) self.assertNotIn('AZEL2', t.colnames()) self.assertNotIn('AZEL1', t.colnames()) self.assertNotIn('UVW_J2000', t.colnames()) self.assertNotIn('LAST1', t.colnames()) self.assertNotIn('LAST2', t.colnames()) self.assertNotIn('HA1', t.colnames()) self.assertNotIn('HA2', t.colnames()) self.assertNotIn('HA', t.colnames()) self.assertNotIn('HA', t.colnames()) self.assertNotIn('HA', t.colnames()) taql("SELECT FROM tabtemp where TIME in (SELECT DISTINCT TIME" + " FROM tabtemp LIMIT 10) GIVING first10.MS AS PLAIN") taql("SELECT FROM tabtemp where TIME in (SELECT DISTINCT TIME" + " FROM tabtemp LIMIT 10 OFFSET 10) GIVING second10.MS AS PLAIN") msconcat(["first10.MS", "second10.MS"], "combined.MS", concatTime=True) spw.close() t.close() tabledelete("tabtemp")
def FindWeights(self, colname=""): ms = table(self.MSName, ack=False) ants = table(ms.getkeyword("ANTENNA"), ack=False) antnames = ants.getcol("NAME") ants.close() nAnt = len(antnames) u, v, _ = ms.getcol("UVW").T A0 = ms.getcol("ANTENNA1") A1 = ms.getcol("ANTENNA2") tarray = ms.getcol("TIME") nbl = np.where(tarray == tarray[0])[0].size warnings.filterwarnings("ignore") warnings.filterwarnings("default") if "RESIDUAL_DATA" not in ms.colnames(): print('reweight: RESIDUAL_DATA not found. Exiting...') sys.exit(1) residualdata = ms.getcol("RESIDUAL_DATA") flags = ms.getcol("FLAG") ms.close() # apply uvcut c_m_s = 2.99792458e8 uvlen = np.sqrt(u**2 + v**2) / c_m_s * self.referencefreq flags[uvlen > self.uvcut[1], :, :] = True flags[uvlen < self.uvcut[0], :, :] = True residualdata[flags] = np.nan residualdata[residualdata == 0] = np.nan # initialise nChan = residualdata.shape[1] nPola = residualdata.shape[2] nt = residualdata.shape[0] / nbl residualdata = residualdata.reshape((nt, nbl, nChan, nPola)) A0 = A0.reshape((nt, nbl))[0, :] A1 = A1.reshape((nt, nbl))[0, :] # make rms and residuals arrays rmsarray = np.zeros((nt, nbl, nChan, 2), dtype=np.complex64) residuals = np.zeros_like(rmsarray, dtype=np.complex64) rmsarray[:, :, :, 0] = residualdata[:, :, :, 1] rmsarray[:, :, :, 1] = residualdata[:, :, :, 2] residuals[:, :, :, 0] = residualdata[:, :, :, 0] residuals[:, :, :, 1] = residualdata[:, :, :, 3] # start calculating the weights CoeffArray = np.zeros((nt, nChan, nAnt)) ant1 = np.arange(nAnt) num_sols_time = int(np.ceil(float(nt) / self.ntSol)) num_sols_freq = int(np.ceil(float(nChan) / self.nchanSol)) tcellsize = self.ntSol for t_i in range(0, nt, self.ntSol): if (t_i == nt - self.ntSol) and (nt % self.ntSol > 0): tcellsize = nt % self.ntSol t_e = t_i + tcellsize fcellsize = self.nchanSol for f_i in range(0, nChan, self.nchanSol): if (f_i == nChan - self.nchanSol) and (nChan % self.nchanSol > 0): fcellsize = nChan % self.nchanSol f_e = f_i + fcellsize # build weights for each antenna in the current time-frequency block for ant in ant1: # set of vis for baselines ant-ant_i set1 = np.where(A0 == ant)[0] # set of vis for baselines ant_i-ant set2 = np.where(A1 == ant)[0] CoeffArray[t_i:t_e, f_i:f_e, ant] = np.sqrt( np.nanmean( np.append(residuals[t_i:t_e, set1, f_i:f_e, :], residuals[t_i:t_e, set2, f_i:f_e, :]) * np.append(residuals[t_i:t_e, set1, f_i:f_e, :], residuals[t_i:t_e, set2, f_i:f_e, :]).conj()) - np.nanstd( np.append(rmsarray[t_i:t_e, set1, f_i:f_e, :], rmsarray[t_i:t_e, set2, f_i:f_e, :]))) if not self.quiet: PrintProgress(t_i, nt) # plot # Nr = 8 # Nc = 8 # xvals = range(nt) # yvals = range(nChan) # figSize = [10+3*Nc, 8+2*Nr] # figgrid, axa = plt.subplots(Nr, Nc, sharex=True, sharey=True, figsize=figSize) # for i in range(nAnt): # ax = axa.flatten()[i] # bbox = ax.get_window_extent().transformed(figgrid.dpi_scale_trans.inverted()) # aspect = ((xvals[-1]-xvals[0])*bbox.height)/((yvals[-1]-yvals[0])*bbox.width) # im = ax.imshow(CoeffArray[:, :, i].transpose(1,0), origin='lower', interpolation="none", cmap=plt.cm.rainbow, norm=None, # extent=[xvals[0],xvals[-1],yvals[0],yvals[-1]], aspect=str(aspect)) # figgrid.colorbar(im, ax=axa.ravel().tolist(), use_gridspec=True, fraction=0.02, pad=0.005, aspect=35) # figgrid.savefig(self.MSName+'_coef1.png', bbox_inches='tight') # plt.close() # get rid of NaNs and low values CoeffArray[np.isnan(CoeffArray)] = np.inf for i in range(nAnt): tempars = CoeffArray[:, :, i] thres = 0.25 * np.median(tempars[np.where(np.isfinite(tempars))]) CoeffArray[:, :, i][tempars < thres] = thres return CoeffArray
# Flagging on concatenated dataset - also flag low-elevation logger.info('Flagging...') MSs = lib_ms.AllMSs(glob.glob('mss_t*/*MS'), s) MSs.run('DPPP '+parset_dir+'/DPPP-flag.parset msin=$pathMS', \ log='$nameMS_DPPP_flag.log', commandType='DPPP') #sys.exit() # for DDFacet # Create time-chunks logger.info('Splitting in time...') for groupname in groupnames: tc = initc ms = groupname + '/' + groupname + '.MS' if not os.path.exists(ms): continue t = pt.table(ms, ack=False) starttime = t[0]['TIME'] endtime = t[t.nrows() - 1]['TIME'] hours = (endtime - starttime) / 3600. logger.debug(ms + ' has length of ' + str(hours) + ' h.') for timerange in np.array_split(t.getcol('TIME'), round(hours)): logger.info('%02i - Splitting timerange %f %f' % (tc, timerange[0], timerange[-1])) t1 = t.query('TIME >= ' + str(timerange[0]) + ' && TIME <= ' + str(timerange[-1]), sortlist='TIME,ANTENNA1,ANTENNA2') splitms = groupname + '/TC%02i.MS' % tc lib_util.check_rm(splitms) t1.copy(splitms, True) t1.close()
def createMS2(MS2name,specdata,time,ra,dec,sysvel,sourcename,project,observer,direction,freq,Tsys,Tsys_time,state_id,path_temp='./temp/',removetemp=True): # modules import casacore.tables as tb # params os.system('rm -rf '+path_temp) os.system('rm -rf '+MS2name) os.system('mkdir -p '+path_temp) # make tables #makeMAIN(MS2name,path_temp+'MAIN',specdata,time,state_id) makeMAIN2(MS2name,specdata,time,state_id) makeANTENNA(MS2name+'/ANTENNA',path_temp+'ANTENNA') makeDATA_DESCRIPTION(MS2name+'/DATA_DESCRIPTION',path_temp+'DATA_DESCRIPTION') makeDOPPLER(MS2name+'/DOPPLER',path_temp+'DOPPLER') makeFEED(MS2name+'/FEED',path_temp+'FEED') makeFIELD(MS2name+'/FIELD',path_temp+'FIELD',ra,dec,sourcename,time) makeFLAG_CMD(MS2name+'/FLAG_CMD',path_temp+'FLAG_CMD') makeFREQ_OFFSET(MS2name+'/FREQ_OFFSET',path_temp+'FREQ_OFFSET') makeHISTORY(MS2name+'/HISTORY',path_temp+'HISTORY') makeOBSERVATION(MS2name+'/OBSERVATION',path_temp+'OBSERVATION',time,project,observer) #makePOINTING(MS2name+'/POINTING',path_temp+'POINTING',direction,time) makePOINTING2(MS2name+'/POINTING',direction,time) makePOLARIZAION(MS2name+'/POLARIZATION',path_temp+'POLARIZATION') makePROCESSOR(MS2name+'/PROCESSOR',path_temp+'PROCESSOR') makeSOURCE(MS2name+'/SOURCE',path_temp+'SOURCE',ra,dec,sysvel,sourcename,time,freq) makeSPECTRAL_WINDOW(MS2name+'/SPECTRAL_WINDOW',path_temp+'SPECTRAL_WINDOW',freq) makeSTATE(MS2name+'/STATE',path_temp+'STATE') makeSYSCAL(MS2name+'/SYSCAL',path_temp+'SYSCAL',Tsys,Tsys_time,time) makeWEATHER(MS2name+'/WEATHER',path_temp+'WEATHER') makeLMT_ARRAY(MS2name+'/LMT_ARRAY',path_temp+'LMT_ARRAY') # put keywords abs_path = os.path.abspath(MS2name) keywords = {'MS_VERSION': 2.0, 'ANTENNA': 'Table: '+abs_path+'/ANTENNA', 'DATA_DESCRIPTION': 'Table: '+abs_path+'/DATA_DESCRIPTION', 'DOPPLER': 'Table: '+abs_path+'/DOPPLER', 'FEED': 'Table: '+abs_path+'/FEED', 'FIELD': 'Table: '+abs_path+'/FIELD', 'FLAG_CMD': 'Table: '+abs_path+'/FLAG_CMD', 'FREQ_OFFSET': 'Table: '+abs_path+'/FREQ_OFFSET', 'HISTORY': 'Table: '+abs_path+'/HISTORY', 'OBSERVATION': 'Table: '+abs_path+'/OBSERVATION', 'POINTING': 'Table: '+abs_path+'/POINTING', 'POLARIZATION': 'Table: '+abs_path+'/POLARIZATION', 'PROCESSOR': 'Table: '+abs_path+'/PROCESSOR', 'SOURCE': 'Table: '+abs_path+'/SOURCE', 'SPECTRAL_WINDOW': 'Table: '+abs_path+'/SPECTRAL_WINDOW', 'STATE': 'Table: '+abs_path+'/STATE', 'SYSCAL': 'Table: '+abs_path+'/SYSCAL', 'WEATHER': 'Table: '+abs_path+'/WEATHER', 'LMT_ARRAY': 'Table: '+abs_path+'/LMT_ARRAY', } returnedTable = tb.table(MS2name,readonly=False) returnedTable.putkeywords(keywords) returnedTable.flush(recursive=True) returnedTable.close() if removetemp: os.system('rm -rf '+path_temp)
def selfcal(vis,minuvw,robust,model='MODEL',outcal_root='',max_sol=600.0,init_sol=30.0,\ incol='DATA',outcol='DATA',caltype='P',nchan=0): if not model: imaging(vis, 1000, 10, minuvw, robust) # need a predict step to deal with sourcedb here if necessary ptant = casatb.table(vis + '/ANTENNA') antenna_list = np.array([], dtype='S') for i in ptant.select('NAME'): antenna_list = np.append(antenna_list, i.values()) nant = len(antenna_list) if caltype == 'P': sol_int_range = np.arange( np.ceil(np.log(max_sol / init_sol) / np.log(3.))) sol_int_range = np.ceil(init_sol * 3.**sol_int_range) nsol = len(sol_int_range) coh = CCRIT * np.ones((nsol, nant)) for i in range(nsol): solint = sol_int_range[i] outcal_root = outcal_root if len(outcal_root) else vis outcal = outcal_root + '_c%d.h5' % i loop3log( vis, '\n--- Beginning pass with solint %.1f sec ---' % (solint)) calib (vis, solint=solint, outcal=outcal, incol=incol, \ outcol=outcol,solmode='P',tsamp=TSAMP,nchan=nchan) snplt(vis, htab=outcal, outpng=outcal) coh[i] = coherence_metric(outcal, antenna_list) loop3log(vis, '\nCoherences by antenna:') for j in range(nant): loop3log(vis, '%.2f ' % (coh[i, j]), cret=not ((j + 1) % 10)) loop3log(vis, ' ') if len(coh[i][coh[i] >= CCRIT]) == 0: # all coherent break # For each antenna in the antenna list, find the selfcal table with # the shortest solution interval that contains coherent solutions. If # there is no such table, report -1 in order to signal that they should # all be set to zero. ncoh = np.ones(nant, dtype=int) * -1 allcoh = np.ones(nant, dtype=float) * CCRIT for i in range(nant): try: ncoh[i] = np.min(np.ravel(np.argwhere(coh[:, i] < CCRIT))) allcoh[i] = coh[:, i][ncoh[i]] except: pass loop3log(vis, '\nCombined coherences: ') for i in range(nant): loop3log(vis, '%.2f ' % (allcoh[i]), cret=not ((i + 1) % 10)) loop3log(vis, '\nSolution number with first coherence (-1=no coh):') for i in range(nant): loop3log(vis, '%d ' % (ncoh[i]), cret=not ((i + 1) % 10)) loop3log(vis, ' ') loop3log(vis, ' ----- Starting edit of this solution ------') # For each selfcal table containing the shortest solution interval with # coherence on some antennas, replace the entries in the first selfcal # table with the interpolated values from that antenna for i in range(1, coh.shape[0]): iant = antenna_list[ncoh == i] loop3log( vis, 'Editing %d antennas to h5parm number %d' % (len(iant), i)) if len(iant): clcal (vis,outcal_root+'_c0.h5',outcal_root+'_c%d.h5'%i,\ ant_interp=iant) # For each antenna without any coherence at all, zero the phase # solutions for that antenna iant = antenna_list[ncoh == -1] if len(iant): loop3log(vis, 'Trying to zero antennas: ', cret=False) for i in range(len(iant)): loop3log(vis, '%s ' % iant[i], cret=False) loop3log(vis, '\n') zerosol(vis, outcal_root + '_c0.h5', iant) else: # amplitude selfcal: only one interval outcal_root = outcal_root if len(outcal_root) else vis outcal = outcal_root + '_c0.h5' % i calib (vis, solint=init_sol, outcal=outcal, incol=incol, \ outcol=outcol,solmode='A', nchan=nchan) snplt(vis, htab=outcal, outpng=outcal, soltab='amplitude000') allcoh = coherence_metric(outcal, antenna_list) loop3log(vis, 'Coherences: \n') for i in range(nant): loop3log(vis, '%.2f ' % (allcoh[i]), cret=not ((i + 1) % 10)) loop3log(vis, ' ') # For each antenna without any coherence at all, zero the amp/phase # solutions for that antenna # corrected bug here (Neal) - used to be < so everything was zeroed iant = antenna_list[allcoh >= CCRIT] print('******', allcoh) print('******>>> ', iant) if len(iant): zerosol(vis, outcal_root + '_c0.h5', ant=iant) # find the maximum baseline length with coherent cal signal cohlength = getcoh_baseline(antenna_list, allcoh, CCRIT) return allcoh, cohlength
def makePOINTING2(tablename,direction,time,nbeam=1,texp=0.2): # modules import casacore.tables as tb # params direction_rad = direction/180.*np.pi n_direction = direction.shape[0] time_end = time.max() # make table colnames = ['DIRECTION', 'ANTENNA_ID', 'INTERVAL', 'NAME', 'NUM_POLY', 'TARGET', 'TIME', 'TIME_ORIGIN', 'TRACKING'] colkeywords = [{'MEASINFO': {'Ref': 'J2000', 'type': 'direction'},'QuantumUnits': np.array(['rad', 'rad'],dtype='|S4')}, {}, {'QuantumUnits': np.array(['s'],dtype='|S2')}, {},{}, {'MEASINFO': {'Ref': 'J2000', 'type': 'direction'},'QuantumUnits': np.array(['rad', 'rad'],dtype='|S4')}, {'MEASINFO': {'Ref': 'UTC', 'type': 'epoch'}, 'QuantumUnits': np.array(['s'],dtype='|S2')}, {'MEASINFO': {'Ref': 'UTC', 'type': 'epoch'}, 'QuantumUnits': np.array(['s'],dtype='|S2')}, {} ] #ndims = [2,1,1,1,1,2,1,1,1] #isarrays = [True,False,False,False,False,True,False,False,False] valuetypes = ['double','int','double','string','int','double','double','double','bool'] ndims = [2,1,1,1,1,-1,1,1,1] descs = [] for colname,colkeyword,valuetype,ndim in zip(colnames,colkeywords,valuetypes,ndims): if (colname=='DIRECTION' or colname=='TARGET'): descs.append(tb.makearrcoldesc(colname,0.0,datamanagertype='StandardStMan',datamanagergroup='StandardStMan',ndim=ndim,keywords=colkeyword,valuetype=valuetype,options=0)) else: descs.append(tb.makescacoldesc(colname,0.0,datamanagertype='StandardStMan',datamanagergroup='StandardStMan',keywords=colkeyword,valuetype=valuetype)) td = tb.maketabdesc(descs=descs) returned_table = tb.table(tablename,tabledesc=td,nrow=n_direction,readonly=False) value = np.zeros([n_direction,1,2],dtype='float64') value[:,0] = direction_rad.copy() returned_table.putcol('DIRECTION',value) value = np.zeros([n_direction],dtype='int32') returned_table.putcol('ANTENNA_ID',value) returned_table.putcol('NUM_POLY',value) value = np.zeros([n_direction],dtype='float64') returned_table.putcol('TIME_ORIGIN',value) value = np.zeros([n_direction],dtype='bool') returned_table.putcol('TRACKING',value) value = np.full([n_direction],texp,dtype='float64') returned_table.putcol('INTERVAL',value) value = time returned_table.putcol('TIME',value) returned_table.flush() returned_table.close()
def main (vis,strategy='P30,P30,P30,A500,A450,A400',startmod='',ith=5.0,\ bandwidth='8MHz',goodness=2.,minuvw=50.0,robust=-1.0): ## format arguments strategy = str(strategy) startmod = str(startmod) ith = float(ith) bandwidth = str(bandwidth) minuvw = float(minuvw) * 1000.0 # convert km -> m robust = float(robust) ## process arguments vis = vis.rstrip('/') vis = vis.split('/')[-1] strategy = strategy.split(',') bw_val = '' bw_unit = '' for c in bandwidth: try: float(c) bw_val = bw_val + c except ValueError: bw_unit = bw_unit + c if bw_unit == 'MHz': bw_val = float(bw_val) * 1e6 ## get bandwidth of vis spec_info = casatb.table(vis + '::SPECTRAL_WINDOW') total_bw = spec_info.getcol('TOTAL_BANDWIDTH')[0] num_chan = spec_info.getcol('NUM_CHAN')[0] spec_info.close() if total_bw < bw_val: wsclean_chans = 0 mfs = '' nchan = 0 else: wsclean_chans = int(np.ceil(total_bw / bw_val)) mfs = '-MFS' nchan = num_chan / wsclean_chans ## make a working directory and go there tmp_dir = 'loop3_' + vis.rstrip('.ms').rstrip('.MS') os.system('mkdir %s' % tmp_dir) os.chdir(tmp_dir) os.system('mv ../%s .' % vis) import bdsf prevstat = 0.0 cohlength = 2.0E6 strategy_type = [] for i in strategy: strategy_type.append(i[0]) ploop, nloop, snver = strategy_type.count('P'), len(strategy), 0 # # PHASE CALIBRATION - run through ploop iterations, exiting if we have convergence # for iloop in range(ploop): fitsmask = vis + '_%02d-mask.fits' % (iloop - 1) if iloop else '' if startmod == '' or iloop: pstr = '******* PHASE LOOP %d running wsclean ************' % iloop loop3log(vis, pstr + '\n') imagr(vis,minuvwm=minuvw,robust=robust,cellsize='0.05asec',domultiscale=True,\ outname=vis+'_%02d'%iloop,channelsout=wsclean_chans,\ fitsmask=fitsmask,dolocalrms=True,maxuvwm=cohlength) else: # Need something here to produce an image from startmod pass # check if there's a source thisstat = measure_statistic2(vis + '_%02d%s-image.fits' % (iloop, mfs)) if thisstat < goodness: pstr = 'SNR is %f, breaking out of loop.' % thisstat loop3log(vis, pstr + '\n') montage_plot('*MFS-image.fits', imscale=0.65, nup='4x2', plot_resid=False) return (0) pstr = '******* PHASE LOOP %d making mask %s_%02d%s-image.fits ********' % ( iloop, vis, iloop, mfs) loop3log(vis, pstr + '\n') stdout = sys.stdout sys.stdout = open('bdsf_chunterings', 'a') img = bdsf.process_image('%s_%02d%s-image.fits' % (vis, iloop, mfs), atrous_do=True, thresh_isl=ith) sys.stdout.close() sys.stdout = stdout img.export_image(img_type='island_mask', outfile='%s_%02d-mask.fits' % (vis, iloop)) # exit loop if clean finishing pstr = '******* PHASE LOOP %d goodness stat %f ************' % ( iloop, thisstat) loop3log(vis, pstr + '\n') if thisstat - prevstat < 0.01: pstr = '****** EXITING PHASE CAL with diff %f *********' % ( thisstat - prevstat) loop3log(vis, pstr + '\n') break else: prevstat = thisstat imagr(vis,minuvwm=minuvw,robust=robust,dopredict=True,fitsmask=fitsmask,\ autothreshold=3,dolocalrms=True,\ outname=vis+'_%02d%s'%(iloop,mfs)) pstr = '******* PHASE LOOP %d making new cal file %s ************' % ( iloop, vis + '_%02d' % iloop) loop3log(vis, pstr + '\n') caltype, sol0 = strategy[iloop][0], float(strategy[iloop][1:]) coh, cohlength = selfcal(vis,minuvw,robust,model='MODEL',incol='DATA',\ outcol='CORRECTED_DATA',outcal_root=vis+'_%02d'%iloop,\ caltype=caltype,init_sol=sol0,nchan=nchan) snver = iloop pstr='******** END PHASE LOOP %d - coherence on %.1f km **********' % \ (iloop,cohlength/1000.) loop3log(vis, pstr + '\n') # Exit at this point if we are not doing amplitude cal if ploop == nloop: exit() # # If we are doing amplitude calibration, we now need to apply the # calibration and write a new MS with a DATA column visA = vis + '_A' # delete all existing files beginning with vis+'_A' os.system('rm -fr %s*' % visA) pstr = '****** APPLYING CALIBRATION TABLE %d\n' % snver loop3log(vis, pstr + '\n') applycal_split(vis, visA, 'sol000', '%s_%02d_c0.h5' % (vis, snver)) init_fitsmask = vis + '_%02d-mask.fits' % iloop init_img = vis + '_%02d%s-image.fits' % (iloop, mfs) pred_img = vis + '_%02d%s' % (iloop, mfs) for iloop in range(ploop, nloop): fitsmask = init_fitsmask if iloop == ploop else visA + '_%02d-mask.fits' % ( iloop - 1) pstr = '******* AMPLITUDE LOOP %d running wsclean ************' % iloop loop3log(vis, pstr + '\n') imagr(visA,minuvwm=minuvw,robust=robust,cellsize='0.05asec',domultiscale=True,\ outname=visA+'_%02d'%iloop,channelsout=wsclean_chans,\ fitsmask=fitsmask,dolocalrms=True,maxuvwm=cohlength) ## check if there's a source thisstat = measure_statistic2(visA + '_%02d%s-image.fits' % (iloop, mfs)) if thisstat < goodness: pstr = 'SNR is %f, breaking out of loop.' % thisstat loop3log(vis, pstr + '\n') montage_plot('*MFS-image.fits', imscale=0.65, nup='4x2', plot_resid=True) return (0) image_bdsf = '%s_%02d%s-image.fits' % (visA, iloop, mfs) pstr = '******* AMPLITUDE LOOP %d making mask %s_%02d%s-image.fits ************' % ( iloop, visA, iloop, mfs) loop3log(vis, pstr + '\n') img = bdsf.process_image(image_bdsf, atrous_do=True, thresh_isl=ith) img.export_image(img_type='island_mask', outfile='%s_%02d-mask.fits' % (visA, iloop)) pstr = '******* AMPLITUDE LOOP %d goodness stat %f ************' % ( iloop, thisstat) loop3log(vis, pstr + '\n') if iloop != ploop and thisstat - prevstat < 0.01: pstr = '****** EXITING AMPLITUDE CAL with diff %f *********' % ( thisstat - prevstat) loop3log(vis, pstr + '\n') break else: prevstat = thisstat imagr(visA,minuvwm=minuvw,dopredict=True,fitsmask=fitsmask,\ autothreshold=3,dolocalrms=True,robust=robust,\ outname=visA+'_%02d%s'%(iloop,mfs)) pstr = '******* AMPLITUDE LOOP %d making new cal file %s ************' % ( iloop, visA + '_%02d' % iloop) loop3log(vis, pstr + '\n') caltype, sol0 = strategy[iloop][0], float(strategy[iloop][1:]) coh,cohlength = selfcal(visA,minuvw,robust,model='MODEL',incol='DATA',\ outcol='CORRECTED_DATA',outcal_root=visA+'_%02d'%iloop,\ caltype=caltype,init_sol=sol0,nchan=nchan) pstr='******** END AMPLITUDE LOOP %d - coherence on %.1f km **********' % \ (iloop,cohlength/1000.) loop3log(vis, pstr + '\n') fitsmask = init_fitsmask if iloop == ploop else visA + '_%02d-mask.fits' % ( iloop - 1) imagr(visA,minuvwm=minuvw,cellsize='0.05asec',domultiscale=True,\ outname=visA+'_final',channelsout=wsclean_chans,robust=robust,\ fitsmask=fitsmask,dolocalrms=True) ## make a model from the final image final_im = glob.glob('*final*image.fits') if len(final_im) > 1: tmp = [a for a in final_im if 'MFS' in a] final_im = tmp img = bdsf.process_image(final_im[0], atrous_do=True, thresh_isl=ith) skyfile = final_im[0].replace('fits', 'skymodel') img.write_catalog(outfile=skyfile, bbs_patches='single', catalog_type='gaul', format='bbs') ## convert it to a sourcedb ss = "makesourcedb in=%s out=%s format='<'" % ( skyfile, skyfile.replace('skymodel', 'sky')) os.system(ss) ## plot things like solutions make_plots(vis) make_plots(visA) ## If we got to this point, self-cal has successfully completed # montage_plot( '*MFS-image.fits', imscale=0.65, nup='4x2', plot_resid=True) pngfile, h5files = cleanup(vis) for h5file in h5files: os.system('mv %s ../' % h5file) # os.system('mv *.pdf ../') # os.system('mv *.png ../') os.system('mv *skymodel ../') os.system('mv *sky ../') os.system('mv %s ../' % vis) print 'Output calibration tables', h5files #return pngfile,h5files return 0
def main(): parser = argparse.ArgumentParser() parser.add_argument('--xm', required=True) parser.add_argument('--ms', required=True) parser.add_argument('--center', required=True) parser.add_argument('--radius', type=float, default=20, help="TEC map radius (degrees)") parser.add_argument('--delta', type=float, default=0.25, help="Pixel size (degrees)") args = parser.parse_args() # Parse image center value center = SkyCoord(args.center, unit=(units.hourangle, units.degree)) # Open measurement set, and find central time and frequency ms = table(args.ms) times = sorted(set(ms.getcol('TIME_CENTROID'))) midtime = (max(times) + min(times)) / 2 # We just starting with the central time nearesttime = min(times, key=lambda x: abs(x - midtime)) freqs = table(args.ms + '::SPECTRAL_WINDOW').getcell( 'CHAN_FREQ', 0) # Assume just one SPW entry midfreq = (max(freqs) + min(freqs)) / 2 print("Calculating TEC values for frequency %f and time %f" % (midfreq, times[0]), file=sys.stderr) # prototype = fits.open(args.prototype)[0] # header = prototype.header # data = prototype.data # wcs = WCS(prototype) # # Calculate image dimension # # We want our TEC image to be at least as large as the original image # # Calculating this is yucky due to projection warping # width, height = data.shape[3], data.shape[2] # [ stokes, freq, dec (y), ra (x) ] # center_x, center_y = width // 2, height // 2 # center_ra, center_dec, _, _ = wcs.all_pix2world([[center_x, center_y, 0, 0]], 0)[0] # # Calculate maximum angular extent of image # corners = wcs.all_pix2world([ # [0, 0, 0, 0], # [width, 0, 0, 0], # [0, height, 0, 0], # [width, height, 0, 0] # ], 0) # max_ra = max(abs(corners.T[0] - center_ra)) # max_dec = max(abs(corners.T[1] - center_dec)) # width, height = int((2 * max_ra) // args.delta) + 1, int((2 * max_dec) // args.delta) + 1 # center_x, center_y = int(width // 2), int(height // 2) # Provision TEC fits file and set up headers width, height = int((2 * args.radius) // args.delta), int( (2 * args.radius) // args.delta) center_x, center_y = int(width // 2), int(height // 2) print("Creating TEC image of dimesions (%d, %d)" % (width, height), file=sys.stdout) data = np.zeros((1, 1, 128, height, width), dtype=np.float) # [time, frequency, antennas, dec, ra] Atec = fits.PrimaryHDU(data) Atec.header['CTYPE1'] = 'RA---SIN' Atec.header['CRPIX1'] = center_x Atec.header['CRVAL1'] = center.ra.deg Atec.header['CDELT1'] = args.delta Atec.header['CUNIT1'] = 'deg' Atec.header['CTYPE2'] = 'DEC--SIN' Atec.header['CRPIX2'] = center_y Atec.header['CRVAl2'] = center.dec.deg Atec.header['CDELT2'] = args.delta Atec.header['CUNIT2'] = 'deg' Atec.header['CTYPE3'] = 'ANTENNA' Atec.header['CRPIX3'] = 1 Atec.header['CRVAL3'] = 0 Atec.header['CTYPE4'] = 'FREQ' Atec.header['CRPIX4'] = 1 Atec.header['CRVAL4'] = midfreq Atec.header['CDELT4'] = 1 Atec.header['CUNIT4'] = 'Hz' Atec.header['CTYPE5'] = 'TIME' Atec.header['CRPIX5'] = 1 Atec.header['CRVAL5'] = midtime # FIXME Atec.header['CDELT5'] = 1 wcs = WCS(Atec.header) # Process crossmatched sources xm = Table.read(args.xm) model_positions = np.radians(np.array([xm['model_ra'], xm['model_dec']])) source_positions = np.radians(np.array([xm['source_ra'], xm['source_dec']])) offsets = np.sin(model_positions - source_positions) # Filter out extreme offsets separations = angular_separation(model_positions[0], model_positions[1], source_positions[0], source_positions[1]) exclusions = separations > 2 * np.median(separations) print("Excluding %d / %d extremal offsets" % (sum(exclusions), len(exclusions))) model_positions = model_positions[:, ~exclusions] source_positions = source_positions[:, ~exclusions] offsets = offsets[:, ~exclusions] model_positions_lm = radec_to_lm(model_positions[0], model_positions[1], center.ra.rad, center.dec.rad) # Get oversampled l,m values for TEC file xx, yy = np.meshgrid(range(0, 3 * width), range(0, 3 * height)) pixels = np.array([xx.flatten(), yy.flatten()]).T ret = wcs.all_pix2world([[x / 3 - 1 / 3, y / 3 - 1 / 3, 0, 0, 0] for x, y in pixels], 0) grid_lm = radec_to_lm(np.radians(ret.T[0]), np.radians(ret.T[1]), center.ra.rad, center.dec.rad) # Compute interpolated position offsets delta_l = griddata(model_positions_lm.T, offsets[0], grid_lm.T, fill_value=0) delta_m = griddata(model_positions_lm.T, offsets[1], grid_lm.T, fill_value=0) delta_l = np.reshape(delta_l, (3 * height, 3 * width)) # [ dec, ra ] delta_m = np.reshape(delta_m, (3 * height, 3 * width)) # Gaussian smooth delta_l = gaussian_filter(delta_l, 3, mode='constant', cval=0) delta_m = gaussian_filter(delta_m, 3, mode='constant', cval=0) # Downsample delta_l = delta_l[1::3, 1::3] delta_m = delta_m[1::3, 1::3] # Create downsampled grid xx, yy = np.meshgrid(range(0, width), range(0, height)) pixels = np.array([xx.flatten(), yy.flatten()]).T ret = wcs.all_pix2world([[x, y, 0, 0, 0] for x, y in pixels], 0) grid_lm = radec_to_lm(np.radians(ret.T[0]), np.radians(ret.T[1]), center.ra.rad, center.dec.rad) # Plot interpolation plt.figure() plt.quiver(model_positions_lm[0], model_positions_lm[1], offsets[0], offsets[1], angles='xy', scale=0.01, scale_units='xy') plt.quiver(grid_lm[0], grid_lm[1], delta_l.flatten(), delta_m.flatten(), angles='xy', scale=0.01, scale_units='xy', color='gray') plt.gca().invert_xaxis() plt.show() # Create debugging fits files for delta, name in [(delta_l, 'delta-l'), (delta_m, 'delta-m')]: hdu = fits.PrimaryHDU(delta) hdu.header = Atec.header.copy() hdu.writeto(name + '.fits', overwrite=True) for i, time in enumerate([midtime]): for ant in range(1, 128): sys.stderr.write("\rCalculating antenna %d..." % ant) sys.stderr.flush() tbl = taql( "select UVW from $ms where TIME_CENTROID = $nearesttime and ANTENNA1 = 0 and ANTENNA2 = $ant" ) if len(tbl) == 0: continue elif len(tbl) > 1: print("Oopsie doodle!") exit(1) u, v = tbl.getcol('UVW')[0][[True, True, False ]] / (299792458 / midfreq) phase = 2 * np.pi * (u * delta_l + v * delta_m) data[i, 0, ant, :, :] = phase / -8.44797245E9 * midfreq print(" Done.", file=sys.stderr) Atec.data = data Atec.writeto('tec.fits', overwrite=True)
def check_phaseref_in_MS(mspath, phaseref, sep_threshold=1., frame='icrs', ack=False): """Check if a given phase-reference point is amongst the phase centre of an MS for any field and direction existing in that MS This function is needed as the Phase centre referencing is NOT clear in the MS format using ASKAP observations. Parameters ========== mspath: str The input MS path phaseref: Astropy coordinate Astropy SkyCoord object with the same frame as the :param frame: parameter sep_threshold: float Maximum allowed separation between the given phaseref and the phasecentres in the MS. The separation is defined in arcesconds. If the phaseref and the phasecentre within the separation, it counts as a match frame: str, optional Reference frame used to calculate the Astropy phase centre. Default: 'icrs' ack: bool, optional Enabling messages of successful interaction with the MS e.g. successful opening of a table Returns ======= IDs: list of lists If the phase reference given matches with at least one of the phasecentre in the MS, the filed index and direction index is returned as a list. Else an empty list is returned. The returned indices are the field followed by direction for each match """ assert type(phaseref) == type(SkyCoord(ra = 0 * u.deg, dec = 0 * u.deg, frame=frame, equinox='J2000')), 'Input phaseref is not an astropy SkyCoord object!' MS = ds.msutil.create_MS_object(mspath, ack=ack) fields_table = casatables.table(mspath + '/FIELD', ack=ack) #Get the reference equinox from the table keywords equinox = fields_table.getcolkeyword('PHASE_DIR','MEASINFO')['Ref'] #Only can convert from radians assert fields_table.getcolkeyword('PHASE_DIR','QuantumUnits')[0] == 'rad', 'Phase centre direction is not in radians!' IDs = [] for d in range(0,np.shape(fields_table.getcol('PHASE_DIR'))[0]): for f in range(0,np.shape(fields_table.getcol('PHASE_DIR'))[1]): pc = fields_table.getcol('PHASE_DIR')[d,f, :] if phaseref.separation(SkyCoord(ra=pc[0] * u.rad, dec=pc[1] * u.rad, frame=frame, equinox=equinox)).arcsecond <= sep_threshold: IDs.append([f,d]) MS.close() return IDs
def copy_column_to_ms(ms, inputcol, outputcol, ms_from=None, use_compression=False): """ Copies one column to another, within an MS file or between two MS files Parameters ---------- ms : str MS file receiving copy inputcol : str Column name to copy from outputcol : str Column name to copy to ms_from : str, optional MS file to copy from. If None, the column is copied internally """ t = pt.table(ms, readonly=False, ack=False) if ms_from is not None: tf = pt.table(ms_from, readonly=False, ack=False) data = tf.getcol(inputcol) desc = tf.getcoldesc(inputcol) else: data = t.getcol(inputcol) desc = t.getcoldesc(inputcol) # Add the output column if needed if outputcol not in t.colnames(): if use_compression: # Set DyscoStMan to be storage manager for DATA and WEIGHT_SPECTRUM # We use a visibility bit rate of 16 and truncation of 1.5 sigma to keep the # compression noise below ~ 0.01 mJy, as estimated from Fig 4 of # Offringa (2016). For the weights, we use a bit rate of 12, as # recommended in Sec 4.4 of Offringa (2016) desc['name'] = outputcol dmi = { 'SPEC': { 'dataBitCount': numpy.uint32(16), 'distribution': 'TruncatedGaussian', 'distributionTruncation': 1.5, 'normalization': 'RF', 'weightBitCount': numpy.uint32(12) }, 'NAME': '{}_dm'.format(outputcol), 'SEQNR': 1, 'TYPE': 'DyscoStMan' } desc['option'] = 1 # make a Direct column t.addcols(desc, dmi) else: desc['name'] = outputcol t.addcols(desc) if use_compression: # Replace flagged values with NaNs before compression flags = t.getcol('FLAG') flagged = numpy.where(flags) data[flagged] = numpy.NaN t.putcol(outputcol, data) t.flush() t.close()
def get_MS_phasecentre_all(mspath, frame='icrs', ack=False): """Get the list of the phase centres for each field and direction of the MS and return a list of astropy skycoord values Both field and direction IDs are expected to increment from zero, and the maximum ID can be the number of unique fields/dds. However, less than the maximum number of valid IDs can occurs and this code can handle that. e.g. one field and one direction ID, but in the PHASE_DIR table, phase centre for two directions are existing, the code chooses the valid one Parameters ========== mspath: str The input MS path frame: str, optional Reference frame used to calculate the Astropy phase centre. Default: 'icrs' ack: bool, optional Enabling messages of successful interaction with the MS e.g. successful opening of a table Returns ======= phasecentres: list of lists containing Astropy skycoords A list of the phasecentres for each field and direction in the MS as a list of lists i.e. each element is a list """ MS = ds.msutil.create_MS_object(mspath, ack=ack) #Get the number of unique fields and data descriptoions (e.g. footprints) fields = np.unique(MS.getcol('FIELD_ID')) dds = np.unique(MS.getcol('DATA_DESC_ID')) fields_table = casatables.table(mspath + '/FIELD', ack=ack) phasecentres = [] #Get the reference equinox from the table keywords equinox = fields_table.getcolkeyword('PHASE_DIR','MEASINFO')['Ref'] #Only can convert from radians assert fields_table.getcolkeyword('PHASE_DIR','QuantumUnits')[0] == 'rad', 'Phase centre direction is not in radians!' i = 0 j = 0 for field in range(0,np.size(fields)): #The number and referencing of fields can be messy if np.shape(fields_table.getcol('PHASE_DIR'))[1] > np.size(fields): field_ID = fields[i] else: field_ID = field directions = [] for dd in range(0,np.size(dds)): #Same for the DDs as the fields if np.shape(fields_table.getcol('PHASE_DIR'))[0] > np.size(dds): dd_ID = dds[i] else: dd_ID = dd pc = fields_table.getcol('PHASE_DIR')[dd_ID,field_ID, :] #Convert to astropy coordinates directions.append(SkyCoord(ra=pc[0] * u.rad, dec=pc[1] * u.rad, frame=frame, equinox=equinox)) j += 1 phasecentres.append(directions) i += 1 MS.close() return phasecentres
def main(input_image, input_skymodel_pb, input_bright_skymodel_pb, output_root, vertices_file, threshisl=5.0, threshpix=7.5, rmsbox=(150, 50), rmsbox_bright=(35, 7), adaptive_rmsbox=True, use_adaptive_threshold=False, adaptive_thresh=75.0, beamMS=None, peel_bright=False): """ Filter the input sky model so that they lie in islands in the image Parameters ---------- input_image : str Filename of input image to use to detect sources for filtering. Ideally, this should be a flat-noise image (i.e., without primary-beam correction) input_skymodel_pb : str Filename of input makesourcedb sky model, with primary-beam correction input_bright_skymodel_pb : str Filename of input makesourcedb sky model of bright sources only, with primary- beam correction output_root : str Root of filename of output makesourcedb sky models. Output filenames will be output_root+'.apparent_sky.txt' and output_root+'.true_sky.txt' vertices_file : str Filename of file with vertices threshisl : float, optional Value of thresh_isl PyBDSF parameter threshpix : float, optional Value of thresh_pix PyBDSF parameter rmsbox : tuple of floats, optional Value of rms_box PyBDSF parameter rmsbox_bright : tuple of floats, optional Value of rms_box_bright PyBDSF parameter adaptive_rmsbox : tuple of floats, optional Value of adaptive_rms_box PyBDSF parameter use_adaptive_threshold : bool, optional If True, use an adaptive threshold estimated from the negative values in the image adaptive_thresh : float, optional If adaptive_rmsbox is True, this value sets the threshold above which a source will use the small rms box peel_bright : bool, optional If True, bright sources were peeled, so add then back before filtering """ if rmsbox is not None and isinstance(rmsbox, str): rmsbox = eval(rmsbox) if isinstance(rmsbox_bright, str): rmsbox_bright = eval(rmsbox_bright) adaptive_rmsbox = misc.string2bool(adaptive_rmsbox) use_adaptive_threshold = misc.string2bool(use_adaptive_threshold) if isinstance(beamMS, str): beamMS = misc.string2list(beamMS) peel_bright = misc.string2bool(peel_bright) # Try to set the TMPDIR evn var to a short path, to ensure we do not hit the length # limits for socket paths (used by the mulitprocessing module). We try a number of # standard paths (the same ones used in the tempfile Python library) old_tmpdir = os.environ["TMPDIR"] for tmpdir in ['/tmp', '/var/tmp', '/usr/tmp']: if os.path.exists(tmpdir): os.environ["TMPDIR"] = tmpdir break # Run PyBDSF to make a mask for grouping if use_adaptive_threshold: # Get an estimate of the rms by running PyBDSF to make an rms map img = bdsf.process_image(input_image, mean_map='zero', rms_box=rmsbox, thresh_pix=threshpix, thresh_isl=threshisl, thresh='hard', adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright, rms_map=True, quiet=True, stop_at='isl') # Find min and max pixels max_neg_val = abs(np.min(img.ch0_arr)) max_neg_pos = np.where(img.ch0_arr == np.min(img.ch0_arr)) max_pos_val = abs(np.max(img.ch0_arr)) max_pos_pos = np.where(img.ch0_arr == np.max(img.ch0_arr)) # Estimate new thresh_isl from min pixel value's sigma, but don't let # it get higher than 1/2 of the peak's sigma threshisl_neg = 2.0 * max_neg_val / img.rms_arr[max_neg_pos][0] max_sigma = max_pos_val / img.rms_arr[max_pos_pos][0] if threshisl_neg > max_sigma / 2.0: threshisl_neg = max_sigma / 2.0 # Use the new threshold only if it is larger than the user-specified one if threshisl_neg > threshisl: threshisl = threshisl_neg img = bdsf.process_image(input_image, mean_map='zero', rms_box=rmsbox, thresh_pix=threshpix, thresh_isl=threshisl, thresh='hard', adaptive_rms_box=adaptive_rmsbox, adaptive_thresh=adaptive_thresh, rms_box_bright=rmsbox_bright, atrous_do=True, atrous_jmax=3, rms_map=True, quiet=True) emptysky = False if img.nisl > 0: maskfile = input_image + '.mask' img.export_image(outfile=maskfile, clobber=True, img_type='island_mask') # Construct polygon needed to trim the mask to the sector header = pyfits.getheader(maskfile, 0) w = wcs.WCS(header) RAind = w.axis_type_names.index('RA') Decind = w.axis_type_names.index('DEC') vertices = misc.read_vertices(vertices_file) RAverts = vertices[0] Decverts = vertices[1] verts = [] for RAvert, Decvert in zip(RAverts, Decverts): ra_dec = np.array([[0.0, 0.0, 0.0, 0.0]]) ra_dec[0][RAind] = RAvert ra_dec[0][Decind] = Decvert verts.append((w.wcs_world2pix(ra_dec, 0)[0][RAind], w.wcs_world2pix(ra_dec, 0)[0][Decind])) hdu = pyfits.open(maskfile, memmap=False) data = hdu[0].data # Rasterize the poly data_rasertize = data[0, 0, :, :] data_rasertize = misc.rasterize(verts, data_rasertize) data[0, 0, :, :] = data_rasertize hdu[0].data = data hdu.writeto(maskfile, overwrite=True) # Now filter the sky model using the mask made above if len(beamMS) > 1: # Select the best MS for the beam attenuation ms_times = [] for ms in beamMS: tab = pt.table(ms, ack=False) ms_times.append(np.mean(tab.getcol('TIME'))) tab.close() ms_times_sorted = sorted(ms_times) mid_time = ms_times_sorted[int(len(ms_times)/2)] beam_ind = ms_times.index(mid_time) else: beam_ind = 0 try: s = lsmtool.load(input_skymodel_pb, beamMS=beamMS[beam_ind]) except astropy.io.ascii.InconsistentTableError: emptysky = True if peel_bright: try: # If bright sources were peeled before imaging, add them back s_bright = lsmtool.load(input_bright_skymodel_pb, beamMS=beamMS[beam_ind]) # Rename the bright sources, removing the '_sector_*' added previously # (otherwise the '_sector_*' text will be added every iteration, # eventually making for very long source names) new_names = [name.split('_sector')[0] for name in s_bright.getColValues('Name')] s_bright.setColValues('Name', new_names) if not emptysky: s.concatenate(s_bright) else: s = s_bright emptysky = False except astropy.io.ascii.InconsistentTableError: pass if not emptysky: s.select('{} == True'.format(maskfile)) # keep only those in PyBDSF masked regions if len(s) == 0: emptysky = True else: # Write out apparent and true-sky models del(img) # helps reduce memory usage s.group(maskfile) # group the sky model by mask islands s.write(output_root+'.true_sky.txt', clobber=True) s.write(output_root+'.apparent_sky.txt', clobber=True, applyBeam=True) else: emptysky = True if emptysky: # No sources cleaned/found in image, so just make a dummy sky model with single, # very faint source at center dummylines = ["Format = Name, Type, Patch, Ra, Dec, I, SpectralIndex, LogarithmicSI, " "ReferenceFrequency='100000000.0', MajorAxis, MinorAxis, Orientation\n"] ra, dec = img.pix2sky((img.shape[-2]/2.0, img.shape[-1]/2.0)) if ra < 0.0: ra += 360.0 ra = misc.ra2hhmmss(ra) sra = str(ra[0]).zfill(2)+':'+str(ra[1]).zfill(2)+':'+str("%.6f" % (ra[2])).zfill(6) dec = misc.dec2ddmmss(dec) decsign = ('-' if dec[3] < 0 else '+') sdec = decsign+str(dec[0]).zfill(2)+'.'+str(dec[1]).zfill(2)+'.'+str("%.6f" % (dec[2])).zfill(6) dummylines.append(',,p1,{0},{1}\n'.format(sra, sdec)) dummylines.append('s0c0,POINT,p1,{0},{1},0.00000001,' '[0.0,0.0],false,100000000.0,,,\n'.format(sra, sdec)) with open(output_root+'.apparent_sky.txt', 'w') as f: f.writelines(dummylines) with open(output_root+'.true_sky.txt', 'w') as f: f.writelines(dummylines) # Set the TMPDIR env var back to its original value os.environ["TMPDIR"] = old_tmpdir
def main(ms_input, input_colname, output_data_colname, output_weights_colname, baseline_file, delta_theta_deg, target_peak_reduction_factor=0.99): """ Pre-average data using a sliding Gaussian kernel in frequency Parameters ---------- ms_input : str MS filename input_colname : str Name of the column in the MS from which the data are read output_data_colname : str Name of the column in the MS into which the averaged data are written output_weights_colname : str Name of the column in the MS into which the averaged data weights are written baseline_file : str Filename of pickled baseline lengths delta_theta_deg : float Radius of calibration region in degrees target_peak_reduction_factor : float, optional Target reduction in peak flux density. Note: this reduction is in addition to any incurred by earlier averaging """ if os.path.exists(baseline_file): f = open(baseline_file, 'r') baseline_dict = pickle.load(f) f.close() else: print('Cannot find baseline_file. Exiting...') sys.exit(1) delta_theta_deg = float(delta_theta_deg) target_peak_reduction_factor = float(target_peak_reduction_factor) ms = pt.table(ms_input, readonly=False, ack=False) ant1_list = ms.getcol('ANTENNA1') ant2_list = ms.getcol('ANTENNA2') data_all = ms.getcol(input_colname) weights_all = ms.getcol('WEIGHT_SPECTRUM') flags = ms.getcol('FLAG') # Get lowest frequency of MS and channel width sw = pt.table(ms_input + '::SPECTRAL_WINDOW', ack=False) freq_hz = sw.col('CHAN_FREQ')[0][0] chan_width_hz = sw.col('CHAN_WIDTH')[0][0] flags[np.isnan(data_all)] = True # flag NaNs weights_all = weights_all * ~flags # set weight of flagged data to 0 # Check that all NaNs are flagged if np.count_nonzero(np.isnan(data_all[~flags])) > 0: logging.error('NaNs in unflagged data in {0}!'.format(ms_input)) sys.exit(1) # Weight data and set bad data to 0 so nans do not propagate data_all = np.nan_to_num(data_all * weights_all) # Iteration on baseline combination for ant in itertools.product(set(ant1_list), set(ant2_list)): if ant[0] >= ant[1]: continue sel1 = np.where(ant1_list == ant[0])[0] sel2 = np.where(ant2_list == ant[1])[0] sel_list = sorted(list(frozenset(sel1).intersection(sel2))) data = data_all[sel_list, :, :] weights = weights_all[sel_list, :, :] # compute the Gaussian sigma from the max bandwidth over which we # can average and avoid significant bandwidth smearing but limited to # no more than 3 MHz (to avoid smoothing over the beam-induced effects) lambda_km = 299792.458 / freq_hz dist_km = baseline_dict['{0}-{1}'.format(ant[0], ant[1])] if dist_km > 0: resolution_deg = lambda_km / dist_km * 180.0 / np.pi stddev_hz = min( 3e6, get_target_bandwidth(freq_hz, delta_theta_deg, resolution_deg, target_peak_reduction_factor) / 4.0) stddev_nchan = stddev_hz / chan_width_hz * np.sqrt(0.5 / dist_km) # smear weighted data and weights dataR = gfilter(np.real(data), stddev_nchan, axis=1) dataI = gfilter(np.imag(data), stddev_nchan, axis=1) weights = gfilter(weights, stddev_nchan, axis=1) # re-create data data = (dataR + 1j * dataI) data[(weights != 0)] /= weights[(weights != 0)] # avoid divbyzero data_all[sel_list, :, :] = data weights_all[sel_list, :, :] = weights # Add the output columns if needed if output_data_colname not in ms.colnames(): desc = ms.getcoldesc(input_colname) desc['name'] = output_data_colname ms.addcols(desc) if output_weights_colname not in ms.colnames(): desc = ms.getcoldesc('WEIGHT_SPECTRUM') desc['name'] = output_weights_colname ms.addcols(desc) ms.putcol(output_data_colname, data_all) ms.putcol('FLAG', flags) # this saves flags of nans, which is always good ms.putcol(output_weights_colname, weights_all) ms.close()
import casacore.tables as tables print 'tmp-compressed.ms:' t = tables.table('tmp-compressed.ms') print t.getdminfo() print 'tmp-uncompressed.ms:' t = tables.table('tmp-uncompressed.ms') print t.getdminfo() print 'tmp-weightcompressed.ms:' t = tables.table('tmp-weightcompressed.ms') print t.getdminfo()
def main(infile, clean_sig=6, map_size=512, pix_size=100, obs_length=900, datacolumn='CORRECTED_DATA', startmod=True, verbose=False, pols='I', catalogue=None): # current working directory current_dir = os.getcwd() ## make a working directory, move the data, and chdir # get filestem to make unique name infile = infile.rstrip('/') tmp = infile.split('/') filestem = tmp[-1].split('_')[0] # working directory work_dir = os.path.join(current_dir, 'difmap_{:s}'.format(filestem)) os.mkdir(work_dir) os.system('cp -r {:s} {:s}'.format(infile, work_dir)) os.chdir(work_dir) ## redefine infile infile = glob.glob(os.path.join(work_dir, tmp[-1]))[0] ## if a catalogue is specified, override the imaging parameters if catalogue is not None: print( 'Catalogue is specified, reading information to set imaging parameters.' ) ## get ra and dec [[[ra, dec]]] = ct.table(infile + '::FIELD', readonly=True).getcol('PHASE_DIR') # => shift for the negative angles before the conversion to deg (so that RA in [0;2pi]) if ra < 0: ra = ra + 2 * np.pi # convert radians to degrees ra_deg = ra / np.pi * 180. dec_deg = dec / np.pi * 180. tgt_coords = SkyCoord(ra_deg, dec_deg, unit='deg') t = Table.read(catalogue, format='csv') ## more flexible RA/DEC column naming mycols = t.colnames ra_col = [val for val in mycols if val == 'RA'] de_col = [val for val in mycols if val == 'DEC'] if len(ra_col) == 1: ra_col = ra_col[0] de_col = de_col[0] else: ## pick LoTSS position ra_col = [val for val in mycols if val == 'RA_LOTSS'][0] de_col = [val for val in mycols if val == 'DEC_LOTSS'][0] coords = SkyCoord(t[ra_col], t[de_col], unit='deg') seps = coords.separation(tgt_coords).value src_idx = np.where(seps == np.min(seps))[0] src_tmp = t[src_idx] ## use LGZ_Size if available, otherwise use DC_Maj if 'LGZ_Size' in src_tmp.colnames: ## in arcsec size_asec = src_tmp['LGZ_Size'][0] else: size_asec = src_tmp['DC_Maj'][0] * 60. * 60. padding = 1.5 possible_map_sizes = np.array([512, 1024, 2048, 4096, 8192]) possible_map_asec = possible_map_sizes * float( pix_size) * 1e-3 ## convert to arcsec possible_idx = np.where(size_asec * adding <= possible_map_asec)[0] if len(possible_idx) >= 1: map_size = possible_map_sizes[np.min(possible_idx)] print( 'Estimated source size {:s}, making image with {:s}x{:s} pixels ({:s}x{:s} arcsec)' .format(str(size_asec), str(map_size), str(map_size), str(map_size * float(pix_size) * 1e-3), str(map_size * float(pix_size) * 1e-3))) else: print( 'Image size exceeds {:s} arcseconds! Are you sure you want to make this image?' .format(str(np.max(possible_map_asec)))) print('Image size too large, aborting.') return total_flux = src_tmp['Total_flux'][0] if pols == 'I': ## use stokes I only for self-cal and imaging (best option) fitsfile = dif_script(infile, pol=pols, clean_sigma=clean_sig, map_size=map_size, pixel_size=pix_size, obs_length=obs_length, datacolumn=datacolumn, startmod=startmod) corpltfile = glob.glob(os.path.join(work_dir, 'CORPLT'))[0] ampI, amperrI, phsI, phserrI, utI, stnI = corplt2array(corpltfile) corpltout = insert_into_filestem( corpltfile.replace('CORPLT', '_CORPLT_I'), filestem) os.system('mv {:s} {:s}'.format(corpltfile, corpltout)) else: ## self-cal the polarisations separately ## create a difmap script fitsfile = dif_script(infile, pol='XX', clean_sigma=clean_sig, map_size=map_size, pixel_size=pix_size, obs_length=obs_length, datacolumn=datacolumn, startmod=startmod) ## plot solutions and get values corpltfile = glob.glob(os.path.join(work_dir, 'CORPLT'))[0] ampXX, amperrXX, phsXX, phserrXX, utXX, stnXX = corplt2array( corpltfile) corpltout = insert_into_filestem( corpltfile.replace('CORPLT', '_CORPLT_XX'), filestem) os.system('mv {:s} {:s}'.format(corpltfile, corpltout)) ## write a difmap script for the YY polarisation and run fitsfile = dif_script(infile, pol='YY', clean_sigma=clean_sig, map_size=map_size, pixel_size=pix_size, obs_length=obs_length, datacolumn=datacolumn, startmod=startmod) ## plot solutions and get values corpltfile = glob.glob(os.path.join(work_dir, 'CORPLT'))[0] ampYY, amperrYY, phsYY, phserrYY, utYY, stnYY = corplt2array( corpltfile) corpltout = insert_into_filestem( corpltfile.replace('CORPLT', '_CORPLT_YY'), filestem) os.system('mv {:s} {:s}'.format(corpltfile, corpltout)) if pols == 'I': ut = utI stn = stnI else: ut = utXX stn = stnXX ## convert time axis to lofar times myms = ct.table(infile) lof_times = myms.getcol('TIME') myms.close() utvec = ut - np.min(ut) time_ax = utvec + np.min(lof_times) ## get frequency axis myspw = ct.table(infile + '::SPECTRAL_WINDOW') freq_ax = np.squeeze(myspw.getcol('CHAN_FREQ')) myspw.close() if pols == 'I': ## split the stokes I correction across XX and YY tmp_amp = np.rollaxis( np.dstack((np.sqrt(ampI / 2.), np.sqrt(ampI / 2.))), 1, 0) tmp_phs = np.rollaxis(np.dstack((phsI, phsI)), 1, 0) else: ## combine XX and YY information and reformat axes tmp_amp = np.rollaxis(np.dstack((ampXX, ampYY)), 1, 0) tmp_phs = np.rollaxis(np.dstack((phsXX, phsYY)), 1, 0) ## expand to fill frequency axis tmp_amp2 = np.expand_dims(tmp_amp, axis=1) tmp_phs2 = np.expand_dims(tmp_phs, axis=1) amp = np.repeat(tmp_amp2, len(freq_ax), axis=1) phs = np.repeat(tmp_phs2, len(freq_ax), axis=1) phs = phs * -1.0 ## difmap has a different convention ## get antenna information new_ants = make_ant_table(stn) ## get pointing information ptg = ct.table(infile + '::FIELD') ptg_dir = ptg.getcol('PHASE_DIR')[0] ptg.close() new_dir = {} new_dir[filestem] = ptg_dir[0] ## write solutions to an h5parm h5parmfile = os.path.join(work_dir, filestem + '_sols.h5') out_h5 = h5parm(h5parmfile, readonly=False) out_solset = out_h5.makeSolset(solsetName='sol000') antenna_table = out_solset.obj._f_get_child('antenna') antenna_table.append(new_ants.items()) out_solset.obj.source.append(new_dir.items()) out_solset.makeSoltab('amplitude', axesNames=['time', 'freq', 'ant', 'pol'], axesVals=[time_ax, freq_ax, stn, ['XX', 'YY']], vals=amp, weights=np.ones_like(amp)) out_solset.makeSoltab('phase', axesNames=['time', 'freq', 'ant', 'pol'], axesVals=[time_ax, freq_ax, stn, ['XX', 'YY']], vals=phs, weights=np.ones_like(phs)) out_h5.close() ## apply the solutions with open('applysols.parset', 'w') as f: f.write('msin={:s}\n'.format(infile)) f.write('msin.datacolumn=DATA\n') f.write('msout=.\n') f.write('msout.datacolumn=CORRECTED_DATA\n') f.write('numthreads=6\n') f.write('steps=[applycal]\n') f.write('applycal.type=applycal\n') f.write('applycal.parmdb={:s}\n'.format(h5parmfile)) f.write('applycal.steps=[applyphs,applyamp]\n') f.write('applycal.applyphs.correction=phase000\n') f.write('applycal.applyamp.correction=amplitude000\n') f.close() ss = 'NDPPP applysols.parset > applysols.log 2>&1' os.system(ss) ## rename files so they have unique names diflogs = glob.glob(os.path.join(work_dir, 'dif*')) for diflog in diflogs: diflogout = insert_into_filestem(diflog, filestem + '_') os.system('mv {:s} {:s}'.format(diflog, diflogout)) ## Make a BBS format skymodel from the difmap XX model ## get reference frequency rf = ct.taql( 'select REF_FREQUENCY from {:s}::SPECTRAL_WINDOW'.format(infile)) ref_freq = rf.getcol('REF_FREQUENCY')[0] # find the model file modfiles = glob.glob(filestem + '*.mod') if pols == 'I': xx_file = modfiles[0] else: xx_file = [mf for mf in modfiles if 'XX' in mf][0] ## read the model xx_mod = read_mod(xx_file) ## find central point xx_cen = find_centre(xx_mod) ## and get the flux xx_flux = [] for mykey in xx_mod.keys(): xx_flux.append(xx_mod[mykey][0]) ## write the model file outfile = os.path.join(work_dir, filestem + '_StokesI.mod') with open(outfile, 'w') as f: f.write( "# (Name, Type, Patch, Ra, Dec, I, Q, U, V, MajorAxis, MinorAxis, Orientation, ReferenceFrequency='{:s}', SpectralIndex='[]') = format\n\n" .format(str(ref_freq))) ## patch name patch_cen = lof_coords( SkyCoord(xx_cen[0], xx_cen[1], frame='icrs', unit='deg')) f.write(", , difmap_cc, {:s}\n".format(patch_cen)) for mykey in xx_mod.keys(): flux = xx_mod[mykey][0] coords = lof_coords(xx_mod[mykey][1]) f.write( "{:s}, POINT, difmap_cc, {:s}, {:s}, 0.0, 0.0, 0.0, 0.00000e+00, 0.00000e+00, 0.00000e+00, {:s}, [-0.8]\n" .format(mykey, coords, str(flux), str(ref_freq))) f.close() ## convert to a sourcedb ss = 'makesourcedb in={:s} out={:s} format="<"'.format( outfile, outfile.replace('mod', 'skymodel')) os.system(ss) ## run wsclean #wsclean_name = filestem + '_wsclean' ## convert the im params to wsclean format #ss = 'wsclean -j 16 -mem 20 -v -reorder -update-model-required -weight uniform -mf-weighting -weighting-rank-filter 3 -name {:s} -size {:s} {:s} -padding 1.4 -scale {:s}asec -channels-out 6 -data-column CORRECTED_DATA -niter 10000 -auto-threshold 3 -auto-mask 5 -mgain 0.8 -join-channels -fit-spectral-pol 3 -fit-beam {:s}'.format( wsclean_name, str(map_size), str(map_size), str(float(pix_size)*0.001), infile ) #os.system( ss ) ## TO DO: move final files ## h5parm, images, log files, and skymodel image_files = glob.glob(os.path.join(work_dir, '*.ps')) log_files = glob.glob(os.path.join(work_dir, '*log')) #wsclean_ims = glob.glob( os.path.join( work_dir, '*wsclean*MFS*fits' ) ) myh5parm = glob.glob(os.path.join(work_dir, '*h5')) skymodel = glob.glob(os.path.join(work_dir, '*skymodel')) #file_list = image_files + log_files + wsclean_ims + myh5parm + skymodel file_list = image_files + log_files + myh5parm + skymodel for myfile in file_list: ff = myfile.split('/')[-1] ss = 'mv {:s} {:s}'.format(myfile, os.path.join(current_dir, ff)) os.system(ss) ## move the infile back and rename it tmp = infile.split('/')[-1] new_file = tmp + '.selfcal' selfcal_file = os.path.join(current_dir, new_file) os.system('cp -r {:s} {:s}'.format(infile, selfcal_file)) print('done')
#!/usr/bin/python # usage: plot_beamcorr.py <MS> import os, sys import numpy as np import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import casacore.tables as pt msname = sys.argv[1] print "Plotting beam of " + msname # get time t = pt.table(msname) times = t.getcol('TIME')[::600] # every 10 min t.close() # get direction t = pt.table(msname + '/FIELD') direction = t.getcol('PHASE_DIR')[0][0] print "Direction:", direction t.close() # get stations t = pt.table(msname + '/ANTENNA') stations = t.getcol('NAME') t.close() print "Stations:", stations import lofar.stationresponse as st
def test_getcol(self): c1 = makescacoldesc(unicode_string, 0) t = table(join(self.workdir, 'ascii'), maketabdesc([c1]), ack=False) t.getcol(unicode_string)