def read_ms(vis): ''' Read a CASA ms file and return a dictionary of amplitude, phase, uvdistance, uvangle, frequency (GHz) and time (MJD). Currently only returns the XX IF channel. vis Name of the visibility (ms) folder ''' ms.open(vis) spwinfo = ms.getspectralwindowinfo() nspw = len(spwinfo.keys()) for i in range(nspw): print('Working on spw', i) ms.selectinit(datadescid=0, reset=True) ms.selectinit(datadescid=i) if i == 0: spw = ms.getdata(['amplitude', 'phase', 'u', 'v', 'axis_info'], ifraxis=True) xxamp = spw['amplitude'] xxpha = spw['phase'] fghz = spw['axis_info']['freq_axis']['chan_freq'][:, 0] / 1e9 band = np.ones_like(fghz) * i mjd = spw['axis_info']['time_axis']['MJDseconds'] / 86400. uvdist = np.sqrt(spw['u'] ** 2 + spw['v'] ** 2) uvang = np.angle(spw['u'] + 1j * spw['v']) else: spw = ms.getdata(['amplitude', 'phase', 'axis_info'], ifraxis=True) xxamp = np.concatenate((xxamp, spw['amplitude']), 1) xxpha = np.concatenate((xxpha, spw['phase']), 1) fg = spw['axis_info']['freq_axis']['chan_freq'][:, 0] / 1e9 fghz = np.concatenate((fghz, fg)) band = np.concatenate((band, np.ones_like(fg) * i)) ms.close() return {'amp': xxamp, 'phase': xxpha, 'fghz': fghz, 'band': band, 'mjd': mjd, 'uvdist': uvdist, 'uvangle': uvang}
def mk_qlook_image(trange, doimport=False, docalib=False, ncpu=10, twidth=12, stokes=None, antenna='0~12', lowcutoff_freq=3.7, imagedir=None, spws=['1~5', '6~10', '11~15', '16~25'], toTb=True, overwrite=True, doslfcal=False, verbose=False): ''' trange: can be 1) a single Time() object: use the entire day 2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00']) 3) a single or a list of UDBms file(s) 4) None -- use current date Time.now() ''' antenna0 = antenna if type(trange) == Time: mslist = trange2ms(trange=trange, doimport=doimport) vis = mslist['ms'] tsts = [l.to_datetime() for l in mslist['tstlist']] if type(trange) == str: try: date = Time(trange) mslist = trange2ms(trange=trange, doimport=doimport) vis = mslist['ms'] tsts = [l.to_datetime() for l in mslist['tstlist']] except: vis = [trange] tsts = [] for v in vis: tb.open(v + '/OBSERVATION') tsts.append( Time(tb.getcell('TIME_RANGE')[0] / 24 / 3600, format='mjd').datetime) tb.close() subdir = [tst.strftime("%Y/%m/%d/") for tst in tsts] for idx, f in enumerate(vis): if f[-1] == '/': vis[idx] = f[:-1] if not stokes: stokes = 'XX' if not imagedir: imagedir = './' imres = { 'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [], 'Spw': [], 'Vis': [], 'Synoptic': { 'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [], 'Spw': [], 'Vis': [] } } for n, msfile in enumerate(vis): msfilebs = os.path.basename(msfile) imdir = imagedir + subdir[n] if not os.path.exists(imdir): os.makedirs(imdir) if doslfcal: slfcalms = './' + msfilebs + '.xx' split(msfile, outputvis=slfcalms, datacolumn='corrected', correlation='XX') cfreqs = getspwfreq(msfile) for spw in spws: antenna = antenna0 if spw == '': continue spwran = [s.zfill(2) for s in spw.split('~')] freqran = [cfreqs[int(s)] for s in spw.split('~')] cfreq = np.mean(freqran) bmsz = max(150. / cfreq, 20.) uvrange = '<10klambda' if doslfcal: slfcal_img = './' + msfilebs + '.slf.spw' + spw.replace( '~', '-') + '.slfimg' slfcal_tb = './' + msfilebs + '.slf.spw' + spw.replace( '~', '-') + '.slftb' try: clean(vis=slfcalms, antenna=antenna, imagename=slfcal_img, spw=spw, mode='mfs', timerange='', imagermode='csclean', psfmode='clark', imsize=[512, 512], cell=['5arcsec'], niter=100, gain=0.05, stokes='I', weighting='natural', restoringbeam=[str(bmsz) + 'arcsec'], pbcor=False, interactive=False, usescratch=True) except: print('error in cleaning spw: ' + spw) break gaincal(vis=slfcalms, refant='0', antenna=antenna, caltable=slfcal_tb, spw=spw, uvrange='', gaintable=[], selectdata=True, timerange='', solint='600s', gaintype='G', calmode='p', combine='', minblperant=3, minsnr=2, append=False) if not os.path.exists(slfcal_tb): print('No solution found in spw: ' + spw) break else: clearcal(slfcalms) delmod(slfcalms) applycal(vis=slfcalms, gaintable=[slfcal_tb], spw=spw, selectdata=True, antenna=antenna, interp='nearest', flagbackup=False, applymode='calonly', calwt=False) msfile = slfcalms imsize = 512 cell = ['5arcsec'] if len(spwran) == 2: spwstr = spwran[0] + '~' + spwran[1] else: spwstr = spwran[0] restoringbeam = ['{0:.1f}arcsec'.format(bmsz)] imagesuffix = '.spw' + spwstr.replace('~', '-') if cfreq > 10.: antenna = antenna + ';!0&1;!0&2' # deselect the shortest baselines # else: # antenna = antenna + ';!0&1' # deselect the shortest baselines res = ptclean3(vis=msfile, imageprefix=imdir, imagesuffix=imagesuffix, twidth=twidth, uvrange=uvrange, spw=spw, ncpu=ncpu, niter=1000, gain=0.05, antenna=antenna, imsize=imsize, cell=cell, stokes=stokes, doreg=True, usephacenter=False, overwrite=overwrite, toTb=toTb, restoringbeam=restoringbeam, specmode="mfs", deconvolver="hogbom", datacolumn='data', pbcor=True) if res: imres['Succeeded'] += res['Succeeded'] imres['BeginTime'] += res['BeginTime'] imres['EndTime'] += res['EndTime'] imres['ImageName'] += res['ImageName'] imres['Spw'] += [spwstr] * len(res['ImageName']) imres['Vis'] += [msfile] * len(res['ImageName']) else: continue if len(vis) == 1: # produce the band-by-band whole-day images ms.open(msfile) ms.selectinit() timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True) tim = timfreq['time'] ms.close() cfreqs = getspwfreq(msfile) imdir = imagedir + subdir[0] if not os.path.exists(imdir): os.makedirs(imdir) for spw in spws: antenna = antenna0 if spw == '': spw = '{:d}~{:d}'.format( next(x[0] for x in enumerate(cfreqs) if x[1] > lowcutoff_freq), len(cfreqs) - 1) spwran = [s.zfill(2) for s in spw.split('~')] freqran = [cfreqs[int(s)] for s in spw.split('~')] cfreq = np.mean(freqran) bmsz = max(150. / cfreq, 20.) uvrange = '' imsize = 512 cell = ['5arcsec'] if len(spwran) == 2: spwstr = spwran[0] + '~' + spwran[1] else: spwstr = spwran[0] restoringbeam = ['{0:.1f}arcsec'.format(bmsz)] imagesuffix = '.synoptic.spw' + spwstr.replace('~', '-') antenna = antenna + ';!0&1' # deselect the shortest baselines res = ptclean3(vis=msfile, imageprefix=imdir, imagesuffix=imagesuffix, twidth=len(tim), uvrange=uvrange, spw=spw, ncpu=1, niter=0, gain=0.05, antenna=antenna, imsize=imsize, cell=cell, stokes=stokes, doreg=True, usephacenter=False, overwrite=overwrite, toTb=toTb, restoringbeam=restoringbeam, specmode="mfs", deconvolver="hogbom", datacolumn='data', pbcor=True) if res: imres['Synoptic']['Succeeded'] += res['Succeeded'] imres['Synoptic']['BeginTime'] += res['BeginTime'] imres['Synoptic']['EndTime'] += res['EndTime'] imres['Synoptic']['ImageName'] += res['ImageName'] imres['Synoptic']['Spw'] += [spwstr] * len(res['ImageName']) imres['Synoptic']['Vis'] += [msfile] * len(res['ImageName']) else: continue # save it for debugging purposes np.savez('imres.npz', imres=imres) return imres
def ptclean3(vis, imageprefix, imagesuffix, ncpu, twidth, doreg, usephacenter, reftime, toTb, overwrite, selectdata, field, spw, timerange, uvrange, antenna, scan, observation, intent, datacolumn, imsize, cell, phasecenter, stokes, projection, startmodel, specmode, reffreq, nchan, start, width, outframe, veltype, restfreq, interpolation, gridder, facets, chanchunks, wprojplanes, vptable, usepointing, mosweight, aterm, psterm, wbawp, conjbeams, cfcache, computepastep, rotatepastep, pblimit, normtype, deconvolver, scales, nterms, smallscalebias, restoration, restoringbeam, pbcor, outlierfile, weighting, robust, npixels, uvtaper, niter, gain, threshold, nsigma, cycleniter, cyclefactor, minpsffraction, maxpsffraction, interactive, usemask, mask, pbmask, sidelobethreshold, noisethreshold, lownoisethreshold, negativethreshold, smoothfactor, minbeamfrac, cutthreshold, growiterations, dogrowprune, minpercentchange, verbose, restart, savemodel, calcres, calcpsf, parallel, subregion): if not (type(ncpu) is int): casalog.post('ncpu should be an integer') ncpu = 8 if doreg: # check if ephem and msinfo exist. If not, generate one on the fly try: ephem = hf.read_horizons(vis=vis) except ValueError: print("error in obtaining ephemeris") try: msinfo = hf.read_msinfo(vis) except ValueError: print("error in getting ms info") else: ephem = None msinfo = None if imageprefix: workdir = os.path.dirname(imageprefix) else: workdir = './' tmpdir = workdir + '/tmp/' if not os.path.exists(tmpdir): os.makedirs(tmpdir) # get number of time pixels ms.open(vis) ms.selectinit() timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True) tim = timfreq['time'] ms.close() if twidth < 1: casalog.post('twidth less than 1. Change to 1') twidth = 1 if twidth > len(tim): casalog.post( 'twidth greater than # of time pixels in the dataset. Change to the timerange of the entire dateset' ) twidth = len(tim) # find out the start and end time index according to the parameter timerange # if not defined (empty string), use start and end from the entire time of the ms if not timerange: btidx = 0 etidx = len(tim) - 1 else: try: (tstart, tend) = timerange.split('~') bt_s = qa.convert(qa.quantity(tstart, 's'), 's')['value'] et_s = qa.convert(qa.quantity(tend, 's'), 's')['value'] # only time is given but not date, add the date (at 0 UT) from the first record if bt_s < 86400. or et_s < 86400.: bt_s += np.fix( qa.convert(qa.quantity(tim[0], 's'), 'd')['value']) * 86400. et_s += np.fix( qa.convert(qa.quantity(tim[0], 's'), 'd')['value']) * 86400. btidx = np.argmin(np.abs(tim - bt_s)) etidx = np.argmin(np.abs(tim - et_s)) # make the indice back to those bracket by the timerange if tim[btidx] < bt_s: btidx += 1 if tim[etidx] > et_s: etidx -= 1 if etidx <= btidx: print("ending time must be greater than starting time") print("reinitiating to the entire time range") btidx = 0 etidx = len(tim) - 1 except ValueError: print("keyword 'timerange' has a wrong format") btstr = qa.time(qa.quantity(tim[btidx], 's'), prec=9, form='fits')[0] etstr = qa.time(qa.quantity(tim[etidx], 's'), prec=9, form='fits')[0] iterable = range(btidx, etidx + 1, twidth) print('First time pixel: ' + btstr) print('Last time pixel: ' + etstr) print(str(len(iterable)) + ' images to clean...') res = [] # partition clnpart = partial( clean_iter, tim, vis, imageprefix, imagesuffix, twidth, doreg, usephacenter, reftime, ephem, msinfo, toTb, overwrite, selectdata, field, spw, uvrange, antenna, scan, observation, intent, datacolumn, imsize, cell, phasecenter, stokes, projection, startmodel, specmode, reffreq, nchan, start, width, outframe, veltype, restfreq, interpolation, gridder, facets, chanchunks, wprojplanes, vptable, usepointing, mosweight, aterm, psterm, wbawp, conjbeams, cfcache, computepastep, rotatepastep, pblimit, normtype, deconvolver, scales, nterms, smallscalebias, restoration, restoringbeam, pbcor, outlierfile, weighting, robust, npixels, uvtaper, niter, gain, threshold, nsigma, cycleniter, cyclefactor, minpsffraction, maxpsffraction, interactive, usemask, mask, pbmask, sidelobethreshold, noisethreshold, lownoisethreshold, negativethreshold, smoothfactor, minbeamfrac, cutthreshold, growiterations, dogrowprune, minpercentchange, verbose, restart, savemodel, calcres, calcpsf, parallel, subregion, tmpdir) timelapse = 0 t0 = time() # parallelization if ncpu > 1: import multiprocessing as mprocs casalog.post('Perform clean in parallel ...') print('Perform clean in parallel ...') pool = mprocs.Pool(ncpu) res = pool.map(clnpart, iterable) pool.close() pool.join() else: casalog.post('Perform clean in single process ...') print('Perform clean in single process ...') for i in iterable: res.append(clnpart(i)) t1 = time() timelapse = t1 - t0 print('It took %f secs to complete' % timelapse) # repackage this into a single dictionary results = { 'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [] } for r in res: results['Succeeded'].append(r[0]) results['BeginTime'].append(r[1]) results['EndTime'].append(r[2]) results['ImageName'].append(r[3]) if os.path.exists(tmpdir): os.system('rm -rf ' + tmpdir) return results
def subvs(vis=None, outputvis=None, timerange=None, spw=None, timoffset=4, windowlen=10, windowtype='flat', splitsel=True, reverse=False, overwrite=False): """Vector-subtraction in UV using selected time ranges and spectral channels as background subvs is a function to do UV vector-subtraction. By selecting gliding averaging window, only the low-frequency signals corresponding to the background continuum emission remain. As a result, a uv subtraction of original dynamic spectrum from the background can improve fine stucture such as fibers. Subvs can be used to subtract the background continuum emission to separate the time-dependent emission, e.g. solar coherent radio bursts. Keyword arguments: vis -- Name of input visibility file default: none; example: vis='sun_type3.ms' outputvis -- Name of output visibility file default: none; example: outputvis='sun_type3.sub.ms' timerange -- Select the time range in the data to be subtracted from. timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' Note: if YYYY/MM/DD is missing date, timerange defaults to the first day in the dataset timerange='09:14:0~09:54:0' picks 40 min on first day timerange='25:00:00~27:30:00' picks 1 hr to 3 hr 30min on next day spw -- Select spectral window/channel. default = '' all the spectral channels. Example: spw='0:1~20' timoffset -- After the convolution, each single channel of smoothed uv vectors in time series are misaligned from original data. Setting the timoffset allow you to shift by the specifed amount of data points. windowlen -- Choose the width the gliding window for smoothing. windowtype -- Choose the type o gliding window. Options available are 'flat', 'hanning', 'hamming', 'bartlett', 'blackman' splitsel -- True of False. default = False. If splitsel = False, then the entire input measurement set is copied as the output measurement set (outputvis), with background subtracted at selected timerange and spectral channels. If splitsel = True,then only the selected timerange and spectral channels are copied into the output measurement set (outputvis). reverse -- True or False. default = False. If reverse = False, then the times indicated by subtime1 and/or subtime2 are treated as background and subtracted; If reverse = True, then reverse the sign of the background-subtracted data. The option can be used for mapping absorptive structure. overwrite -- True or False. default = False. If overwrite = True and outputvis already exists, the selected subtime and spw in the already existing output measurement set will be replaced with subtracted visibilities """ #check the visbility ms if not outputvis or outputvis.isspace(): raise ValueError, 'Please specify outputvis.' if os.path.exists(outputvis): if overwrite: print "The already existing output measurement set will be updated." else: raise ValueError, "Output MS %s already exists - will not overwrite." % outputvis else: if not splitsel: shutil.copytree(vis, outputvis) else: ms.open(vis, nomodify=True) ms.split(outputvis, spw=spw, time=timerange, whichcol='DATA') ms.close() # check and specify time range and channel if timerange and (type(timerange) == str): [btimeo, etimeo] = timerange.split('~') btimeosec = qa.getvalue(qa.convert(qa.totime(btimeo), 's')) etimeosec = qa.getvalue(qa.convert(qa.totime(etimeo), 's')) timebinosec = etimeosec - btimeosec if timebinosec < 0: raise Exception, 'Negative timebin! Please check the "timerange" parameter.' casalog.post('Selected timerange: ' + timerange + ' as the time for UV subtraction.') else: casalog.post( 'Output timerange not specified, using the entire timerange') if spw and (type(spw) == str): [spwid, chanran] = spw.split(':') [bchan, echan] = chanran.split('~') nchan = int(echan) - int(bchan) + 1 else: casalog.post('spw not specified, use all frequency channels') #select data range to be smoothed ms.open(vis, nomodify=True) #Select the spw id ms.msselect({'time': timerange}) ms.selectinit(datadescid=int(spwid)) ms.selectchannel(nchan, int(bchan), 1, 1) rec = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True) #print 'shape of the frequency matrix ',rec1['axis_info']['freq_axis']['chan_freq'].shape sz = rec['data'].shape print 'dimension of selected background for smoothing', rec['data'].shape #the data shape is (n_pol,n_channel,n_baseline,n_time), no need to reshape #rec1['data']=rec1['data'].reshape(sz1[0],sz1[1],sz1[2],nspw,sz1[3]/nspw,order='F') #print 'reshaped rec1 ', rec1['data'].shape if not (timoffset and (type(timoffset) == int)): timoffset = int(4) for i in range(rec['data'].shape[0]): for j in range(rec['data'].shape[1]): for k in range(rec['data'].shape[2]): rec['data'][i, j, k, :] = task_smooth.smooth(rec['data'][i, j, k, :], timoffset, windowlen, windowtype) casalog.post('Smoothing the visibilities in timerange: ' + timerange) ms.close() #do UV subtraction, according to timerange and spw ms.open(outputvis, nomodify=False) if not splitsel: #outputvis is identical to input visibility, do the selection if timerange and (type(timerange == str)): ms.msselect({'time': timerange}) if spw and (type(spw) == str): ms.selectinit(datadescid=int(spwid)) nchan = int(echan) - int(bchan) + 1 ms.selectchannel(nchan, int(bchan), 1, 1) else: #outputvis is splitted, selections have already applied, select all the data ms.selectinit(datadescid=0) orec = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True) b_rows = orec['data'].shape[2] nchan = orec['data'].shape[1] #szo=orec['data'].shape print 'dimension of output data', orec['data'].shape #orec['data']=orec['data'].reshape(szo[0],szo[1],szo[2],nspw,szo[3]/nspw,order='F') #print 'reshaped rec1 ', orec['data'].shape t_rows = orec['data'].shape[3] casalog.post('Number of baselines: ' + str(b_rows)) casalog.post('Number of spectral channels: ' + str(nchan)) casalog.post('Number of time pixels: ' + str(t_rows)) casalog.post('Subtracting background defined in timerange: ' + timerange) for i in range(t_rows): orec['data'][:, :, :, i] -= rec['data'][:, :, :, i] if reverse: orec['data'][:, :, :, i] = -orec['data'][:, :, :, i] del orec['time'] del orec['axis_info'] ms.putdata(orec) ms.close()
def ptclean(vis, imageprefix, ncpu, twidth, doreg, overwrite, ephemfile, msinfofile, outlierfile, field, spw, selectdata, timerange, uvrange, antenna, scan, observation, intent, mode, resmooth, gridmode, wprojplanes, facets, cfcache, rotpainc, painc, aterm, psterm, mterm, wbawp, conjbeams, epjtable, interpolation, niter, gain, threshold, psfmode, imagermode, ftmachine, mosweight, scaletype, multiscale, negcomponent, smallscalebias, interactive, mask, nchan, start, width, outframe, veltype, imsize, cell, phasecenter, restfreq, stokes, weighting, robust, uvtaper, outertaper, innertaper, modelimage, restoringbeam, pbcor, minpb, usescratch, noise, npixels, npercycle, cyclefactor, cyclespeedup, nterms, reffreq, chaniter, flatnoise, allowchunk): if not (type(ncpu) is int): casalog.post('ncpu should be an integer') ncpu = 8 if doreg: # check if ephemfile and msinfofile exist try: ephem = vla_prep.read_horizons(ephemfile=ephemfile) except ValueError: print("error in reading ephemeris file") if not os.path.isfile(msinfofile): print("msinfofile does not exist!") else: ephem = None # get number of time pixels ms.open(vis) ms.selectinit() timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True) tim = timfreq['time'] # dt = tim[1]-tim[0] #need to change to median of all time intervals dt = np.median(np.diff(tim)) freq = timfreq['axis_info']['freq_axis']['chan_freq'].flatten() ms.close() if twidth < 1 or twidth > len(tim): casalog.post( 'twidth not between 1 and # of time pixels in the dataset. Change to 1' ) twidth = 1 # find out the start and end time index according to the parameter timerange # if not defined (empty string), use start and end from the entire time of the ms if not timerange: btidx = 0 etidx = len(tim) - 1 else: try: (tstart, tend) = timerange.split('~') bt_s = qa.convert(qa.quantity(tstart, 's'), 's')['value'] et_s = qa.convert(qa.quantity(tend, 's'), 's')['value'] # only time is given but not date, add the date (at 0 UT) from the first record if bt_s < 86400. or et_s < 86400.: bt_s += np.fix( qa.convert(qa.quantity(tim[0], 's'), 'd')['value']) * 86400. et_s += np.fix( qa.convert(qa.quantity(tim[0], 's'), 'd')['value']) * 86400. btidx = np.argmin(np.abs(tim - bt_s)) etidx = np.argmin(np.abs(tim - et_s)) # make the indice back to those bracket by the timerange if tim[btidx] < bt_s: btidx += 1 if tim[etidx] > et_s: etidx -= 1 if etidx <= btidx: print "ending time must be greater than starting time" print "reinitiating to the entire time range" btidx = 0 etidx = len(tim) - 1 except ValueError: print "keyword 'timerange' has a wrong format" btstr = qa.time(qa.quantity(tim[btidx], 's'), prec=9, form='fits')[0] etstr = qa.time(qa.quantity(tim[etidx], 's'), prec=9, form='fits')[0] iterable = range(btidx, etidx + 1, twidth) print 'First time pixel: ' + btstr print 'Last time pixel: ' + etstr print str(len(iterable)) + ' images to clean...' res = [] # partition clnpart = partial( clean_iter, tim, freq, vis, imageprefix, ncpu, twidth, doreg, overwrite, ephemfile, ephem, msinfofile, outlierfile, field, spw, selectdata, uvrange, antenna, scan, observation, intent, mode, resmooth, gridmode, wprojplanes, facets, cfcache, rotpainc, painc, aterm, psterm, mterm, wbawp, conjbeams, epjtable, interpolation, niter, gain, threshold, psfmode, imagermode, ftmachine, mosweight, scaletype, multiscale, negcomponent, smallscalebias, interactive, mask, nchan, start, width, outframe, veltype, imsize, cell, phasecenter, restfreq, stokes, weighting, robust, uvtaper, outertaper, innertaper, modelimage, restoringbeam, pbcor, minpb, usescratch, noise, npixels, npercycle, cyclefactor, cyclespeedup, nterms, reffreq, chaniter, flatnoise, allowchunk) timelapse = 0 t0 = time() # parallelization para = 1 if para: casalog.post('Perform clean in parallel ...') pool = mp.Pool(ncpu) # res = pool.map_async(clnpart, iterable) res = pool.map(clnpart, iterable) pool.close() pool.join() else: for i in iterable: res.append(clnpart(i)) t1 = time() timelapse = t1 - t0 print 'It took %f secs to complete' % timelapse # repackage this into a single dictionary results = {'succeeded': [], 'timestamps': [], 'imagenames': []} for r in res: results['succeeded'].append(r[0]) results['timestamps'].append(r[1]) results['imagenames'].append(r[2]) return results
def get_dspec(vis=None, savespec=True, specfile=None, bl='', uvrange='', field='', scan='', datacolumn='data', domedian=False, timeran=None, spw=None, timebin='0s', verbose=False): from split_cli import split_cli as split msfile = vis if not spw: spw = '' if not timeran: timeran = '' if not bl: bl = '' if domedian: if not uvrange: uvrange = '0.2~0.8km' else: uvrange = '' # Open the ms and plot dynamic spectrum if verbose: print 'Splitting selected data...' vis_spl = './tmpms.splitted' if os.path.exists(vis_spl): os.system('rm -rf ' + vis_spl) ms.open(msfile, nomodify=False) ms.split(outputms=vis_spl, whichcol=datacolumn, time=timeran, spw=spw, baseline=bl, field=field, scan=scan, uvrange=uvrange, timebin=timebin) ms.close() # split(vis=msfile, outputvis=vis_spl, timerange=timeran, antenna=bl, field=field, scan=scan, spw=spw, # uvrange=uvrange, timebin=timebin, datacolumn=datacolumn) ms.open(vis_spl, nomodify=False) if verbose: print 'Regridding into a single spectral window...' # print 'Reading data spw by spw' ms.cvel(outframe='LSRK', mode='frequency', interp='nearest') ms.selectinit(datadescid=0, reset=True) data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True) ms.close() os.system('rm -rf ' + vis_spl) specamp = data['amplitude'] (npol, nfreq, nbl, ntim) = specamp.shape if verbose: print 'npol, nfreq, nbl, ntime:', data['amplitude'].shape spec = np.swapaxes(specamp, 2, 1) freq = data['axis_info']['freq_axis']['chan_freq'].reshape(nfreq) tim = data['time'] if domedian: if verbose: print('doing median of all the baselines') # mask zero values before median spec_masked = np.ma.masked_where(spec < 1e-9, spec) spec_med = np.ma.filled(np.ma.median(spec_masked, axis=1), fill_value=0.) nbl = 1 ospec = spec_med.reshape((npol, nbl, nfreq, ntim)) else: ospec = spec # Save the dynamic spectral data if savespec: if not specfile: specfile = msfile + '.dspec.npz' if os.path.exists(specfile): os.system('rm -rf ' + specfile) np.savez(specfile, spec=ospec, tim=tim, freq=freq, timeran=timeran, spw=spw, bl=bl, uvrange=uvrange) if verbose: print 'Median dynamic spectrum saved as: ' + specfile return { 'spec': ospec, 'tim': tim, 'freq': freq, 'timeran': timeran, 'spw': spw, 'bl': bl, 'uvrange': uvrange }
def subvs(vis=None, outputvis=None, timerange=None, spw=None, mode=None, subtime1=None, subtime2=None, smoothaxis=None, smoothtype=None, smoothwidth=None, splitsel=None, reverse=None, overwrite=None): """Perform vector subtraction for visibilities Keyword arguments: vis -- Name of input visibility file (MS) default: none; example: vis='ngc5921.ms' outputvis -- Name of output uv-subtracted visibility file (MS) default: none; example: outputvis='ngc5921_src.ms' timerange -- Time range of performing the UV subtraction: default='' means all times. examples: timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' timerange = 'hh:mm:ss~hh:mm:ss' spw -- Select spectral window/channel. default = '' all the spectral channels. Example: spw='0:1~20' mode -- operation mode default 'linear' mode = 'linear': use a linear fit for the background to be subtracted mode = 'lowpass': act as a lowpass filter---smooth the data using different smooth types and window sizes. Can be performed along either time or frequency axis mode = 'highpass': act as a highpass filter---smooth the data first, and subtract the smoothed data from the original. Can be performed along either time or frequency axis mode = 'linear' expandable parameters: subtime1 -- Time range 1 of the background to be subtracted from the data default='' means all times. format: timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' timerange = 'hh:mm:ss~hh:mm:ss' subtime2 -- Time range 2 of the backgroud to be subtracted from the data default='' means all times. examples: timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' timerange = 'hh:mm:ss~hh:mm:ss' mode = 'lowpass' or 'highpass' expandable parameters: smoothaxis -- axis of smooth Default: 'time' smoothaxis = 'time': smooth is along the time axis smoothaxis = 'freq': smooth is along the frequency axis smoothtype -- type of the smooth depending on the convolving kernel Default: 'flat' smoothtype = 'flat': convolving kernel is a flat rectangle, equivalent to a boxcar moving smooth smoothtype = 'hanning': Hanning smooth kernel. See numpy.hanning smoothtype = 'hamming': Hamming smooth kernel. See numpy.hamming smoothtype = 'bartlett': Bartlett smooth kernel. See numpy.bartlett smoothtype = 'blackman': Blackman smooth kernel. See numpy.blackman smoothwidth -- width of the smooth kernel Default: 5 Examples: smoothwidth=5, meaning the width is 5 pixels splitsel -- True or False. default = False. If splitsel = False, then the entire input measurement set is copied as the output measurement set (outputvis), with background subtracted at selected timerange and spectral channels. If splitsel = True,then only the selected timerange and spectral channels are copied into the output measurement set (outputvis). reverse -- True or False. default = False. If reverse = False, then the times indicated by subtime1 and/or subtime2 are treated as background and subtracted; If reverse = True, then reverse the sign of the background-subtracted data. The option can be used for mapping absorptive structure. overwrite -- True or False. default = False. If overwrite = True and outputvis already exists, the selected subtime and spw in the output measurment set will be replaced with background subtracted visibilities """ # check the visbility ms casalog.post('input parameters:') casalog.post('vis: ' + vis) casalog.post('outputvis: ' + outputvis) casalog.post('smoothaxis: ' + smoothaxis) casalog.post('smoothtype: ' + smoothtype) casalog.post('smoothwidth: ' + str(smoothwidth)) if not outputvis or outputvis.isspace(): raise (ValueError, 'Please specify outputvis') if os.path.exists(outputvis): if overwrite: print( "The already existing output measurement set will be updated.") else: raise (ValueError, "Output MS %s already exists - will not overwrite." % outputvis) else: if not splitsel: shutil.copytree(vis, outputvis) else: ms.open(vis, nomodify=True) ms.split(outputvis, spw=spw, time=timerange, whichcol='DATA') ms.close() if timerange and (type(timerange) == str): [btimeo, etimeo] = timerange.split('~') btimeosec = qa.getvalue(qa.convert(qa.totime(btimeo), 's')) etimeosec = qa.getvalue(qa.convert(qa.totime(etimeo), 's')) timebinosec = etimeosec - btimeosec if timebinosec < 0: raise Exception( 'Negative timebin! Please check the "timerange" parameter.') casalog.post('Selected timerange: ' + timerange + ' as the time for UV subtraction.') else: casalog.post( 'Output timerange not specified, using the entire timerange') if spw and (type(spw) == str): [spwid, chanran] = spw.split(':') [bchan, echan] = chanran.split('~') else: casalog.post('spw not specified, use all frequency channels') # read the output data datams = mstool() datams.open(outputvis, nomodify=False) if not splitsel: # outputvis is identical to input visibility, do the selection if timerange and (type(timerange == str)): datams.msselect({'time': timerange}) if spw and (type(spw) == str): datams.selectinit(datadescid=int(spwid)) nchan = int(echan) - int(bchan) + 1 datams.selectchannel(nchan, int(bchan), 1, 1) if not spw and not timerange: # data selection is not made print('selecting all spws and times') datams.selectinit(reset=True) else: # outputvis is splitted, selections have already applied, select all the data print('split the selected spws and times') datams.selectinit(reset=True) orec = datams.getdata(['data', 'time', 'axis_info'], ifraxis=True) nchan = orec['data'].shape[1] nbl = orec['data'].shape[2] # szo=orec['data'].shape print('dimension of output data', orec['data'].shape) # orec['data']=orec['data'].reshape(szo[0],szo[1],szo[2],nspw,szo[3]/nspw,order='F') # print('reshaped rec1 ', orec['data'].shape) ntim = orec['data'].shape[3] casalog.post('Number of baselines: ' + str(nbl)) casalog.post('Number of spectral channels: ' + str(nchan)) casalog.post('Number of time pixels: ' + str(ntim)) try: if mode == 'linear': # define and check the background time ranges if subtime1 and (type(subtime1) == str): [bsubtime1, esubtime1] = subtime1.split('~') bsubtime1sec = qa.getvalue( qa.convert(qa.totime(bsubtime1), 's')) esubtime1sec = qa.getvalue( qa.convert(qa.totime(esubtime1), 's')) timebin1sec = esubtime1sec - bsubtime1sec if timebin1sec < 0: raise Exception( 'Negative timebin! Please check the "subtime1" parameter.' ) casalog.post('Selected timerange 1: ' + subtime1 + ' as background for uv subtraction.') else: raise Exception( 'Please enter at least one timerange as the background') if subtime2 and (type(subtime2) == str): [bsubtime2, esubtime2] = subtime2.split('~') bsubtime2sec = qa.getvalue( qa.convert(qa.totime(bsubtime2), 's')) esubtime2sec = qa.getvalue( qa.convert(qa.totime(esubtime2), 's')) timebin2sec = esubtime2sec - bsubtime2sec if timebin2sec < 0: raise Exception( 'Negative timebin! Please check the "subtime2" parameter.' ) timebin2 = str(timebin2sec) + 's' casalog.post('Selected timerange 2: ' + subtime2 + ' as background for uv subtraction.') # plus 1s is to ensure averaging over the entire timerange else: casalog.post( 'Timerange 2 not selected, using only timerange 1 as background' ) # Select the background indicated by subtime1 ms.open(vis, nomodify=True) # Select the spw id ms.msselect({'time': subtime1}) if spw and (type(spw) == str): ms.selectinit(datadescid=int(spwid)) nchan = int(echan) - int(bchan) + 1 ms.selectchannel(nchan, int(bchan), 1, 1) rec1 = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True) # print('shape of the frequency matrix ',rec1['axis_info']['freq_axis']['chan_freq'].shape) sz1 = rec1['data'].shape print('dimension of selected background 1', rec1['data'].shape) # the data shape is (n_pol,n_channel,n_baseline,n_time), no need to reshape # rec1['data']=rec1['data'].reshape(sz1[0],sz1[1],sz1[2],nspw,sz1[3]/nspw,order='F') # print('reshaped rec1 ', rec1['data'].shape) rec1avg = np.average(rec1['data'], axis=3) casalog.post('Averaging the visibilities in subtime1: ' + subtime1) ms.close() if subtime2 and (type(subtime2) == str): ms.open(vis, nomodify=True) # Select the spw id ms.msselect({'time': subtime2}) if spw and (type(spw) == str): ms.selectinit(datadescid=0) nchan = int(echan) - int(bchan) + 1 ms.selectchannel(nchan, int(bchan), 1, 1) rec2 = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True) sz2 = rec2['data'].shape print('dimension of selected background 2', rec2['data'].shape) # rec2['data']=rec2['data'].reshape(sz2[0],sz2[1],sz2[2],nspw,sz2[3]/nspw,order='F') # print('reshaped rec1 ', rec2['data'].shape) rec2avg = np.average(rec2['data'], axis=3) ms.close() casalog.post('Averaged the visibilities in subtime2: ' + subtime2) if subtime1 and (not subtime2): casalog.post( 'Only "subtime1" is defined, subtracting background defined in subtime1: ' + subtime1) t1 = (np.amax(rec1['time']) + np.amin(rec1['time'])) / 2. print('t1: ', qa.time(qa.quantity(t1, 's'), form='ymd', prec=10)) for i in range(ntim): orec['data'][:, :, :, i] -= rec1avg if reverse: orec['data'][:, :, :, i] = -orec['data'][:, :, :, i] if subtime1 and subtime2 and (type(subtime2) == str): casalog.post( 'Both subtime1 and subtime2 are specified, doing linear interpolation between "subtime1" and "subtime2"' ) t1 = (np.amax(rec1['time']) + np.amin(rec1['time'])) / 2. t2 = (np.amax(rec2['time']) + np.amin(rec2['time'])) / 2. touts = orec['time'] print('t1: ', qa.time(qa.quantity(t1, 's'), form='ymd', prec=10)) print('t2: ', qa.time(qa.quantity(t2, 's'), form='ymd', prec=10)) for i in range(ntim): tout = touts[i] if tout > np.amax([t1, t2]): tout = np.amax([t1, t2]) elif tout < np.amin([t1, t2]): tout = np.amin([t1, t2]) orec['data'][:, :, :, i] -= (rec2avg - rec1avg) * ( tout - t1) / (t2 - t1) + rec1avg if reverse: orec['data'][:, :, :, i] = -orec['data'][:, :, :, i] elif mode == 'highpass': if smoothtype != 'flat' and smoothtype != 'hanning' and smoothtype != 'hamming' and smoothtype != 'bartlett' and smoothtype != 'blackman': raise Exception('Unknown smoothtype ' + str(smoothtype)) if smoothaxis == 'time': if smoothwidth <= 0 or smoothwidth >= ntim: raise Exception( 'Specified smooth width is <=0 or >= the total number of ' + smoothaxis) else: for i in range(orec['data'].shape[0]): for j in range(nchan): for k in range(nbl): orec['data'][i, j, k, :] -= signalsmooth.smooth( orec['data'][i, j, k, :], smoothwidth, smoothtype) if smoothaxis == 'freq': if smoothwidth <= 0 or smoothwidth >= nchan: raise Exception( 'Specified smooth width is <=0 or >= the total number of ' + smoothaxis) else: for i in range(orec['data'].shape[0]): for j in range(nbl): for k in range(ntim): orec['data'][i, :, j, k] -= signalsmooth.smooth( orec['data'][i, :, j, k], smoothwidth, smoothtype) elif mode == 'lowpass': if smoothtype != 'flat' and smoothtype != 'hanning' and smoothtype != 'hamming' and smoothtype != 'bartlett' and smoothtype != 'blackman': raise Exception('Unknown smoothtype ' + str(smoothtype)) if smoothaxis == 'time': if smoothwidth <= 0 or smoothwidth >= ntim: raise Exception( 'Specified smooth width is <=0 or >= the total number of ' + smoothaxis) else: for i in range(orec['data'].shape[0]): for j in range(nchan): for k in range(nbl): orec['data'][i, j, k, :] = signalsmooth.smooth( orec['data'][i, j, k, :], smoothwidth, smoothtype) if smoothaxis == 'freq': if smoothwidth <= 0 or smoothwidth >= nchan: raise Exception( 'Specified smooth width is <=0 or >= the total number of ' + smoothaxis) else: for i in range(orec['data'].shape[0]): for j in range(nbl): for k in range(ntim): orec['data'][i, :, j, k] = signalsmooth.smooth( orec['data'][i, :, j, k], smoothwidth, smoothtype) else: raise Exception('Unknown mode' + str(mode)) except Exception as instance: print('*** Error ***', instance) # orec['data']=orec['data'].reshape(szo[0],szo[1],szo[2],szo[3],order='F') # put the modified data back into the output visibility set del orec['time'] del orec['axis_info'] # ms.open(outputvis,nomodify=False) # if not splitsel: # outputvis is identical to input visibility, do the selection # if timerange and (type(timerange==str)): # datams.msselect({'time':timerange}) # if spw and (type(spw)==str): # datams.selectinit(datadescid=int(spwid)) # nchan=int(echan)-int(bchan)+1 # datams.selectchannel(nchan,int(bchan),1,1) # if not spw and not timerange: # data selection is not made # datams.selectinit(datadescid=0) # else: # outputvis is splitted, selections have already applied, select all the data # datams.selectinit(datadescid=0) datams.putdata(orec) datams.close()
def subvs(vis=None, outputvis=None, timerange=None, spw=None, subtime1=None, subtime2=None, splitsel=True, reverse=False, overwrite=False): """Perform vector subtraction for visibilities Keyword arguments: vis -- Name of input visibility file (MS) default: none; example: vis='ngc5921.ms' outputvis -- Name of output uv-subtracted visibility file (MS) default: none; example: outputvis='ngc5921_src.ms' timerange -- Time range of performing the UV subtraction: default='' means all times. examples: timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' timerange = 'hh:mm:ss~hh:mm:ss' spw -- Select spectral window/channel. default = '' all the spectral channels. Example: spw='0:1~20' subtime1 -- Time range 1 of the background to be subtracted from the data default='' means all times. format: timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' timerange = 'hh:mm:ss~hh:mm:ss' subtime2 -- Time range 2 of the backgroud to be subtracted from the data default='' means all times. examples: timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss' timerange = 'hh:mm:ss~hh:mm:ss' splitsel -- True or False. default = False. If splitsel = False, then the entire input measurement set is copied as the output measurement set (outputvis), with background subtracted at selected timerange and spectral channels. If splitsel = True,then only the selected timerange and spectral channels are copied into the output measurement set (outputvis). reverse -- True or False. default = False. If reverse = False, then the times indicated by subtime1 and/or subtime2 are treated as background and subtracted; If reverse = True, then reverse the sign of the background-subtracted data. The option can be used for mapping absorptive structure. overwrite -- True or False. default = False. If overwrite = True and outputvis already exists, the selected subtime and spw in the output measurment set will be replaced with background subtracted visibilities """ #check the visbility ms if not outputvis or outputvis.isspace(): raise ValueError, 'Please specify outputvis' if os.path.exists(outputvis): if overwrite: print "The already existing output measurement set will be updated." else: raise ValueError, "Output MS %s already exists - will not overwrite." % outputvis else: if not splitsel: shutil.copytree(vis, outputvis) else: ms.open(vis, nomodify=True) ms.split(outputvis, spw=spw, time=timerange, whichcol='DATA') ms.close() #define and check the time ranges if subtime1 and (type(subtime1) == str): [bsubtime1, esubtime1] = subtime1.split('~') bsubtime1sec = qa.getvalue(qa.convert(qa.totime(bsubtime1), 's')) esubtime1sec = qa.getvalue(qa.convert(qa.totime(esubtime1), 's')) timebin1sec = esubtime1sec - bsubtime1sec if timebin1sec < 0: raise Exception, 'Negative timebin! Please check the "subtime1" parameter.' casalog.post('Selected timerange 1: ' + subtime1 + ' as background for uv subtraction.') else: raise Exception, 'Please enter at least one timerange as the background' if subtime2 and (type(subtime2) == str): [bsubtime2, esubtime2] = subtime2.split('~') bsubtime2sec = qa.getvalue(qa.convert(qa.totime(bsubtime2), 's')) esubtime2sec = qa.getvalue(qa.convert(qa.totime(esubtime2), 's')) timebin2sec = esubtime2sec - bsubtime2sec if timebin2sec < 0: raise Exception, 'Negative timebin! Please check the "subtime2" parameter.' timebin2 = str(timebin2sec) + 's' casalog.post('Selected timerange 2: ' + subtime2 + ' as background for uv subtraction.') #plus 1s is to ensure averaging over the entire timerange else: casalog.post( 'Timerange 2 not selected, using only timerange 1 as background') if timerange and (type(timerange) == str): [btimeo, etimeo] = timerange.split('~') btimeosec = qa.getvalue(qa.convert(qa.totime(btimeo), 's')) etimeosec = qa.getvalue(qa.convert(qa.totime(etimeo), 's')) timebinosec = etimeosec - btimeosec if timebinosec < 0: raise Exception, 'Negative timebin! Please check the "timerange" parameter.' casalog.post('Selected timerange: ' + timerange + ' as the time for UV subtraction.') else: casalog.post( 'Output timerange not specified, using the entire timerange') if spw and (type(spw) == str): [spwid, chanran] = spw.split(':') [bchan, echan] = chanran.split('~') else: casalog.post('spw not specified, use all frequency channels') #Select the background indicated by subtime1 ms.open(vis, nomodify=True) #Select the spw id ms.msselect({'time': subtime1}) if spw and (type(spw) == str): ms.selectinit(datadescid=int(spwid)) nchan = int(echan) - int(bchan) + 1 ms.selectchannel(nchan, int(bchan), 1, 1) rec1 = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True) #print 'shape of the frequency matrix ',rec1['axis_info']['freq_axis']['chan_freq'].shape sz1 = rec1['data'].shape print 'dimension of selected background 1', rec1['data'].shape #the data shape is (n_pol,n_channel,n_baseline,n_time), no need to reshape #rec1['data']=rec1['data'].reshape(sz1[0],sz1[1],sz1[2],nspw,sz1[3]/nspw,order='F') #print 'reshaped rec1 ', rec1['data'].shape rec1avg = np.average(rec1['data'], axis=3) casalog.post('Averaging the visibilities in subtime1: ' + subtime1) ms.close() if subtime2 and (type(subtime2) == str): ms.open(vis, nomodify=True) #Select the spw id ms.msselect({'time': subtime2}) if spw and (type(spw) == str): ms.selectinit(datadescid=0) nchan = int(echan) - int(bchan) + 1 ms.selectchannel(nchan, int(bchan), 1, 1) rec2 = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True) sz2 = rec2['data'].shape print 'dimension of selected background 2', rec2['data'].shape #rec2['data']=rec2['data'].reshape(sz2[0],sz2[1],sz2[2],nspw,sz2[3]/nspw,order='F') #print 'reshaped rec1 ', rec2['data'].shape rec2avg = np.average(rec2['data'], axis=3) ms.close() casalog.post('Averaged the visibilities in subtime2: ' + subtime2) #do UV subtraction, according to timerange and spw ms.open(outputvis, nomodify=False) if not splitsel: #outputvis is identical to input visibility, do the selection if timerange and (type(timerange == str)): ms.msselect({'time': timerange}) if spw and (type(spw) == str): ms.selectinit(datadescid=int(spwid)) nchan = int(echan) - int(bchan) + 1 ms.selectchannel(nchan, int(bchan), 1, 1) else: #outputvis is splitted, selections have already applied, select all the data ms.selectinit(datadescid=0) orec = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True) b_rows = orec['data'].shape[2] nchan = orec['data'].shape[1] #szo=orec['data'].shape print 'dimension of output data', orec['data'].shape #orec['data']=orec['data'].reshape(szo[0],szo[1],szo[2],nspw,szo[3]/nspw,order='F') #print 'reshaped rec1 ', orec['data'].shape t_rows = orec['data'].shape[3] casalog.post('Number of baselines: ' + str(b_rows)) casalog.post('Number of spectral channels: ' + str(nchan)) casalog.post('Number of time pixels: ' + str(t_rows)) if subtime1 and (not subtime2): casalog.post( 'Only "subtime1" is defined, subtracting background defined in subtime1: ' + subtime1) t1 = (np.amax(rec1['time']) + np.amin(rec1['time'])) / 2. print 't1: ', qa.time(qa.quantity(t1, 's'), form='ymd', prec=10) for i in range(t_rows): orec['data'][:, :, :, i] -= rec1avg if reverse: orec['data'][:, :, :, i] = -orec['data'][:, :, :, i] if subtime1 and subtime2 and (type(subtime2) == str): casalog.post( 'Both subtime1 and subtime2 are specified, doing linear interpolation between "subtime1" and "subtime2"' ) t1 = (np.amax(rec1['time']) + np.amin(rec1['time'])) / 2. t2 = (np.amax(rec2['time']) + np.amin(rec2['time'])) / 2. touts = orec['time'] print 't1: ', qa.time(qa.quantity(t1, 's'), form='ymd', prec=10) print 't2: ', qa.time(qa.quantity(t2, 's'), form='ymd', prec=10) for i in range(t_rows): tout = touts[i] if tout > np.amax([t1, t2]): tout = np.amax([t1, t2]) elif tout < np.amin([t1, t2]): tout = np.amin([t1, t2]) orec['data'][:, :, :, i] -= (rec2avg - rec1avg) * (tout - t1) / (t2 - t1) + rec1avg if reverse: orec['data'][:, :, :, i] = -orec['data'][:, :, :, i] #orec['data']=orec['data'].reshape(szo[0],szo[1],szo[2],szo[3],order='F') #put the modified data back into the output visibility set del orec['time'] del orec['axis_info'] ms.putdata(orec) ms.close()
def get_dspec(vis=None, savespec=True, specfile=None, bl='', uvrange='', field='', scan='', datacolumn='data', domedian=False, timeran=None, spw=None, timebin='0s', regridfreq=False, fillnan=None, verbose=False): # from split_cli import split_cli as split if vis.endswith('/'): vis = vis[:-1] msfile = vis if not spw: spw = '' if not timeran: timeran = '' if not bl: bl = '' if domedian: if not uvrange: uvrange = '0.2~0.8km' bl = '' else: uvrange = '' # Open the ms and plot dynamic spectrum if verbose: print('Splitting selected data...') vis_spl = './tmpms.splitted' if os.path.exists(vis_spl): os.system('rm -rf ' + vis_spl) # split(vis=msfile, outputvis=vis_spl, timerange=timeran, antenna=bl, field=field, scan=scan, spw=spw, # uvrange=uvrange, timebin=timebin, datacolumn=datacolumn) ms.open(msfile, nomodify=False) ms.split(outputms=vis_spl, whichcol=datacolumn, time=timeran, spw=spw, baseline=bl, field=field, scan=scan, uvrange=uvrange, timebin=timebin) ms.close() if verbose: print('Regridding into a single spectral window...') # print('Reading data spw by spw') try: tb.open(vis_spl + '/POLARIZATION') corrtype = tb.getcell('CORR_TYPE', 0) pols = [stokesenum[p] for p in corrtype] tb.close() except: pols = [] if regridfreq: ms.open(vis_spl, nomodify=False) ms.cvel(outframe='LSRK', mode='frequency', interp='nearest') ms.selectinit(datadescid=0, reset=True) data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True) specamp = data['amplitude'] freq = data['axis_info']['freq_axis']['chan_freq'] else: ms.open(vis_spl) ms.selectinit(datadescid=0, reset=True) spwinfo = ms.getspectralwindowinfo() specamp = [] freq = [] time = [] for descid in range(len(spwinfo.keys())): ms.selectinit(datadescid=0, reset=True) ms.selectinit(datadescid=descid) data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True) specamp_ = data['amplitude'] freq_ = data['axis_info']['freq_axis']['chan_freq'] time_ = data['time'] if fillnan is not None: flag_ = ms.getdata(['flag', 'time', 'axis_info'], ifraxis=True)['flag'] if type(fillnan) in [int, float, long]: specamp_[flag_] = float(fillnan) else: specamp_[flag_] = 0.0 specamp.append(specamp_) freq.append(freq_) time.append(time_) specamp = np.concatenate(specamp, axis=1) freq = np.concatenate(freq, axis=0) ms.selectinit(datadescid=0, reset=True) ms.close() os.system('rm -rf ' + vis_spl) (npol, nfreq, nbl, ntim) = specamp.shape freq = freq.reshape(nfreq) if verbose: print('npol, nfreq, nbl, ntime:', (npol, nfreq, nbl, ntim)) spec = np.swapaxes(specamp, 2, 1) tim = data['time'] if domedian: if verbose: print('doing median of all the baselines') # mask zero values before median spec_masked = np.ma.masked_where(spec < 1e-9, spec) spec_med = np.ma.filled(np.ma.median(spec_masked, axis=1), fill_value=0.) nbl = 1 ospec = spec_med.reshape((npol, nbl, nfreq, ntim)) else: ospec = spec # Save the dynamic spectral data if savespec: if not specfile: specfile = msfile + '.dspec.npz' if os.path.exists(specfile): os.system('rm -rf ' + specfile) np.savez(specfile, spec=ospec, tim=tim, freq=freq, timeran=timeran, spw=spw, bl=bl, uvrange=uvrange, pol=pols) if verbose: print('Median dynamic spectrum saved as: ' + specfile) return {'spec': ospec, 'tim': tim, 'freq': freq, 'timeran': timeran, 'spw': spw, 'bl': bl, 'uvrange': uvrange, 'pol': pols}
def get_dspec(vis=None, savespec=True, specfile=None, bl='', uvrange='', field='', scan='', datacolumn='data', domedian=False, timeran=None, spw=None, timebin='0s', regridfreq=False, fillnan=None, verbose=False, usetbtool=False): # from split_cli import split_cli as split if vis.endswith('/'): vis = vis[:-1] msfile = vis if not spw: spw = '' if not timeran: timeran = '' if domedian: if not uvrange: uvrange = '0.2~0.8km' # bl = '' else: uvrange = '' if not bl: bl = '' else: uvrange = '' # Open the ms and plot dynamic spectrum if verbose: print('Splitting selected data...') if usetbtool: try: tb.open(vis + '/POLARIZATION') corrtype = tb.getcell('CORR_TYPE', 0) pols = [stokesenum[p] for p in corrtype] tb.close() except: pols = [] antmask = [] if uvrange is not '' or bl is not '': ms.open(vis) ms.selectinit(datadescid=0) mdata = ms.metadata() antlist = mdata.antennaids() mdata.done() staql = {'uvdist': uvrange, 'baseline': bl, 'spw': spw, 'field': field, 'scan': scan, 'timerange': timeran} ### todo the selection only works for uvrange and bl. To make the selection of other items works, ## I need to make mask for other items. a = ms.msselect(staql) mdata = ms.metadata() baselines = mdata.baselines() for lidx, l in enumerate(antlist): antmask.append(baselines[l][antlist[lidx:]]) antmask = np.hstack(antmask) mdata.done() ms.close() tb.open(vis) spwtb = tbtool() spwtb.open(vis + '/SPECTRAL_WINDOW') ptb = tbtool() ptb.open(vis + '/POLARIZATION') ms.open(vis) spwlist = [] mdata = ms.metadata() nspw = mdata.nspw() nbl = mdata.nbaselines() + mdata.nantennas() nscans = mdata.nscans() spw_nfrq = [] # List of number of frequencies in each spw for i in range(nspw): spw_nfrq.append(mdata.nchan(i)) spw_nfrq = np.array(spw_nfrq) nf = np.sum(spw_nfrq) smry = mdata.summary() scan_ntimes = [] # List of number of times in each scan for iscan in range(nscans): scan_ntimes.append( smry['observationID=0']['arrayID=0']['scan=' + str(iscan)]['fieldID=0']['nrows'] / nspw / nbl) scan_ntimes = np.array(scan_ntimes) scan_ntimes_integer = scan_ntimes.astype(np.int) if len(np.where(scan_ntimes % scan_ntimes_integer != 0)[0]) != 0: # if True: scan_ntimes = [] # List of number of times in each scan for iscan in range(nscans): scan_ntimes.append( len(smry['observationID=0']['arrayID=0']['scan=' + str(iscan)]['fieldID=0'].keys()) - 6) scan_ntimes = np.array(scan_ntimes) else: scan_ntimes = scan_ntimes_integer nt = np.sum(scan_ntimes) times = tb.getcol('TIME') if times[nbl] - times[0] != 0: # This is frequency/scan sort order order = 'f' elif times[nbl * nspw - 1] - times[0] != 0: # This is time sort order order = 't' npol = ptb.getcol('NUM_CORR', 0, 1)[0] ptb.close() freq = np.zeros(nf, float) times = np.zeros(nt, float) if order == 't': specamp = np.zeros((npol, nf, nbl, nt), np.complex) for j in range(nt): fptr = 0 # Loop over spw for i in range(nspw): # Get channel frequencies for this spw (annoyingly comes out as shape (nf, 1) cfrq = spwtb.getcol('CHAN_FREQ', i, 1)[:, 0] if j == 0: # Only need this the first time through spwlist += [i] * len(cfrq) if i == 0: times[j] = tb.getcol('TIME', nbl * (i + nspw * j), 1) # Get the time spec_ = tb.getcol('DATA', nbl * (i + nspw * j), nbl) # Get complex data for this spw flag = tb.getcol('FLAG', nbl * (i + nspw * j), nbl) # Get flags for this spw nfrq = len(cfrq) # Apply flags if type(fillnan) in [int, float]: spec_[flag] = float(fillnan) else: spec_[flag] = 0.0 # Insert data for this spw into larger array specamp[:, fptr:fptr + nfrq, :, j] = spec_ freq[fptr:fptr + nfrq] = cfrq fptr += nfrq else: specf = np.zeros((npol, nf, nt, nbl), np.complex) # Array indexes are swapped iptr = 0 for j in range(nscans): # Loop over scans for i in range(nspw): # Loop over spectral windows s = scan_ntimes[j] f = spw_nfrq[i] s1 = np.sum(scan_ntimes[:j]) # Start time index s2 = np.sum(scan_ntimes[:j + 1]) # End time index f1 = np.sum(spw_nfrq[:i]) # Start freq index f2 = np.sum(spw_nfrq[:i + 1]) # End freq index spec_ = tb.getcol('DATA', iptr, nbl * s) flag = tb.getcol('FLAG', iptr, nbl * s) if j == 0: cfrq = spwtb.getcol('CHAN_FREQ', i, 1)[:, 0] freq[f1:f2] = cfrq spwlist += [i] * len(cfrq) times[s1:s2] = tb.getcol('TIME', iptr, nbl * s).reshape(s, nbl)[:, 0] # Get the times iptr += nbl * s # Apply flags if type(fillnan) in [int, float]: spec_[flag] = float(fillnan) else: spec_[flag] = 0.0 # Insert data for this spw into larger array specf[:, f1:f2, s1:s2] = spec_.reshape(npol, f, s, nbl) # Swap the array indexes back to the desired order specamp = np.swapaxes(specf, 2, 3) tb.close() spwtb.close() ms.close() if len(antmask) > 0: specamp = specamp[:, :, np.where(antmask)[0], :] (npol, nfreq, nbl, ntim) = specamp.shape tim = times else: # Open the ms and plot dynamic spectrum if verbose: print('Splitting selected data...') vis_spl = './tmpms.splitted' if os.path.exists(vis_spl): os.system('rm -rf ' + vis_spl) # split(vis=msfile, outputvis=vis_spl, timerange=timeran, antenna=bl, field=field, scan=scan, spw=spw, # uvrange=uvrange, timebin=timebin, datacolumn=datacolumn) try: from split_cli import split_cli as split split(vis=msfile, outputvis=vis_spl, datacolumn=datacolumn, timerange=timeran, spw=spw, antenna=bl, field=field, scan=scan, uvrange=uvrange, timebin=timebin) except: ms.open(msfile, nomodify=True) ms.split(outputms=vis_spl, whichcol=datacolumn, time=timeran, spw=spw, baseline=bl, field=field, scan=scan, uvrange=uvrange, timebin=timebin) ms.close() if verbose: print('Regridding into a single spectral window...') # print('Reading data spw by spw') try: tb.open(vis_spl + '/POLARIZATION') corrtype = tb.getcell('CORR_TYPE', 0) pols = [stokesenum[p] for p in corrtype] tb.close() except: pols = [] if regridfreq: ms.open(vis_spl, nomodify=False) ms.cvel(outframe='LSRK', mode='frequency', interp='nearest') ms.selectinit(datadescid=0, reset=True) data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True) specamp = data['amplitude'] freq = data['axis_info']['freq_axis']['chan_freq'] else: ms.open(vis_spl) ms.selectinit(datadescid=0, reset=True) spwinfo = ms.getspectralwindowinfo() specamp = [] freq = [] time = [] for descid in range(len(spwinfo.keys())): ms.selectinit(datadescid=0, reset=True) ms.selectinit(datadescid=descid) data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True) specamp_ = data['amplitude'] freq_ = data['axis_info']['freq_axis']['chan_freq'] time_ = data['time'] if fillnan is not None: flag_ = ms.getdata(['flag', 'time', 'axis_info'], ifraxis=True)['flag'] if type(fillnan) in [int, float, long]: specamp_[flag_] = float(fillnan) else: specamp_[flag_] = 0.0 specamp.append(specamp_) freq.append(freq_) time.append(time_) specamp = np.concatenate(specamp, axis=1) freq = np.concatenate(freq, axis=0) ms.selectinit(datadescid=0, reset=True) ms.close() os.system('rm -rf ' + vis_spl) (npol, nfreq, nbl, ntim) = specamp.shape freq = freq.reshape(nfreq) tim = data['time'] if verbose: print('npol, nfreq, nbl, ntime:', (npol, nfreq, nbl, ntim)) spec = np.swapaxes(specamp, 2, 1) if domedian: if verbose: print('doing median of all the baselines') # mask zero values before median # spec_masked = np.ma.masked_where(spec < 1e-9, spec) # spec_masked2 = np.ma.masked_invalid(spec) # spec_masked = np.ma.masked_array(spec, mask=np.logical_or(spec_masked.mask, spec_masked2.mask)) # spec_med = np.ma.filled(np.ma.median(spec_masked, axis=1), fill_value=0.) spec = np.abs(spec) spec_med = np.nanmedian(spec, axis=1) nbl = 1 ospec = spec_med.reshape((npol, nbl, nfreq, ntim)) else: ospec = spec # Save the dynamic spectral data if savespec: if not specfile: specfile = msfile + '.dspec.npz' if os.path.exists(specfile): os.system('rm -rf ' + specfile) np.savez(specfile, spec=ospec, tim=tim, freq=freq, timeran=timeran, spw=spw, bl=bl, uvrange=uvrange, pol=pols) if verbose: print('Median dynamic spectrum saved as: ' + specfile) return {'spec': ospec, 'tim': tim, 'freq': freq, 'timeran': timeran, 'spw': spw, 'bl': bl, 'uvrange': uvrange, 'pol': pols}
def pltvs(vis='', outputvis='', plttimerange='', pltspw1='', pltspw2='', pltspw3='', timoffset=4, windowlen=10, windowtype='hamming', pol='LL', bl='19&22'): ms.open(vis, nomodify=True) """ This function can do two steps: (1) plot the uv amplitude vs. time on three selected channel and compare the original and smoothed signals. You can select appropriate window length and type, and the offset time to match the peaks and valleys for the three channel curves. (2) Confirm the use of specified window length, type and offset time and smooth the data channel by channel. The background-subtracted measurement set is then generated. """ timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True) tim = timfreq['time'] timerange = str(qa.time(qa.quantity(tim[0], 's'), prec=8)[0]) + '~' + str( qa.time(qa.quantity(tim[-1], 's'), prec=8)[0]) # check plotting timerange if plttimerange and (type(plttimerange) == str): print 'plotting the specified timerange: ', plttimerange else: plttimerange = timerange print 'plotting the entire timerange: ', plttimerange if pltspw1 and (type(pltspw1) == str): print 'Using the specified channel 1:', pltspw1 else: pltspw1 = '0:400' if pltspw2 and (type(pltspw2) == str): print 'Using the specified channel 2:', pltspw2 else: pltspw2 = '0:500' if pltspw3 and (type(pltspw3) == str): print 'Using the specified channel 3:', pltspw3 else: pltspw3 = '0:600' [spwid1, chan1] = pltspw1.split(':') [spwid2, chan2] = pltspw2.split(':') [spwid3, chan3] = pltspw3.split(':') chanid = [chan1, chan2, chan3] if not (spwid1 == spwid2 and spwid1 == spwid3): print 'Please use the same spectral window' exit() if not (timoffset and (type(timoffset) == int)): timoffset = int(4) # initialize the loop-out status status = 'n' while True: timoffset = int( raw_input("Please specify the offset after smoothing:")) windowlen = int( raw_input("Please specify window width for smoothing:")) windowtype = str( raw_input( "Please specify window type for smoothing: (e.g. 'flat', 'hanning', 'hamming', 'bartlett', 'blackman')" )) pol = str(raw_input("Please specify polarization: (e.g. RR/LL)")) bl = str(raw_input("Please specify baseline: (e.g. '19&22')")) j = 331 for i in range(len(chanid)): ms.selectinit(datadescid=int(spwid1)) if bl and (type(bl) == str): ms.msselect({'baseline': bl}) if timerange and (type(timerange) == str): ms.msselect({'time': timerange}) if chanid[i] and (type(chanid[i]) == str): ms.selectchannel(1, int(chanid[i]), 1, 1) specdata = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True) if pol == 'RR': spec = specdata['data'][0, 0, 0, :] if pol == 'LL': spec = specdata['data'][1, 0, 0, :] ms.selectinit(datadescid=int(spwid1)) if bl and (type(bl) == str): ms.msselect({'baseline': bl}) if plttimerange and (type(plttimerange) == str): ms.msselect({'time': plttimerange}) if chanid[i] and (type(chanid[i]) == str): ms.selectchannel(1, int(chanid[i]), 1, 1) specdata_plt = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True) if pol == 'RR': spec_plt = specdata_plt['data'][0, 0, 0, :] if pol == 'LL': spec_plt = specdata_plt['data'][1, 0, 0, :] spec_plt_smooth = task_smooth.smooth(spec_plt, timoffset, windowlen, windowtype) spec_smooth = task_smooth.smooth(spec, timoffset, windowlen, windowtype) spec_plt_amp = np.absolute(spec_plt) spec_plt_smooth_amp = np.absolute(spec_plt_smooth) #spec_plt_amp=sqrt(spec_plt.real**2+spec_plt.imag**2) #spec_plt_smooth_amp=sqrt(spec_plt_smooth.real**2+spec_plt_smooth.imag**2) #print len(spec) #print len(spec_smooth) #print type(spec) #print spec[0] sp1 = fft(spec) sp2 = fft(spec_smooth) sp3 = sp1 - sp2 freq1 = fftfreq(len(sp1), d=0.001) freq2 = fftfreq(len(sp2), d=0.001) freq3 = fftfreq(len(sp3), d=0.001) freq1_index = np.argsort(freq1) freq2_index = np.argsort(freq2) freq3_index = np.argsort(freq3) #print min(freq1),max(freq1) subplot(j) plot(spec_plt_amp) plot(spec_plt_smooth_amp) title("Signal vs Time") j = j + 1 subplot(j) plot(freq1[freq1_index], log10(sp1[freq1_index])) plot(freq2[freq2_index], log10(sp2[freq2_index])) ylim([0, 6]) title("FFT signal vs Frequency") j = j + 1 subplot(j) #plot(subspec) plot(freq3[freq3_index], log10(sp3[freq3_index])) ylim([0, 6]) title("FFT smoothed signal vs Frequency") j = j + 1 #print "number of original data points: ",len(spec) #print "number of smoothed data points: ",len(spec_smooth) status = str( raw_input("Confirm to use current parameters? (y/n/abort) ")) if status == 'y': flag1 = str( raw_input("Smooth all the channels and time range? (y/n) ")) if flag1 == 'y': smtimerange = '' smspw = '' splitsel = False else: print 'confirm using window width: ', windowlen print 'confirm using window type: ', windowtype smtimerange = str( raw_input( "Please specify the time range for smoothing (HH:MM:SS) :" )) smspw = str( raw_input( "Please specify spectral window and channel (e.g. 0:0~1032) :" )) splitsel = True break elif status == 'abort': print 'Abort background subtraction.' sys.exit() if not outputvis: outputvis = str(timoffset) + '_' + str(windowlen) + '_' + str( windowtype) + '.ms' print "Generating output: ", outputvis ms.close() result1 = subvs(vis, outputvis, smtimerange, smspw, timoffset, windowlen, windowtype, '', '', '')