def __init__(self, filename, nints=1, nskip=0, ddid=-1, selectpol=['XX','YY']):
"""Initializes the class "mwa". This creates new object containing data and metadata for a set of files in a directory.
It also includes functions to manipulate data and do some analysis, like making lightcurves, etc.
Note that this uses CASA libraries in a way that requires it to be run from within "casapy". Other options exist elsewhere.
Default is to read in first file (alphabetically) in the directory. Use nints and nskip to control where to start and number of files to read. this assumes that the alphabetical order is the time order.
Examples of usage in python/casapy:
import mwavis
obs = mwavis.mwa('directory') -- create observation object for first file in a directory.
print obs.data.shape -- see the structure of data read in. dimensions are (time, baseline, channel, polarization)
results = obs.bisplc(show=1) -- create a bispectrum lightcurve and show any candidate transients. results are returned in 'return' object
"""
# critical parameters. need to edit these
ants = range(64) # set what antennas to use. default is to use "range" to specify all antennas with range(n_ant+1)
self.chans = n.array(range(64)) # set what channesl to use. default is to use range to select all channels.
self.track0 = [n.zeros(len(self.chans)), self.chans]
# open first file and read a bit of data to define data structure
self.file = filename
print 'Reading ', self.file
ms.open(self.file)
spwinfo = ms.getspectralwindowinfo()
summary = ms.summary()
# read in multiple subbands ("data id" in casa parlance). mwa probably doesn't use this.
if ddid < 0:
ddidlist = range(len(spwinfo['spwInfo']))
else:
ddidlist = [ddid]
freq = n.array([])
for ddid in ddidlist:
nch = spwinfo['spwInfo'][str(ddid)]['NumChan']
ch0 = spwinfo['spwInfo'][str(ddid)]['Chan1Freq']
chw = spwinfo['spwInfo'][str(ddid)]['ChanWidth']
freq = n.concatenate( (freq, (ch0 + chw * n.arange(nch)) * 1e-9) )
self.freq = freq[self.chans]
self.nchan = len(self.freq)
# read data into data structure. start with subband 0, then iterate over higher ones. mwa probably only has 0.
ms.selectinit(datadescid=0) # reset select params for later data selection
selection = {'antenna1': ants, 'antenna2': ants}
ms.select(items = selection)
print 'Reading SB %d, polarization %s...' % (0, selectpol)
ms.selectpolarization(selectpol)
da = ms.getdata(['data','axis_info'], ifraxis=True)
if da == {}:
print 'No data found.'
return 1
newda = n.transpose(da['data'], axes=[3,2,1,0]) # if using multi-pol data.
if len(ddidlist) > 1:
for ddid in ddidlist[1:]:
ms.selectinit(datadescid=ddid) # reset select params for later data selection
ms.select(items = selection)
print 'Reading SB %d, polarization %s...' % (ddid, selectpol)
ms.selectpolarization(selectpol)
da = ms.getdata(['data','axis_info'], ifraxis=True)
newda = n.concatenate( (newda, n.transpose(da['data'], axes=[3,2,1,0])), axis=2 )
ms.close()
rawdata = newda # array for collecting raw data
# check pol and baseline dimensions of data
self.npol_orig = da['data'].shape[0]
self.npol = len(selectpol)
self.nbl = da['data'].shape[2]
print 'Initializing %d of %d polarizations' % (self.npol, self.npol_orig)
print 'Initializing nchan:', self.nchan
print 'Initializing nbl:', self.nbl
self.nskip = int(nskip*self.nbl) # number of iterations to skip (for reading in different parts of buffer)
# set number of antennas and names of baselines
bls = da['axis_info']['ifr_axis']['ifr_shortname']
self.blarr = n.array([[bls[i].split('-')[0],bls[i].split('-')[1]] for i in range(len(bls))])
self.ants = n.unique(self.blarr)
self.nants = len(self.ants)
print 'Initializing nants:', self.nants
# set integration time and time axis
ti = da['axis_info']['time_axis']['MJDseconds']
self.inttime = n.mean([ti[i+1] - ti[i] for i in range(len(ti)-1)])
print 'Initializing integration time (s):', self.inttime
self.reltime = ti - ti[0]
if len(msfiles) > 1:
for file in msfiles[1:]:
ms.open(file)
print 'Reading ', file
spwinfo = ms.getspectralwindowinfo()
# read in multiple subbands ("data id" in casa parlance). mwa probably doesn't use this.
if ddid < 0:
ddidlist = range(len(spwinfo['spwInfo']))
else:
ddidlist = [ddid]
# read data into data structure. start with subband 0, then iterate over higher ones. mwa probably only has 0.
ms.selectinit(datadescid=0) # reset select params for later data selection
ms.select(items = selection)
print 'Reading SB %d, polarization %s...' % (0, selectpol)
ms.selectpolarization(selectpol)
da = ms.getdata(['data','axis_info'], ifraxis=True)
if da == {}:
print 'No data found.'
return 1
newda = n.transpose(da['data'], axes=[3,2,1,0]) # if using multi-pol data.
if len(ddidlist) > 1:
for ddid in ddidlist[1:]:
ms.selectinit(datadescid=ddid) # reset select params for later data selection
ms.select(items = selection)
print 'Reading SB %d, polarization %s...' % (ddid, selectpol)
ms.selectpolarization(selectpol)
da = ms.getdata(['data','axis_info'], ifraxis=True)
newda = n.concatenate( (newda, n.transpose(da['data'], axes=[3,2,1,0])), axis=2 )
ms.close()
rawdata = n.concatenate( (rawdata, newda), axis=0)
# create data structures
self.rawdata = rawdata
self.data = rawdata[:,:,self.chans] # remove channels ignored earlier
self.dataph = (self.data.mean(axis=3).mean(axis=1)).real # create dataph, which is sum over all baselines. akin to calculating tied-array beam (possibly without calibration)
self.min = self.dataph.min()
self.max = self.dataph.max()
print 'Shape of rawdata, data:'
print self.rawdata.shape, self.data.shape
print 'Dataph min, max:'
print self.min, self.max
def pipeline(asdmfile, workdir='', mode='prep', intentfilter='OBSERVE_TARGET'):
""" Pipeline to simplify quasi-blind searching over many scans and/or files.
mode of 'frb' does multi-node processing of frb data.
mode of 'any' does search over all scans of arbitrary observation.
"""
if workdir == '':
workdir = os.chdir(workdir)
# strip out path
if '/' in asdmfile:
asdmfile = asdmfile.split('/')[-1]
# data filler steps
# asdmfile = search(datadir) # daemon process?
msfile = asdmfile + '.ms'
telcalfile = asdmfile + '.GN'
try:
goodscans = prep(asdmfile, workdir=workdir, intentfilter=intentfilter) # set up asdm and telcal files, get scans of interest (tuple with scann
msfile2 = asdm2ms(asdmfile, msfile, goodscans[scan][0])
except:
raise
if mode == 'prep':
# just filling data...
return msfile2
elif mode == 'frb':
# set good channels
# nch = 64
# edgechan = n.round(0.05*nch).astype('int')
# chans = n.arange(edgechan, nch-edgechan, dtype='int')
dmarr = [0,19.2033,38.4033,57.6025,76.8036,96.0093,115.222,134.445,153.68,172.93,192.198,211.486,230.797,250.133,269.498,288.894,308.323,327.788,347.292,366.837,386.426,406.062,425.747,445.484,465.276,485.125,505.033,525.005,545.042,565.147,585.322,605.571,625.896,646.3,666.786,687.355,708.012,728.759,749.598,770.532,791.565,812.699,833.936,855.28,876.733,898.299,919.979,941.778,963.697,985.741,1007.91,1030.21,1052.64,1075.21,1097.92,1120.76,1143.76,1166.9,1190.19,1213.63,1237.23,1260.99,1284.92,1309.01,1333.27,1357.7,1382.31,1407.09,1432.06,1457.22,1482.56,1508.1,1533.83,1559.76,1585.89,1612.23,1638.77,1665.53,1692.51,1719.7,1747.12,1774.77,1802.64,1830.75,1859.1,1887.69,1916.53,1945.61,1974.95,2004.54,2034.39,2064.51,2094.9,2125.56,2156.49,2187.71,2219.21,2250.99,2283.07,2315.45,2348.13,2381.12,2414.41,2448.02,2481.94,2516.19,2550.77,2585.68,2620.92,2656.51,2692.44,2728.72,2765.36,2802.36,2839.72,2877.45,2915.55,2954.04,2992.91]
chans = range(23,31)+range(32,50)+range(70,128)
d = leanpipe.pipe_thread(filename=msfile, nints=nints, nskip=0, iterint=200, spw=[0,1], chans=chans, dmarr=dmarr, fwhmsurvey=0.5, fwhmfield=0.5, selectpol=['RR','LL'], scan=0, datacol='data', size=25600, res=50, sigma_image=6., searchtype='imageallstat', telcalfile=telcalfile, filtershape=None, savecands=True)
elif mode == 'any':
# read structure of data
os.chdir(workdir)
ms.open(msfile)
scans = ms.getscansummary()
spwinfo = ms.getspectralwindowinfo()
summary = ms.summary()
ms.close()
print 'Searching over %d field(s) in %d scans and %d spws.' % (summary['nfields'], len(scans.keys()), len(spwinfo))
# search
scanlist = scans.keys()
for scan in scanlist:
nints = int(n.round((scans[scan]['0']['EndTime']-scans[scan]['0']['BeginTime'])*24*3600/scans[scan]['0']['IntegrationTime'])-1)
spws = scans[scan]['0']['SpwIds']
for spw in spws:
nch = spwinfo[str(spw)]['NumChan']
print
print 'For scan %d and spw %d, %d integrations and %d channels' % (int(scan), spws[spw], nints, nch)
edgechan = n.round(0.05*nch).astype('int')
chans = n.arange(edgechan, nch-edgechan, dtype='int')
d = leanpipe.pipe_thread(filename=msfile, nints=nints, nskip=0, iterint=min(nints,200), spw=[spws[spw]], chans=chans, dmarr=range(0,300,30), fwhmsurvey=0.5, fwhmfield=0.5, selectpol=['RR','LL'], scan=int(scanlist.index(scan)), datacol='data', size=50000, res=100, sigma_bisp=5., sigma_image=5., calibratedfilter=True, specmodfilter=1.5, searchtype='imageallstat', telcalfile=telcalfile, telcalcalibrator='3C48', filtershape='b')
def __init__(self, filename, nints=1, nskip=0, selectpol=['XX','YY'], datacol='corrected'):
"""Initializes the class "lofar". This creates new object containing data and metadata for a set of files in a directory.
It also includes functions to manipulate data and do some analysis, like making lightcurves, etc.
Note that this uses CASA libraries in a way that requires it to be run from within "casapy". Other options exist elsewhere.
Default is to read in first file (alphabetically) in the directory. Use nints and nskip to control where to start and number of files to read. this assumes that the alphabetical order is the time order.
Examples of usage in python/casapy:
import lofarvis
obs = lofarvis.lofar('directory') -- create observation object for first file in a directory.
print obs.data.shape -- see the structure of data read in. dimensions are (time, baseline, channel, polarization)
results = obs.bisplc(show=1) -- create a bispectrum lightcurve and show any candidate transients. results are returned in 'return' object
"""
# critical parameters. need to edit these
ants = range(25) # set what antennas to use. default is to use "range" to specify all antennas with range(n_ant+1)
self.chans = n.array(range(1)) # set what channesl to use. default is to use range to select all channels.
self.track0 = [n.zeros(len(self.chans)), self.chans]
# open file and read a bit of data to define data structure
self.file = filename
print 'Reading ', self.file
ms.open(self.file)
spwinfo = ms.getspectralwindowinfo()
summary = ms.summary()
scansummary = ms.getscansummary()
starttime_mjd = scansummary['summary']['0']['0']['BeginTime']
starttime0 = qa.getvalue(qa.convert(qa.time(qa.quantity(starttime_mjd+0/(24.*60*60),'d'),form=['ymd'], prec=9), 's'))
stoptime0 = qa.getvalue(qa.convert(qa.time(qa.quantity(starttime_mjd+0.5/(24.*60*60), 'd'), form=['ymd'], prec=9), 's'))
ms.selectinit(datadescid=0) # initialize to initialize params
selection = {'time': [starttime0, stoptime0], 'antenna1': ants, 'antenna2': ants}
da = ms.getdata([datacol,'axis_info', 'u', 'v', 'w'], ifraxis=True)
ms.close()
self.npol_orig = da[datacol].shape[0]
self.npol = len(selectpol)
self.nbl = da[datacol].shape[2]
print 'Initializing %d of %d polarizations' % (self.npol, self.npol_orig)
print 'Initializing nbl:', self.nbl
# good baselines
bls = da['axis_info']['ifr_axis']['ifr_shortname']
self.blarr = n.array([[bls[i].split('-')[0],bls[i].split('-')[1]] for i in range(len(bls))])
self.ants = n.unique(self.blarr)
self.nants = len(self.ants)
self.nants0 = len(self.ants)
print 'Initializing nants:', self.nants
self.nskip = int(nskip*self.nbl) # number of iterations to skip (for reading in different parts of buffer)
# set integration time
ti0 = da['axis_info']['time_axis']['MJDseconds']
self.inttime = scansummary['summary']['0']['0']['IntegrationTime']
self.inttime0 = self.inttime
print 'Initializing integration time (s):', self.inttime
nch = spwinfo['spwInfo']['0']['NumChan']
ch0 = spwinfo['spwInfo']['0']['Chan1Freq']
chw = spwinfo['spwInfo']['0']['ChanWidth']
freq = (ch0 + chw * n.arange(nch)) * 1e-9
self.freq = freq[self.chans]
self.nchan = len(self.freq)
# read data into data structure. start with subband 0, then iterate over higher ones. lofar probably only has 0.
timeskip = self.inttime*nskip
starttime = qa.getvalue(qa.convert(qa.time(qa.quantity(starttime_mjd+timeskip/(24.*60*60),'d'),form=['ymd'], prec=9), 's'))
stoptime = qa.getvalue(qa.convert(qa.time(qa.quantity(starttime_mjd+(timeskip+nints*self.inttime)/(24.*60*60), 'd'), form=['ymd'], prec=9), 's'))
print 'First integration:', qa.time(qa.quantity(starttime_mjd,'d'),form=['ymd'],prec=9)
print
print 'Reading times', qa.time(qa.quantity(starttime_mjd+timeskip/(24.*60*60),'d'),form=['hms'], prec=9), 'to', qa.time(qa.quantity(starttime_mjd+(timeskip+nints*self.inttime)/(24.*60*60), 'd'), form=['hms'], prec=9)
ms.open(self.file)
ms.selectinit(datadescid=0) # reset select params for later data selection
selection = {'time': [starttime, stoptime], 'antenna1': ants, 'antenna2': ants}
ms.select(items = selection)
print 'Reading SB %d, polarization %s...' % (0, selectpol)
ms.selectpolarization(selectpol)
da = ms.getdata([datacol,'axis_info', 'u', 'v', 'w'], ifraxis=True)
u = da['u']; v = da['v']; w = da['w']
if da == {}:
print 'No data found.'
return 1
newda = n.transpose(da[datacol], axes=[3,2,1,0]) # if using multi-pol data.
ms.close()
rawdata = newda # array for collecting raw data
# create data structures
self.u = u.transpose() * (-self.freq.mean()*1e9/3e8) # uvw are in m on ground. scale by -wavelenth to get projected lamba uvw (as in miriad?)
self.v = v.transpose() * (-self.freq.mean()*1e9/3e8)
self.w = w.transpose() * (-self.freq.mean()*1e9/3e8)
self.rawdata = rawdata
self.data = rawdata[:,:,self.chans] # remove channels ignored earlier
self.dataph = (self.data.mean(axis=3).mean(axis=1)).real # create dataph, which is sum over all baselines. akin to calculating tied-array beam (possibly without calibration)
self.min = self.dataph.min()
self.max = self.dataph.max()
print 'Shape of rawdata, data:'
print self.rawdata.shape, self.data.shape
print 'Dataph min, max:'
print self.min, self.max
ti = da['axis_info']['time_axis']['MJDseconds']
self.reltime = ti - ti[0]
def read_data_from_ms(msfn, viscol="DATA", noisecol='SIGMA',
mode='tot',noise_est = False):
"""
Reads polarization or total intensity data in visibility and noise arrays.
Args:
msfn: Name of the MeasurementSet file from which to read the data
viscol: A string with the name of the MS column from which to read the
data [DATASET_STRING]
noisecol: A string with the name of the MS column from which to read
the noise or weights ['SIGMA']
mode: Flag to set whether the function should read in
polarization data ('pol') or total intensity data ('tot')
Returns:
vis
noise
"""
m = M.Messenger(2)
if mode == 'pol':
m.header2("Reading polarization data from the MeasurementSet...")
if mode == 'tot':
m.header2("Reading total intensity data from the MeasurementSet...")
viscol = viscol.lower()
noisecol = noisecol.lower()
if noise_est:
m.message("Performing simple noise estimate")
computenoise(msfn,viscol,m,minsamp=10)
ms.open(msfn)
meta = ms.metadata()
nspw = range(meta.nspw())
nchan = []
nvis = []
u = []
v = []
freq = []
if mode == 'pol':
Qvis = []
Uvis = []
Qsigma = []
Usigma = []
lambs2 = []
if mode == 'tot':
vis = []
sigma = []
# the Q,U OR the I part of the S Jones matrix (hence Spart)
# from the Stokes enumeration defined in the casa core libraries
# http://www.astron.nl/casacore/trunk/casacore/doc/html \
#/classcasa_1_1Stokes.html#e3cb0ef26262eb3fdfbef8273c455e0c
# this defines which polarization type the data columns correspond to
corr = ms.getdata(['axis_info'])['axis_info']['corr_axis']
corr_announce = "Correlation type detected to be "
ii = complex(0, 1)
if mode == 'pol':
if corr[0] == 'RR': # RR, RL, LR, LL
QSpart = np.array([0, 0.5, 0.5, 0])
USpart = np.array([0, -0.5 * ii, 0.5 * ii, 0])
corr_announce += "RR, RL, LR, LL"
elif corr[0] == 'I': # I, Q, U, V
QSpart = np.array([0, 1., 0, 0])
USpart = np.array([0, 0, 1., 0])
corr_announce += "I, Q, U, V"
elif corr[0] == 'XX': # XX, XY, YX, YY
QSpart = np.array([0.5, 0, 0, -0.5])
USpart = np.array([0, 0.5, 0.5, 0])
corr_announce += "XX, XY, YX, YY"
if mode == 'tot':
if corr[0] == 'RR': # RR, RL, LR, LL
Spart = np.array([0.5, 0, 0, 0.5])
corr_announce += "RR, RL, LR, LL"
elif corr[0] == 'I': # I, Q, U, V
Spart = np.array([1., 0, 0, 0])
corr_announce += "I, Q, U, V"
elif corr[0] == 'XX': # XX, XY, YX, YY
Spart = np.array([0.5, 0, 0, 0.5])
corr_announce += "XX, XY, YX, YY"
print corr_announce
for spw in nspw:
nchan.append(meta.nchan(spw))
ms.selectinit(datadescid=spw)
temp = ms.getdata([viscol,"axis_info"],ifraxis=False)
data_temp = temp[viscol]
info_temp = temp["axis_info"]
s_temp = ms.getdata(["sigma"], ifraxis=False)['sigma']
flags = 1. - ms.getdata(["flag"])['flag']
if not(np.sum(flags[0])==np.sum(flags[1])==np.sum(flags[2])==np.sum(flags[3])):
m.warn('Warning: Different flags for different correlations/channels.'+
'Hard flag is applied: If any correlation is flagged, this gets'+
'extended to all.')
maximum = np.ones(np.shape(flags[0]))
for i in range(4):
if flags[i].sum() < maximum.sum():
maximum = flags[i]
flag = maximum
else:
flag = flags[0]
#Start reading data.
if mode == 'tot':
vis_temp = flag * (Spart[0] * data_temp[0] + Spart[1] * data_temp[1] + Spart[2] *\
data_temp[2] + Spart[3] * data_temp[3])
sigma_temp = flag * (Spart[0] * s_temp[0] + Spart[1] * s_temp[1] + Spart[2] * s_temp[2] +\
Spart[3] * s_temp[3])
vis.append(vis_temp)
sigma.append(sigma_temp)
nvis.append(len(vis_temp[0]))
if mode == 'pol':
Qvis_temp = flag * (QSpart[0] * data_temp[0] + QSpart[1] * data_temp[1] + QSpart[2] *\
data_temp[2] + QSpart[3] * data_temp[3])
Qsigma_temp = flag * (QSpart[0] * s_temp[0] + QSpart[1] * s_temp[1] + QSpart[2] * s_temp[2] +\
QSpart[3] * s_temp[3])
Qvis.append(Qvis_temp)
Qsigma.append(Qsigma_temp)
Uvis_temp = flag * (USpart[0] * data_temp[0] + USpart[1] * data_temp[1] + USpart[2] *\
data_temp[2] + USpart[3] * data_temp[3])
Usigma_temp = flag * (USpart[0] * s_temp[0] + USpart[1] * s_temp[1] + USpart[2] * s_temp[2] +\
USpart[3] * s_temp[3])
Uvis.append(Uvis_temp)
Usigma.append(Usigma_temp)
nvis.append(len(Uvis_temp[0]))
#uvflat give all uv coordinates for the chosen spw in m
uflat = ms.getdata(['u'])['u']
vflat = ms.getdata(['v'])['v']
freqtemp = info_temp["freq_axis"]["chan_freq"]
freq.append(freqtemp)
lamb = C / freqtemp
if mode == 'pol':
lambs2.append(lamb**2 / PI)
#calculates uv coordinates per channel in #lambda
utemp = np.array([uflat/k for k in lamb])
vtemp = np.array([vflat/k for k in lamb])
#Reads the uv coordates into lists. Delete functions take care of flags.
u.append(utemp)
v.append(vtemp)
try:
summary = ms.summary()
except:
print "Warning: Could not create a summary"
summary = None
ms.close()
if mode =='tot':
return vis, sigma, u, v, freq, nchan, nspw, nvis, summary
if mode == 'pol':
return Qvis, Qsigma, Uvis, Usigma, freq, lamb, u, v, nchan, nspw, nvis
