def _figure_of_merit(t, raw, resids, cc):
"""
Figure of merit for deconvolution that combines the positivity of the
residuals, the skewness of the clean components, the RMS of the
residuals, and the number of noise-like points
From:
Bhat, N., Cordes, J., & Chatterjee, S. 2003, ApJ, 584, 782.
"""
# Use robust methods to estimate the off-pulse mean and standard deviation
m = robust.mean(raw)
s = robust.std(raw)
# Find the fraction of noise-like points in the residuals
n = len(numpy.where(numpy.abs(resids - m) <= 3 * s)[0])
n = float(n) / raw.size
# Compute the positivity and skewness
f = _positivity(t, raw, resids, cc)
g = _skewness(t, raw, resids, cc)
# Get the standard deviation of all of the residuals relative to the estimated
# off-pulse value
r = resids.std()
r /= s
# The figure-of-metric
return (f + g) / 2.0 + r - n
def _positivity(t, raw, resids, cc):
"""
Compute the positivity of the residual profile following Equation 10 of
Bhat, N., Cordes, J., & Chatterjee, S. 2003, ApJ,
584, 782.
"""
# Weight factor
m = 1.0
# Tuning factor to find when the CLEANed profile goes negative
x = 3. / 2.
# Get the off-pulsar standard deviation
sigma = robust.std(raw)
temp = -resids - x * sigma
f = (resids**2 * numpy.where(temp >= 0, 1, 0)).mean()
f *= m / sigma**2
return f
def test_std(self):
"""Test the outlier-resistant standard deviation function."""
self.assertAlmostEqual(robust.std(self.a), 0.929675, 6)
self.assertAlmostEqual(robust.std(self.a, zero=True), 0.960737, 6)
b = self.a.reshape(2, -1)
self.assertAlmostEqual(robust.std(b, axis=1)[0], 0.923335, 6)
self.assertAlmostEqual(robust.std(b, axis=1)[1], 1.11836, 5)
b = numpy.ma.array(self.a,
mask=numpy.zeros(self.a.size, dtype=numpy.bool))
b.mask[:b.size // 2] = False
b.mask[b.size // 2:] = True
self.assertAlmostEqual(robust.std(b), 0.923335, 6)
b.mask[:b.size // 2] = True
b.mask[b.size // 2:] = False
self.assertAlmostEqual(robust.std(b, zero=True), 1.10655, 5)
def main(args):
t0=time.time()
#nChunks = 10000#10000, the size of a file.
#nFramesAvg = 1*4#200, 50 frames per pol, the subintergration time.
nChunks = 3000 #10000 #the size of a file.
nFramesAvg = 4*16 #the intergration time.
fcl = 6000+7000
fch = fcl + 3343
for offset_i in range(0, 1409):# one offset = nChunks*nFramesAvg skiped
#for offset_i in range(1500*2, 1500*3):# one offset = nChunks*nFramesAvg skiped
#for offset_i in range(1500*4, 1500*5):# one offset = nChunks*nFramesAvg skiped
offset = nChunks*nFramesAvg*offset_i
# Parse command line options
config = parseOptions(args)
# Length of the FFT
#LFFT = config['LFFT']
LFFT = 4096*16
# Build the DRX file
try:
#drxFile = drsu.getFileByName(config['args'][0], config['args'][1])
fh = open(config['args'][0], "rb")
nFramesFile = os.path.getsize(config['args'][0]) / drx.FrameSize
except:
print config['args']
sys.exit(1)
#drxFile.open()
#nFramesFile = drxFile.size / drx.FrameSize
while True:
try:
junkFrame = drx.readFrame(fh)
try:
srate = junkFrame.getSampleRate()
#t0 = junkFrame.getTime()
break
except ZeroDivisionError:
pass
except errors.syncError:
fh.seek(-drx.FrameSize+1, 1)
fh.seek(-drx.FrameSize, 1)
beam,tune,pol = junkFrame.parseID()
beams = drx.getBeamCount(fh)
tunepols = drx.getFramesPerObs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
config['offset'] = offset/srate/beampols*4096
if offset != 0:
fh.seek(offset*drx.FrameSize, 1)
# Make sure that the file chunk size contains is an integer multiple
# of the FFT length so that no data gets dropped. This needs to
# take into account the number of beampols in the data, the FFT length,
# and the number of samples per frame.
maxFrames = int(1.0*config['maxFrames']/beampols*4096/float(LFFT))*LFFT/4096*beampols
# Number of frames to integrate over
# nFramesAvg = int(config['average'] * srate / 4096 * beampols)
# nFramesAvg = int(1.0 * nFramesAvg / beampols*4096/float(LFFT))*LFFT/4096*beampols
config['average'] = 1.0 * nFramesAvg / beampols * 4096 / srate
maxFrames = nFramesAvg
# Number of remaining chunks (and the correction to the number of
# frames to read in).
# nChunks = int(round(config['duration'] / config['average']))
config['duration']=nChunks*config['average']
if nChunks == 0:
nChunks = 1
nFrames = nFramesAvg*nChunks
# Date & Central Frequnecy
beginDate = ephem.Date(unix_to_utcjd(junkFrame.getTime()) - DJD_OFFSET)
centralFreq1 = 0.0
centralFreq2 = 0.0
for i in xrange(4):
junkFrame = drx.readFrame(fh)
b,t,p = junkFrame.parseID()
if p == 0 and t == 1:
try:
centralFreq1 = junkFrame.getCentralFreq()
except AttributeError:
from lsl.common.dp import fS
centralFreq1 = fS * ((junkFrame.data.flags>>32) & (2**32-1)) / 2**32
elif p == 0 and t == 2:
try:
centralFreq2 = junkFrame.getCentralFreq()
except AttributeError:
from lsl.common.dp import fS
centralFreq2 = fS * ((junkFrame.data.flags>>32) & (2**32-1)) / 2**32
else:
pass
fh.seek(-4*drx.FrameSize, 1)
config['freq1'] = centralFreq1
config['freq2'] = centralFreq2
# File summary
print "Filename: %s" % config['args']
print "Date of First Frame: %s" % str(beginDate)
print "Beams: %i" % beams
print "Tune/Pols: %i %i %i %i" % tunepols
print "Sample Rate: %i Hz" % srate
print "Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" % (centralFreq1, centralFreq2)
print "Frames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate)
print "---"
print "Offset: %.3f s (%i frames)" % (config['offset'], offset)
print "Integration: %.3f s (%i frames; %i frames per beam/tune/pol)" % (config['average'], nFramesAvg, nFramesAvg / beampols)
print "Duration: %.3f s (%i frames; %i frames per beam/tune/pol)" % (config['average']*nChunks, nFrames, nFrames / beampols)
print "Chunks: %i" % nChunks
#sys.exit()
# Sanity check
if nFrames > (nFramesFile - offset):
raise RuntimeError("Requested integration time+offset is greater than file length")
# Estimate clip level (if needed)
if config['estimate']:
filePos = fh.tell()
# Read in the first 100 frames for each tuning/polarization
count = {0:0, 1:0, 2:0, 3:0}
data = numpy.zeros((4, 4096*100), dtype=numpy.csingle)
for i in xrange(4*100):
try:
cFrame = drx.readFrame(fh, Verbose=False)
except errors.eofError:
break
except errors.syncError:
continue
beam,tune,pol = cFrame.parseID()
aStand = 2*(tune-1) + pol
data[aStand, count[aStand]*4096:(count[aStand]+1)*4096] = cFrame.data.iq
count[aStand] += 1
# Go back to where we started
fh.seek(filePos)
# Compute the robust mean and standard deviation for I and Q for each
# tuning/polarization
meanI = []
meanQ = []
stdsI = []
stdsQ = []
#for i in xrange(4):
for i in xrange(2):
meanI.append( robust.mean(data[i,:].real) )
meanQ.append( robust.mean(data[i,:].imag) )
stdsI.append( robust.std(data[i,:].real) )
stdsQ.append( robust.std(data[i,:].imag) )
# Come up with the clip levels based on 4 sigma
clip1 = (meanI[0] + meanI[1] + meanQ[0] + meanQ[1]) / 4.0
#clip2 = (meanI[2] + meanI[3] + meanQ[2] + meanQ[3]) / 4.0
clip2 = 0
clip1 += 5*(stdsI[0] + stdsI[1] + stdsQ[0] + stdsQ[1]) / 4.0
#clip2 += 5*(stdsI[2] + stdsI[3] + stdsQ[2] + stdsQ[3]) / 4.0
clip2 += 0
clip1 = int(round(clip1))
clip2 = int(round(clip2))
# Report again
else:
clip1 = config['clip']
clip2 = config['clip']
# Master loop over all of the file chunks
#masterSpectra = numpy.zeros((nChunks, 4, LFFT-1))
masterSpectra = numpy.zeros((nChunks, 4, fch-fcl))
masterTimes = numpy.zeros(nChunks)
for i in xrange(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i*maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
if framesRemaining%(nFrames/10)==0:
print "Working on chunk %i, %i frames remaining" % (i, framesRemaining)
count = {0:0, 1:0, 2:0, 3:0}
data = numpy.zeros((4,framesWork*4096/beampols), dtype=numpy.csingle)
# If there are fewer frames than we need to fill an FFT, skip this chunk
if data.shape[1] < LFFT:
break
# Inner loop that actually reads the frames into the data array
if framesRemaining%(nFrames/10)==0:
print "Working on %.1f ms of data" % ((framesWork*4096/beampols/srate)*1000.0)
for j in xrange(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.readFrame(fh, Verbose=False)
except errors.eofError:
print "EOF Error"
break
except errors.syncError:
print "Sync Error"
continue
beam,tune,pol = cFrame.parseID()
aStand = 2*(tune-1) + pol
if j is 0:
cTime = cFrame.getTime()
data[aStand, count[aStand]*4096:(count[aStand]+1)*4096] = cFrame.data.iq
count[aStand] += 1
# Calculate the spectra for this block of data and then weight the results by
# the total number of frames read. This is needed to keep the averages correct.
#freq, tempSpec1 = fxc.SpecMaster(data[:2,:], LFFT=LFFT, window=config['window'], verbose=config['verbose'], SampleRate=srate, ClipLevel=clip1)
freq, tempSpec1 = fxc.SpecMaster(data[:2,:], LFFT=LFFT, window=config['window'], verbose=config['verbose'], SampleRate=srate)
#freq, tempSpec2 = fxc.SpecMaster(data[2:,:], LFFT=LFFT, window=config['window'], verbose=config['verbose'], SampleRate=srate, ClipLevel=clip2)
freq, tempSpec2 = fxc.SpecMaster(data[2:,:], LFFT=LFFT, window=config['window'], verbose=config['verbose'], SampleRate=srate)
# Save the results to the various master arrays
masterTimes[i] = cTime
masterSpectra[i,0,:] = tempSpec1[0,fcl:fch]
masterSpectra[i,1,:] = tempSpec1[1,fcl:fch]
masterSpectra[i,2,:] = tempSpec2[0,fcl:fch]
masterSpectra[i,3,:] = tempSpec2[1,fcl:fch]
# We don't really need the data array anymore, so delete it
del(data)
#drxFile.close()
# Now that we have read through all of the chunks, perform the final averaging by
# dividing by all of the chunks
outname = "%s_%i_fft_offset_%.9i_frames" % (config['args'][0], beam,offset)
# numpy.savez(outname, freq=freq, freq1=freq+config['freq1'], freq2=freq+config['freq2'], times=masterTimes, spec=masterSpectra, tInt=(maxFrames*4096/beampols/srate), srate=srate, standMapper=[4*(beam-1) + i for i in xrange(masterSpectra.shape[1])])
#print 'fInt = ',(freq+config['freq1'])[1]-(freq+config['freq1'])[0]
#print 'tInt = ',maxFrames*4096/beampols/srate
masterSpectra[:,0,:]=masterSpectra[:,0:2,:].mean(1)
masterSpectra[:,1,:]=masterSpectra[:,2:4,:].mean(1)
numpy.save(outname[-46:], masterSpectra[:,1,:])
#numpy.save(outname[-46:], masterSpectra[:,0:2,:])
#numpy.save(outname[-46:], masterSpectra)
# spec = numpy.squeeze( (masterWeight*masterSpectra).sum(axis=0) / masterWeight.sum(axis=0) )
# offset_i = offset_i + 1
print time.time()-t0
def main(args):
fh = open(args.filename, "rb")
nFramesFile = os.path.getsize(args.filename) // drx.FRAME_SIZE
while True:
junkFrame = drx.read_frame(fh)
try:
srate = junkFrame.sample_rate
break
except ZeroDivisionError:
pass
fh.seek(-drx.FRAME_SIZE, 1)
beams = drx.get_beam_count(fh)
tunepols = drx.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(round(args.offset * srate / 4096 * beampols))
offset = int(1.0 * offset / beampols) * beampols
args.offset = 1.0 * offset / beampols * 4096 / srate
fh.seek(offset * drx.FRAME_SIZE)
# Make sure that the file chunk size contains is an intger multiple
# of the beampols.
maxFrames = int((19144 * 4) / beampols) * beampols
# Setup the statistics data set
if args.stats:
if args.plot_range < 0.1:
args.plot_range = 0.5
# Number of frames to integrate over
nFrames = int(args.plot_range * srate / 4096 * beampols)
nFrames = int(1.0 * nFrames / beampols) * beampols
args.plot_range = 1.0 * nFrames / beampols * 4096 / srate
# Number of remaining chunks
nChunks = int(math.ceil(1.0 * (nFrames) / maxFrames))
# File summary
print("Filename: %s" % args.filename)
print("Beams: %i" % beams)
print("Tune/Pols: %i %i %i %i" % tunepols)
print("Sample Rate: %i Hz" % srate)
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.offset, offset))
print("Plot time: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.plot_range, nFrames, nFrames // beampols))
print("Chunks: %i" % nChunks)
# Sanity check
if offset > nFramesFile:
raise RuntimeError("Requested offset is greater than file length")
if nFrames > (nFramesFile - offset):
raise RuntimeError(
"Requested integration time+offset is greater than file length")
# Align the file handle so that the first frame read in the
# main analysis loop is from tuning 1, polarization 0
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
while 2 * (t - 1) + p != 0:
junkFrame = drx.read_frame(fh)
b, t, p = junkFrame.id
fh.seek(-drx.FRAME_SIZE, 1)
# Master loop over all of the file chuncks
standMapper = []
for i in xrange(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
print("Working on chunk %i, %i frames remaining" %
(i, framesRemaining))
count = {0: 0, 1: 0, 2: 0, 3: 0}
data = numpy.zeros((beampols, framesWork * 4096 // beampols),
dtype=numpy.csingle)
# Inner loop that actually reads the frames into the data array
print("Working on %.1f ms of data" %
((framesWork * 4096 / beampols / srate) * 1000.0))
t0 = time.time()
for j in xrange(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = drx.read_frame(fh, verbose=False)
except errors.EOFError:
break
except errors.SyncError:
#print("WARNING: Mark 5C sync error on frame #%i" % (int(fh.tell())/drx.FRAME_SIZE-1))
continue
beam, tune, pol = cFrame.id
aStand = 2 * (tune - 1) + pol
data[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = numpy.abs(cFrame.payload.data)**2
# Update the counters so that we can average properly later on
count[aStand] += 1
# Statistics
print("Running robust statistics")
means = [robust.mean(data[i, :]) for i in xrange(data.shape[0])]
stds = [robust.std(data[i, :]) for i in xrange(data.shape[0])]
if args.stats:
## Report statistics
print("Mean: %s" % ' '.join(["%.3f" % m for m in means]))
print("StdD: %s" % ' '.join(["%.3f" % s for s in stds]))
print("Levels:")
## Count'em up
j = 0
counts = [
1,
] * data.shape[0]
while (means[i] + j * stds[i] <= 98) and max(counts) != 0:
counts = [
len(
numpy.where(
numpy.abs(data[i, :] - means[i]) >= j *
stds[i])[0]) for i in xrange(data.shape[0])
]
print(" %2isigma (%5.1f%%): %s" %
(j, 100.0 * (1 - erf(j / numpy.sqrt(2))), ' '.join([
"%7i (%5.1f%%)" % (c, 100.0 * c / data.shape[1])
for c in counts
])))
j += 1
## Why j-2? Well, j is 1 more than the last iteration. So, that last iteration
## is j-1, which is always filled with 0s by construction. So, the last crazy
## bin is j-2.
jP = j - 2
if jP > 20:
counts = [
len(
numpy.where(
numpy.abs(data[i, :] - means[i]) >= jP *
stds[i])[0]) for i in xrange(data.shape[0])
]
for i in xrange(data.shape[0]):
if counts[i] > 0:
break
if counts[i] == 1:
print(
" -> Clip-o-rama likely occuring with %i %i-sigma detection on tuning %i, pol %i"
% (counts[i], jP, i // 2 + 1, i % 2))
else:
print(
" -> Clip-o-rama likely occuring with %i %i-sigma detections on tuning %i, pol %i"
% (counts[i], jP, i // 2 + 1, i % 2))
else:
outfile = os.path.splitext(args.filename)[0]
outfile = '%s.txt' % outfile
fh = open(outfile, 'w')
# Plot possible clip-o-rama and flag it
print("Computing power derivatives w.r.t. time")
deriv = numpy.zeros_like(data)
for i in xrange(data.shape[0]):
deriv[i, :] = numpy.roll(data[i, :], -1) - data[i, :]
# The plots: This is setup for the current configuration of 20 beampols
print("Plotting")
fig = plt.figure()
figsX = int(round(math.sqrt(beampols)))
figsY = beampols // figsX
for i in xrange(data.shape[0]):
ax = fig.add_subplot(figsX, figsY, i + 1)
ax.plot(args.offset + numpy.arange(0, data.shape[1]) / srate,
data[i, :])
## Mark areas of crazy derivatives
bad = numpy.where(
deriv[i, :] > 20 * stds[i] * numpy.sqrt(2))[0]
for j in bad:
fh.write(
"Clip-o-rama on tuning %i, pol. %i at %.6f seconds\n" %
(i // 2 + 1, i % 2, args.offset + j / srate))
print("Clip-o-rama on tuning %i, pol. %i at %.6f seconds" %
(i // 2 + 1, i % 2, args.offset + j / srate))
ax.vlines(args.offset + j / srate,
-10,
100,
linestyle='--',
color='red',
linewidth=2.0)
## Mark areas of crazy power levels
bad = numpy.where(data[i, :] == 98)[0]
for j in bad:
fh.write(
"Saturation on tuning %i, pol. %i at %.6f seconds\n" %
(i // 2 + 1, i % 2, args.offset + j / srate))
print("Saturation on tuning %i, pol. %i at %.6f seconds" %
(i // 2 + 1, i % 2, args.offset + j / srate))
ax.vlines(args.offset + j / srate,
-10,
100,
linestyle='-.',
color='red')
ax.set_ylim([-10, 100])
ax.set_title('Beam %i, Tune. %i, Pol. %i' %
(beam, i // 2 + 1, i % 2))
ax.set_xlabel('Time [seconds]')
ax.set_ylabel('I$^2$ + Q$^2$')
fh.close()
if args.do_plot:
plt.show()
def main(args):
filenames = args.filenames
filenames.sort()
fig1 = plt.figure()
ax1 = fig1.add_subplot(2, 1, 1)
ax2 = fig1.add_subplot(2, 1, 2)
fig2 = plt.figure()
ax3 = fig2.add_subplot(2, 1, 1)
ax4 = fig2.add_subplot(2, 1, 2)
times = []
vis1 = []
vis2 = []
for filename in filenames:
dataDict = numpy.load(filename)
tStart = datetime.utcfromtimestamp(dataDict['tStart'])
tInt = dataDict['tInt']
try:
srate = dataDict['srate']
except KeyError:
srate = 19.6e6
freq1 = dataDict['freq1']
vis1.append(dataDict['vis1'][1, :])
freq2 = dataDict['freq2']
vis2.append(dataDict['vis2'][1, :])
times.append(tStart)
dataDict.close()
N = srate * tInt / (len(freq1) + 1)
print("Got %i files from %s to %s (%s)" %
(len(filenames), times[0].strftime("%Y/%m/%d %H:%M:%S"),
times[-1].strftime("%Y/%m/%d %H:%M:%S"), (times[-1] - times[0])))
iTimes = []
for i in xrange(1, len(times)):
dt = times[i] - times[i - 1]
iTimes.append(dt.days * 24 * 3600 + dt.seconds + dt.microseconds / 1e6)
iTimes = numpy.array(iTimes)
print(" -> Interval: %.3f +/- %.3f seconds (%.3f to %.3f seconds)" %
(iTimes.mean(), iTimes.std(), iTimes.min(), iTimes.max()))
print("Number of frequency channels: %i (~%.1f Hz/channel)" %
(len(freq1) + 1, freq1[1] - freq1[0]))
dTimes = []
for t in times:
dTimes.append((t - times[0]).seconds)
freq1 /= 1e6
freq2 /= 1e6
vis1 = numpy.array(vis1)
vis1 = numpy.ma.array(vis1, mask=~numpy.isfinite(vis1))
vis2 = numpy.array(vis2)
vis2 = numpy.ma.array(vis2, mask=~numpy.isfinite(vis2))
sk = numpy.zeros(freq1.size)
for i in xrange(vis1.shape[1]):
sk[i] = spectralKurtosis(numpy.abs(vis1[:, i])**2, N=N)
skM = robust.mean(sk)
skS = robust.std(sk)
bad = numpy.where(numpy.abs(sk - skM) > 4 * skS)[0]
#vis1.mask[:,bad] = True
sk = numpy.zeros_like(freq2)
for i in xrange(vis2.shape[1]):
sk[i] = spectralKurtosis(numpy.abs(vis2[:, i])**2, N=N)
skM = robust.mean(sk)
skS = robust.std(sk)
bad = numpy.where(numpy.abs(sk - skM) > 4 * skS)[0]
#vis2.mask[:,bad] = True
data = 1.0 * numpy.abs(vis1)
data = data.ravel()
data.sort()
vmin1 = data[int(round(0.15 * len(data)))]
vmax1 = data[int(round(0.85 * len(data)))]
print('Plot range for tuning 1:', vmin1, vmax1)
data = 1.0 * numpy.abs(vis2)
data = data.ravel()
data.sort()
vmin2 = data[int(round(0.15 * len(data)))]
vmax2 = data[int(round(0.85 * len(data)))]
print('Plot range for tuning 2:', vmin2, vmax2)
ax1.imshow(numpy.abs(vis1),
extent=(freq1[0], freq1[-1], dTimes[0], dTimes[-1]),
origin='lower',
vmin=vmin1,
vmax=vmax1)
ax2.imshow(numpy.abs(vis2),
extent=(freq2[0], freq2[-1], dTimes[0], dTimes[-1]),
origin='lower',
vmin=vmin2,
vmax=vmax2)
ax1.axis('auto')
ax2.axis('auto')
fig1.suptitle("%s to %s UTC" % (times[0].strftime("%Y/%m/%d %H:%M"),
times[-1].strftime("%Y/%m/%d %H:%M")))
ax1.set_xlabel('Frequency [MHz]')
ax2.set_xlabel('Frequency [MHz]')
ax1.set_ylabel('Elapsed Time [s]')
ax2.set_ylabel('Elapsed Time [s]')
ax3.plot(freq1, numpy.abs(vis1).mean(axis=0))
ax4.plot(freq2, numpy.abs(vis2).mean(axis=0))
fig2.suptitle("%s to %s UTC" % (times[0].strftime("%Y/%m/%d %H:%M"),
times[-1].strftime("%Y/%m/%d %H:%M")))
ax3.set_xlabel('Frequency [MHz]')
ax4.set_xlabel('Frequency [MHz]')
ax3.set_ylabel('Mean Vis. Amp. [arb.]')
ax4.set_ylabel('Mean Vis. Amp. [arb.]')
plt.show()
def main(args):
# Length of the FFT and the window to use
LFFT = args.fft_length
if args.bartlett:
window = numpy.bartlett
elif args.blackman:
window = numpy.blackman
elif args.hanning:
window = numpy.hanning
else:
window = fxc.null_window
args.window = window
fh = open(args.filename, "rb")
usrp.FRAME_SIZE = usrp.get_frame_size(fh)
nFramesFile = os.path.getsize(args.filename) // usrp.FRAME_SIZE
junkFrame = usrp.read_frame(fh)
srate = junkFrame.sample_rate
t0i, t0f = junkFrame.time
fh.seek(-usrp.FRAME_SIZE, 1)
beams = usrp.get_beam_count(fh)
tunepols = usrp.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(args.skip * srate / junkFrame.payload.data.size * beampols)
offset = int(1.0 * offset / beampols) * beampols
fh.seek(offset * usrp.FRAME_SIZE, 1)
# Iterate on the offsets until we reach the right point in the file. This
# is needed to deal with files that start with only one tuning and/or a
# different sample rate.
while True:
## Figure out where in the file we are and what the current tuning/sample
## rate is
junkFrame = usrp.read_frame(fh)
srate = junkFrame.sample_rate
t1i, t1f = junkFrame.time
tunepols = usrp.get_frames_per_obs(fh)
tunepol = tunepols[0] + tunepols[1] + tunepols[2] + tunepols[3]
beampols = tunepol
fh.seek(-usrp.FRAME_SIZE, 1)
## See how far off the current frame is from the target
tDiff = t1i - (t0i + args.skip) + t1f - t0f
## Half that to come up with a new seek parameter
tCorr = -tDiff / 2.0
cOffset = int(tCorr * srate / junkFrame.payload.data.size * beampols)
cOffset = int(1.0 * cOffset / beampols) * beampols
offset += cOffset
## If the offset is zero, we are done. Otherwise, apply the offset
## and check the location in the file again/
if cOffset is 0:
break
fh.seek(cOffset * usrp.FRAME_SIZE, 1)
# Update the offset actually used
args.skip = t1i - t0i + t1f - t0f
offset = int(
round(args.skip * srate / junkFrame.payload.data.size * beampols))
offset = int(1.0 * offset / beampols) * beampols
# Make sure that the file chunk size contains is an integer multiple
# of the FFT length so that no data gets dropped. This needs to
# take into account the number of beampols in the data, the FFT length,
# and the number of samples per frame.
maxFrames = int(
1.0 * 28000 / beampols * junkFrame.payload.data.size /
float(LFFT)) * LFFT / junkFrame.payload.data.size * beampols
# Number of frames to integrate over
nFramesAvg = int(args.average * srate / junkFrame.payload.data.size *
beampols)
nFramesAvg = int(
1.0 * nFramesAvg / beampols * junkFrame.payload.data.size /
float(LFFT)) * LFFT / junkFrame.payload.data.size * beampols
args.average = 1.0 * nFramesAvg / beampols * junkFrame.payload.data.size / srate
maxFrames = nFramesAvg
# Number of remaining chunks (and the correction to the number of
# frames to read in).
nChunks = int(round(args.duration / args.average))
if nChunks == 0:
nChunks = 1
nFrames = nFramesAvg * nChunks
# Date & Central Frequnecy
beginDate = junkFrame.time.datetime
central_freq1 = 0.0
central_freq2 = 0.0
for i in range(4):
junkFrame = usrp.read_frame(fh)
b, t, p = junkFrame.id
if p == 0 and t == 1:
central_freq1 = junkFrame.central_freq
elif p == 0 and t == 2:
central_freq2 = junkFrame.central_freq
else:
pass
fh.seek(-4 * usrp.FRAME_SIZE, 1)
central_freq1 = central_freq1
central_freq2 = central_freq2
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Beams: %i" % beams)
print("Tune/Pols: %i %i %i %i" % tunepols)
print("Sample Rate: %i Hz" % srate)
print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" %
(central_freq1, central_freq2))
print("Frames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / beampols *
junkFrame.payload.data.size / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.skip, offset))
print("Integration: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.average, nFramesAvg, nFramesAvg / beampols))
print("Duration: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.average * nChunks, nFrames, nFrames / beampols))
print("Chunks: %i" % nChunks)
# Sanity check
if nFrames > (nFramesFile - offset):
raise RuntimeError(
"Requested integration time+offset is greater than file length")
# Estimate clip level (if needed)
if args.estimate_clip_level:
filePos = fh.tell()
# Read in the first 100 frames for each tuning/polarization
count = {0: 0, 1: 0, 2: 0, 3: 0}
data = numpy.zeros((4, junkFrame.payload.data.size * 100),
dtype=numpy.csingle)
for i in range(4 * 100):
try:
cFrame = usrp.read_frame(fh, Verbose=False)
except errors.EOFError:
break
except errors.SyncError:
continue
beam, tune, pol = cFrame.id
aStand = 2 * (tune - 1) + pol
data[aStand, count[aStand] *
junkFrame.payload.data.size:(count[aStand] + 1) *
junkFrame.payload.data.size] = cFrame.payload.data
count[aStand] += 1
# Go back to where we started
fh.seek(filePos)
# Correct the DC bias
for j in range(data.shape[0]):
data[j, :] -= data[j, :].mean()
# Compute the robust mean and standard deviation for I and Q for each
# tuning/polarization
meanI = []
meanQ = []
stdsI = []
stdsQ = []
for i in range(4):
meanI.append(robust.mean(data[i, :].real))
meanQ.append(robust.mean(data[i, :].imag))
stdsI.append(robust.std(data[i, :].real))
stdsQ.append(robust.std(data[i, :].imag))
# Report
print("Statistics:")
for i in range(4):
print(" Mean %i: %.3f + %.3f j" % (i + 1, meanI[i], meanQ[i]))
print(" Std %i: %.3f + %.3f j" % (i + 1, stdsI[i], stdsQ[i]))
# Come up with the clip levels based on 4 sigma
clip1 = (meanI[0] + meanI[1] + meanQ[0] + meanQ[1]) / 4.0
clip2 = (meanI[2] + meanI[3] + meanQ[2] + meanQ[3]) / 4.0
clip1 += 5 * (stdsI[0] + stdsI[1] + stdsQ[0] + stdsQ[1]) / 4.0
clip2 += 5 * (stdsI[2] + stdsI[3] + stdsQ[2] + stdsQ[3]) / 4.0
clip1 = int(round(clip1))
clip2 = int(round(clip2))
# Report again
print("Clip Levels:")
print(" Tuning 1: %i" % clip1)
print(" Tuning 2: %i" % clip2)
else:
clip1 = args.clip_level
clip2 = args.clip_level
# Master loop over all of the file chunks
masterWeight = numpy.zeros((nChunks, 4, LFFT))
masterSpectra = numpy.zeros((nChunks, 4, LFFT))
masterTimes = numpy.zeros(nChunks)
for i in range(nChunks):
# Find out how many frames remain in the file. If this number is larger
# than the maximum of frames we can work with at a time (maxFrames),
# only deal with that chunk
framesRemaining = nFrames - i * maxFrames
if framesRemaining > maxFrames:
framesWork = maxFrames
else:
framesWork = framesRemaining
print("Working on chunk %i, %i frames remaining" %
(i, framesRemaining))
count = {0: 0, 1: 0, 2: 0, 3: 0}
data = numpy.zeros(
(4, framesWork * junkFrame.payload.data.size // beampols),
dtype=numpy.csingle)
# If there are fewer frames than we need to fill an FFT, skip this chunk
if data.shape[1] < LFFT:
break
# Inner loop that actually reads the frames into the data array
print("Working on %.1f ms of data" %
((framesWork * junkFrame.payload.data.size / beampols / srate) *
1000.0))
for j in range(framesWork):
# Read in the next frame and anticipate any problems that could occur
try:
cFrame = usrp.read_frame(fh, Verbose=False)
except errors.EOFError:
break
except errors.SyncError:
continue
beam, tune, pol = cFrame.id
aStand = 2 * (tune - 1) + pol
if j is 0:
cTime = cFrame.time
data[aStand,
count[aStand] * cFrame.payload.data.size:(count[aStand] + 1) *
cFrame.payload.data.size] = cFrame.payload.data
count[aStand] += 1
# Correct the DC bias
for j in range(data.shape[0]):
data[j, :] -= data[j, :].mean()
# Calculate the spectra for this block of data and then weight the results by
# the total number of frames read. This is needed to keep the averages correct.
freq, tempSpec1 = fxc.SpecMaster(data[:2, :],
LFFT=LFFT,
window=args.window,
verbose=True,
sample_rate=srate,
clip_level=clip1)
freq, tempSpec2 = fxc.SpecMaster(data[2:, :],
LFFT=LFFT,
window=args.window,
verbose=True,
sample_rate=srate,
clip_level=clip2)
# Save the results to the various master arrays
masterTimes[i] = cTime
masterSpectra[i, 0, :] = tempSpec1[0, :]
masterSpectra[i, 1, :] = tempSpec1[1, :]
masterSpectra[i, 2, :] = tempSpec2[0, :]
masterSpectra[i, 3, :] = tempSpec2[1, :]
masterWeight[i, 0, :] = int(count[0] * cFrame.payload.data.size / LFFT)
masterWeight[i, 1, :] = int(count[1] * cFrame.payload.data.size / LFFT)
masterWeight[i, 2, :] = int(count[2] * cFrame.payload.data.size / LFFT)
masterWeight[i, 3, :] = int(count[3] * cFrame.payload.data.size / LFFT)
# We don't really need the data array anymore, so delete it
del (data)
# Now that we have read through all of the chunks, perform the final averaging by
# dividing by all of the chunks
outname = os.path.split(args.filename)[1]
outname = os.path.splitext(outname)[0]
outname = '%s-waterfall.npz' % outname
numpy.savez(outname,
freq=freq,
freq1=freq + central_freq1,
freq2=freq + central_freq2,
times=masterTimes,
spec=masterSpectra,
tInt=(maxFrames * cFrame.payload.data.size / beampols / srate),
srate=srate,
standMapper=[
4 * (beam - 1) + i for i in range(masterSpectra.shape[1])
])
spec = numpy.squeeze(
(masterWeight * masterSpectra).sum(axis=0) / masterWeight.sum(axis=0))
# The plots: This is setup for the current configuration of 20 beampols
fig = plt.figure()
figsX = int(round(math.sqrt(1)))
figsY = 1 // figsX
# Put the frequencies in the best units possible
freq, units = bestFreqUnits(freq)
for i in range(1):
ax = fig.add_subplot(figsX, figsY, i + 1)
currSpectra = numpy.squeeze(numpy.log10(masterSpectra[:, i, :]) * 10.0)
currSpectra = numpy.where(numpy.isfinite(currSpectra), currSpectra,
-10)
ax.imshow(currSpectra,
interpolation='nearest',
extent=(freq.min(), freq.max(), args.skip + 0,
args.skip + args.average * nChunks),
origin='lower')
print(currSpectra.min(), currSpectra.max())
ax.axis('auto')
ax.set_title('Beam %i, Tune. %i, Pol. %i' % (beam, i // 2 + 1, i % 2))
ax.set_xlabel('Frequency Offset [%s]' % units)
ax.set_ylabel('Time [s]')
ax.set_xlim([freq.min(), freq.max()])
print("RBW: %.4f %s" % ((freq[1] - freq[0]), units))
if True:
plt.show()
# Save spectra image if requested
if args.output is not None:
fig.savefig(args.output)
def main(args):
# Parse the command line
## Baseline list
if args.baseline is not None:
## Fill the baseline list with the conjugates, if needed
newBaselines = []
for pair in args.baseline:
newBaselines.append((pair[1], pair[0]))
args.baseline.extend(newBaselines)
## Search limits
args.delay_window = [float(v) for v in args.delay_window.split(',', 1)]
args.rate_window = [float(v) for v in args.rate_window.split(',', 1)]
## Filenames
filenames = args.filename
filenames.sort()
if args.limit != -1:
filenames = filenames[:args.limit]
nInt = len(filenames)
dataDict = numpy.load(filenames[0])
tInt = dataDict['tInt']
nBL, nchan = dataDict['vis1XX'].shape
freq = dataDict['freq1']
junk0, refSrc, junk1, junk2, junk3, junk4, antennas = read_correlator_configuration(
dataDict)
dataDict.close()
# Make sure the reference antenna is in there
if args.ref_ant is None:
args.ref_ant = antennas[0].stand.id
else:
found = False
for ant in antennas:
if ant.stand.id == args.ref_ant:
found = True
break
if not found:
raise RuntimeError("Cannot file reference antenna %i in the data" %
args.ref_ant)
bls = []
l = 0
cross = []
for i in xrange(0, len(antennas), 2):
ant1 = antennas[i].stand.id
for j in xrange(i, len(antennas), 2):
ant2 = antennas[j].stand.id
if ant1 != ant2:
bls.append((ant1, ant2))
cross.append(l)
l += 1
nBL = len(cross)
if args.decimate > 1:
if nchan % args.decimate != 0:
raise RuntimeError(
"Invalid freqeunce decimation factor: %i %% %i = %i" %
(nchan, args.decimate, nchan % args.decimate))
nchan /= args.decimate
freq.shape = (freq.size / args.decimate, args.decimate)
freq = freq.mean(axis=1)
times = numpy.zeros(nInt, dtype=numpy.float64)
visXX = numpy.zeros((nInt, nBL, nchan), dtype=numpy.complex64)
if not args.y_only:
visXY = numpy.zeros((nInt, nBL, nchan), dtype=numpy.complex64)
visYX = numpy.zeros((nInt, nBL, nchan), dtype=numpy.complex64)
visYY = numpy.zeros((nInt, nBL, nchan), dtype=numpy.complex64)
for i, filename in enumerate(filenames):
dataDict = numpy.load(filename)
tStart = dataDict['tStart']
cvisXX = dataDict['vis1XX'][cross, :]
cvisXY = dataDict['vis1XY'][cross, :]
cvisYX = dataDict['vis1YX'][cross, :]
cvisYY = dataDict['vis1YY'][cross, :]
if args.decimate > 1:
cvisXX.shape = (cvisXX.shape[0], cvisXX.shape[1] // args.decimate,
args.decimate)
cvisXX = cvisXX.mean(axis=2)
cvisXY.shape = (cvisXY.shape[0], cvisXY.shape[1] // args.decimate,
args.decimate)
cvisXY = cvisXY.mean(axis=2)
cvisYX.shape = (cvisYX.shape[0], cvisYX.shape[1] // args.decimate,
args.decimate)
cvisYX = cvisYX.mean(axis=2)
cvisYY.shape = (cvisYY.shape[0], cvisYY.shape[1] // args.decimate,
args.decimate)
cvisYY = cvisYY.mean(axis=2)
visXX[i, :, :] = cvisXX
if not args.y_only:
visXY[i, :, :] = cvisXY
visYX[i, :, :] = cvisYX
visYY[i, :, :] = cvisYY
times[i] = tStart
dataDict.close()
print("Got %i files from %s to %s (%.1f s)" %
(len(filenames), datetime.utcfromtimestamp(
times[0]).strftime("%Y/%m/%d %H:%M:%S"),
datetime.utcfromtimestamp(times[-1]).strftime("%Y/%m/%d %H:%M:%S"),
(times[-1] - times[0])))
iTimes = numpy.zeros(nInt - 1, dtype=times.dtype)
for i in xrange(1, len(times)):
iTimes[i - 1] = times[i] - times[i - 1]
print(
" -> Interval: %.3f +/- %.3f seconds (%.3f to %.3f seconds)" %
(robust.mean(iTimes), robust.std(iTimes), iTimes.min(), iTimes.max()))
iSize = int(round(args.interval / robust.mean(iTimes)))
print(" -> Chunk size is %i intervals (%.3f seconds)" %
(iSize, iSize * robust.mean(iTimes)))
iCount = times.size // iSize
print(" -> Working with %i chunks of data" % iCount)
print("Number of frequency channels: %i (~%.1f Hz/channel)" %
(len(freq), freq[1] - freq[0]))
dTimes = times - times[0]
ref_time = (int(times[0]) / 60) * 60
dMax = 1.0 / (freq[1] - freq[0]) / 4
dMax = int(dMax * 1e6) * 1e-6
if -dMax * 1e6 > args.delay_window[0]:
args.delay_window[0] = -dMax * 1e6
if dMax * 1e6 < args.delay_window[1]:
args.delay_window[1] = dMax * 1e6
rMax = 1.0 / robust.mean(iTimes) / 4
rMax = int(rMax * 1e2) * 1e-2
if -rMax * 1e3 > args.rate_window[0]:
args.rate_window[0] = -rMax * 1e3
if rMax * 1e3 < args.rate_window[1]:
args.rate_window[1] = rMax * 1e3
dres = 1.0
nDelays = int((args.delay_window[1] - args.delay_window[0]) / dres)
while nDelays < 50:
dres /= 10
nDelays = int((args.delay_window[1] - args.delay_window[0]) / dres)
while nDelays > 5000:
dres *= 10
nDelays = int((args.delay_window[1] - args.delay_window[0]) / dres)
nDelays += (nDelays + 1) % 2
rres = 10.0
nRates = int((args.rate_window[1] - args.rate_window[0]) / rres)
while nRates < 50:
rres /= 10
nRates = int((args.rate_window[1] - args.rate_window[0]) / rres)
while nRates > 5000:
rres *= 10
nRates = int((args.rate_window[1] - args.rate_window[0]) / rres)
nRates += (nRates + 1) % 2
print("Searching delays %.1f to %.1f us in steps of %.2f us" %
(args.delay_window[0], args.delay_window[1], dres))
print(" rates %.1f to %.1f mHz in steps of %.2f mHz" %
(args.rate_window[0], args.rate_window[1], rres))
print(" ")
delay = numpy.linspace(args.delay_window[0] * 1e-6,
args.delay_window[1] * 1e-6, nDelays) # s
drate = numpy.linspace(args.rate_window[0] * 1e-3,
args.rate_window[1] * 1e-3, nRates) # Hz
# Find RFI and trim it out. This is done by computing average visibility
# amplitudes (a "spectrum") and running a median filter in frequency to extract
# the bandpass. After the spectrum has been bandpassed, 3sigma features are
# trimmed. Additionally, area where the bandpass fall below 10% of its mean
# value are also masked.
spec = numpy.median(numpy.abs(visXX.mean(axis=0)), axis=0)
spec += numpy.median(numpy.abs(visYY.mean(axis=0)), axis=0)
smth = spec * 0.0
winSize = int(250e3 / (freq[1] - freq[0]))
winSize += ((winSize + 1) % 2)
for i in xrange(smth.size):
mn = max([0, i - winSize // 2])
mx = min([i + winSize, smth.size])
smth[i] = numpy.median(spec[mn:mx])
smth /= robust.mean(smth)
bp = spec / smth
good = numpy.where((smth > 0.1) & (
numpy.abs(bp - robust.mean(bp)) < 3 * robust.std(bp)))[0]
nBad = nchan - len(good)
print("Masking %i of %i channels (%.1f%%)" %
(nBad, nchan, 100.0 * nBad / nchan))
freq2 = freq * 1.0
freq2.shape += (1, )
dirName = os.path.basename(os.path.dirname(filenames[0]))
for b in xrange(len(bls)):
## Skip over baselines that are not in the baseline list (if provided)
if args.baseline is not None:
if bls[b] not in args.baseline:
continue
## Skip over baselines that don't include the reference antenna
elif bls[b][0] != args.ref_ant and bls[b][1] != args.ref_ant:
continue
## Check and see if we need to conjugate the visibility, i.e., switch from
## baseline (*,ref) to baseline (ref,*)
doConj = False
if bls[b][1] == args.ref_ant:
doConj = True
## Figure out which polarizations to process
if bls[b][0] not in (51, 52) and bls[b][1] not in (51, 52):
### Standard VLA-VLA baseline
polToUse = ('XX', 'XY', 'YX', 'YY')
visToUse = (visXX, visXY, visYX, visYY)
else:
### LWA-LWA or LWA-VLA baseline
if args.y_only:
polToUse = ('YX', 'YY')
visToUse = (visYX, visYY)
else:
polToUse = ('XX', 'XY', 'YX', 'YY')
visToUse = (visXX, visXY, visYX, visYY)
blName = bls[b]
if doConj:
blName = (bls[b][1], bls[b][0])
blName = '%s-%s' % ('EA%02i' %
blName[0] if blName[0] < 51 else 'LWA%i' %
(blName[0] - 50), 'EA%02i' %
blName[1] if blName[1] < 51 else 'LWA%i' %
(blName[1] - 50))
fig = plt.figure()
fig.suptitle('%s @ %s' % (blName, refSrc.name))
fig.subplots_adjust(hspace=0.001)
axR = fig.add_subplot(4, 1, 1)
axD = fig.add_subplot(4, 1, 2, sharex=axR)
axP = fig.add_subplot(4, 1, 3, sharex=axR)
axA = fig.add_subplot(4, 1, 4, sharex=axR)
markers = {'XX': 's', 'YY': 'o', 'XY': 'v', 'YX': '^'}
for pol, vis in zip(polToUse, visToUse):
for i in xrange(iCount):
subStart, subStop = times[iSize * i], times[iSize * (i + 1) -
1]
if (subStop - subStart) > 1.1 * args.interval:
continue
subTime = times[iSize * i:iSize * (i + 1)].mean()
dTimes2 = dTimes[iSize * i:iSize * (i + 1)] * 1.0
dTimes2.shape += (1, )
subData = vis[iSize * i:iSize * (i + 1), b, good] * 1.0
subPhase = vis[iSize * i:iSize * (i + 1), b, good] * 1.0
if doConj:
subData = subData.conj()
subPhase = subPhase.conj()
subData = numpy.dot(
subData,
numpy.exp(-2j * numpy.pi * freq2[good, :] * delay))
subData /= freq2[good, :].size
amp = numpy.dot(subData.T,
numpy.exp(-2j * numpy.pi * dTimes2 * drate))
amp = numpy.abs(amp / dTimes2.size)
subPhase = numpy.angle(subPhase.mean()) * 180 / numpy.pi
subPhase %= 360
if subPhase > 180:
subPhase -= 360
best = numpy.where(amp == amp.max())
if amp.max() > 0:
bsnr = (amp[best] - amp.mean())[0] / amp.std()
bdly = delay[best[0][0]] * 1e6
brat = drate[best[1][0]] * 1e3
axR.plot(subTime - ref_time,
brat,
linestyle='',
marker=markers[pol],
color='k')
axD.plot(subTime - ref_time,
bdly,
linestyle='',
marker=markers[pol],
color='k')
axP.plot(subTime - ref_time,
subPhase,
linestyle='',
marker=markers[pol],
color='k')
axA.plot(subTime - ref_time,
amp.max() * 1e3,
linestyle='',
marker=markers[pol],
color='k',
label=pol)
# Legend and reference marks
handles, labels = axA.get_legend_handles_labels()
axA.legend(handles[::iCount], labels[::iCount], loc=0)
oldLim = axR.get_xlim()
for ax in (axR, axD, axP):
ax.hlines(0, oldLim[0], oldLim[1], linestyle=':', alpha=0.5)
axR.set_xlim(oldLim)
# Turn off redundant x-axis tick labels
xticklabels = axR.get_xticklabels() + axD.get_xticklabels(
) + axP.get_xticklabels()
plt.setp(xticklabels, visible=False)
for ax in (axR, axD, axP, axA):
ax.set_xlabel(
'Elapsed Time [s since %s]' %
datetime.utcfromtimestamp(ref_time).strftime('%Y%b%d %H:%M'))
# Flip the y axis tick labels on every other plot
for ax in (axR, axP):
ax.yaxis.set_label_position('right')
ax.tick_params(axis='y',
which='both',
labelleft='off',
labelright='on')
# Get the labels
axR.set_ylabel('Rate [mHz]')
axD.set_ylabel('Delay [$\\mu$s]')
axP.set_ylabel('Phase [$^\\circ$]')
axA.set_ylabel('Amp.$\\times10^3$')
# Set the y ranges
axR.set_ylim((-max([abs(v) for v in axR.get_ylim()]),
max([abs(v) for v in axR.get_ylim()])))
axD.set_ylim((-max([abs(v) for v in axD.get_ylim()]),
max([abs(v) for v in axD.get_ylim()])))
ax.set_ylim((-180, 180))
axA.set_ylim((0, axA.get_ylim()[1]))
plt.draw()
plt.show()
def main(args):
# Parse the command line
## Baseline list
if args.baseline is not None:
## Fill the baseline list with the conjugates, if needed
newBaselines = []
for pair in args.baseline:
newBaselines.append( (pair[1],pair[0]) )
args.baseline.extend(newBaselines)
## Search limits
args.delay_window = [float(v) for v in args.delay_window.split(',', 1)]
args.rate_window = [float(v) for v in args.rate_window.split(',', 1)]
figs = {}
first = True
for filename in args.filename:
print("Working on '%s'" % os.path.basename(filename))
# Open the FITS IDI file and access the UV_DATA extension
hdulist = astrofits.open(filename, mode='readonly')
andata = hdulist['ANTENNA']
fqdata = hdulist['FREQUENCY']
fgdata = None
for hdu in hdulist[1:]:
if hdu.header['EXTNAME'] == 'FLAG':
fgdata = hdu
uvdata = hdulist['UV_DATA']
# Pull out various bits of information we need to flag the file
## Antenna look-up table
antLookup = {}
for an, ai in zip(andata.data['ANNAME'], andata.data['ANTENNA_NO']):
antLookup[an] = ai
## Frequency and polarization setup
nBand, nFreq, nStk = uvdata.header['NO_BAND'], uvdata.header['NO_CHAN'], uvdata.header['NO_STKD']
stk0 = uvdata.header['STK_1']
## Baseline list
bls = uvdata.data['BASELINE']
## Time of each integration
obsdates = uvdata.data['DATE']
obstimes = uvdata.data['TIME']
inttimes = uvdata.data['INTTIM']
## Source list
srcs = uvdata.data['SOURCE']
## Band information
fqoffsets = fqdata.data['BANDFREQ'].ravel()
## Frequency channels
freq = (numpy.arange(nFreq)-(uvdata.header['CRPIX3']-1))*uvdata.header['CDELT3']
freq += uvdata.header['CRVAL3']
## UVW coordinates
try:
u, v, w = uvdata.data['UU'], uvdata.data['VV'], uvdata.data['WW']
except KeyError:
u, v, w = uvdata.data['UU---SIN'], uvdata.data['VV---SIN'], uvdata.data['WW---SIN']
uvw = numpy.array([u, v, w]).T
## The actual visibility data
flux = uvdata.data['FLUX'].astype(numpy.float32)
# Convert the visibilities to something that we can easily work with
nComp = flux.shape[1] // nBand // nFreq // nStk
if nComp == 2:
## Case 1) - Just real and imaginary data
flux = flux.view(numpy.complex64)
else:
## Case 2) - Real, imaginary data + weights (drop the weights)
flux = flux[:,0::nComp] + 1j*flux[:,1::nComp]
flux.shape = (flux.shape[0], nBand, nFreq, nStk)
# Find unique baselines, times, and sources to work with
ubls = numpy.unique(bls)
utimes = numpy.unique(obstimes)
usrc = numpy.unique(srcs)
# Convert times to real times
times = utcjd_to_unix(obsdates + obstimes)
times = numpy.unique(times)
# Build a mask
mask = numpy.zeros(flux.shape, dtype=numpy.bool)
if fgdata is not None and not args.drop:
reltimes = obsdates - obsdates[0] + obstimes
maxtimes = reltimes + inttimes / 2.0 / 86400.0
mintimes = reltimes - inttimes / 2.0 / 86400.0
bls_ant1 = bls//256
bls_ant2 = bls%256
for row in fgdata.data:
ant1, ant2 = row['ANTS']
## Only deal with flags that we need for the plots
process_flag = False
if ant1 != ant2 or ant1 == 0 or ant2 == 0:
if ant1 == 0 and ant2 == 0:
process_flag = True
elif args.baseline is not None:
if ant2 == 0 and ant1 in [a0 for a0,a1 in args.baseline]:
process_flag = True
elif (ant1,ant2) in args.baseline:
process_flag = True
elif args.ref_ant is not None:
if ant1 == args.ref_ant or ant2 == args.ref_ant:
process_flag = True
else:
process_flag = True
if not process_flag:
continue
tStart, tStop = row['TIMERANG']
band = row['BANDS']
try:
len(band)
except TypeError:
band = [band,]
cStart, cStop = row['CHANS']
if cStop == 0:
cStop = -1
pol = row['PFLAGS'].astype(numpy.bool)
if ant1 == 0 and ant2 == 0:
btmask = numpy.where( ( (maxtimes >= tStart) & (mintimes <= tStop) ) )[0]
elif ant1 == 0 or ant2 == 0:
ant1 = max([ant1, ant2])
btmask = numpy.where( ( (bls_ant1 == ant1) | (bls_ant2 == ant1) ) \
& ( (maxtimes >= tStart) & (mintimes <= tStop) ) )[0]
else:
btmask = numpy.where( ( (bls_ant1 == ant1) & (bls_ant2 == ant2) ) \
& ( (maxtimes >= tStart) & (mintimes <= tStop) ) )[0]
for b,v in enumerate(band):
if not v:
continue
mask[btmask,b,cStart-1:cStop,:] |= pol
# Make sure the reference antenna is in there
if first:
if args.ref_ant is None:
bl = bls[0]
i,j = (bl>>8)&0xFF, bl&0xFF
args.ref_ant = i
else:
found = False
for bl in bls:
i,j = (bl>>8)&0xFF, bl&0xFF
if i == args.ref_ant:
found = True
break
if not found:
raise RuntimeError("Cannot file reference antenna %i in the data" % args.ref_ant)
plot_bls = []
cross = []
for i in xrange(len(ubls)):
bl = ubls[i]
ant1, ant2 = (bl>>8)&0xFF, bl&0xFF
if ant1 != ant2:
if args.baseline is not None:
if (ant1,ant2) in args.baseline:
plot_bls.append( bl )
cross.append( i )
elif args.ref_ant is not None:
if ant1 == args.ref_ant or ant2 == args.ref_ant:
plot_bls.append( bl )
cross.append( i )
else:
plot_bls.append( bl )
cross.append( i )
nBL = len(cross)
# Decimation, if needed
if args.decimate > 1:
if nFreq % args.decimate != 0:
raise RuntimeError("Invalid freqeunce decimation factor: %i %% %i = %i" % (nFreq, args.decimate, nFreq%args.decimate))
nFreq //= args.decimate
freq.shape = (freq.size//args.decimate, args.decimate)
freq = freq.mean(axis=1)
flux.shape = (flux.shape[0], flux.shape[1], flux.shape[2]//args.decimate, args.decimate, flux.shape[3])
flux = flux.mean(axis=3)
mask.shape = (mask.shape[0], mask.shape[1], mask.shape[2]//args.decimate, args.decimate, mask.shape[3])
mask = mask.mean(axis=3)
good = numpy.arange(freq.size//8, freq.size*7//8) # Inner 75% of the band
iSize = int(round(args.interval/robust.mean(inttimes)))
print(" -> Chunk size is %i intervals (%.3f seconds)" % (iSize, iSize*robust.mean(inttimes)))
iCount = times.size//iSize
print(" -> Working with %i chunks of data" % iCount)
print("Number of frequency channels: %i (~%.1f Hz/channel)" % (len(freq), freq[1]-freq[0]))
dTimes = times - times[0]
if first:
ref_time = (int(times[0]) / 60) * 60
dMax = 1.0/(freq[1]-freq[0])/4
dMax = int(dMax*1e6)*1e-6
if -dMax*1e6 > args.delay_window[0]:
args.delay_window[0] = -dMax*1e6
if dMax*1e6 < args.delay_window[1]:
args.delay_window[1] = dMax*1e6
rMax = 1.0/robust.mean(inttimes)/4
rMax = int(rMax*1e2)*1e-2
if -rMax*1e3 > args.rate_window[0]:
args.rate_window[0] = -rMax*1e3
if rMax*1e3 < args.rate_window[1]:
args.rate_window[1] = rMax*1e3
dres = 1.0
nDelays = int((args.delay_window[1]-args.delay_window[0])/dres)
while nDelays < 50:
dres /= 10
nDelays = int((args.delay_window[1]-args.delay_window[0])/dres)
while nDelays > 5000:
dres *= 10
nDelays = int((args.delay_window[1]-args.delay_window[0])/dres)
nDelays += (nDelays + 1) % 2
rres = 10.0
nRates = int((args.rate_window[1]-args.rate_window[0])/rres)
while nRates < 50:
rres /= 10
nRates = int((args.rate_window[1]-args.rate_window[0])/rres)
while nRates > 5000:
rres *= 10
nRates = int((args.rate_window[1]-args.rate_window[0])/rres)
nRates += (nRates + 1) % 2
print("Searching delays %.1f to %.1f us in steps of %.2f us" % (args.delay_window[0], args.delay_window[1], dres))
print(" rates %.1f to %.1f mHz in steps of %.2f mHz" % (args.rate_window[0], args.rate_window[1], rres))
print(" ")
delay = numpy.linspace(args.delay_window[0]*1e-6, args.delay_window[1]*1e-6, nDelays) # s
drate = numpy.linspace(args.rate_window[0]*1e-3, args.rate_window[1]*1e-3, nRates ) # Hz
# Find RFI and trim it out. This is done by computing average visibility
# amplitudes (a "spectrum") and running a median filter in frequency to extract
# the bandpass. After the spectrum has been bandpassed, 3sigma features are
# trimmed. Additionally, area where the bandpass fall below 10% of its mean
# value are also masked.
spec = numpy.median(numpy.abs(flux[:,0,:,0]), axis=0)
spec += numpy.median(numpy.abs(flux[:,0,:,1]), axis=0)
smth = spec*0.0
winSize = int(250e3/(freq[1]-freq[0]))
winSize += ((winSize+1)%2)
for i in xrange(smth.size):
mn = max([0, i-winSize//2])
mx = min([i+winSize, smth.size])
smth[i] = numpy.median(spec[mn:mx])
smth /= robust.mean(smth)
bp = spec / smth
good = numpy.where( (smth > 0.1) & (numpy.abs(bp-robust.mean(bp)) < 3*robust.std(bp)) )[0]
nBad = nFreq - len(good)
print("Masking %i of %i channels (%.1f%%)" % (nBad, nFreq, 100.0*nBad/nFreq))
freq2 = freq*1.0
freq2.shape += (1,)
dirName = os.path.basename( filename )
for b in xrange(len(plot_bls)):
bl = plot_bls[b]
valid = numpy.where( bls == bl )[0]
i,j = (bl>>8)&0xFF, bl&0xFF
dTimes = obsdates[valid] + obstimes[valid]
dTimes = numpy.array([utcjd_to_unix(v) for v in dTimes])
## Skip over baselines that are not in the baseline list (if provided)
if args.baseline is not None:
if (i,j) not in args.baseline:
continue
## Skip over baselines that don't include the reference antenna
elif i != args.ref_ant and j != args.ref_ant:
continue
## Check and see if we need to conjugate the visibility, i.e., switch from
## baseline (*,ref) to baseline (ref,*)
doConj = False
if j == args.ref_ant:
doConj = True
## Figure out which polarizations to process
if args.cross_hands:
polToUse = ('XX', 'XY', 'YY')
visToUse = (0, 2, 1)
else:
polToUse = ('XX', 'YY')
visToUse = (0, 1)
blName = (i, j)
if doConj:
blName = (j, i)
blName = '%s-%s' % ('EA%02i' % blName[0] if blName[0] < 51 else 'LWA%i' % (blName[0]-50),
'EA%02i' % blName[1] if blName[1] < 51 else 'LWA%i' % (blName[1]-50))
if first or blName not in figs:
fig = plt.figure()
fig.suptitle('%s' % blName)
fig.subplots_adjust(hspace=0.001)
axR = fig.add_subplot(2, 1, 1)
axD = fig.add_subplot(2, 1, 2, sharex=axR)
figs[blName] = (fig, axR, axD)
fig, axR, axD = figs[blName]
markers = {'XX':'s', 'YY':'o', 'XY':'v', 'YX':'^'}
for pol,vis in zip(polToUse, visToUse):
for i in xrange(iCount):
subStart, subStop = dTimes[iSize*i], dTimes[iSize*(i+1)-1]
if (subStop - subStart) > 1.1*args.interval:
continue
subTime = dTimes[iSize*i:iSize*(i+1)].mean()
dTimes2 = dTimes[iSize*i:iSize*(i+1)]*1.0
dTimes2 -= dTimes2[0]
dTimes2.shape += (1,)
subData = flux[valid,...][iSize*i:iSize*(i+1),0,good,vis]*1.0
subPhase = flux[valid,...][iSize*i:iSize*(i+1),0,good,vis]*1.0
if doConj:
subData = subData.conj()
subPhase = subPhase.conj()
subData = numpy.dot(subData, numpy.exp(-2j*numpy.pi*freq2[good,:]*delay))
subData /= freq2[good,:].size
amp = numpy.dot(subData.T, numpy.exp(-2j*numpy.pi*dTimes2*drate))
amp = numpy.abs(amp / dTimes2.size)
subPhase = numpy.angle(subPhase.mean()) * 180/numpy.pi
subPhase %= 360
if subPhase > 180:
subPhase -= 360
best = numpy.where( amp == amp.max() )
if amp.max() > 0:
bsnr = (amp[best]-amp.mean())[0]/amp.std()
bdly = delay[best[0][0]]*1e6
brat = drate[best[1][0]]*1e3
c = axR.scatter(subTime-ref_time, brat, c=bsnr, marker=markers[pol],
cmap='gist_yarg', vmin=3, vmax=40)
c = axD.scatter(subTime-ref_time, bdly, c=bsnr, marker=markers[pol],
cmap='gist_yarg', vmin=3, vmax=40)
first = False
for blName in figs:
fig, axR, axD = figs[blName]
# Colorbar
cb = fig.colorbar(c, ax=axR, orientation='horizontal')
cb.set_label('SNR')
# Legend and reference marks
handles = []
for pol in polToUse:
handles.append(Line2D([0,], [0,], linestyle='', marker=markers[pol], color='k', label=pol))
axR.legend(handles=handles, loc=0)
oldLim = axR.get_xlim()
for ax in (axR, axD):
ax.hlines(0, oldLim[0], oldLim[1], linestyle=':', alpha=0.5)
axR.set_xlim(oldLim)
# Set the labels
axR.set_ylabel('Rate [mHz]')
axD.set_ylabel('Delay [$\\mu$s]')
for ax in (axR, axD):
ax.set_xlabel('Elapsed Time [s since %s]' % datetime.utcfromtimestamp(ref_time).strftime('%Y%b%d %H:%M'))
# Set the y ranges
axR.set_ylim((-max([100, max([abs(v) for v in axR.get_ylim()])]), max([100, max([abs(v) for v in axR.get_ylim()])])))
axD.set_ylim((-max([0.5, max([abs(v) for v in axD.get_ylim()])]), max([0.5, max([abs(v) for v in axD.get_ylim()])])))
# No-go regions for the delays
xlim, ylim = axD.get_xlim(), axD.get_ylim()
axD.add_patch(Box(xy=(xlim[0],ylim[0]), width=xlim[1]-xlim[0], height=-0.5001-ylim[0],
fill=True, color='red', alpha=0.2))
axD.add_patch(Box(xy=(xlim[0],0.5001), width=xlim[1]-xlim[0], height=ylim[1]-0.5001,
fill=True, color='red', alpha=0.2))
axD.set_xlim(xlim)
axD.set_ylim(ylim)
fig.tight_layout()
plt.draw()
if args.save_images:
fig.savefig('sniffer-%s.png' % blName)
if not args.save_images:
plt.show()
def main(args):
# Parse the command line
## Baseline list
if args.baseline is not None:
## Fill the baseline list with the conjugates, if needed
newBaselines = []
for pair in args.baseline:
newBaselines.append((pair[1], pair[0]))
args.baseline.extend(newBaselines)
## Search limits
args.delay_window = [float(v) for v in args.delay_window.split(',', 1)]
args.rate_window = [float(v) for v in args.rate_window.split(',', 1)]
## Filenames
filenames = args.filename
filenames.sort()
if args.limit != -1:
filenames = filenames[:args.limit]
nInt = len(filenames)
dataDict = numpy.load(filenames[0])
tInt = dataDict['tInt']
nBL, nchan = dataDict['vis1XX'].shape
freq = dataDict['freq1']
junk0, refSrc, junk1, junk2, junk3, junk4, antennas = read_correlator_configuration(
dataDict)
dataDict.close()
# Make sure the reference antenna is in there
if args.ref_ant is None:
args.ref_ant = antennas[0].stand.id
else:
found = False
for ant in antennas:
if ant.stand.id == args.ref_ant:
found = True
break
if not found:
raise RuntimeError("Cannot file reference antenna %i in the data" %
args.ref_ant)
bls = []
l = 0
cross = []
for i in xrange(0, len(antennas), 2):
ant1 = antennas[i].stand.id
for j in xrange(i, len(antennas), 2):
ant2 = antennas[j].stand.id
if ant1 != ant2:
bls.append((ant1, ant2))
cross.append(l)
l += 1
nBL = len(cross)
if args.decimate > 1:
if nchan % args.decimate != 0:
raise RuntimeError(
"Invalid freqeunce decimation factor: %i %% %i = %i" %
(nchan, args.decimate, nchan % args.decimate))
nchan //= args.decimate
freq.shape = (freq.size // args.decimate, args.decimate)
freq = freq.mean(axis=1)
times = numpy.zeros(nInt, dtype=numpy.float64)
visXX = numpy.zeros((nInt, nBL, nchan), dtype=numpy.complex64)
if not args.y_only:
visXY = numpy.zeros((nInt, nBL, nchan), dtype=numpy.complex64)
visYX = numpy.zeros((nInt, nBL, nchan), dtype=numpy.complex64)
visYY = numpy.zeros((nInt, nBL, nchan), dtype=numpy.complex64)
for i, filename in enumerate(filenames):
dataDict = numpy.load(filename)
tStart = dataDict['tStart']
cvisXX = dataDict['vis1XX'][cross, :]
cvisXY = dataDict['vis1XY'][cross, :]
cvisYX = dataDict['vis1YX'][cross, :]
cvisYY = dataDict['vis1YY'][cross, :]
if args.decimate > 1:
cvisXX.shape = (cvisXX.shape[0], cvisXX.shape[1] // args.decimate,
args.decimate)
cvisXX = cvisXX.mean(axis=2)
cvisXY.shape = (cvisXY.shape[0], cvisXY.shape[1] // args.decimate,
args.decimate)
cvisXY = cvisXY.mean(axis=2)
cvisYX.shape = (cvisYX.shape[0], cvisYX.shape[1] // args.decimate,
args.decimate)
cvisYX = cvisYX.mean(axis=2)
cvisYY.shape = (cvisYY.shape[0], cvisYY.shape[1] // args.decimate,
args.decimate)
cvisYY = cvisYY.mean(axis=2)
visXX[i, :, :] = cvisXX
if not args.y_only:
visXY[i, :, :] = cvisXY
visYX[i, :, :] = cvisYX
visYY[i, :, :] = cvisYY
times[i] = tStart
dataDict.close()
print("Got %i files from %s to %s (%.1f s)" %
(len(filenames), datetime.utcfromtimestamp(
times[0]).strftime("%Y/%m/%d %H:%M:%S"),
datetime.utcfromtimestamp(times[-1]).strftime("%Y/%m/%d %H:%M:%S"),
(times[-1] - times[0])))
iTimes = numpy.zeros(nInt - 1, dtype=times.dtype)
for i in xrange(1, len(times)):
iTimes[i - 1] = times[i] - times[i - 1]
print(" -> Interval: %.3f +/- %.3f seconds (%.3f to %.3f seconds)" %
(iTimes.mean(), iTimes.std(), iTimes.min(), iTimes.max()))
print("Number of frequency channels: %i (~%.1f Hz/channel)" %
(len(freq), freq[1] - freq[0]))
dTimes = times - times[0]
dMax = 1.0 / (freq[1] - freq[0]) / 4
dMax = int(dMax * 1e6) * 1e-6
if -dMax * 1e6 > args.delay_window[0]:
args.delay_window[0] = -dMax * 1e6
if dMax * 1e6 < args.delay_window[1]:
args.delay_window[1] = dMax * 1e6
rMax = 1.0 / iTimes.mean() / 4
rMax = int(rMax * 1e2) * 1e-2
if -rMax * 1e3 > args.rate_window[0]:
args.rate_window[0] = -rMax * 1e3
if rMax * 1e3 < args.rate_window[1]:
args.rate_window[1] = rMax * 1e3
dres = 1.0
nDelays = int((args.delay_window[1] - args.delay_window[0]) / dres)
while nDelays < 50:
dres /= 10
nDelays = int((args.delay_window[1] - args.delay_window[0]) / dres)
while nDelays > 5000:
dres *= 10
nDelays = int((args.delay_window[1] - args.delay_window[0]) / dres)
nDelays += (nDelays + 1) % 2
rres = 10.0
nRates = int((args.rate_window[1] - args.rate_window[0]) / rres)
while nRates < 50:
rres /= 10
nRates = int((args.rate_window[1] - args.rate_window[0]) / rres)
while nRates > 5000:
rres *= 10
nRates = int((args.rate_window[1] - args.rate_window[0]) / rres)
nRates += (nRates + 1) % 2
print("Searching delays %.1f to %.1f us in steps of %.2f us" %
(args.delay_window[0], args.delay_window[1], dres))
print(" rates %.1f to %.1f mHz in steps of %.2f mHz" %
(args.rate_window[0], args.rate_window[1], rres))
print(" ")
delay = numpy.linspace(args.delay_window[0] * 1e-6,
args.delay_window[1] * 1e-6, nDelays) # s
drate = numpy.linspace(args.rate_window[0] * 1e-3,
args.rate_window[1] * 1e-3, nRates) # Hz
# Find RFI and trim it out. This is done by computing average visibility
# amplitudes (a "spectrum") and running a median filter in frequency to extract
# the bandpass. After the spectrum has been bandpassed, 3sigma features are
# trimmed. Additionally, area where the bandpass fall below 10% of its mean
# value are also masked.
spec = numpy.median(numpy.abs(visXX.mean(axis=0)), axis=0)
spec += numpy.median(numpy.abs(visYY.mean(axis=0)), axis=0)
smth = spec * 0.0
winSize = int(250e3 / (freq[1] - freq[0]))
winSize += ((winSize + 1) % 2)
for i in xrange(smth.size):
mn = max([0, i - winSize // 2])
mx = min([i + winSize // 2 + 1, smth.size])
smth[i] = numpy.median(spec[mn:mx])
smth /= robust.mean(smth)
bp = spec / smth
good = numpy.where((smth > 0.1) & (
numpy.abs(bp - robust.mean(bp)) < 3 * robust.std(bp)))[0]
nBad = nchan - len(good)
print("Masking %i of %i channels (%.1f%%)" %
(nBad, nchan, 100.0 * nBad / nchan))
if args.plot:
fig = plt.figure()
ax = fig.gca()
ax.plot(freq / 1e6, numpy.log10(spec) * 10)
ax.plot(freq[good] / 1e6, numpy.log10(spec[good]) * 10)
ax.set_title('Mean Visibility Amplitude')
ax.set_xlabel('Frequency [MHz]')
ax.set_ylabel('PSD [arb. dB]')
plt.draw()
freq2 = freq * 1.0
freq2.shape += (1, )
dTimes2 = dTimes * 1.0
dTimes2.shape += (1, )
dirName = os.path.basename(os.path.dirname(filenames[0]))
print("%3s %9s %2s %6s %9s %11s" %
('#', 'BL', 'Pl', 'S/N', 'Delay', 'Rate'))
for b in xrange(len(bls)):
## Skip over baselines that are not in the baseline list (if provided)
if args.baseline is not None:
if bls[b] not in args.baseline:
continue
## Skip over baselines that don't include the reference antenna
elif bls[b][0] != args.ref_ant and bls[b][1] != args.ref_ant:
continue
## Check and see if we need to conjugate the visibility, i.e., switch from
## baseline (*,ref) to baseline (ref,*)
doConj = False
if bls[b][1] == args.ref_ant:
doConj = True
## Figure out which polarizations to process
if bls[b][0] not in (51, 52) and bls[b][1] not in (51, 52):
### Standard VLA-VLA baseline
polToUse = ('XX', 'YY')
visToUse = (visXX, visYY)
else:
### LWA-LWA or LWA-VLA baseline
if args.y_only:
polToUse = ('YX', 'YY')
visToUse = (visYX, visYY)
else:
polToUse = ('XX', 'XY', 'YX', 'YY')
visToUse = (visXX, visXY, visYX, visYY)
if args.plot:
fig = plt.figure()
axs = {}
axs['XX'] = fig.add_subplot(2, 2, 1)
axs['YY'] = fig.add_subplot(2, 2, 2)
axs['XY'] = fig.add_subplot(2, 2, 3)
axs['YX'] = fig.add_subplot(2, 2, 4)
for pol, vis in zip(polToUse, visToUse):
subData = vis[:, b, good] * 1.0
if doConj:
subData = subData.conj()
subData = numpy.dot(
subData, numpy.exp(-2j * numpy.pi * freq2[good, :] * delay))
subData /= freq2[good, :].size
amp = numpy.dot(subData.T,
numpy.exp(-2j * numpy.pi * dTimes2 * drate))
amp = numpy.abs(amp / dTimes2.size)
blName = bls[b]
if doConj:
blName = (bls[b][1], bls[b][0])
blName = '%s-%s' % ('EA%02i' %
blName[0] if blName[0] < 51 else 'LWA%i' %
(blName[0] - 50), 'EA%02i' %
blName[1] if blName[1] < 51 else 'LWA%i' %
(blName[1] - 50))
best = numpy.where(amp == amp.max())
if amp.max() > 0:
bsnr = (amp[best] - amp.mean())[0] / amp.std()
bdly = delay[best[0][0]] * 1e6
brat = drate[best[1][0]] * 1e3
print("%3i %9s %2s %6.2f %6.2f us %7.2f mHz" %
(b, blName, pol, bsnr, bdly, brat))
else:
print("%3i %9s %2s %6s %9s %11s" %
(b, blName, pol, '----', '----', '----'))
if args.plot:
axs[pol].imshow(amp,
origin='lower',
interpolation='nearest',
extent=(drate[0] * 1e3, drate[-1] * 1e3,
delay[0] * 1e6, delay[-1] * 1e6),
cmap='gray_r')
axs[pol].plot(drate[best[1][0]] * 1e3,
delay[best[0][0]] * 1e6,
linestyle='',
marker='x',
color='r',
ms=15,
alpha=0.75)
if args.plot:
fig.suptitle(dirName)
for pol in axs.keys():
ax = axs[pol]
ax.axis('auto')
ax.set_title(pol)
ax.set_xlabel('Rate [mHz]')
ax.set_ylabel('Delay [$\\mu$s]')
fig.suptitle("%s @ %s" % (blName, refSrc.name))
plt.draw()
if args.plot:
plt.show()
def unscatter(t,
raw,
tScatMin,
tScatMax,
tScatStep,
gain=0.05,
max_iter=10000,
screen=thin,
verbose=True):
"""
Multi-path scattering deconvolution method based on the method
presented in Bhat, N., Cordes, J., & Chatterjee, S. 2003, ApJ,
584, 782.
Inputs:
1) t: List of times in seconds the pulse profile corresponds to
2) raw: the raw (scattered) pulse profile over time
3) tScatMin: minimum scattering time to search
4) tScatMax: maximum scattering time to search
5) tScatStep: time step for tScat search
Options:
* gain: CLEAN loop gain (default is 0.05)
* max_iter: maximum number of iterations to use (default is 10000)
* screen: pulse broadening function (default is thin)
Outputs (as a tuple):
1) tScat: best-fit scattering time in seconds
2) merit: figure of merit for the deconvolution/scattering time fit
3) cleand: CLEANed pulsar profile as a function of time
"""
iList = {}
meritList = {}
ccList = {}
residList = {}
# Off-pulsar standard deviation
sigma = robust.std(raw)
# Loop over tScat values
best = numpy.inf
bestTau = None
for tScat in numpy.arange(tScatMin, tScatMax, tScatStep):
## Setup the temporary variables
working = raw * 1.0
cc = raw * 0.0
## Iterate...
i = 0
while i < max_iter:
### Find the peak and make sure it is really a peak
peak = numpy.where(working == working.max())[0][0]
if working.max() < 3. / 2. * sigma:
break
### Generate the clean component
tPeak = t[peak]
tRel = t - tPeak
toRemove = screen(tRel, tScat)
toRemove /= toRemove.sum()
toRemove *= gain * working.max()
### Remove and continue
cc[peak] += toRemove.sum()
working -= toRemove
i += 1
## Evaluate and save
iList[tScat] = i
meritList[tScat] = _figure_of_merit(t, raw, working, cc)
ccList[tScat] = cc
residList[tScat] = working
## Compare
if meritList[tScat] < best:
best = meritList[tScat]
bestTau = tScat
# Make sure we have something to report
if tScat is None:
raise RuntimeError(
"No good solution found, consider changing the search range.")
# Select out what we need, i.e., the best
tScat = bestTau
i = iList[bestTau]
merit = meritList[bestTau]
cc = ccList[bestTau]
resids = residList[bestTau]
# Report on the findings
if verbose:
print("Multi-path Scattering Results:")
print(" Iterations Used: %i of %i" % (i, max_iter))
print(" Best-fit Scattering time: %.3f ms" % (tScat * 1000.0, ))
print(" Figure-of-merit: %.5f" % merit)
# Restore the profile using a Gaussian with a sigma value of 5 time steps
sigmaRestore = 5 * (t[1] - t[0])
restoreFunction = numpy.exp(-t**2 / (2 * sigmaRestore**2))
restoreFunction /= restoreFunction.sum()
out = numpy.convolve(cc, restoreFunction, 'same')
# Add back in the residuals
out += resids
# Done!
return tScat, merit, out
def estimateClipLevel(fh, beampols):
"""
Read in a set of 100 frames and come up with the 4-sigma clip levels
for each tuning. These clip levels are returned as a two-element
tuple.
"""
filePos = fh.tell()
# Read in the first 100 frames for each tuning/polarization
count = {0:0, 1:0, 2:0, 3:0}
data = numpy.zeros((4, 4096*100), dtype=numpy.csingle)
for i in xrange(beampols*100):
try:
cFrame = drx.readFrame(fh, Verbose=False)
except errors.eofError:
break
except errors.syncError:
continue
beam,tune,pol = cFrame.parseID()
aStand = 2*(tune-1) + pol
data[aStand, count[aStand]*4096:(count[aStand]+1)*4096] = cFrame.data.iq
count[aStand] += 1
# Go back to where we started
fh.seek(filePos)
# Compute the robust mean and standard deviation for I and Q for each
# tuning/polarization
meanI = []
meanQ = []
stdsI = []
stdsQ = []
for i in xrange(4):
meanI.append( robust.mean(data[i,:].real) )
meanQ.append( robust.mean(data[i,:].imag) )
stdsI.append( robust.std(data[i,:].real) )
stdsQ.append( robust.std(data[i,:].imag) )
# Report
print "Statistics:"
for i in xrange(4):
print " Mean %i: %.3f + %.3f j" % (i+1, meanI[i], meanQ[i])
print " Std %i: %.3f + %.3f j" % (i+1, stdsI[i], stdsQ[i])
# Come up with the clip levels based on 4 sigma
clip1 = (meanI[0] + meanI[1] + meanQ[0] + meanQ[1]) / 4.0
clip2 = (meanI[2] + meanI[3] + meanQ[2] + meanQ[3]) / 4.0
clip1 += 5*(stdsI[0] + stdsI[1] + stdsQ[0] + stdsQ[1]) / 4.0
clip2 += 5*(stdsI[2] + stdsI[3] + stdsQ[2] + stdsQ[3]) / 4.0
clip1 = int(round(clip1))
clip2 = int(round(clip2))
# Report again
print "Clip Levels:"
print " Tuning 1: %i" % clip1
print " Tuning 2: %i" % clip2
return clip1, clip2
def main(args):
# Parse the command line
## Baseline list
if args.baseline is not None:
## Fill the baseline list with the conjugates, if needed
newBaselines = []
for pair in args.baseline:
newBaselines.append( (pair[1],pair[0]) )
args.baseline.extend(newBaselines)
## Search limits
args.delay_window = [float(v) for v in args.delay_window.split(',', 1)]
args.rate_window = [float(v) for v in args.rate_window.split(',', 1)]
print("Working on '%s'" % os.path.basename(args.filename))
# Open the FITS IDI file and access the UV_DATA extension
hdulist = astrofits.open(args.filename, mode='readonly')
andata = hdulist['ANTENNA']
fqdata = hdulist['FREQUENCY']
uvdata = hdulist['UV_DATA']
# Verify we can flag this data
if uvdata.header['STK_1'] > 0:
raise RuntimeError("Cannot flag data with STK_1 = %i" % uvdata.header['STK_1'])
if uvdata.header['NO_STKD'] < 4:
raise RuntimeError("Cannot flag data with NO_STKD = %i" % uvdata.header['NO_STKD'])
# Pull out various bits of information we need to flag the file
## Antenna look-up table
antLookup = {}
for an, ai in zip(andata.data['ANNAME'], andata.data['ANTENNA_NO']):
antLookup[an] = ai
## Frequency and polarization setup
nBand, nFreq, nStk = uvdata.header['NO_BAND'], uvdata.header['NO_CHAN'], uvdata.header['NO_STKD']
## Baseline list
bls = uvdata.data['BASELINE']
## Time of each integration
obsdates = uvdata.data['DATE']
obstimes = uvdata.data['TIME']
inttimes = uvdata.data['INTTIM']
## Source list
srcs = uvdata.data['SOURCE']
## Band information
fqoffsets = fqdata.data['BANDFREQ'].ravel()
## Frequency channels
freq = (numpy.arange(nFreq)-(uvdata.header['CRPIX3']-1))*uvdata.header['CDELT3']
freq += uvdata.header['CRVAL3']
## UVW coordinates
try:
u, v, w = uvdata.data['UU'], uvdata.data['VV'], uvdata.data['WW']
except KeyError:
u, v, w = uvdata.data['UU---SIN'], uvdata.data['VV---SIN'], uvdata.data['WW---SIN']
uvw = numpy.array([u, v, w]).T
## The actual visibility data
flux = uvdata.data['FLUX'].astype(numpy.float32)
# Convert the visibilities to something that we can easily work with
nComp = flux.shape[1] // nBand // nFreq // nStk
if nComp == 2:
## Case 1) - Just real and imaginary data
flux = flux.view(numpy.complex64)
else:
## Case 2) - Real, imaginary data + weights (drop the weights)
flux = flux[:,0::nComp] + 1j*flux[:,1::nComp]
flux.shape = (flux.shape[0], nBand, nFreq, nStk)
# Find unique baselines, times, and sources to work with
ubls = numpy.unique(bls)
utimes = numpy.unique(obstimes)
usrc = numpy.unique(srcs)
# Convert times to real times
times = utcjd_to_unix(obsdates + obstimes)
times = numpy.unique(times)
# Find unique scans to work on, making sure that there are no large gaps
blocks = []
for src in usrc:
valid = numpy.where( src == srcs )[0]
blocks.append( [valid[0],valid[0]] )
for v in valid[1:]:
if v == blocks[-1][1] + 1 \
and (obsdates[v] - obsdates[blocks[-1][1]] + obstimes[v] - obstimes[blocks[-1][1]])*86400 < 10*inttimes[v]:
blocks[-1][1] = v
else:
blocks.append( [v,v] )
blocks.sort()
# Make sure the reference antenna is in there
if args.ref_ant is None:
bl = ubls[0]
ant1, ant2 = (bl>>8)&0xFF, bl&0xFF
args.ref_ant = ant1
else:
found = False
for bl in ubls:
ant1, ant2 = (bl>>8)&0xFF, bl&0xFF
if ant1 == args.ref_ant:
found = True
break
if not found:
raise RuntimeError("Cannot file reference antenna %i in the data" % args.ref_ant)
search_bls = []
cross = []
for i in xrange(len(ubls)):
bl = ubls[i]
ant1, ant2 = (bl>>8)&0xFF, bl&0xFF
if ant1 != ant2:
search_bls.append( bl )
cross.append( i )
nBL = len(cross)
iTimes = numpy.zeros(times.size-1, dtype=times.dtype)
for i in xrange(1, len(times)):
iTimes[i-1] = times[i] - times[i-1]
print(" -> Interval: %.3f +/- %.3f seconds (%.3f to %.3f seconds)" % (iTimes.mean(), iTimes.std(), iTimes.min(), iTimes.max()))
print("Number of frequency channels: %i (~%.1f Hz/channel)" % (len(freq), freq[1]-freq[0]))
dTimes = times - times[0]
dMax = 1.0/(freq[1]-freq[0])/4
dMax = int(dMax*1e6)*1e-6
if -dMax*1e6 > args.delay_window[0]:
args.delay_window[0] = -dMax*1e6
if dMax*1e6 < args.delay_window[1]:
args.delay_window[1] = dMax*1e6
rMax = 1.0/iTimes.mean()/4
rMax = int(rMax*1e2)*1e-2
if -rMax*1e3 > args.rate_window[0]:
args.rate_window[0] = -rMax*1e3
if rMax*1e3 < args.rate_window[1]:
args.rate_window[1] = rMax*1e3
dres = 1.0
nDelays = int((args.delay_window[1]-args.delay_window[0])/dres)
while nDelays < 50:
dres /= 10
nDelays = int((args.delay_window[1]-args.delay_window[0])/dres)
while nDelays > 5000:
dres *= 10
nDelays = int((args.delay_window[1]-args.delay_window[0])/dres)
nDelays += (nDelays + 1) % 2
rres = 10.0
nRates = int((args.rate_window[1]-args.rate_window[0])/rres)
while nRates < 50:
rres /= 10
nRates = int((args.rate_window[1]-args.rate_window[0])/rres)
while nRates > 5000:
rres *= 10
nRates = int((args.rate_window[1]-args.rate_window[0])/rres)
nRates += (nRates + 1) % 2
print("Searching delays %.1f to %.1f us in steps of %.2f us" % (args.delay_window[0], args.delay_window[1], dres))
print(" rates %.1f to %.1f mHz in steps of %.2f mHz" % (args.rate_window[0], args.rate_window[1], rres))
print(" ")
delay = numpy.linspace(args.delay_window[0]*1e-6, args.delay_window[1]*1e-6, nDelays) # s
drate = numpy.linspace(args.rate_window[0]*1e-3, args.rate_window[1]*1e-3, nRates ) # Hz
# Find RFI and trim it out. This is done by computing average visibility
# amplitudes (a "spectrum") and running a median filter in frequency to extract
# the bandpass. After the spectrum has been bandpassed, 3sigma features are
# trimmed. Additionally, area where the bandpass fall below 10% of its mean
# value are also masked.
spec = numpy.median(numpy.abs(flux[:,0,:,0]), axis=0)
spec += numpy.median(numpy.abs(flux[:,0,:,1]), axis=0)
smth = spec*0.0
winSize = int(250e3/(freq[1]-freq[0]))
winSize += ((winSize+1)%2)
for i in xrange(smth.size):
mn = max([0, i-winSize//2])
mx = min([i+winSize//2+1, smth.size])
smth[i] = numpy.median(spec[mn:mx])
smth /= robust.mean(smth)
bp = spec / smth
good = numpy.where( (smth > 0.1) & (numpy.abs(bp-robust.mean(bp)) < 3*robust.std(bp)) )[0]
nBad = nFreq - len(good)
print("Masking %i of %i channels (%.1f%%)" % (nBad, nFreq, 100.0*nBad/nFreq))
if args.plot:
fig = plt.figure()
ax = fig.gca()
ax.plot(freq/1e6, numpy.log10(spec)*10)
ax.plot(freq[good]/1e6, numpy.log10(spec[good])*10)
ax.set_title('Mean Visibility Amplitude')
ax.set_xlabel('Frequency [MHz]')
ax.set_ylabel('PSD [arb. dB]')
plt.draw()
freq2 = freq*1.0
freq2.shape += (1,)
dTimes2 = dTimes*1.0
dTimes2.shape += (1,)
# NOTE: Assumed linear data
polMapper = {'XX':0, 'YY':1, 'XY':2, 'YX':3}
print("%3s %9s %2s %6s %9s %11s" % ('#', 'BL', 'Pl', 'S/N', 'Delay', 'Rate'))
for b in xrange(len(search_bls)):
bl = search_bls[b]
ant1, ant2 = (bl>>8)&0xFF, bl&0xFF
## Skip over baselines that are not in the baseline list (if provided)
if args.baseline is not None:
if (ant1, ant2) not in args.baseline:
continue
## Skip over baselines that don't include the reference antenna
elif ant1 != args.ref_ant and ant2 != args.ref_ant:
continue
## Check and see if we need to conjugate the visibility, i.e., switch from
## baseline (*,ref) to baseline (ref,*)
doConj = False
if ant2 == args.ref_ant:
doConj = True
## Figure out which polarizations to process
if ant1 not in (51, 52) and ant2 not in (51, 52):
### Standard VLA-VLA baseline
polToUse = ('XX', 'YY')
else:
### LWA-LWA or LWA-VLA baseline
if args.y_only:
polToUse = ('YX', 'YY')
else:
polToUse = ('XX', 'XY', 'YX', 'YY')
if args.plot:
fig = plt.figure()
axs = {}
axs['XX'] = fig.add_subplot(2, 2, 1)
axs['YY'] = fig.add_subplot(2, 2, 2)
axs['XY'] = fig.add_subplot(2, 2, 3)
axs['YX'] = fig.add_subplot(2, 2, 4)
valid = numpy.where( bls == bl )[0]
for pol in polToUse:
subData = flux[valid,0,:,polMapper[pol]]*1.0
subData = subData[:,good]
if doConj:
subData = subData.conj()
subData = numpy.dot(subData, numpy.exp(-2j*numpy.pi*freq2[good,:]*delay))
subData /= freq2[good,:].size
amp = numpy.dot(subData.T, numpy.exp(-2j*numpy.pi*dTimes2*drate))
amp = numpy.abs(amp / dTimes2.size)
blName = (ant1, ant2)
if doConj:
blName = (ant2, ant1)
blName = '%s-%s' % ('EA%02i' % blName[0] if blName[0] < 51 else 'LWA%i' % (blName[0]-50),
'EA%02i' % blName[1] if blName[1] < 51 else 'LWA%i' % (blName[1]-50))
best = numpy.where( amp == amp.max() )
if amp.max() > 0:
bsnr = (amp[best]-amp.mean())[0]/amp.std()
bdly = delay[best[0][0]]*1e6
brat = drate[best[1][0]]*1e3
print("%3i %9s %2s %6.2f %6.2f us %7.2f mHz" % (b, blName, pol, bsnr, bdly, brat))
else:
print("%3i %9s %2s %6s %9s %11s" % (b, blName, pol, '----', '----', '----'))
if args.plot:
axs[pol].imshow(amp, origin='lower', interpolation='nearest',
extent=(drate[0]*1e3, drate[-1]*1e3, delay[0]*1e6, delay[-1]*1e6),
cmap='gray_r')
axs[pol].plot(drate[best[1][0]]*1e3, delay[best[0][0]]*1e6, linestyle='', marker='x', color='r', ms=15, alpha=0.75)
if args.plot:
fig.suptitle(os.path.basename(args.filename))
for pol in axs.keys():
ax = axs[pol]
ax.axis('auto')
ax.set_title(pol)
ax.set_xlabel('Rate [mHz]')
ax.set_ylabel('Delay [$\\mu$s]')
fig.suptitle("%s" % blName)
plt.draw()
if args.plot:
plt.show()
def estimateClipLevel(fh, beampols):
"""
Read in a set of 100 frames and come up with the 4-sigma clip levels
for each tuning. These clip levels are returned as a two-element
tuple.
"""
filePos = fh.tell()
# Read in the first 100 frames for each tuning/polarization
count = {0: 0, 1: 0, 2: 0, 3: 0}
data = numpy.zeros((4, 4096 * 100), dtype=numpy.csingle)
for i in xrange(beampols * 100):
try:
cFrame = drx.readFrame(fh, Verbose=False)
except errors.eofError:
break
except errors.syncError:
continue
beam, tune, pol = cFrame.parseID()
aStand = 2 * (tune - 1) + pol
data[aStand,
count[aStand] * 4096:(count[aStand] + 1) * 4096] = cFrame.data.iq
count[aStand] += 1
# Go back to where we started
fh.seek(filePos)
# Compute the robust mean and standard deviation for I and Q for each
# tuning/polarization
meanI = []
meanQ = []
stdsI = []
stdsQ = []
for i in xrange(4):
meanI.append(robust.mean(data[i, :].real))
meanQ.append(robust.mean(data[i, :].imag))
stdsI.append(robust.std(data[i, :].real))
stdsQ.append(robust.std(data[i, :].imag))
# Report
print "Statistics:"
for i in xrange(4):
print " Mean %i: %.3f + %.3f j" % (i + 1, meanI[i], meanQ[i])
print " Std %i: %.3f + %.3f j" % (i + 1, stdsI[i], stdsQ[i])
# Come up with the clip levels based on 4 sigma
clip1 = (meanI[0] + meanI[1] + meanQ[0] + meanQ[1]) / 4.0
clip2 = (meanI[2] + meanI[3] + meanQ[2] + meanQ[3]) / 4.0
clip1 += 5 * (stdsI[0] + stdsI[1] + stdsQ[0] + stdsQ[1]) / 4.0
clip2 += 5 * (stdsI[2] + stdsI[3] + stdsQ[2] + stdsQ[3]) / 4.0
clip1 = int(round(clip1))
clip2 = int(round(clip2))
# Report again
print "Clip Levels:"
print " Tuning 1: %i" % clip1
print " Tuning 2: %i" % clip2
return clip1, clip2