def m5stat(fn, fmt, nframes, offset):
"""Reports statistics for file"""
# Open file
try:
m5file = m5lib.new_mark5_stream_file(fn, ctypes.c_longlong(offset))
m5fmt = m5lib.new_mark5_format_generic_from_string(fmt)
ms = m5lib.new_mark5_stream_absorb(m5file, m5fmt)
dms = ms.contents
except:
print ("Error: problem opening or decoding %s\n" % (fn))
return 1
# Read sample data
nsamples = dms.framesamples*nframes
pdata = m5lib.helpers.make_decoder_array(ms, nsamples, dtype=ctypes.c_float)
m5lib.mark5_stream_decode(ms, nsamples, pdata)
# Statistics
A = 8.0*(numpy.pi-3.0) / (3.0*numpy.pi*(4.0-numpy.pi))
eps = 0.1
hist_limits = [[], # histogram ranges for
[-1.0-eps,0,+1.0+eps], # 1-bit data
[-3.3359-eps,-1.0-eps,0,+1.0+eps,+3.3359+eps], # 2-bit data
[-1.0-eps,0-eps,0+eps,+1.0-eps,+1.0+eps], # 3-bit data
[x/2.95-eps for x in range(-8,8+1)], # 4-bit data
[], [], [],
[(x*2-255)/71.0-eps for x in range(0,255+1)] ] # 8-bit data
print (' Ch mean std skew kurt gfact state counts from -HiMag to +HiMag')
for i in range(dms.nchan):
d = pdata[i][0:nsamples]
m0 = numpy.mean(d)
m1 = numpy.std(d)
m2 = stats.skew(d)
m3 = stats.kurtosis(d, fisher=False, bias=False)
(hcounts,hlimits) = numpy.histogram(d, bins=hist_limits[dms.nbit])
hpdf = [x/float(sum(hcounts)) for x in hcounts]
if dms.nbit==2:
x = hpdf[1] + hpdf[2]
else:
x = hpdf[0] + hpdf[1]
if dms.nbit==2 or dms.nbit==1:
k = numpy.log(1.0-x*x)
g = numpy.sqrt( -4.0/(A*numpy.pi) - k
+ 2.0*numpy.sqrt( 2.0**(2.0/(A*numpy.pi)+0.5*k) - k/A)) / 0.91
else:
g = float('NaN')
hstr = ' / '.join(['%4.1f' % (100.0*x) for x in hpdf])
print ('%3d %+6.2f %+6.2f %+5.2f %+5.2f %+6.2f %s' % (i,m0,m1,m2,m3,g,hstr))
print 'based on %u samples (%u invalid)' % (dms.framesamples*(dms.nvalidatepass+dms.nvalidatefail),dms.framesamples*dms.nvalidatefail)
return 0
def m5time(fn, fmt, offset):
"""Reports first timestamp in file and verifies additional timestamps are consistent."""
try:
m5file = m5lib.new_mark5_stream_file(fn, ctypes.c_longlong(offset))
m5fmt = m5lib.new_mark5_format_generic_from_string(fmt)
ms = m5lib.new_mark5_stream_absorb(m5file, m5fmt)
dms = ms.contents
except:
print("Error: problem opening or decoding %s\n" % (fn))
return EXIT_FAILURE
if not (dms.format in fmt_supported):
print('File format is not supported by m5ime.py')
return EXIT_FAILURE
mjd = dms.mjd
sec = dms.sec + 1e-9 * dms.ns
now = datetime.utcnow()
now_mjd = date_to_mjd(now)
if dms.format in [m5lib.MK5_FORMAT_VLBA, m5lib.MK5_FORMAT_MARK5B]:
mjd = mjd + (now_mjd - (now_mjd % 1000))
if (mjd > now_mjd):
mjd = mjd - 1000
(r, ss) = divmod(sec, 60)
(hh, mm) = divmod(r, 60)
mjd_curr = ms.contents.mjd
for ii in range(Ncheck):
m5lib.mark5_stream_next_frame(ms)
mjd_next = ms.contents.mjd
if ((mjd_next - mjd_curr) >= 2) or (mjd_curr > now_mjd):
print(
'Possibly wrong file format specified : MJD in frame %d jumped from %d to %d.'
% (ii, mjd_curr, mjd_next))
return EXIT_FAILURE
mjd_curr = mjd_next
print('MJD = %d/%02d:%02d:%05.2f' % (mjd, hh, mm, ss))
return 0
def m5subband(fn, fmt, fout, if_nr, factor, Ldft, start_bin, stop_bin, offset):
"""Extracts narrow-band signal out from file"""
# Derived settings
nin = Ldft
nout = stop_bin - start_bin + 1
#Lout = next_pow2(2*(nout-nout%2)) # time-domain output data will be somewhat oversampled
Lout = next_even(
2 * (nout - nout % 2)
) # time-domain output data will be closer to critically sampled
iter = 0
# Open file
try:
m5file = m5lib.new_mark5_stream_file(fn, ctypes.c_longlong(offset))
m5fmt = m5lib.new_mark5_format_generic_from_string(fmt)
ms = m5lib.new_mark5_stream_absorb(m5file, m5fmt)
dms = ms.contents
m5lib.mark5_stream_fix_mjd(ms, refMJD_Mark5B)
(mjd, sec, ns) = m5lib.helpers.get_sample_time(ms)
except:
print('Error: problem opening or decoding %s\n' % (fn))
return 1
# Safety checks
if (if_nr < 0) or (if_nr >= dms.nchan) or (factor < 0) or (
factor > 32) or (Ldft < 2) or (start_bin > stop_bin) or (stop_bin
>= Ldft):
print('Error: invalid command line arguments')
return 1
if (Ldft % factor) > 0:
print(
'Error: length of DFT (Ldft=%u) must be divisible by overlap-add factor (factor=%u)'
% (Ldft, factor))
return 1
if (Lout % factor) > 0:
print(
'Error: length derived for output IDFT (Lout=%u) does not divide the overlap-add factor (factor=%u)'
% (Lout, factor))
return 1
# Get storage for raw sample data from m5lib.mark5_stream_decode()
pdata = m5lib.helpers.make_decoder_array(ms, nin, dtype=ctypes.c_float)
if_data = ctypes.cast(pdata[if_nr], ctypes.POINTER(ctypes.c_float * nin))
# Numpy 2D arrays for processed data
fp = 'float32'
cp = 'complex64' # complex64 is 2 x float32
flt_in = numpy.zeros(shape=(factor, nin), dtype=fp)
flt_out = numpy.zeros(shape=(factor, Lout), dtype=cp)
iconcat = numpy.array([0.0 for x in range(2 * nin)], dtype=fp)
oconcat = numpy.array([0.0 + 0.0j for x in range(2 * Lout)], dtype=cp)
# Coefficient for coherent phase connection between overlapped input segments
r = float(start_bin) / float(factor)
rfrac = r - numpy.floor(r)
rot_f0 = numpy.exp(2j * numpy.pi * rfrac)
if (abs(numpy.imag(rot_f0)) < 1e-5):
# set near-zero values to zero
rot_f0 = numpy.real(rot_f0) + 0.0j
rot_f = rot_f0**0.0
# Window functions for DFT and IDFT
win_in = numpy.cos(
(numpy.pi / nin) * (numpy.linspace(0, nin - 1, nin) - 0.5 * (nin - 1)))
win_in = numpy.resize(win_in.astype(fp), new_shape=(factor, nin))
win_out = numpy.cos((numpy.pi / Lout) *
(numpy.linspace(0, Lout - 1, Lout) - 0.5 * (Lout - 1)))
win_out = numpy.resize(win_out.astype(fp), new_shape=(factor, Lout))
# Prepare VDIF output file with reduced data rate and same starting timestamp
bwout = float(dms.samprate) * (nout / float(nin))
fsout = 2 * bwout
outMbps = fsout * 1e-6 * 32 # 32 for real-valued data, 64 for complex data
vdiffmt = 'VDIF_8192-%u-1-32' % (outMbps)
if not (int(outMbps) == outMbps):
print('*** Warning: output rate is non-integer (%e Ms/s)! ***' %
(outMbps))
(vdifref,
vdifsec) = m5lib.helpers.get_VDIF_time_from_MJD(mjd, sec + 1e-9 * ns)
vdif = m5lib.writers.VDIFEncapsulator()
vdif.open(fout, format=vdiffmt, complex=False, station='SB')
vdif.set_time(vdifref, vdifsec, framenr=0)
vdiffmt = vdif.get_format()
# Report
bw = float(dms.samprate) * 0.5
print('Input file : start MJD %u/%.6f sec' % (mjd, sec + ns * 1e-9))
print(
'Bandwidth : %u kHz in, %.2f kHz out, bandwidth reduction of ~%.2f:1'
% (1e-3 * bw, nout * 1e-3 * bw / nin, float(nin) / nout))
print('Input side : %u-point DFT with %u bins (%u...%u) extracted' %
(nin, nout, start_bin, stop_bin))
print('Output side : %u-point IDFT with %u-point zero padding' %
(Lout, Lout - nout))
print('Overlap : %u samples on input, %u samples on output' %
(nin - nin / factor, Lout - Lout / factor))
print('Phasors : %s^t : %s ...' %
(str(rot_f0), str([rot_f0**t for t in range(factor + 2)])))
print('Output file : rate %.3f Mbps, %u fps, format %s' %
(outMbps, vdif.get_fps(), vdif.get_format()))
# Do filtering
print('Filtering...')
while True:
# Get next full slice of data
rc = m5lib.mark5_stream_decode(ms, nin, pdata)
if (rc < 0):
print('\n<EOF> status=%d' % (rc))
return 0
in_new = numpy.frombuffer(if_data.contents, dtype='float32')
# Debug: replace data with noise + tone
if False:
t = iter * nin + numpy.array(range(nin))
f = (start_bin + numpy.floor(nout / 2.0)) / float(nin)
in_new = numpy.random.standard_normal(
size=in_new.size) + 10 * numpy.sin(2 * numpy.pi * f * t)
in_new = in_new.astype('float32')
# Feed the window-overlap-DFT processing input stage
iconcat = numpy.concatenate([iconcat[0:nin], in_new]) # [old,new]
for ii in range(factor):
iconcat = numpy.roll(iconcat, -nin / factor)
flt_in[ii] = iconcat[0:nin]
# Window and do 1D DFT of 2D array
flt_in = numpy.multiply(flt_in, win_in)
F = numpy.fft.fft(flt_in)
# Copy the desired bins and fix DC/Nyquist bins
for ii in range(factor):
flt_out[ii][0:nout] = F[ii][start_bin:(start_bin + nout)]
flt_out[ii][0] = 0.0 # numpy.real(flt_out[ii][0])
flt_out[ii][nout - 1] = 0.0 # numpy.real(flt_out[ii][nout-1])
# Do inverse 1D DFT and window the result
F = numpy.fft.ifft(flt_out)
F = numpy.multiply(F, win_out)
# Reconstruct time domain signal by shifting and stacking overlapped segments coherently
for ii in range(factor):
oconcat[Lout:] = oconcat[Lout:] + F[ii] * rot_f
rot_f = rot_f * rot_f0
oconcat = numpy.roll(oconcat, -Lout / factor)
# note: numpy has a circular shift (numpy.roll), but no "shift array left/right" function,
# so we need to zero out the undesired values shifted back in by the circular shift:
oconcat[(-Lout / factor):] = 0
# Output real part of complex time domain data
# (If suppression of upper Nyquist is zone desired, should write out both real&imag)
vdif.write(numpy.real(oconcat[0:Lout]).view('float32').tostring())
# Reporting
if (iter % 100) == 0:
(mjd, sec, ns) = m5lib.helpers.get_sample_time(ms)
T_abs = sec + 1e-9 * ns
T_count = 1e-9 * dms.framens * dms.nvalidatepass
print('Iter %7d : %u/%f : %u : %f sec\r' %
(iter, mjd, T_abs, dms.nvalidatepass, T_count)),
iter = iter + 1
vdif.close()
return 0
def m5subband(fn, fmt, fout, if_nr, factor, Ldft, start_bin, stop_bin, offset):
"""Extracts narrow-band signal out from file"""
# Derived settings
nin = Ldft
nout = stop_bin - start_bin + 1
#Lout = next_pow2(2*(nout-nout%2)) # time-domain output data will be somewhat oversampled
Lout = next_even(2*(nout-nout%2)) # time-domain output data will be closer to critically sampled
iter = 0
# Open file
try:
m5file = m5lib.new_mark5_stream_file(fn, ctypes.c_longlong(offset))
m5fmt = m5lib.new_mark5_format_generic_from_string(fmt)
ms = m5lib.new_mark5_stream_absorb(m5file, m5fmt)
dms = ms.contents
m5lib.mark5_stream_fix_mjd(ms, refMJD_Mark5B)
(mjd,sec,ns) = m5lib.helpers.get_sample_time(ms)
except:
print ('Error: problem opening or decoding %s\n' % (fn))
return 1
# Safety checks
if (if_nr<0) or (if_nr>=dms.nchan) or (factor<0) or (factor>32) or (Ldft<2) or (start_bin>stop_bin) or (stop_bin>=Ldft):
print ('Error: invalid command line arguments')
return 1
if (Ldft % factor)>0:
print ('Error: length of DFT (Ldft=%u) must be divisible by overlap-add factor (factor=%u)' % (Ldft,factor))
return 1
if (Lout % factor)>0:
print ('Error: length derived for output IDFT (Lout=%u) does not divide the overlap-add factor (factor=%u)' % (Lout,factor))
return 1
# Get storage for raw sample data from m5lib.mark5_stream_decode()
pdata = m5lib.helpers.make_decoder_array(ms, nin, dtype=ctypes.c_float)
if_data = ctypes.cast(pdata[if_nr], ctypes.POINTER(ctypes.c_float*nin))
# Numpy 2D arrays for processed data
fp = 'float32'
cp = 'complex64' # complex64 is 2 x float32
flt_in = numpy.zeros(shape=(factor,nin), dtype=fp)
flt_out = numpy.zeros(shape=(factor,Lout), dtype=cp)
iconcat = numpy.array([0.0 for x in range(2*nin)], dtype=fp)
oconcat = numpy.array([0.0+0.0j for x in range(2*Lout)], dtype=cp)
# Coefficient for coherent phase connection between overlapped input segments
r = float(start_bin)/float(factor)
rfrac = r - numpy.floor(r)
rot_f0 = numpy.exp(2j*numpy.pi*rfrac)
if (abs(numpy.imag(rot_f0)) < 1e-5):
# set near-zero values to zero
rot_f0 = numpy.real(rot_f0) + 0.0j
rot_f = rot_f0**0.0
# Window functions for DFT and IDFT
win_in = numpy.cos((numpy.pi/nin)*(numpy.linspace(0,nin-1,nin) - 0.5*(nin-1)))
win_in = numpy.resize(win_in.astype(fp), new_shape=(factor,nin))
win_out = numpy.cos((numpy.pi/Lout)*(numpy.linspace(0,Lout-1,Lout) - 0.5*(Lout-1)))
win_out = numpy.resize(win_out.astype(fp), new_shape=(factor,Lout))
# Prepare VDIF output file with reduced data rate and same starting timestamp
bwout = float(dms.samprate)*(nout/float(nin))
fsout = 2*bwout
outMbps = fsout*1e-6 * 32 # 32 for real-valued data, 64 for complex data
vdiffmt = 'VDIF_8192-%u-1-32' % (outMbps)
if not(int(outMbps) == outMbps):
print ('*** Warning: output rate is non-integer (%e Ms/s)! ***' % (outMbps))
(vdifref,vdifsec) = m5lib.helpers.get_VDIF_time_from_MJD(mjd,sec+1e-9*ns)
vdif = m5lib.writers.VDIFEncapsulator()
vdif.open(fout, format=vdiffmt, complex=False, station='SB')
vdif.set_time(vdifref,vdifsec, framenr=0)
vdiffmt = vdif.get_format()
# Report
bw = float(dms.samprate)*0.5
print ('Input file : start MJD %u/%.6f sec' % (mjd,sec+ns*1e-9))
print ('Bandwidth : %u kHz in, %.2f kHz out, bandwidth reduction of ~%.2f:1' % (1e-3*bw, nout*1e-3*bw/nin, float(nin)/nout))
print ('Input side : %u-point DFT with %u bins (%u...%u) extracted' % (nin,nout,start_bin,stop_bin))
print ('Output side : %u-point IDFT with %u-point zero padding' % (Lout,Lout-nout))
print ('Overlap : %u samples on input, %u samples on output' % (nin-nin/factor,Lout-Lout/factor))
print ('Phasors : %s^t : %s ...' % (str(rot_f0), str([rot_f0**t for t in range(factor+2)])))
print ('Output file : rate %.3f Mbps, %u fps, format %s'
% (outMbps,vdif.get_fps(),vdif.get_format()) )
# Do filtering
print ('Filtering...')
while True:
# Get next full slice of data
rc = m5lib.mark5_stream_decode(ms, nin, pdata)
if (rc < 0):
print ('\n<EOF> status=%d' % (rc))
return 0
in_new = numpy.frombuffer(if_data.contents, dtype='float32')
# Debug: replace data with noise + tone
if False:
t = iter*nin + numpy.array(range(nin))
f = (start_bin + numpy.floor(nout/2.0)) / float(nin)
in_new = numpy.random.standard_normal(size=in_new.size) + 10*numpy.sin(2*numpy.pi * f*t)
in_new = in_new.astype('float32')
# Feed the window-overlap-DFT processing input stage
iconcat = numpy.concatenate([iconcat[0:nin],in_new]) # [old,new]
for ii in range(factor):
iconcat = numpy.roll(iconcat, -nin/factor)
flt_in[ii] = iconcat[0:nin]
# Window and do 1D DFT of 2D array
flt_in = numpy.multiply(flt_in,win_in)
F = numpy.fft.fft(flt_in)
# Copy the desired bins and fix DC/Nyquist bins
for ii in range(factor):
flt_out[ii][0:nout] = F[ii][start_bin:(start_bin+nout)]
flt_out[ii][0] = 0.0 # numpy.real(flt_out[ii][0])
flt_out[ii][nout-1] = 0.0 # numpy.real(flt_out[ii][nout-1])
# Do inverse 1D DFT and window the result
F = numpy.fft.ifft(flt_out)
F = numpy.multiply(F,win_out)
# Reconstruct time domain signal by shifting and stacking overlapped segments coherently
for ii in range(factor):
oconcat[Lout:] = oconcat[Lout:] + F[ii]*rot_f
rot_f = rot_f * rot_f0
oconcat = numpy.roll(oconcat, -Lout/factor)
# note: numpy has a circular shift (numpy.roll), but no "shift array left/right" function,
# so we need to zero out the undesired values shifted back in by the circular shift:
oconcat[(-Lout/factor):] = 0
# Output real part of complex time domain data
# (If suppression of upper Nyquist is zone desired, should write out both real&imag)
vdif.write(numpy.real(oconcat[0:Lout]).view('float32').tostring())
# Reporting
if (iter % 100)==0:
(mjd,sec,ns) = m5lib.helpers.get_sample_time(ms)
T_abs = sec + 1e-9*ns
T_count = 1e-9*dms.framens * dms.nvalidatepass
print ('Iter %7d : %u/%f : %u : %f sec\r' % (iter, mjd,T_abs, dms.nvalidatepass, T_count)),
iter = iter + 1
vdif.close()
return 0
def m5selfcorr(fn, fmt):
Nint = 500
nfft = 1024
offset = 0
# Open file
try:
m5file = m5lib.new_mark5_stream_file(fn, ctypes.c_longlong(offset))
m5fmt = m5lib.new_mark5_format_generic_from_string(fmt)
ms = m5lib.new_mark5_stream_absorb(m5file, m5fmt)
dms = ms.contents
except:
print ('Error: problem opening or decoding %s using format %s\n' % (fn,fmt))
return 1
basename = os.path.basename(fn)
basename = os.path.splitext(basename)[0]
# Collection of vectors for mark5access decode() raw sample output data
pdata = m5lib.helpers.make_decoder_array(ms, nfft, dtype=ctypes.c_float)
# Result arrays
ch_data = [ctypes.cast(pdata[ii], ctypes.POINTER(ctypes.c_float*nfft)) for ii in range(dms.nchan)]
xspecs = numpy.zeros(shape=(dms.nchan*dms.nchan,nfft), dtype='complex128')
# Process the recorded data
iter = 0
while (iter < Nint):
iter = iter + 1
# Read data
rc = m5lib.mark5_stream_decode(ms, nfft, pdata)
if (rc < 0):
print ('\n<EOF> status=%d' % (rc))
return 0
# Calculate normalized cross-correlations
for ii in range(dms.nchan):
for jj in range(dms.nchan):
# print ('Computing pair %d--%d...' % (ii,jj))
k = ii*dms.nchan + jj
a = numpy.frombuffer(ch_data[ii].contents, dtype='float32')
b = numpy.frombuffer(ch_data[jj].contents, dtype='float32')
A = numpy.fft.fft(a)
B = numpy.fft.fft(b)
xspecs[k] = numpy.add(xspecs[k], A*numpy.conj(B))
# Calculate lag spectra
xspecs /= Nint*nfft
lagspecs = numpy.zeros_like(xspecs)
normspecs = numpy.zeros_like(xspecs)
for ii in range(dms.nchan):
for jj in range(dms.nchan):
k = ii*dms.nchan + jj
xc = numpy.fft.ifft(xspecs[k])
lagspecs[k] = numpy.fft.fftshift(xc)
xc = xspecs[k]
ac1 = xspecs[ii*dms.nchan + ii]
ac2 = xspecs[jj*dms.nchan + jj]
norm = numpy.sqrt(numpy.multiply(ac1,ac2))
normspecs[k] = numpy.divide(xc, norm)
figsize = (5*dms.nchan,5*dms.nchan)
# Show lag spectra
ymin = numpy.amin(numpy.abs(lagspecs))
ymax = numpy.amax(numpy.abs(lagspecs))
pylab.figure(figsize=figsize,dpi=75)
for ii in range(dms.nchan):
for jj in range(dms.nchan):
if (jj < ii):
continue
print ('Plotting lag spec. %d--%d' % (ii,jj))
if (ii == jj):
linespec = 'kx-'
else:
linespec = 'bx-'
k = ii*dms.nchan + jj
pylab.subplot(dms.nchan, dms.nchan, k+1)
pylab.ylim([ymin,ymax])
pylab.xlim([0,nfft-1])
pylab.plot(numpy.abs(lagspecs[k]), linespec)
pylab.title('IF %d x %d' % (ii+1,jj+1))
if (ii==jj):
pylab.xlabel('Lag')
pylab.ylabel('Cross-corr.')
pylab.savefig(basename + '.lag.png')
# Show normalized xcorr spectra: phase
pylab.figure(figsize=figsize,dpi=75)
for ii in range(dms.nchan):
for jj in range(dms.nchan):
if (jj < ii):
continue
print ('Plotting xcorr spec. phase %d--%d' % (ii,jj))
if (ii == jj):
linespec = 'kx'
else:
linespec = 'bx'
k = ii*dms.nchan + jj
pylab.subplot(dms.nchan, dms.nchan, k+1)
pylab.plot(numpy.angle(normspecs[k],deg=True), linespec)
#pylab.ylim([-numpy.pi,+numpy.pi])
pylab.ylim([-180.0,+180.0])
pylab.xlim([0,nfft-1])
pylab.title('IF %d x %d' % (ii+1,jj+1))
if (ii==jj):
pylab.xlabel('FFT bin')
pylab.ylabel('Phase (deg)')
pylab.savefig(basename + '.phase.png')
# Show normalized xcorr spectra: amp
ymin = numpy.amin(numpy.abs(normspecs))
ymax = numpy.amax(numpy.abs(normspecs))
pylab.figure(figsize=figsize,dpi=75)
for ii in range(dms.nchan):
for jj in range(dms.nchan):
if (jj < ii):
continue
print ('Plotting xcorr spec. amp %d--%d' % (ii,jj))
if (ii == jj):
linespec = 'kx:'
else:
linespec = 'bx:'
k = ii*dms.nchan + jj
pylab.subplot(dms.nchan, dms.nchan, k+1)
pylab.plot(numpy.abs(normspecs[k]), linespec)
pylab.ylim([ymin,ymax])
pylab.xlim([0,nfft-1])
pylab.title('IF %d x %d' % (ii+1,jj+1))
if (ii==jj):
pylab.xlabel('FFT bin')
pylab.ylabel('Coherence')
pylab.savefig(basename + '.coherence.png')
# Show autocorrelations (without normalization)
pylab.figure(figsize=figsize,dpi=75)
Nnyq = int(nfft/2+1)
for ii in range(dms.nchan):
k = ii*dms.nchan + ii
print ('Plotting auto power spec. %d' % (ii))
pylab.subplot(dms.nchan,1, ii+1)
ac = numpy.abs(xspecs[k])
pylab.semilogy(ac[0:Nnyq], 'k-')
pylab.xlim([0,Nnyq-1])
pylab.title('Auto power spectrum IF %d' % (ii+1))
if ii==(dms.nchan-1):
pylab.xlabel('FFT bin')
pylab.savefig(basename + '.autospec.png')
pylab.show()
def m5stat(fn, fmt, nframes, offset):
"""Reports statistics for file"""
# Open file
try:
m5file = m5lib.new_mark5_stream_file(fn, ctypes.c_longlong(offset))
m5fmt = m5lib.new_mark5_format_generic_from_string(fmt)
ms = m5lib.new_mark5_stream_absorb(m5file, m5fmt)
dms = ms.contents
except:
print("Error: problem opening or decoding %s\n" % (fn))
return 1
# Read sample data
nsamples = dms.framesamples * nframes
pdata = m5lib.helpers.make_decoder_array(ms,
nsamples,
dtype=ctypes.c_float)
m5lib.mark5_stream_decode(ms, nsamples, pdata)
# Statistics
A = 8.0 * (numpy.pi - 3.0) / (3.0 * numpy.pi * (4.0 - numpy.pi))
eps = 0.1
hist_limits = [
[], # histogram ranges for
[-1.0 - eps, 0, +1.0 + eps], # 1-bit data
[-3.3359 - eps, -1.0 - eps, 0, +1.0 + eps,
+3.3359 + eps], # 2-bit data
[-1.0 - eps, 0 - eps, 0 + eps, +1.0 - eps, +1.0 + eps], # 3-bit data
[x / 2.95 - eps for x in range(-8, 8 + 1)], # 4-bit data
[],
[],
[],
[(x * 2 - 255) / 71.0 - eps for x in range(0, 255 + 1)]
] # 8-bit data
print(
' Ch mean std skew kurt gfact state counts from -HiMag to +HiMag'
)
for i in range(dms.nchan):
d = pdata[i][0:nsamples]
m0 = numpy.mean(d)
m1 = numpy.std(d)
m2 = stats.skew(d)
m3 = stats.kurtosis(d, fisher=False, bias=False)
(hcounts, hlimits) = numpy.histogram(d, bins=hist_limits[dms.nbit])
hpdf = [x / float(sum(hcounts)) for x in hcounts]
if dms.nbit == 2:
x = hpdf[1] + hpdf[2]
else:
x = hpdf[0] + hpdf[1]
if dms.nbit == 2 or dms.nbit == 1:
k = numpy.log(1.0 - x * x)
g = numpy.sqrt(-4.0 / (A * numpy.pi) - k +
2.0 * numpy.sqrt(2.0**(2.0 /
(A * numpy.pi) + 0.5 * k) -
k / A)) / 0.91
else:
g = float('NaN')
hstr = ' / '.join(['%4.1f' % (100.0 * x) for x in hpdf])
print('%3d %+6.2f %+6.2f %+5.2f %+5.2f %+6.2f %s' %
(i, m0, m1, m2, m3, g, hstr))
print 'based on %u samples (%u invalid)' % (
dms.framesamples * (dms.nvalidatepass + dms.nvalidatefail),
dms.framesamples * dms.nvalidatefail)
return 0
def m5spec(fn, fmt, fout, T_int_ms, nfft, offset):
"""Form time-averaged autocorrelation spectra"""
# Open file
try:
m5file = m5lib.new_mark5_stream_file(fn, ctypes.c_longlong(offset))
m5fmt = m5lib.new_mark5_format_generic_from_string(fmt)
ms = m5lib.new_mark5_stream_absorb(m5file, m5fmt)
dms = ms.contents
except:
print('Error: problem opening or decoding %s\n' % (fn))
return 1
# Settings
nint = numpy.round(float(dms.samprate) * T_int_ms * 1e-3 / float(nfft))
Tint = float(nint * nfft) / float(dms.samprate)
df = float(dms.samprate) / float(nfft)
iter = 0
print(
'Averaging a total of %u DFTs, each with %u points, for a %f millisecond time average.'
% (nint, nfft, Tint * 1e3))
# Collection of vectors for mark5access decode() raw sample output data
pdata = m5lib.helpers.make_decoder_array(ms, nfft, dtype=ctypes.c_float)
# Result arrays
ch_data = [
ctypes.cast(pdata[ii], ctypes.POINTER(ctypes.c_float * nfft))
for ii in range(dms.nchan)
]
freqs = numpy.linspace(0.0, dms.samprate * 1e-6, num=nfft)
specs = numpy.zeros(shape=(dms.nchan, nfft), dtype='float64')
# Process the recorded data
while True:
# Read data
rc = m5lib.mark5_stream_decode(ms, nfft, pdata)
if (rc < 0):
print('\n<EOF> status=%d' % (rc))
return 0
# Averaging of 'Abs(FFT(x))'
for ii in range(dms.nchan):
x = numpy.frombuffer(ch_data[ii].contents, dtype='float32')
specs[ii] += numpy.abs(numpy.fft.fft(x))
# Save data and plot at the end of an averaging period
iter = iter + 1
if (iter % nint) == 0:
layouts = [
(None, None),
(1, 1),
(1, 2),
(1, 3),
(2, 2),
(1, 5), # 1 to 5 channels
(2, 3),
(2, 4),
(2, 4),
(3, 3),
(2, 5),
(3, 4),
(2, 6), # 6 to 12 channels
(3, 5),
(3, 5),
(3, 5),
(4, 4)
] # 13 to 16 channels
N = int(numpy.floor(float(nfft) / 2 + 1))
M = int(numpy.min([16, dms.nchan]))
rows, cols = layouts[M]
f = freqs[0:N]
s = specs[:, 0:N]
s /= float(nint)
s[:, 0] = s[:, 1]
s[:, -1] = s[:, -2]
writeOut(fout, f, s)
a_min = numpy.amin(s)
a_max = numpy.amax(s)
pylab.gcf().set_facecolor('white')
for ch in range(M):
pylab.subplot(rows, cols, ch + 1)
pylab.plot(f, abs(s[ch]), 'k-')
pylab.title('Channel %d' % (ch + 1))
pylab.axis('tight')
pylab.ylim([a_min, a_max])
if ch >= (M - cols):
pylab.xlabel('Frequency (MHz)')
else:
pylab.gca().set_xticklabels([])
if (ch % rows) == 0:
pylab.ylabel('Amplitude (Sum|FFT(x)|)')
else:
pylab.gca().set_yticklabels([])
pylab.show()
# Current version: stop after 1 integration period
break
return 0
def m5subband(fn, fmt, fout, if_nr, factor, start_MHz, stop_MHz, offset):
"""Extracts narrow-band signal out from file"""
# Hard-coded settings
refMJD_Mark5B = 57000 # reference MJD for Mark5B input data
Ldft_out = 64 # number of DFT points for output transform
m5_t = ctypes.c_float # mark5access output type for decoded samples
fp_t = 'float32' # numpy array type for DFT input
cp_t = 'complex64' # numpy array type for DFT output, complex64 is 2 x float32
nbits_out = 8 # quantization (32 or 8 bits) to use in output VDIF file
stats_done = False
out_std = 1.0
# Open the VLBI recording
try:
m5file = m5lib.new_mark5_stream_file(fn, ctypes.c_longlong(offset))
m5fmt = m5lib.new_mark5_format_generic_from_string(fmt)
ms = m5lib.new_mark5_stream_absorb(m5file, m5fmt)
dms = ms.contents
m5lib.mark5_stream_fix_mjd(ms, refMJD_Mark5B)
(mjd, sec, ns) = m5lib.helpers.get_sample_time(ms)
except:
print('Error: problem opening or decoding %s\n' % (fn))
return 1
if (factor < 0) or (factor > 16) or (if_nr < 0) or (if_nr >= dms.nchan):
print('Error: invalid arguments')
return 1
# Derive input-side DFT length
bw_in = float(dms.samprate) * 0.5 * 1e-6
bw_out = numpy.abs(stop_MHz - start_MHz)
R = bw_in / bw_out
Ldft_in = int(R * Ldft_out + 0.5)
radix = [2, 3, 5, 7, 11]
while not (any([(Ldft_in % rx) == 0
for rx in radix])) and not ((Ldft_in % factor) == 0):
Ldft_in = Ldft_in + 1
R = Ldft_in / float(Ldft_out)
start_bin = int(Ldft_in * start_MHz / bw_in)
stop_bin = int(Ldft_in * stop_MHz / bw_in)
start_MHz = (bw_in * start_bin) / Ldft_in
stop_MHz = (bw_in * stop_bin) / Ldft_in
bw_out = numpy.abs(stop_MHz - start_MHz)
Ldft_in = 2 * Ldft_in # due to r2c DFT rather than c2c DFT
# Increase output-side DFT's input length, i.e., use zero padding
Ldft_copy = Ldft_out
#Ldft_out = next_pow2(2*(Ldft_out-Ldft_out%2)) # time-domain output data will be somewhat oversampled
Ldft_out = next_even(
2 * (Ldft_out - Ldft_out % 2)
) # time-domain output data will be closer to critically sampled
print(
'Derived settings : actual extraction range %.3f--%.3f MHz, %d-point DFT bins %d--%d (%d bins) padded to %d-point IDFT\n'
% (start_MHz, stop_MHz, Ldft_in, start_bin, stop_bin, Ldft_copy,
Ldft_out))
# Safety checks
if (Ldft_in < Ldft_out) or (start_bin > stop_bin) or (stop_bin >= Ldft_in):
print('Error: invalid derived settings')
return 1
if (Ldft_in % factor) > 0:
print(
'Error: length of input-side DFT (%u) must be divisible by overlap-add factor (quality factor %u)'
% (Ldft_in, factor))
return 1
if (Ldft_out % factor) > 0:
print(
'Error: length derived for output-side IDFT (%u) does not divide the overlap-add factor (quality factor %u)'
% (Ldft_out, factor))
return 1
# Get storage for raw sample data from m5lib.mark5_stream_decode()
pdata = m5lib.helpers.make_decoder_array(ms, Ldft_in, dtype=m5_t)
if_data = ctypes.cast(pdata[if_nr], ctypes.POINTER(m5_t * Ldft_in))
# Numpy 2D arrays for processed data
flt_in = numpy.zeros(shape=(factor, Ldft_in), dtype=fp_t)
flt_out = numpy.zeros(shape=(factor, Ldft_out), dtype=cp_t)
iconcat = numpy.array([0.0 for x in range(2 * Ldft_in)], dtype=fp_t)
oconcat = numpy.array([0.0 + 0.0j for x in range(2 * Ldft_out)],
dtype=cp_t)
# Coefficient for coherent phase connection between overlapped input segments
r = float(start_bin) / float(factor)
rfrac = r - numpy.floor(r)
rot_f0 = numpy.exp(2j * numpy.pi * rfrac)
if (abs(numpy.imag(rot_f0)) < 1e-5):
# set near-zero values to zero
rot_f0 = numpy.real(rot_f0) + 0.0j
rot_f = rot_f0**0.0
# Window functions for DFT and IDFT
win_in = numpy.cos(
(numpy.pi / Ldft_in) * (numpy.linspace(0, Ldft_in - 1, Ldft_in) - 0.5 *
(Ldft_in - 1)))
win_in = numpy.resize(win_in.astype(fp_t), new_shape=(factor, Ldft_in))
win_out = numpy.cos(
(numpy.pi / Ldft_out) *
(numpy.linspace(0, Ldft_out - 1, Ldft_out) - 0.5 * (Ldft_out - 1)))
win_out = numpy.resize(win_out.astype(fp_t), new_shape=(factor, Ldft_out))
# Prepare VDIF output file with reduced data rate and same starting timestamp
bwout = float(dms.samprate) * (Ldft_copy / float(Ldft_in))
fsout = 2 * bwout
outMbps = fsout * 1e-6 * nbits_out
vdiffmt = 'VDIF_8192-%u-1-%d' % (outMbps, nbits_out)
#if not(int(outMbps) == outMbps):
# print ('*** Warning: output rate is non-integer (%e Ms/s)! ***' % (outMbps))
(vdifref,
vdifsec) = m5lib.helpers.get_VDIF_time_from_MJD(mjd, sec + 1e-9 * ns)
vdif = m5lib.writers.VDIFEncapsulator()
vdif.open(fout, format=vdiffmt, complex=False, station='SB')
vdif.set_time(vdifref, vdifsec, framenr=0)
vdiffmt = vdif.get_format()
# Report
bw = float(dms.samprate) * 0.5
print('Input file : start MJD %u/%.6f sec' % (mjd, sec + ns * 1e-9))
print(
'Bandwidth : %u kHz in, %.2f kHz out, bandwidth reduction of ~%.2f:1'
% (1e-3 * bw, Ldft_copy * 1e-3 * bw / Ldft_in,
float(Ldft_in) / Ldft_copy))
print('Input side : %u-point DFT with %u bins (%u...%u) extracted' %
(Ldft_in, Ldft_copy, start_bin, stop_bin))
print('Output side : %u-point IDFT with %u-point zero padding' %
(Ldft_out, Ldft_out - Ldft_copy))
print('Overlap : %u samples on input, %u samples on output' %
(Ldft_in - Ldft_in / factor, Ldft_out - Ldft_out / factor))
print('Phasors : %s^t : %s ...' %
(str(rot_f0), str([rot_f0**t for t in range(factor + 2)])))
print('Output file : rate %.3f Mbps, %u fps, format %s' %
(outMbps, vdif.get_fps(), vdif.get_format()))
# Do filtering
print('Filtering...')
iter = 0
while True:
# Get next full slice of data
rc = m5lib.mark5_stream_decode(ms, Ldft_in, pdata)
if (rc < 0):
print('\n<EOF> status=%d' % (rc))
return 0
in_new = numpy.frombuffer(if_data.contents, dtype='float32')
# Debug: replace data with noise + tone
if False:
t = iter * Ldft_in + numpy.array(range(Ldft_in))
f = (start_bin + numpy.floor(Ldft_copy / 2.0)) / float(Ldft_in)
in_new = numpy.random.standard_normal(
size=in_new.size) + 10 * numpy.sin(2 * numpy.pi * f * t)
in_new = in_new.astype('float32')
# Feed the window-overlap input-side DFT processing input stage
iconcat = numpy.concatenate([iconcat[0:Ldft_in], in_new]) # [old,new]
for ii in range(factor):
iconcat = numpy.roll(iconcat, -Ldft_in / factor)
flt_in[ii] = iconcat[0:Ldft_in]
# Window and do 1D DFT of 2D array
flt_in = numpy.multiply(flt_in, win_in)
F = numpy.fft.fft(flt_in)
# Copy the desired bins and fix DC/Nyquist bins
for ii in range(factor):
flt_out[ii][0:Ldft_copy] = F[ii][start_bin:(start_bin + Ldft_copy)]
flt_out[ii][0] = 0.0 # numpy.real(flt_out[ii][0])
flt_out[ii][Ldft_copy -
1] = 0.0 # numpy.real(flt_out[ii][Ldft_copy-1])
# Do inverse 1D DFT of 2D array and window the result
F = numpy.fft.ifft(flt_out)
F = numpy.multiply(F, win_out)
# Reconstruct time domain signal by shifting and stacking overlapped segments coherently
for ii in range(factor):
oconcat[Ldft_out:] = oconcat[Ldft_out:] + F[ii] * rot_f
rot_f = rot_f * rot_f0
oconcat = numpy.roll(oconcat, -Ldft_out / factor)
# note: numpy has a circular shift (numpy.roll), but no "shift array left/right" function,
# so we need to zero out the undesired values shifted back in by the circular shift:
oconcat[(-Ldft_out / factor):] = 0
# Output real part of complex time domain data
# (If suppression of upper Nyquist is zone desired, should write out both real&imag)
if not stats_done:
out_std = numpy.std(numpy.real(oconcat[0:Ldft_out]))
stats_done = True
if nbits_out == 32:
raw = numpy.real(oconcat[0:Ldft_out]).view('float32').tostring()
vdif.write(raw)
elif nbits_out == 8:
a = numpy.real(oconcat[0:Ldft_out]) * (127.5 / (8 * out_std))
a8 = a.astype(numpy.int8)
vdif.write(a8.view('int8').tostring())
elif nbits_out == 2:
# TODO: add actual conversion to 4-level (first float to int8, then divide by some integer?)
# TODO: add some "fast" means of packing four 2-bit samples into a byte in Python
# TODO: ensure Ldft_out * 2-bit is still an 8-byte multiple to meet VDIF requirements
pass
# Reporting
if (iter % 100) == 0:
(mjd, sec, ns) = m5lib.helpers.get_sample_time(ms)
T_abs = sec + 1e-9 * ns
T_count = 1e-9 * dms.framens * dms.nvalidatepass
print('Iter %7d : %u/%f : %u : %f sec\r' %
(iter, mjd, T_abs, dms.nvalidatepass, T_count)),
iter = iter + 1
vdif.close()
return 0
def m5tone(fn,
fmt,
fout,
if_nr,
Tint_sec,
tonefreq_Hz,
Ldft,
offset,
doPlot=False,
doFast=True):
"""Extracts a single tone from the desired channel in a VLBI recording"""
# Open file
try:
m5file = m5lib.new_mark5_stream_file(fn, ctypes.c_longlong(offset))
m5fmt = m5lib.new_mark5_format_generic_from_string(fmt)
ms = m5lib.new_mark5_stream_absorb(m5file, m5fmt)
dms = ms.contents
except:
print('Error: problem opening or decoding %s\n' % (fn))
return 1
# Get storage for raw sample data from m5lib.mark5_stream_decode()
pdata = m5lib.helpers.make_decoder_array(ms, Ldft, dtype=ctypes.c_float)
if_data = ctypes.cast(pdata[if_nr - 1],
ctypes.POINTER(ctypes.c_float * Ldft))
# Derived settings
Lnyq = numpy.floor(Ldft / 2 - 1)
nint = numpy.round(float(dms.samprate) * Tint_sec / float(Ldft))
Tint = float(nint * Ldft) / float(dms.samprate)
pcbin = Ldft * float(tonefreq_Hz) / float(dms.samprate)
iter = 0
# Safety checks
if (if_nr < 1) or (if_nr > dms.nchan):
print(
'Error: requested nonexistent channel %d (file has channels 1...%d).'
% (if_nr, dms.nchan))
return 1
if (tonefreq_Hz <= 0) or (tonefreq_Hz >= 0.5 * dms.samprate):
print(
'Error: tone frequency of %u Hz not within channel bandwidth of %.1f Hz.'
% (tonefreq_Hz, 0.5 * dms.samprate))
return 1
if (Ldft < 16):
print('Error: length of DFT (Ldft=%d) must be >=16 points.' % (Ldft))
return 1
if (pcbin != int(pcbin)):
print(
'Error: tone bin is not an integer (bin=%f). Adjust Ldft or frequency.'
% (pcbin))
return 1
# Spectral data
winf = numpy.kaiser(Ldft, 7.0) # Kaiser window function
spec = numpy.zeros(shape=(Ldft), dtype='complex64') # Accumulated spectrum
tavg = numpy.zeros(shape=(Ldft),
dtype='float32') # Accumulated time domain data
pcamp = 0.0 # Total abs amplitude
history = {'amp': [], 'phase': [], 'coh': [], 'T': [], 'MJD': []}
# Plotting
Lsgram = 8
specgram = numpy.zeros(shape=(Lsgram, Lnyq), dtype='complex64')
if doPlot:
pylab.ion()
pylab.figure()
pylab.gcf().set_facecolor('white')
# Report
print(
'Tone at %.3f kHz in the %.3f MHz wide band lands in %.3f kHz-wide bin %u.'
% (tonefreq_Hz * 1e-3, dms.samprate * 0.5e-6,
1e-3 * dms.samprate / float(Ldft), pcbin))
print(
'Integrating for %.2f milliseconds with %u spectra per integration.' %
(Tint * 1e3, nint))
if doFast:
print(
"Note: Using fast ingration, coherence 'r' will not actually be calculated."
)
# Detect tone phase and amplitude
(mjd0, sec0, ns0) = m5lib.helpers.get_frame_time(ms)
while True:
# Get next full slice of data
rc = m5lib.mark5_stream_decode(ms, Ldft, pdata)
if (rc < 0):
print('\n<EOF> status=%d' % (rc))
return 0
dd = numpy.frombuffer(if_data.contents, dtype='float32')
# Extract the tone
if not (doFast):
# Brute force method, benefit is that coherence can be measured
ddwin = numpy.multiply(dd, winf)
F = numpy.fft.fft(ddwin)
spec = numpy.add(spec, F)
pcamp = pcamp + numpy.abs(F[pcbin])
else:
# Faster method, but cannot measure coherence
tavg = numpy.add(tavg, dd)
# Report the result at end of each averaging period
iter = iter + 1
if (iter % nint) == 0:
# Timestamp at mid-point of integration
T_count = Tint * (
(iter / nint) - 0.5) # data-second, relative time
(mjd1, sec1, ns1) = m5lib.helpers.get_sample_time(ms)
T_stamp = mjd1 + (sec1 + ns1 * 1e-9 - Tint /
2.0) / 86400.0 # fractional MJD, absolute time
# Extract final tone amp, phase, coherence
if doFast:
spec = numpy.fft.fft(numpy.multiply(tavg, winf))
pctone = spec[pcbin] / float(nint * Ldft)
pcamp = pcamp / float(nint * Ldft)
pctone_A = numpy.abs(pctone)
pctone_ph = numpy.angle(pctone, deg=True)
if doFast:
pctone_C = 1.0
else:
pctone_C = pctone_A / pcamp
# Store results
print('%.9f mjd : %.6f sec : %e /_ %+.2f deg : r=%.3f' %
(T_stamp, T_count, pctone_A, pctone_ph, pctone_C))
history['amp'].append(pctone_A)
history['phase'].append(pctone_ph)
history['coh'].append(pctone_C)
history['T'].append(T_count)
history['MJD'].append(T_stamp)
line = '%.9f %.6f %e %+.2f %.3f\n' % (T_stamp, T_count, pctone_A,
pctone_ph, pctone_C)
fout.write(line)
# Plotting
if doPlot:
specgram[1:] = specgram[0:-1]
specgram[0] = numpy.abs(spec[0:Lnyq])
if doPlot and ((iter / nint) % Lsgram) == 0:
pylab.clf()
pylab.subplot(311)
pylab.plot(history['T'], history['amp'], 'rx')
# pylab.plot(numpy.real(pcvec))
pylab.xlabel('Time (s)')
pylab.ylabel('Amplitude')
pylab.subplot(312)
pylab.plot(history['T'], history['phase'], 'gx')
pylab.ylim(-180.0, 180.0)
pylab.xlabel('Time (s)')
pylab.ylabel('Phase (deg)')
pylab.subplot(313)
xyextent = [
0, 0.5e-6 * dms.samprate, T_count * 1e3,
(T_count + Lsgram) * Tint * 1e3
]
pylab.imshow(numpy.abs(specgram),
aspect='auto',
extent=xyextent)
Fpeak = 1e-3 * dms.samprate * float(numpy.argmax(
specgram[0])) / Ldft
pylab.text(Fpeak, Lsgram / 2, 'Peak at %.3f kHz' % (Fpeak))
pylab.xlabel('Frequency (MHz)')
pylab.ylabel('Time (ms)')
pylab.draw() # non-blocking
pylab.draw()
# Clear accumulated values
spec = numpy.zeros_like(spec)
tavg = numpy.zeros_like(tavg)
pcamp = 0.0
return 0