def _hilbert(self, data):
pci1 = fft.fft2(fft.fftshift(np.float32(data)))
pci2 = fft.ifftshift(pci1) * self.filter1
fpci0 = fft.ifftshift(fft.ifft2(fft.fftshift(pci2)))
fpci = np.imag(fpci0)
result = fpci
return result
def _hilbert(self, data):
pci1 = fft.fft2(fft.fftshift(np.float32(data)))
pci2 = fft.ifftshift(pci1)*self.filter1
fpci0 = fft.ifftshift(fft.ifft2(fft.fftshift(pci2)))
fpci = np.imag(fpci0)
result = fpci
return result
def _paganin(self, data):
pci1 = fft.fft2(np.float32(data))
pci2 = fft.fftshift(pci1) / self.filtercomplex
fpci = np.abs(fft.ifft2(pci2))
result = -0.5 * self.parameters['Ratio'] * np.log(
fpci + self.parameters['increment'])
return result
def _paganin(self, data):
pci1 = fft.fft2(np.float32(data))
pci2 = fft.fftshift(pci1) / self.filtercomplex
fpci = np.abs(fft.ifft2(pci2))
result = -0.5 * self.parameters['Ratio'] * np.log(
fpci + self.parameters['increment'])
return result
def _fine_search(self, sino, raw_cor):
(Nrow, Ncol) = sino.shape
centerfliplr = (Ncol + 1.0) / 2.0 - 1.0
# Use to shift the sino2 to the raw CoR
shiftsino = np.int16(2 * (raw_cor - centerfliplr))
sino2 = np.roll(np.fliplr(sino[1:]), shiftsino, axis=1)
lefttake = 0
righttake = Ncol - 1
search_rad = self.parameters['search_radius']
if raw_cor <= centerfliplr:
lefttake = np.ceil(search_rad + 1)
righttake = np.floor(2 * raw_cor - search_rad - 1)
else:
lefttake = np.ceil(raw_cor - (Ncol - 1 - raw_cor) + search_rad + 1)
righttake = np.floor(Ncol - 1 - search_rad - 1)
Ncol1 = righttake - lefttake + 1
mask = self._create_mask(2 * Nrow - 1, Ncol1,
0.5 * self.parameters['ratio'] * Ncol)
numshift = np.int16((2 * search_rad + 1.0) / self.parameters['step'])
listshift = np.linspace(-search_rad, search_rad, num=numshift)
listmetric = np.zeros(len(listshift), dtype=np.float32)
num1 = 0
for i in listshift:
logging.debug("list shift %d", i)
sino2a = ndi.interpolation.shift(sino2, (0, i), prefilter=False)
sinojoin = np.vstack((sino, sino2a))
listmetric[num1] = np.sum(
np.abs(
fft.fftshift(fft.fft2(
sinojoin[:, lefttake:righttake + 1]))) * mask)
num1 = num1 + 1
minpos = np.argmin(listmetric)
rotcenter = raw_cor + listshift[minpos] / 2.0
return rotcenter, listmetric
def _coarse_search(self, sino):
# search minsearch to maxsearch in 1 pixel steps
smin, smax = self.parameters['search_area']
(Nrow, Ncol) = sino.shape
centre_fliplr = (Ncol - 1.0) / 2.0
# check angles here to determine if a sinogram should be chopped off.
# Copy the sinogram and flip left right, the purpose is to make a full
# [0;2Pi] sinogram
sino2 = np.fliplr(sino[1:])
# This image is used for compensating the shift of sino2
compensateimage = np.zeros((Nrow - 1, Ncol), dtype=np.float32)
# Start coarse search in which the shift step is 1
compensateimage[:] = np.flipud(sino)[1:]
start_shift = self._get_start_shift(centre_fliplr) * 2
list_shift = np.arange(smin, smax + 1) * 2 - start_shift
list_metric = np.zeros(len(list_shift), dtype=np.float32)
mask = self._create_mask(2 * Nrow - 1, Ncol,
0.5 * self.parameters['ratio'] * Ncol)
count = 0
for i in list_shift:
sino2a = np.roll(sino2, i, axis=1)
if i >= 0:
sino2a[:, 0:i] = compensateimage[:, 0:i]
else:
sino2a[:, i:] = compensateimage[:, i:]
fft_out = fft.fft2(np.vstack((sino, sino2a)))
temp = np.sum(np.abs(fft.fftshift(fft_out)) * mask)
list_metric[count] = temp
count += 1
minpos = np.argmin(list_metric)
rot_centre = centre_fliplr + list_shift[minpos] / 2.0
return rot_centre, list_metric
def simcheck(data, nphases):
norients = data.shape[0]//nphases # need to use integer divide
# check user input before proceeding
assert nphases*norients == data.shape[0]
newshape = norients, nphases, data.shape[-2], data.shape[-1]
# FT data only along spatial dimensions
ft_data = fftshift(
fftn(ifftshift(
data, axes=(1, 2)),
axes=(1, 2)),
axes=(1, 2))
# average only along phase, **not** orientation
# This should be the equivalent of the FT of the average image per each
# phase (i.e it should be symmetric as the phases will have averaged out)
ft_data_avg = ft_data.reshape(newshape).mean(1)
# Do the same, but take the absolute value before averaging, in this case
# the signal should add up because the phase has been removed
ft_data_avg_abs = np.abs(ft_data).reshape(newshape).mean(1)
# Take the difference of the average power and the power of the average
ft_data_diff = ft_data_avg_abs-abs(ft_data_avg)
return ft_data_diff, ft_data_avg_abs, ft_data_avg
def _coarse_search(self, sino, list_shift):
# search minsearch to maxsearch in 1 pixel steps
list_metric = np.zeros(len(list_shift), dtype=np.float32)
(Nrow, Ncol) = sino.shape
# check angles to determine if a sinogram should be chopped off.
# Copy the sinogram and flip left right, to make a full [0:2Pi] sino
sino2 = np.fliplr(sino[1:])
# This image is used for compensating the shift of sino2
compensateimage = np.zeros((Nrow - 1, Ncol), dtype=np.float32)
# Start coarse search in which the shift step is 1
compensateimage[:] = np.flipud(sino)[1:]
mask = self._create_mask(2 * Nrow - 1, Ncol,
0.5 * self.parameters['ratio'] * Ncol)
count = 0
for i in list_shift:
sino2a = np.roll(sino2, i, axis=1)
if i >= 0:
sino2a[:, 0:i] = compensateimage[:, 0:i]
else:
sino2a[:, i:] = compensateimage[:, i:]
list_metric[count] = np.sum(
np.abs(fft.fftshift(fft.fft2(np.vstack(
(sino, sino2a))))) * mask)
count += 1
return list_metric
def _fine_search(self, sino, raw_cor):
(Nrow, Ncol) = sino.shape
centerfliplr = (Ncol + 1.0)/2.0-1.0
# Use to shift the sino2 to the raw CoR
shiftsino = np.int16(2*(raw_cor-centerfliplr))
sino2 = np.roll(np.fliplr(sino[1:]), shiftsino, axis=1)
lefttake = 0
righttake = Ncol-1
search_rad = self.parameters['search_radius']
if raw_cor <= centerfliplr:
lefttake = np.ceil(search_rad+1)
righttake = np.floor(2*raw_cor-search_rad-1)
else:
lefttake = np.ceil(raw_cor-(Ncol-1-raw_cor)+search_rad+1)
righttake = np.floor(Ncol-1-search_rad-1)
Ncol1 = righttake-lefttake + 1
mask = self._create_mask(2*Nrow-1, Ncol1,
0.5*self.parameters['ratio']*Ncol)
numshift = np.int16((2*search_rad+1.0)/self.parameters['step'])
listshift = np.linspace(-search_rad, search_rad, num=numshift)
listmetric = np.zeros(len(listshift), dtype=np.float32)
num1 = 0
for i in listshift:
logging.debug("list shift %d", i)
sino2a = ndi.interpolation.shift(sino2, (0, i), prefilter=False)
sinojoin = np.vstack((sino, sino2a))
listmetric[num1] = np.sum(np.abs(fft.fftshift(
fft.fft2(sinojoin[:, lefttake:righttake + 1])))*mask)
num1 = num1 + 1
minpos = np.argmin(listmetric)
rotcenter = raw_cor + listshift[minpos]/2.0
return rotcenter, listmetric
def _coarse_search(self, sino):
# search minsearch to maxsearch in 1 pixel steps
smin, smax = self.parameters['search_area']
logging.debug("SMIN and SMAX %d %d", smin, smax)
(Nrow, Ncol) = sino.shape
centre_fliplr = (Ncol - 1.0)/2.0
# check angles here to determine if a sinogram should be chopped off.
# Copy the sinogram and flip left right, the purpose is to make a full
# [0;2Pi] sinogram
sino2 = np.fliplr(sino[1:])
# This image is used for compensating the shift of sino2
compensateimage = np.zeros((Nrow-1, Ncol), dtype=np.float32)
# Start coarse search in which the shift step is 1
compensateimage[:] = sino[-1]
start_shift = self._get_start_shift(centre_fliplr)*2
list_shift = np.arange(smin, smax + 1)*2 - start_shift
logging.debug("%s", list_shift)
list_metric = np.zeros(len(list_shift), dtype=np.float32)
mask = self._create_mask(2*Nrow-1, Ncol,
0.5*self.parameters['ratio']*Ncol)
count = 0
for i in list_shift:
logging.debug("list shift %d", i)
sino2a = np.roll(sino2, i, axis=1)
if i >= 0:
sino2a[:, 0:i] = compensateimage[:, 0:i]
else:
sino2a[:, i:] = compensateimage[:, i:]
list_metric[count] = np.sum(
np.abs(fft.fftshift(fft.fft2(np.vstack((sino, sino2a)))))*mask)
count += 1
minpos = np.argmin(list_metric)
rot_centre = centre_fliplr + list_shift[minpos]/2.0
return rot_centre, list_metric
def FT_1D(ls, P, axis=-1):
"""
Fourier transform the complex linear polarization spectrum
P(lambda^2) to obtain the Faraday dispersion function F(phi).
The function uses the FFT to approximate the continuous
Fourier transform of a discretely sampled function.
FT: F(phi) = integral[ P(ls) exp(-2*i*phi*ls) dls]
IFT: P(ls) = integral[ F(phi) exp(2*i*phi*ls) dphi]
Function returns phi and F, which approximate F(phi).
Parameters
----------
ls : array_like
regularly sampled array of lambda_squared.
ls is assumed to be regularly spaced, i.e.
ls = ls0 + Dls * np.arange(N)
P : array_like
Complex linear polarization spectrum.
axis : int
axis along which to perform fourier transform.
Returns
-------
phi : ndarray
Faraday depth of the calculated Faraday dispersion function.
F : ndarray
Complex Faraday dispersion function.
"""
assert ls.ndim == 1
assert P.shape[axis] == ls.shape[0]
N = int(len(ls))
if N % 2 != 0:
raise ValueError("number of samples must be even")
ls = ls / np.pi
Dls = ls[1] - ls[0]
Dphi = 1. / (N * Dls)
ls0 = ls[int(N / 2)]
phi = Dphi * (np.arange(N) - N / 2)
shape = np.ones(P.ndim, dtype=int)
shape[axis] = N
phase = np.ones(N)
phase[1::2] = -1
phase = phase.reshape(shape)
# F = Dls * fft.fft(P * phase, axis=axis)
F = Dls * fft.fftshift(fft.fft(P, axis=axis), axes=axis) #*np.pi
F *= phase
F *= np.exp(-2j * np.pi * ls0 * phi.reshape(shape))
F *= np.exp(-1j * np.pi * N / 2)
return phi, F
def process_frames(self, data):
sino = data[0]
sino2 = np.fliplr(sino[1:])
(Nrow, Ncol) = sino.shape
mask = self._create_mask(
2*Nrow-1, Ncol, 0.5*self.parameters['ratio']*Ncol)
FT1 = fft.fftshift(fft.fft2(np.vstack((sino, sino2))))
sino = fft.ifft2(fft.ifftshift(FT1 - FT1*mask))
return sino[0:Nrow].real
def sampling_op_forward(image, mask_as_image):
""" Sampling operator
:param image: Assumed to be an array with shape [N,N]
:param mask_as_image:
:param mask_as_image: Mask of shape [N,N] where middle of the image
corresponds to zero frequency. Values should be 0,1 or
True/False.
:return: An array with shape [N,N] where some of the data have been zeroed out.
"""
N = max(image.shape)
fourier_coeff = fftw.fftshift(fftw.fft2(image)) / N
fourier_coeff_zero = np.multiply(mask_as_image, fourier_coeff)
return fourier_coeff_zero
def spectral_whitening(tr,
smooth=None,
filter=None,
waterlevel=1e-8,
mask_again=True):
"""
Apply spectral whitening to data
Data is divided by its smoothed (Default: None) amplitude spectrum.
:param tr: trace to manipulate
:param smooth: length of smoothing window in Hz
(default None -> no smoothing)
:param filter: filter spectrum with bandpass after whitening
(tuple with min and max frequency)
:param waterlevel: waterlevel relative to mean of spectrum
:param mask_again: weather to mask array after this operation again and
set the corresponding data to 0
:return: whitened data
"""
sr = tr.stats.sampling_rate
data = tr.data
data = _fill_array(data, fill_value=0)
mask = np.ma.getmask(data)
nfft = next_fast_len(len(data))
spec = fft(data, nfft)
spec_ampl = np.abs(spec)
spec_ampl /= np.max(spec_ampl)
if smooth:
smooth = int(smooth * nfft / sr)
spec_ampl = ifftshift(smooth_func(fftshift(spec_ampl), smooth))
# save guard against division by 0
spec_ampl[spec_ampl < waterlevel] = waterlevel
spec /= spec_ampl
if filter is not None:
spec *= _filter_resp(*filter, sr=sr, N=len(spec), whole=True)[1]
ret = np.real(ifft(spec, nfft)[:len(data)])
if mask_again:
ret = _fill_array(ret, mask=mask, fill_value=0)
tr.data = ret
return tr
def _coarse_search(self, sino, list_shift):
# search minsearch to maxsearch in 1 pixel steps
list_metric = np.zeros(len(list_shift), dtype=np.float32)
(Nrow, Ncol) = sino.shape
# check angles to determine if a sinogram should be chopped off.
# Copy the sinogram and flip left right, to make a full [0:2Pi] sino
sino2 = np.fliplr(sino[1:])
# This image is used for compensating the shift of sino2
compensateimage = np.zeros((Nrow-1, Ncol), dtype=np.float32)
# Start coarse search in which the shift step is 1
compensateimage[:] = np.flipud(sino)[1:]
mask = self._create_mask(2*Nrow-1, Ncol,
0.5*self.parameters['ratio']*Ncol)
count = 0
for i in list_shift:
sino2a = np.roll(sino2, i, axis=1)
if i >= 0:
sino2a[:, 0:i] = compensateimage[:, 0:i]
else:
sino2a[:, i:] = compensateimage[:, i:]
list_metric[count] = np.sum(
np.abs(fft.fftshift(fft.fft2(np.vstack((sino, sino2a)))))*mask)
count += 1
return list_metric
def plot(self):
try:
self.plotErrorLabel.setText('')
plot_groups = []
# Read Groups
defined_groups = str(self.groupsTextEdit.toPlainText()).replace(
'\n', '').replace(' ', '').split(';')
defined_groups = defined_groups[0:-1]
# Parse each group as if defining a constraint group (recylcing code from PINTS)
for group in defined_groups:
group_info = group.split(',')
group_name = group_info[0]
group_members = group_info[1:]
new_group = CSTGroup('', group_name, group_members, 0, 0)
plot_groups.append(new_group)
# Gather plotting variables
sfactor = float(self.sfactorInput.text())
pfactor = np.deg2rad(float(self.pfactorInput.text()))
VSHIFT = float(self.VSHIFTInput.text())
HSHIFT = float(self.HSHIFTInput.text())
FT1 = int(self.ft1Input.text())
lb = float(self.lbInput.text())
b0 = float(self.b0Input.text())
dwell_time = float(self.dwellTimeInput.text())
acq_time = int(self.nInput.text()) * dwell_time
fs = 1 / dwell_time
t = np.arange(0, acq_time, dwell_time)
tableau10 = [(31, 119, 180), (255, 127, 14), (44, 160, 44),
(214, 39, 40), (148, 103, 189), (140, 86, 75),
(227, 119, 194), (127, 127, 127), (188, 189, 34),
(23, 190, 207)]
for i in range(len(tableau10)):
r, g, b = tableau10[i]
tableau10[i] = (r / 255., g / 255., b / 255.)
# Calculate Summed Spectra
fit_spec_sum = []
fit_fid_sum = []
for metabolite in self.fit_out.metabolites_list:
fid = self.fit_out.metabolites[metabolite].getFID(
0, b0, t, 0, 1, pfactor, 0, lb)
if not (self.extrap0CheckBox.isChecked()):
fid = fid[FT1:]
spec = fftw.fftshift(fftw.fft(fid))
n = np.size(fid)
f = np.arange(+n // 2, -n // 2, -1) * (fs / n) * (1 / b0)
fit_f = -f
# fit_f, spec = self.fit_out.metabolites[metabolite].getSpec(0, b0, t, 0, 1, pfactor, 0, lb, fs)
fit_fid_sum.append(fid)
fit_spec_sum.append(spec)
fit_fid_sum = np.sum(np.array(fit_fid_sum), axis=0)
fit_spec_sum = np.real(np.sum(np.array(fit_spec_sum), axis=0))
# Calculate In-Vivo Spectra
invivo_dat_temp = copy.copy(self.invivo_dat)
invivo_dat_temp.signal = invivo_dat_temp.signal[
0:np.size(t)] * np.exp(1j * pfactor) * np.exp(-sp.pi * lb * t)
if not (self.extrap0CheckBox.isChecked()):
invivo_dat_temp.signal = invivo_dat_temp.signal[FT1:]
else:
invivo_dat_temp.signal = np.hstack(
(fit_fid_sum[0:FT1], invivo_dat_temp.signal[FT1:]))
invivo_f, invivo_spec = invivo_dat_temp.getSpec()
invivo_spec = np.real(invivo_spec)
# Calculate Fitted Spectra
fit_spec = []
fit_spec_names = []
for group in plot_groups:
group_spec = []
for member in group.members:
fid = self.fit_out.metabolites[member].getFID(
0, b0, t, 0, 1, pfactor, 0, lb)
if not (self.extrap0CheckBox.isChecked()):
fid = fid[FT1:]
spec = fftw.fftshift(fftw.fft(fid))
n = np.size(fid)
f = np.arange(+n // 2, -n // 2, -1) * (fs / n) * (1 / b0)
fit_f = -f
# fit_f, spec = self.fit_out.metabolites[member].getSpec(0, b0, t, 0, 1, pfactor, 0, lb, fs)
group_spec.append(spec)
group_spec = np.real(np.sum(np.array(group_spec), axis=0))
fit_spec.append(group_spec)
fit_spec_names.append(group.name)
plt.figure(1)
plt.clf()
ax = plt.subplot(111)
ax.spines['top'].set_visible(False)
ax.spines['bottom'].set_visible(True)
ax.spines['right'].set_visible(False)
ax.spines['left'].set_visible(False)
ax.get_xaxis().tick_bottom()
ax.get_yaxis().set_visible(False)
plt.xlim(5, 0)
for i, spec in enumerate(fit_spec):
plt.plot(
np.array(-fit_f) + sfactor,
np.real(fit_spec[-(i + 1)]) -
i * VSHIFT * np.amax(np.real(fit_spec_sum)),
color=tableau10[-((i) % np.size(tableau10, axis=0) + 1)],
lw=1.5,
alpha=0.8)
ax.text(5, -i * VSHIFT * np.amax(np.real(fit_spec_sum)),
fit_spec_names[-(i + 1)])
print(np.size(fit_f), np.size(fit_spec_sum), np.size(invivo_spec))
plt.plot(
np.array(-fit_f) + sfactor,
(np.real(fit_spec_sum) - np.roll(
np.real(invivo_spec)[0:np.size(fit_f)], int(HSHIFT))) +
np.amax(np.real(invivo_spec)) +
3.0 * VSHIFT * np.amax(np.real(invivo_spec)),
color=tableau10[2],
lw=1.5,
alpha=0.8)
plt.plot(
np.array(-fit_f) + sfactor,
np.roll(np.real(invivo_spec)[0:np.size(fit_f)], int(HSHIFT)) +
1.5 * VSHIFT * np.amax(np.real(invivo_spec)),
color=tableau10[0],
lw=1.5)
plt.plot(np.array(-fit_f) + sfactor,
np.real(fit_spec_sum) +
1.5 * VSHIFT * np.amax(np.real(invivo_spec)),
color=tableau10[1],
lw=1.5,
alpha=0.8)
plt.xlabel('ppm')
self.canvas[0].draw()
except Exception as e:
self.plotErrorLabel.setText(
'>> ERROR: ' + str(e) +
'\n>> Could not plot. Please try again.')
def cifftn(data,axes):
'''Centered ifft'''
return fftpack.fftshift(fftpack.ifftn(fftpack.ifftshift(data,axes),axes=axes))
def fold(fh1,
dtype,
samplerate,
fedge,
fedge_at_top,
nchan,
nt,
ntint,
nskip,
ngate,
ntbin,
ntw,
dm,
fref,
phasepol,
dedisperse='incoherent',
do_waterfall=True,
do_foldspec=True,
verbose=True,
progress_interval=100,
comm=None):
"""FFT GMRT data, fold by phase/time and make a waterfall series
Parameters
----------
fh1 : file handle
handle to file holding voltage timeseries
dtype : numpy dtype or '4bit' or '1bit'
way the data are stored in the file
samplerate : float
rate at which samples were originally taken and thus double the
band width (frequency units)
fedge : float
edge of the frequency band (frequency units)
fedge_at_top: bool
whether edge is at top (True) or bottom (False)
nchan : int
number of frequency channels for FFT
nt, ntint : int
total number nt of sets, each containing ntint samples in each file
hence, total # of samples is nt*ntint, with each sample containing
a single polarisation
nskip : int
number of records (nchan*ntint*2 for phased data w/ np.int8 real,imag)
ngate, ntbin : int
number of phase and time bins to use for folded spectrum
ntbin should be an integer fraction of nt
ntw : int
number of time samples to combine for waterfall (does not have to be
integer fraction of nt)
dm : float
dispersion measure of pulsar, used to correct for ism delay
(column number density)
fref: float
reference frequency for dispersion measure
phasepol : callable
function that returns the pulsar phase for time in seconds relative to
start of part of the file that is read (i.e., ignoring nhead)
dedisperse : None or string
None, 'incoherent', 'coherent', 'by-channel'
do_waterfall, do_foldspec : bool
whether to construct waterfall, folded spectrum (default: True)
verbose : bool
whether to give some progress information (default: True)
progress_interval : int
Ping every progress_interval sets
comm : MPI communicator (default: None)
"""
if comm is None:
rank = 0
size = 1
else:
rank = comm.rank
size = comm.size
# initialize folded spectrum and waterfall
foldspec2 = np.zeros((nchan, ngate, ntbin))
nwsize = nt * ntint // ntw
waterfall = np.zeros((nchan, nwsize))
# size in bytes of records read from file (simple for ARO: 1 byte/sample)
# double since we need to get ntint samples after FFT
itemsize = {np.int8: 2}[dtype]
recsize = nchan * ntint * itemsize
if verbose:
print('Reading from {}'.format(fh1))
if nskip > 0:
if verbose:
print('Skipping {0} {1}-byte records'.format(nskip, recsize))
if size == 1:
fh1.seek(nskip * recsize)
foldspec = np.zeros((nchan, ngate, ntbin), dtype=np.int)
icount = np.zeros((nchan, ngate, ntbin), dtype=np.int)
dt1 = (1. / samplerate).to(u.s)
# but include 2*nchan real-valued samples used for each FFT
# (or, equivalently, real and imag for channels)
dtsample = nchan * 2 * dt1
tstart = dt1 * nskip * recsize
# pre-calculate time delay due to dispersion in coarse channels
freq = fftshift(fftfreq(nchan, 2. * dt1.value)) * u.Hz
freq = (fedge - (freq - freq[0]) if fedge_at_top else fedge +
(freq - freq[0]))
# [::2] sets frequency channels to numerical recipes ordering
dt = (dispersion_delay_constant * dm * (1. / freq**2 - 1. / fref**2)).to(
u.s).value
# if dedisperse in {'coherent', 'by-channel'}:
# # pre-calculate required turns due to dispersion
# fcoh = (fedge - fftfreq(nchan*ntint, 2.*dt1)
# if fedge_at_top
# else
# fedge + fftfreq(nchan*ntint, 2.*dt1))
# # set frequency relative to which dispersion is coherently corrected
# if dedisperse == 'coherent':
# _fref = fref
# else:
# _fref = np.repeat(freq.value, ntint) * freq.unit
# # (check via eq. 5.21 and following in
# # Lorimer & Kramer, Handbook of Pulsar Astrono
# dang = (dispersion_delay_constant * dm * fcoh *
# (1./_fref-1./fcoh)**2) * 360. * u.deg
# # order of frequencies is r[0], r[1],i[1],...r[n-1],i[n-1],r[n]
# # for 0 and n need only real part, but for 1...n-1 need real, imag
# # so just get shifts for r[1], r[2], ..., r[n-1]
# dang = dang.to(u.rad).value[1:-1:2]
# dd_coh = np.exp(dang * 1j).conj().astype(np.complex64)
for j in xrange(rank, nt, size):
if verbose and j % progress_interval == 0:
print('Doing {:6d}/{:6d}; time={:18.12f}'.format(
j + 1, nt,
(tstart + dtsample * j * ntint).value)) # time since start
# just in case numbers were set wrong -- break if file ends
# better keep at least the work done
try:
if size > 1:
fh1.seek((nskip + j) * recsize)
# data just a series of byte pairs, of real and imag
raw = fromfile(fh1, dtype, recsize)
except (EOFError, IOError) as exc:
print("Hit {}; writing pgm's".format(exc))
break
if verbose == 'very':
print("Read {} items".format(raw.size), end="")
vals = raw.astype(np.float32).view(np.complex64).squeeze()
# if dedisperse in {'coherent', 'by-channel'}:
# fine = rfft(vals, axis=0, overwrite_x=True, **_fftargs)
# fine_cmplx = fine[1:-1].view(np.complex64)
# fine_cmplx *= dd_coh # this overwrites parts of fine, as intended
# vals = irfft(fine, axis=0, overwrite_x=True, **_fftargs)
# if verbose == 'very':
# print("... dedispersed", end="")
chan = vals.reshape(-1, nchan)
if verbose == 'very':
print("... power", end="")
power = chan.real**2 + chan.imag**2
# current sample positions in stream
isr = j * ntint + np.arange(ntint)
if do_waterfall:
# loop over corresponding positions in waterfall
for iw in xrange(isr[0] // ntw, isr[-1] // ntw + 1):
if iw < nwsize: # add sum of corresponding samples
waterfall[:, iw] += np.sum(power[isr // ntw == iw], axis=0)
if verbose == 'very':
print("... waterfall", end="")
if do_foldspec:
tsample = (tstart + isr * dtsample).value # times since start
ibin = j * ntbin // nt # bin in the time series: 0..ntbin-1
for k in xrange(nchan):
if dedisperse == 'coherent':
t = tsample # already dedispersed
else:
t = tsample - dt[k] # dedispersed times
phase = phasepol(t) # corresponding PSR phases
iphase = np.remainder(phase * ngate, ngate).astype(np.int)
# sum and count samples by phase bin
foldspec[k, :, ibin] += np.bincount(iphase, power[:, k], ngate)
icount[k, :, ibin] += np.bincount(iphase, None, ngate)
if verbose == 'very':
print("... folded", end="")
if 0: #done in gmrt.py (j+1)*ntbin//nt > ibin: # last addition to bin?
# get normalised flux in each bin (where any were added)
nonzero = icount > 0
nfoldspec = np.where(nonzero, foldspec / icount, 0.)
# subtract phase average and store
nfoldspec -= np.where(
nonzero,
np.sum(nfoldspec, 1, keepdims=True) /
np.sum(nonzero, 1, keepdims=True), 0)
foldspec2[:, :, ibin] = nfoldspec
# reset for next iteration
foldspec *= 0
icount *= 0
if verbose == 'very':
print("... added", end="")
if verbose == 'very':
print("... done")
if verbose:
print('read {0:6d} out of {1:6d}'.format(j + 1, nt))
if 0: # done in gmrt.py do_waterfall:
nonzero = waterfall == 0.
waterfall -= np.where(
nonzero,
np.sum(waterfall, 1, keepdims=True) /
np.sum(nonzero, 1, keepdims=True), 0.)
return foldspec, icount, waterfall
def fold(fh1, dtype, samplerate, fedge, fedge_at_top, nchan,
nt, ntint, nskip, ngate, ntbin, ntw, dm, fref, phasepol,
dedisperse='incoherent',
do_waterfall=True, do_foldspec=True, verbose=True,
progress_interval=100, comm=None):
"""FFT GMRT data, fold by phase/time and make a waterfall series
Parameters
----------
fh1 : file handle
handle to file holding voltage timeseries
dtype : numpy dtype or '4bit' or '1bit'
way the data are stored in the file
samplerate : float
rate at which samples were originally taken and thus double the
band width (frequency units)
fedge : float
edge of the frequency band (frequency units)
fedge_at_top: bool
whether edge is at top (True) or bottom (False)
nchan : int
number of frequency channels for FFT
nt, ntint : int
total number nt of sets, each containing ntint samples in each file
hence, total # of samples is nt*ntint, with each sample containing
a single polarisation
nskip : int
number of records (nchan*ntint*2 for phased data w/ np.int8 real,imag)
ngate, ntbin : int
number of phase and time bins to use for folded spectrum
ntbin should be an integer fraction of nt
ntw : int
number of time samples to combine for waterfall (does not have to be
integer fraction of nt)
dm : float
dispersion measure of pulsar, used to correct for ism delay
(column number density)
fref: float
reference frequency for dispersion measure
phasepol : callable
function that returns the pulsar phase for time in seconds relative to
start of part of the file that is read (i.e., ignoring nhead)
dedisperse : None or string
None, 'incoherent', 'coherent', 'by-channel'
do_waterfall, do_foldspec : bool
whether to construct waterfall, folded spectrum (default: True)
verbose : bool
whether to give some progress information (default: True)
progress_interval : int
Ping every progress_interval sets
comm : MPI communicator (default: None)
"""
if comm is None:
rank = 0
size = 1
else:
rank = comm.rank
size = comm.size
# initialize folded spectrum and waterfall
foldspec2 = np.zeros((nchan, ngate, ntbin))
nwsize = nt*ntint//ntw
waterfall = np.zeros((nchan, nwsize))
# size in bytes of records read from file (simple for ARO: 1 byte/sample)
# double since we need to get ntint samples after FFT
itemsize = {np.int8: 2}[dtype]
recsize = nchan*ntint*itemsize
if verbose:
print('Reading from {}'.format(fh1))
if nskip > 0:
if verbose:
print('Skipping {0} {1}-byte records'.format(nskip, recsize))
if size == 1:
fh1.seek(nskip * recsize)
foldspec = np.zeros((nchan, ngate, ntbin), dtype=np.int)
icount = np.zeros((nchan, ngate, ntbin), dtype=np.int)
dt1 = (1./samplerate).to(u.s)
# but include 2*nchan real-valued samples used for each FFT
# (or, equivalently, real and imag for channels)
dtsample = nchan * 2 * dt1
tstart = dt1 * nskip * recsize
# pre-calculate time delay due to dispersion in coarse channels
freq = fftshift(fftfreq(nchan, 2.*dt1.value)) * u.Hz
freq = (fedge - (freq-freq[0])
if fedge_at_top
else fedge + (freq-freq[0]))
# [::2] sets frequency channels to numerical recipes ordering
dt = (dispersion_delay_constant * dm *
(1./freq**2 - 1./fref**2)).to(u.s).value
# if dedisperse in {'coherent', 'by-channel'}:
# # pre-calculate required turns due to dispersion
# fcoh = (fedge - fftfreq(nchan*ntint, 2.*dt1)
# if fedge_at_top
# else
# fedge + fftfreq(nchan*ntint, 2.*dt1))
# # set frequency relative to which dispersion is coherently corrected
# if dedisperse == 'coherent':
# _fref = fref
# else:
# _fref = np.repeat(freq.value, ntint) * freq.unit
# # (check via eq. 5.21 and following in
# # Lorimer & Kramer, Handbook of Pulsar Astrono
# dang = (dispersion_delay_constant * dm * fcoh *
# (1./_fref-1./fcoh)**2) * 360. * u.deg
# # order of frequencies is r[0], r[1],i[1],...r[n-1],i[n-1],r[n]
# # for 0 and n need only real part, but for 1...n-1 need real, imag
# # so just get shifts for r[1], r[2], ..., r[n-1]
# dang = dang.to(u.rad).value[1:-1:2]
# dd_coh = np.exp(dang * 1j).conj().astype(np.complex64)
for j in xrange(rank, nt, size):
if verbose and j % progress_interval == 0:
print('Doing {:6d}/{:6d}; time={:18.12f}'.format(
j+1, nt, (tstart+dtsample*j*ntint).value)) # time since start
# just in case numbers were set wrong -- break if file ends
# better keep at least the work done
try:
if size > 1:
fh1.seek((nskip + j) * recsize)
# data just a series of byte pairs, of real and imag
raw = fromfile(fh1, dtype, recsize)
except(EOFError, IOError) as exc:
print("Hit {}; writing pgm's".format(exc))
break
if verbose == 'very':
print("Read {} items".format(raw.size), end="")
vals = raw.astype(np.float32).view(np.complex64).squeeze()
# if dedisperse in {'coherent', 'by-channel'}:
# fine = rfft(vals, axis=0, overwrite_x=True, **_fftargs)
# fine_cmplx = fine[1:-1].view(np.complex64)
# fine_cmplx *= dd_coh # this overwrites parts of fine, as intended
# vals = irfft(fine, axis=0, overwrite_x=True, **_fftargs)
# if verbose == 'very':
# print("... dedispersed", end="")
chan = vals.reshape(-1, nchan)
if verbose == 'very':
print("... power", end="")
power = chan.real**2+chan.imag**2
# current sample positions in stream
isr = j*ntint + np.arange(ntint)
if do_waterfall:
# loop over corresponding positions in waterfall
for iw in xrange(isr[0]//ntw, isr[-1]//ntw + 1):
if iw < nwsize: # add sum of corresponding samples
waterfall[:,iw] += np.sum(power[isr//ntw == iw],
axis=0)
if verbose == 'very':
print("... waterfall", end="")
if do_foldspec:
tsample = (tstart + isr*dtsample).value # times since start
ibin = j*ntbin//nt # bin in the time series: 0..ntbin-1
for k in xrange(nchan):
if dedisperse == 'coherent':
t = tsample # already dedispersed
else:
t = tsample - dt[k] # dedispersed times
phase = phasepol(t) # corresponding PSR phases
iphase = np.remainder(phase*ngate,
ngate).astype(np.int)
# sum and count samples by phase bin
foldspec[k,:,ibin] += np.bincount(iphase, power[:,k], ngate)
icount[k,:,ibin] += np.bincount(iphase, None, ngate)
if verbose == 'very':
print("... folded", end="")
if 0: #done in gmrt.py (j+1)*ntbin//nt > ibin: # last addition to bin?
# get normalised flux in each bin (where any were added)
nonzero = icount > 0
nfoldspec = np.where(nonzero, foldspec/icount, 0.)
# subtract phase average and store
nfoldspec -= np.where(nonzero,
np.sum(nfoldspec, 1, keepdims=True) /
np.sum(nonzero, 1, keepdims=True), 0)
foldspec2[:,:,ibin] = nfoldspec
# reset for next iteration
foldspec *= 0
icount *= 0
if verbose == 'very':
print("... added", end="")
if verbose == 'very':
print("... done")
if verbose:
print('read {0:6d} out of {1:6d}'.format(j+1, nt))
if 0: # done in gmrt.py do_waterfall:
nonzero = waterfall == 0.
waterfall -= np.where(nonzero,
np.sum(waterfall, 1, keepdims=True) /
np.sum(nonzero, 1, keepdims=True), 0.)
return foldspec, icount, waterfall
def fftshift(self, a, axes=None):
return scipy_fft.fftshift(a, axes=axes)
def fold(file1, samplerate, fmid, nchan,
nt, ntint, nhead, ngate, ntbin, ntw, dm, fref, phasepol,
coherent=False, do_waterfall=True, do_foldspec=True, verbose=True,
progress_interval=100):
"""FFT Effelsberg data, fold by phase/time and make a waterfall series
Parameters
----------
file1 : string
name of the file holding voltage timeseries
samplerate : float
rate at which samples were originally taken and thus band width
(frequency units))
fmid : float
mid point of the frequency band (frequency units)
nchan : int
number of frequency channels for FFT
nt, ntint : int
total number nt of sets, each containing ntint samples in each file
hence, total # of samples is nt*(2*ntint), with each sample containing
real,imag for two polarisations
nhead : int
number of bytes to skip before reading (usually 4096 for Effelsberg)
ngate, ntbin : int
number of phase and time bins to use for folded spectrum
ntbin should be an integer fraction of nt
ntw : int
number of time samples to combine for waterfall (does not have to be
integer fraction of nt)
dm : float
dispersion measure of pulsar, used to correct for ism delay
(column number density)
fref: float
reference frequency for dispersion measure
phasepol : callable
function that returns the pulsar phase for time in seconds relative to
start of part of the file that is read (i.e., ignoring nhead)
do_waterfall, do_foldspec : bool
whether to construct waterfall, folded spectrum (default: True)
verbose : bool
whether to give some progress information (default: True)
progress_interval : int
Ping every progress_interval sets
"""
# initialize folded spectrum and waterfall
foldspec2 = np.zeros((nchan, ngate, ntbin))
nwsize = nt*ntint//ntw
waterfall = np.zeros((nchan, nwsize))
# size in bytes of records read from file (each nchan contains 4 bytes:
# real,imag for 2 polarisations).
recsize = 4*nchan*ntint
if verbose:
print('Reading from {}'.format(file1))
myopen = gzip.open if '.gz' in file1 else open
with myopen(file1, 'rb', recsize) as fh1:
if nhead > 0:
if verbose:
print('Skipping {0} bytes'.format(nhead))
fh1.seek(nhead)
foldspec = np.zeros((nchan, ngate))
icount = np.zeros((nchan, ngate))
# gosh, fftpack has everything; used to calculate with:
# fband / nchan * (np.mod(np.arange(nchan)+nchan/2, nchan)-nchan/2)
if coherent:
# pre-calculate required turns due to dispersion
fcoh = (fmid +
fftfreq(nchan*ntint, (1./samplerate).to(u.s).value) * u.Hz)
# (check via eq. 5.21 and following in
# Lorimer & Kramer, Handbook of Pulsar Astrono
dang = (dispersion_delay_constant * dm * fcoh *
(1./fref-1./fcoh)**2) * 360. * u.deg
dedisperse = np.exp(dang.to(u.rad).value * 1j
).conj().astype(np.complex64)
else:
# pre-calculate time delay due to dispersion
freq = fmid + fftfreq(nchan, (1./samplerate).to(u.s).value) * u.Hz
dt = (dispersion_delay_constant * dm *
(1./freq**2 - 1./fref**2)).to(u.s).value
dtsample = (nchan/samplerate).to(u.s).value
for j in xrange(nt):
if verbose and (j+1) % progress_interval == 0:
print('Doing {:6d}/{:6d}; time={:18.12f}'.format(
j+1, nt, dtsample*j*ntint)) # equivalent time since start
# just in case numbers were set wrong -- break if file ends
# better keep at least the work done
try:
# data stored as series of two two-byte complex numbers,
# one for each polarization
raw = np.fromstring(fh1.read(recsize),
dtype=np.int8).reshape(-1,2,2)
except:
break
# use view for fast conversion from float to complex
vals = raw.astype(np.float32).view(np.complex64).squeeze()
# vals[i_int * i_block, i_pol]
if coherent:
fine = fft(vals, axis=0, overwrite_x=True, **_fftargs)
fine *= dedisperse[:,np.newaxis]
vals = ifft(fine, axis=0, overwrite_x=True, **_fftargs)
chan = fft(vals.reshape(-1, nchan, 2), axis=1, overwrite_x=True,
**_fftargs)
# chan[i_int, i_block, i_pol]
power = np.sum(chan.real**2+chan.imag**2, axis=-1)
# current sample positions in stream
isr = j*ntint + np.arange(ntint)
if do_waterfall:
# loop over corresponding positions in waterfall
for iw in xrange(isr[0]//ntw, isr[-1]//ntw + 1):
if iw < nwsize: # add sum of corresponding samples
waterfall[:,iw] += np.sum(power[isr//ntw == iw],
axis=0)
if do_foldspec:
tsample = dtsample*isr # times since start
for k in xrange(nchan):
if coherent:
t = tsample # already dedispersed
else:
t = tsample - dt[k] # dedispersed times
phase = phasepol(t) # corresponding PSR phases
iphase = np.remainder(phase*ngate,
ngate).astype(np.int)
# sum and count samples by phase bin
foldspec[k] += np.bincount(iphase, power[:,k], ngate)
icount[k] += np.bincount(iphase, None, ngate)
ibin = j*ntbin//nt # bin in the time series: 0..ntbin-1
if (j+1)*ntbin//nt > ibin: # last addition to bin?
# get normalised flux in each bin (where any were added)
nonzero = icount > 0
nfoldspec = np.where(nonzero, foldspec/icount, 0.)
# subtract phase average and store
nfoldspec -= np.where(nonzero,
np.sum(nfoldspec, 1, keepdims=True) /
np.sum(nonzero, 1, keepdims=True), 0)
foldspec2[:,:,ibin] = nfoldspec
# reset for next iteration
foldspec *= 0
icount *= 0
if verbose:
print('read {0:6d} out of {1:6d}'.format(j+1, nt))
if do_foldspec:
# swap two halfs in frequency, so that freq increases monotonically
foldspec2 = fftshift(foldspec2, axes=0)
if do_waterfall:
nonzero = waterfall == 0.
waterfall -= np.where(nonzero,
np.sum(waterfall, 1, keepdims=True) /
np.sum(nonzero, 1, keepdims=True), 0.)
# swap two halfs in frequency, so that freq increases monotonically
waterfall = fftshift(waterfall, axes=0)
return foldspec2, waterfall