def time_max(self):
""" Maximal observed time.
`astropy.Time` object
"""
times_per_block = self.nffte * self.fftlen * self.nfft2int
sec_dt_block = 5.12e-6 * times_per_block
return to_unix(self._timestamps[-1] + sec_dt_block)
def time(self, t):
tmin = self.desctab['tmin']
tmax = self.desctab['tmax']
if t is None:
t = [tmin.min(), tmax.max()]
else:
t = [to_unix(ti).unix for ti in t]
if len(t) != 2:
raise IndexError('time should be a length 2 list')
self._time = t
self._tmask = ((tmin <= t[0]) * (t[0] <= tmax)) +\
((t[0] <= tmin) * (tmin <= t[1]))
return
def _get_time(self, id_min, id_max):
""" Recover the time for a given block selection
Parameters
----------
id_min : int
Index of starting time block
id_max : int
Index of ending time block
Returns
-------
time : `astropy.Time`
Time object array
"""
n_times = (id_max - id_min) * self.nffte
t = np.arange(n_times, dtype='float64')
dt = t * self.dt,
dt += self._timestamps[id_min]
return to_unix(np.squeeze(dt))
def average(self,
stokes='I',
time=None,
freq=None,
beam=None,
dt=None,
df=None):
""" Average a dynamic spectrum in time and frequency
Parameters
----------
time : list
Length-2 list of time range (ISO or ISOT format)
e.g.:
`time=['2019-03-20T11:59:00.0', '2019-03-20T12:20:00.0']`
Default: `None` whole time range selection.
stokes : str
Stokes parameter required (I, Q, U, V, fracV)
Default: `'I'`
freq : list
Length-2 list of frequency range (in MHz)
e.g.:
`freq=[30, 35]`
Default: `None` whole frequency range selection.
beam : int
Beam index, refer to observation setup to see the
details of the different observed beams.
Default: `None` consider index 0.
dt : float
Time steps in seconds
df : float
Frequency steps in MHz
Returns
-------
spec : `SpecData`
SpecData object containing the time, the frequency and the data
"""
self.beam = beam
self.time = time
self.freq = freq
time_min, time_max = self.time
freq_min, freq_max = self.freq
if dt is None:
dt = (time_max - time_min) / 1000
if df is None:
df = (freq_max - freq_min) / 500
# Prepare the final array
nt = int((time_max - time_min) // dt)
nf = int((freq_max - freq_min) // df)
available_memory = psutil.virtual_memory().available
avg_data_size = nt * nf * np.dtype(np.float32).itemsize
if avg_data_size > available_memory:
raise MemoryError('Try to increase dt and/or df')
averaged_data = np.zeros((nt, nf), dtype='float32')
averaged_time = np.zeros(nt) #, dtype='float32')
averaged_freq = np.zeros(nf) #, dtype='float32')
# Loop over the time and seelct corresponding data
bar = ProgressBar(valmax=nt, title='Averaging spectra...')
for i in range(nt):
spec = self.select(
stokes=stokes,
time=[time_min + i * dt, time_min + (i + 1) * dt],
freq=freq,
beam=beam)
# Averaging data in time
d = np.squeeze(np.mean(spec.data, axis=0))
averaged_time[i] = to_unix(np.mean(spec.time.unix)).unix
# Averaging in frequency
d = rebin1d(d, nf)
if i == 0:
averaged_freq[:] = rebin1d(spec.freq, nf)
# Storing into final array
averaged_data[i, :] = d
bar.update()
# return averaged_time, averaged_freq, averaged_data
return SpecData(data=averaged_data,
time=to_unix(averaged_time),
freq=averaged_freq,
stokes=stokes)
def select(self,
stokes='I',
time=None,
freq=None,
beam=None,
bp_corr=True):
""" Select data within a lane file.
If the selection appears to be too big regarding
available memory, an error should be raised.
However as rough estimate, try to avoid time range
of more than 15 min and/or frequency range of more
than 10 MHz...
Parameters
----------
time : list
Length-2 list of time range (ISO or ISOT format)
e.g.:
`time=['2019-03-20T11:59:00.0', '2019-03-20T12:20:00.0']`
Default: `None` whole time range selection.
stokes : str
Stokes parameter required (I, Q, U, V, fracV)
Default: `'I'`
freq : list
Length-2 list of frequency range (in MHz)
e.g.:
`freq=[30, 35]`
Default: `None` whole frequency range selection.
beam : int
Beam index, refer to observation setup to see the
details of the different observed beams.
Default: `None` consider index 0.
bp_corr : bool or int, optional, default: `True
Compute the bandpass correction.
`False`: do not compute any correction
`True``: compute the correction with Kaiser coefficients
`'median'`: compute a medianed correction
`'fft'`: correct the bandpass using FFT
Returns
-------
spec : `SpecData`
SpecData object containing the time, the frequency and the data
"""
self.beam = beam
self.time = time
self.freq = freq
if self.time[1] - self.time[0] < self.dt:
raise ValueError('Time interval selected < {} sec'.format(self.dt))
if self.freq[1] - self.freq[0] < self.df:
raise ValueError('Frequency interval selected < {} MHz'.format(
self.df))
tmin_idx = self._t2bidx(time=self.time[0], order='low')
tmax_idx = self._t2bidx(time=self.time[1], order='high')
fmin_idx = self._f2bidx(frequency=self.freq[0], order='low')
fmax_idx = self._f2bidx(frequency=self.freq[1], order='high')
self._check_memory(nt=tmax_idx + 1 - tmin_idx,
nf=fmax_idx + 1 - fmin_idx)
times = self._get_time(id_min=tmin_idx, id_max=tmax_idx + 1)
t_mask = (times >= to_unix(self.time[0])) &\
(times < to_unix(self.time[1]))
freqs = self._get_freq(id_min=fmin_idx, id_max=fmax_idx + 1)
f_mask = (freqs >= self.freq[0]) &\
(freqs < self.freq[1])
spectrum = NenuStokes(data=self.memdata['data'],
stokes=stokes,
nffte=self.nffte,
fftlen=self.fftlen,
bp_corr=bp_corr)[tmin_idx:tmax_idx + 1,
fmin_idx:fmax_idx + 1, ]
return SpecData(data=spectrum[t_mask, :][:, f_mask],
time=times[t_mask],
freq=freqs[f_mask],
stokes=stokes)
def time_min(self):
""" Minimal observed time.
`astropy.Time` object
"""
return to_unix(self._timestamps[0])
def select(self, stokes='I', time=None, freq=None, beam=None):
""" Select data within a lane file.
If the selection appears to be too big regarding
available memory, an error should be raised.
However as rough estimate, try to avoid time range
of more than 15 min and/or frequency range of more
than 10 MHz...
Parameters
----------
time : list
Length-2 list of time range (ISO or ISOT format)
e.g.:
`time=['2019-03-20T11:59:00.0', '2019-03-20T12:20:00.0']`
Default: `None` whole time range selection.
stokes : str
Stokes parameter required (I, Q, U, V, fracV)
Default: `'I'`
freq : list
Length-2 list of frequency range (in MHz)
e.g.:
`freq=[30, 35]`
Default: `None` whole frequency range selection.
beam : int
Beam index, refer to observation setup to see the
details of the different observed beams.
Default: `None` consider index 0.
Returns
-------
spec : `SpecData`
SpecData object containing the time, the frequency and the data
"""
self.beam = beam
self.time = time
self.freq = freq
tmin_idx = self._t2bidx(time=self.time[0], order='low')
tmax_idx = self._t2bidx(time=self.time[1], order='high')
fmin_idx = self._f2bidx(frequency=self.freq[0], order='low')
fmax_idx = self._f2bidx(frequency=self.freq[1], order='high')
self._check_memory(nt=tmax_idx + 1 - tmin_idx,
nf=fmax_idx + 1 - fmin_idx)
times = self._get_time(id_min=tmin_idx, id_max=tmax_idx + 1)
t_mask = (times >= to_unix(self.time[0])) &\
(times <= to_unix(self.time[1]))
freqs = self._get_freq(id_min=fmin_idx, id_max=fmax_idx + 1)
f_mask = (freqs >= self.freq[0]) &\
(freqs <= self.freq[1])
spectrum = NenuStokes(data=self.memdata['data'],
stokes=stokes,
nffte=self.nffte,
fftlen=self.fftlen)[tmin_idx:tmax_idx + 1,
fmin_idx:fmax_idx + 1, ]
# return (
# times[t_mask],
# freqs[f_mask],
# spectrum[t_mask, :][:, f_mask]
# )
return SpecData(data=spectrum[t_mask, :][:, f_mask],
time=times[t_mask],
freq=freqs[f_mask],
stokes=stokes)
def select(self, stokes='I', **kwargs):
""" Select among the data stored in the directory according
to a time range, a frequency range and a beam index
and return them converted in the chosen Stokes parameter.
NenuFAR-TF data can be spread over several lane files.
This will select the data nonetheless and concatenate
what is needed to ouptut a single `SpecData` object.
This may be usefull to call for `.info()` method prior
to select the data in order to know the time and
frequency boundaries of the observation as well as
recorded beam indices.
Parameters
----------
stokes : {'I', 'Q', 'U', 'V', 'fracV'}, optional, default: 'I'
Stokes parameter value to convert raw data to.
Other Parameters
----------------
**kwargs
Data selection can be applied on three parameters,
namely `freq`, `time` and `beam`.
- freq : list, optional, default: [fmin, fmax]
Frequency range in MHz passed as a lenght-2
list.
- time : list, optional, default: [tmin, tmax]
Time range in ISOT or ISO format passed as
a length-2 list.
- beam : int, optional, default: 0
Beam index.
Returns
-------
spec : `~.stokes.SpecData`
The selected data are returned via a `SpecData`
instance, with a set of methods and attributes
to easily get times and amplitudes in various
units as well as some basic dynamic spectrum
analysis tools.
Examples
--------
::
# Load the module
from nenupytf.read import Spectrum
# Creates an instance for the observation stored
# in a given repository
s = Spectrum('/path/to/observation/')
# Display main informations
s.info()
# Data selection
spec = s.select(
freq=[35, 40],
time=['2019-11-04T12:15:55.0000000', '2019-11-04T12:15:57.0000000'],
beam=0
)
# Plot the data
from nenupytf.display import plotdb
plotdb(spec)
"""
self._parameters(**kwargs)
mask = self._bmask * self._fmask * self._tmask
for f in self.desctab[mask]['file']:
l = Lane(f)
if not 'spec' in locals():
spec = l.select(stokes=stokes,
time=[to_unix(t).isot for t in self.time],
freq=self.freq,
beam=self.beam)
else:
# We assume here that the time is the same,
# only frequency is spread over different lanes.
# This will concatenate spectra from different
# lanes:
spec = spec & l.select(
stokes=stokes,
time=[to_unix(t).isot for t in self.time],
freq=self.freq,
beam=self.beam)
del l
return spec
def average(self, stokes='I', df=1, dt=1, **kwargs):
"""
Parameters
----------
stokes : {'I', 'Q', 'U', 'V', 'fracV'}, optional, default: 'I'
Stokes parameter value to convert raw data to.
df : float
Frequency step in MHz on which average.
dt : float
Time step in seconds on which average.
Other Parameters
----------------
**kwargs
Data selection can be applied on three parameters,
namely `freq`, `time` and `beam`.
- freq : list, optional, default: [fmin, fmax]
Frequency range in MHz passed as a lenght-2
list.
- time : list, optional, default: [tmin, tmax]
Time range in ISOT or ISO format passed as
a length-2 list.
- beam : int, optional, default: 0
Beam index.
"""
self._parameters(**kwargs)
# Keep track of inputs parameters
beam = self.beam
freq = self.freq.copy()
time = self.time.copy()
start, stop = time
# # No predefined array
# bar = ProgressBar(
# valmax=(stop-start)/dt,
# title='Averaging...')
# while start <= stop:
# if not 'spec' in locals():
# spec = self.select(
# stokes=stokes,
# time=[start, start+dt],
# freq=freq,
# beam=beam
# ).tmean().frebin((freq[1]-freq[0])/df)
# else:
# spec = spec | self.select(
# stokes=stokes,
# time=[start, start+dt],
# freq=freq,
# beam=beam
# ).tmean().frebin((freq[1]-freq[0])/df)
# start += dt
# bar.update()
# Predefined array
ntimes = int(np.ceil((stop - start) / dt))
nfreqs = int((freq[1] - freq[0]) / df)
avg_data = np.zeros((ntimes, nfreqs))
avg_time = np.zeros(ntimes)
avg_freq = np.zeros(nfreqs)
i = 0
bar = ProgressBar(valmax=ntimes, title='Averaging...')
while start < stop:
tmp_spec = self.select(stokes=stokes,
time=[start, start + dt],
freq=freq,
beam=beam).tmean().frebin(
(freq[1] - freq[0]) / df)
avg_data[i, :] = tmp_spec.amp
avg_time[i] = start + dt / 2
avg_freq = tmp_spec.freq
i += 1
start += dt
bar.update()
spec = SpecData(data=avg_data,
time=to_unix(avg_time),
freq=avg_freq,
stokes=stokes)
return spec
def average(self, stokes='I', df=1., dt=1., bp_corr=True, **kwargs):
r""" Average in time and frequency *NenuFAR/UnDySPuTeD*
high rate time-frequency data.
Data are read by time blocks of size `dt` before
averaging by computing the mean over time. Then, the
spectrum is rebinned to a reconstructed frequency
axis with spaces around `df`.
:param stokes:
Stokes parameter value to convert raw data to,
allowed values are `{'I', 'Q', 'U', 'V', 'fracV',
'XX', 'YY'}`, defaults to `'I'`
:type stokes: str, optional
:param df:
Frequency resolution in MHz on which the
averaging is performed, defaults to `1.`
:type df: int, float, optional
:param dt:
Time resolution in seconds on which the average
is performed, defaults to `1.`
:type dt: int, float, optional
:param bp_corr:
Compute the bandpass correction, defaults to
`True`, possible values are
* `False`: do not compute any correction
* `True`: compute the correction with Kaiser
coefficients
* `'median'`: compute a medianed correction
* `'fft'`: correct the bandpass using FFT
:type bp_corr: bool, str, optional
:param \**kwargs:
See below for keyword arguments:
:param freq:
Frequency range in MHz passed as a lenght-2 list.
:type freq: list, optional
:param time:
Time range in ISOT or ISO format passed as a
length-2 list.
:type time: list, optional
:param beam:
Beam index.
:type beam: int, optional
:returns: `SpecData` object, embedding the stacked
averaged spectra
:rtype: :class:`.SpecData`
:Example:
>>> from nenupytf.read import Spectrum
>>> s = Spectrum('/path/to/observation/')
>>> spec = s.average(
time=['2019-10-03 14:30:00', '2019-10-03 18:31:01.34'],
freq=[34.5, 40],
dt=0.01,
df=0.5,
stokes='I'
)
.. seealso:: :func:`select()` :class:`.SpecData`
.. warning:: This may take a significant time to
process depending on the time and frequency
ranges and the required time and frequency
resolution.
"""
self._parameters(**kwargs)
# Keep track of inputs parameters
beam = self.beam
freq = self.freq.copy()
time = self.time.copy()
start, stop = time
# Predefined array
ntimes = int(np.ceil((stop - start) / dt))
nfreqs = int((freq[1] - freq[0]) / df)
avg_data = np.zeros((ntimes, nfreqs))
avg_time = np.zeros(ntimes)
avg_freq = np.zeros(nfreqs)
i = 0
bar = ProgressBar(valmax=ntimes, title='Averaging...')
while start < stop - dt:
tmp_spec = self.select(
stokes=stokes,
time=[start, start + dt],
freq=freq,
beam=beam,
bp_corr=bp_corr,
).tmean().frebin((freq[1] - freq[0]) / df)
avg_data[i, :] = tmp_spec.amp
avg_time[i] = start + dt / 2
avg_freq = tmp_spec.freq
i += 1
start += dt
bar.update()
spec = SpecData(data=avg_data,
time=to_unix(avg_time),
freq=avg_freq,
stokes=stokes)
return spec
def select(self, stokes='I', bp_corr=True, **kwargs):
r""" Select among the data stored in the directory
according to a :attr:`Spectrum.time` range, a
:attr:`Spectrum.freq` range and a
:attr:`Spectrum.beam` index and return them converted
in the chosen Stokes parameter. NenuFAR-TF data can
be spread over several lane files. This will select
the data nonetheless and concatenate what is needed
to ouptut a single :class:`.SpecData` object.
This may be usefull to call for :func:`ObsRepo.info()`
method prior to select the data in order to know the
time and frequency boundaries of the observation as
well as recorded beam indices.
:param stokes:
Stokes parameter value to convert raw data to,
allowed values are `{'I', 'Q', 'U', 'V', 'fracV',
'XX', 'YY'}`, defaults to `'I'`
:type stokes: str, optional
:param bp_corr:
Compute the bandpass correction, defaults to
`True`, possible values are
* `False`: do not compute any correction
* `True`: compute the correction with Kaiser
coefficients
* `'median'`: compute a medianed correction
* `'fft'`: correct the bandpass using FFT
:type bp_corr: bool, str, optional
:param \**kwargs:
See below for keyword arguments:
:param freq:
Frequency range in MHz passed as a lenght-2 list.
:type freq: list, optional
:param time:
Time range in ISOT or ISO format passed as a
length-2 list.
:type time: list, optional
:param beam:
Beam index.
:type beam: int, optional
:returns: `SpecData` object, embedding the stacked
averaged spectra
:rtype: :class:`.SpecData`
:Example:
>>> from nenupytf.read import Spectrum
>>> s = Spectrum('/path/to/observation/')
>>> spec = s.select(
time=['2019-10-03 14:30:00', '2019-10-03 14:30:10.34'],
freq=[34.5, 40],
stokes='I'
)
.. seealso:: :func:`average()` :class:`.SpecData`
.. warning:: This may take a significant time to
process depending on the time and frequency
ranges.
"""
self._parameters(**kwargs)
mask = self._bmask * self._fmask * self._tmask
if all(~mask):
raise ValueError('Empty selection, check parameter ranges')
for f in self.desctab[mask]['file']:
l = Lane(f)
if not 'spec' in locals():
spec = l.select(stokes=stokes,
time=[to_unix(t).isot for t in self.time],
freq=self.freq,
beam=self.beam,
bp_corr=bp_corr)
else:
# We assume here that the time is the same,
# only frequency is spread over different lanes.
# This will concatenate spectra from different
# lanes:
spec = spec & l.select(
stokes=stokes,
time=[to_unix(t).isot for t in self.time],
freq=self.freq,
beam=self.beam,
bp_corr=bp_corr)
del l
return spec
