def time_max(self): """ Maximal observed time. `astropy.Time` object """ times_per_block = self.nffte * self.fftlen * self.nfft2int sec_dt_block = 5.12e-6 * times_per_block return to_unix(self._timestamps[-1] + sec_dt_block)
def time(self, t): tmin = self.desctab['tmin'] tmax = self.desctab['tmax'] if t is None: t = [tmin.min(), tmax.max()] else: t = [to_unix(ti).unix for ti in t] if len(t) != 2: raise IndexError('time should be a length 2 list') self._time = t self._tmask = ((tmin <= t[0]) * (t[0] <= tmax)) +\ ((t[0] <= tmin) * (tmin <= t[1])) return
def _get_time(self, id_min, id_max): """ Recover the time for a given block selection Parameters ---------- id_min : int Index of starting time block id_max : int Index of ending time block Returns ------- time : `astropy.Time` Time object array """ n_times = (id_max - id_min) * self.nffte t = np.arange(n_times, dtype='float64') dt = t * self.dt, dt += self._timestamps[id_min] return to_unix(np.squeeze(dt))
def average(self, stokes='I', time=None, freq=None, beam=None, dt=None, df=None): """ Average a dynamic spectrum in time and frequency Parameters ---------- time : list Length-2 list of time range (ISO or ISOT format) e.g.: `time=['2019-03-20T11:59:00.0', '2019-03-20T12:20:00.0']` Default: `None` whole time range selection. stokes : str Stokes parameter required (I, Q, U, V, fracV) Default: `'I'` freq : list Length-2 list of frequency range (in MHz) e.g.: `freq=[30, 35]` Default: `None` whole frequency range selection. beam : int Beam index, refer to observation setup to see the details of the different observed beams. Default: `None` consider index 0. dt : float Time steps in seconds df : float Frequency steps in MHz Returns ------- spec : `SpecData` SpecData object containing the time, the frequency and the data """ self.beam = beam self.time = time self.freq = freq time_min, time_max = self.time freq_min, freq_max = self.freq if dt is None: dt = (time_max - time_min) / 1000 if df is None: df = (freq_max - freq_min) / 500 # Prepare the final array nt = int((time_max - time_min) // dt) nf = int((freq_max - freq_min) // df) available_memory = psutil.virtual_memory().available avg_data_size = nt * nf * np.dtype(np.float32).itemsize if avg_data_size > available_memory: raise MemoryError('Try to increase dt and/or df') averaged_data = np.zeros((nt, nf), dtype='float32') averaged_time = np.zeros(nt) #, dtype='float32') averaged_freq = np.zeros(nf) #, dtype='float32') # Loop over the time and seelct corresponding data bar = ProgressBar(valmax=nt, title='Averaging spectra...') for i in range(nt): spec = self.select( stokes=stokes, time=[time_min + i * dt, time_min + (i + 1) * dt], freq=freq, beam=beam) # Averaging data in time d = np.squeeze(np.mean(spec.data, axis=0)) averaged_time[i] = to_unix(np.mean(spec.time.unix)).unix # Averaging in frequency d = rebin1d(d, nf) if i == 0: averaged_freq[:] = rebin1d(spec.freq, nf) # Storing into final array averaged_data[i, :] = d bar.update() # return averaged_time, averaged_freq, averaged_data return SpecData(data=averaged_data, time=to_unix(averaged_time), freq=averaged_freq, stokes=stokes)
def select(self, stokes='I', time=None, freq=None, beam=None, bp_corr=True): """ Select data within a lane file. If the selection appears to be too big regarding available memory, an error should be raised. However as rough estimate, try to avoid time range of more than 15 min and/or frequency range of more than 10 MHz... Parameters ---------- time : list Length-2 list of time range (ISO or ISOT format) e.g.: `time=['2019-03-20T11:59:00.0', '2019-03-20T12:20:00.0']` Default: `None` whole time range selection. stokes : str Stokes parameter required (I, Q, U, V, fracV) Default: `'I'` freq : list Length-2 list of frequency range (in MHz) e.g.: `freq=[30, 35]` Default: `None` whole frequency range selection. beam : int Beam index, refer to observation setup to see the details of the different observed beams. Default: `None` consider index 0. bp_corr : bool or int, optional, default: `True Compute the bandpass correction. `False`: do not compute any correction `True``: compute the correction with Kaiser coefficients `'median'`: compute a medianed correction `'fft'`: correct the bandpass using FFT Returns ------- spec : `SpecData` SpecData object containing the time, the frequency and the data """ self.beam = beam self.time = time self.freq = freq if self.time[1] - self.time[0] < self.dt: raise ValueError('Time interval selected < {} sec'.format(self.dt)) if self.freq[1] - self.freq[0] < self.df: raise ValueError('Frequency interval selected < {} MHz'.format( self.df)) tmin_idx = self._t2bidx(time=self.time[0], order='low') tmax_idx = self._t2bidx(time=self.time[1], order='high') fmin_idx = self._f2bidx(frequency=self.freq[0], order='low') fmax_idx = self._f2bidx(frequency=self.freq[1], order='high') self._check_memory(nt=tmax_idx + 1 - tmin_idx, nf=fmax_idx + 1 - fmin_idx) times = self._get_time(id_min=tmin_idx, id_max=tmax_idx + 1) t_mask = (times >= to_unix(self.time[0])) &\ (times < to_unix(self.time[1])) freqs = self._get_freq(id_min=fmin_idx, id_max=fmax_idx + 1) f_mask = (freqs >= self.freq[0]) &\ (freqs < self.freq[1]) spectrum = NenuStokes(data=self.memdata['data'], stokes=stokes, nffte=self.nffte, fftlen=self.fftlen, bp_corr=bp_corr)[tmin_idx:tmax_idx + 1, fmin_idx:fmax_idx + 1, ] return SpecData(data=spectrum[t_mask, :][:, f_mask], time=times[t_mask], freq=freqs[f_mask], stokes=stokes)
def time_min(self): """ Minimal observed time. `astropy.Time` object """ return to_unix(self._timestamps[0])
def select(self, stokes='I', time=None, freq=None, beam=None): """ Select data within a lane file. If the selection appears to be too big regarding available memory, an error should be raised. However as rough estimate, try to avoid time range of more than 15 min and/or frequency range of more than 10 MHz... Parameters ---------- time : list Length-2 list of time range (ISO or ISOT format) e.g.: `time=['2019-03-20T11:59:00.0', '2019-03-20T12:20:00.0']` Default: `None` whole time range selection. stokes : str Stokes parameter required (I, Q, U, V, fracV) Default: `'I'` freq : list Length-2 list of frequency range (in MHz) e.g.: `freq=[30, 35]` Default: `None` whole frequency range selection. beam : int Beam index, refer to observation setup to see the details of the different observed beams. Default: `None` consider index 0. Returns ------- spec : `SpecData` SpecData object containing the time, the frequency and the data """ self.beam = beam self.time = time self.freq = freq tmin_idx = self._t2bidx(time=self.time[0], order='low') tmax_idx = self._t2bidx(time=self.time[1], order='high') fmin_idx = self._f2bidx(frequency=self.freq[0], order='low') fmax_idx = self._f2bidx(frequency=self.freq[1], order='high') self._check_memory(nt=tmax_idx + 1 - tmin_idx, nf=fmax_idx + 1 - fmin_idx) times = self._get_time(id_min=tmin_idx, id_max=tmax_idx + 1) t_mask = (times >= to_unix(self.time[0])) &\ (times <= to_unix(self.time[1])) freqs = self._get_freq(id_min=fmin_idx, id_max=fmax_idx + 1) f_mask = (freqs >= self.freq[0]) &\ (freqs <= self.freq[1]) spectrum = NenuStokes(data=self.memdata['data'], stokes=stokes, nffte=self.nffte, fftlen=self.fftlen)[tmin_idx:tmax_idx + 1, fmin_idx:fmax_idx + 1, ] # return ( # times[t_mask], # freqs[f_mask], # spectrum[t_mask, :][:, f_mask] # ) return SpecData(data=spectrum[t_mask, :][:, f_mask], time=times[t_mask], freq=freqs[f_mask], stokes=stokes)
def select(self, stokes='I', **kwargs): """ Select among the data stored in the directory according to a time range, a frequency range and a beam index and return them converted in the chosen Stokes parameter. NenuFAR-TF data can be spread over several lane files. This will select the data nonetheless and concatenate what is needed to ouptut a single `SpecData` object. This may be usefull to call for `.info()` method prior to select the data in order to know the time and frequency boundaries of the observation as well as recorded beam indices. Parameters ---------- stokes : {'I', 'Q', 'U', 'V', 'fracV'}, optional, default: 'I' Stokes parameter value to convert raw data to. Other Parameters ---------------- **kwargs Data selection can be applied on three parameters, namely `freq`, `time` and `beam`. - freq : list, optional, default: [fmin, fmax] Frequency range in MHz passed as a lenght-2 list. - time : list, optional, default: [tmin, tmax] Time range in ISOT or ISO format passed as a length-2 list. - beam : int, optional, default: 0 Beam index. Returns ------- spec : `~.stokes.SpecData` The selected data are returned via a `SpecData` instance, with a set of methods and attributes to easily get times and amplitudes in various units as well as some basic dynamic spectrum analysis tools. Examples -------- :: # Load the module from nenupytf.read import Spectrum # Creates an instance for the observation stored # in a given repository s = Spectrum('/path/to/observation/') # Display main informations s.info() # Data selection spec = s.select( freq=[35, 40], time=['2019-11-04T12:15:55.0000000', '2019-11-04T12:15:57.0000000'], beam=0 ) # Plot the data from nenupytf.display import plotdb plotdb(spec) """ self._parameters(**kwargs) mask = self._bmask * self._fmask * self._tmask for f in self.desctab[mask]['file']: l = Lane(f) if not 'spec' in locals(): spec = l.select(stokes=stokes, time=[to_unix(t).isot for t in self.time], freq=self.freq, beam=self.beam) else: # We assume here that the time is the same, # only frequency is spread over different lanes. # This will concatenate spectra from different # lanes: spec = spec & l.select( stokes=stokes, time=[to_unix(t).isot for t in self.time], freq=self.freq, beam=self.beam) del l return spec
def average(self, stokes='I', df=1, dt=1, **kwargs): """ Parameters ---------- stokes : {'I', 'Q', 'U', 'V', 'fracV'}, optional, default: 'I' Stokes parameter value to convert raw data to. df : float Frequency step in MHz on which average. dt : float Time step in seconds on which average. Other Parameters ---------------- **kwargs Data selection can be applied on three parameters, namely `freq`, `time` and `beam`. - freq : list, optional, default: [fmin, fmax] Frequency range in MHz passed as a lenght-2 list. - time : list, optional, default: [tmin, tmax] Time range in ISOT or ISO format passed as a length-2 list. - beam : int, optional, default: 0 Beam index. """ self._parameters(**kwargs) # Keep track of inputs parameters beam = self.beam freq = self.freq.copy() time = self.time.copy() start, stop = time # # No predefined array # bar = ProgressBar( # valmax=(stop-start)/dt, # title='Averaging...') # while start <= stop: # if not 'spec' in locals(): # spec = self.select( # stokes=stokes, # time=[start, start+dt], # freq=freq, # beam=beam # ).tmean().frebin((freq[1]-freq[0])/df) # else: # spec = spec | self.select( # stokes=stokes, # time=[start, start+dt], # freq=freq, # beam=beam # ).tmean().frebin((freq[1]-freq[0])/df) # start += dt # bar.update() # Predefined array ntimes = int(np.ceil((stop - start) / dt)) nfreqs = int((freq[1] - freq[0]) / df) avg_data = np.zeros((ntimes, nfreqs)) avg_time = np.zeros(ntimes) avg_freq = np.zeros(nfreqs) i = 0 bar = ProgressBar(valmax=ntimes, title='Averaging...') while start < stop: tmp_spec = self.select(stokes=stokes, time=[start, start + dt], freq=freq, beam=beam).tmean().frebin( (freq[1] - freq[0]) / df) avg_data[i, :] = tmp_spec.amp avg_time[i] = start + dt / 2 avg_freq = tmp_spec.freq i += 1 start += dt bar.update() spec = SpecData(data=avg_data, time=to_unix(avg_time), freq=avg_freq, stokes=stokes) return spec
def average(self, stokes='I', df=1., dt=1., bp_corr=True, **kwargs): r""" Average in time and frequency *NenuFAR/UnDySPuTeD* high rate time-frequency data. Data are read by time blocks of size `dt` before averaging by computing the mean over time. Then, the spectrum is rebinned to a reconstructed frequency axis with spaces around `df`. :param stokes: Stokes parameter value to convert raw data to, allowed values are `{'I', 'Q', 'U', 'V', 'fracV', 'XX', 'YY'}`, defaults to `'I'` :type stokes: str, optional :param df: Frequency resolution in MHz on which the averaging is performed, defaults to `1.` :type df: int, float, optional :param dt: Time resolution in seconds on which the average is performed, defaults to `1.` :type dt: int, float, optional :param bp_corr: Compute the bandpass correction, defaults to `True`, possible values are * `False`: do not compute any correction * `True`: compute the correction with Kaiser coefficients * `'median'`: compute a medianed correction * `'fft'`: correct the bandpass using FFT :type bp_corr: bool, str, optional :param \**kwargs: See below for keyword arguments: :param freq: Frequency range in MHz passed as a lenght-2 list. :type freq: list, optional :param time: Time range in ISOT or ISO format passed as a length-2 list. :type time: list, optional :param beam: Beam index. :type beam: int, optional :returns: `SpecData` object, embedding the stacked averaged spectra :rtype: :class:`.SpecData` :Example: >>> from nenupytf.read import Spectrum >>> s = Spectrum('/path/to/observation/') >>> spec = s.average( time=['2019-10-03 14:30:00', '2019-10-03 18:31:01.34'], freq=[34.5, 40], dt=0.01, df=0.5, stokes='I' ) .. seealso:: :func:`select()` :class:`.SpecData` .. warning:: This may take a significant time to process depending on the time and frequency ranges and the required time and frequency resolution. """ self._parameters(**kwargs) # Keep track of inputs parameters beam = self.beam freq = self.freq.copy() time = self.time.copy() start, stop = time # Predefined array ntimes = int(np.ceil((stop - start) / dt)) nfreqs = int((freq[1] - freq[0]) / df) avg_data = np.zeros((ntimes, nfreqs)) avg_time = np.zeros(ntimes) avg_freq = np.zeros(nfreqs) i = 0 bar = ProgressBar(valmax=ntimes, title='Averaging...') while start < stop - dt: tmp_spec = self.select( stokes=stokes, time=[start, start + dt], freq=freq, beam=beam, bp_corr=bp_corr, ).tmean().frebin((freq[1] - freq[0]) / df) avg_data[i, :] = tmp_spec.amp avg_time[i] = start + dt / 2 avg_freq = tmp_spec.freq i += 1 start += dt bar.update() spec = SpecData(data=avg_data, time=to_unix(avg_time), freq=avg_freq, stokes=stokes) return spec
def select(self, stokes='I', bp_corr=True, **kwargs): r""" Select among the data stored in the directory according to a :attr:`Spectrum.time` range, a :attr:`Spectrum.freq` range and a :attr:`Spectrum.beam` index and return them converted in the chosen Stokes parameter. NenuFAR-TF data can be spread over several lane files. This will select the data nonetheless and concatenate what is needed to ouptut a single :class:`.SpecData` object. This may be usefull to call for :func:`ObsRepo.info()` method prior to select the data in order to know the time and frequency boundaries of the observation as well as recorded beam indices. :param stokes: Stokes parameter value to convert raw data to, allowed values are `{'I', 'Q', 'U', 'V', 'fracV', 'XX', 'YY'}`, defaults to `'I'` :type stokes: str, optional :param bp_corr: Compute the bandpass correction, defaults to `True`, possible values are * `False`: do not compute any correction * `True`: compute the correction with Kaiser coefficients * `'median'`: compute a medianed correction * `'fft'`: correct the bandpass using FFT :type bp_corr: bool, str, optional :param \**kwargs: See below for keyword arguments: :param freq: Frequency range in MHz passed as a lenght-2 list. :type freq: list, optional :param time: Time range in ISOT or ISO format passed as a length-2 list. :type time: list, optional :param beam: Beam index. :type beam: int, optional :returns: `SpecData` object, embedding the stacked averaged spectra :rtype: :class:`.SpecData` :Example: >>> from nenupytf.read import Spectrum >>> s = Spectrum('/path/to/observation/') >>> spec = s.select( time=['2019-10-03 14:30:00', '2019-10-03 14:30:10.34'], freq=[34.5, 40], stokes='I' ) .. seealso:: :func:`average()` :class:`.SpecData` .. warning:: This may take a significant time to process depending on the time and frequency ranges. """ self._parameters(**kwargs) mask = self._bmask * self._fmask * self._tmask if all(~mask): raise ValueError('Empty selection, check parameter ranges') for f in self.desctab[mask]['file']: l = Lane(f) if not 'spec' in locals(): spec = l.select(stokes=stokes, time=[to_unix(t).isot for t in self.time], freq=self.freq, beam=self.beam, bp_corr=bp_corr) else: # We assume here that the time is the same, # only frequency is spread over different lanes. # This will concatenate spectra from different # lanes: spec = spec & l.select( stokes=stokes, time=[to_unix(t).isot for t in self.time], freq=self.freq, beam=self.beam, bp_corr=bp_corr) del l return spec