def _check_signal_specification_v1(obj): # [implicit] - it must be a dataframe if not isinstance(obj, pd.DataFrame): raise SignalSpecificationError('Signals must be a dataframe', SignalSpecificationErrorCode.BAD_TYPE) dataframe = obj # [1] - dimensions (probably not necessary with dataframes) if len(dataframe.shape) != 2: # pragma: no cover raise SignalSpecificationError('Dataframe must have two dimensions', SignalSpecificationErrorCode.BAD_SHAPE) # [2] - datetime index index = dataframe.index if not pd.core.dtypes.common.is_datetime64_any_dtype(index): raise SignalSpecificationError('Dataframe must have a datetime index', SignalSpecificationErrorCode.BAD_TYPE) # [3] - uniform time support # Use dsu.pandas_helpers.estimate_rate, since that code already raises # a DSUException on the same cases that we want to test if dataframe.shape[0] >= 2: from dsu.pandas_helpers import estimate_rate from dsu.exceptions import DSUException try: estimate_rate(dataframe) except DSUException as ex: raise SignalSpecificationError(f'Invalid dataframe index: {ex}', SignalSpecificationErrorCode.BAD_SAMPLING) from ex # [4] - known columns # Use mne channel names for standard 10/05 to avoid writing them down from mne.channels import make_standard_montage known_columns = ( make_standard_montage('standard_1005').ch_names + # EEG ['I', 'II', 'III', 'aVR', 'aVL', 'aVF'] + # ECG ['PPG', 'GSR', 'PZT'] ) for col in dataframe.columns: if col == 'sample_number': continue if col not in known_columns: raise SignalSpecificationError(f'Unexpected "{col}" column', SignalSpecificationErrorCode.UNKNOWN_COLUMN_NAME) # [5] - column types for col in known_columns: if col not in dataframe.columns: continue if not np.issubdtype(dataframe[col].dtype, np.number): raise SignalSpecificationError(f'Column "{col}" must be of numeric dtype', SignalSpecificationErrorCode.INCORRECT_COLUMN_TYPE) if 'sample_number' in dataframe.columns: if not np.issubdtype(dataframe['sample_number'], np.integer): raise SignalSpecificationError('Column "sample_number" must be of integer dtype', SignalSpecificationErrorCode.INCORRECT_COLUMN_TYPE)
def run(self, *, signals: pd.DataFrame) -> FileAdapter: if signals.empty: raise SoftPreconditionFailed('Input signals are empty') if 'PPG' not in signals.columns: raise SoftPreconditionFailed( 'Input signals do not have a PPG column') output_file = self.default_outputs() fs = int(estimate_rate(signals)) bands = (0.5, 11) self.logger.info( 'Band-pass filtering signal between %.2f -- %.2f Hz ' 'with a FIR filter of order %d', *bands, fs) filtered = filtfilt_signal( signals, order=fs, frequencies=bands, filter_type='bandpass', filter_design='fir', ) scaled = scale_signal(filtered, method='robust') self.logger.info('Cleaned PPG signal, input shape %s, output shape %s', signals.shape, scaled.shape) with pd.HDFStore(output_file.file, 'w') as store: scaled.to_hdf(store, self.output_hdf5_key) return output_file
def run(self, *, signals: pd.DataFrame) -> FileAdapter: if signals.empty: raise SoftPreconditionFailed('Input signals are empty') if self.column not in signals.columns: raise SoftPreconditionFailed( 'Input signals do not have a PPG column') output_file = self.default_outputs() # Step 1: calculate SSF fs = estimate_rate(signals) window_samples = int(self.window_fraction * fs) ppg = signals[self.column] ppg_ssf = ssf(ppg, win=window_samples) df_ssf = pd.DataFrame({'PPG_SSF': ppg_ssf}, index=signals.index) self.logger.info( 'Calculated SSF signal, input shape %s, output shape %s', signals.shape, ppg_ssf.shape) # Step 2: detect peak with adaptive threshold peaks, thresh = detect_ssf_peaks(df_ssf.PPG_SSF, threshold_percentage=0.50) # Step 3: convert to PP intervals and post-process them df_interval = peak_to_nn(peaks).rename(columns={'interval': 'NN'}) # Step 4: interpolate NN df_interpolated = nn_interpolation(df_interval, fs=fs, column='NN') with pd.HDFStore(output_file.file, 'w') as store: df_ssf.to_hdf(store, self.ssf_output_hdf5_key) df_interval.to_hdf(store, self.ssf_nn_output_hdf5_key) df_interpolated.to_hdf(store, self.ssf_nni_output_hdf5_key) return output_file
def run(self, *, signal: pd.DataFrame) -> FileAdapter: output_file = self.default_outputs() if self.column not in signal: raise SoftPreconditionFailed(f'Input dataframe does not have column "{self.column}"') x = signal[self.column] fs = int(estimate_rate(x)) properties, _ = extract_all_peaks(x, window_size=fs) with pd.HDFStore(output_file.file, 'w') as store: properties.to_hdf(store, self.output_hdf5_key) return output_file
def run(self, signals: pd.DataFrame) -> FileAdapter: """Extract and pre-process signals""" logger.info('Extracting Nexus signal %s -> %s on file %s', self.source_column, self.target_column, prefect.context.run_kwargs['signals']) raw = ( signals[[self.source_column]] .rename(columns={self.source_column: self.target_column}) ) # Estimate the sampling frequency: weird signals that have a heavy jitter # will fail here early and raise a ValueError. See issue #44 try: fs = estimate_rate(raw) except DSUException as ex: logger.warning('Failed to estimate rate: %s, raising a precondition fail', ex) raise SoftPreconditionFailed(str(ex)) from ex logger.debug('Uniform resampling from %.3f Hz to %d Hz', fs, self.sampling_rate) # Uniform sampling, with linear interpolation. # sample-and-hold is not a good strategy, see issue 48: # https://github.com/OpenMindInnovation/iguazu/issues/48 raw_uniform = uniform_sampling(raw, self.sampling_rate, interpolation_kind='linear') # Create the annotations companion dataframe and mark any nan as a # "unknown" problem since it must come from the device / driver. # idx_sparse = raw_uniform.isna().any(axis='columns') #raw_annotations = raw_uniform.loc[idx_sparse].isna().replace({True: 'unknown', False: ''}) # I have changed my mind: sparse complicates the code, and we are only saving so little space raw_annotations = raw_uniform.isna().replace({True: 'unknown', False: ''}) n_samples = raw_uniform.shape[0] n_nans = (raw_annotations != '').sum() logger.debug('Finished standardization of Nexus signal %s -> %s. ' 'Result has %d samples (%.1f seconds, %.1f minutes) ' '%d samples are NaN (%.1f %%).', self.source_column, self.target_column, n_samples, n_samples / self.sampling_rate, n_samples / self.sampling_rate / 60, n_nans, 100 * n_nans / n_samples) if n_samples > 0: logger.debug('Extract of result:\n%s', raw_uniform.to_string(max_rows=5)) return self.save(raw_uniform, raw_annotations)
def respiration_clean(data, column='PZT'): ''' # todo: does this function belong to dsu? Parameters ---------- data column Returns ------- ''' sampling_rate = estimate_rate(data) data.loc[:, column] = nk.rsp_clean(data[column], sampling_rate, method='BioSPPy') return data
def bandpower(data, bands, epoch_size, epoch_overlap, fs=None, scaling='density', relative=False): # Note: add to doc that epoch_size and epoch_overlap is in seconds # also, # TODO: add on documentation of estimate_rate that the output is in Hz if isinstance(data, (pd.DataFrame, pd.Series)) and isinstance(data.index, (pd.TimedeltaIndex, pd.DatetimeIndex)): datetime_index = True fs = fs or int(estimate_rate(data)) if isinstance(data, pd.DataFrame): columns = data.columns x = np.asarray(data.values) else: columns = [data.name] x = np.asarray(data.values)[:, np.newaxis] else: datetime_index = False x = np.asarray(data) if x.ndim not in (1, 2): raise ValueError('bandpower only supports 1- or 2-dimensional data') elif x.ndim == 1: x = x[:, np.newaxis] columns = [f'x{i+1}' for i in range(x.shape[1])] fs = fs or 1 logger.debug('Calculating %s spectra with fs=%dHz on data shaped as %s', 'relative' if relative else 'absolute', fs, data.shape) x = x.T nsamples = x.shape[1] nperseg = int(epoch_size * fs) noverlap = int(epoch_overlap * fs) if nperseg > nsamples: raise ValueError('Epoch size is larger than data') # To whom it may concern: according to the _spectral_helper code, the # difference between scaling='density' and 'spectrum' is just how the # window adjusts the final value. One uses the sum of squared values, the # other the square of the sum. The 'density' also divides by the fs (which # is why the units change to V^2 / Hz). # In my opinion, this is not very important as long as we are consistent freqs, t, psd = _spectral_helper(x, x, fs=fs, window='hann', nperseg=nperseg, noverlap=noverlap, detrend='constant', return_onesided=True, scaling=scaling, mode='psd') # TODO: psd or stft ??? # Manage index if datetime_index: index = data.index[0] + pd.to_timedelta(t, unit='s') else: # index = pd.Index((t * (nperseg - noverlap)).astype(int), name='sample') n = index.shape[0] powers = [] rel_suffix = '_rel' if relative else '_abs' for name, (f_start, f_stop) in bands.items(): f_start = f_start or 0 f_stop = f_stop or fs / 2 idx = (freqs >= f_start) & (freqs < f_stop) if idx.sum() == 0: logger.warning('Band %s is empty for fs=%d and nperseg=%d', name, fs, nperseg) result = pd.DataFrame(data=[[np.nan] * len(columns)] * n, columns=columns, index=index) else: logger.debug('Calculating band power for %s with %d bins', name, idx.sum()) if idx.sum() <= 1: logger.warning('Band power for %s will be zero because there ' 'are not enough frequency points to calculate an ' 'integral', name) # TODO: we should manage the 1-bin case, but how ? # pxx axes: (column, freq, time) # bp = psd[:, idx, :].sum(axis=1) bp = simps(psd[:, idx, :], freqs[idx], axis=1) if relative: bp /= simps(psd, freqs, axis=1) # power_sum axes: (column, time) result = pd.DataFrame(data=bp.T, # (time, column) columns=columns, index=index) powers.append(result.add_suffix(f'_{name}{rel_suffix}')) return pd.concat(powers, axis='columns')
def galvanic_cvx(signals, annotations, column=None, warmup_duration=15, threshold_scr=4.0, cvxeda_params=None, epoch_size=None, epoch_overlap=None): """ Separate galvanic components using a convex deconvolution. This function separates the phasic (SCR) and tonic (SCL) galvanic components using the cvxEDA algorithm. For large signals, the underlying library (cvx) is particularly heavy on memory usage. Also, since it uses C code, it holds and does not relase the Python global interpreter lock (GIL). This makes everything more difficult, in particular for Iguazu. In order to manage this, one can do the algorithm by epochs using both the `epoch_size` and `epoch_overlap` parameters. Parameters ---------- data: pd.DataFrame Dataframe containing the preprocessed GSR in channel given by column_name. column: str | None Name of column where the data of interest are located. If None, the first column is considered. warmup_duration: float Duration at beginning and end of the data to label as 'bad'==True. threshold_scr: Maximum acceptable amplitude of SCR component. cvxeda_params: Keywords arguments to apply cvxEDA algorithm. epoch_size: float Size in seconds of the epoch size. When set to ``None``, cvxEDA will be applied only once on the whole signal. epoch_overlap: float Size in seconds of the epoch overlap. When set to ``None``, cvxEDA will be applied only once on the whole signal. Returns ------- data: pd.DataFrame Dataframe with columns: `..._SCR`, `..._SCL` and `bad`. Examples -------- .. image:: ../source/_static/examples/galvanic_functions/io_deconvolution.png """ cvxeda_params = cvxeda_params or {} # extract SCR and SCL component using deconvolution toolbox cvxEDA # if no column is specified, consider the first one column = column or signals.columns[-1] n = signals.shape[0] idx_epochs = np.arange(n)[np.newaxis, :] idx_warmup = slice(0, n) if epoch_size is not None and epoch_overlap is not None: fs = estimate_rate(signals) n_warmup = int(warmup_duration * fs) n_epoch = int(epoch_size * fs) + n_warmup n_overlap = int(epoch_overlap * fs) logger.debug( 'Attempting to epoch signal of %d samples into epochs ' 'of %d samples and %d overlap', n, n_epoch, n_overlap) if n < n_epoch + n_overlap: # Data is not big enough for window logger.debug( 'Cannot epoch into epochs of %d samples, signal is not ' 'large enough. Falling back to complete implementation.', n_epoch) else: idx_epochs = sliding_window(np.arange(n), size=n_epoch, stepsize=n_overlap) idx_warmup = slice(n_warmup, -n_warmup) logger.debug('cvxEDA epoched implementation with %d epochs', len(idx_epochs)) else: logger.debug('cvxEDA complete implementation') epochs = [] for i, idx in enumerate(idx_epochs): logger.debug('Epoch %d / %d', i + 1, len(idx_epochs)) chunk = signals.iloc[idx][[column]].dropna() if not chunk.empty: chunk = (apply_cvxEDA( chunk, **cvxeda_params).iloc[idx_warmup].rename_axis( index='epoched_index').reset_index()) epochs.append(chunk) if not epochs: return pd.DataFrame(), pd.DataFrame() # or None? signals = (pd.concat( epochs, ignore_index=False).groupby('epoched_index').mean().rename_axis( index=signals.index.name)) # add an annotation rejection boolean on amplitude criteria # todo: helpers with inputs: data, annotations, and index or bool condition, that sets values in data to NaN and annotate in annotations annotations.loc[signals[signals[column + '_SCR'] >= threshold_scr].index, 'GSR'] = 'CVX SCR outlier' warm_up_timedelta = warmup_duration * np.timedelta64(1, 's') annotations.loc[:signals.index[0] + warm_up_timedelta, 'GSR'] = 'CVX warm up' annotations.loc[signals.index[-1] - warm_up_timedelta:, 'GSR'] = 'CVX warm up' # replace column string name by 'gsr' for lisibility purpose signals.columns = signals.columns.str.replace(column, 'GSR') return signals, annotations
def respiration_sequence_features(data, events, column='PZT', known_sequences=None): # nk.rsp_peaks(pzt_signal['PZT'].values, sampling_rate=sampling_rate, method="BioSPPy") sampling_rate = estimate_rate(data) # Extract peak using neurokit BioSPPy method _index = data.index _, info = nk.rsp_peaks(data[column], sampling_rate=sampling_rate, method="BioSPPy") # Truncate so that first and last events are troughs RSP_Troughs = info['RSP_Troughs'] RSP_Peaks = info['RSP_Peaks'] # at this point, check that there are some peaks/trough if RSP_Troughs.size == 0 or RSP_Peaks.size == 0: raise NoRespirationPeaks( 'No peaks/trough could be detected in PZT signal. ') RSP_Peaks = RSP_Peaks[(RSP_Peaks > RSP_Troughs[0]) & (RSP_Peaks <= RSP_Troughs[-1])] # Estimate Inspiration and Expiration durations, cycle (I+E) durations and amplitude I_duration = (RSP_Peaks - RSP_Troughs[:-1]) / sampling_rate E_duration = (RSP_Troughs[1:] - RSP_Peaks) / sampling_rate IE_duration = np.ediff1d(RSP_Troughs) / sampling_rate amplitude = data.iloc[RSP_Peaks].values - data.iloc[ RSP_Troughs[:-1]].values IE_ratio = I_duration / E_duration # Back to Dataframe format to allow extracting feature between events I_duration_df = pd.DataFrame(index=_index[RSP_Peaks], columns=['I_duration'], data=I_duration) E_duration_df = pd.DataFrame(index=_index[RSP_Troughs[1:]], columns=['E_duration'], data=E_duration) IE_duration_df = pd.DataFrame(index=_index[RSP_Troughs[1:]], columns=['IE_duration'], data=IE_duration) amplitude_df = pd.DataFrame(index=_index[RSP_Peaks], columns=['IE_amplitude'], data=amplitude) IE_ratio_df = pd.DataFrame(index=_index[RSP_Peaks], columns=['IE_ratio'], data=IE_ratio) cycles_df = pd.concat([ I_duration_df, E_duration_df, IE_duration_df, amplitude_df, IE_ratio_df ], axis=1) known_sequences = known_sequences or VALID_SEQUENCE_KEYS features = [] # for name, row in events.T.iterrows(): # transpose due to https://github.com/OpenMindInnovation/iguazu/issues/54 for index, row in events.iterrows(): logger.debug('Processing sequence %s at %s', row.id, index) if row.id not in known_sequences: continue begin = row.begin end = row.end cycles_sequence = cycles_df.loc[begin:end].copy() # extract features on sequence sequence_features = dataclass_to_dataframe( respiration_features(cycles_sequence)).rename_axis( index='id').reset_index() sequence_features.insert(0, 'reference', row.id) features.append(sequence_features) if len(features) > 0: features = pd.concat(features, axis='index', ignore_index=True, sort=False) logger.info('Generated a feature dataframe of shape %s', features.shape) else: logger.info('No features were generated') features = pd.DataFrame(columns=['id', 'reference', 'value']) return features
def detect_ssf_peaks(signal, *, fs=None, max_bpm=180, baseline_length=3, threshold_percentage=0.70, peak_memory_size=5): """ Detect peaks on a SSF signal This function performs the second part of the SSF-based PPG algorithm presented in Jang et al. (2014). "A Real-Time Pulse Peak Detection Algorithm for the Photoplethysmogram." International Journal of Electronics and Electrical Engineering, https://doi.org/10.12720/ijeee.2.1.45-49 with some particularities that have been left out from this paper but are available on its previous iteration in Jang et al. (2014). "A robust method for pulse peak determination in a digital volume pulse waveform with a wandering baseline." IEEE Transactions on Biomedical Circuits and Systems, https://doi.org/10.1109/TBCAS.2013.2295102 Briefly, it detects peaks on a SSF signal and then calculates and updates an adaptive threshold in order to keep only the most prominent peaks which correspond to PPG systolic peaks. The threshold is adaptative to account for a wandering baseline present on long PPG recordings. Initially, the threshold is set to a fraction of the max peaks on a baseline period. After this period, the threshold is updated every time a new peak is detected over the threshold. The timing of the SSF signal is important for this algorithm. For this reason, the input cannot be a numpy array; it must be a :py:class:`pandas.Series` whose index are timestamps. Parameters ---------- signal: pd.Series Input SSF signal with timestamps as index fs: float, optional Sampling frequency. If not set, it will be deduced from the `signal` index. max_bpm: float Expected maximum number for beats per minute. This value is used to avoid peaks that are too close to each other. baseline_length: float Length of the baseline period to initialize the adaptive threshold. threshold_percentage: float Fraction of the maximum or median peak used to update the threshold. peak_memory_size: int Number of peaks to keep in memory to update the threshold. Returns ------- peak_series: pd.Series A series with the PPG values where peaks were detected. The index of this series can be used to estimate beat-to-beat segments. threshold_series: pd.Series A series with the values of the dynamic threshold. """ if fs is None: fs = estimate_rate(signal) if threshold_percentage < 0: raise ValueError('threshold_percentage must be positive') # Some synonyms for shorter code min_dist = int(max_bpm * 60 / fs) memsize = peak_memory_size memratio = threshold_percentage # Peak detection on complete signal, all at once i_peaks, props = scipy.signal.find_peaks(signal.values, distance=min_dist, prominence=1e-2, plateau_size=0) # Use the left edge of the peak since SSF often a plateau idx_peaks = signal.index[props['left_edges']] # Calculate initial threshold on a baseline. # On the original paper, this is done on the first 3 seconds of the SSF signal, # using 70% of the max peak on this baseline baseline_start = signal.index[0] baseline_end = signal.index[0] + np.timedelta64(baseline_length, 's') baseline = signal[baseline_start:baseline_end] threshold = np.nan_to_num(baseline[idx_peaks].max() * memratio, nan=0) # Keep a peak and threshold series peaks_series = baseline[baseline > threshold][idx_peaks].dropna().rename( 'peaks') threshold_series = pd.Series([threshold], index=[baseline_start], name='threshold') for index, value in signal.loc[baseline_end:][idx_peaks].items(): if value > threshold: peaks_series.at[index] = value # Update the threshold to use the last N peaks. On the original paper # the details are lost but there is a reference that indicates that # it should be the 70% the median of the last 5 peaks threshold = np.nan_to_num(peaks_series.tail(n=memsize).median() * memratio, nan=0) threshold_series.at[index] = threshold return peaks_series, threshold_series