def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
"""Read a chunk of raw data"""
input_fname = self._filenames[fi]
data_ = np.genfromtxt(input_fname,
delimiter=',',
comments='%',
skip_footer=1)
"""
Dealing with the missing data
-----------------------------
When recording with OpenBCI over Bluetooth, it is possible for some of
the data packets, samples, to not be recorded. This does not happen
often but it poses a problem for maintaining proper sampling periods.
OpenBCI data format combats this by providing a counter on the sample
to know which ones are missing.
Solution
--------
Interpolate the missing samples by resampling the surrounding samples.
1. Find where the missing samples are.
2. Deal with the counter reset (resets after cycling a byte).
3. Resample given the diffs.
4. Insert resampled data in the array using the diff indices
(index + 1).
5. If number of missing samples is greater than the missing_tol, Values
are replaced with np.nan.
"""
# counter goes from 0 to 255, maxdiff is 255.
# make diff one like others.
missing_tol = self._raw_extras[fi]['missing_tol']
diff = np.abs(np.diff(data_[:, 0]))
diff = np.mod(diff, 126) - 1
missing_idx = np.where(diff != 0)[0]
missing_samps = diff[missing_idx].astype(int)
if missing_samps.size:
missing_nsamps = np.sum(missing_samps, dtype=int)
missing_cumsum = np.insert(np.cumsum(missing_samps), 0, 0)[:-1]
missing_data = np.empty((missing_nsamps, data_.shape[-1]),
dtype=float)
insert_idx = list()
for idx_, nn, ii in zip(missing_idx, missing_samps,
missing_cumsum):
missing_data[ii:ii + nn] = np.nanmean(data_[(idx_,
idx_ + 1), :],
axis=0)
if nn > missing_tol:
missing_data[ii:ii + nn] *= np.nan
warnings.warn('The number of missing samples exceeded the '
'missing_tol threshold.')
insert_idx.append([idx_] * nn)
insert_idx = np.hstack(insert_idx)
data_ = np.insert(data_, insert_idx, missing_data, axis=0)
# data_ dimensions are samples by channels. transpose for MNE.
data_ = data_[start:stop, 1:].T
_mult_cal_one(data, data_, idx, cals, mult)
def _read_segment_file(self, data, idx, fi, start, stop,
cals, mult):
"""Read a chunk of raw data"""
input_fname = self._filenames[fi]
data_ = np.genfromtxt(input_fname, delimiter=',', comments='%',
skip_footer=1)
"""
Dealing with the missing data
-----------------------------
When recording with OpenBCI over Bluetooth, it is possible for some of
the data packets, samples, to not be recorded. This does not happen
often but it poses a problem for maintaining proper sampling periods.
OpenBCI data format combats this by providing a counter on the sample
to know which ones are missing.
Solution
--------
Interpolate the missing samples by resampling the surrounding samples.
1. Find where the missing samples are.
2. Deal with the counter reset (resets after cycling a byte).
3. Resample given the diffs.
4. Insert resampled data in the array using the diff indices
(index + 1).
5. If number of missing samples is greater than the missing_tol, Values
are replaced with np.nan.
"""
# counter goes from 0 to 255, maxdiff is 255.
# make diff one like others.
missing_tol = self._raw_extras[fi]['missing_tol']
diff = np.abs(np.diff(data_[:, 0]))
diff = np.mod(diff, 126) - 1
missing_idx = np.where(diff != 0)[0]
missing_samps = diff[missing_idx].astype(int)
if missing_samps.size:
missing_nsamps = np.sum(missing_samps, dtype=int)
missing_cumsum = np.insert(np.cumsum(missing_samps), 0, 0)[:-1]
missing_data = np.empty((missing_nsamps, data_.shape[-1]),
dtype=float)
insert_idx = list()
for idx_, nn, ii in zip(missing_idx, missing_samps,
missing_cumsum):
missing_data[ii:ii + nn] = np.nanmean(data_[(idx_, idx_ + 1), :], axis=0)
if nn > missing_tol:
missing_data[ii:ii + nn] *= np.nan
warnings.warn('The number of missing samples exceeded the '
'missing_tol threshold.')
insert_idx.append([idx_] * nn)
insert_idx = np.hstack(insert_idx)
data_ = np.insert(data_, insert_idx, missing_data, axis=0)
# data_ dimensions are samples by channels. transpose for MNE.
data_ = data_[start:stop, 1:].T
_mult_cal_one(data, data_, idx, cals, mult)
def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
one = np.full((8, stop - start), np.nan)
one[idx] = np.arange(1, 9)[idx, np.newaxis]
_mult_cal_one(data, one, idx, cals, mult)