class SignalProcessor(object):
def __init__(self, verbose=False):
config = ConfigProvider().getProcessingConfig()
self.maxNaNValues = config.get("maxNaNValues")
self.lowerFreq = config.get("lowerFreq")
self.upperFreq = config.get("upperFreq")
self.samplingRate = ConfigProvider().getEmotivConfig().get("samplingRate")
self.qualUtil = QualityUtil()
self.sigUtil = SignalUtil()
self.verbose = verbose
def process(self, raw, quality):
raw = self._replaceBadQuality(raw, quality)
raw = self._replaceSequences(raw)
raw = self._replaceOutliners(raw)
raw = self._normalize(raw)
invalid = self.qualUtil.isInvalidData(raw)
return raw, invalid
def _replaceBadQuality(self, raw, quality):
if self.verbose:
print "badQuality: %d" % self.qualUtil.countBadQuality(raw, quality)
raw = self.qualUtil.replaceBadQuality(raw, quality, NaN)
self._printNaNCount(raw)
return raw
def _replaceSequences(self, raw):
if self.verbose:
print "sequences: %d" % self.qualUtil.countSequences(raw)
raw = self.qualUtil.replaceSequences(raw)
self._printNaNCount(raw)
return raw
def _replaceOutliners(self, raw):
if self.verbose:
print "outliners: %d" % self.qualUtil.countOutliners(raw)
raw = self.qualUtil.replaceOutliners(raw, NaN)
self._printNaNCount(raw)
return raw
def _normalize(self, raw):
if self.verbose:
print "normalize: min %.2f max %.2f" % (self.sigUtil.minimum(raw),self.sigUtil.maximum(raw))
raw = self.sigUtil.normalize(raw)
if self.verbose:
print "normalize: min %.2f max %.2f" % (self.sigUtil.minimum(raw),self.sigUtil.maximum(raw))
self._printNaNCount(raw)
return raw
def _printNaNCount(self, raw):
if self.verbose:
print "NaN count: %s" % self.qualUtil.countNans(raw)
def __init__(self, verbose=False):
config = ConfigProvider().getProcessingConfig()
self.maxNaNValues = config.get("maxNaNValues")
self.lowerFreq = config.get("lowerFreq")
self.upperFreq = config.get("upperFreq")
self.samplingRate = ConfigProvider().getEmotivConfig().get("samplingRate")
self.qualUtil = QualityUtil()
self.sigUtil = SignalUtil()
self.verbose = verbose
class SignalPreProcessor(object):
def __init__(self):
config = ConfigProvider().getProcessingConfig()
self.lowerFreq = config.get("lowerFreq")
self.upperFreq = config.get("upperFreq")
self.samplingRate = ConfigProvider().getEmotivConfig().get("samplingRate")
self.sigUtil = SignalUtil()
def process(self, raw):
return self.sigUtil.butterBandpassFilter(raw, self.lowerFreq, self.upperFreq, self.samplingRate)
class BandpassFilteredSignalPlotter(RawSignalPlotter):
def __init__(self, person, eegData, signals, filePath, save=True, plot=True, logScale=False):
RawSignalPlotter.__init__(self, person, eegData, signals, filePath, save, plot, logScale, name="processedAlpha")
self.chain = SignalProcessor()
self.sigUtil = SignalUtil()
def _getData(self, signal):
raw = self.eegData.getColumn(signal)
qual = self.eegData.getQuality(signal)
return self.sigUtil.butterBandpassFilter(raw, 0.5, 50, 128)
def __init__(self):
self.signalUtil = SignalUtil()
class EEGUtil(object):
'''
This class does useful things with EEG signals
Like splitting by channel
DELTA = 0.5 - 4hz
THETA = 4 - 8hz
ALPHA = 8 - 13hz
BETA = 13 - 30hz
GAMMA = 30 - 99hz
'''
channel_ranges = {
"delta": DELTA_RANGE,
"theta": THETA_RANGE,
"alpha": ALPHA_RANGE,
"beta": BETA_RANGE,
"gamma": GAMMA_RANGE}
def __init__(self):
self.signalUtil = SignalUtil()
def getChannels(self, fft):
'''
Get a fft signal split by channels
{
"delta": [1, 2, 3],
...
"gamma": [5, 6, 7]
}
:param array fft: eeg data with performed fft
:return: split channels as map
:rtype: dict
'''
channels = {}
for label, freqRange in EEGUtil.channel_ranges.iteritems():
channels[label] = fft[slice(*self._getSliceParam(freqRange))]
return channels
def _getSliceParam(self, freqRange):
return (int(ceil(freqRange[0])), int(ceil(freqRange[1])))
def getDeltaChannel(self, fft):
'''
Get delta channel of a fft signal
:param array fft: eeg data with performed fft
:return: split from 0.5-4hz
:rtype: array
'''
return fft[slice(*self._getSliceParam(self.channel_ranges["delta"]))]
def getThetaChannel(self, fft):
'''
Get theta channel of a fft signal
:param array fft: eeg data with performed fft
:return: split from 4-8hz
:rtype: array
'''
return fft[slice(*self.channel_ranges["theta"])]
def getAlphaChannel(self, fft):
'''
Get alpha channel of a fft signal
:param array fft: eeg data with performed fft
:return: split from 8-13hz
:rtype: array
'''
return fft[slice(*self.channel_ranges["alpha"])]
def getBetaChannel(self, fft):
'''
Get delta channel of a fft signal
:param array fft: eeg data with performed fft
:return: split from 13-30hz
:rtype: array
'''
return fft[slice(*self.channel_ranges["beta"])]
def getGammaChannel(self, fft):
'''
Get delta channel of a fft signal
:param array fft: eeg data with performed fft
:return: split from 30-99hz
:rtype: array
'''
return fft[slice(*self.channel_ranges["gamma"])]
def getWaves(self, eeg, samplingRate):
'''
Get a eeg signal split by waves
{
"delta": [1, 2, 3],
...
"gamma": [5, 6, 7]
}
:param numpy.array eeg: eeg data with performed butterworth filter
:return: split channels as map
:rtype: dict
'''
waves = {}
for label, (lowcut, highcut) in EEGUtil.channel_ranges.iteritems():
waves[label] = self.signalUtil.butterBandpassFilter(eeg, lowcut, highcut, samplingRate)
return waves
def getDeltaWaves(self, eeg, samplingRate):
'''
Get band pass filtered delta waves (0.5-4hz) of a eeg signal
:param numpy.array eeg raw eeg data
:return: filtered signal
:rtype: numpy.array
'''
lowcut, highcut = self.channel_ranges["delta"]
return self.signalUtil.butterBandpassFilter(eeg, lowcut, highcut, samplingRate)
def getThetaWaves(self, eeg, samplingRate):
'''
Get band pass filtered theta waves (4-8hz) of an eeg signal
:param numpy.array eeg: raw eeg data
:return: filtered signal
:rtype: numpy.array
'''
lowcut, highcut = self.channel_ranges["theta"]
return self.signalUtil.butterBandpassFilter(eeg, lowcut, highcut, samplingRate)
def getAlphaWaves(self, eeg, samplingRate):
'''
Get band pass filtered alpha waves (8-13hz) of an eeg signal
:param numpy.array eeg raw eeg data
:return: filtered signal
:rtype: numpy.array
'''
lowcut, highcut = self.channel_ranges["alpha"]
return self.signalUtil.butterBandpassFilter(eeg, lowcut, highcut, samplingRate)
def getBetaWaves(self, eeg, samplingRate):
'''
Get band pass filtered beta waves (13-30hz) of an eeg signal
:param numpy.array eeg raw eeg data
:return: filtered signal
:rtype: numpy.array
'''
lowcut, highcut = self.channel_ranges["beta"]
return self.signalUtil.butterBandpassFilter(eeg, lowcut, highcut, samplingRate)
def getGammaWaves(self, eeg, samplingRate):
'''
Get band pass filtered gamma waves (30-99hz) or 30-samplingRate/2 of a eeg signal
:param numpy.array eeg raw eeg data
:return: filtered signal
:rtype: numpy.array
'''
lowcut, highcut = self.channel_ranges["gamma"]
return self.signalUtil.butterBandpassFilter(eeg, lowcut, highcut, samplingRate)
def __init__(self):
config = ConfigProvider().getProcessingConfig()
self.lowerFreq = config.get("lowerFreq")
self.upperFreq = config.get("upperFreq")
self.samplingRate = ConfigProvider().getEmotivConfig().get("samplingRate")
self.sigUtil = SignalUtil()
class BandPassExample(object):
def __init__(self):
self.su = SignalUtil()
def getSignal(self, a, t, f0):
x = 0.1 * np.sin(2 * np.pi * 1.2 * np.sqrt(t))
x += 0.01 * np.cos(2 * np.pi * 312 * t + 0.1)
x += a * np.cos(2 * np.pi * f0 * t + .11)
x += 0.03 * np.cos(2 * np.pi * 2000 * t)
return x
def getEEGSignal(self):
scriptPath = os.path.dirname(os.path.abspath(__file__))
return EEGTableFileUtil().readFile(scriptPath + "/../../examples/example_1024.csv")
def plotBBF(self, fs, lowcut, highcut):
# Plot the frequency response for a few different orders.
plt.figure(1)
plt.clf()
for order in [2, 4, 8]:
b, a = self.su.butterBandpass(lowcut, highcut, fs, order=order)
w, h = freqz(b, a, worN=2000)
plt.doPlot((fs * 0.5 / np.pi) * w, abs(h), label="order = %d" % order)
plt.doPlot([0, 0.5 * fs], [np.sqrt(0.5), np.sqrt(0.5)], '--', label='sqrt(0.5)')
plt.xlabel('Frequency (Hz)')
plt.ylabel('Gain')
plt.grid(True)
plt.legend(loc='best')
def getDuration(self, data):
time = data.getColumn("Timestamp")
duration = time[len(time) - 1] - time[0]
return duration
def plotEEGSignal(self):
# Filter a noisy signal.
data = self.getEEGSignal()
fs = data.getSamplingRate();
x = data.getColumn("F3")
x = SignalUtil().normalize(x)
duration = self.getDuration(data)
t = np.linspace(0, duration, len(x), endpoint=False)
plt.figure(2)
plt.clf()
plt.doPlot(t, x, label='normalized signal')
for label, (lowcut, highcut) in EEGUtil().channel_ranges.iteritems():
if label == "alpha":
y = self.su.butterBandpassFilter(x, lowcut, highcut, fs, order=6)
plt.doPlot(t, y, label='%s (%d - %dHz)' % (label, lowcut, highcut))
plt.xlabel('time (seconds)')
plt.grid(True)
plt.axis('tight')
plt.legend(loc='upper left')
def plotSoundSignal(self):
# Filter a noisy signal.
path = "../../examples/"
fs, s1 = wavfile.read(path + '12000hz.wav')
if len(s1.shape) == 2:
s1 = s1[:,0]
n = float(len(s1))
t = arange(0, n, 1)
t = t / fs
t = t * 1000 #scale to milliseconds
plt.figure(2)
plt.clf()
plt.doPlot(t, s1, label='sound signal')
y = self.su.butterBandpassFilter(s1, 11000, 13000, fs, order=6)
#wavfile.write(path + '12000hz_cut.wav', fs, y)
plt.doPlot(t, y, label='%s (%d - %dHz)' % ("12000Hz", 11000, 13000))
plt.xlabel('time (seconds)')
plt.grid(True)
plt.axis('tight')
plt.legend(loc='upper left')
def plotSignal(self, fs, lowcut, highcut):
# Filter a noisy signal.
T = 0.05
nsamples = T * fs
t = np.linspace(0, T, nsamples, endpoint=False)
a = 0.02
f0 = 600.0 # frequency
x = self.getSignal(a, t, f0)
plt.figure(2)
plt.clf()
plt.doPlot(t, x, label='Noisy signal')
y = self.butterBandpassFilter(x, lowcut, highcut, fs, order=6)
plt.doPlot(t, y, label='Filtered signal (%g Hz)' % f0)
plt.xlabel('time (seconds)')
plt.hlines([-a, a], 0, T, linestyles='--')
plt.grid(True)
plt.axis('tight')
plt.legend(loc='upper left')
def main(self):
# Sample rate and desired cutoff frequencies (in Hz).
fs = 5000.0
lowcut = 500.0
highcut = 1250.0
self.plotBBF(fs, lowcut, highcut)
#self.plotSignal(fs, lowcut, highcut)
self.plotEEGSignal()
#self.plotSoundSignal()
plt.show()
def __init__(self, person, eegData, signals, filePath, save=True, plot=True, logScale=False):
RawSignalPlotter.__init__(self, person, eegData, signals, filePath, save, plot, logScale, name="processedAlpha")
self.chain = SignalProcessor()
self.sigUtil = SignalUtil()
def setUp(self):
self.util = SignalUtil()
self._setSignalParams()
class TestFrequencyFilter(unittest.TestCase):
def setUp(self):
self.util = SignalUtil()
self._setSignalParams()
def _setSignalParams(self, offset=0, length=-1):
self.samplingRate, self.data= self._readSoundFile(offset=offset, length=length)
self.n = len(self.data)
self.nUniquePts = ceil((self.n+1)/2.0)
self.freqArray = np.arange(0, self.nUniquePts, 1.0) * (self.samplingRate / self.n)
def test_butterBandpass(self):
samplingRate = 32
i = samplingRate / 4
b, a = self.util.butterBandpass(i-1, i+1, samplingRate)
_, h = freqz(b, a, worN=samplingRate*4)
h = abs(h)
self.assertEqual(np.argmax(h), len(h)/2)
self.assertAlmostEqual(max(h), 1, delta=0.1)
self.assertAlmostEqual(h[0], 0, delta=0.01)
self.assertAlmostEqual(h[len(h)-1], 0, delta=0.01)
def test_butterBandpass_egde(self):
# no signal allowed
b, a = self.util.butterBandpass(5, 5, 16)
_, h = freqz(b, a, worN=32)
self.assertEquals(np.count_nonzero(abs(h)), 0)
# everything gets through (except 0)
b, a = self.util.butterBandpass(0, 8, 16)
_, h = freqz(b, a, worN=16)
self.assertAlmostEqual(sum(abs(h)), len(h)-1, delta = 1)
b, a = self.util.butterBandpass(0.5, 8, 16)
_, h = freqz(b, a, worN=16)
self.assertAlmostEqual(sum(abs(h)), len(h)-1, delta = 1)
b, a = self.util.butterBandpass(-1, 9, 16)
_, h = freqz(b, a, worN=16)
self.assertAlmostEqual(sum(abs(h)), len(h)-1, delta = 1)
def test_butterBandpassFilter(self):
x = [1, -1, 1, -1]
_ = self.util.butterBandpassFilter(x, 1, 2, 10)
def _readSoundFile(self, fileName='12000hz.wav', offset=0, length=-1):
samplingRate, data = wavfile.read(PATH + fileName)
if len(data.shape) == 2:
data = data[:,0]
if(length > -1):
data = data[offset:offset+length]
data = self.util.normalize(data)
return samplingRate, data
def _getMaxFrequency(self, data, freqArray):
fft = FFTUtil().fft(data)
return freqArray[np.argmax(fft)]
def test_butterBandpassFilter_original(self):
freqMax = self._getMaxFrequency(self.data, self.freqArray)
self.assertAlmostEqual(freqMax, 12000.0, delta= 1000)
def test_butterBandpassFilter_filterNoCut(self):
cut = self.util.butterBandpassFilter(self.data, 1000, self.samplingRate/2-1000, self.samplingRate)
freqMax = self._getMaxFrequency(cut, self.freqArray)
self.assertAlmostEqual(freqMax, 12000.0, delta= 1000)
def test_butterBandpassFilter_filterAroundMax(self):
cut = self.util.butterBandpassFilter(self.data, 11000, 13000, self.samplingRate)
freqMax = self._getMaxFrequency(cut, self.freqArray)
self.assertAlmostEqual(freqMax, 12000.0, delta= 1000)
def test_butterBandpassFilter_filterAboveMax(self):
cut = self.util.butterBandpassFilter(self.data, 0, 2000, self.samplingRate)
freqMax = self._getMaxFrequency(cut, self.freqArray)
self.assertNotEqual(freqMax, 12000.0)
self.assertAlmostEqual(freqMax, 1000.0, delta= 1000)
def test_butterBandpassFilter_filterBeyondMax(self):
cut = self.util.butterBandpassFilter(self.data, self.samplingRate/2-2000, self.samplingRate/2, self.samplingRate)
freqMax = self._getMaxFrequency(cut, self.freqArray)
self.assertNotEqual(freqMax, 12000.0)
self.assertAlmostEqual(freqMax, self.samplingRate/2-1000, delta= 1000)
def test_butterBandpassFilter_short(self):
self._setSignalParams(length=4)
freqMax = self._getMaxFrequency(self.data, self.freqArray)
self.assertAlmostEqual(freqMax, 12000.0, delta= 1000)
cut = self.util.butterBandpassFilter(self.data, 1000, self.samplingRate/2-1000, self.samplingRate)
freqMax = self._getMaxFrequency(cut, self.freqArray)
self.assertAlmostEqual(freqMax, 12000.0, delta= 1000)
def test_butterBandpassFilter_originalAndShort(self):
cutOrig = self.util.butterBandpassFilter(self.data, 1000, self.samplingRate/2-1000, self.samplingRate)
freqMaxOrig = self._getMaxFrequency(self.data, self.freqArray)
self._setSignalParams(length=4)
cutShort = self.util.butterBandpassFilter(self.data, 1000, self.samplingRate/2-1000, self.samplingRate)
freqMaxShort = self._getMaxFrequency(self.data, self.freqArray)
self.assertAlmostEqual(freqMaxShort, freqMaxOrig, delta=1000)
for i in range(len(cutShort)):
self.assertAlmostEqual(cutOrig[i], cutShort[i], delta=0.02)
def setUp(self):
self.util = SignalUtil()
class TestSignalUtil(unittest.TestCase):
def setUp(self):
self.util = SignalUtil()
def test_normalize(self):
testList = np.array([0, -5, 1, 10])
normList = self.util.normalize(testList)
self.assertEqual(len(testList), len(normList))
self.assertTrue(max(normList) <= 1)
self.assertTrue(min(normList) >= -1)
def test_normalize_value(self):
norm = ConfigProvider().getProcessingConfig().get("normalize")
testList = np.array([0, -5, 1, 10])
normList = self.util.normalize(testList, norm)
self.assertEqual(len(testList), len(normList))
self.assertItemsEqual(normList, testList / norm)
def test_normalize_zero(self):
testList = np.array([0, 0, 0, 0])
normList = self.util.normalize(testList)
self.assertEqual(len(testList), len(normList))
self.assertTrue(max(normList) <= 1)
self.assertTrue(min(normList) >= -1)
self.assertTrue(sameEntries(testList, normList))
def test_normalize_NaN(self):
testList = np.array([np.NaN, -2, -1, 0, np.NaN, 1, 2, np.NaN])
normList = self.util.normalize(testList)
self.assertEqual(len(testList), len(normList))
self.assertTrue(np.nanmax(normList) <= 1)
self.assertTrue(np.nanmin(normList) >= -1)
def test_energy(self):
testList = np.array([1, 2, 3, 4])
energy = self.util.energy(testList)
self.assertEqual(energy, 30)
def test_maximum(self):
testList = np.array([-5, 1, 2, 3, 4])
maximum = self.util.maximum(testList)
self.assertEqual(maximum, 4)
def test_minimum(self):
testList = np.array([-5, 1, 2, 3, 6])
minimum = self.util.minimum(testList)
self.assertEqual(minimum, -5)
def test_mean(self):
testList = np.array([0, 1, 2, 3, 4])
mean = self.util.mean(testList)
self.assertEqual(mean, 2)
def test_var(self):
testList = np.array([0, 1, 2, 3, 4])
var = self.util.var(testList)
self.assertEqual(var, 2)
def test_std(self):
testList = np.array([0, 1, 2, 3, 4])
std = self.util.std(testList)
self.assertEqual(std, sqrt(self.util.var(testList)))
def test_zcr(self):
testList = np.array([1, -1, 1, -1, 1])
zcr = self.util.zcr(testList)
self.assertEqual(zcr, 4)
def test_zcr_zeros(self):
testList = np.array([0, 0, 0, 0, 0])
zcr = self.util.zcr(testList)
self.assertEqual(zcr, 0)
testList = np.array([1, 0, -1, 0, 1, 0, -1])
zcr = self.util.zcr(testList)
self.assertEqual(zcr, 3)
def test_zcr_zeroChanges(self):
testList = np.array([1, 1, 1, 1, 1])
zcr = self.util.zcr(testList)
self.assertEqual(zcr, 0)
testList = np.array([-1, -1, -1, -1, -1])
zcr = self.util.zcr(testList)
self.assertEqual(zcr, 0)
def test_nan_onOtherFunctions(self):
norm = self.util.normalize(TEST_DATA_NAN)
self.assertItemsEqual(np.isnan(norm), np.isnan(TEST_DATA_NAN))
maxi = self.util.maximum(TEST_DATA_NAN)
self.assertTrue(np.isnan(maxi))
mini = self.util.minimum(TEST_DATA_NAN)
self.assertTrue(np.isnan(mini))
mean = self.util.mean(TEST_DATA_NAN)
self.assertTrue(np.isnan(mean))
var = self.util.var(TEST_DATA_NAN)
self.assertTrue(np.isnan(var))
std = self.util.std(TEST_DATA_NAN)
self.assertTrue(np.isnan(std))
energy = self.util.energy(TEST_DATA_NAN)
self.assertTrue(np.isnan(energy))
zcr = self.util.zcr(TEST_DATA_NAN)
self.assertTrue(np.isnan(zcr))
def test_mixed_onOtherFunctions(self):
norm = self.util.normalize(TEST_DATA_MIXED)
self.assertItemsEqual(np.isnan(norm), np.isnan(TEST_DATA_MIXED))
maxi = self.util.maximum(TEST_DATA_MIXED)
self.assertEquals(maxi, 1.0)
mini = self.util.minimum(TEST_DATA_MIXED)
self.assertEquals(mini, 0.0)
mean = self.util.mean(TEST_DATA_MIXED)
self.assertEquals(mean, 0.5)
var = self.util.var(TEST_DATA_MIXED)
self.assertEquals(var, 0.25)
std = self.util.std(TEST_DATA_MIXED)
self.assertEquals(std, 0.5)
energy = self.util.energy(TEST_DATA_MIXED)
self.assertEquals(energy, 2.0)
zcr = self.util.zcr(TEST_DATA_MIXED)
self.assertEquals(zcr, 0)
