def setUp(self):
self.chain = SignalProcessor()
self.qualUtil = QualityUtil()
config = ConfigProvider().getProcessingConfig()
self.upperBound = config.get("upperBound")
self.lowerBound = config.get("lowerBound")
self.minQuality = config.get("minQual")
self.maxNaNValues = config.get("maxNaNValues")
def createEmotivDataCollector(collectedQueue):
collectorConfig = ConfigProvider().getCollectorConfig()
fields = collectorConfig.get("eegFields") + collectorConfig.get(
"gyroFields")
windowSize = collectorConfig.get("windowSeconds")
windowCount = collectorConfig.get("windowCount")
return EEGDataCollector(EmotivConnector(), collectedQueue, fields,
windowSize, windowCount)
def setUp(self):
self.chain = SignalProcessor()
self.qualUtil = QualityUtil()
config = ConfigProvider().getProcessingConfig()
self.upperBound = config.get("upperBound")
self.lowerBound = config.get("lowerBound")
self.minQuality = config.get("minQual")
self.maxNaNValues = config.get("maxNaNValues")
def __init__(self, verbose=False):
config = ConfigProvider().getProcessingConfig()
self.maxNaNValues = config.get("maxNaNValues")
self.lowerFreq = config.get("lowerFreq")
self.upperFreq = config.get("upperFreq")
self.samplingRate = ConfigProvider().getEmotivConfig().get("samplingRate")
self.qualUtil = QualityUtil()
self.sigUtil = SignalUtil()
self.verbose = verbose
def createDemoEEGDataCollector(demoFile, collectedQueue):
collectorConfig = ConfigProvider().getCollectorConfig()
fields = collectorConfig.get("eegFields") + collectorConfig.get(
"gyroFields")
windowSeconds = collectorConfig.get("windowSeconds")
windowCount = collectorConfig.get("windowCount")
datasource = Factory.createDummyPacketSource(demoFile)
return EEGDataCollector(datasource, collectedQueue, fields,
windowSeconds, windowCount, 128)
def __init__(self, verbose=False):
config = ConfigProvider().getProcessingConfig()
self.maxNaNValues = config.get("maxNaNValues")
self.lowerBound = config.get("lowerBound")
self.upperBound = config.get("upperBound")
self.normalize = config.get("normalize")
self.samplingRate = ConfigProvider().getEmotivConfig().get("samplingRate")
self.qualUtil = QualityUtil()
self.sigUtil = SignalUtil()
self.verbose = verbose
def _initPoSDBoS(demo):
posdbos = PoSDBoS()
posdbos.demo = demo
posdbos.running = True
posdbosConfig = ConfigProvider().getPoSDBoSConfig()
posdbos.drowsyMinCount = posdbosConfig.get("drowsyMinCount")
posdbos.awakeMinCount = posdbosConfig.get("awakeMinCount")
posdbos.classified = [0, 0]
posdbos.curClass = 0
posdbos.classCount = 0
posdbos.found = 0
return posdbos
def __init__(self):
config = ConfigProvider().getProcessingConfig()
self.lowerFreq = config.get("lowerFreq")
self.upperFreq = config.get("upperFreq")
self.samplingRate = ConfigProvider().getEmotivConfig().get(
"samplingRate")
self.sigUtil = SignalUtil()
def normGyroData(self):
config = ConfigProvider().getProcessingConfig()
for gyroField in self.getGyroHeader():
gyroCol = self.getColumn(gyroField) - config.get(
gyroField.lower() + "Ground")
self.setColumn(gyroField, gyroCol)
def __init__(self):
self.su = SignalUtil()
self.windowSeconds = ConfigProvider().getCollectorConfig().get(
"windowSeconds")
config = ConfigProvider().getProcessingConfig()
self.xMax = config.get("xMax")
self.yMax = config.get("yMax")
def getForTesting():
self = TestFactory
app = self._get()
app.nn = self.loadNeuralNetwork(scriptPath + "/test_data/test", False)
app.dm = DrowsinessMonitor()
collectorConfig = ConfigProvider().getCollectorConfig()
fields = collectorConfig.get("eegFields") + collectorConfig.get(
"gyroFields")
windowSeconds = collectorConfig.get("windowSeconds")
windowCount = collectorConfig.get("windowCount")
samplingRate = 128
filePath = scriptPath + "/test_data/example_1024.csv"
app.dc = self.createTestDataCollector(app.collectedQueue, fields,
windowSeconds, samplingRate,
windowCount, filePath)
app.dp = self.createDataProcessor(app.collectedQueue,
app.extractedQueue)
return app
class QualityUtil(object):
"""removes signal data with low quality"""
def __init__(self):
self.config = ConfigProvider().getProcessingConfig()
self.upperBound = self.config.get("upperBound")
self.lowerBound = self.config.get("lowerBound")
self.minQuality = self.config.get("minQual")
self.maxSeqLength = self.config.get("maxSeqLength")
self.maxNaNValues = self.config.get("maxNaNValues")
self.windowSeconds = ConfigProvider().getCollectorConfig().get(
"windowSeconds")
def _copyArray(self, data):
return copy(data[:])
def replaceOutliners(self,
data,
value=None,
lowerBound=None,
upperBound=None):
"""outliner values beyond 'lowerBound' and above 'upperBound' will be set to 'value'
if value is not set, the values will be set to upper and lower bound
inplace method
:param numpy.array data: list of values
:param float lowerBound: values < this param will be set to 'value'
:param float upperBound: values > this param will be set to 'value'
:param float value: default value for all outside the bounds
:return: data without outliners
:rtype: numpy.array
"""
if lowerBound == None:
lowerBound = self.lowerBound
if upperBound == None:
upperBound = self.upperBound
with errstate(invalid='ignore'
): #avoid warning because of DEFAULT_REPLACE_VALUE value
ret = self._copyArray(data)
if value == None:
#http://stackoverflow.com/questions/41329691/pythonic-way-to-replace-list-values-with-upper-and-lower-bound/41329750#41329750
clip(ret, lowerBound, upperBound, out=ret)
else:
# http://stackoverflow.com/questions/41654038/replace-list-values-above-and-below-thresholds-with-default-value-in-python/41654071#41654071
ret[(ret < lowerBound) | (ret > upperBound)] = value
return ret
def countOutliners(self, data, lowerBound=None, upperBound=None):
"""counts the outliner values beyond 'lowerBound' and above 'upperBound'
:param numpy.array data: list of values
:param float lowerBound: values < this param will be set to 'value'
:param float upperBound: values > this param will be set to 'value'
:return: number of outliners in data
:rtype: int
"""
if lowerBound == None:
lowerBound = self.lowerBound
if upperBound == None:
upperBound = self.upperBound
cdata = self.replaceOutliners(copy(data[:]), DEFAULT_REPLACE_VALUE,
lowerBound, upperBound)
return count_nonzero(isnan(cdata))
def replaceBadQuality(self, data, quality, value, threshold=None):
"""replaces values from data with value where quality < threshold
inplace method
:param numpy.array data: list of values
:param numpy.array quality: list of quality
:param float threshold: param to compare quality with
:param float value: param to replace data values
:return: data without bad quality values
:rtype: numpy.array
"""
if len(data) != len(quality):
raise ValueError("data and quality must have the same length")
if threshold == None:
threshold = self.minQuality
ret = self._copyArray(data)
ret[quality < threshold] = value
return ret
def countBadQuality(self, data, quality, threshold=None):
"""counts values from data with value where quality < threshold
inplace method
:param numpy.array data: list of values
:param numpy.array quality: list of quality
:param float threshold: param to compare quality with
:return: number of data with bad quality values
:rtype: int
"""
if len(data) != len(quality):
raise ValueError("data and quality must have the same length")
if threshold == None:
threshold = self.minQuality
count = 0
for _, qual in enumerate(quality):
if qual < threshold:
count += 1
return count
def countZeros(self, data):
'''calculates the number of countZeros in data
:param numpy.array data: list of values
:return: zero count
:rtype: int
'''
return len(data) - count_nonzero(data)
def replaceNans(self, data):
'''replaces NaNs in data with zero
:param numpy.array data: list of values
:return: data without Nan
:rtype: numpy.array
'''
return nan_to_num(self._copyArray(data))
def countNans(self, data):
'''calculates the number of NaNs in data
:param numpy.array data: list of values
:return: NaN count
:rtype: int
'''
return count_nonzero(isnan(data))
def isInvalidData(self, data):
'''considers a data set invalid, if there are more NaNs than maxNaNValues in the set
:param numpy.array data: list of values
:return: invalid
:rtype: boolean
'''
nonZero = count_nonzero(isnan(data))
#if nonZero > self.maxNaNValues:
# print nonZero
nonZero = round(nonZero / self.windowSeconds)
return self.maxNaNValues < nonZero
def replaceZeroSequences(self, data):
'''replaces zero sequences, which is an unwanted artefact, with DEFAULT_REPLACE_VALUE
see http://stackoverflow.com/questions/38584956/replace-a-zero-sequence-with-other-value
:param numpy.array data: list of values
:return: zero sequences replaced data
:rtype: numpy.array
'''
a_extm = hstack((True, data != 0, True))
mask = a_extm == binary_closing(a_extm,
structure=ones(self.maxSeqLength))
return where(~a_extm[1:-1] & mask[1:-1], DEFAULT_REPLACE_VALUE, data)
def countSequences(self, data):
seqList = self._getSequenceList(data)
return len([s for s in seqList if len(s) >= self.maxSeqLength])
def replaceSequences(self, data):
'''replaces any sequences of more than MAX_SEQUENCE_LENGTH same numbers in a row with DEFAULT_REPLACE_VALUE
see http://stackoverflow.com/questions/38584956/replace-a-zero-sequence-with-other-value
:param numpy.array data: list of values
:return: sequences replaced data
:rtype: numpy.array
'''
ret = self._copyArray(data)
seqList = self._getSequenceList(ret)
return array([item for l in seqList for item in l])
def _getSequenceList(self, data):
return array(
[self._getSequence(value, it) for value, it in groupby(data)])
def _getSequence(self, value, it):
itLen = sum(1 for _ in it) # length of iterator
if itLen >= self.maxSeqLength:
return [DEFAULT_REPLACE_VALUE] * itLen
else:
return [value] * itLen
def __init__(self):
config = ConfigProvider().getProcessingConfig()
self.lowerFreq = config.get("lowerFreq")
self.upperFreq = config.get("upperFreq")
self.samplingRate = ConfigProvider().getEmotivConfig().get("samplingRate")
self.sigUtil = SignalUtil()
class MNEUtilTest(BaseTest):
def setUp(self):
self.mne = MNEUtil()
self.config = ConfigProvider().getEmotivConfig()
self.eegData = self._readData()
def _readData(self):
return FileUtil().getDto(self.getData1024CSV())
def _createTestData(self):
header = ["F3", "F4", "AF3", "AF4"]
data = np.random.rand(4,512)
filePath = "test"
return self.mne.createMNEObject(data, header, None, [], filePath, 128)
def test__mergeData(self):
eegData = np.array([
[1, 1, 1, 1],
[2, 2, 2, 2],
[3, 3, 3, 3],
[4, 4, 4, 4]
])
gyroData = np.array([
[5, 5, 5, 5],
[6, 6, 6, 6]
])
mergedData = self.mne._mergeData(eegData, gyroData)
self.assertEquals(mergedData.shape, (6, 4))
def test_createMNEObjectFromDto_creation(self):
self.mne.createMNEObjectFromEEGDto(self.eegData)
def test_createMNEObject_creation(self):
self.mne.createMNEObject(self.eegData.getEEGData(), self.eegData.getEEGHeader(), self.eegData.getGyroData(), self.eegData.getGyroHeader(), self.eegData.filePath, self.eegData.getSamplingRate())
def test_createMNEObjectFromDto_getChannels(self):
channels = ["AF3", "F3"]
raw = self.mne.createMNEObjectFromEEGDto(self.eegData)
chanObj = self.mne.getChannels(raw, channels)
self.assertEqual(chanObj.info["nchan"], len(channels))
def test__createInfo_creation(self):
eegChannels = self.config.get("eegFields")
gyroChannels = self.config.get("gyroFields")
channels = eegChannels + gyroChannels
samplingRate = self.config.get("samplingRate")
info = self.mne._createEEGInfo(eegChannels, gyroChannels, "testFile", 128)
self.assertEquals(info["sfreq"], samplingRate)
self.assertEquals(info["nchan"], len(channels))
self.assertItemsEqual(info["ch_names"], channels)
@unittest.skip("work in progress")
def test_createMNEEpochsObject(self):
epochs = self.mne.createMNEEpochsObject(self.eegData, 1)
self.assertEqual(len(epochs.get_data()), 15)
def test__createEventsArray_overlapping(self):
raw = self._createTestData()
event_id = dict(drowsy=1)
events1 = self.mne._createEventsArray(raw, 1, False)
epochs1 = mne.Epochs(raw, events=events1, event_id=event_id, tmin=0.0, tmax=0.99, add_eeg_ref=True)
events2 = self.mne._createEventsArray(raw, 1)
epochs2 = mne.Epochs(raw, events=events2, event_id=event_id, tmin=0.0, tmax=0.99, add_eeg_ref=True)
for i in range(0, len(events1)):
data1 = epochs1[i].get_data()
data2 = epochs2[i*2].get_data()
self.assertTrue((data1 == data2).all())
@unittest.skip("todo")
def test_ICA(self):
raw = self.mne.createMNEObjectFromEEGDto(self.eegData)
logging.info(self.mne.ICA(raw))
from config.config import ConfigProvider
from posdbos.util.eog_extractor import EOGExtractor
from posdbos.util.file_util import FileUtil
from posdbos.util.mne_util import MNEUtil, MNEPlotter
from posdbos.processor.mne_processor import MNEProcessor, SignalPreProcessor, FreqProcessor
from numpy import count_nonzero, mean
warnings.filterwarnings(action='ignore')
mneUtil = MNEUtil()
mnePlotter = MNEPlotter()
fileUtil = FileUtil()
#eogExtractor = EOGExtractor()
procConfig = ConfigProvider().getProcessingConfig()
FILE_PATH = "E:/thesis/experiment/%s/"
sFreq = procConfig.get("resamplingRate")
icCount = procConfig.get("icCount")
probands = ConfigProvider().getExperimentConfig().get("probands")
eegFields = ConfigProvider().getEmotivConfig().get("eegFields")
def _filter(raw):
start = time.time()
mneUtil.bandpassFilterData(raw)
dur = time.time() - start
print "filter EEG: %.2f" % dur
def _resample(raw):
start = time.time()
raw.resample(sFreq, npad='auto', n_jobs=8, verbose=True)
def __init__(self):
"""This class does signal processor with raw signals"""
config = ConfigProvider().getProcessingConfig()
self.xGround = config.get("xGround")
self.yGround = config.get("yGround")
def rawWithNormedGyroAll(fileName="EEGICA", outName="EEGNormGyro"):
config = ConfigProvider().getProcessingConfig()
xGround, yGround = config.get("xGround"), config.get("yGround")
for proband in probands:
rawWithNormedGyro(proband, xGround, yGround, fileName, outName)
:organization: Reutlingen University
'''
import logging
logging.basicConfig(
level=logging.INFO,
format=
'%(asctime)s.%(msecs)03d %(levelname)-8s %(module)s.%(funcName)s:%(lineno)d %(message)s',
datefmt='%H:%M:%S')
from posdbos.util.file_util import FileUtil
import threading
from config.config import ConfigProvider
from posdbos.factory import Factory
exConfig = ConfigProvider().getExperimentConfig()
probands = exConfig.get("probands")
experimentDir = exConfig.get("filePath")
fileUtil = FileUtil()
def getFilePaths(fileName):
filePaths = []
for proband in probands:
filePath = "%s%s/" % (experimentDir, proband)
filePaths.append(filePath + fileName)
return filePaths
def splitDtos(filePaths):
awakes, drowsies = [], []
class QualityUtil(object):
"""removes signal data with low quality"""
def __init__(self):
self.config = ConfigProvider().getProcessingConfig()
self.upperBound = self.config.get("upperBound")
self.lowerBound = self.config.get("lowerBound")
self.minQuality = self.config.get("minQual")
self.maxSeqLength = self.config.get("maxSeqLength")
self.maxNaNValues = self.config.get("maxNaNValues")
def _copyArray(self, data):
return copy(data[:])
def replaceOutliners(self, data, value=None, lowerBound=None, upperBound=None):
"""outliner values beyond 'lowerBound' and above 'upperBound' will be set to 'value'
if value is not set, the values will be set to upper and lower bound
inplace method
:param numpy.array data: list of values
:param float lowerBound: values < this param will be set to 'value'
:param float upperBound: values > this param will be set to 'value'
:param float value: default value for all outside the bounds
:return: data without outliners
:rtype: numpy.array
"""
if lowerBound == None:
lowerBound=self.lowerBound
if upperBound == None:
upperBound=self.upperBound
#TODO could be nicer / faster?
# http://stackoverflow.com/questions/19666626/replace-all-elements-of-python-numpy-array-that-are-greater-than-some-value
with errstate(invalid='ignore'): #avoid warning because of DEFAULT_REPLACE_VALUE value
ret = self._copyArray(data)
if value == None:
ret[ret > upperBound] = upperBound
ret[ret < lowerBound] = lowerBound
else:
ret[ret > upperBound] = value
ret[ret < lowerBound] = value
return ret
def countOutliners(self, data, lowerBound=None, upperBound=None):
"""counts the outliner values beyond 'lowerBound' and above 'upperBound'
:param numpy.array data: list of values
:param float lowerBound: values < this param will be set to 'value'
:param float upperBound: values > this param will be set to 'value'
:return: number of outliners in data
:rtype: int
"""
if lowerBound == None:
lowerBound=self.lowerBound
if upperBound == None:
upperBound=self.upperBound
cdata = copy(data[:])
with errstate(invalid='ignore'):
cdata[cdata > upperBound] = DEFAULT_REPLACE_VALUE
cdata[cdata < lowerBound] = DEFAULT_REPLACE_VALUE
return count_nonzero(isnan(cdata))
def replaceBadQuality(self, data, quality, value, threshold=None):
"""replaces values from data with value where quality < threshold
inplace method
:param numpy.array data: list of values
:param numpy.array quality: list of quality
:param float threshold: param to compare quality with
:param float value: param to replace data values
:return: data without bad quality values
:rtype: numpy.array
"""
if len(data) != len(quality):
raise ValueError("data and quality must have the same length")
if threshold == None:
threshold = self.minQuality
#TODO make me nice
ret = self._copyArray(data)
for i, qual in enumerate(quality):
if qual < threshold:
ret[i] = value
return ret
def countBadQuality(self, data, quality, threshold=None):
"""counts values from data with value where quality < threshold
inplace method
:param numpy.array data: list of values
:param numpy.array quality: list of quality
:param float threshold: param to compare quality with
:return: number of data with bad quality values
:rtype: int
"""
if len(data) != len(quality):
raise ValueError("data and quality must have the same length")
if threshold == None:
threshold = self.minQuality
count = 0
for _, qual in enumerate(quality):
if qual < threshold:
count += 1
return count
def countZeros(self, data):
'''calculates the number of countZeros in data
:param numpy.array data: list of values
:return: zero count
:rtype: int
'''
return len(data) - count_nonzero(data)
def replaceNans(self, data):
'''replaces NaNs in data with zero
:param numpy.array data: list of values
:return: data without Nan
:rtype: numpy.array
'''
return nan_to_num(self._copyArray(data))
def countNans(self, data):
'''calculates the number of NaNs in data
:param numpy.array data: list of values
:return: NaN count
:rtype: int
'''
return count_nonzero(isnan(data))
def isInvalidData(self, data):
'''considers a data set invalid, if there are more NaNs than maxNaNValues in the set
:param numpy.array data: list of values
:return: invalid
:rtype: boolean
'''
return self.maxNaNValues < count_nonzero(isnan(data))
def replaceZeroSequences(self, data):
'''replaces zero sequences, which is an unwanted artefact, with DEFAULT_REPLACE_VALUE
see http://stackoverflow.com/questions/38584956/replace-a-zero-sequence-with-other-value
:param numpy.array data: list of values
:return: zero sequences replaced data
:rtype: numpy.array
'''
a_extm = hstack((True,data!=0,True))
mask = a_extm == binary_closing(a_extm,structure=ones(self.maxSeqLength))
return where(~a_extm[1:-1] & mask[1:-1],DEFAULT_REPLACE_VALUE, data)
def countSequences(self, data):
seqList = self._getSequenceList(data)
return len([s for s in seqList if len(s) >= self.maxSeqLength])
def replaceSequences(self, data):
'''replaces any sequences of more than MAX_SEQUENCE_LENGTH same numbers in a row with DEFAULT_REPLACE_VALUE
see http://stackoverflow.com/questions/38584956/replace-a-zero-sequence-with-other-value
:param numpy.array data: list of values
:return: sequences replaced data
:rtype: numpy.array
'''
ret = self._copyArray(data)
seqList = self._getSequenceList(ret)
return array( [ item for l in seqList for item in l ] )
def _getSequenceList(self, data):
return array([self._getSequence(value, it) for value, it in groupby(data)])
def _getSequence(self, value, it):
itLen = sum(1 for _ in it) # length of iterator
if itLen>=self.maxSeqLength:
return [ DEFAULT_REPLACE_VALUE ]*itLen
else:
return [ value ]*itLen