def setUp(self): self.chain = SignalProcessor() self.qualUtil = QualityUtil() config = ConfigProvider().getProcessingConfig() self.upperBound = config.get("upperBound") self.lowerBound = config.get("lowerBound") self.minQuality = config.get("minQual") self.maxNaNValues = config.get("maxNaNValues")
def createEmotivDataCollector(collectedQueue): collectorConfig = ConfigProvider().getCollectorConfig() fields = collectorConfig.get("eegFields") + collectorConfig.get( "gyroFields") windowSize = collectorConfig.get("windowSeconds") windowCount = collectorConfig.get("windowCount") return EEGDataCollector(EmotivConnector(), collectedQueue, fields, windowSize, windowCount)
def setUp(self): self.chain = SignalProcessor() self.qualUtil = QualityUtil() config = ConfigProvider().getProcessingConfig() self.upperBound = config.get("upperBound") self.lowerBound = config.get("lowerBound") self.minQuality = config.get("minQual") self.maxNaNValues = config.get("maxNaNValues")
def __init__(self, verbose=False): config = ConfigProvider().getProcessingConfig() self.maxNaNValues = config.get("maxNaNValues") self.lowerFreq = config.get("lowerFreq") self.upperFreq = config.get("upperFreq") self.samplingRate = ConfigProvider().getEmotivConfig().get("samplingRate") self.qualUtil = QualityUtil() self.sigUtil = SignalUtil() self.verbose = verbose
def createDemoEEGDataCollector(demoFile, collectedQueue): collectorConfig = ConfigProvider().getCollectorConfig() fields = collectorConfig.get("eegFields") + collectorConfig.get( "gyroFields") windowSeconds = collectorConfig.get("windowSeconds") windowCount = collectorConfig.get("windowCount") datasource = Factory.createDummyPacketSource(demoFile) return EEGDataCollector(datasource, collectedQueue, fields, windowSeconds, windowCount, 128)
def __init__(self, verbose=False): config = ConfigProvider().getProcessingConfig() self.maxNaNValues = config.get("maxNaNValues") self.lowerBound = config.get("lowerBound") self.upperBound = config.get("upperBound") self.normalize = config.get("normalize") self.samplingRate = ConfigProvider().getEmotivConfig().get("samplingRate") self.qualUtil = QualityUtil() self.sigUtil = SignalUtil() self.verbose = verbose
def _initPoSDBoS(demo): posdbos = PoSDBoS() posdbos.demo = demo posdbos.running = True posdbosConfig = ConfigProvider().getPoSDBoSConfig() posdbos.drowsyMinCount = posdbosConfig.get("drowsyMinCount") posdbos.awakeMinCount = posdbosConfig.get("awakeMinCount") posdbos.classified = [0, 0] posdbos.curClass = 0 posdbos.classCount = 0 posdbos.found = 0 return posdbos
def __init__(self): config = ConfigProvider().getProcessingConfig() self.lowerFreq = config.get("lowerFreq") self.upperFreq = config.get("upperFreq") self.samplingRate = ConfigProvider().getEmotivConfig().get( "samplingRate") self.sigUtil = SignalUtil()
def normGyroData(self): config = ConfigProvider().getProcessingConfig() for gyroField in self.getGyroHeader(): gyroCol = self.getColumn(gyroField) - config.get( gyroField.lower() + "Ground") self.setColumn(gyroField, gyroCol)
def __init__(self): self.su = SignalUtil() self.windowSeconds = ConfigProvider().getCollectorConfig().get( "windowSeconds") config = ConfigProvider().getProcessingConfig() self.xMax = config.get("xMax") self.yMax = config.get("yMax")
def getForTesting(): self = TestFactory app = self._get() app.nn = self.loadNeuralNetwork(scriptPath + "/test_data/test", False) app.dm = DrowsinessMonitor() collectorConfig = ConfigProvider().getCollectorConfig() fields = collectorConfig.get("eegFields") + collectorConfig.get( "gyroFields") windowSeconds = collectorConfig.get("windowSeconds") windowCount = collectorConfig.get("windowCount") samplingRate = 128 filePath = scriptPath + "/test_data/example_1024.csv" app.dc = self.createTestDataCollector(app.collectedQueue, fields, windowSeconds, samplingRate, windowCount, filePath) app.dp = self.createDataProcessor(app.collectedQueue, app.extractedQueue) return app
class QualityUtil(object): """removes signal data with low quality""" def __init__(self): self.config = ConfigProvider().getProcessingConfig() self.upperBound = self.config.get("upperBound") self.lowerBound = self.config.get("lowerBound") self.minQuality = self.config.get("minQual") self.maxSeqLength = self.config.get("maxSeqLength") self.maxNaNValues = self.config.get("maxNaNValues") self.windowSeconds = ConfigProvider().getCollectorConfig().get( "windowSeconds") def _copyArray(self, data): return copy(data[:]) def replaceOutliners(self, data, value=None, lowerBound=None, upperBound=None): """outliner values beyond 'lowerBound' and above 'upperBound' will be set to 'value' if value is not set, the values will be set to upper and lower bound inplace method :param numpy.array data: list of values :param float lowerBound: values < this param will be set to 'value' :param float upperBound: values > this param will be set to 'value' :param float value: default value for all outside the bounds :return: data without outliners :rtype: numpy.array """ if lowerBound == None: lowerBound = self.lowerBound if upperBound == None: upperBound = self.upperBound with errstate(invalid='ignore' ): #avoid warning because of DEFAULT_REPLACE_VALUE value ret = self._copyArray(data) if value == None: #http://stackoverflow.com/questions/41329691/pythonic-way-to-replace-list-values-with-upper-and-lower-bound/41329750#41329750 clip(ret, lowerBound, upperBound, out=ret) else: # http://stackoverflow.com/questions/41654038/replace-list-values-above-and-below-thresholds-with-default-value-in-python/41654071#41654071 ret[(ret < lowerBound) | (ret > upperBound)] = value return ret def countOutliners(self, data, lowerBound=None, upperBound=None): """counts the outliner values beyond 'lowerBound' and above 'upperBound' :param numpy.array data: list of values :param float lowerBound: values < this param will be set to 'value' :param float upperBound: values > this param will be set to 'value' :return: number of outliners in data :rtype: int """ if lowerBound == None: lowerBound = self.lowerBound if upperBound == None: upperBound = self.upperBound cdata = self.replaceOutliners(copy(data[:]), DEFAULT_REPLACE_VALUE, lowerBound, upperBound) return count_nonzero(isnan(cdata)) def replaceBadQuality(self, data, quality, value, threshold=None): """replaces values from data with value where quality < threshold inplace method :param numpy.array data: list of values :param numpy.array quality: list of quality :param float threshold: param to compare quality with :param float value: param to replace data values :return: data without bad quality values :rtype: numpy.array """ if len(data) != len(quality): raise ValueError("data and quality must have the same length") if threshold == None: threshold = self.minQuality ret = self._copyArray(data) ret[quality < threshold] = value return ret def countBadQuality(self, data, quality, threshold=None): """counts values from data with value where quality < threshold inplace method :param numpy.array data: list of values :param numpy.array quality: list of quality :param float threshold: param to compare quality with :return: number of data with bad quality values :rtype: int """ if len(data) != len(quality): raise ValueError("data and quality must have the same length") if threshold == None: threshold = self.minQuality count = 0 for _, qual in enumerate(quality): if qual < threshold: count += 1 return count def countZeros(self, data): '''calculates the number of countZeros in data :param numpy.array data: list of values :return: zero count :rtype: int ''' return len(data) - count_nonzero(data) def replaceNans(self, data): '''replaces NaNs in data with zero :param numpy.array data: list of values :return: data without Nan :rtype: numpy.array ''' return nan_to_num(self._copyArray(data)) def countNans(self, data): '''calculates the number of NaNs in data :param numpy.array data: list of values :return: NaN count :rtype: int ''' return count_nonzero(isnan(data)) def isInvalidData(self, data): '''considers a data set invalid, if there are more NaNs than maxNaNValues in the set :param numpy.array data: list of values :return: invalid :rtype: boolean ''' nonZero = count_nonzero(isnan(data)) #if nonZero > self.maxNaNValues: # print nonZero nonZero = round(nonZero / self.windowSeconds) return self.maxNaNValues < nonZero def replaceZeroSequences(self, data): '''replaces zero sequences, which is an unwanted artefact, with DEFAULT_REPLACE_VALUE see http://stackoverflow.com/questions/38584956/replace-a-zero-sequence-with-other-value :param numpy.array data: list of values :return: zero sequences replaced data :rtype: numpy.array ''' a_extm = hstack((True, data != 0, True)) mask = a_extm == binary_closing(a_extm, structure=ones(self.maxSeqLength)) return where(~a_extm[1:-1] & mask[1:-1], DEFAULT_REPLACE_VALUE, data) def countSequences(self, data): seqList = self._getSequenceList(data) return len([s for s in seqList if len(s) >= self.maxSeqLength]) def replaceSequences(self, data): '''replaces any sequences of more than MAX_SEQUENCE_LENGTH same numbers in a row with DEFAULT_REPLACE_VALUE see http://stackoverflow.com/questions/38584956/replace-a-zero-sequence-with-other-value :param numpy.array data: list of values :return: sequences replaced data :rtype: numpy.array ''' ret = self._copyArray(data) seqList = self._getSequenceList(ret) return array([item for l in seqList for item in l]) def _getSequenceList(self, data): return array( [self._getSequence(value, it) for value, it in groupby(data)]) def _getSequence(self, value, it): itLen = sum(1 for _ in it) # length of iterator if itLen >= self.maxSeqLength: return [DEFAULT_REPLACE_VALUE] * itLen else: return [value] * itLen
def __init__(self): config = ConfigProvider().getProcessingConfig() self.lowerFreq = config.get("lowerFreq") self.upperFreq = config.get("upperFreq") self.samplingRate = ConfigProvider().getEmotivConfig().get("samplingRate") self.sigUtil = SignalUtil()
class MNEUtilTest(BaseTest): def setUp(self): self.mne = MNEUtil() self.config = ConfigProvider().getEmotivConfig() self.eegData = self._readData() def _readData(self): return FileUtil().getDto(self.getData1024CSV()) def _createTestData(self): header = ["F3", "F4", "AF3", "AF4"] data = np.random.rand(4,512) filePath = "test" return self.mne.createMNEObject(data, header, None, [], filePath, 128) def test__mergeData(self): eegData = np.array([ [1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4] ]) gyroData = np.array([ [5, 5, 5, 5], [6, 6, 6, 6] ]) mergedData = self.mne._mergeData(eegData, gyroData) self.assertEquals(mergedData.shape, (6, 4)) def test_createMNEObjectFromDto_creation(self): self.mne.createMNEObjectFromEEGDto(self.eegData) def test_createMNEObject_creation(self): self.mne.createMNEObject(self.eegData.getEEGData(), self.eegData.getEEGHeader(), self.eegData.getGyroData(), self.eegData.getGyroHeader(), self.eegData.filePath, self.eegData.getSamplingRate()) def test_createMNEObjectFromDto_getChannels(self): channels = ["AF3", "F3"] raw = self.mne.createMNEObjectFromEEGDto(self.eegData) chanObj = self.mne.getChannels(raw, channels) self.assertEqual(chanObj.info["nchan"], len(channels)) def test__createInfo_creation(self): eegChannels = self.config.get("eegFields") gyroChannels = self.config.get("gyroFields") channels = eegChannels + gyroChannels samplingRate = self.config.get("samplingRate") info = self.mne._createEEGInfo(eegChannels, gyroChannels, "testFile", 128) self.assertEquals(info["sfreq"], samplingRate) self.assertEquals(info["nchan"], len(channels)) self.assertItemsEqual(info["ch_names"], channels) @unittest.skip("work in progress") def test_createMNEEpochsObject(self): epochs = self.mne.createMNEEpochsObject(self.eegData, 1) self.assertEqual(len(epochs.get_data()), 15) def test__createEventsArray_overlapping(self): raw = self._createTestData() event_id = dict(drowsy=1) events1 = self.mne._createEventsArray(raw, 1, False) epochs1 = mne.Epochs(raw, events=events1, event_id=event_id, tmin=0.0, tmax=0.99, add_eeg_ref=True) events2 = self.mne._createEventsArray(raw, 1) epochs2 = mne.Epochs(raw, events=events2, event_id=event_id, tmin=0.0, tmax=0.99, add_eeg_ref=True) for i in range(0, len(events1)): data1 = epochs1[i].get_data() data2 = epochs2[i*2].get_data() self.assertTrue((data1 == data2).all()) @unittest.skip("todo") def test_ICA(self): raw = self.mne.createMNEObjectFromEEGDto(self.eegData) logging.info(self.mne.ICA(raw))
from config.config import ConfigProvider from posdbos.util.eog_extractor import EOGExtractor from posdbos.util.file_util import FileUtil from posdbos.util.mne_util import MNEUtil, MNEPlotter from posdbos.processor.mne_processor import MNEProcessor, SignalPreProcessor, FreqProcessor from numpy import count_nonzero, mean warnings.filterwarnings(action='ignore') mneUtil = MNEUtil() mnePlotter = MNEPlotter() fileUtil = FileUtil() #eogExtractor = EOGExtractor() procConfig = ConfigProvider().getProcessingConfig() FILE_PATH = "E:/thesis/experiment/%s/" sFreq = procConfig.get("resamplingRate") icCount = procConfig.get("icCount") probands = ConfigProvider().getExperimentConfig().get("probands") eegFields = ConfigProvider().getEmotivConfig().get("eegFields") def _filter(raw): start = time.time() mneUtil.bandpassFilterData(raw) dur = time.time() - start print "filter EEG: %.2f" % dur def _resample(raw): start = time.time() raw.resample(sFreq, npad='auto', n_jobs=8, verbose=True)
def __init__(self): """This class does signal processor with raw signals""" config = ConfigProvider().getProcessingConfig() self.xGround = config.get("xGround") self.yGround = config.get("yGround")
def rawWithNormedGyroAll(fileName="EEGICA", outName="EEGNormGyro"): config = ConfigProvider().getProcessingConfig() xGround, yGround = config.get("xGround"), config.get("yGround") for proband in probands: rawWithNormedGyro(proband, xGround, yGround, fileName, outName)
:organization: Reutlingen University ''' import logging logging.basicConfig( level=logging.INFO, format= '%(asctime)s.%(msecs)03d %(levelname)-8s %(module)s.%(funcName)s:%(lineno)d %(message)s', datefmt='%H:%M:%S') from posdbos.util.file_util import FileUtil import threading from config.config import ConfigProvider from posdbos.factory import Factory exConfig = ConfigProvider().getExperimentConfig() probands = exConfig.get("probands") experimentDir = exConfig.get("filePath") fileUtil = FileUtil() def getFilePaths(fileName): filePaths = [] for proband in probands: filePath = "%s%s/" % (experimentDir, proband) filePaths.append(filePath + fileName) return filePaths def splitDtos(filePaths): awakes, drowsies = [], []
class QualityUtil(object): """removes signal data with low quality""" def __init__(self): self.config = ConfigProvider().getProcessingConfig() self.upperBound = self.config.get("upperBound") self.lowerBound = self.config.get("lowerBound") self.minQuality = self.config.get("minQual") self.maxSeqLength = self.config.get("maxSeqLength") self.maxNaNValues = self.config.get("maxNaNValues") def _copyArray(self, data): return copy(data[:]) def replaceOutliners(self, data, value=None, lowerBound=None, upperBound=None): """outliner values beyond 'lowerBound' and above 'upperBound' will be set to 'value' if value is not set, the values will be set to upper and lower bound inplace method :param numpy.array data: list of values :param float lowerBound: values < this param will be set to 'value' :param float upperBound: values > this param will be set to 'value' :param float value: default value for all outside the bounds :return: data without outliners :rtype: numpy.array """ if lowerBound == None: lowerBound=self.lowerBound if upperBound == None: upperBound=self.upperBound #TODO could be nicer / faster? # http://stackoverflow.com/questions/19666626/replace-all-elements-of-python-numpy-array-that-are-greater-than-some-value with errstate(invalid='ignore'): #avoid warning because of DEFAULT_REPLACE_VALUE value ret = self._copyArray(data) if value == None: ret[ret > upperBound] = upperBound ret[ret < lowerBound] = lowerBound else: ret[ret > upperBound] = value ret[ret < lowerBound] = value return ret def countOutliners(self, data, lowerBound=None, upperBound=None): """counts the outliner values beyond 'lowerBound' and above 'upperBound' :param numpy.array data: list of values :param float lowerBound: values < this param will be set to 'value' :param float upperBound: values > this param will be set to 'value' :return: number of outliners in data :rtype: int """ if lowerBound == None: lowerBound=self.lowerBound if upperBound == None: upperBound=self.upperBound cdata = copy(data[:]) with errstate(invalid='ignore'): cdata[cdata > upperBound] = DEFAULT_REPLACE_VALUE cdata[cdata < lowerBound] = DEFAULT_REPLACE_VALUE return count_nonzero(isnan(cdata)) def replaceBadQuality(self, data, quality, value, threshold=None): """replaces values from data with value where quality < threshold inplace method :param numpy.array data: list of values :param numpy.array quality: list of quality :param float threshold: param to compare quality with :param float value: param to replace data values :return: data without bad quality values :rtype: numpy.array """ if len(data) != len(quality): raise ValueError("data and quality must have the same length") if threshold == None: threshold = self.minQuality #TODO make me nice ret = self._copyArray(data) for i, qual in enumerate(quality): if qual < threshold: ret[i] = value return ret def countBadQuality(self, data, quality, threshold=None): """counts values from data with value where quality < threshold inplace method :param numpy.array data: list of values :param numpy.array quality: list of quality :param float threshold: param to compare quality with :return: number of data with bad quality values :rtype: int """ if len(data) != len(quality): raise ValueError("data and quality must have the same length") if threshold == None: threshold = self.minQuality count = 0 for _, qual in enumerate(quality): if qual < threshold: count += 1 return count def countZeros(self, data): '''calculates the number of countZeros in data :param numpy.array data: list of values :return: zero count :rtype: int ''' return len(data) - count_nonzero(data) def replaceNans(self, data): '''replaces NaNs in data with zero :param numpy.array data: list of values :return: data without Nan :rtype: numpy.array ''' return nan_to_num(self._copyArray(data)) def countNans(self, data): '''calculates the number of NaNs in data :param numpy.array data: list of values :return: NaN count :rtype: int ''' return count_nonzero(isnan(data)) def isInvalidData(self, data): '''considers a data set invalid, if there are more NaNs than maxNaNValues in the set :param numpy.array data: list of values :return: invalid :rtype: boolean ''' return self.maxNaNValues < count_nonzero(isnan(data)) def replaceZeroSequences(self, data): '''replaces zero sequences, which is an unwanted artefact, with DEFAULT_REPLACE_VALUE see http://stackoverflow.com/questions/38584956/replace-a-zero-sequence-with-other-value :param numpy.array data: list of values :return: zero sequences replaced data :rtype: numpy.array ''' a_extm = hstack((True,data!=0,True)) mask = a_extm == binary_closing(a_extm,structure=ones(self.maxSeqLength)) return where(~a_extm[1:-1] & mask[1:-1],DEFAULT_REPLACE_VALUE, data) def countSequences(self, data): seqList = self._getSequenceList(data) return len([s for s in seqList if len(s) >= self.maxSeqLength]) def replaceSequences(self, data): '''replaces any sequences of more than MAX_SEQUENCE_LENGTH same numbers in a row with DEFAULT_REPLACE_VALUE see http://stackoverflow.com/questions/38584956/replace-a-zero-sequence-with-other-value :param numpy.array data: list of values :return: sequences replaced data :rtype: numpy.array ''' ret = self._copyArray(data) seqList = self._getSequenceList(ret) return array( [ item for l in seqList for item in l ] ) def _getSequenceList(self, data): return array([self._getSequence(value, it) for value, it in groupby(data)]) def _getSequence(self, value, it): itLen = sum(1 for _ in it) # length of iterator if itLen>=self.maxSeqLength: return [ DEFAULT_REPLACE_VALUE ]*itLen else: return [ value ]*itLen