def test_getSingleWaves(self):
eegData = EEGTableFileUtil().readFile(PATH + "example_32.csv")
eeg = eegData.getColumn("F3")
nEeg = len(eeg)
samplingRate = eegData.getSamplingRate()
waves = self.util.getWaves(eeg, samplingRate)
delta = self.util.getDeltaWaves(eeg, samplingRate)
self.assertEqual(len(delta), nEeg)
self.assertTrue(all([x in waves["delta"] for x in delta]))
theta = self.util.getThetaWaves(eeg, samplingRate)
self.assertEqual(len(theta), nEeg)
self.assertTrue(all([x in waves["theta"] for x in theta]))
alpha = self.util.getAlphaWaves(eeg, samplingRate)
self.assertEqual(len(alpha), nEeg)
self.assertTrue(all([x in waves["alpha"] for x in alpha]))
beta = self.util.getBetaWaves(eeg, samplingRate)
self.assertEqual(len(beta), nEeg)
self.assertTrue(all([x in waves["alpha"] for x in alpha]))
gamma = self.util.getGammaWaves(eeg, samplingRate)
self.assertEqual(len(gamma), nEeg)
self.assertTrue(all([x in waves["gamma"] for x in gamma]))
def test_getWaves(self):
eegData = EEGTableFileUtil().readFile(PATH + "example_32.csv")
eeg = eegData.getColumn("F3")
nEeg = len(eeg)
waves = self.util.getWaves(eeg, eegData.getSamplingRate())
self.assertEqual(len(waves), 5)
for _, wave in waves.iteritems():
self.assertEqual(len(wave), nEeg)
class EEGTableConverter(object):
def __init__(self, filePath=None, infinite=True):
'''
Reads data from ./../../examples/example_4096.csv and builds the data structure
'''
self.reader = EEGTableFileUtil()
self.filepath = filePath
self.infinite = infinite
self.hasMore = False
if filePath == None:
self.filepath = scriptPath + "/../../examples/example_4096.csv"
#self.filepath = scriptPath + "/../../../captured_data/janis/2016-07-12-11-15_EEG_1.csv"
self.data = None
self.index = 0
def convert(self):
self._readHeader()
self._readRawData()
self.data = self._buildDataStructure()
def _readHeader(self):
self.header = self.reader.readHeader(self.filepath)
fields = self.header[:]
fields.remove("Timestamp")
fields.remove("Unknown")
self.fields = filter(lambda x: not (x.startswith("Q")), fields)
def _readRawData(self):
self.rawData = self.reader.readData(self.filepath)
self.len = len(self.rawData)
if self.len > 0:
self.hasMore = True
print "Using %d dummy datasets" % self.len
def dequeue(self):
pass
def _getNextIndex(self):
self.index += 1
if self.index >= len(self.data) and not self.infinite:
self.hasMore = False
self.index %= self.len
def close(self):
pass
def __init__(self, networkFile=None, demo=False, demoFile=None):
'''Main class for drowsiness detection
:param string networkFile: file name of the saved neural network (path: "/../../data/<networkFile>.nn")
'''
self.demo = demo
self.running = True
self.config = ConfigProvider()
self._initNeuralNetwork(networkFile)
self._initFeatureExtractor(demoFile)
self.dm = DrowsinessMonitor()
self.fileUtil = EEGTableFileUtil()
def __init__(self, filePath=None, infinite=True):
'''
Reads data from ./../../examples/example_4096.csv and builds the data structure
'''
self.reader = EEGTableFileUtil()
self.filepath = filePath
self.infinite = infinite
self.hasMore = False
if filePath == None:
self.filepath = scriptPath + "/../../examples/example_4096.csv"
#self.filepath = scriptPath + "/../../../captured_data/janis/2016-07-12-11-15_EEG_1.csv"
self.data = None
self.index = 0
class PoSDBoS(object):
def __init__(self, networkFile=None, demo=False, demoFile=None):
'''Main class for drowsiness detection
:param string networkFile: file name of the saved neural network (path: "/../../data/<networkFile>.nn")
'''
self.demo = demo
self.running = True
self.config = ConfigProvider()
self._initPoSDBoS()
self._initNeuralNetwork(networkFile)
self._initFeatureExtractor(demoFile)
self.dm = DrowsinessMonitor()
self.fileUtil = EEGTableFileUtil()
def _initPoSDBoS(self):
posdbosConfig = self.config.getPoSDBoSConfig()
self.drowsyMinCount = posdbosConfig.get("drowsyMinCount")
self.awakeMinCount = posdbosConfig.get("awakeMinCount")
self.classified = [0, 0]
self.curClass = 0
self.classCount = 0
self.found = 0
def _initNeuralNetwork(self, networkFile):
nnCreate = self.config.getNNInitConfig()
self.nn = NeuralNetwork()
if networkFile == None:
self.nn.createNew(**nnCreate)
else:
self.nn.load(networkFile)
def _initFeatureExtractor(self, demoFile):
self.demoFile = demoFile
collector = self._initDataCollector(self.demoFile)
self.fe = FeatureExtractor(collector)
self.inputQueue = self.fe.extractQueue
def _initDataCollector(self, demoFile):
collectorConfig = self.config.getCollectorConfig()
if self.demo:
return DummyDataCollector(demoFile, **collectorConfig)
else:
return EEGDataCollector(None, **collectorConfig)
def close(self):
self.running = False
def run(self):
fet = threading.Thread(target=self.fe.start)
fet.start()
dmt = threading.Thread(target=self.dm.run)
dmt.start()
features = []
total = 0
start = time.time()
c = []
while self.running and dmt.is_alive():
try:
#awake = 0, drowsy = 1
data = self.inputQueue.get(timeout=1)
features.append(data)
clazz = self.nn.activate(data, True)
c.append([clazz, clazz])
self.setStatus(clazz)
total += 1
except Empty:
print "needed %sms for %d windows" % (time.time() - start, total)
pass
except KeyboardInterrupt:
self.close()
except Exception as e:
print e.message
self.close()
#self.writeFeature(c)
self.fe.close()
self.dm.close()
dmt.join()
def setStatus(self, clazz):
self.classified[clazz] += 1
if self.curClass == clazz:
self.classCount += 1
else:
self.curClass = clazz
self.classCount = 0
info = "class %d row (%s)" % (clazz, str(self.classCount))
if clazz == 1 and self.classCount >= self.drowsyMinCount:
self.dm.setStatus(clazz, info)
self.found += 1
elif clazz == 0 and self.classCount >= self.awakeMinCount:
self.dm.setStatus(clazz, info)
def writeFeature(self, data):
filePath = scriptPath + "/../data/" + "classes.csv"
#filePath = scriptPath + "/../data/" + "drowsy_full_.csv"
header = ["clazz", "clazz2"]
#start = 4
#end = start + len(data[0])/6
#for field in self.config.getCollectorConfig().get("fields"):
# header.extend([str(x) + "Hz" + field for x in range(start, end)])
self.fileUtil.writeFile(filePath, data, header)
def _readData(self):
self.reader = EEGTableFileUtil()
self.eegData = self.reader.readFile(self.filePath)
class SignalStatisticUtil(object):
'''
class to show some statistical values for a channel
'''
def __init__(self, person, filePath, signals=None, save=True, plot=True, logScale=False):
self.person = person
self.filePath = filePath
self._initStatsDict()
self._readData()
self._initSignals(signals)
self.su = SignalUtil()
self.qu = QualityUtil()
self.eu = EEGUtil()
self._initFields()
self.save = save
self._initPlotter(person, plot, logScale)
self.ssPrint = SignalStatisticPrinter(person)
self.preProcessor = SignalPreProcessor()
self.processor = SignalProcessor()
self.windowSize = ConfigProvider().getCollectorConfig().get("windowSize")
def _initStatsDict(self):
self.stats = OrderedDict()
self.stats[GENERAL_KEY] = OrderedDict()
self.stats[SIGNALS_KEY] = OrderedDict()
def _readData(self):
self.reader = EEGTableFileUtil()
self.eegData = self.reader.readFile(self.filePath)
def _initSignals(self, signals):
if not signals:
signals = ConfigProvider().getEmotivConfig().get("eegFields")
self.signals = signals
def _initFields(self):
self.statFields = STAT_FIELDS
self.statFields["max"][METHOD] = self.su.maximum
self.statFields["min"][METHOD] = self.su.minimum
self.statFields["mean"][METHOD] = self.su.mean
self.statFields["std"][METHOD] = self.su.std
self.statFields["var"][METHOD] = self.su.var
self.statFields["zeros"][METHOD] = self.qu.countZeros
self.statFields["seq"][METHOD] = self.qu.countSequences
self.statFields["out"][METHOD] = self.qu.countOutliners
self.statFields["nrgy"][METHOD] = self.su.energy
self.statFields["zcr"][METHOD] = self.su.zcr
def _initPlotter(self, person, plot, logScale):
self.plotter = []
for clazz in PLOTTER:
plotter = clazz(person, self.eegData, self.signals, self.filePath, self.save, plot, logScale)
thread = multiprocessing.Process(target=plotter.doPlot)
self.plotter.append(thread)
def main(self):
self.doPlot()
self.collect_stats()
self.printStats()
def doPlot(self):
for thread in self.plotter:
thread.start()
def collect_stats(self):
self.collectGeneralStats()
for signal in self.signals:
self.stats[SIGNALS_KEY][signal] = {}
self.collectRawStats(signal)
def collectGeneralStats(self):
self._addGeneralStatValue("file path", self.filePath)
self._addGeneralStatValue("sampleRate", ("%f.2" % self.eegData.getSamplingRate()))
self._addGeneralStatValue("dataLength", ("%d" % self.eegData.len))
self._addGeneralStatValue("bound", ("%d - %d" % (self.qu.lowerBound, self.qu.upperBound)))
self._addGeneralTimeStat("start time", "getStartTime", TIME_FORMAT_STRING)
self._addGeneralTimeStat("end time", "getEndTime", TIME_FORMAT_STRING)
self._addGeneralTimeStat("duration", "getDuration", DURATION_FORMAT_STRING)
def _addGeneralTimeStat(self, name, method, formatString):
time = getattr(self.eegData, method)()
value = self._buildFormattedTime(time, formatString)
self._addGeneralStatValue(name, value)
def _buildFormattedTime(self, time, formatString):
value = datetime.fromtimestamp(time).strftime(formatString)
return value
def _addGeneralStatValue(self, name, value):
self.stats[GENERAL_KEY][name] = value
def collectRawStats(self, signal):
data = self.eegData.getColumn(signal)
self._collectSignalStat(signal, RAW_KEY, data)
def _collectSignalStat(self, signal, category, data):
self.stats[SIGNALS_KEY][signal][category] = OrderedDict()
for field, attributes in self.statFields.iteritems():
fieldValues = []
for window in self.getWindows(data):
fieldValues.append(self._getSignalStat(signal, category, field, attributes["method"], window, 0))
merged = self._mergeValues(fieldValues, field)
self._addSignalStatValue(signal, category, field, merged)
def getWindows(self, raw):
windows = []
for start in range(0, len(raw), self.windowSize / 2):
end = start + self.windowSize
if end <= len(raw):
windows.append(raw[start:end])
return windows
def _getSignalStat(self, signal, category, name, method, raw, decPlace=2):
return method(raw)
def _addSignalStatValue(self, signal, category, name, value):
self.stats[SIGNALS_KEY][signal][category][name] = value
def _mergeValues(self, values, field):
typ = self.statFields[field][TYPE]
if typ == MAX_TYPE:
return nanmax(values)
if typ == MIN_TYPE:
return nanmin(values)
if typ == AGGREGATION_TYPE:
return nansum(values)
if typ == MEAN_TYPE:
return nanmean(values)
if typ == DIFF_TYPE:
return nanmean(values)
def setStats(self, stats):
self.stats = stats
def printStats(self):
content = self.ssPrint.getSignalStatsString(self.stats)
print content
if self.save:
filePath = getNewFileName(self.filePath, "txt", "_stats")
self.ssPrint.saveStats(filePath, content)
def setUp(self):
self.reader = EEGTableFileUtil()
class TestEEGTableFileUtil(unittest.TestCase):
def setUp(self):
self.reader = EEGTableFileUtil()
def test_readData(self):
file_path = PATH + "example_32.csv"
if os.path.isfile(file_path):
self.reader.readData(file_path)
else:
print "'%s' not found" % file_path
def test_readHeader(self):
file_path = PATH + "example_32.csv"
if os.path.isfile(file_path):
self.reader.readHeader(file_path)
else:
print "'%s' not found" % file_path
def test_readFile(self):
file_path = PATH + "example_32.csv"
if os.path.isfile(file_path):
self.reader.readFile(file_path)
else:
print "'%s' not found" % file_path
def test_writeFile(self):
filePath = PATH + "test.csv"
header= ["A", "B", "C"]
data = np.array([[1.123456789, 2, 3], [-4.123456789, 5, 6], [7.123456789, 8, 99.123]])
self.reader.writeFile(filePath, data, header)
if os.path.isfile(filePath):
read = self.reader.readFile(filePath)
for i in range(len(data)):
for j in range(len(data[i])):
self.assertAlmostEqual(data[i, j], read.data[i, j], delta= 0.001)
removeFile(filePath)
def test_writeStructredFile(self):
filePath = PATH + "test_structured.csv"
data = {
"A": {
"value": [1, 2, 3],
"quality": [-1, -1, -1]
},
"B": {
"value": [4, 5, 6],
"quality": [-2, -2, -2]
},
"C": {
"value": [7, 8, 9],
"quality": [-3, -3, -3]
}
}
self.reader.writeStructredFile(filePath, data)
if os.path.isfile(filePath):
read = self.reader.readFile(filePath)
for key, values in data.iteritems():
self.assertTrue(sameEntries(values["value"], read.getColumn(key)))
removeFile(filePath)
def test_readFile_NaNValues(self):
eegData = self.reader.readFile(PATH + "example_32_empty.csv")
emptyCol = eegData.getColumn("Y")
self.assertTrue(np.isnan(emptyCol).any())
nonEmptyCol = eegData.getColumn("F3")
self.assertFalse(np.isnan(nonEmptyCol).any())
def test_readFile_SeparatorFallback(self):
eegData = self.reader.readFile(PATH + "example_32_empty.csv")
semicolonData = eegData.getColumn("F3")
eegData = self.reader.readFile(PATH + "example_32_comma.csv")
commaData = eegData.getColumn("F3")
self.assertTrue((semicolonData == commaData).all())
def __init__(self, files=[]):
self.files = files
self.fileUtil = EEGTableFileUtil()
class NetworkDataUtil(object):
'''
Class to load and separate data for a neural network
'''
def __init__(self, files=[]):
self.files = files
self.fileUtil = EEGTableFileUtil()
def get(self, separate=True):
values0, values1 = self.readFiles(self.files)
if separate:
return self.buildTestSet(values0, values1)
else:
return self.buildFullTestSet(values0, values1)
def getNInput(self):
return self.nInputs
def readFiles(self, files):
return self.fileUtil.readData(files[0]), self.fileUtil.readData(files[1])
def buildFullTestSet(self, values0, values1):
values0 = self._addClass(values0, 0.)
values1 = self._addClass(values1, 1.)
rawData = self._postprocData(values0, values1)
self.nInputs = len(rawData[0])-1
return self.createData(self.nInputs, rawData)
def buildTestSet(self, values0, values1):
train0, test0 = self._preprocData(values0, 0.)
train1, test1 = self._preprocData(values1, 1.)
trainRawData = self._postprocData(train0, train1)
testRawData = self._postprocData(test0, test1)
self.nInputs = len(trainRawData[0])-1
trainData = self.createData(self.nInputs, trainRawData)
testData = self.createData(self.nInputs, testRawData)
return trainData, testData
def _preprocData(self, values, clazz):
np.random.shuffle(values)
values0 = self._addClass(values, clazz)
return self._separateData(values0)
def _addClass(self, values, clazz):
shape = values.shape
clazzArray = np.full((shape[0], shape[1]+1), clazz)
clazzArray[:,:-1] = values
return clazzArray
def _separateData(self, values):
l = len(values)
d = (2 * l / 3)
return values[0:d], values[d:]
def _postprocData(self, values0, values1):
values = np.concatenate((values0, values1), axis=0)
np.random.shuffle(values)
return values
def createXORData(self):
values = [[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0]]
return self.createData(2, values)
def createData(self, nInput, values):
ds = SupervisedDataSet(nInput, N_OUTPUT)
for value in values:
ds.addSample(value[:nInput], value[nInput])
return ds
class PoSDBoS(object):
def __init__(self, networkFile=None, demo=False, demoFile=None):
'''Main class for drowsiness detection
:param string networkFile: file name of the saved neural network (path: "/../../data/<networkFile>.nn")
'''
self.demo = demo
self.running = True
self.config = ConfigProvider()
self._initNeuralNetwork(networkFile)
self._initFeatureExtractor(demoFile)
self.dm = DrowsinessMonitor()
self.fileUtil = EEGTableFileUtil()
def _initNeuralNetwork(self, networkFile):
nn_conf = self.config.getNeuralNetworkConfig()
self.nn = NeuralNetwork()
if networkFile == None:
self.nn.createNew(nn_conf["nInputs"], nn_conf["nHiddenLayers"], nn_conf["nOutput"], nn_conf["bias"])
else:
self.nn.load(networkFile)
def _initFeatureExtractor(self, demoFile):
collector = self._initDataCollector(demoFile)
self.fe = FeatureExtractor(collector)
self.inputQueue = self.fe.extractQueue
def _initDataCollector(self, demoFile):
collectorConfig = self.config.getCollectorConfig()
if self.demo:
return DummyDataCollector(demoFile, **collectorConfig)
else:
return EEGDataCollector(None, **collectorConfig)
def close(self):
self.running = False
def run(self):
fet = threading.Thread(target=self.fe.start)
fet.start()
dmt = threading.Thread(target=self.dm.run)
dmt.start()
features = []
while self.running and dmt.is_alive():
try:
data = self.inputQueue.get(timeout=1)
features.append(data)
x = random.randint(1, 10)%2
y = random.randint(1, 10)%2
data = (x, y)
clazz = self.nn.activate(data)
info = "%d XOR %d is %d; queue: %d" % (x, y, clazz, self.inputQueue.qsize())
self.dm.setStatus(clazz, info)
#sleep(1)
except Empty:
pass
#if self.demo:
# self.close()
except KeyboardInterrupt:
self.close()
except Exception as e:
print e.message
self.close()
self.writeFeature(features)
self.fe.close()
self.dm.close()
dmt.join()
def writeFeature(self, data):
filePath = scriptPath + "/../data/" + "test.csv"
header = []
for field in ["F3", "F4", "F7", "F8"]:
for i in range(1, 5):
header.append("%s_%s" % (field ,str(i)))
self.fileUtil.writeFile(filePath, data, header)
self._plotSignal(signal, self.data[:,i], axes[i])
self._configurePlot()
self.savePlot()
self.showPlot()
print "plotting done"
def _initPlot(self):
signalCount = self._calcSignalCount()
fig, axes = plt.subplots(signalCount, figsize=self.figsize, dpi=80, sharex=True, sharey=False)
return fig, axes
def _plotSignal(self, header, data, axis):
axis.yaxis.set_label_position("right")
axis.set_ylabel(header)
axis.plot(data)
mean = np.nanmean(data)
print header, mean
axis.plot([mean]*len(data))
if __name__ == '__main__': # pragma: no cover
#filePath = scriptPath + "/../../data/awake_full_.csv"
filePath = scriptPath + "/../../data/classes.csv"
fileUtil = EEGTableFileUtil()
fp = FeaturePlotter(fileUtil.readData(filePath), fileUtil.readHeader(filePath), filePath)
fp.doPlot()
