class EEGTableConverter(object): def __init__(self, filePath=None, infinite=True): ''' Reads data from ./../../examples/example_4096.csv and builds the data structure ''' self.reader = EEGTableFileUtil() self.filepath = filePath self.infinite = infinite self.hasMore = False if filePath == None: self.filepath = scriptPath + "/../../examples/example_4096.csv" #self.filepath = scriptPath + "/../../../captured_data/janis/2016-07-12-11-15_EEG_1.csv" self.data = None self.index = 0 def convert(self): self._readHeader() self._readRawData() self.data = self._buildDataStructure() def _readHeader(self): self.header = self.reader.readHeader(self.filepath) fields = self.header[:] fields.remove("Timestamp") fields.remove("Unknown") self.fields = filter(lambda x: not (x.startswith("Q")), fields) def _readRawData(self): self.rawData = self.reader.readData(self.filepath) self.len = len(self.rawData) if self.len > 0: self.hasMore = True print "Using %d dummy datasets" % self.len def dequeue(self): pass def _getNextIndex(self): self.index += 1 if self.index >= len(self.data) and not self.infinite: self.hasMore = False self.index %= self.len def close(self): pass
class NetworkDataUtil(object): ''' Class to load and separate data for a neural network ''' def __init__(self, files=[]): self.files = files self.fileUtil = EEGTableFileUtil() def get(self, separate=True): values0, values1 = self.readFiles(self.files) if separate: return self.buildTestSet(values0, values1) else: return self.buildFullTestSet(values0, values1) def getNInput(self): return self.nInputs def readFiles(self, files): return self.fileUtil.readData(files[0]), self.fileUtil.readData(files[1]) def buildFullTestSet(self, values0, values1): values0 = self._addClass(values0, 0.) values1 = self._addClass(values1, 1.) rawData = self._postprocData(values0, values1) self.nInputs = len(rawData[0])-1 return self.createData(self.nInputs, rawData) def buildTestSet(self, values0, values1): train0, test0 = self._preprocData(values0, 0.) train1, test1 = self._preprocData(values1, 1.) trainRawData = self._postprocData(train0, train1) testRawData = self._postprocData(test0, test1) self.nInputs = len(trainRawData[0])-1 trainData = self.createData(self.nInputs, trainRawData) testData = self.createData(self.nInputs, testRawData) return trainData, testData def _preprocData(self, values, clazz): np.random.shuffle(values) values0 = self._addClass(values, clazz) return self._separateData(values0) def _addClass(self, values, clazz): shape = values.shape clazzArray = np.full((shape[0], shape[1]+1), clazz) clazzArray[:,:-1] = values return clazzArray def _separateData(self, values): l = len(values) d = (2 * l / 3) return values[0:d], values[d:] def _postprocData(self, values0, values1): values = np.concatenate((values0, values1), axis=0) np.random.shuffle(values) return values def createXORData(self): values = [[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0]] return self.createData(2, values) def createData(self, nInput, values): ds = SupervisedDataSet(nInput, N_OUTPUT) for value in values: ds.addSample(value[:nInput], value[nInput]) return ds
class TestEEGTableFileUtil(unittest.TestCase): def setUp(self): self.reader = EEGTableFileUtil() def test_readData(self): file_path = PATH + "example_32.csv" if os.path.isfile(file_path): self.reader.readData(file_path) else: print "'%s' not found" % file_path def test_readHeader(self): file_path = PATH + "example_32.csv" if os.path.isfile(file_path): self.reader.readHeader(file_path) else: print "'%s' not found" % file_path def test_readFile(self): file_path = PATH + "example_32.csv" if os.path.isfile(file_path): self.reader.readFile(file_path) else: print "'%s' not found" % file_path def test_writeFile(self): filePath = PATH + "test.csv" header= ["A", "B", "C"] data = np.array([[1.123456789, 2, 3], [-4.123456789, 5, 6], [7.123456789, 8, 99.123]]) self.reader.writeFile(filePath, data, header) if os.path.isfile(filePath): read = self.reader.readFile(filePath) for i in range(len(data)): for j in range(len(data[i])): self.assertAlmostEqual(data[i, j], read.data[i, j], delta= 0.001) removeFile(filePath) def test_writeStructredFile(self): filePath = PATH + "test_structured.csv" data = { "A": { "value": [1, 2, 3], "quality": [-1, -1, -1] }, "B": { "value": [4, 5, 6], "quality": [-2, -2, -2] }, "C": { "value": [7, 8, 9], "quality": [-3, -3, -3] } } self.reader.writeStructredFile(filePath, data) if os.path.isfile(filePath): read = self.reader.readFile(filePath) for key, values in data.iteritems(): self.assertTrue(sameEntries(values["value"], read.getColumn(key))) removeFile(filePath) def test_readFile_NaNValues(self): eegData = self.reader.readFile(PATH + "example_32_empty.csv") emptyCol = eegData.getColumn("Y") self.assertTrue(np.isnan(emptyCol).any()) nonEmptyCol = eegData.getColumn("F3") self.assertFalse(np.isnan(nonEmptyCol).any()) def test_readFile_SeparatorFallback(self): eegData = self.reader.readFile(PATH + "example_32_empty.csv") semicolonData = eegData.getColumn("F3") eegData = self.reader.readFile(PATH + "example_32_comma.csv") commaData = eegData.getColumn("F3") self.assertTrue((semicolonData == commaData).all())
self._plotSignal(signal, self.data[:,i], axes[i]) self._configurePlot() self.savePlot() self.showPlot() print "plotting done" def _initPlot(self): signalCount = self._calcSignalCount() fig, axes = plt.subplots(signalCount, figsize=self.figsize, dpi=80, sharex=True, sharey=False) return fig, axes def _plotSignal(self, header, data, axis): axis.yaxis.set_label_position("right") axis.set_ylabel(header) axis.plot(data) mean = np.nanmean(data) print header, mean axis.plot([mean]*len(data)) if __name__ == '__main__': # pragma: no cover #filePath = scriptPath + "/../../data/awake_full_.csv" filePath = scriptPath + "/../../data/classes.csv" fileUtil = EEGTableFileUtil() fp = FeaturePlotter(fileUtil.readData(filePath), fileUtil.readHeader(filePath), filePath) fp.doPlot()