class EEGTableConverter(object):
def __init__(self, filePath=None, infinite=True):
'''
Reads data from ./../../examples/example_4096.csv and builds the data structure
'''
self.reader = EEGTableFileUtil()
self.filepath = filePath
self.infinite = infinite
self.hasMore = False
if filePath == None:
self.filepath = scriptPath + "/../../examples/example_4096.csv"
#self.filepath = scriptPath + "/../../../captured_data/janis/2016-07-12-11-15_EEG_1.csv"
self.data = None
self.index = 0
def convert(self):
self._readHeader()
self._readRawData()
self.data = self._buildDataStructure()
def _readHeader(self):
self.header = self.reader.readHeader(self.filepath)
fields = self.header[:]
fields.remove("Timestamp")
fields.remove("Unknown")
self.fields = filter(lambda x: not (x.startswith("Q")), fields)
def _readRawData(self):
self.rawData = self.reader.readData(self.filepath)
self.len = len(self.rawData)
if self.len > 0:
self.hasMore = True
print "Using %d dummy datasets" % self.len
def dequeue(self):
pass
def _getNextIndex(self):
self.index += 1
if self.index >= len(self.data) and not self.infinite:
self.hasMore = False
self.index %= self.len
def close(self):
pass
class NetworkDataUtil(object):
'''
Class to load and separate data for a neural network
'''
def __init__(self, files=[]):
self.files = files
self.fileUtil = EEGTableFileUtil()
def get(self, separate=True):
values0, values1 = self.readFiles(self.files)
if separate:
return self.buildTestSet(values0, values1)
else:
return self.buildFullTestSet(values0, values1)
def getNInput(self):
return self.nInputs
def readFiles(self, files):
return self.fileUtil.readData(files[0]), self.fileUtil.readData(files[1])
def buildFullTestSet(self, values0, values1):
values0 = self._addClass(values0, 0.)
values1 = self._addClass(values1, 1.)
rawData = self._postprocData(values0, values1)
self.nInputs = len(rawData[0])-1
return self.createData(self.nInputs, rawData)
def buildTestSet(self, values0, values1):
train0, test0 = self._preprocData(values0, 0.)
train1, test1 = self._preprocData(values1, 1.)
trainRawData = self._postprocData(train0, train1)
testRawData = self._postprocData(test0, test1)
self.nInputs = len(trainRawData[0])-1
trainData = self.createData(self.nInputs, trainRawData)
testData = self.createData(self.nInputs, testRawData)
return trainData, testData
def _preprocData(self, values, clazz):
np.random.shuffle(values)
values0 = self._addClass(values, clazz)
return self._separateData(values0)
def _addClass(self, values, clazz):
shape = values.shape
clazzArray = np.full((shape[0], shape[1]+1), clazz)
clazzArray[:,:-1] = values
return clazzArray
def _separateData(self, values):
l = len(values)
d = (2 * l / 3)
return values[0:d], values[d:]
def _postprocData(self, values0, values1):
values = np.concatenate((values0, values1), axis=0)
np.random.shuffle(values)
return values
def createXORData(self):
values = [[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0]]
return self.createData(2, values)
def createData(self, nInput, values):
ds = SupervisedDataSet(nInput, N_OUTPUT)
for value in values:
ds.addSample(value[:nInput], value[nInput])
return ds
class TestEEGTableFileUtil(unittest.TestCase):
def setUp(self):
self.reader = EEGTableFileUtil()
def test_readData(self):
file_path = PATH + "example_32.csv"
if os.path.isfile(file_path):
self.reader.readData(file_path)
else:
print "'%s' not found" % file_path
def test_readHeader(self):
file_path = PATH + "example_32.csv"
if os.path.isfile(file_path):
self.reader.readHeader(file_path)
else:
print "'%s' not found" % file_path
def test_readFile(self):
file_path = PATH + "example_32.csv"
if os.path.isfile(file_path):
self.reader.readFile(file_path)
else:
print "'%s' not found" % file_path
def test_writeFile(self):
filePath = PATH + "test.csv"
header= ["A", "B", "C"]
data = np.array([[1.123456789, 2, 3], [-4.123456789, 5, 6], [7.123456789, 8, 99.123]])
self.reader.writeFile(filePath, data, header)
if os.path.isfile(filePath):
read = self.reader.readFile(filePath)
for i in range(len(data)):
for j in range(len(data[i])):
self.assertAlmostEqual(data[i, j], read.data[i, j], delta= 0.001)
removeFile(filePath)
def test_writeStructredFile(self):
filePath = PATH + "test_structured.csv"
data = {
"A": {
"value": [1, 2, 3],
"quality": [-1, -1, -1]
},
"B": {
"value": [4, 5, 6],
"quality": [-2, -2, -2]
},
"C": {
"value": [7, 8, 9],
"quality": [-3, -3, -3]
}
}
self.reader.writeStructredFile(filePath, data)
if os.path.isfile(filePath):
read = self.reader.readFile(filePath)
for key, values in data.iteritems():
self.assertTrue(sameEntries(values["value"], read.getColumn(key)))
removeFile(filePath)
def test_readFile_NaNValues(self):
eegData = self.reader.readFile(PATH + "example_32_empty.csv")
emptyCol = eegData.getColumn("Y")
self.assertTrue(np.isnan(emptyCol).any())
nonEmptyCol = eegData.getColumn("F3")
self.assertFalse(np.isnan(nonEmptyCol).any())
def test_readFile_SeparatorFallback(self):
eegData = self.reader.readFile(PATH + "example_32_empty.csv")
semicolonData = eegData.getColumn("F3")
eegData = self.reader.readFile(PATH + "example_32_comma.csv")
commaData = eegData.getColumn("F3")
self.assertTrue((semicolonData == commaData).all())
self._plotSignal(signal, self.data[:,i], axes[i])
self._configurePlot()
self.savePlot()
self.showPlot()
print "plotting done"
def _initPlot(self):
signalCount = self._calcSignalCount()
fig, axes = plt.subplots(signalCount, figsize=self.figsize, dpi=80, sharex=True, sharey=False)
return fig, axes
def _plotSignal(self, header, data, axis):
axis.yaxis.set_label_position("right")
axis.set_ylabel(header)
axis.plot(data)
mean = np.nanmean(data)
print header, mean
axis.plot([mean]*len(data))
if __name__ == '__main__': # pragma: no cover
#filePath = scriptPath + "/../../data/awake_full_.csv"
filePath = scriptPath + "/../../data/classes.csv"
fileUtil = EEGTableFileUtil()
fp = FeaturePlotter(fileUtil.readData(filePath), fileUtil.readHeader(filePath), filePath)
fp.doPlot()
