class LogWriter(object): """ This provides a writer to the SOFIE HDF FORMAT system. Use with caution, Pytables does not support concurrent writes to an HDF file. (Currently not used in the ExperimentControl Package.)""" previousParsedValue = None def __init__(self, filename='', runName='test'): self.is_open = False self.open(filename, runName) def __del__(self): if self.is_open: self.fd.close() self.fdP.close() def open(self, filename='', runName='test'): if filename == '': filename = datetime.datetime.now().isoformat() + '.h5' if self.is_open == True: self.close() self.fd = AntRawDataAccess(filename, runName + '/' + SPARKSUBDIRECTORY) self.fdP = AntParsedDataAccess(filename, runName + '/' + SPARKSUBDIRECTORY) self.is_open = True #TODO write meta information... def close(self): if self.is_open: self.fd.close() self.fdP.close() self.is_open = False def _logEvent(self, event, data=None): timestamp = time.time() if event == EVENT_READ and len(data) > 0: #logging.debug(len(data)) information = {'packet': data, 'timestamp': timestamp} try: #logging.debug(information) informationRaw = ant_sample(information) if informationRaw == []: logging.debug('NOT A BROADCAST') return logging.debug('Information Raw:' + str(informationRaw)) informationParsed = ant_sample_interpret(informationRaw,\ self.previousParsedValue) self.previousParsedValue = informationParsed self.fd.write(informationRaw) self.fdP.write(informationParsed) except exceptions.UnknownParseError, e: pass except exceptions.ChecksumParseError, e: logging.debug('CHECKSUM ERROR:' + e.message)
class LogWriter(object): """ This provides a writer to the SOFIE HDF FORMAT system. Use with caution, Pytables does not support concurrent writes to an HDF file. (Currently not used in the ExperimentControl Package.)""" previousParsedValue = None; def __init__(self, filename='',runName='test'): self.is_open = False self.open(filename,runName) def __del__(self): if self.is_open: self.fd.close() self.fdP.close() def open(self, filename='',runName='test'): if filename == '': filename = datetime.datetime.now().isoformat() + '.h5' if self.is_open == True: self.close() self.fd = AntRawDataAccess(filename,runName+'/'+SPARKSUBDIRECTORY) self.fdP = AntParsedDataAccess(filename,runName+'/'+SPARKSUBDIRECTORY) self.is_open = True #TODO write meta information... def close(self): if self.is_open: self.fd.close() self.fdP.close() self.is_open = False def _logEvent(self, event, data=None): timestamp = time.time() if event == EVENT_READ and len(data)>0: #logging.debug(len(data)) information = {'packet':data,'timestamp':timestamp} try: #logging.debug(information) informationRaw = ant_sample(information) if informationRaw == []: logging.debug('NOT A BROADCAST') return logging.debug('Information Raw:'+str(informationRaw)) informationParsed = ant_sample_interpret(informationRaw,\ self.previousParsedValue) self.previousParsedValue = informationParsed self.fd.write(informationRaw) self.fdP.write(informationParsed) except exceptions.UnknownParseError,e: pass except exceptions.ChecksumParseError,e: logging.debug('CHECKSUM ERROR:'+e.message)
def open(self, filename='', runName='test'): if filename == '': filename = datetime.datetime.now().isoformat() + '.h5' if self.is_open == True: self.close() self.fd = AntRawDataAccess(filename, runName + '/' + SPARKSUBDIRECTORY) self.fdP = AntParsedDataAccess(filename, runName + '/' + SPARKSUBDIRECTORY) self.is_open = True
def test_parse_speed_cadence(self): #Read Test Data if exists('sample-tmp.h5'): unlink('sample-tmp.h5') copyfile('test-data/sample.h5','sample-tmp.h5') antAccess = AntRawDataAccess('sample-tmp.h5','/test/goldenembeded'); antWriter = AntParsedDataAccess('sample-tmp.h5','/test/goldenembeded'); firstSpeed=0; cumulativeRev=0; i=0 previousRow = None for row in antAccess: #logging.debug('Type Row:'+str(type(row))+' Row: '+str(row)) parsedRow = parse_sample_interpret(row,previousRow) logging.debug(row['Timestamp']) cspMSB = row['CumulativeSpeedRevolutionCountMSB'] cspLSB = row['CumulativeSpeedRevolutionCountLSB'] self.assertEqual(parsedRow['CumulativeSpeedRevolutionCount'],+\ cspMSB*256+cspLSB) self.assertEqual(parsedRow['BikeCadence'],0) logging.debug('SPEED:'+str(i)+':'+str(parsedRow['BikeSpeed'])) if i>19 and i < 33: self.assertTrue(parsedRow['BikeSpeed']>0) i+=1; self.assertEqual(parsedRow['BikeCadence'],0) secondSpeed = parsedRow['BikeSpeedEventTime'] for column in parsedRow.values(): #logging.debug('Column: '+str(column)) self.assertTrue((type(column)==int) or (type(column)==float) ) cumulativeSecond = parsedRow['CumulativeSpeedRevolutionCount'] if secondSpeed > firstSpeed: self.assertGreater(cumulativeSecond, cumulativeRev, 'Greater') firstSpeed = secondSpeed cumulativeRev = cumulativeSecond logging.debug(parsedRow) antWriter.write(parsedRow) previousRow = parsedRow antAccess.close() i=0 for row in antWriter: if i == 0: self.assertEqual(row['CumulativeSpeedRevolutionCount'],231*256+120) self.assertEqual(row['BikeCadence'],0) i+=1
def open(self, filename='',runName='test'): if filename == '': filename = datetime.datetime.now().isoformat() + '.h5' if self.is_open == True: self.close() self.fd = AntRawDataAccess(filename,runName+'/'+SPARKSUBDIRECTORY) self.fdP = AntParsedDataAccess(filename,runName+'/'+SPARKSUBDIRECTORY) self.is_open = True
def importdata(infile, outfile, runName, description, shouldParse, shouldCsv): """ Standard function used to import data into the file """ print '\n-------------------------------\nImporting information' + \ '\n-------------------------------\n' if(infile == None): print "Specify the in file (--infile filename)" exit() if(outfile == None): print "Specify the out file (--outfile filename)" exit() if(runName == None): print "Specify the rest run name (--runName 01CornerTestRun) " exit() extension = os.path.splitext(infile) infileName = extension[0] extension = extension[1] csvOut = infileName + '-sofie.csv' csvOutParsed = infileName + '-sofie-parsed.csv' if extension == '.gce': runNameExtended = getGESubDirectory(runName) theOriginalFile = GE.open(infile) parser = ant_sample parser_interpret = ant_sample_interpret theWriter = AntRawDataAccess(outfile, runName=runNameExtended, description=description) elif extension == '.ant': runNameExtended = getSparkSubDirectory(runName) theOriginalFile = SF.open(infile) parser = ant_sample parser_interpret = ant_sample_interpret theWriter = AntRawDataAccess(outfile, runName=runNameExtended, description=description) elif extension == '.c3d' or extension == '.C3D': runNameExtended = getC3DSubDirectory(runName) theOriginalFile = C3.open(infile) parser = c3d_sample parser_interpret = c3d_sample_interpret theWriter = C3DDataAccess(outfile, runName=runNameExtended, description=description, allowDuplicates=True, videoSamplingRate=theOriginalFile.getVideoSamplingRate(), videoFrames=theOriginalFile.getVideoFrames(), scale=theOriginalFile.getScale()) elif extension == '.csv': runNameExtended = getCSVImportSubDirectory(runName) theOriginalFile = scf.open(infile) parser = csv_sample parser_interpret = csv_sample_interpret theWriter = SofieCsvPyTableAccess(outfile, runName=runNameExtended, description=description, fieldNames=theOriginalFile.getTableStructure()) elif extension == '.tab': runNameExtended = getAdamsSubDirectory(runName) theOriginalFile = adams.open(infile) parser = csv_sample parser_interpret = csv_sample_interpret theWriter = SofieCsvPyTableAccess(outfile, runName=runNameExtended, description=description, fieldNames=theOriginalFile.getTableStructure()) elif extension == '.bag': runNameExtended = getBagSubDirectory(runName) theWriter = SofieCsvPyTableAccess(outfile, runName=runNameExtended, description=description, fieldNames=['A HACK']) theWriter.writeFile(infile) print '\n-------------------------------\n{0} Saved to file'.format(outfile) theWriter.close() return else: raise FileNotSupportedException('File not Supported: ' + str(extension)) print '\n-------------------------------\nSaving to file', print outfile if shouldParse: print 'File is going to be parsed' print 'CSV Parsed File: ' + csvOutParsed if shouldCsv: print 'CSV File: ' + csvOut + '\n-----------------------------------\n' awritercsv = SofieCsvAccess(csvOut) for theLine in theOriginalFile: logging.debug(theLine) try: parsedLine = parser(theLine) except Exception, e: print 'Parsing Failed: ' + str(e) continue try: logging.debug(parsedLine) theWriter.write(parsedLine) print '.', if shouldCsv: awritercsv.write(parsedLine) print 'c', except AntRawParserLengthNotCorrect, e: print 'x'
def test_getRunsAndGetRunsMetaFromFile(self): filename ='simple-test-data.h5' writer = AntRawDataAccess(filename, '/test') #self.assertEqual(writer.getRuns(),[]); writerData = ('1', '2', '3', '4', '5', '6', '7', '8', '9', 111111.101221) logging.info('Writing the following data: ' + str(writerData)) writer.write(writerData) logging.info('Came Back from writing.') writer.close() writer = AntRawDataAccess(filename, '/test') self.assertEqual(writer.getRuns(), ['/RunMeta','/test/AntRawData']); writer.setRunName('again'); writerData = ('1', '2', '3', '4', '5', '6', '7', '8', '9', 111111.101221) logging.info('Writing the following data: ' + str(writerData)) writer.write(writerData) logging.info('Came Back from writing.') self.assertEqual(writer.getRuns(), ['/RunMeta','/again/AntRawData', '/test/AntRawData']); self.assertEqual(writer.getRuns('again'), ['/again/AntRawData']); self.assertEqual(writer.getRuns('agai'), []); self.assertEqual(writer.getRuns('agai'), []); self.assertEqual(writer.getRuns('again/AntRawData'), ['/again/AntRawData']); writer.close() #Testing RUNMETA static functions. theRuns = SofieCsvPyTableAccess.getRunsInTheFile('simple-test-data.h5') logging.info('The RUNS: '+ '|'.join(theRuns)) expected = {'runSubject':'Our Subject','runNotes':'Test\n\nttt'} SofieCsvPyTableAccess.setRunMetaInFile(filename,'again',expected) theRunMeta = SofieCsvPyTableAccess.getRunMetaInFile(filename,'again') logging.info('The Run INFO: {0}'.format(theRunMeta)) self.assertTrue(all(item in theRunMeta.items() for item in expected.items()))
def test_setTable(self): writer = AntRawDataAccess('simple-test-data.h5', '/') self.assertIsNone(writer.setTable('/', 'HEELLO')); self.assertEqual(writer.getRun(), '/');
def test_SofiePyTableAccessReadLine(self): logging.info('AntPyTableAccess ReadLine') writer = AntRawDataAccess('simple-test-data.h5', '/test') writerData = ('1', '2', '3', '4', '5', '6', '7', '8', '9', 111111.101221) logging.info('Writing the following data: ' + str(writerData)) writer.write(writerData) logging.info('Came Back from writing.') writer.close() writerPacket = [int(number) for number in writerData if isinstance(number, str)] writer = AntRawDataAccess('simple-test-data.h5', '/test') #self.assertRaises(SetTheTableFirstException, writer.readline) data = writer.readlines(); logging.debug('Returned data') logging.debug(data) logging.debug(type(data)) data = list(data[0]) logging.debug('The Data' + str(data)) logging.debug(type(data)) expected = writerPacket + [writerData[-1]] logging.debug('Expected Data' + str(expected)) self.assertEqual(expected[0:8], data[0:8]) self.assertEqual(float(expected[9]), float(data[9])) writerData1 = ('2', '2', '3', '4', '5', '6', '7', '8', '9', 222222.02221) writer.write(writerData1) writer.close() writer = AntRawDataAccess('simple-test-data.h5', '/test') data = writer.readlines(); data = list(data[0]) self.assertEqual(expected, data) dataR = writer.readlines(); data = list(dataR[0]) data1 = list(dataR[1]) logging.debug(dataR) writerPacket1 = [int(number) for number in writerData1 if isinstance(number, str)] expected1 = writerPacket1 + [writerData1[-1]] self.assertEqual(expected, data) self.assertEqual(expected1, data1) logging.debug('Reading the rows:') for row in writer.table: logging.debug(row) logging.debug('Reading the rows Writer:') for row in writer: logging.debug(row['CumulativeSpeedRevolutionCountMSB']) self.assertEqual(2, row['BikeCadenceEventTimeLSB']) self.assertEqual(3, row['BikeCadenceEventTimeMSB']) self.assertEqual(4, row['CumulativeCadenceRevolutionCountLSB']) self.assertEqual(5, row['CumulativeCadenceRevolutionCountMSB']) self.assertEqual(6, row['BikeSpeedEventTimeLSB']) self.assertEqual(7, row['BikeSpeedEventTimeMSB']) self.assertEqual(8, row['CumulativeSpeedRevolutionCountLSB']) self.assertEqual(9, row['CumulativeSpeedRevolutionCountMSB']) logging.debug(row['Timestamp']) self.assertTrue((row['Timestamp'] == 111111.101221) or \ (row['Timestamp'] == 222222.02221)) self.assertTrue((row['ChanID'] == 1) or \ (row['ChanID'] == 2)) self.assertEqual(writer.getDuplicateWrites(), 0) writer.write(writerData1) self.assertEqual(writer.getDuplicateWrites(), 1) writer.allowDuplicates() writer.write(writerData1) self.assertEqual(writer.getDuplicateWrites(), 1) dataR = writer.readlines(); self.assertEqual(len(dataR), 3) writer.doNotAllowDuplicates() writer.write(writerData1) self.assertEqual(writer.getDuplicateWrites(), 2) dataR = writer.readlines(); self.assertEqual(len(dataR), 3)