class LogWriter(object):
""" This provides a writer to the SOFIE HDF FORMAT system. Use with caution, Pytables
does not support concurrent writes to an HDF file. (Currently not used in the ExperimentControl Package.)"""
previousParsedValue = None
def __init__(self, filename='', runName='test'):
self.is_open = False
self.open(filename, runName)
def __del__(self):
if self.is_open:
self.fd.close()
self.fdP.close()
def open(self, filename='', runName='test'):
if filename == '':
filename = datetime.datetime.now().isoformat() + '.h5'
if self.is_open == True:
self.close()
self.fd = AntRawDataAccess(filename, runName + '/' + SPARKSUBDIRECTORY)
self.fdP = AntParsedDataAccess(filename,
runName + '/' + SPARKSUBDIRECTORY)
self.is_open = True
#TODO write meta information...
def close(self):
if self.is_open:
self.fd.close()
self.fdP.close()
self.is_open = False
def _logEvent(self, event, data=None):
timestamp = time.time()
if event == EVENT_READ and len(data) > 0:
#logging.debug(len(data))
information = {'packet': data, 'timestamp': timestamp}
try:
#logging.debug(information)
informationRaw = ant_sample(information)
if informationRaw == []:
logging.debug('NOT A BROADCAST')
return
logging.debug('Information Raw:' + str(informationRaw))
informationParsed = ant_sample_interpret(informationRaw,\
self.previousParsedValue)
self.previousParsedValue = informationParsed
self.fd.write(informationRaw)
self.fdP.write(informationParsed)
except exceptions.UnknownParseError, e:
pass
except exceptions.ChecksumParseError, e:
logging.debug('CHECKSUM ERROR:' + e.message)
class LogWriter(object):
""" This provides a writer to the SOFIE HDF FORMAT system. Use with caution, Pytables
does not support concurrent writes to an HDF file. (Currently not used in the ExperimentControl Package.)"""
previousParsedValue = None;
def __init__(self, filename='',runName='test'):
self.is_open = False
self.open(filename,runName)
def __del__(self):
if self.is_open:
self.fd.close()
self.fdP.close()
def open(self, filename='',runName='test'):
if filename == '':
filename = datetime.datetime.now().isoformat() + '.h5'
if self.is_open == True:
self.close()
self.fd = AntRawDataAccess(filename,runName+'/'+SPARKSUBDIRECTORY)
self.fdP = AntParsedDataAccess(filename,runName+'/'+SPARKSUBDIRECTORY)
self.is_open = True
#TODO write meta information...
def close(self):
if self.is_open:
self.fd.close()
self.fdP.close()
self.is_open = False
def _logEvent(self, event, data=None):
timestamp = time.time()
if event == EVENT_READ and len(data)>0:
#logging.debug(len(data))
information = {'packet':data,'timestamp':timestamp}
try:
#logging.debug(information)
informationRaw = ant_sample(information)
if informationRaw == []:
logging.debug('NOT A BROADCAST')
return
logging.debug('Information Raw:'+str(informationRaw))
informationParsed = ant_sample_interpret(informationRaw,\
self.previousParsedValue)
self.previousParsedValue = informationParsed
self.fd.write(informationRaw)
self.fdP.write(informationParsed)
except exceptions.UnknownParseError,e:
pass
except exceptions.ChecksumParseError,e:
logging.debug('CHECKSUM ERROR:'+e.message)
def open(self, filename='', runName='test'):
if filename == '':
filename = datetime.datetime.now().isoformat() + '.h5'
if self.is_open == True:
self.close()
self.fd = AntRawDataAccess(filename, runName + '/' + SPARKSUBDIRECTORY)
self.fdP = AntParsedDataAccess(filename,
runName + '/' + SPARKSUBDIRECTORY)
self.is_open = True
def test_parse_speed_cadence(self):
#Read Test Data
if exists('sample-tmp.h5'):
unlink('sample-tmp.h5')
copyfile('test-data/sample.h5','sample-tmp.h5')
antAccess = AntRawDataAccess('sample-tmp.h5','/test/goldenembeded');
antWriter = AntParsedDataAccess('sample-tmp.h5','/test/goldenembeded');
firstSpeed=0;
cumulativeRev=0;
i=0
previousRow = None
for row in antAccess:
#logging.debug('Type Row:'+str(type(row))+' Row: '+str(row))
parsedRow = parse_sample_interpret(row,previousRow)
logging.debug(row['Timestamp'])
cspMSB = row['CumulativeSpeedRevolutionCountMSB']
cspLSB = row['CumulativeSpeedRevolutionCountLSB']
self.assertEqual(parsedRow['CumulativeSpeedRevolutionCount'],+\
cspMSB*256+cspLSB)
self.assertEqual(parsedRow['BikeCadence'],0)
logging.debug('SPEED:'+str(i)+':'+str(parsedRow['BikeSpeed']))
if i>19 and i < 33:
self.assertTrue(parsedRow['BikeSpeed']>0)
i+=1;
self.assertEqual(parsedRow['BikeCadence'],0)
secondSpeed = parsedRow['BikeSpeedEventTime']
for column in parsedRow.values():
#logging.debug('Column: '+str(column))
self.assertTrue((type(column)==int) or (type(column)==float) )
cumulativeSecond = parsedRow['CumulativeSpeedRevolutionCount']
if secondSpeed > firstSpeed:
self.assertGreater(cumulativeSecond, cumulativeRev, 'Greater')
firstSpeed = secondSpeed
cumulativeRev = cumulativeSecond
logging.debug(parsedRow)
antWriter.write(parsedRow)
previousRow = parsedRow
antAccess.close()
i=0
for row in antWriter:
if i == 0:
self.assertEqual(row['CumulativeSpeedRevolutionCount'],231*256+120)
self.assertEqual(row['BikeCadence'],0)
i+=1
def open(self, filename='',runName='test'):
if filename == '':
filename = datetime.datetime.now().isoformat() + '.h5'
if self.is_open == True:
self.close()
self.fd = AntRawDataAccess(filename,runName+'/'+SPARKSUBDIRECTORY)
self.fdP = AntParsedDataAccess(filename,runName+'/'+SPARKSUBDIRECTORY)
self.is_open = True
def importdata(infile, outfile, runName, description, shouldParse, shouldCsv):
"""
Standard function used to import data into the file
"""
print '\n-------------------------------\nImporting information' + \
'\n-------------------------------\n'
if(infile == None):
print "Specify the in file (--infile filename)"
exit()
if(outfile == None):
print "Specify the out file (--outfile filename)"
exit()
if(runName == None):
print "Specify the rest run name (--runName 01CornerTestRun) "
exit()
extension = os.path.splitext(infile)
infileName = extension[0]
extension = extension[1]
csvOut = infileName + '-sofie.csv'
csvOutParsed = infileName + '-sofie-parsed.csv'
if extension == '.gce':
runNameExtended = getGESubDirectory(runName)
theOriginalFile = GE.open(infile)
parser = ant_sample
parser_interpret = ant_sample_interpret
theWriter = AntRawDataAccess(outfile,
runName=runNameExtended,
description=description)
elif extension == '.ant':
runNameExtended = getSparkSubDirectory(runName)
theOriginalFile = SF.open(infile)
parser = ant_sample
parser_interpret = ant_sample_interpret
theWriter = AntRawDataAccess(outfile,
runName=runNameExtended,
description=description)
elif extension == '.c3d' or extension == '.C3D':
runNameExtended = getC3DSubDirectory(runName)
theOriginalFile = C3.open(infile)
parser = c3d_sample
parser_interpret = c3d_sample_interpret
theWriter = C3DDataAccess(outfile,
runName=runNameExtended,
description=description,
allowDuplicates=True,
videoSamplingRate=theOriginalFile.getVideoSamplingRate(),
videoFrames=theOriginalFile.getVideoFrames(),
scale=theOriginalFile.getScale())
elif extension == '.csv':
runNameExtended = getCSVImportSubDirectory(runName)
theOriginalFile = scf.open(infile)
parser = csv_sample
parser_interpret = csv_sample_interpret
theWriter = SofieCsvPyTableAccess(outfile,
runName=runNameExtended,
description=description,
fieldNames=theOriginalFile.getTableStructure())
elif extension == '.tab':
runNameExtended = getAdamsSubDirectory(runName)
theOriginalFile = adams.open(infile)
parser = csv_sample
parser_interpret = csv_sample_interpret
theWriter = SofieCsvPyTableAccess(outfile,
runName=runNameExtended,
description=description,
fieldNames=theOriginalFile.getTableStructure())
elif extension == '.bag':
runNameExtended = getBagSubDirectory(runName)
theWriter = SofieCsvPyTableAccess(outfile,
runName=runNameExtended,
description=description,
fieldNames=['A HACK'])
theWriter.writeFile(infile)
print '\n-------------------------------\n{0} Saved to file'.format(outfile)
theWriter.close()
return
else:
raise FileNotSupportedException('File not Supported: ' + str(extension))
print '\n-------------------------------\nSaving to file',
print outfile
if shouldParse:
print 'File is going to be parsed'
print 'CSV Parsed File: ' + csvOutParsed
if shouldCsv:
print 'CSV File: ' + csvOut + '\n-----------------------------------\n'
awritercsv = SofieCsvAccess(csvOut)
for theLine in theOriginalFile:
logging.debug(theLine)
try:
parsedLine = parser(theLine)
except Exception, e:
print 'Parsing Failed: ' + str(e)
continue
try:
logging.debug(parsedLine)
theWriter.write(parsedLine)
print '.',
if shouldCsv:
awritercsv.write(parsedLine)
print 'c',
except AntRawParserLengthNotCorrect, e:
print 'x'
def test_getRunsAndGetRunsMetaFromFile(self):
filename ='simple-test-data.h5'
writer = AntRawDataAccess(filename, '/test')
#self.assertEqual(writer.getRuns(),[]);
writerData = ('1', '2', '3', '4', '5', '6', '7', '8', '9', 111111.101221)
logging.info('Writing the following data: ' + str(writerData))
writer.write(writerData)
logging.info('Came Back from writing.')
writer.close()
writer = AntRawDataAccess(filename, '/test')
self.assertEqual(writer.getRuns(),
['/RunMeta','/test/AntRawData']);
writer.setRunName('again');
writerData = ('1', '2', '3', '4', '5', '6', '7', '8', '9', 111111.101221)
logging.info('Writing the following data: ' + str(writerData))
writer.write(writerData)
logging.info('Came Back from writing.')
self.assertEqual(writer.getRuns(),
['/RunMeta','/again/AntRawData', '/test/AntRawData']);
self.assertEqual(writer.getRuns('again'),
['/again/AntRawData']);
self.assertEqual(writer.getRuns('agai'),
[]);
self.assertEqual(writer.getRuns('agai'),
[]);
self.assertEqual(writer.getRuns('again/AntRawData'),
['/again/AntRawData']);
writer.close()
#Testing RUNMETA static functions.
theRuns = SofieCsvPyTableAccess.getRunsInTheFile('simple-test-data.h5')
logging.info('The RUNS: '+ '|'.join(theRuns))
expected = {'runSubject':'Our Subject','runNotes':'Test\n\nttt'}
SofieCsvPyTableAccess.setRunMetaInFile(filename,'again',expected)
theRunMeta = SofieCsvPyTableAccess.getRunMetaInFile(filename,'again')
logging.info('The Run INFO: {0}'.format(theRunMeta))
self.assertTrue(all(item in theRunMeta.items() for item in expected.items()))
def test_setTable(self):
writer = AntRawDataAccess('simple-test-data.h5', '/')
self.assertIsNone(writer.setTable('/', 'HEELLO'));
self.assertEqual(writer.getRun(), '/');
def test_SofiePyTableAccessReadLine(self):
logging.info('AntPyTableAccess ReadLine')
writer = AntRawDataAccess('simple-test-data.h5', '/test')
writerData = ('1', '2', '3', '4', '5', '6', '7', '8', '9', 111111.101221)
logging.info('Writing the following data: ' + str(writerData))
writer.write(writerData)
logging.info('Came Back from writing.')
writer.close()
writerPacket = [int(number) for number in writerData if isinstance(number, str)]
writer = AntRawDataAccess('simple-test-data.h5', '/test')
#self.assertRaises(SetTheTableFirstException, writer.readline)
data = writer.readlines();
logging.debug('Returned data')
logging.debug(data)
logging.debug(type(data))
data = list(data[0])
logging.debug('The Data' + str(data))
logging.debug(type(data))
expected = writerPacket + [writerData[-1]]
logging.debug('Expected Data' + str(expected))
self.assertEqual(expected[0:8], data[0:8])
self.assertEqual(float(expected[9]), float(data[9]))
writerData1 = ('2', '2', '3', '4', '5', '6', '7', '8', '9', 222222.02221)
writer.write(writerData1)
writer.close()
writer = AntRawDataAccess('simple-test-data.h5', '/test')
data = writer.readlines();
data = list(data[0])
self.assertEqual(expected, data)
dataR = writer.readlines();
data = list(dataR[0])
data1 = list(dataR[1])
logging.debug(dataR)
writerPacket1 = [int(number) for number in writerData1 if isinstance(number, str)]
expected1 = writerPacket1 + [writerData1[-1]]
self.assertEqual(expected, data)
self.assertEqual(expected1, data1)
logging.debug('Reading the rows:')
for row in writer.table:
logging.debug(row)
logging.debug('Reading the rows Writer:')
for row in writer:
logging.debug(row['CumulativeSpeedRevolutionCountMSB'])
self.assertEqual(2, row['BikeCadenceEventTimeLSB'])
self.assertEqual(3, row['BikeCadenceEventTimeMSB'])
self.assertEqual(4, row['CumulativeCadenceRevolutionCountLSB'])
self.assertEqual(5, row['CumulativeCadenceRevolutionCountMSB'])
self.assertEqual(6, row['BikeSpeedEventTimeLSB'])
self.assertEqual(7, row['BikeSpeedEventTimeMSB'])
self.assertEqual(8, row['CumulativeSpeedRevolutionCountLSB'])
self.assertEqual(9, row['CumulativeSpeedRevolutionCountMSB'])
logging.debug(row['Timestamp'])
self.assertTrue((row['Timestamp'] == 111111.101221) or \
(row['Timestamp'] == 222222.02221))
self.assertTrue((row['ChanID'] == 1) or \
(row['ChanID'] == 2))
self.assertEqual(writer.getDuplicateWrites(), 0)
writer.write(writerData1)
self.assertEqual(writer.getDuplicateWrites(), 1)
writer.allowDuplicates()
writer.write(writerData1)
self.assertEqual(writer.getDuplicateWrites(), 1)
dataR = writer.readlines();
self.assertEqual(len(dataR), 3)
writer.doNotAllowDuplicates()
writer.write(writerData1)
self.assertEqual(writer.getDuplicateWrites(), 2)
dataR = writer.readlines();
self.assertEqual(len(dataR), 3)