def add(self, tell, lrType, theF):
"""Add an IFLR."""
assert (self.canAdd(lrType))
#print('add()', tell, lrType, theF)
# Read the nth word of the Logical record and treat this
# as the X axis value.
skip = 0
myRm = self._logPass.dfsr.ebs.recordingMode
if myRm:
# Indirect X
myXrc = self._logPass.dfsr.ebs.depthRepCode
else:
# Select Rep Code from the channel
myXrc = self._logPass.dfsr.dsbBlocks[
self._logPass.xAxisIndex].repCode
if myRm == 0 and self._logPass.xAxisIndex != 0:
# Have to skip before reading X axis
skip += theF.skipLrBytes(
self._logPass.type01Plan.chOffset(
frame=0,
ch=self._logPass.xAxisIndex,
))
myLisSize = RepCode.lisSize(myXrc)
myXval = RepCode.readBytes(myXrc, theF.readLrBytes(myLisSize))
skip += myLisSize
skip += theF.skipToNextLr()
self._logPass.addType01Data(tell, lrType, skip, myXval)
def test_30(self):
"""TestRepCodeFrom68.test_30(): from68(0x3fc00000) -> 0.25 (Defect found during FrameSet testing)."""
self.assertEqual(RepCode.from68(0x40000000), 0.0)
v = 0x3fc00000
# print()
# print('0x{:08x}'.format(v), self.splitBits68(v), RepCode.from68(v))
self.assertEqual(RepCode.from68(v), 0.25)
def test_11__(self):
"""TestRepCodeTo68.test_11__(): to68() -1e40 is min."""
# print()
# print('-1e40 = 0x{:08X} {:s}'.format(RepCode.to68(-1e40), self.splitBits68(RepCode.to68(-1e40))))
# print(' Min = 0x{:08X} {:g}'.format(RepCode.to68(RepCode.minValue(68)), RepCode.minValue(68)))
self.assertEqual(RepCode.to68(-1e40),
RepCode.to68(RepCode.minValue(68)))
def test_time_03(self):
"""TestRepCodeFrom68Time.test_time_03(): tests conversion of 1e6 of same word - RepCode.from68 code."""
i = 0
num = 1e6
tS = time.perf_counter()
while i < num:
RepCode.from68(0xBBB38000)
i += 1
self.writeTimeToStdErr(tS, 68, num)
def test_time_03(self):
"""TestRepCodeTo68Time.test_time_03(): tests conversion of 1e6 of same word - RepCode.to68 code."""
i = 0
num = 1e6
tS = time.clock()
while i < num:
RepCode.to68(153.0)
i += 1
self.writeTimeToStdErr(tS, 68, num)
def test_time_20(self):
"""TestRepCodeFrom68Time.test_time_20(): 1e5 word conversion from FileRead: """
i = 0
numWords = 1e5
myWord = b'\x44\x4c\x80\x00'
wordsInPr = int(
(PhysRec.PR_MAX_LENGTH - PhysRec.PR_PRH_LENGTH) / len(myWord))
# Suc no Pre: 1
# Pre no Suc: 2
# Suc and Pre: 3
prContStart = PhysRec.PR_PRH_LEN_FORMAT.pack(PhysRec.PR_PRH_LENGTH + len(myWord)*wordsInPr) \
+ PhysRec.PR_PRH_ATTR_FORMAT.pack(1) \
+ (myWord * wordsInPr)
# Absent Physical Record trailer
prContBody = PhysRec.PR_PRH_LEN_FORMAT.pack(PhysRec.PR_PRH_LENGTH + len(myWord)*wordsInPr) \
+ PhysRec.PR_PRH_ATTR_FORMAT.pack(3) \
+ (myWord * wordsInPr)
# Absent Physical Record trailer
prContEnd = PhysRec.PR_PRH_LEN_FORMAT.pack(PhysRec.PR_PRH_LENGTH + len(myWord)*wordsInPr) \
+ PhysRec.PR_PRH_ATTR_FORMAT.pack(2) \
+ (myWord * wordsInPr)
# Absent Physical Record trailer
# How many physical records
numPr = int(numWords / wordsInPr)
numPrBody = numPr - 2
assert (numPrBody >= 0)
# Python code first
myBy = io.BytesIO(prContStart + prContBody * numPrBody + prContEnd)
myFile = File.FileRead(theFile=myBy,
theFileId='MyFile',
keepGoing=False)
i = 0
tS = time.clock()
while i < wordsInPr * numPr:
pRepCode.read68(myFile)
i += 1
tE_P = time.clock() - tS
self.assertFalse(myFile.hasLd())
sys.stderr.write('Python: %.3f %8.0f words/S ' %
(tE_P, numWords / tE_P))
# Now Cython code
myBy = io.BytesIO(prContStart + prContBody * numPrBody + prContEnd)
myFile = File.FileRead(theFile=myBy,
theFileId='MyFile',
keepGoing=False)
i = 0
tS = time.clock()
while i < wordsInPr * numPr:
RepCode.read68(myFile)
i += 1
tE_C = time.clock() - tS
self.assertFalse(myFile.hasLd())
sys.stderr.write('Cython: %.3f %8.0f words/S ' %
(tE_C, numWords / tE_C))
sys.stderr.write('%.1f%% (x%.1f) ' % ((100.0 *
(tE_C / tE_P)), tE_P / tE_C))
def test_00(self):
"""TestRepCodeTo68LowExponent.test_30(): to68(0.0) -> 0x40000000 and to68(0.25) -> 0x3fc00000 (Defect found during FrameSet testing)."""
# print()
# print('pRepCode.to68(math.ldexp(-1, -128))', '0x{:08x}'.format(pRepCode.to68(math.ldexp(-1, -128))))
# print('pRepCode.to68(math.ldexp(+1, -128))', '0x{:08x}'.format(pRepCode.to68(math.ldexp(1, -128))))
self.assertEqual(RepCode.to68(0.0), 0x40000000)
self.assertEqual(RepCode.to68(0.25), 0x3fc00000)
self.assertEqual(RepCode.from68(0x40000000), 0.0)
# print('0x3fc00000', RepCode.from68(0x3fc00000))
self.assertEqual(RepCode.from68(0x3fc00000), 0.25)
def test_minmax(self):
"""TestRepCodeFrom68.test_minmax(): min/max."""
#print
#print RepCode.minMaxValue(68)
sys.stderr.write(str(RepCode.minMaxValue(68)))
sys.stderr.write(' ')
self.assertEqual(RepCode.minMaxValue(68), (
-1.7014118346046923e+38,
+1.7014116317805963e+38,
))
def test_04(self):
"""TestRepCodeTo68LowExponent.test_04(): Special exponent tests, full range."""
for e in range(-150, 128, 1):
rcWord = pRepCode.to68(math.ldexp(0.5, e))
self.assertEqual((0.5, e), math.frexp(pRepCode.from68(rcWord)))
for e in range(-150, 128, 1):
rcWord = cRepCode.to68(math.ldexp(0.5, e))
self.assertEqual((0.5, e), math.frexp(cRepCode.from68(rcWord)))
for e in range(-150, 128, 1):
rcWord = RepCode.to68(math.ldexp(0.5, e))
self.assertEqual((0.5, e), math.frexp(RepCode.from68(rcWord)))
def test_10__(self):
"""TestRepCodeTo68.test_10__(): to68() <3.50325e-46 is zero."""
v = 3.50325e-46
# print()
# print('1e-40 = 0x{:08X} {:s} {:g}'.format(
# RepCode.to68(v),
# self.splitBits68(RepCode.to68(v)),
# RepCode.from68(RepCode.to68(v))),
# )
self.assertEqual(0x00000001, RepCode.to68(v))
# Now reduce v slightly and we should see zero
self.assertEqual(RepCode.to68(0.99 * v), RepCode.to68(0.0))
def test_min(self):
"""TestRepCodeFrom68.test_min(): min by bit stuffing."""
myWord = 1
myWord <<= 8
myWord |= 1
myWord <<= 23
# 0 exponent
myWord = 0x80000000
#print
#print RepCode.from68(myWord)
self.assertEqual(RepCode.from68(myWord), RepCode.minValue(68))
self.assertEqual(pRepCode.from68(myWord), RepCode.minValue(68))
self.assertEqual(cRepCode.from68(myWord), RepCode.minValue(68))
def _hdrTrippleToLogicalRecord(theData):
# Record header
myB = [
bytes([34, 0]),
bytes([73, 65, 4, 0]),
bytes('TYPE', 'ascii'),
bytes(' ', 'ascii'),
bytes('CONS', 'ascii'),
]
# Insert record contents
CODE_INT = 73
CODE_FLOAT = 68
for m, v, u in theData:
if u is None:
u = b' '
myB.append(bytes([0, 65, 4, 0]))
myB.append(b'MNEM') # Mnem
myB.append(b' ') # Units
myB.append(m)
myB.append(bytes([69, 65, 4, 0]))
myB.append(b'STAT')
myB.append(b' ')
myB.append(b'ALLO')
myB.append(bytes([69, 65, 4, 0]))
myB.append(b'PUNI')
myB.append(b' ')
myB.append(u)
myB.append(bytes([69, 65, 4, 0]))
myB.append(b'TUNI')
myB.append(b' ')
myB.append(u)
assert (type(v) in (bytes, int, float))
if isinstance(v, bytes):
myB.append(bytes([69, 65, len(v), 0]))
myB.append(b'VALU')
myB.append(u)
myB.append(v)
elif isinstance(v, int):
myB.append(bytes([69, CODE_INT, RepCode.lisSize(CODE_INT), 0]))
myB.append(b'VALU')
myB.append(u)
myB.append(RepCode.writeBytes(v, CODE_INT))
elif isinstance(v, float):
myB.append(bytes([69, CODE_FLOAT, RepCode.lisSize(CODE_FLOAT), 0]))
myB.append(b'VALU')
myB.append(u)
myB.append(RepCode.writeBytes(v, CODE_FLOAT))
# Use self._retFilePrS() as size could be large
myBaseFile = BaseTestClasses.TestBaseFile()
myF = myBaseFile._retFilePrS(b''.join(myB))
return LogiRec.LrTableRead(myF)
def test_minmax(self):
"""TestRepCodeTo68Basic.test_minmax(): min/max."""
myMinMax = RepCode.to68(RepCode.minMaxValue(68)[0]), RepCode.to68(
RepCode.minMaxValue(68)[1])
# print()
# print((RepCode.minValue(68)))
# print((RepCode.from68(0xFFC00001)))
# print((RepCode.from68(0xFFFFFFFF)))
#print '(0x%X, 0x%X)' % myMinMax
sys.stderr.write('(0x%X, 0x%X)' % myMinMax)
sys.stderr.write(' ')
self.assertEqual(
myMinMax,
(0xFFC00000, 0x7FFFFFFF),
)
def __init__(self, dfsr):
if dfsr.ebs.recordingMode:
self._indirectSize = RepCode.lisSize(dfsr.ebs.depthRepCode)
else:
self._indirectSize = 0
# Individual channel sizes
self._channelSizes = [b.size for b in dfsr.dsbBlocks]
# Sum the frame size
self._frameSize = sum(self._channelSizes)
# Set up skip lists [integer_byte_length, ...]
# Distance from start of frame to start of channel
self._skipToChStart = []
# Distance end of channel to end of frame
self._skipToFrameEnd = []
toEnd = self._frameSize
fromStart = 0
for b in dfsr.dsbBlocks:
self._skipToChStart.append(fromStart)
toEnd -= b.size
self._skipToFrameEnd.append(toEnd)
fromStart += b.size
# Integrity checks
assert (toEnd == 0)
assert (len(self._channelSizes) == len(self._skipToChStart))
assert (len(self._channelSizes) == len(self._skipToFrameEnd))
assert (sum(self._channelSizes) == self._frameSize)
assert (fromStart == self._frameSize)
def writeTimeToStdErr(self, start, rc, numWords):
tE = time.perf_counter() - start
siz = numWords * RepCode.lisSize(rc)
# sys.stderr.write('Time: {:.3f} Rate {:8.0f} words/S '.format(tE, numWords/tE))
# sys.stderr.write(' Cost: {:.3f} (ms/MB)'.format((tE*1024)/(siz/(1024*1024))))
print('Time: {:.3f} Rate {:8.0f} words/S Cost: {:.3f} (ms/MB)'.format(
tE, numWords / tE, (tE * 1024) / (siz / (1024 * 1024))))
def encode(self):
"""Encode my value to my RepCode returning a bytes object. May raise an ExceptionEngVal."""
try:
return RepCode.writeBytes(self.value, self.rc)
#return RepCode.toRepCode(self.rc, self.value)
except RepCode.ExceptionRepCode as err:
raise ExceptionEngVal('EngVal.encode(): {:s}'.format(str(err)))
def test_01(self):
"""TestCreateList: Create list of 2m RepCode 68 from 64MB of bytes."""
b = self._getBytes()
myTs = TimeSpace()
myTs.startEvent(NUM_MB * self._mb)
l = [RepCode.readBytes68(b[i:i+len(self._word)]) for i in range(0,len(b),len(self._word))]
myTs.endEvent()
myTs.writeCostStdErr()
def test_max(self):
"""TestRepCodeFrom68.test_max(): max by bit stuffing."""
myWord = 0x7FFFFFFF
f = RepCode.from68(myWord)
#m,e = math.frexp(f)
#print
#print 'Max:', f
#print 'm,e:', m,e
#print 'm: shifted 23', m * (1<<23)
#print 'm: 1.0 / (1.0 - m)', 1.0 / (1.0 - m)
#print ' 2**23', 2**23
#print ' 1<<23', 1<<23
#print('dir(RepCode)')
#print(dir(RepCode))
self.assertEqual(RepCode.from68(myWord), RepCode.maxValue(68))
self.assertEqual(cRepCode.from68(myWord), RepCode.maxValue(68))
self.assertEqual(pRepCode.from68(myWord), RepCode.maxValue(68))
def test_02(self):
"""TestCreateListArrayFloat: Create array('f') of 2m RepCode 68 from 64MB of bytes via a list."""
b = self._getBytes()
myTs = TimeSpace()
myTs.startEvent(NUM_MB * self._mb)
l = array.array('f', [RepCode.readBytes68(b[i:i+len(self._word)]) for i in range(0,len(b),len(self._word))])
myTs.endEvent()
myTs.writeCostStdErr()
def val(self, f, s=0):
assert (s == 0), 'ChValsXaxis.val() called with non-zero sample index.'
d = (self._fSpace * f) + self.noise()
if self._xDec:
r = self._xStart - d
else:
r = self._xStart + d
if RepCode.isInt(self.rc):
r = int(r + 0.5)
return r
def test_02(self):
"""TestCreateArrayDouble: Create array('d') of 2m RepCode 68 from 64MB of bytes."""
b = self._getBytes()
myTs = TimeSpace()
myTs.startEvent(NUM_MB * self._mb)
l = array.array('d')
for i in range(0,len(b),len(self._word)):
l.append(RepCode.readBytes68(b[i:i+len(self._word)]))
myTs.endEvent()
myTs.writeCostStdErr()
def test_time_11(self):
"""TestRepCodeTo68Time.test_time_11(): tests conversion of 1e5 random words - Python code."""
i = 0
num = 1e5
tS = time.clock()
myMin, myMax = RepCode.minMaxValue(68)
while i < num:
val = myMin + random.random() * (myMax - myMin)
pRepCode.to68(val)
i += 1
self.writeTimeToStdErr(tS, 68, num)
def test_01(self):
"""TestFileType0.test_01(): 8MB Type 0 data Repcode 68, 4 byte reads."""
tS = time.perf_counter()
# Iterate through file
wordCntr = 0
while not self._file.isEOF:
lrh = self._file.readLrBytes(2)
while self._file.hasLd():
v = RepCode.read68(self._file)
wordCntr += 1
self.writeCostToStderr(tS, 4 * wordCntr, 'Words', wordCntr)
def lrBytes(self, fFrom, numFrames):
"""The Logical Record bytes for the frame range."""
# Start with LRH
myB = bytearray(bytes([self._ebs.dataType, 0]))
if not self.isDirectX:
# Generate indirect X value
xVal = self._chXVal.val(fFrom)
#print('LogPassGen.lrBytes()', xVal)
myB.extend(RepCode.writeBytes(xVal, self._chXVal.rc))
# Add frame data
myB.extend(self._normalAlternateData(fFrom, numFrames))
return myB
def _ReadFileWithBufferSizeRepCode(self, bufSiz):
"""Reads a file with a particular buffer size."""
tS = time.perf_counter()
# Iterate through file
wordCntr = 0
while not self._file.isEOF:
lrh = self._file.readLrBytes(2)
while self._file.hasLd():
b = self._file.readLrBytes(bufSiz)
for i in range(0, bufSiz, 4):
v = RepCode.readBytes68(b[i:i + 4])
wordCntr += 1
self.writeCostToStderr(tS, 4 * wordCntr, 'Words', wordCntr)
def test_12__(self):
"""TestRepCodeFrom68.test_12__(): read68(0xBBB38000) -> -153.0."""
myBy = io.BytesIO(
PhysRec.PR_PRH_LEN_FORMAT.pack(PhysRec.PR_PRH_LENGTH + 4) \
+ PhysRec.PR_PRH_ATTR_FORMAT.pack(0) \
+ b'\xbb\xb3\x80\x00' \
# Absent Physical Record trailer
)
myFile = File.FileRead(theFile=myBy,
theFileId='MyFile',
keepGoing=True)
self.assertEqual(RepCode.read68(myFile), -153.0)
self.assertFalse(myFile.hasLd())
def test_02(self):
"""TestCreateArrayDouble_01: Create array('d') of RepCode 68 consuming bytes as we go."""
sys.stderr.write('Disabled as this os O(N**2) and takes 6mins for just 4MB.')
return
assert(0)
b = bytearray(self._getBytes())
myTs = TimeSpace()
myTs.startEvent(NUM_MB * self._mb)
l = array.array('d')
r = len(self._word)
for i in range(0,len(b),r):
l.append(RepCode.readBytes68(b[:r]))
del b[:r]
myTs.endEvent()
myTs.writeCostStdErr()
def __init__(self, ebSet, chList, xStart, xRepCode, xNoise):
"""Constructor.
ebsBytes - a EntryBlockSet object.
chList - a list of Channel objects or objects derived from ChGenBase.
(xStart, xRepCode, xNoise) control X axis generation. An X axis value
generator (ChValsXaxis) is created from this + the ebSet.frameSpacing
+ ebSet.upDown."""
# Need numeric frame spacing
if ebSet.frameSpacing is None:
raise ExceptionLisGen(
'LogPassGen.__init__() must have non-None frame spacing.')
self._ebs = ebSet
self._chXVal = ChValsXaxis(
xStart,
self._ebs.frameSpacing,
xDec=not self._ebs.xInc,
rc=xRepCode,
noise=xNoise,
)
if self._ebs.recordingMode == 0:
# Direct X, prepend an Xaxis channel and chValsX
if self._ebs.optLogScale == 0:
xName = b'TIME'
else:
xName = b'DEPT'
self._chS = [
Channel(
ChannelSpec(xName, b'ServID', b'ServOrdN',
self._ebs.frameSpacingUnits, 45310011, 256,
RepCode.lisSize(self._chXVal.rc), 1,
self._chXVal.rc),
self._chXVal,
),
]
self._chS += chList
else:
# Indirect X axis so we generate our own X values
self._chS = chList
def test_05(self):
"""TestRepCodeTo68LowExponent.test_02(): to68() exponent <-150 made to zero."""
self.assertEqual(0x40000000, pRepCode.to68(math.ldexp(0.5, -151)))
self.assertEqual(0x40000000, cRepCode.to68(math.ldexp(0.5, -151)))
self.assertEqual(0x40000000, RepCode.to68(math.ldexp(0.5, -151)))
def test_02_02(self):
"""TestRepCodeTo68LowExponent.test_02_02(): from68() mantissa depression - RepCode module."""
# print('0x00000001', RepCode.from68(0x00000001))
self.assertEqual((0.5, -150), math.frexp(RepCode.from68(0x00000001)))
self.assertEqual((0.5, -149), math.frexp(RepCode.from68(0x00000002)))
self.assertEqual((0.5, -148), math.frexp(RepCode.from68(0x00000004)))
self.assertEqual((0.5, -147), math.frexp(RepCode.from68(0x00000008)))
self.assertEqual((0.5, -146), math.frexp(RepCode.from68(0x00000010)))
self.assertEqual((0.5, -145), math.frexp(RepCode.from68(0x00000020)))
self.assertEqual((0.5, -144), math.frexp(RepCode.from68(0x00000040)))
self.assertEqual((0.5, -143), math.frexp(RepCode.from68(0x00000080)))
self.assertEqual((0.5, -142), math.frexp(RepCode.from68(0x00000100)))
self.assertEqual((0.5, -141), math.frexp(RepCode.from68(0x00000200)))
self.assertEqual((0.5, -140), math.frexp(RepCode.from68(0x00000400)))
self.assertEqual((0.5, -139), math.frexp(RepCode.from68(0x00000800)))
self.assertEqual((0.5, -138), math.frexp(RepCode.from68(0x00001000)))
self.assertEqual((0.5, -137), math.frexp(RepCode.from68(0x00002000)))
self.assertEqual((0.5, -136), math.frexp(RepCode.from68(0x00004000)))
self.assertEqual((0.5, -135), math.frexp(RepCode.from68(0x00008000)))
self.assertEqual((0.5, -134), math.frexp(RepCode.from68(0x00010000)))
self.assertEqual((0.5, -133), math.frexp(RepCode.from68(0x00020000)))
self.assertEqual((0.5, -132), math.frexp(RepCode.from68(0x00040000)))
self.assertEqual((0.5, -131), math.frexp(RepCode.from68(0x00080000)))
self.assertEqual((0.5, -130), math.frexp(RepCode.from68(0x00100000)))
self.assertEqual((0.5, -129), math.frexp(RepCode.from68(0x00200000)))
self.assertEqual((0.5, -128), math.frexp(RepCode.from68(0x00400000)))
