def testInitWithRadius(self):
"""
Verifies you can use radius to specify a log encoder
"""
# Create the encoder
le = LogEncoder(w=1,
radius=1,
minval=1,
maxval=10000,
name="amount",
forced=True)
self.assertEqual(le.encoder.n, 5)
# Verify a a couple powers of 10 are encoded as expected
value = 1.0
output = le.encode(value)
expected = [1, 0, 0, 0, 0]
# Convert to numpy array
expected = numpy.array(expected, dtype="uint8")
self.assertTrue(numpy.array_equal(output, expected))
value = 100.0
output = le.encode(value)
expected = [0, 0, 1, 0, 0]
# Convert to numpy array
expected = numpy.array(expected, dtype="uint8")
self.assertTrue(numpy.array_equal(output, expected))
def testInitWithRadius(self):
"""
Verifies you can use radius to specify a log encoder
"""
# Create the encoder
le = LogEncoder(w=1,
radius=1,
minval=1,
maxval=10000,
name="amount",
forced=True)
self.assertEqual(le.encoder.n, 5)
# Verify a a couple powers of 10 are encoded as expected
value = 1.0
output = le.encode(value)
expected = [1, 0, 0, 0, 0]
# Convert to numpy array
expected = cupy.array(expected, dtype="uint8")
self.assertTrue(numpy.array_equal(output, expected))
value = 100.0
output = le.encode(value)
expected = [0, 0, 1, 0, 0]
# Convert to numpy array
expected = cupy.array(expected, dtype="uint8")
self.assertTrue(numpy.array_equal(output, expected))
def testGetBucketValues(self):
"""
Verify that the values of buckets are as expected for given
init params
"""
# Create the encoder
le = LogEncoder(w=5,
resolution=0.1,
minval=1,
maxval=10000,
name="amount",
forced=True)
# Build our expected values
inc = 0.1
exp = 0
expected = []
# Incrementing to exactly 4.0 runs into fp issues
while exp <= 4.0001:
val = 10 ** exp
expected.append(val)
exp += inc
expected = numpy.array(expected)
actual = numpy.array(le.getBucketValues())
numpy.testing.assert_almost_equal(expected, actual, 7)
def testInitWithRadius(self):
'''
Verifies you can use radius to specify a log encoder
'''
# Create the encoder
le = LogEncoder(w=1,
radius=1,
minval=1,
maxval=10000,
name="amount")
self.assertEqual(le.encoder.n, 5)
#######################################################################
# Verify a a couple powers of 10 are encoded as expected
value = 1.0
output = le.encode(value)
expected = [1, 0, 0, 0, 0]
# Convert to numpy array
expected = numpy.array(expected, dtype='uint8')
self.assertTrue((output == expected).all())
value = 100.0
output = le.encode(value)
expected = [0, 0, 1, 0, 0]
# Convert to numpy array
expected = numpy.array(expected, dtype='uint8')
self.assertTrue((output == expected).all())
def testGetBucketValues(self):
"""
Verify that the values of buckets are as expected for given
init params
"""
# Create the encoder
le = LogEncoder(w=5,
resolution=0.1,
minval=1,
maxval=10000,
name="amount",
forced=True)
# Build our expected values
inc = 0.1
exp = 0
expected = []
# Incrementing to exactly 4.0 runs into fp issues
while exp <= 4.0001:
val = 10**exp
expected.append(val)
exp += inc
expected = cupy.array(expected)
actual = cupy.array(le.getBucketValues())
numpy.testing.assert_almost_equal(expected, actual, 7)
def testInitWithRadius(self):
'''
Verifies you can use radius to specify a log encoder
'''
# Create the encoder
le = LogEncoder(w=1, radius=1, minval=1, maxval=10000, name="amount")
self.assertEqual(le.encoder.n, 5)
#######################################################################
# Verify a a couple powers of 10 are encoded as expected
value = 1.0
output = le.encode(value)
expected = [1, 0, 0, 0, 0]
# Convert to numpy array
expected = numpy.array(expected, dtype='uint8')
self.assertTrue((output == expected).all())
value = 100.0
output = le.encode(value)
expected = [0, 0, 1, 0, 0]
# Convert to numpy array
expected = numpy.array(expected, dtype='uint8')
self.assertTrue((output == expected).all())
def testReadWrite(self):
le = LogEncoder(w=5, resolution=0.1, minval=1, maxval=10000, name="amount", forced=True)
originalValue = le.encode(1.0)
proto1 = LogEncoderProto.new_message()
le.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = LogEncoderProto.read(f)
encoder = LogEncoder.read(proto2)
self.assertIsInstance(encoder, LogEncoder)
self.assertEqual(encoder.minScaledValue, le.minScaledValue)
self.assertEqual(encoder.maxScaledValue, le.maxScaledValue)
self.assertEqual(encoder.minval, le.minval)
self.assertEqual(encoder.maxval, le.maxval)
self.assertEqual(encoder.name, le.name)
self.assertEqual(encoder.verbosity, le.verbosity)
self.assertEqual(encoder.clipInput, le.clipInput)
self.assertEqual(encoder.width, le.width)
self.assertEqual(encoder.description, le.description)
self.assertIsInstance(encoder.encoder, ScalarEncoder)
self.assertTrue(numpy.array_equal(encoder.encode(1), originalValue))
self.assertEqual(le.decode(encoder.encode(1)), encoder.decode(le.encode(1)))
# Feed in a new value and ensure the encodings match
result1 = le.encode(10)
result2 = encoder.encode(10)
self.assertTrue(numpy.array_equal(result1, result2))
def testInitWithN(self):
"""
Verifies you can use N to specify a log encoder
"""
# Create the encoder
n = 100
le = LogEncoder(n=n, forced=True)
self.assertEqual(le.encoder.n, n)
def testInitWithN(self):
'''
Verifies you can use N to specify a log encoder
'''
# Create the encoder
n = 100
le = LogEncoder(n=n)
self.assertEqual(le.encoder.n, n)
def testMinvalMaxVal(self):
'''
Verifies unusual instances of minval and maxval are handled properly
'''
self.assertRaises(ValueError, LogEncoder, n=100, minval=0, maxval=-100, forced=True)
self.assertRaises(ValueError, LogEncoder, n=100, minval=0, maxval=1e-07, forced=True)
le = LogEncoder(n=100, minval=42, maxval=1.3e12, forced=True)
expectedRadius = 0.552141792732
expectedResolution = 0.110428358546
self.assertAlmostEqual(le.encoder.radius, expectedRadius)
self.assertAlmostEqual(le.encoder.resolution, expectedResolution)
def testReadWrite(self):
le = LogEncoder(w=5,
resolution=0.1,
minval=1,
maxval=10000,
name="amount",
forced=True)
originalValue = le.encode(1.0)
proto1 = LogEncoderProto.new_message()
le.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = LogEncoderProto.read(f)
encoder = LogEncoder.read(proto2)
self.assertIsInstance(encoder, LogEncoder)
self.assertEqual(encoder.minScaledValue, le.minScaledValue)
self.assertEqual(encoder.maxScaledValue, le.maxScaledValue)
self.assertEqual(encoder.minval, le.minval)
self.assertEqual(encoder.maxval, le.maxval)
self.assertEqual(encoder.name, le.name)
self.assertEqual(encoder.verbosity, le.verbosity)
self.assertEqual(encoder.clipInput, le.clipInput)
self.assertEqual(encoder.width, le.width)
self.assertEqual(encoder.description, le.description)
self.assertIsInstance(encoder.encoder, ScalarEncoder)
self.assertTrue(numpy.array_equal(encoder.encode(1), originalValue))
self.assertEqual(le.decode(encoder.encode(1)),
encoder.decode(le.encode(1)))
# Feed in a new value and ensure the encodings match
result1 = le.encode(10)
result2 = encoder.encode(10)
self.assertTrue(numpy.array_equal(result1, result2))
def testLogEncoder(self):
# Create the encoder
# use of forced=True is not recommended, but is used in the example for
# readibility, see scalar.py
le = LogEncoder(w=5,
resolution=0.1,
minval=1,
maxval=10000,
name="amount",
forced=True)
# Verify we're setting the description properly
self.assertEqual(le.getDescription(), [("amount", 0)])
# Verify we're getting the correct field types
types = le.getDecoderOutputFieldTypes()
self.assertEqual(types[0], FieldMetaType.float)
# Verify the encoder ends up with the correct width
#
# 10^0 -> 10^4 => 0 -> 4; With a resolution of 0.1
# 41 possible values plus padding = 4 = width 45
self.assertEqual(le.getWidth(), 45)
# Verify we have the correct number of possible values
self.assertEqual(len(le.getBucketValues()), 41)
# Verify closeness calculations
testTuples = [([1], [10000], 0.0),
([1], [1000], 0.25),
([1], [1], 1.0),
([1], [-200], 1.0)]
for tp in testTuples:
expected = tp[0]
actual = tp[1]
expectedResult = tp[2]
self.assertEqual(le.closenessScores(expected, actual),
expectedResult,
"exp: %s act: %s expR: %s" % (str(expected),
str(actual),
str(expectedResult)))
# Verify a value of 1.0 is encoded as expected
value = 1.0
output = le.encode(value)
# Our expected encoded representation of the value 1 is the first
# w bits on in an array of len width.
expected = [1, 1, 1, 1, 1] + 40 * [0]
# Convert to numpy array
expected = numpy.array(expected, dtype="uint8")
self.assertTrue(numpy.array_equal(output, expected))
# Test reverse lookup
decoded = le.decode(output)
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertTrue(numpy.array_equal(ranges[0], [1, 1]))
# Verify an input representing a missing value is handled properly
mvOutput = le.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
self.assertEqual(sum(mvOutput), 0)
# Test top-down for all values
value = le.minval
while value <= le.maxval:
output = le.encode(value)
topDown = le.topDownCompute(output)
# Do the scaling by hand here.
scaledVal = math.log10(value)
# Find the range of values that would also produce this top down
# output.
minTopDown = math.pow(10, (scaledVal - le.encoder.resolution))
maxTopDown = math.pow(10, (scaledVal + le.encoder.resolution))
# Verify the range surrounds this scaled val
self.assertGreaterEqual(topDown.value, minTopDown)
self.assertLessEqual(topDown.value, maxTopDown)
# Test bucket support
bucketIndices = le.getBucketIndices(value)
topDown = le.getBucketInfo(bucketIndices)[0]
# Verify our reconstructed value is in the valid range
self.assertGreaterEqual(topDown.value, minTopDown)
self.assertLessEqual(topDown.value, maxTopDown)
# Same for the scalar value
self.assertGreaterEqual(topDown.scalar, minTopDown)
self.assertLessEqual(topDown.scalar, maxTopDown)
# That the encoding portion of our EncoderResult matched the result of
# encode()
self.assertTrue(numpy.array_equal(topDown.encoding, output))
# Verify our reconstructed value is the same as the bucket value
bucketValues = le.getBucketValues()
self.assertEqual(topDown.value,
bucketValues[bucketIndices[0]])
# Next value
scaledVal += le.encoder.resolution / 4.0
value = math.pow(10, scaledVal)
# Verify next power of 10 encoding
output = le.encode(100)
# increase of 2 decades = 20 decibels
# bit 0, 1 are padding; bit 3 is 1, ..., bit 22 is 20 (23rd bit)
expected = 20 * [0] + [1, 1, 1, 1, 1] + 20 * [0]
expected = numpy.array(expected, dtype="uint8")
self.assertTrue(numpy.array_equal(output, expected))
# Test reverse lookup
decoded = le.decode(output)
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertTrue(numpy.array_equal(ranges[0], [100, 100]))
# Verify next power of 10 encoding
output = le.encode(10000)
expected = 40 * [0] + [1, 1, 1, 1, 1]
expected = numpy.array(expected, dtype="uint8")
self.assertTrue(numpy.array_equal(output, expected))
# Test reverse lookup
decoded = le.decode(output)
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertTrue(numpy.array_equal(ranges[0], [10000, 10000]))
print "################# BEGIN TESTING #################"; print ### General performance of the encoder import math import numpy from nupic.encoders.logenc import LogEncoder test = LogEncoder(w=5, minval=1e-07, maxval=10000, resolution=0.5, periodic=False, clipInput=True) example_10 = test.encode(10) example_990 = test.encode(990) print "1e-07 =", test.encode(1e-07) print "10 = ", example_10 print "100 = ", test.encode(100) print "990 = ", example_990 print "1000 = ", test.encode(1000) print "10000 = ", test.encode(10000); print print "Decode 10: ", test.decode(example_10) print "Deocde 990: ", test.decode(example_990); print print "getBucketValues(self):", test.getBucketValues(); print print "getBucketInfo(self, buckets):", test.getBucketInfo([16]); print print "getBucketIndices(10):", test.getBucketIndices(10) print "getBucketIndices(1000):", test.getBucketIndices(1000) print "getBucketIndices(1e-07):", test.getBucketIndices(1e-07); print
def testLogEncoder(self):
# Create the encoder
# use of forced=True is not recommended, but is used in the example for
# readibility, see scalar.py
le = LogEncoder(w=5,
resolution=0.1,
minval=1,
maxval=10000,
name="amount",
forced=True)
# Verify we're setting the description properly
self.assertEqual(le.getDescription(), [("amount", 0)])
# Verify we're getting the correct field types
types = le.getDecoderOutputFieldTypes()
self.assertEqual(types[0], FieldMetaType.float)
# Verify the encoder ends up with the correct width
#
# 10^0 -> 10^4 => 0 -> 4; With a resolution of 0.1
# 41 possible values plus padding = 4 = width 45
self.assertEqual(le.getWidth(), 45)
# Verify we have the correct number of possible values
self.assertEqual(len(le.getBucketValues()), 41)
# Verify closeness calculations
testTuples = [([1], [10000], 0.0), ([1], [1000], 0.25),
([1], [1], 1.0), ([1], [-200], 1.0)]
for tp in testTuples:
expected = tp[0]
actual = tp[1]
expectedResult = tp[2]
self.assertEqual(
le.closenessScores(expected, actual), expectedResult,
"exp: %s act: %s expR: %s" %
(str(expected), str(actual), str(expectedResult)))
# Verify a value of 1.0 is encoded as expected
value = 1.0
output = le.encode(value)
# Our expected encoded representation of the value 1 is the first
# w bits on in an array of len width.
expected = [1, 1, 1, 1, 1] + 40 * [0]
# Convert to numpy array
expected = cupy.array(expected, dtype="uint8")
self.assertTrue(numpy.array_equal(output, expected))
# Test reverse lookup
decoded = le.decode(output)
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertTrue(numpy.array_equal(ranges[0], [1, 1]))
# Verify an input representing a missing value is handled properly
mvOutput = le.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
self.assertEqual(sum(mvOutput), 0)
# Test top-down for all values
value = le.minval
while value <= le.maxval:
output = le.encode(value)
topDown = le.topDownCompute(output)
# Do the scaling by hand here.
scaledVal = math.log10(value)
# Find the range of values that would also produce this top down
# output.
minTopDown = math.pow(10, (scaledVal - le.encoder.resolution))
maxTopDown = math.pow(10, (scaledVal + le.encoder.resolution))
# Verify the range surrounds this scaled val
self.assertGreaterEqual(topDown.value, minTopDown)
self.assertLessEqual(topDown.value, maxTopDown)
# Test bucket support
bucketIndices = le.getBucketIndices(value)
topDown = le.getBucketInfo(bucketIndices)[0]
# Verify our reconstructed value is in the valid range
self.assertGreaterEqual(topDown.value, minTopDown)
self.assertLessEqual(topDown.value, maxTopDown)
# Same for the scalar value
self.assertGreaterEqual(topDown.scalar, minTopDown)
self.assertLessEqual(topDown.scalar, maxTopDown)
# That the encoding portion of our EncoderResult matched the result of
# encode()
self.assertTrue(numpy.array_equal(topDown.encoding, output))
# Verify our reconstructed value is the same as the bucket value
bucketValues = le.getBucketValues()
self.assertEqual(topDown.value, bucketValues[bucketIndices[0]])
# Next value
scaledVal += le.encoder.resolution / 4.0
value = math.pow(10, scaledVal)
# Verify next power of 10 encoding
output = le.encode(100)
# increase of 2 decades = 20 decibels
# bit 0, 1 are padding; bit 3 is 1, ..., bit 22 is 20 (23rd bit)
expected = 20 * [0] + [1, 1, 1, 1, 1] + 20 * [0]
expected = cupy.array(expected, dtype="uint8")
self.assertTrue(numpy.array_equal(output, expected))
# Test reverse lookup
decoded = le.decode(output)
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertTrue(numpy.array_equal(ranges[0], [100, 100]))
# Verify next power of 10 encoding
output = le.encode(10000)
expected = 40 * [0] + [1, 1, 1, 1, 1]
expected = cupy.array(expected, dtype="uint8")
self.assertTrue(numpy.array_equal(output, expected))
# Test reverse lookup
decoded = le.decode(output)
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertTrue(numpy.array_equal(ranges[0], [10000, 10000]))
def testLogEncoder(self):
print "Testing LogEncoder...",
l = LogEncoder(w=5, resolution=1, minval=1, maxval=10000, name="amount")
self.assertEqual(l.getDescription(), [("amount", 0)])
# -------------------------------------------------------------------
# 10^0 -> 10^4 => 0 decibels -> 40 decibels;
# 41 possible decibel values plus padding=4 = width 45
self.assertEqual(l.getWidth(), 45)
value = 1.0
output = l.encode(value)
expected = [1, 1, 1, 1, 1] + 40 * [0]
expected = numpy.array(expected, dtype='uint8')
self.assertTrue((output == expected).all())
# Test reverse lookup
decoded = l.decode(output)
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded)
self.assertTrue(len(ranges) == 1 and numpy.array_equal(ranges[0], [1, 1]))
# MISSING VALUE
mvOutput = l.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
self.assertEqual(sum(mvOutput), 0)
# Test top-down
value = l.minval
while value <= l.maxval:
output = l.encode(value)
print "output of %f =>" % (value), output
topDown = l.topDownCompute(output)
print "topdown =>", topDown
scaledVal = 10 * math.log10(value)
minTopDown = math.pow(10, (scaledVal-l.encoder.resolution) / 10.0)
maxTopDown = math.pow(10, (scaledVal+l.encoder.resolution) / 10.0)
self.assertTrue(topDown.value >= minTopDown and topDown.value <= maxTopDown)
# Test bucket support
bucketIndices = l.getBucketIndices(value)
print "bucket index =>", bucketIndices[0]
topDown = l.getBucketInfo(bucketIndices)[0]
self.assertTrue(topDown.value >= minTopDown and topDown.value <= maxTopDown)
self.assertTrue(topDown.scalar >= minTopDown and topDown.scalar <= maxTopDown)
self.assertTrue((topDown.encoding == output).all())
self.assertEqual(topDown.value, l.getBucketValues()[bucketIndices[0]])
# Next value
scaledVal += l.encoder.resolution/4
value = math.pow(10, scaledVal / 10.0)
# -------------------------------------------------------------------
output = l.encode(100)
# increase of 2 decades = 20 decibels
# bit 0, 1 are padding; bit 3 is 1, ..., bit 22 is 20 (23rd bit)
expected = 20 * [0] + [1, 1, 1, 1, 1] + 20 * [0]
expected = numpy.array(expected, dtype='uint8')
self.assertTrue((output == expected).all())
# Test reverse lookup
decoded = l.decode(output)
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertTrue(len(ranges) == 1 and numpy.array_equal(ranges[0], [100, 100]))
print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded)
# -------------------------------------------------------------------
output = l.encode(10000)
expected = 40 * [0] + [1, 1, 1, 1, 1]
expected = numpy.array(expected, dtype='uint8')
self.assertTrue((output == expected).all())
# Test reverse lookup
decoded = l.decode(output)
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertTrue(len(ranges) == 1 and numpy.array_equal(ranges[0], [10000, 10000]))
print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded)