def testSetFieldStats(self):
"""Test setting the min and max using setFieldStats"""
def _dumpParams(enc):
return (enc.n, enc.w, enc.minval, enc.maxval, enc.resolution,
enc.windowSize, enc._learningEnabled, enc.recordNum,
enc.radius, enc.rangeInternal, enc.padding, enc.nInternal)
sfs = AdaptiveScalarEncoder(name='scalar',
n=14,
w=5,
minval=1,
maxval=10,
periodic=False)
reg = AdaptiveScalarEncoder(name='scalar',
n=14,
w=5,
minval=1,
maxval=100,
periodic=False)
self.assertTrue(_dumpParams(sfs) != _dumpParams(reg), "Params should not be equal, "\
"since the two encoders were instantiated with different values.")
# set the min and the max using sFS to 1,100 respectively.
sfs.setFieldStats('this', {"this": {"min": 1, "max": 100}})
#Now the parameters for both should be the same
self.assertEqual(_dumpParams(sfs), _dumpParams(reg), "Params should now be equal, "\
"but they are not. sFS should be equivalent to initialization.")
def testMissingValues(self): """missing values""" # forced: it's strongly recommended to use w>=21, in the example we force skip the check for readib. mv = AdaptiveScalarEncoder(name='mv', n=14, w=3, minval=1, maxval=8, periodic=False, forced=True) empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA) print "\nEncoded missing data \'None\' as %s" % empty self.assertEqual(empty.sum(), 0)
def setUp(self):
self._l = AdaptiveScalarEncoder(name='scalar',
n=14,
w=5,
minval=1,
maxval=10,
periodic=False)
def __init__(self,
w,
minval=None,
maxval=None,
periodic=False,
n=0,
radius=0,
resolution=0,
name=None,
verbosity=0,
clipInput=True):
"""[ScalarEncoder class method override]"""
self._learningEnabled = True
self._stateLock = False
self.width = 0
self.encoders = None
self.description = []
self.name = name
if periodic:
#Delta scalar encoders take non-periodic inputs only
raise Exception('Delta encoder does not encode periodic inputs')
assert n != 0 #An adaptive encoder can only be intialized using n
self._adaptiveScalarEnc = AdaptiveScalarEncoder(w=w,
n=n,
minval=minval,
maxval=maxval,
clipInput=True,
name=name,
verbosity=verbosity)
self.width += self._adaptiveScalarEnc.getWidth()
self.n = self._adaptiveScalarEnc.n
self._prevAbsolute = None #how many inputs have been sent to the encoder?
self._prevDelta = None
def setUp(self):
# forced: it's strongly recommended to use w>=21, in the example we force
# skip the check for readibility
self._l = AdaptiveScalarEncoder(name="scalar",
n=14,
w=5,
minval=1,
maxval=10,
periodic=False,
forced=True)
def testMissingValues(self):
"""missing values"""
mv = AdaptiveScalarEncoder(name='mv',
n=14,
w=3,
minval=1,
maxval=8,
periodic=False)
empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
print "\nEncoded missing data \'None\' as %s" % empty
self.assertEqual(empty.sum(), 0)
def testSetFieldStats(self):
"""Test setting the min and max using setFieldStats"""
def _dumpParams(enc):
return (enc.n, enc.w, enc.minval, enc.maxval, enc.resolution,
enc._learningEnabled, enc.recordNum,
enc.radius, enc.rangeInternal, enc.padding, enc.nInternal)
sfs = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10,
periodic=False, forced=True)
reg = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=100,
periodic=False, forced=True)
self.assertTrue(_dumpParams(sfs) != _dumpParams(reg), "Params should not be equal, "\
"since the two encoders were instantiated with different values.")
# set the min and the max using sFS to 1,100 respectively.
sfs.setFieldStats('this',{"this":{"min":1,"max":100}})
#Now the parameters for both should be the same
self.assertEqual(_dumpParams(sfs), _dumpParams(reg), "Params should now be equal, "\
"but they are not. sFS should be equivalent to initialization.")
def testNonPeriodicEncoderMinMaxNotSpec(self):
"""Non-periodic encoder, min and max not specified"""
l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=None, maxval=None,
periodic=False, forced=True)
def _verify(v, encoded, expV=None):
if expV is None:
expV = v
self.assertTrue((l.encode(v) == numpy.array(encoded, dtype=defaultDtype)).all())
self.assertTrue(abs(l.getBucketInfo(l.getBucketIndices(v))[0].value - expV) <= \
l.resolution/2)
def _verifyNot(v, encoded):
self.assertFalse((l.encode(v) == numpy.array(encoded, dtype=defaultDtype)).all())
_verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(2, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(3, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0])
_verify(-9, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-8, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-7, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-6, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-5, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0])
_verify(0, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
_verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0])
_verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(11, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(12, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(13, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(14, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(15, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
#"""Test switching learning off"""
l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10,
periodic=False, forced=True)
_verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(10, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
l.setLearning(False)
_verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], expV=20)
_verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1)
_verify(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1)
l.setLearning(True)
_verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verifyNot(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verifyNot(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
def __new__(self, w, minval=None, maxval=None, periodic=False, n=0, radius=0,
resolution=0, name=None, verbosity=0, clipInput=False,
space="absolute"):
self._encoder = None
if space == "absolute":
ret = AdaptiveScalarEncoder(w,minval,maxval,periodic,n,radius,
resolution,name,verbosity,clipInput)
else:
ret = DeltaEncoder(w,minval,maxval,periodic,n,radius,resolution,name,verbosity,clipInput)
return ret
def read(cls, proto):
encoder = object.__new__(cls)
encoder.width = proto.width
encoder.name = proto.name or None
encoder.n = proto.n
encoder._adaptiveScalarEnc = (AdaptiveScalarEncoder.read(
proto.adaptiveScalarEnc))
encoder._prevAbsolute = proto.prevAbsolute
encoder._prevDelta = proto.prevDelta
encoder._stateLock = proto.stateLock
return encoder
def read(cls, proto):
encoder = object.__new__(cls)
encoder.width = proto.width
encoder.name = proto.name or None
encoder.n = proto.n
encoder._adaptiveScalarEnc = (
AdaptiveScalarEncoder.read(proto.adaptiveScalarEnc)
)
encoder._prevAbsolute = proto.prevAbsolute
encoder._prevDelta = proto.prevDelta
encoder._stateLock = proto.stateLock
return encoder
def testNonPeriodicEncoderMinMaxNotSpec(self):
"""Non-periodic encoder, min and max not specified"""
l = AdaptiveScalarEncoder(name="scalar", n=14, w=5, minval=None,
maxval=None, periodic=False, forced=True)
def _verify(v, encoded, expV=None):
if expV is None:
expV = v
self.assertTrue(numpy.array_equal(
l.encode(v),
numpy.array(encoded, dtype=defaultDtype)))
self.assertLessEqual(
abs(l.getBucketInfo(l.getBucketIndices(v))[0].value - expV),
l.resolution/2)
def _verifyNot(v, encoded):
self.assertFalse(numpy.array_equal(
l.encode(v), numpy.array(encoded, dtype=defaultDtype)))
_verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(2, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(3, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0])
_verify(-9, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-8, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-7, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-6, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-5, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0])
_verify(0, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
_verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0])
_verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(11, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(12, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(13, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(14, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(15, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
#"""Test switching learning off"""
l = AdaptiveScalarEncoder(name="scalar", n=14, w=5, minval=1, maxval=10,
periodic=False, forced=True)
_verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(10, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
l.setLearning(False)
_verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], expV=20)
_verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1)
_verify(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1)
l.setLearning(True)
_verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verifyNot(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verifyNot(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
def __init__(self, w, minval=None, maxval=None, periodic=False, n=0, radius=0,
resolution=0, name=None, verbosity=0, clipInput=True):
"""[ScalarEncoder class method override]"""
self._learningEnabled = True
self._stateLock = False
self.width = 0
self.encoders = None
self.description = []
self.name = name
if periodic:
#Delta scalar encoders take non-periodic inputs only
raise Exception('Delta encoder does not encode periodic inputs')
assert n!=0 #An adaptive encoder can only be intialized using n
self._adaptiveScalarEnc = AdaptiveScalarEncoder(w=w, n=n, minval=minval,
maxval=maxval, clipInput=True, name=name, verbosity=verbosity)
self.width+=self._adaptiveScalarEnc.getWidth()
self.n = self._adaptiveScalarEnc.n
self._prevAbsolute = None #how many inputs have been sent to the encoder?
self._prevDelta = None
def testReadWrite(self):
originalValue = self._l.encode(1)
proto1 = AdaptiveScalarEncoderProto.new_message()
self._l.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = AdaptiveScalarEncoderProto.read(f)
encoder = AdaptiveScalarEncoder.read(proto2)
self.assertIsInstance(encoder, AdaptiveScalarEncoder)
self.assertEqual(encoder.recordNum, self._l.recordNum)
self.assertDictEqual(encoder.slidingWindow.__dict__,
self._l.slidingWindow.__dict__)
self.assertEqual(encoder.w, self._l.w)
self.assertEqual(encoder.minval, self._l.minval)
self.assertEqual(encoder.maxval, self._l.maxval)
self.assertEqual(encoder.periodic, self._l.periodic)
self.assertEqual(encoder.n, self._l.n)
self.assertEqual(encoder.radius, self._l.radius)
self.assertEqual(encoder.resolution, self._l.resolution)
self.assertEqual(encoder.name, self._l.name)
self.assertEqual(encoder.verbosity, self._l.verbosity)
self.assertEqual(encoder.clipInput, self._l.clipInput)
self.assertTrue(numpy.array_equal(encoder.encode(1), originalValue))
self.assertEqual(self._l.decode(encoder.encode(1)),
encoder.decode(self._l.encode(1)))
# Feed in a new value and ensure the encodings match
result1 = self._l.encode(7)
result2 = encoder.encode(7)
self.assertTrue(numpy.array_equal(result1, result2))
def testReadWrite(self):
originalValue = self._l.encode(1)
proto1 = AdaptiveScalarEncoderProto.new_message()
self._l.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = AdaptiveScalarEncoderProto.read(f)
encoder = AdaptiveScalarEncoder.read(proto2)
self.assertIsInstance(encoder, AdaptiveScalarEncoder)
self.assertEqual(encoder.recordNum, self._l.recordNum)
self.assertDictEqual(encoder.slidingWindow.__dict__,
self._l.slidingWindow.__dict__)
self.assertEqual(encoder.w, self._l.w)
self.assertEqual(encoder.minval, self._l.minval)
self.assertEqual(encoder.maxval, self._l.maxval)
self.assertEqual(encoder.periodic, self._l.periodic)
self.assertEqual(encoder.n, self._l.n)
self.assertEqual(encoder.radius, self._l.radius)
self.assertEqual(encoder.resolution, self._l.resolution)
self.assertEqual(encoder.name, self._l.name)
self.assertEqual(encoder.verbosity, self._l.verbosity)
self.assertEqual(encoder.clipInput, self._l.clipInput)
self.assertTrue(numpy.array_equal(encoder.encode(1), originalValue))
self.assertEqual(self._l.decode(encoder.encode(1)),
encoder.decode(self._l.encode(1)))
# Feed in a new value and ensure the encodings match
result1 = self._l.encode(7)
result2 = encoder.encode(7)
self.assertTrue(numpy.array_equal(result1, result2))
def testMissingValues(self): """missing values""" mv = AdaptiveScalarEncoder(name='mv', n=14, w=3, minval=1, maxval=8, periodic=False) empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA) print "\nEncoded missing data \'None\' as %s" % empty self.assertEqual(empty.sum(), 0)
class DeltaEncoder(AdaptiveScalarEncoder):
"""
This is an implementation of a delta encoder. The delta encoder encodes differences between # to_note: so basically different input values can have
successive scalar values instead of encoding the actual values. It returns an actual value when # the same representation. The only value that matters
decoding and not a delta. # is the difference between the current input and the last input
""" # problem_with_this_approach: the fact that it uses adaptive scalar encoder makes learning highly improbable, since with each
# new maximum or minimum value, the whole encoding scheme changes, and this change in the encoded representation can easily
# mess whatever the machine has learned so far. Worse, the machine will not even recognize why it is wrong. The developers
# are really going over and abusing the knowledge that the brain can handle various kind of data. Yes, the brain can handle new
# encoded representation, however it takes a lot of time, and the change in encoded representation is not this arbitrary. With
# this kind of arbirtrary (encoding scheme changes merely whenever new max or min input is presented), even the brain's learning
# algorithm will be messed up.
def __init__(self, w, minval=None, maxval=None, periodic=False, n=0, radius=0,
resolution=0, name=None, verbosity=0, clipInput=True, forced=False):
"""[ScalarEncoder class method override]"""
self._learningEnabled = True
self._stateLock = False
self.width = 0
self.encoders = None
self.description = []
self.name = name
if periodic:
#Delta scalar encoders take non-periodic inputs only
raise Exception('Delta encoder does not encode periodic inputs')
assert n!=0 #An adaptive encoder can only be intialized using n
self._adaptiveScalarEnc = AdaptiveScalarEncoder(w=w, n=n, minval=minval,
maxval=maxval, clipInput=True, name=name, verbosity=verbosity, forced=forced)
self.width+=self._adaptiveScalarEnc.getWidth()
self.n = self._adaptiveScalarEnc.n
self._prevAbsolute = None #how many inputs have been sent to the encoder?
self._prevDelta = None
def encodeIntoArray(self, input, output, learn=None):
if not isinstance(input, numbers.Number):
raise TypeError(
"Expected a scalar input but got input of type %s" % type(input))
if learn is None:
learn = self._learningEnabled
if input == SENTINEL_VALUE_FOR_MISSING_DATA:
output[0:self.n] = 0
else:
#make the first delta zero so that the delta ranges are not messed up.
if self._prevAbsolute==None:
self._prevAbsolute= input
delta = input - self._prevAbsolute
self._adaptiveScalarEnc.encodeIntoArray(delta, output, learn) # to_note: generate a representation for the difference between the current
if not self._stateLock: # input and the last input.
self._prevAbsolute = input
self._prevDelta = delta
return output
############################################################################
def setStateLock(self, lock):
self._stateLock = lock
############################################################################
def setFieldStats(self, fieldName, fieldStatistics):
pass
############################################################################
def getBucketIndices(self, input, learn=None):
return self._adaptiveScalarEnc.getBucketIndices(input, learn)
############################################################################
def getBucketInfo(self, buckets):
return self._adaptiveScalarEnc.getBucketInfo(buckets)
############################################################################
def topDownCompute(self, encoded):
"""[ScalarEncoder class method override]"""
#Decode to delta scalar
if self._prevAbsolute==None or self._prevDelta==None:
return [EncoderResult(value=0, scalar=0,
encoding=numpy.zeros(self.n))]
ret = self._adaptiveScalarEnc.topDownCompute(encoded)
if self._prevAbsolute != None:
ret = [EncoderResult(value=ret[0].value+self._prevAbsolute,
scalar=ret[0].scalar+self._prevAbsolute,
encoding=ret[0].encoding)] # problem_with_this_approach: encoded houses the value of delta, so it should
# ret[0].value+=self._prevAbsolute # the decoding scheme in topDownCompute will generate delta. If we add that delta
# ret[0].scalar+=self._prevAbsolute # to the previous absolute scalar, we will get a completely useless result
return ret
@classmethod
def read(cls, proto):
encoder = object.__new__(cls)
encoder.width = proto.width
encoder.name = proto.name or None
encoder.n = proto.n
encoder._adaptiveScalarEnc = (
AdaptiveScalarEncoder.read(proto.adaptiveScalarEnc)
)
encoder._prevAbsolute = proto.prevAbsolute
encoder._prevDelta = proto.prevDelta
encoder._stateLock = proto.stateLock
return encoder
def write(self, proto):
proto.width = self.width
proto.name = self.name or ""
proto.n = self.n
self._adaptiveScalarEnc.write(proto.adaptiveScalarEnc)
proto.prevAbsolute = self._prevAbsolute
proto.prevDelta = self._prevDelta
proto.stateLock = self._stateLock
class DeltaEncoder(AdaptiveScalarEncoder):
"""
This is an implementation of a delta encoder. The delta encoder encodes differences between
successive scalar values instead of encoding the actual values. It returns an actual value when
decoding and not a delta.
"""
def __init__(self,
w,
minval=None,
maxval=None,
periodic=False,
n=0,
radius=0,
resolution=0,
name=None,
verbosity=0,
clipInput=True):
"""[ScalarEncoder class method override]"""
self._learningEnabled = True
self._stateLock = False
self.width = 0
self.encoders = None
self.description = []
self.name = name
if periodic:
#Delta scalar encoders take non-periodic inputs only
raise Exception('Delta encoder does not encode periodic inputs')
assert n != 0 #An adaptive encoder can only be intialized using n
self._adaptiveScalarEnc = AdaptiveScalarEncoder(w=w,
n=n,
minval=minval,
maxval=maxval,
clipInput=True,
name=name,
verbosity=verbosity)
self.width += self._adaptiveScalarEnc.getWidth()
self.n = self._adaptiveScalarEnc.n
self._prevAbsolute = None #how many inputs have been sent to the encoder?
self._prevDelta = None
def encodeIntoArray(self, input, output, learn=None):
if learn is None:
learn = self._learningEnabled
if input == SENTINEL_VALUE_FOR_MISSING_DATA:
output[0:self.n] = 0
else:
#make the first delta zero so that the delta ranges are not messed up.
if self._prevAbsolute == None:
self._prevAbsolute = input
delta = input - self._prevAbsolute
self._adaptiveScalarEnc.encodeIntoArray(delta, output, learn)
if not self._stateLock:
self._prevAbsolute = input
self._prevDelta = delta
return output
############################################################################
def setStateLock(self, lock):
self._stateLock = lock
############################################################################
def setFieldStats(self, fieldName, fieldStatistics):
pass
############################################################################
def isDelta(self):
return True
############################################################################
def getBucketIndices(self, input, learn=None):
return self._adaptiveScalarEnc.getBucketIndices(input, learn)
############################################################################
def getBucketInfo(self, buckets):
return self._adaptiveScalarEnc.getBucketInfo(buckets)
############################################################################
def topDownCompute(self, encoded):
"""[ScalarEncoder class method override]"""
#Decode to delta scalar
if self._prevAbsolute == None or self._prevDelta == None:
return [
EncoderResult(value=0, scalar=0, encoding=numpy.zeros(self.n))
]
ret = self._adaptiveScalarEnc.topDownCompute(encoded)
if self._prevAbsolute != None:
ret = [
EncoderResult(value=ret[0].value + self._prevAbsolute,
scalar=ret[0].scalar + self._prevAbsolute,
encoding=ret[0].encoding)
]
# ret[0].value+=self._prevAbsolute
# ret[0].scalar+=self._prevAbsolute
return ret
class AdaptiveScalarTest(unittest.TestCase):
"""Tests for AdaptiveScalarEncoder"""
def setUp(self):
# forced: it's strongly recommended to use w>=21, in the example we force
# skip the check for readibility
self._l = AdaptiveScalarEncoder(name="scalar", n=14, w=5, minval=1,
maxval=10, periodic=False, forced=True)
def testMissingValues(self):
"""missing values"""
# forced: it's strongly recommended to use w>=21, in the example we force
# skip the check for readib.
mv = AdaptiveScalarEncoder(name="mv", n=14, w=3, minval=1, maxval=8,
periodic=False, forced=True)
empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
self.assertEqual(empty.sum(), 0)
def testNonPeriodicEncoderMinMaxSpec(self):
"""Non-periodic encoder, min and max specified"""
self.assertTrue(numpy.array_equal(
self._l.encode(1),
numpy.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(
self._l.encode(2),
numpy.array([0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(
self._l.encode(10),
numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
dtype=defaultDtype)))
def testTopDownDecode(self):
"""Test the input description generation and topDown decoding"""
l = self._l
v = l.minval
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertLess(abs(rangeMin - v), l.resolution)
topDown = l.topDownCompute(output)[0]
self.assertLessEqual(abs(topDown.value - v), l.resolution)
# Test bucket support
bucketIndices = l.getBucketIndices(v)
topDown = l.getBucketInfo(bucketIndices)[0]
self.assertLessEqual(abs(topDown.value - v), l.resolution / 2)
self.assertEqual(topDown.value, l.getBucketValues()[bucketIndices[0]])
self.assertEqual(topDown.scalar, topDown.value)
self.assertTrue(numpy.array_equal(topDown.encoding, output))
# Next value
v += l.resolution / 4
def testFillHoles(self):
"""Make sure we can fill in holes"""
l=self._l
decoded = l.decode(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1]))
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertSequenceEqual(ranges[0], [10, 10])
decoded = l.decode(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1]))
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertSequenceEqual(ranges[0], [10, 10])
def testNonPeriodicEncoderMinMaxNotSpec(self):
"""Non-periodic encoder, min and max not specified"""
l = AdaptiveScalarEncoder(name="scalar", n=14, w=5, minval=None,
maxval=None, periodic=False, forced=True)
def _verify(v, encoded, expV=None):
if expV is None:
expV = v
self.assertTrue(numpy.array_equal(
l.encode(v),
numpy.array(encoded, dtype=defaultDtype)))
self.assertLessEqual(
abs(l.getBucketInfo(l.getBucketIndices(v))[0].value - expV),
l.resolution/2)
def _verifyNot(v, encoded):
self.assertFalse(numpy.array_equal(
l.encode(v), numpy.array(encoded, dtype=defaultDtype)))
_verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(2, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(3, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0])
_verify(-9, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-8, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-7, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-6, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-5, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0])
_verify(0, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
_verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0])
_verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(11, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(12, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(13, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(14, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(15, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
#"""Test switching learning off"""
l = AdaptiveScalarEncoder(name="scalar", n=14, w=5, minval=1, maxval=10,
periodic=False, forced=True)
_verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(10, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
l.setLearning(False)
_verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], expV=20)
_verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1)
_verify(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1)
l.setLearning(True)
_verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verifyNot(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verifyNot(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
def testSetFieldStats(self):
"""Test setting the min and max using setFieldStats"""
def _dumpParams(enc):
return (enc.n, enc.w, enc.minval, enc.maxval, enc.resolution,
enc._learningEnabled, enc.recordNum,
enc.radius, enc.rangeInternal, enc.padding, enc.nInternal)
sfs = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10,
periodic=False, forced=True)
reg = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=100,
periodic=False, forced=True)
self.assertNotEqual(_dumpParams(sfs), _dumpParams(reg),
("Params should not be equal, since the two encoders "
"were instantiated with different values."))
# set the min and the max using sFS to 1,100 respectively.
sfs.setFieldStats("this", {"this":{"min":1, "max":100}})
#Now the parameters for both should be the same
self.assertEqual(_dumpParams(sfs), _dumpParams(reg),
("Params should now be equal, but they are not. sFS "
"should be equivalent to initialization."))
def testReadWrite(self):
originalValue = self._l.encode(1)
proto1 = AdaptiveScalarEncoderProto.new_message()
self._l.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = AdaptiveScalarEncoderProto.read(f)
encoder = AdaptiveScalarEncoder.read(proto2)
self.assertIsInstance(encoder, AdaptiveScalarEncoder)
self.assertEqual(encoder.recordNum, self._l.recordNum)
self.assertDictEqual(encoder.slidingWindow.__dict__,
self._l.slidingWindow.__dict__)
self.assertEqual(encoder.w, self._l.w)
self.assertEqual(encoder.minval, self._l.minval)
self.assertEqual(encoder.maxval, self._l.maxval)
self.assertEqual(encoder.periodic, self._l.periodic)
self.assertEqual(encoder.n, self._l.n)
self.assertEqual(encoder.radius, self._l.radius)
self.assertEqual(encoder.resolution, self._l.resolution)
self.assertEqual(encoder.name, self._l.name)
self.assertEqual(encoder.verbosity, self._l.verbosity)
self.assertEqual(encoder.clipInput, self._l.clipInput)
self.assertTrue(numpy.array_equal(encoder.encode(1), originalValue))
self.assertEqual(self._l.decode(encoder.encode(1)),
encoder.decode(self._l.encode(1)))
# Feed in a new value and ensure the encodings match
result1 = self._l.encode(7)
result2 = encoder.encode(7)
self.assertTrue(numpy.array_equal(result1, result2))
class AdaptiveScalarTest(unittest.TestCase):
"""Tests for AdaptiveScalarEncoder"""
def setUp(self):
# forced: it's strongly recommended to use w>=21, in the example we force skip the check for readibility
self._l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10,
periodic=False, forced=True)
def testMissingValues(self):
"""missing values"""
# forced: it's strongly recommended to use w>=21, in the example we force skip the check for readib.
mv = AdaptiveScalarEncoder(name='mv', n=14, w=3, minval=1, maxval=8, periodic=False, forced=True)
empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
print "\nEncoded missing data \'None\' as %s" % empty
self.assertEqual(empty.sum(), 0)
def testNonPeriodicEncoderMinMaxSpec(self):
"""Non-periodic encoder, min and max specified"""
self.assertTrue((self._l.encode(1) == numpy.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((self._l.encode(2) == numpy.array([0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((self._l.encode(10) == numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
dtype=defaultDtype)).all())
def testTopDownDecode(self):
"""Test the input description generation and topDown decoding"""
l=self._l
v = l.minval
print "\nTesting non-periodic encoder decoding, resolution of %f..." % \
l.resolution
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
print "decoding", output, "(%f)=>" % v, l.decodedToStr(decoded)
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertTrue(abs(rangeMin - v) < l.resolution)
topDown = l.topDownCompute(output)[0]
print "topdown =>", topDown
self.assertTrue(abs(topDown.value - v) <= l.resolution)
# Test bucket support
bucketIndices = l.getBucketIndices(v)
print "bucket index =>", bucketIndices[0]
topDown = l.getBucketInfo(bucketIndices)[0]
self.assertTrue(abs(topDown.value - v) <= l.resolution / 2)
self.assertEqual(topDown.value, l.getBucketValues()[bucketIndices[0]])
self.assertEqual(topDown.scalar, topDown.value)
self.assertTrue((topDown.encoding == output).all())
# Next value
v += l.resolution / 4
def testFillHoles(self):
"""Make sure we can fill in holes"""
l=self._l
decoded = l.decode(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertSequenceEqual(ranges[0], [10, 10])
print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded)
decoded = l.decode(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertSequenceEqual(ranges[0], [10, 10])
print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded)
def testNonPeriodicEncoderMinMaxNotSpec(self):
"""Non-periodic encoder, min and max not specified"""
l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=None, maxval=None,
periodic=False, forced=True)
def _verify(v, encoded, expV=None):
if expV is None:
expV = v
self.assertTrue((l.encode(v) == numpy.array(encoded, dtype=defaultDtype)).all())
self.assertTrue(abs(l.getBucketInfo(l.getBucketIndices(v))[0].value - expV) <= \
l.resolution/2)
def _verifyNot(v, encoded):
self.assertFalse((l.encode(v) == numpy.array(encoded, dtype=defaultDtype)).all())
_verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(2, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(3, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0])
_verify(-9, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-8, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-7, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-6, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-5, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0])
_verify(0, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
_verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0])
_verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(11, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(12, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(13, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(14, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(15, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
#"""Test switching learning off"""
l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10,
periodic=False, forced=True)
_verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(10, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
l.setLearning(False)
_verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], expV=20)
_verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1)
_verify(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1)
l.setLearning(True)
_verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verifyNot(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verifyNot(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
def testSetFieldStats(self):
"""Test setting the min and max using setFieldStats"""
def _dumpParams(enc):
return (enc.n, enc.w, enc.minval, enc.maxval, enc.resolution,
enc._learningEnabled, enc.recordNum,
enc.radius, enc.rangeInternal, enc.padding, enc.nInternal)
sfs = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10,
periodic=False, forced=True)
reg = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=100,
periodic=False, forced=True)
self.assertTrue(_dumpParams(sfs) != _dumpParams(reg), "Params should not be equal, "\
"since the two encoders were instantiated with different values.")
# set the min and the max using sFS to 1,100 respectively.
sfs.setFieldStats('this',{"this":{"min":1,"max":100}})
#Now the parameters for both should be the same
self.assertEqual(_dumpParams(sfs), _dumpParams(reg), "Params should now be equal, "\
"but they are not. sFS should be equivalent to initialization.")
def setUp(self):
# forced: it's strongly recommended to use w>=21, in the example we force skip the check for readibility
self._l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10,
periodic=False, forced=True)
class DeltaEncoder(AdaptiveScalarEncoder):
"""
This is an implementation of a delta encoder. The delta encoder encodes differences between
successive scalar values instead of encoding the actual values. It returns an actual value when
decoding and not a delta.
"""
def __init__(self, w, minval=None, maxval=None, periodic=False, n=0, radius=0,
resolution=0, name=None, verbosity=0, clipInput=True):
"""[ScalarEncoder class method override]"""
self._learningEnabled = True
self._stateLock = False
self.width = 0
self.encoders = None
self.description = []
self.name = name
if periodic:
#Delta scalar encoders take non-periodic inputs only
raise Exception('Delta encoder does not encode periodic inputs')
assert n!=0 #An adaptive encoder can only be intialized using n
self._adaptiveScalarEnc = AdaptiveScalarEncoder(w=w, n=n, minval=minval,
maxval=maxval, clipInput=True, name=name, verbosity=verbosity)
self.width+=self._adaptiveScalarEnc.getWidth()
self.n = self._adaptiveScalarEnc.n
self._prevAbsolute = None #how many inputs have been sent to the encoder?
self._prevDelta = None
def encodeIntoArray(self, input, output, learn=None):
if learn is None:
learn = self._learningEnabled
if input == SENTINEL_VALUE_FOR_MISSING_DATA:
output[0:self.n] = 0
else:
#make the first delta zero so that the delta ranges are not messed up.
if self._prevAbsolute==None:
self._prevAbsolute= input
delta = input - self._prevAbsolute
self._adaptiveScalarEnc.encodeIntoArray(delta, output, learn)
if not self._stateLock:
self._prevAbsolute = input
self._prevDelta = delta
return output
############################################################################
def setStateLock(self, lock):
self._stateLock = lock
############################################################################
def setFieldStats(self, fieldName, fieldStatistics):
pass
############################################################################
def isDelta(self):
return True
############################################################################
def getBucketIndices(self, input, learn=None):
return self._adaptiveScalarEnc.getBucketIndices(input, learn)
############################################################################
def getBucketInfo(self, buckets):
return self._adaptiveScalarEnc.getBucketInfo(buckets)
############################################################################
def topDownCompute(self, encoded):
"""[ScalarEncoder class method override]"""
#Decode to delta scalar
if self._prevAbsolute==None or self._prevDelta==None:
return [EncoderResult(value=0, scalar=0,
encoding=numpy.zeros(self.n))]
ret = self._adaptiveScalarEnc.topDownCompute(encoded)
if self._prevAbsolute != None:
ret = [EncoderResult(value=ret[0].value+self._prevAbsolute,
scalar=ret[0].scalar+self._prevAbsolute,
encoding=ret[0].encoding)]
# ret[0].value+=self._prevAbsolute
# ret[0].scalar+=self._prevAbsolute
return ret
class AdaptiveScalarTest(unittest.TestCase):
"""Tests for AdaptiveScalarEncoder"""
def setUp(self):
self._l = AdaptiveScalarEncoder(name='scalar',
n=14,
w=5,
minval=1,
maxval=10,
periodic=False)
def testMissingValues(self):
"""missing values"""
mv = AdaptiveScalarEncoder(name='mv',
n=14,
w=3,
minval=1,
maxval=8,
periodic=False)
empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
print "\nEncoded missing data \'None\' as %s" % empty
self.assertEqual(empty.sum(), 0)
def testNonPeriodicEncoderMinMaxSpec(self):
"""Non-periodic encoder, min and max specified"""
self.assertTrue((self._l.encode(1) == numpy.array(
[1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((self._l.encode(2) == numpy.array(
[0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((self._l.encode(10) == numpy.array(
[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
dtype=defaultDtype)).all())
def testTopDownDecode(self):
"""Test the input description generation and topDown decoding"""
l = self._l
v = l.minval
print "\nTesting non-periodic encoder decoding, resolution of %f..." % \
l.resolution
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
print "decoding", output, "(%f)=>" % v, l.decodedToStr(decoded)
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertTrue(abs(rangeMin - v) < l.resolution)
topDown = l.topDownCompute(output)[0]
print "topdown =>", topDown
self.assertTrue(abs(topDown.value - v) <= l.resolution)
# Test bucket support
bucketIndices = l.getBucketIndices(v)
print "bucket index =>", bucketIndices[0]
topDown = l.getBucketInfo(bucketIndices)[0]
self.assertTrue(abs(topDown.value - v) <= l.resolution / 2)
self.assertEqual(topDown.value,
l.getBucketValues()[bucketIndices[0]])
self.assertEqual(topDown.scalar, topDown.value)
self.assertTrue((topDown.encoding == output).all())
# Next value
v += l.resolution / 4
def testFillHoles(self):
"""Make sure we can fill in holes"""
l = self._l
decoded = l.decode(
numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertSequenceEqual(ranges[0], [10, 10])
print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded)
decoded = l.decode(
numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertSequenceEqual(ranges[0], [10, 10])
print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded)
def testNonPeriodicEncoderMinMaxNotSpec(self):
"""Non-periodic encoder, min and max not specified"""
l = AdaptiveScalarEncoder(name='scalar',
n=14,
w=5,
minval=None,
maxval=None,
periodic=False)
def _verify(v, encoded, expV=None):
if expV is None:
expV = v
self.assertTrue(
(l.encode(v) == numpy.array(encoded,
dtype=defaultDtype)).all())
self.assertTrue(abs(l.getBucketInfo(l.getBucketIndices(v))[0].value - expV) <= \
l.resolution/2)
def _verifyNot(v, encoded):
self.assertFalse(
(l.encode(v) == numpy.array(encoded,
dtype=defaultDtype)).all())
_verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(2, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(3, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0])
_verify(-9, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-8, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-7, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-6, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-5, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0])
_verify(0, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
_verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0])
_verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(11, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(12, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(13, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(14, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(15, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
#"""Test switching learning off"""
l = AdaptiveScalarEncoder(name='scalar',
n=14,
w=5,
minval=1,
maxval=10,
periodic=False)
_verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(10, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
l.setLearning(False)
_verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], expV=20)
_verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1)
_verify(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1)
l.setLearning(True)
_verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verifyNot(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verifyNot(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
def testSetFieldStats(self):
"""Test setting the min and max using setFieldStats"""
def _dumpParams(enc):
return (enc.n, enc.w, enc.minval, enc.maxval, enc.resolution,
enc.windowSize, enc._learningEnabled, enc.recordNum,
enc.radius, enc.rangeInternal, enc.padding, enc.nInternal)
sfs = AdaptiveScalarEncoder(name='scalar',
n=14,
w=5,
minval=1,
maxval=10,
periodic=False)
reg = AdaptiveScalarEncoder(name='scalar',
n=14,
w=5,
minval=1,
maxval=100,
periodic=False)
self.assertTrue(_dumpParams(sfs) != _dumpParams(reg), "Params should not be equal, "\
"since the two encoders were instantiated with different values.")
# set the min and the max using sFS to 1,100 respectively.
sfs.setFieldStats('this', {"this": {"min": 1, "max": 100}})
#Now the parameters for both should be the same
self.assertEqual(_dumpParams(sfs), _dumpParams(reg), "Params should now be equal, "\
"but they are not. sFS should be equivalent to initialization.")
def setUp(self):
self._l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10,
periodic=False)
class AdaptiveScalarTest(unittest.TestCase):
"""Tests for AdaptiveScalarEncoder"""
def setUp(self):
# forced: it's strongly recommended to use w>=21, in the example we force
# skip the check for readibility
self._l = AdaptiveScalarEncoder(name="scalar",
n=14,
w=5,
minval=1,
maxval=10,
periodic=False,
forced=True)
def testMissingValues(self):
"""missing values"""
# forced: it's strongly recommended to use w>=21, in the example we force
# skip the check for readib.
mv = AdaptiveScalarEncoder(name="mv",
n=14,
w=3,
minval=1,
maxval=8,
periodic=False,
forced=True)
empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
self.assertEqual(empty.sum(), 0)
def testNonPeriodicEncoderMinMaxSpec(self):
"""Non-periodic encoder, min and max specified"""
self.assertTrue(
numpy.array_equal(
self._l.encode(1),
numpy.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(
numpy.array_equal(
self._l.encode(2),
numpy.array([0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(
numpy.array_equal(
self._l.encode(10),
numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
dtype=defaultDtype)))
def testTopDownDecode(self):
"""Test the input description generation and topDown decoding"""
l = self._l
v = l.minval
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertLess(abs(rangeMin - v), l.resolution)
topDown = l.topDownCompute(output)[0]
self.assertLessEqual(abs(topDown.value - v), l.resolution)
# Test bucket support
bucketIndices = l.getBucketIndices(v)
topDown = l.getBucketInfo(bucketIndices)[0]
self.assertLessEqual(abs(topDown.value - v), l.resolution / 2)
self.assertEqual(topDown.value,
l.getBucketValues()[bucketIndices[0]])
self.assertEqual(topDown.scalar, topDown.value)
self.assertTrue(numpy.array_equal(topDown.encoding, output))
# Next value
v += l.resolution / 4
def testFillHoles(self):
"""Make sure we can fill in holes"""
l = self._l
decoded = l.decode(
numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1]))
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertSequenceEqual(ranges[0], [10, 10])
decoded = l.decode(
numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1]))
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertSequenceEqual(ranges[0], [10, 10])
def testNonPeriodicEncoderMinMaxNotSpec(self):
"""Non-periodic encoder, min and max not specified"""
l = AdaptiveScalarEncoder(name="scalar",
n=14,
w=5,
minval=None,
maxval=None,
periodic=False,
forced=True)
def _verify(v, encoded, expV=None):
if expV is None:
expV = v
self.assertTrue(
numpy.array_equal(l.encode(v),
numpy.array(encoded, dtype=defaultDtype)))
self.assertLessEqual(
abs(l.getBucketInfo(l.getBucketIndices(v))[0].value - expV),
l.resolution / 2)
def _verifyNot(v, encoded):
self.assertFalse(
numpy.array_equal(l.encode(v),
numpy.array(encoded, dtype=defaultDtype)))
_verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(2, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(3, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0])
_verify(-9, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-8, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-7, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-6, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(-5, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0])
_verify(0, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
_verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0])
_verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(11, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(12, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(13, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(14, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(15, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
#"""Test switching learning off"""
l = AdaptiveScalarEncoder(name="scalar",
n=14,
w=5,
minval=1,
maxval=10,
periodic=False,
forced=True)
_verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(10, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
l.setLearning(False)
_verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], expV=20)
_verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1)
_verify(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1)
l.setLearning(True)
_verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verifyNot(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
_verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
_verifyNot(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
def testSetFieldStats(self):
"""Test setting the min and max using setFieldStats"""
def _dumpParams(enc):
return (enc.n, enc.w, enc.minval, enc.maxval, enc.resolution,
enc._learningEnabled, enc.recordNum, enc.radius,
enc.rangeInternal, enc.padding, enc.nInternal)
sfs = AdaptiveScalarEncoder(name='scalar',
n=14,
w=5,
minval=1,
maxval=10,
periodic=False,
forced=True)
reg = AdaptiveScalarEncoder(name='scalar',
n=14,
w=5,
minval=1,
maxval=100,
periodic=False,
forced=True)
self.assertNotEqual(
_dumpParams(sfs), _dumpParams(reg),
("Params should not be equal, since the two encoders "
"were instantiated with different values."))
# set the min and the max using sFS to 1,100 respectively.
sfs.setFieldStats("this", {"this": {"min": 1, "max": 100}})
#Now the parameters for both should be the same
self.assertEqual(_dumpParams(sfs), _dumpParams(reg),
("Params should now be equal, but they are not. sFS "
"should be equivalent to initialization."))
@unittest.skipUnless(
capnp, "pycapnp is not installed, skipping serialization test.")
def testReadWrite(self):
originalValue = self._l.encode(1)
proto1 = AdaptiveScalarEncoderProto.new_message()
self._l.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = AdaptiveScalarEncoderProto.read(f)
encoder = AdaptiveScalarEncoder.read(proto2)
self.assertIsInstance(encoder, AdaptiveScalarEncoder)
self.assertEqual(encoder.recordNum, self._l.recordNum)
self.assertDictEqual(encoder.slidingWindow.__dict__,
self._l.slidingWindow.__dict__)
self.assertEqual(encoder.w, self._l.w)
self.assertEqual(encoder.minval, self._l.minval)
self.assertEqual(encoder.maxval, self._l.maxval)
self.assertEqual(encoder.periodic, self._l.periodic)
self.assertEqual(encoder.n, self._l.n)
self.assertEqual(encoder.radius, self._l.radius)
self.assertEqual(encoder.resolution, self._l.resolution)
self.assertEqual(encoder.name, self._l.name)
self.assertEqual(encoder.verbosity, self._l.verbosity)
self.assertEqual(encoder.clipInput, self._l.clipInput)
self.assertTrue(numpy.array_equal(encoder.encode(1), originalValue))
self.assertEqual(self._l.decode(encoder.encode(1)),
encoder.decode(self._l.encode(1)))
# Feed in a new value and ensure the encodings match
result1 = self._l.encode(7)
result2 = encoder.encode(7)
self.assertTrue(numpy.array_equal(result1, result2))
class DeltaEncoder(AdaptiveScalarEncoder):
"""
This is an implementation of a delta encoder. The delta encoder encodes differences between
successive scalar values instead of encoding the actual values. It returns an actual value when
decoding and not a delta.
"""
def __init__(self, w, minval=None, maxval=None, periodic=False, n=0, radius=0,
resolution=0, name=None, verbosity=0, clipInput=True, forced=False):
"""[ScalarEncoder class method override]"""
self._learningEnabled = True
self._stateLock = False
self.width = 0
self.encoders = None
self.description = []
self.name = name
if periodic:
#Delta scalar encoders take non-periodic inputs only
raise Exception('Delta encoder does not encode periodic inputs')
assert n!=0 #An adaptive encoder can only be intialized using n
self._adaptiveScalarEnc = AdaptiveScalarEncoder(w=w, n=n, minval=minval,
maxval=maxval, clipInput=True, name=name, verbosity=verbosity, forced=forced)
self.width+=self._adaptiveScalarEnc.getWidth()
self.n = self._adaptiveScalarEnc.n
self._prevAbsolute = None #how many inputs have been sent to the encoder?
self._prevDelta = None
def encodeIntoArray(self, input, output, learn=None):
if not isinstance(input, numbers.Number):
raise TypeError(
"Expected a scalar input but got input of type {0!s}".format(type(input)))
if learn is None:
learn = self._learningEnabled
if input == SENTINEL_VALUE_FOR_MISSING_DATA:
output[0:self.n] = 0
else:
#make the first delta zero so that the delta ranges are not messed up.
if self._prevAbsoluteisNone:
self._prevAbsolute= input
delta = input - self._prevAbsolute
self._adaptiveScalarEnc.encodeIntoArray(delta, output, learn)
if not self._stateLock:
self._prevAbsolute = input
self._prevDelta = delta
return output
def setStateLock(self, lock):
self._stateLock = lock
def setFieldStats(self, fieldName, fieldStatistics):
pass
def getBucketIndices(self, input, learn=None):
return self._adaptiveScalarEnc.getBucketIndices(input, learn)
def getBucketInfo(self, buckets):
return self._adaptiveScalarEnc.getBucketInfo(buckets)
def topDownCompute(self, encoded):
"""[ScalarEncoder class method override]"""
#Decode to delta scalar
if self._prevAbsoluteisNone or self._prevDeltaisNone:
return [EncoderResult(value=0, scalar=0,
encoding=numpy.zeros(self.n))]
ret = self._adaptiveScalarEnc.topDownCompute(encoded)
if self._prevAbsolute is not None:
ret = [EncoderResult(value=ret[0].value+self._prevAbsolute,
scalar=ret[0].scalar+self._prevAbsolute,
encoding=ret[0].encoding)]
# ret[0].value+=self._prevAbsolute
# ret[0].scalar+=self._prevAbsolute
return ret
@classmethod
def read(cls, proto):
encoder = object.__new__(cls)
encoder.width = proto.width
encoder.name = proto.name or None
encoder.n = proto.n
encoder._adaptiveScalarEnc = (
AdaptiveScalarEncoder.read(proto.adaptiveScalarEnc)
)
encoder._prevAbsolute = proto.prevAbsolute
encoder._prevDelta = proto.prevDelta
encoder._stateLock = proto.stateLock
return encoder
def write(self, proto):
proto.width = self.width
proto.name = self.name or ""
proto.n = self.n
self._adaptiveScalarEnc.write(proto.adaptiveScalarEnc)
proto.prevAbsolute = self._prevAbsolute
proto.prevDelta = self._prevDelta
proto.stateLock = self._stateLock