def testSetFieldStats(self): """Test setting the min and max using setFieldStats""" def _dumpParams(enc): return (enc.n, enc.w, enc.minval, enc.maxval, enc.resolution, enc.windowSize, enc._learningEnabled, enc.recordNum, enc.radius, enc.rangeInternal, enc.padding, enc.nInternal) sfs = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False) reg = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=100, periodic=False) self.assertTrue(_dumpParams(sfs) != _dumpParams(reg), "Params should not be equal, "\ "since the two encoders were instantiated with different values.") # set the min and the max using sFS to 1,100 respectively. sfs.setFieldStats('this', {"this": {"min": 1, "max": 100}}) #Now the parameters for both should be the same self.assertEqual(_dumpParams(sfs), _dumpParams(reg), "Params should now be equal, "\ "but they are not. sFS should be equivalent to initialization.")
def testMissingValues(self): """missing values""" # forced: it's strongly recommended to use w>=21, in the example we force skip the check for readib. mv = AdaptiveScalarEncoder(name='mv', n=14, w=3, minval=1, maxval=8, periodic=False, forced=True) empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA) print "\nEncoded missing data \'None\' as %s" % empty self.assertEqual(empty.sum(), 0)
def setUp(self): self._l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False)
def __init__(self, w, minval=None, maxval=None, periodic=False, n=0, radius=0, resolution=0, name=None, verbosity=0, clipInput=True): """[ScalarEncoder class method override]""" self._learningEnabled = True self._stateLock = False self.width = 0 self.encoders = None self.description = [] self.name = name if periodic: #Delta scalar encoders take non-periodic inputs only raise Exception('Delta encoder does not encode periodic inputs') assert n != 0 #An adaptive encoder can only be intialized using n self._adaptiveScalarEnc = AdaptiveScalarEncoder(w=w, n=n, minval=minval, maxval=maxval, clipInput=True, name=name, verbosity=verbosity) self.width += self._adaptiveScalarEnc.getWidth() self.n = self._adaptiveScalarEnc.n self._prevAbsolute = None #how many inputs have been sent to the encoder? self._prevDelta = None
def setUp(self): # forced: it's strongly recommended to use w>=21, in the example we force # skip the check for readibility self._l = AdaptiveScalarEncoder(name="scalar", n=14, w=5, minval=1, maxval=10, periodic=False, forced=True)
def testMissingValues(self): """missing values""" mv = AdaptiveScalarEncoder(name='mv', n=14, w=3, minval=1, maxval=8, periodic=False) empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA) print "\nEncoded missing data \'None\' as %s" % empty self.assertEqual(empty.sum(), 0)
def testSetFieldStats(self): """Test setting the min and max using setFieldStats""" def _dumpParams(enc): return (enc.n, enc.w, enc.minval, enc.maxval, enc.resolution, enc._learningEnabled, enc.recordNum, enc.radius, enc.rangeInternal, enc.padding, enc.nInternal) sfs = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False, forced=True) reg = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=100, periodic=False, forced=True) self.assertTrue(_dumpParams(sfs) != _dumpParams(reg), "Params should not be equal, "\ "since the two encoders were instantiated with different values.") # set the min and the max using sFS to 1,100 respectively. sfs.setFieldStats('this',{"this":{"min":1,"max":100}}) #Now the parameters for both should be the same self.assertEqual(_dumpParams(sfs), _dumpParams(reg), "Params should now be equal, "\ "but they are not. sFS should be equivalent to initialization.")
def testNonPeriodicEncoderMinMaxNotSpec(self): """Non-periodic encoder, min and max not specified""" l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=None, maxval=None, periodic=False, forced=True) def _verify(v, encoded, expV=None): if expV is None: expV = v self.assertTrue((l.encode(v) == numpy.array(encoded, dtype=defaultDtype)).all()) self.assertTrue(abs(l.getBucketInfo(l.getBucketIndices(v))[0].value - expV) <= \ l.resolution/2) def _verifyNot(v, encoded): self.assertFalse((l.encode(v) == numpy.array(encoded, dtype=defaultDtype)).all()) _verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(2, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(3, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) _verify(-9, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-8, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-7, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-6, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-5, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) _verify(0, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) _verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0]) _verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(11, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(12, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(13, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(14, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(15, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) #"""Test switching learning off""" l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False, forced=True) _verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(10, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) l.setLearning(False) _verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], expV=20) _verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1) _verify(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1) l.setLearning(True) _verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verifyNot(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verifyNot(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
def __new__(self, w, minval=None, maxval=None, periodic=False, n=0, radius=0, resolution=0, name=None, verbosity=0, clipInput=False, space="absolute"): self._encoder = None if space == "absolute": ret = AdaptiveScalarEncoder(w,minval,maxval,periodic,n,radius, resolution,name,verbosity,clipInput) else: ret = DeltaEncoder(w,minval,maxval,periodic,n,radius,resolution,name,verbosity,clipInput) return ret
def read(cls, proto): encoder = object.__new__(cls) encoder.width = proto.width encoder.name = proto.name or None encoder.n = proto.n encoder._adaptiveScalarEnc = (AdaptiveScalarEncoder.read( proto.adaptiveScalarEnc)) encoder._prevAbsolute = proto.prevAbsolute encoder._prevDelta = proto.prevDelta encoder._stateLock = proto.stateLock return encoder
def read(cls, proto): encoder = object.__new__(cls) encoder.width = proto.width encoder.name = proto.name or None encoder.n = proto.n encoder._adaptiveScalarEnc = ( AdaptiveScalarEncoder.read(proto.adaptiveScalarEnc) ) encoder._prevAbsolute = proto.prevAbsolute encoder._prevDelta = proto.prevDelta encoder._stateLock = proto.stateLock return encoder
def testNonPeriodicEncoderMinMaxNotSpec(self): """Non-periodic encoder, min and max not specified""" l = AdaptiveScalarEncoder(name="scalar", n=14, w=5, minval=None, maxval=None, periodic=False, forced=True) def _verify(v, encoded, expV=None): if expV is None: expV = v self.assertTrue(numpy.array_equal( l.encode(v), numpy.array(encoded, dtype=defaultDtype))) self.assertLessEqual( abs(l.getBucketInfo(l.getBucketIndices(v))[0].value - expV), l.resolution/2) def _verifyNot(v, encoded): self.assertFalse(numpy.array_equal( l.encode(v), numpy.array(encoded, dtype=defaultDtype))) _verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(2, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(3, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) _verify(-9, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-8, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-7, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-6, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-5, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) _verify(0, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) _verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0]) _verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(11, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(12, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(13, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(14, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(15, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) #"""Test switching learning off""" l = AdaptiveScalarEncoder(name="scalar", n=14, w=5, minval=1, maxval=10, periodic=False, forced=True) _verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(10, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) l.setLearning(False) _verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], expV=20) _verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1) _verify(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1) l.setLearning(True) _verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verifyNot(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verifyNot(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
def __init__(self, w, minval=None, maxval=None, periodic=False, n=0, radius=0, resolution=0, name=None, verbosity=0, clipInput=True): """[ScalarEncoder class method override]""" self._learningEnabled = True self._stateLock = False self.width = 0 self.encoders = None self.description = [] self.name = name if periodic: #Delta scalar encoders take non-periodic inputs only raise Exception('Delta encoder does not encode periodic inputs') assert n!=0 #An adaptive encoder can only be intialized using n self._adaptiveScalarEnc = AdaptiveScalarEncoder(w=w, n=n, minval=minval, maxval=maxval, clipInput=True, name=name, verbosity=verbosity) self.width+=self._adaptiveScalarEnc.getWidth() self.n = self._adaptiveScalarEnc.n self._prevAbsolute = None #how many inputs have been sent to the encoder? self._prevDelta = None
def testReadWrite(self): originalValue = self._l.encode(1) proto1 = AdaptiveScalarEncoderProto.new_message() self._l.write(proto1) # Write the proto to a temp file and read it back into a new proto with tempfile.TemporaryFile() as f: proto1.write(f) f.seek(0) proto2 = AdaptiveScalarEncoderProto.read(f) encoder = AdaptiveScalarEncoder.read(proto2) self.assertIsInstance(encoder, AdaptiveScalarEncoder) self.assertEqual(encoder.recordNum, self._l.recordNum) self.assertDictEqual(encoder.slidingWindow.__dict__, self._l.slidingWindow.__dict__) self.assertEqual(encoder.w, self._l.w) self.assertEqual(encoder.minval, self._l.minval) self.assertEqual(encoder.maxval, self._l.maxval) self.assertEqual(encoder.periodic, self._l.periodic) self.assertEqual(encoder.n, self._l.n) self.assertEqual(encoder.radius, self._l.radius) self.assertEqual(encoder.resolution, self._l.resolution) self.assertEqual(encoder.name, self._l.name) self.assertEqual(encoder.verbosity, self._l.verbosity) self.assertEqual(encoder.clipInput, self._l.clipInput) self.assertTrue(numpy.array_equal(encoder.encode(1), originalValue)) self.assertEqual(self._l.decode(encoder.encode(1)), encoder.decode(self._l.encode(1))) # Feed in a new value and ensure the encodings match result1 = self._l.encode(7) result2 = encoder.encode(7) self.assertTrue(numpy.array_equal(result1, result2))
def testReadWrite(self): originalValue = self._l.encode(1) proto1 = AdaptiveScalarEncoderProto.new_message() self._l.write(proto1) # Write the proto to a temp file and read it back into a new proto with tempfile.TemporaryFile() as f: proto1.write(f) f.seek(0) proto2 = AdaptiveScalarEncoderProto.read(f) encoder = AdaptiveScalarEncoder.read(proto2) self.assertIsInstance(encoder, AdaptiveScalarEncoder) self.assertEqual(encoder.recordNum, self._l.recordNum) self.assertDictEqual(encoder.slidingWindow.__dict__, self._l.slidingWindow.__dict__) self.assertEqual(encoder.w, self._l.w) self.assertEqual(encoder.minval, self._l.minval) self.assertEqual(encoder.maxval, self._l.maxval) self.assertEqual(encoder.periodic, self._l.periodic) self.assertEqual(encoder.n, self._l.n) self.assertEqual(encoder.radius, self._l.radius) self.assertEqual(encoder.resolution, self._l.resolution) self.assertEqual(encoder.name, self._l.name) self.assertEqual(encoder.verbosity, self._l.verbosity) self.assertEqual(encoder.clipInput, self._l.clipInput) self.assertTrue(numpy.array_equal(encoder.encode(1), originalValue)) self.assertEqual(self._l.decode(encoder.encode(1)), encoder.decode(self._l.encode(1))) # Feed in a new value and ensure the encodings match result1 = self._l.encode(7) result2 = encoder.encode(7) self.assertTrue(numpy.array_equal(result1, result2))
def testMissingValues(self): """missing values""" mv = AdaptiveScalarEncoder(name='mv', n=14, w=3, minval=1, maxval=8, periodic=False) empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA) print "\nEncoded missing data \'None\' as %s" % empty self.assertEqual(empty.sum(), 0)
class DeltaEncoder(AdaptiveScalarEncoder): """ This is an implementation of a delta encoder. The delta encoder encodes differences between # to_note: so basically different input values can have successive scalar values instead of encoding the actual values. It returns an actual value when # the same representation. The only value that matters decoding and not a delta. # is the difference between the current input and the last input """ # problem_with_this_approach: the fact that it uses adaptive scalar encoder makes learning highly improbable, since with each # new maximum or minimum value, the whole encoding scheme changes, and this change in the encoded representation can easily # mess whatever the machine has learned so far. Worse, the machine will not even recognize why it is wrong. The developers # are really going over and abusing the knowledge that the brain can handle various kind of data. Yes, the brain can handle new # encoded representation, however it takes a lot of time, and the change in encoded representation is not this arbitrary. With # this kind of arbirtrary (encoding scheme changes merely whenever new max or min input is presented), even the brain's learning # algorithm will be messed up. def __init__(self, w, minval=None, maxval=None, periodic=False, n=0, radius=0, resolution=0, name=None, verbosity=0, clipInput=True, forced=False): """[ScalarEncoder class method override]""" self._learningEnabled = True self._stateLock = False self.width = 0 self.encoders = None self.description = [] self.name = name if periodic: #Delta scalar encoders take non-periodic inputs only raise Exception('Delta encoder does not encode periodic inputs') assert n!=0 #An adaptive encoder can only be intialized using n self._adaptiveScalarEnc = AdaptiveScalarEncoder(w=w, n=n, minval=minval, maxval=maxval, clipInput=True, name=name, verbosity=verbosity, forced=forced) self.width+=self._adaptiveScalarEnc.getWidth() self.n = self._adaptiveScalarEnc.n self._prevAbsolute = None #how many inputs have been sent to the encoder? self._prevDelta = None def encodeIntoArray(self, input, output, learn=None): if not isinstance(input, numbers.Number): raise TypeError( "Expected a scalar input but got input of type %s" % type(input)) if learn is None: learn = self._learningEnabled if input == SENTINEL_VALUE_FOR_MISSING_DATA: output[0:self.n] = 0 else: #make the first delta zero so that the delta ranges are not messed up. if self._prevAbsolute==None: self._prevAbsolute= input delta = input - self._prevAbsolute self._adaptiveScalarEnc.encodeIntoArray(delta, output, learn) # to_note: generate a representation for the difference between the current if not self._stateLock: # input and the last input. self._prevAbsolute = input self._prevDelta = delta return output ############################################################################ def setStateLock(self, lock): self._stateLock = lock ############################################################################ def setFieldStats(self, fieldName, fieldStatistics): pass ############################################################################ def getBucketIndices(self, input, learn=None): return self._adaptiveScalarEnc.getBucketIndices(input, learn) ############################################################################ def getBucketInfo(self, buckets): return self._adaptiveScalarEnc.getBucketInfo(buckets) ############################################################################ def topDownCompute(self, encoded): """[ScalarEncoder class method override]""" #Decode to delta scalar if self._prevAbsolute==None or self._prevDelta==None: return [EncoderResult(value=0, scalar=0, encoding=numpy.zeros(self.n))] ret = self._adaptiveScalarEnc.topDownCompute(encoded) if self._prevAbsolute != None: ret = [EncoderResult(value=ret[0].value+self._prevAbsolute, scalar=ret[0].scalar+self._prevAbsolute, encoding=ret[0].encoding)] # problem_with_this_approach: encoded houses the value of delta, so it should # ret[0].value+=self._prevAbsolute # the decoding scheme in topDownCompute will generate delta. If we add that delta # ret[0].scalar+=self._prevAbsolute # to the previous absolute scalar, we will get a completely useless result return ret @classmethod def read(cls, proto): encoder = object.__new__(cls) encoder.width = proto.width encoder.name = proto.name or None encoder.n = proto.n encoder._adaptiveScalarEnc = ( AdaptiveScalarEncoder.read(proto.adaptiveScalarEnc) ) encoder._prevAbsolute = proto.prevAbsolute encoder._prevDelta = proto.prevDelta encoder._stateLock = proto.stateLock return encoder def write(self, proto): proto.width = self.width proto.name = self.name or "" proto.n = self.n self._adaptiveScalarEnc.write(proto.adaptiveScalarEnc) proto.prevAbsolute = self._prevAbsolute proto.prevDelta = self._prevDelta proto.stateLock = self._stateLock
class DeltaEncoder(AdaptiveScalarEncoder): """ This is an implementation of a delta encoder. The delta encoder encodes differences between successive scalar values instead of encoding the actual values. It returns an actual value when decoding and not a delta. """ def __init__(self, w, minval=None, maxval=None, periodic=False, n=0, radius=0, resolution=0, name=None, verbosity=0, clipInput=True): """[ScalarEncoder class method override]""" self._learningEnabled = True self._stateLock = False self.width = 0 self.encoders = None self.description = [] self.name = name if periodic: #Delta scalar encoders take non-periodic inputs only raise Exception('Delta encoder does not encode periodic inputs') assert n != 0 #An adaptive encoder can only be intialized using n self._adaptiveScalarEnc = AdaptiveScalarEncoder(w=w, n=n, minval=minval, maxval=maxval, clipInput=True, name=name, verbosity=verbosity) self.width += self._adaptiveScalarEnc.getWidth() self.n = self._adaptiveScalarEnc.n self._prevAbsolute = None #how many inputs have been sent to the encoder? self._prevDelta = None def encodeIntoArray(self, input, output, learn=None): if learn is None: learn = self._learningEnabled if input == SENTINEL_VALUE_FOR_MISSING_DATA: output[0:self.n] = 0 else: #make the first delta zero so that the delta ranges are not messed up. if self._prevAbsolute == None: self._prevAbsolute = input delta = input - self._prevAbsolute self._adaptiveScalarEnc.encodeIntoArray(delta, output, learn) if not self._stateLock: self._prevAbsolute = input self._prevDelta = delta return output ############################################################################ def setStateLock(self, lock): self._stateLock = lock ############################################################################ def setFieldStats(self, fieldName, fieldStatistics): pass ############################################################################ def isDelta(self): return True ############################################################################ def getBucketIndices(self, input, learn=None): return self._adaptiveScalarEnc.getBucketIndices(input, learn) ############################################################################ def getBucketInfo(self, buckets): return self._adaptiveScalarEnc.getBucketInfo(buckets) ############################################################################ def topDownCompute(self, encoded): """[ScalarEncoder class method override]""" #Decode to delta scalar if self._prevAbsolute == None or self._prevDelta == None: return [ EncoderResult(value=0, scalar=0, encoding=numpy.zeros(self.n)) ] ret = self._adaptiveScalarEnc.topDownCompute(encoded) if self._prevAbsolute != None: ret = [ EncoderResult(value=ret[0].value + self._prevAbsolute, scalar=ret[0].scalar + self._prevAbsolute, encoding=ret[0].encoding) ] # ret[0].value+=self._prevAbsolute # ret[0].scalar+=self._prevAbsolute return ret
class AdaptiveScalarTest(unittest.TestCase): """Tests for AdaptiveScalarEncoder""" def setUp(self): # forced: it's strongly recommended to use w>=21, in the example we force # skip the check for readibility self._l = AdaptiveScalarEncoder(name="scalar", n=14, w=5, minval=1, maxval=10, periodic=False, forced=True) def testMissingValues(self): """missing values""" # forced: it's strongly recommended to use w>=21, in the example we force # skip the check for readib. mv = AdaptiveScalarEncoder(name="mv", n=14, w=3, minval=1, maxval=8, periodic=False, forced=True) empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA) self.assertEqual(empty.sum(), 0) def testNonPeriodicEncoderMinMaxSpec(self): """Non-periodic encoder, min and max specified""" self.assertTrue(numpy.array_equal( self._l.encode(1), numpy.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], dtype=defaultDtype))) self.assertTrue(numpy.array_equal( self._l.encode(2), numpy.array([0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0], dtype=defaultDtype))) self.assertTrue(numpy.array_equal( self._l.encode(10), numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], dtype=defaultDtype))) def testTopDownDecode(self): """Test the input description generation and topDown decoding""" l = self._l v = l.minval while v < l.maxval: output = l.encode(v) decoded = l.decode(output) (fieldsDict, _) = decoded self.assertEqual(len(fieldsDict), 1) (ranges, _) = fieldsDict.values()[0] self.assertEqual(len(ranges), 1) (rangeMin, rangeMax) = ranges[0] self.assertEqual(rangeMin, rangeMax) self.assertLess(abs(rangeMin - v), l.resolution) topDown = l.topDownCompute(output)[0] self.assertLessEqual(abs(topDown.value - v), l.resolution) # Test bucket support bucketIndices = l.getBucketIndices(v) topDown = l.getBucketInfo(bucketIndices)[0] self.assertLessEqual(abs(topDown.value - v), l.resolution / 2) self.assertEqual(topDown.value, l.getBucketValues()[bucketIndices[0]]) self.assertEqual(topDown.scalar, topDown.value) self.assertTrue(numpy.array_equal(topDown.encoding, output)) # Next value v += l.resolution / 4 def testFillHoles(self): """Make sure we can fill in holes""" l=self._l decoded = l.decode(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1])) (fieldsDict, _) = decoded self.assertEqual(len(fieldsDict), 1) (ranges, _) = fieldsDict.values()[0] self.assertEqual(len(ranges), 1) self.assertSequenceEqual(ranges[0], [10, 10]) decoded = l.decode(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1])) (fieldsDict, _) = decoded self.assertEqual(len(fieldsDict), 1) (ranges, _) = fieldsDict.values()[0] self.assertEqual(len(ranges), 1) self.assertSequenceEqual(ranges[0], [10, 10]) def testNonPeriodicEncoderMinMaxNotSpec(self): """Non-periodic encoder, min and max not specified""" l = AdaptiveScalarEncoder(name="scalar", n=14, w=5, minval=None, maxval=None, periodic=False, forced=True) def _verify(v, encoded, expV=None): if expV is None: expV = v self.assertTrue(numpy.array_equal( l.encode(v), numpy.array(encoded, dtype=defaultDtype))) self.assertLessEqual( abs(l.getBucketInfo(l.getBucketIndices(v))[0].value - expV), l.resolution/2) def _verifyNot(v, encoded): self.assertFalse(numpy.array_equal( l.encode(v), numpy.array(encoded, dtype=defaultDtype))) _verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(2, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(3, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) _verify(-9, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-8, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-7, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-6, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-5, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) _verify(0, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) _verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0]) _verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(11, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(12, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(13, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(14, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(15, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) #"""Test switching learning off""" l = AdaptiveScalarEncoder(name="scalar", n=14, w=5, minval=1, maxval=10, periodic=False, forced=True) _verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(10, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) l.setLearning(False) _verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], expV=20) _verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1) _verify(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1) l.setLearning(True) _verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verifyNot(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verifyNot(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) def testSetFieldStats(self): """Test setting the min and max using setFieldStats""" def _dumpParams(enc): return (enc.n, enc.w, enc.minval, enc.maxval, enc.resolution, enc._learningEnabled, enc.recordNum, enc.radius, enc.rangeInternal, enc.padding, enc.nInternal) sfs = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False, forced=True) reg = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=100, periodic=False, forced=True) self.assertNotEqual(_dumpParams(sfs), _dumpParams(reg), ("Params should not be equal, since the two encoders " "were instantiated with different values.")) # set the min and the max using sFS to 1,100 respectively. sfs.setFieldStats("this", {"this":{"min":1, "max":100}}) #Now the parameters for both should be the same self.assertEqual(_dumpParams(sfs), _dumpParams(reg), ("Params should now be equal, but they are not. sFS " "should be equivalent to initialization.")) def testReadWrite(self): originalValue = self._l.encode(1) proto1 = AdaptiveScalarEncoderProto.new_message() self._l.write(proto1) # Write the proto to a temp file and read it back into a new proto with tempfile.TemporaryFile() as f: proto1.write(f) f.seek(0) proto2 = AdaptiveScalarEncoderProto.read(f) encoder = AdaptiveScalarEncoder.read(proto2) self.assertIsInstance(encoder, AdaptiveScalarEncoder) self.assertEqual(encoder.recordNum, self._l.recordNum) self.assertDictEqual(encoder.slidingWindow.__dict__, self._l.slidingWindow.__dict__) self.assertEqual(encoder.w, self._l.w) self.assertEqual(encoder.minval, self._l.minval) self.assertEqual(encoder.maxval, self._l.maxval) self.assertEqual(encoder.periodic, self._l.periodic) self.assertEqual(encoder.n, self._l.n) self.assertEqual(encoder.radius, self._l.radius) self.assertEqual(encoder.resolution, self._l.resolution) self.assertEqual(encoder.name, self._l.name) self.assertEqual(encoder.verbosity, self._l.verbosity) self.assertEqual(encoder.clipInput, self._l.clipInput) self.assertTrue(numpy.array_equal(encoder.encode(1), originalValue)) self.assertEqual(self._l.decode(encoder.encode(1)), encoder.decode(self._l.encode(1))) # Feed in a new value and ensure the encodings match result1 = self._l.encode(7) result2 = encoder.encode(7) self.assertTrue(numpy.array_equal(result1, result2))
class AdaptiveScalarTest(unittest.TestCase): """Tests for AdaptiveScalarEncoder""" def setUp(self): # forced: it's strongly recommended to use w>=21, in the example we force skip the check for readibility self._l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False, forced=True) def testMissingValues(self): """missing values""" # forced: it's strongly recommended to use w>=21, in the example we force skip the check for readib. mv = AdaptiveScalarEncoder(name='mv', n=14, w=3, minval=1, maxval=8, periodic=False, forced=True) empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA) print "\nEncoded missing data \'None\' as %s" % empty self.assertEqual(empty.sum(), 0) def testNonPeriodicEncoderMinMaxSpec(self): """Non-periodic encoder, min and max specified""" self.assertTrue((self._l.encode(1) == numpy.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], dtype=defaultDtype)).all()) self.assertTrue((self._l.encode(2) == numpy.array([0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0], dtype=defaultDtype)).all()) self.assertTrue((self._l.encode(10) == numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], dtype=defaultDtype)).all()) def testTopDownDecode(self): """Test the input description generation and topDown decoding""" l=self._l v = l.minval print "\nTesting non-periodic encoder decoding, resolution of %f..." % \ l.resolution while v < l.maxval: output = l.encode(v) decoded = l.decode(output) print "decoding", output, "(%f)=>" % v, l.decodedToStr(decoded) (fieldsDict, fieldNames) = decoded self.assertEqual(len(fieldsDict), 1) (ranges, desc) = fieldsDict.values()[0] self.assertEqual(len(ranges), 1) (rangeMin, rangeMax) = ranges[0] self.assertEqual(rangeMin, rangeMax) self.assertTrue(abs(rangeMin - v) < l.resolution) topDown = l.topDownCompute(output)[0] print "topdown =>", topDown self.assertTrue(abs(topDown.value - v) <= l.resolution) # Test bucket support bucketIndices = l.getBucketIndices(v) print "bucket index =>", bucketIndices[0] topDown = l.getBucketInfo(bucketIndices)[0] self.assertTrue(abs(topDown.value - v) <= l.resolution / 2) self.assertEqual(topDown.value, l.getBucketValues()[bucketIndices[0]]) self.assertEqual(topDown.scalar, topDown.value) self.assertTrue((topDown.encoding == output).all()) # Next value v += l.resolution / 4 def testFillHoles(self): """Make sure we can fill in holes""" l=self._l decoded = l.decode(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1])) (fieldsDict, fieldNames) = decoded self.assertEqual(len(fieldsDict), 1) (ranges, desc) = fieldsDict.values()[0] self.assertEqual(len(ranges), 1) self.assertSequenceEqual(ranges[0], [10, 10]) print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded) decoded = l.decode(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1])) (fieldsDict, fieldNames) = decoded self.assertEqual(len(fieldsDict), 1) (ranges, desc) = fieldsDict.values()[0] self.assertEqual(len(ranges), 1) self.assertSequenceEqual(ranges[0], [10, 10]) print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded) def testNonPeriodicEncoderMinMaxNotSpec(self): """Non-periodic encoder, min and max not specified""" l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=None, maxval=None, periodic=False, forced=True) def _verify(v, encoded, expV=None): if expV is None: expV = v self.assertTrue((l.encode(v) == numpy.array(encoded, dtype=defaultDtype)).all()) self.assertTrue(abs(l.getBucketInfo(l.getBucketIndices(v))[0].value - expV) <= \ l.resolution/2) def _verifyNot(v, encoded): self.assertFalse((l.encode(v) == numpy.array(encoded, dtype=defaultDtype)).all()) _verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(2, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(3, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) _verify(-9, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-8, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-7, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-6, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-5, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) _verify(0, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) _verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0]) _verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(11, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(12, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(13, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(14, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(15, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) #"""Test switching learning off""" l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False, forced=True) _verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(10, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) l.setLearning(False) _verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], expV=20) _verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1) _verify(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1) l.setLearning(True) _verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verifyNot(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verifyNot(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) def testSetFieldStats(self): """Test setting the min and max using setFieldStats""" def _dumpParams(enc): return (enc.n, enc.w, enc.minval, enc.maxval, enc.resolution, enc._learningEnabled, enc.recordNum, enc.radius, enc.rangeInternal, enc.padding, enc.nInternal) sfs = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False, forced=True) reg = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=100, periodic=False, forced=True) self.assertTrue(_dumpParams(sfs) != _dumpParams(reg), "Params should not be equal, "\ "since the two encoders were instantiated with different values.") # set the min and the max using sFS to 1,100 respectively. sfs.setFieldStats('this',{"this":{"min":1,"max":100}}) #Now the parameters for both should be the same self.assertEqual(_dumpParams(sfs), _dumpParams(reg), "Params should now be equal, "\ "but they are not. sFS should be equivalent to initialization.")
def setUp(self): # forced: it's strongly recommended to use w>=21, in the example we force skip the check for readibility self._l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False, forced=True)
class DeltaEncoder(AdaptiveScalarEncoder): """ This is an implementation of a delta encoder. The delta encoder encodes differences between successive scalar values instead of encoding the actual values. It returns an actual value when decoding and not a delta. """ def __init__(self, w, minval=None, maxval=None, periodic=False, n=0, radius=0, resolution=0, name=None, verbosity=0, clipInput=True): """[ScalarEncoder class method override]""" self._learningEnabled = True self._stateLock = False self.width = 0 self.encoders = None self.description = [] self.name = name if periodic: #Delta scalar encoders take non-periodic inputs only raise Exception('Delta encoder does not encode periodic inputs') assert n!=0 #An adaptive encoder can only be intialized using n self._adaptiveScalarEnc = AdaptiveScalarEncoder(w=w, n=n, minval=minval, maxval=maxval, clipInput=True, name=name, verbosity=verbosity) self.width+=self._adaptiveScalarEnc.getWidth() self.n = self._adaptiveScalarEnc.n self._prevAbsolute = None #how many inputs have been sent to the encoder? self._prevDelta = None def encodeIntoArray(self, input, output, learn=None): if learn is None: learn = self._learningEnabled if input == SENTINEL_VALUE_FOR_MISSING_DATA: output[0:self.n] = 0 else: #make the first delta zero so that the delta ranges are not messed up. if self._prevAbsolute==None: self._prevAbsolute= input delta = input - self._prevAbsolute self._adaptiveScalarEnc.encodeIntoArray(delta, output, learn) if not self._stateLock: self._prevAbsolute = input self._prevDelta = delta return output ############################################################################ def setStateLock(self, lock): self._stateLock = lock ############################################################################ def setFieldStats(self, fieldName, fieldStatistics): pass ############################################################################ def isDelta(self): return True ############################################################################ def getBucketIndices(self, input, learn=None): return self._adaptiveScalarEnc.getBucketIndices(input, learn) ############################################################################ def getBucketInfo(self, buckets): return self._adaptiveScalarEnc.getBucketInfo(buckets) ############################################################################ def topDownCompute(self, encoded): """[ScalarEncoder class method override]""" #Decode to delta scalar if self._prevAbsolute==None or self._prevDelta==None: return [EncoderResult(value=0, scalar=0, encoding=numpy.zeros(self.n))] ret = self._adaptiveScalarEnc.topDownCompute(encoded) if self._prevAbsolute != None: ret = [EncoderResult(value=ret[0].value+self._prevAbsolute, scalar=ret[0].scalar+self._prevAbsolute, encoding=ret[0].encoding)] # ret[0].value+=self._prevAbsolute # ret[0].scalar+=self._prevAbsolute return ret
class AdaptiveScalarTest(unittest.TestCase): """Tests for AdaptiveScalarEncoder""" def setUp(self): self._l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False) def testMissingValues(self): """missing values""" mv = AdaptiveScalarEncoder(name='mv', n=14, w=3, minval=1, maxval=8, periodic=False) empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA) print "\nEncoded missing data \'None\' as %s" % empty self.assertEqual(empty.sum(), 0) def testNonPeriodicEncoderMinMaxSpec(self): """Non-periodic encoder, min and max specified""" self.assertTrue((self._l.encode(1) == numpy.array( [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], dtype=defaultDtype)).all()) self.assertTrue((self._l.encode(2) == numpy.array( [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0], dtype=defaultDtype)).all()) self.assertTrue((self._l.encode(10) == numpy.array( [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], dtype=defaultDtype)).all()) def testTopDownDecode(self): """Test the input description generation and topDown decoding""" l = self._l v = l.minval print "\nTesting non-periodic encoder decoding, resolution of %f..." % \ l.resolution while v < l.maxval: output = l.encode(v) decoded = l.decode(output) print "decoding", output, "(%f)=>" % v, l.decodedToStr(decoded) (fieldsDict, fieldNames) = decoded self.assertEqual(len(fieldsDict), 1) (ranges, desc) = fieldsDict.values()[0] self.assertEqual(len(ranges), 1) (rangeMin, rangeMax) = ranges[0] self.assertEqual(rangeMin, rangeMax) self.assertTrue(abs(rangeMin - v) < l.resolution) topDown = l.topDownCompute(output)[0] print "topdown =>", topDown self.assertTrue(abs(topDown.value - v) <= l.resolution) # Test bucket support bucketIndices = l.getBucketIndices(v) print "bucket index =>", bucketIndices[0] topDown = l.getBucketInfo(bucketIndices)[0] self.assertTrue(abs(topDown.value - v) <= l.resolution / 2) self.assertEqual(topDown.value, l.getBucketValues()[bucketIndices[0]]) self.assertEqual(topDown.scalar, topDown.value) self.assertTrue((topDown.encoding == output).all()) # Next value v += l.resolution / 4 def testFillHoles(self): """Make sure we can fill in holes""" l = self._l decoded = l.decode( numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1])) (fieldsDict, fieldNames) = decoded self.assertEqual(len(fieldsDict), 1) (ranges, desc) = fieldsDict.values()[0] self.assertEqual(len(ranges), 1) self.assertSequenceEqual(ranges[0], [10, 10]) print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded) decoded = l.decode( numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1])) (fieldsDict, fieldNames) = decoded self.assertEqual(len(fieldsDict), 1) (ranges, desc) = fieldsDict.values()[0] self.assertEqual(len(ranges), 1) self.assertSequenceEqual(ranges[0], [10, 10]) print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded) def testNonPeriodicEncoderMinMaxNotSpec(self): """Non-periodic encoder, min and max not specified""" l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=None, maxval=None, periodic=False) def _verify(v, encoded, expV=None): if expV is None: expV = v self.assertTrue( (l.encode(v) == numpy.array(encoded, dtype=defaultDtype)).all()) self.assertTrue(abs(l.getBucketInfo(l.getBucketIndices(v))[0].value - expV) <= \ l.resolution/2) def _verifyNot(v, encoded): self.assertFalse( (l.encode(v) == numpy.array(encoded, dtype=defaultDtype)).all()) _verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(2, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(3, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) _verify(-9, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-8, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-7, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-6, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-5, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) _verify(0, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) _verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0]) _verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(11, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(12, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(13, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(14, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(15, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) #"""Test switching learning off""" l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False) _verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(10, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) l.setLearning(False) _verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], expV=20) _verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1) _verify(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1) l.setLearning(True) _verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verifyNot(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verifyNot(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) def testSetFieldStats(self): """Test setting the min and max using setFieldStats""" def _dumpParams(enc): return (enc.n, enc.w, enc.minval, enc.maxval, enc.resolution, enc.windowSize, enc._learningEnabled, enc.recordNum, enc.radius, enc.rangeInternal, enc.padding, enc.nInternal) sfs = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False) reg = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=100, periodic=False) self.assertTrue(_dumpParams(sfs) != _dumpParams(reg), "Params should not be equal, "\ "since the two encoders were instantiated with different values.") # set the min and the max using sFS to 1,100 respectively. sfs.setFieldStats('this', {"this": {"min": 1, "max": 100}}) #Now the parameters for both should be the same self.assertEqual(_dumpParams(sfs), _dumpParams(reg), "Params should now be equal, "\ "but they are not. sFS should be equivalent to initialization.")
def setUp(self): self._l = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False)
class AdaptiveScalarTest(unittest.TestCase): """Tests for AdaptiveScalarEncoder""" def setUp(self): # forced: it's strongly recommended to use w>=21, in the example we force # skip the check for readibility self._l = AdaptiveScalarEncoder(name="scalar", n=14, w=5, minval=1, maxval=10, periodic=False, forced=True) def testMissingValues(self): """missing values""" # forced: it's strongly recommended to use w>=21, in the example we force # skip the check for readib. mv = AdaptiveScalarEncoder(name="mv", n=14, w=3, minval=1, maxval=8, periodic=False, forced=True) empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA) self.assertEqual(empty.sum(), 0) def testNonPeriodicEncoderMinMaxSpec(self): """Non-periodic encoder, min and max specified""" self.assertTrue( numpy.array_equal( self._l.encode(1), numpy.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], dtype=defaultDtype))) self.assertTrue( numpy.array_equal( self._l.encode(2), numpy.array([0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0], dtype=defaultDtype))) self.assertTrue( numpy.array_equal( self._l.encode(10), numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], dtype=defaultDtype))) def testTopDownDecode(self): """Test the input description generation and topDown decoding""" l = self._l v = l.minval while v < l.maxval: output = l.encode(v) decoded = l.decode(output) (fieldsDict, _) = decoded self.assertEqual(len(fieldsDict), 1) (ranges, _) = fieldsDict.values()[0] self.assertEqual(len(ranges), 1) (rangeMin, rangeMax) = ranges[0] self.assertEqual(rangeMin, rangeMax) self.assertLess(abs(rangeMin - v), l.resolution) topDown = l.topDownCompute(output)[0] self.assertLessEqual(abs(topDown.value - v), l.resolution) # Test bucket support bucketIndices = l.getBucketIndices(v) topDown = l.getBucketInfo(bucketIndices)[0] self.assertLessEqual(abs(topDown.value - v), l.resolution / 2) self.assertEqual(topDown.value, l.getBucketValues()[bucketIndices[0]]) self.assertEqual(topDown.scalar, topDown.value) self.assertTrue(numpy.array_equal(topDown.encoding, output)) # Next value v += l.resolution / 4 def testFillHoles(self): """Make sure we can fill in holes""" l = self._l decoded = l.decode( numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1])) (fieldsDict, _) = decoded self.assertEqual(len(fieldsDict), 1) (ranges, _) = fieldsDict.values()[0] self.assertEqual(len(ranges), 1) self.assertSequenceEqual(ranges[0], [10, 10]) decoded = l.decode( numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1])) (fieldsDict, _) = decoded self.assertEqual(len(fieldsDict), 1) (ranges, _) = fieldsDict.values()[0] self.assertEqual(len(ranges), 1) self.assertSequenceEqual(ranges[0], [10, 10]) def testNonPeriodicEncoderMinMaxNotSpec(self): """Non-periodic encoder, min and max not specified""" l = AdaptiveScalarEncoder(name="scalar", n=14, w=5, minval=None, maxval=None, periodic=False, forced=True) def _verify(v, encoded, expV=None): if expV is None: expV = v self.assertTrue( numpy.array_equal(l.encode(v), numpy.array(encoded, dtype=defaultDtype))) self.assertLessEqual( abs(l.getBucketInfo(l.getBucketIndices(v))[0].value - expV), l.resolution / 2) def _verifyNot(v, encoded): self.assertFalse( numpy.array_equal(l.encode(v), numpy.array(encoded, dtype=defaultDtype))) _verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(2, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(3, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) _verify(-9, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-8, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-7, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-6, [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(-5, [0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) _verify(0, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) _verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0]) _verify(8, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(11, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(12, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(13, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(14, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(15, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) #"""Test switching learning off""" l = AdaptiveScalarEncoder(name="scalar", n=14, w=5, minval=1, maxval=10, periodic=False, forced=True) _verify(1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verify(10, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(10, [0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) l.setLearning(False) _verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], expV=20) _verify(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1) _verify(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], expV=1) l.setLearning(True) _verify(30, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verifyNot(20, [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) _verify(-10, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) _verifyNot(-1, [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]) def testSetFieldStats(self): """Test setting the min and max using setFieldStats""" def _dumpParams(enc): return (enc.n, enc.w, enc.minval, enc.maxval, enc.resolution, enc._learningEnabled, enc.recordNum, enc.radius, enc.rangeInternal, enc.padding, enc.nInternal) sfs = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False, forced=True) reg = AdaptiveScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=100, periodic=False, forced=True) self.assertNotEqual( _dumpParams(sfs), _dumpParams(reg), ("Params should not be equal, since the two encoders " "were instantiated with different values.")) # set the min and the max using sFS to 1,100 respectively. sfs.setFieldStats("this", {"this": {"min": 1, "max": 100}}) #Now the parameters for both should be the same self.assertEqual(_dumpParams(sfs), _dumpParams(reg), ("Params should now be equal, but they are not. sFS " "should be equivalent to initialization.")) @unittest.skipUnless( capnp, "pycapnp is not installed, skipping serialization test.") def testReadWrite(self): originalValue = self._l.encode(1) proto1 = AdaptiveScalarEncoderProto.new_message() self._l.write(proto1) # Write the proto to a temp file and read it back into a new proto with tempfile.TemporaryFile() as f: proto1.write(f) f.seek(0) proto2 = AdaptiveScalarEncoderProto.read(f) encoder = AdaptiveScalarEncoder.read(proto2) self.assertIsInstance(encoder, AdaptiveScalarEncoder) self.assertEqual(encoder.recordNum, self._l.recordNum) self.assertDictEqual(encoder.slidingWindow.__dict__, self._l.slidingWindow.__dict__) self.assertEqual(encoder.w, self._l.w) self.assertEqual(encoder.minval, self._l.minval) self.assertEqual(encoder.maxval, self._l.maxval) self.assertEqual(encoder.periodic, self._l.periodic) self.assertEqual(encoder.n, self._l.n) self.assertEqual(encoder.radius, self._l.radius) self.assertEqual(encoder.resolution, self._l.resolution) self.assertEqual(encoder.name, self._l.name) self.assertEqual(encoder.verbosity, self._l.verbosity) self.assertEqual(encoder.clipInput, self._l.clipInput) self.assertTrue(numpy.array_equal(encoder.encode(1), originalValue)) self.assertEqual(self._l.decode(encoder.encode(1)), encoder.decode(self._l.encode(1))) # Feed in a new value and ensure the encodings match result1 = self._l.encode(7) result2 = encoder.encode(7) self.assertTrue(numpy.array_equal(result1, result2))
class DeltaEncoder(AdaptiveScalarEncoder): """ This is an implementation of a delta encoder. The delta encoder encodes differences between successive scalar values instead of encoding the actual values. It returns an actual value when decoding and not a delta. """ def __init__(self, w, minval=None, maxval=None, periodic=False, n=0, radius=0, resolution=0, name=None, verbosity=0, clipInput=True, forced=False): """[ScalarEncoder class method override]""" self._learningEnabled = True self._stateLock = False self.width = 0 self.encoders = None self.description = [] self.name = name if periodic: #Delta scalar encoders take non-periodic inputs only raise Exception('Delta encoder does not encode periodic inputs') assert n!=0 #An adaptive encoder can only be intialized using n self._adaptiveScalarEnc = AdaptiveScalarEncoder(w=w, n=n, minval=minval, maxval=maxval, clipInput=True, name=name, verbosity=verbosity, forced=forced) self.width+=self._adaptiveScalarEnc.getWidth() self.n = self._adaptiveScalarEnc.n self._prevAbsolute = None #how many inputs have been sent to the encoder? self._prevDelta = None def encodeIntoArray(self, input, output, learn=None): if not isinstance(input, numbers.Number): raise TypeError( "Expected a scalar input but got input of type {0!s}".format(type(input))) if learn is None: learn = self._learningEnabled if input == SENTINEL_VALUE_FOR_MISSING_DATA: output[0:self.n] = 0 else: #make the first delta zero so that the delta ranges are not messed up. if self._prevAbsoluteisNone: self._prevAbsolute= input delta = input - self._prevAbsolute self._adaptiveScalarEnc.encodeIntoArray(delta, output, learn) if not self._stateLock: self._prevAbsolute = input self._prevDelta = delta return output def setStateLock(self, lock): self._stateLock = lock def setFieldStats(self, fieldName, fieldStatistics): pass def getBucketIndices(self, input, learn=None): return self._adaptiveScalarEnc.getBucketIndices(input, learn) def getBucketInfo(self, buckets): return self._adaptiveScalarEnc.getBucketInfo(buckets) def topDownCompute(self, encoded): """[ScalarEncoder class method override]""" #Decode to delta scalar if self._prevAbsoluteisNone or self._prevDeltaisNone: return [EncoderResult(value=0, scalar=0, encoding=numpy.zeros(self.n))] ret = self._adaptiveScalarEnc.topDownCompute(encoded) if self._prevAbsolute is not None: ret = [EncoderResult(value=ret[0].value+self._prevAbsolute, scalar=ret[0].scalar+self._prevAbsolute, encoding=ret[0].encoding)] # ret[0].value+=self._prevAbsolute # ret[0].scalar+=self._prevAbsolute return ret @classmethod def read(cls, proto): encoder = object.__new__(cls) encoder.width = proto.width encoder.name = proto.name or None encoder.n = proto.n encoder._adaptiveScalarEnc = ( AdaptiveScalarEncoder.read(proto.adaptiveScalarEnc) ) encoder._prevAbsolute = proto.prevAbsolute encoder._prevDelta = proto.prevDelta encoder._stateLock = proto.stateLock return encoder def write(self, proto): proto.width = self.width proto.name = self.name or "" proto.n = self.n self._adaptiveScalarEnc.write(proto.adaptiveScalarEnc) proto.prevAbsolute = self._prevAbsolute proto.prevDelta = self._prevDelta proto.stateLock = self._stateLock