def __init__(self, debugSensor=False, debugMotor=False, **kwargs):
"""
@param debugSensor (bool) Controls whether sensor encodings are contiguous
@param debugMotor (bool) Controls whether motor encodings are contiguous
"""
super(OneDUniverse, self).__init__(**kwargs)
self.debugSensor = debugSensor
self.debugMotor = debugMotor
self.sensorPatternMachine = ConsecutivePatternMachine(
self.nSensor, self.wSensor)
self.sensorEncoder = SDRCategoryEncoder(self.nSensor,
self.wSensor,
forced=True)
self.motorPatternMachine = ConsecutivePatternMachine(
self.nMotor, self.wMotor)
self.motorEncoder = SDRCategoryEncoder(self.nMotor,
self.wMotor,
forced=True)
# This pool is a human friendly representation of sensory values
self.elementCodes = (
range(0x0041, 0x005A + 1) + # A-Z
range(0x0061, 0x007A + 1) + # a-z
range(0x0030, 0x0039 + 1) + # 0-9
range(0x00C0, 0x036F + 1) # Many others
)
self.numDecodedElements = len(self.elementCodes)
def __init__(self, debugSensor=False, debugMotor=False, **kwargs):
"""
@param debugSensor (bool) Controls whether sensor encodings are contiguous
@param debugMotor (bool) Controls whether motor encodings are contiguous
"""
super(OneDUniverse, self).__init__(**kwargs)
self.debugSensor = debugSensor
self.debugMotor = debugMotor
self.sensorPatternMachine = ConsecutivePatternMachine(self.nSensor,
self.wSensor)
self.sensorEncoder = SDRCategoryEncoder(self.nSensor, self.wSensor,
forced=True)
self.motorPatternMachine = ConsecutivePatternMachine(self.nMotor,
self.wMotor)
self.motorEncoder = SDRCategoryEncoder(self.nMotor, self.wMotor,
forced=True)
# This pool is a human friendly representation of sensory values
self.elementCodes = (
range(0x0041, 0x005A+1) + # A-Z
range(0x0061, 0x007A+1) + # a-z
range(0x0030, 0x0039+1) + # 0-9
range(0x00C0, 0x036F+1) # Many others
)
self.numDecodedElements = len(self.elementCodes)
def initializeEncoder(Nelements, seed):
# initialize classifiers
encoder = SDRCategoryEncoder(1024,
40,
categoryList=range(0, Nelements),
encoderSeed=seed)
return encoder
def __init__(self):
self.lat = ScalarEncoder(name='latitude', w=3, n=100, minval=-90, maxval=90,
periodic=False)
self.long= ScalarEncoder(name='longitude', w=3, n=100, minval=-180, maxval=180,
periodic=True)
self.timeenc= DateEncoder(season=0, dayOfWeek=1, weekend=3, timeOfDay=5)
self.likes = ScalarEncoder(name='likes', w=3, n=50, minval=0, maxval=100000,
periodic=False)
self.people = ScalarEncoder(name='numpeople', w=3, n=20, minval=0, maxval=100,
periodic=False)
self.categories = SDRCategoryEncoder(n=87, w=3, categoryList = None,
name="cats", verbosity=0)
self.run()
def initialize(self, useRandomEncoder):
"""
Initialize the various data structures.
"""
self.setRandomSeed(self.seed)
self.dim = numpy.shape(self.spatialConfig)[-1]
self.spatialMap = dict(
zip(map(tuple, list(self.spatialConfig)),
self.sensoryInputElements))
self.lengthMotorInput1D = (2*self.maxDisplacement + 1) * \
self.numActiveBitsMotorInput
uniqueSensoryElements = list(set(self.sensoryInputElementsPool))
if useRandomEncoder:
self.sensoryEncoder = SDRCategoryEncoder(
n=1024,
w=self.numActiveBitsSensoryInput,
categoryList=uniqueSensoryElements,
forced=True)
self.lengthSensoryInput = self.sensoryEncoder.getWidth()
else:
self.lengthSensoryInput = (len(self.sensoryInputElementsPool)+1) * \
self.numActiveBitsSensoryInput
self.sensoryEncoder = CategoryEncoder(
w=self.numActiveBitsSensoryInput,
categoryList=uniqueSensoryElements,
forced=True)
motorEncoder1D = ScalarEncoder(n=self.lengthMotorInput1D,
w=self.numActiveBitsMotorInput,
minval=-self.maxDisplacement,
maxval=self.maxDisplacement,
clipInput=True,
forced=True)
self.motorEncoder = VectorEncoder(length=self.dim,
encoder=motorEncoder1D)
def testSDRCategoryEncoder(self):
print "Testing CategoryEncoder...",
# make sure we have > 16 categories so that we have to grow our sdrs
categories = [
"ES", "S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10",
"S11", "S12", "S13", "S14", "S15", "S16", "S17", "S18", "S19",
"GB", "US"
]
fieldWidth = 100
bitsOn = 10
s = SDRCategoryEncoder(n=fieldWidth,
w=bitsOn,
categoryList=categories,
name="foo",
verbosity=0)
# internal check
self.assertEqual(s.sdrs.shape, (32, fieldWidth))
# ES
es = s.encode("ES")
self.assertEqual(es.sum(), bitsOn)
self.assertEqual(es.shape, (fieldWidth, ))
self.assertEqual(es.sum(), bitsOn)
x = s.decode(es)
self.assertIsInstance(x[0], dict)
self.assertTrue("foo" in x[0])
self.assertEqual(x[0]["foo"][1], "ES")
topDown = s.topDownCompute(es)
self.assertEqual(topDown.value, 'ES')
self.assertEqual(topDown.scalar, 1)
self.assertEqual(topDown.encoding.sum(), bitsOn)
# ----------------------------------------------------------------------
# Test topdown compute
for v in categories:
output = s.encode(v)
topDown = s.topDownCompute(output)
self.assertEqual(topDown.value, v)
self.assertEqual(topDown.scalar, s.getScalars(v)[0])
bucketIndices = s.getBucketIndices(v)
print "bucket index =>", bucketIndices[0]
topDown = s.getBucketInfo(bucketIndices)[0]
self.assertEqual(topDown.value, v)
self.assertEqual(topDown.scalar, s.getScalars(v)[0])
self.assertTrue((topDown.encoding == output).all())
self.assertEqual(topDown.value,
s.getBucketValues()[bucketIndices[0]])
# Unknown
unknown = s.encode("ASDFLKJLK")
self.assertEqual(unknown.sum(), bitsOn)
self.assertEqual(unknown.shape, (fieldWidth, ))
self.assertEqual(unknown.sum(), bitsOn)
x = s.decode(unknown)
self.assertEqual(x[0]["foo"][1], "<UNKNOWN>")
topDown = s.topDownCompute(unknown)
self.assertEqual(topDown.value, "<UNKNOWN>")
self.assertEqual(topDown.scalar, 0)
# US
us = s.encode("US")
self.assertEqual(us.sum(), bitsOn)
self.assertEqual(us.shape, (fieldWidth, ))
self.assertEqual(us.sum(), bitsOn)
x = s.decode(us)
self.assertEqual(x[0]["foo"][1], "US")
topDown = s.topDownCompute(us)
self.assertEqual(topDown.value, "US")
self.assertEqual(topDown.scalar, len(categories))
self.assertEqual(topDown.encoding.sum(), bitsOn)
# empty field
empty = s.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
self.assertEqual(empty.sum(), 0)
self.assertEqual(empty.shape, (fieldWidth, ))
self.assertEqual(empty.sum(), 0)
# make sure it can still be decoded after a change
bit = s.random.randint(0, s.getWidth() - 1)
us[bit] = 1 - us[bit]
x = s.decode(us)
self.assertEqual(x[0]["foo"][1], "US")
# add two reps together
newrep = ((us + unknown) > 0).astype('uint8')
x = s.decode(newrep)
name = x[0]["foo"][1]
if name != "US <UNKNOWN>" and name != "<UNKNOWN> US":
othercategory = name.replace("US", "")
othercategory = othercategory.replace("<UNKNOWN>", "")
othercategory = othercategory.replace(" ", "")
otherencoded = s.encode(othercategory)
print "Got: %s instead of US/unknown" % name
print "US: %s" % us
print "unknown: %s" % unknown
print "Sum: %s" % newrep
print "%s: %s" % (othercategory, s.encode(othercategory))
print "Matches with US: %d" % (us * newrep).sum()
print "Matches with unknown: %d" % (unknown * newrep).sum()
print "Matches with %s: %d" % (othercategory,
(otherencoded * newrep).sum())
raise RuntimeError("Decoding failure")
# serialization
import cPickle as pickle
t = pickle.loads(pickle.dumps(s))
self.assertTrue((t.encode("ES") == es).all())
self.assertTrue((t.encode("GB") == s.encode("GB")).all())
# Test autogrow
s = SDRCategoryEncoder(n=fieldWidth,
w=bitsOn,
categoryList=None,
name="bar")
es = s.encode("ES")
self.assertEqual(es.shape, (fieldWidth, ))
self.assertEqual(es.sum(), bitsOn)
x = s.decode(es)
self.assertIsInstance(x[0], dict)
self.assertTrue("bar" in x[0])
self.assertEqual(x[0]["bar"][1], "ES")
us = s.encode("US")
self.assertEqual(us.shape, (fieldWidth, ))
self.assertEqual(us.sum(), bitsOn)
x = s.decode(us)
self.assertEqual(x[0]["bar"][1], "US")
es2 = s.encode("ES")
self.assertTrue((es2 == es).all())
us2 = s.encode("US")
self.assertTrue((us2 == us).all())
# make sure it can still be decoded after a change
bit = s.random.randint(0, s.getWidth() - 1)
us[bit] = 1 - us[bit]
x = s.decode(us)
self.assertEqual(x[0]["bar"][1], "US")
# add two reps together
newrep = ((us + es) > 0).astype('uint8')
x = s.decode(newrep)
name = x[0]["bar"][1]
self.assertTrue(name == "US ES" or name == "ES US")
# Catch duplicate categories
caughtException = False
newcategories = categories[:]
self.assertTrue("ES" in newcategories)
newcategories.append("ES")
try:
s = SDRCategoryEncoder(n=fieldWidth,
w=bitsOn,
categoryList=newcategories,
name="foo")
except RuntimeError, e:
caughtException = True
def testAutogrow(self):
"""testing auto-grow"""
fieldWidth = 100
bitsOn = 10
s = SDRCategoryEncoder(n=fieldWidth, w=bitsOn, name="foo", verbosity=2)
encoded = numpy.zeros(fieldWidth)
self.assertEqual(s.topDownCompute(encoded).value, "<UNKNOWN>")
s.encodeIntoArray("catA", encoded)
self.assertEqual(encoded.sum(), bitsOn)
self.assertEqual(s.getScalars('catA'), 1)
catA = encoded.copy()
s.encodeIntoArray("catB", encoded)
self.assertEqual(encoded.sum(), bitsOn)
self.assertEqual(s.getScalars('catB'), 2)
catB = encoded.copy()
self.assertEqual(s.topDownCompute(catA).value, 'catA')
self.assertEqual(s.topDownCompute(catB).value, 'catB')
s.encodeIntoArray(SENTINEL_VALUE_FOR_MISSING_DATA, encoded)
self.assertEqual(sum(encoded), 0)
self.assertEqual(s.topDownCompute(encoded).value, "<UNKNOWN>")
#Test Disabling Learning and autogrow
s.setLearning(False)
s.encodeIntoArray("catC", encoded)
self.assertEqual(encoded.sum(), bitsOn)
self.assertEqual(s.getScalars('catC'), 0)
self.assertEqual(s.topDownCompute(encoded).value, "<UNKNOWN>")
s.setLearning(True)
s.encodeIntoArray("catC", encoded)
self.assertEqual(encoded.sum(), bitsOn)
self.assertEqual(s.getScalars('catC'), 3)
self.assertEqual(s.topDownCompute(encoded).value, "catC")
def testSDRCategoryEncoder(self):
print "Testing CategoryEncoder...",
# make sure we have > 16 categories so that we have to grow our sdrs
categories = ["ES", "S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8",
"S9","S10", "S11", "S12", "S13", "S14", "S15", "S16",
"S17", "S18", "S19", "GB", "US"]
fieldWidth = 100
bitsOn = 10
s = SDRCategoryEncoder(n=fieldWidth, w=bitsOn, categoryList = categories,
name="foo", verbosity=0, forced=True)
# internal check
self.assertEqual(s.sdrs.shape, (32, fieldWidth))
# ES
es = s.encode("ES")
self.assertEqual(es.sum(), bitsOn)
self.assertEqual(es.shape, (fieldWidth,))
self.assertEqual(es.sum(), bitsOn)
x = s.decode(es)
self.assertIsInstance(x[0], dict)
self.assertTrue("foo" in x[0])
self.assertEqual(x[0]["foo"][1], "ES")
topDown = s.topDownCompute(es)
self.assertEqual(topDown.value, 'ES')
self.assertEqual(topDown.scalar, 1)
self.assertEqual(topDown.encoding.sum(), bitsOn)
# ----------------------------------------------------------------------
# Test topdown compute
for v in categories:
output = s.encode(v)
topDown = s.topDownCompute(output)
self.assertEqual(topDown.value, v)
self.assertEqual(topDown.scalar, s.getScalars(v)[0])
bucketIndices = s.getBucketIndices(v)
print "bucket index =>", bucketIndices[0]
topDown = s.getBucketInfo(bucketIndices)[0]
self.assertEqual(topDown.value, v)
self.assertEqual(topDown.scalar, s.getScalars(v)[0])
self.assertTrue((topDown.encoding == output).all())
self.assertEqual(topDown.value, s.getBucketValues()[bucketIndices[0]])
# Unknown
unknown = s.encode("ASDFLKJLK")
self.assertEqual(unknown.sum(), bitsOn)
self.assertEqual(unknown.shape, (fieldWidth,))
self.assertEqual(unknown.sum(), bitsOn)
x = s.decode(unknown)
self.assertEqual(x[0]["foo"][1], "<UNKNOWN>")
topDown = s.topDownCompute(unknown)
self.assertEqual(topDown.value, "<UNKNOWN>")
self.assertEqual(topDown.scalar, 0)
# US
us = s.encode("US")
self.assertEqual(us.sum(), bitsOn)
self.assertEqual(us.shape, (fieldWidth,))
self.assertEqual(us.sum(), bitsOn)
x = s.decode(us)
self.assertEqual(x[0]["foo"][1], "US")
topDown = s.topDownCompute(us)
self.assertEqual(topDown.value, "US")
self.assertEqual(topDown.scalar, len(categories))
self.assertEqual(topDown.encoding.sum(), bitsOn)
# empty field
empty = s.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
self.assertEqual(empty.sum(), 0)
self.assertEqual(empty.shape, (fieldWidth,))
self.assertEqual(empty.sum(), 0)
# make sure it can still be decoded after a change
bit = s.random.randint(0, s.getWidth()-1)
us[bit] = 1 - us[bit]
x = s.decode(us)
self.assertEqual(x[0]["foo"][1], "US")
# add two reps together
newrep = ((us + unknown) > 0).astype('uint8')
x = s.decode(newrep)
name =x[0]["foo"][1]
if name != "US <UNKNOWN>" and name != "<UNKNOWN> US":
othercategory = name.replace("US", "")
othercategory = othercategory.replace("<UNKNOWN>", "")
othercategory = othercategory.replace(" ", "")
otherencoded = s.encode(othercategory)
print "Got: %s instead of US/unknown" % name
print "US: %s" % us
print "unknown: %s" % unknown
print "Sum: %s" % newrep
print "%s: %s" % (othercategory, s.encode(othercategory))
print "Matches with US: %d" % (us * newrep).sum()
print "Matches with unknown: %d" % (unknown * newrep).sum()
print "Matches with %s: %d" % (othercategory,
(otherencoded * newrep).sum())
raise RuntimeError("Decoding failure")
# serialization
import cPickle as pickle
t = pickle.loads(pickle.dumps(s))
self.assertTrue((t.encode("ES") == es).all())
self.assertTrue((t.encode("GB") == s.encode("GB")).all())
# Test autogrow
s = SDRCategoryEncoder(n=fieldWidth, w=bitsOn, categoryList = None, name="bar", forced=True)
es = s.encode("ES")
self.assertEqual(es.shape, (fieldWidth,))
self.assertEqual(es.sum(), bitsOn)
x = s.decode(es)
self.assertIsInstance(x[0], dict)
self.assertTrue("bar" in x[0])
self.assertEqual(x[0]["bar"][1], "ES")
us = s.encode("US")
self.assertEqual(us.shape, (fieldWidth,))
self.assertEqual(us.sum(), bitsOn)
x = s.decode(us)
self.assertEqual(x[0]["bar"][1], "US")
es2 = s.encode("ES")
self.assertTrue((es2 == es).all())
us2 = s.encode("US")
self.assertTrue((us2 == us).all())
# make sure it can still be decoded after a change
bit = s.random.randint(0, s.getWidth()-1)
us[bit] = 1 - us[bit]
x = s.decode(us)
self.assertEqual(x[0]["bar"][1], "US")
# add two reps together
newrep = ((us + es) > 0).astype('uint8')
x = s.decode(newrep)
name =x[0]["bar"][1]
self.assertTrue(name == "US ES" or name == "ES US")
# Catch duplicate categories
caughtException = False
newcategories = categories[:]
self.assertTrue("ES" in newcategories)
newcategories.append("ES")
try:
s = SDRCategoryEncoder(n=fieldWidth, w=bitsOn, categoryList = newcategories, name="foo", forced=True)
except RuntimeError, e:
caughtException = True
def testAutogrow(self):
"""testing auto-grow"""
fieldWidth = 100
bitsOn = 10
s = SDRCategoryEncoder(n=fieldWidth, w=bitsOn, name="foo", verbosity=2, forced=True)
encoded = numpy.zeros(fieldWidth)
self.assertEqual(s.topDownCompute(encoded).value, "<UNKNOWN>")
s.encodeIntoArray("catA", encoded)
self.assertEqual(encoded.sum(), bitsOn)
self.assertEqual(s.getScalars('catA'), 1)
catA = encoded.copy()
s.encodeIntoArray("catB", encoded)
self.assertEqual(encoded.sum(), bitsOn)
self.assertEqual(s.getScalars('catB'), 2)
catB = encoded.copy()
self.assertEqual(s.topDownCompute(catA).value, 'catA')
self.assertEqual(s.topDownCompute(catB).value, 'catB')
s.encodeIntoArray(SENTINEL_VALUE_FOR_MISSING_DATA, encoded)
self.assertEqual(sum(encoded), 0)
self.assertEqual(s.topDownCompute(encoded).value, "<UNKNOWN>")
#Test Disabling Learning and autogrow
s.setLearning(False)
s.encodeIntoArray("catC", encoded)
self.assertEqual(encoded.sum(), bitsOn)
self.assertEqual(s.getScalars('catC'), 0)
self.assertEqual(s.topDownCompute(encoded).value, "<UNKNOWN>")
s.setLearning(True)
s.encodeIntoArray("catC", encoded)
self.assertEqual(encoded.sum(), bitsOn)
self.assertEqual(s.getScalars('catC'), 3)
self.assertEqual(s.topDownCompute(encoded).value, "catC")
class FourSquareAnomalyDetector():
def __init__(self):
self.lat = ScalarEncoder(name='latitude', w=3, n=100, minval=-90, maxval=90,
periodic=False)
self.long= ScalarEncoder(name='longitude', w=3, n=100, minval=-180, maxval=180,
periodic=True)
self.timeenc= DateEncoder(season=0, dayOfWeek=1, weekend=3, timeOfDay=5)
self.likes = ScalarEncoder(name='likes', w=3, n=50, minval=0, maxval=100000,
periodic=False)
self.people = ScalarEncoder(name='numpeople', w=3, n=20, minval=0, maxval=100,
periodic=False)
self.categories = SDRCategoryEncoder(n=87, w=3, categoryList = None,
name="cats", verbosity=0)
self.run()
def run(self):
check1=Checkin(10,100,datetime.datetime.utcnow(),12,5,"cafe")
check2=Checkin(10,100,datetime.datetime.utcnow(),12,5,"cafe")
check3=Checkin(10,100,datetime.datetime.utcnow(),12,5,"cafe")
check4=Checkin(10,100,datetime.datetime.utcnow(),12,5,"cafe")
check5=Checkin(10,100,datetime.datetime.utcnow(),12,5,"cafe")
check6=Checkin(10,100,datetime.datetime.utcnow(),12,5,"cafe")
check7=Checkin(10,100,datetime.datetime.utcnow(),12,5,"cafe")
check8=Checkin(10,100,datetime.datetime.utcnow(),12,5,"cafe")
list_of_unencoded_checkins=[check1,check2,check3,check4,check5,check6,check7,check8]
list_of_encoded_checkins=[]
for check in list_of_unencoded_checkins:
print check
list_of_encoded_checkins.append(self.encode(check))
print self.LastAnomalyScore(list_of_encoded_checkins)
def createModel(self):
return ModelFactory.create(model_params.MODEL_PARAMS)
def encode(self, checkin):
print checkin
latenc=self.lat.encode(checkin.latitude)
longenc=self.long.encode(checkin.longitude)
timenc=self.timeenc.encode(checkin.time)
likeenc=self.likes.encode(checkin.likes)
peoplenc=self.people.encode(checkin.people)
for cat in checkin.categories:
try:
catenc=numpy.logical_or(catenc,self.categories.encode(cat))
except:
catenc=self.categories.encode(cat)
checkinsdr=numpy.concatenate((latenc,longenc,timenc,likeenc,peoplenc,catenc))
print checkinsdr
print type(checkinsdr)
return checkinsdr
def LastAnomalyScore(self, checkin_list):
model = self.createModel()
model.enableInference({'predictedField': 'checkin'})
last_anomaly = 0
for i, record in enumerate(checkin_list, start=1):
modelInput = {"checkin": record}
result = model.run(modelInput)
anomalyScore = result.inferences['anomalyScore']
last_anomaly = anomalyScore
return last_anomaly
class OneDUniverse(AbstractUniverse):
def __init__(self, debugSensor=False, debugMotor=False, **kwargs):
"""
@param debugSensor (bool) Controls whether sensor encodings are contiguous
@param debugMotor (bool) Controls whether motor encodings are contiguous
"""
super(OneDUniverse, self).__init__(**kwargs)
self.debugSensor = debugSensor
self.debugMotor = debugMotor
self.sensorPatternMachine = ConsecutivePatternMachine(
self.nSensor, self.wSensor)
self.sensorEncoder = SDRCategoryEncoder(self.nSensor,
self.wSensor,
forced=True)
self.motorPatternMachine = ConsecutivePatternMachine(
self.nMotor, self.wMotor)
self.motorEncoder = SDRCategoryEncoder(self.nMotor,
self.wMotor,
forced=True)
# This pool is a human friendly representation of sensory values
self.elementCodes = (
range(0x0041, 0x005A + 1) + # A-Z
range(0x0061, 0x007A + 1) + # a-z
range(0x0030, 0x0039 + 1) + # 0-9
range(0x00C0, 0x036F + 1) # Many others
)
self.numDecodedElements = len(self.elementCodes)
def encodeSensorValue(self, sensorValue):
"""
@param sensorValue (object) Sensor value
@return (set) Sensor pattern
"""
if self.debugSensor:
return self.sensorPatternMachine.get(sensorValue)
else:
return set(self.sensorEncoder.encode(sensorValue).nonzero()[0])
def decodeSensorValue(self, sensorValue):
"""
@param sensorValue (object) Sensor value
@return (string) Human viewable representation of sensorValue
"""
if sensorValue < len(self.elementCodes):
return unichr(self.elementCodes[sensorValue])
else:
return unichr(0x003F) # Character: ?
def encodeMotorValue(self, motorValue):
"""
@param motorValue (object) Motor value
@return (set) Motor pattern
"""
if self.debugMotor:
numMotorValues = self.nMotor / self.wMotor
motorRadius = (numMotorValues - 1) / 2
return self.motorPatternMachine.get(motorValue + motorRadius)
else:
return set(self.motorEncoder.encode(motorValue).nonzero()[0])
class OneDUniverse(AbstractUniverse):
def __init__(self, debugSensor=False, debugMotor=False, **kwargs):
"""
@param debugSensor (bool) Controls whether sensor encodings are contiguous
@param debugMotor (bool) Controls whether motor encodings are contiguous
"""
super(OneDUniverse, self).__init__(**kwargs)
self.debugSensor = debugSensor
self.debugMotor = debugMotor
self.sensorPatternMachine = ConsecutivePatternMachine(self.nSensor,
self.wSensor)
self.sensorEncoder = SDRCategoryEncoder(self.nSensor, self.wSensor,
forced=True)
self.motorPatternMachine = ConsecutivePatternMachine(self.nMotor,
self.wMotor)
self.motorEncoder = SDRCategoryEncoder(self.nMotor, self.wMotor,
forced=True)
# This pool is a human friendly representation of sensory values
self.elementCodes = (
range(0x0041, 0x005A+1) + # A-Z
range(0x0061, 0x007A+1) + # a-z
range(0x0030, 0x0039+1) + # 0-9
range(0x00C0, 0x036F+1) # Many others
)
self.numDecodedElements = len(self.elementCodes)
def encodeSensorValue(self, sensorValue):
"""
@param sensorValue (object) Sensor value
@return (set) Sensor pattern
"""
if self.debugSensor:
return self.sensorPatternMachine.get(sensorValue)
else:
return set(self.sensorEncoder.encode(sensorValue).nonzero()[0])
def decodeSensorValue(self, sensorValue):
"""
@param sensorValue (object) Sensor value
@return (string) Human viewable representation of sensorValue
"""
if sensorValue < len(self.elementCodes):
return unichr(self.elementCodes[sensorValue])
else:
return unichr(0x003F) # Character: ?
def encodeMotorValue(self, motorValue):
"""
@param motorValue (object) Motor value
@return (set) Motor pattern
"""
if self.debugMotor:
numMotorValues = self.nMotor / self.wMotor
motorRadius = (numMotorValues - 1) / 2
return self.motorPatternMachine.get(motorValue + motorRadius)
else:
return set(self.motorEncoder.encode(motorValue).nonzero()[0])
def testSDRCategoryEncoder(self):
# make sure we have > 16 categories so that we have to grow our sdrs
categories = [
"ES", "S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10",
"S11", "S12", "S13", "S14", "S15", "S16", "S17", "S18", "S19",
"GB", "US"
]
fieldWidth = 100
bitsOn = 10
s = SDRCategoryEncoder(n=fieldWidth,
w=bitsOn,
categoryList=categories,
name="foo",
verbosity=0,
forced=True)
# internal check
self.assertEqual(s.sdrs.shape, (32, fieldWidth))
# ES
es = s.encode("ES")
self.assertEqual(es.sum(), bitsOn)
self.assertEqual(es.shape, (fieldWidth, ))
self.assertEqual(es.sum(), bitsOn)
x = s.decode(es)
self.assertIsInstance(x[0], dict)
self.assertTrue("foo" in x[0])
self.assertEqual(x[0]["foo"][1], "ES")
topDown = s.topDownCompute(es)
self.assertEqual(topDown.value, "ES")
self.assertEqual(topDown.scalar, 1)
self.assertEqual(topDown.encoding.sum(), bitsOn)
# ----------------------------------------------------------------------
# Test topdown compute
for v in categories:
output = s.encode(v)
topDown = s.topDownCompute(output)
self.assertEqual(topDown.value, v)
self.assertEqual(topDown.scalar, s.getScalars(v)[0])
bucketIndices = s.getBucketIndices(v)
topDown = s.getBucketInfo(bucketIndices)[0]
self.assertEqual(topDown.value, v)
self.assertEqual(topDown.scalar, s.getScalars(v)[0])
self.assertTrue(numpy.array_equal(topDown.encoding, output))
self.assertEqual(topDown.value,
s.getBucketValues()[bucketIndices[0]])
# Unknown
unknown = s.encode("ASDFLKJLK")
self.assertEqual(unknown.sum(), bitsOn)
self.assertEqual(unknown.shape, (fieldWidth, ))
self.assertEqual(unknown.sum(), bitsOn)
x = s.decode(unknown)
self.assertEqual(x[0]["foo"][1], "<UNKNOWN>")
topDown = s.topDownCompute(unknown)
self.assertEqual(topDown.value, "<UNKNOWN>")
self.assertEqual(topDown.scalar, 0)
# US
us = s.encode("US")
self.assertEqual(us.sum(), bitsOn)
self.assertEqual(us.shape, (fieldWidth, ))
self.assertEqual(us.sum(), bitsOn)
x = s.decode(us)
self.assertEqual(x[0]["foo"][1], "US")
topDown = s.topDownCompute(us)
self.assertEqual(topDown.value, "US")
self.assertEqual(topDown.scalar, len(categories))
self.assertEqual(topDown.encoding.sum(), bitsOn)
# empty field
empty = s.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
self.assertEqual(empty.sum(), 0)
self.assertEqual(empty.shape, (fieldWidth, ))
self.assertEqual(empty.sum(), 0)
# make sure it can still be decoded after a change
bit = s.random.getUInt32(s.getWidth() - 1)
us[bit] = 1 - us[bit]
x = s.decode(us)
self.assertEqual(x[0]["foo"][1], "US")
# add two reps together
newrep = ((us + unknown) > 0).astype(numpy.uint8)
x = s.decode(newrep)
name = x[0]["foo"][1]
if name != "US <UNKNOWN>" and name != "<UNKNOWN> US":
othercategory = name.replace("US", "")
othercategory = othercategory.replace("<UNKNOWN>", "")
othercategory = othercategory.replace(" ", "")
otherencoded = s.encode(othercategory)
raise RuntimeError("Decoding failure")
# serialization
# TODO: Remove pickle-based serialization tests -- issues #1419 and #1420
import cPickle as pickle
t = pickle.loads(pickle.dumps(s))
self.assertTrue((t.encode("ES") == es).all())
self.assertTrue((t.encode("GB") == s.encode("GB")).all())
# Test autogrow
s = SDRCategoryEncoder(n=fieldWidth,
w=bitsOn,
categoryList=None,
name="bar",
forced=True)
es = s.encode("ES")
self.assertEqual(es.shape, (fieldWidth, ))
self.assertEqual(es.sum(), bitsOn)
x = s.decode(es)
self.assertIsInstance(x[0], dict)
self.assertTrue("bar" in x[0])
self.assertEqual(x[0]["bar"][1], "ES")
us = s.encode("US")
self.assertEqual(us.shape, (fieldWidth, ))
self.assertEqual(us.sum(), bitsOn)
x = s.decode(us)
self.assertEqual(x[0]["bar"][1], "US")
es2 = s.encode("ES")
self.assertTrue(numpy.array_equal(es2, es))
us2 = s.encode("US")
self.assertTrue(numpy.array_equal(us2, us))
# make sure it can still be decoded after a change
bit = s.random.getUInt32(s.getWidth() - 1)
us[bit] = 1 - us[bit]
x = s.decode(us)
self.assertEqual(x[0]["bar"][1], "US")
# add two reps together
newrep = ((us + es) > 0).astype(numpy.uint8)
x = s.decode(newrep)
name = x[0]["bar"][1]
self.assertTrue(name == "US ES" or name == "ES US")
# Catch duplicate categories
caughtException = False
newcategories = categories[:]
self.assertTrue("ES" in newcategories)
newcategories.append("ES")
try:
s = SDRCategoryEncoder(n=fieldWidth,
w=bitsOn,
categoryList=newcategories,
name="foo",
forced=True)
except RuntimeError, e:
caughtException = True
def testReadWrite(self):
categories = [
"ES", "S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10",
"S11", "S12", "S13", "S14", "S15", "S16", "S17", "S18", "S19",
"GB", "US"
]
fieldWidth = 100
bitsOn = 10
original = SDRCategoryEncoder(n=fieldWidth,
w=bitsOn,
categoryList=categories,
name="foo",
verbosity=0,
forced=True)
# internal check
self.assertEqual(original.sdrs.shape, (32, fieldWidth))
# ES
es = original.encode("ES")
self.assertEqual(es.sum(), bitsOn)
self.assertEqual(es.shape, (fieldWidth, ))
self.assertEqual(es.sum(), bitsOn)
decoded = original.decode(es)
proto1 = SDRCategoryEncoderProto.new_message()
original.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = SDRCategoryEncoderProto.read(f)
encoder = SDRCategoryEncoder.read(proto2)
self.assertIsInstance(encoder, SDRCategoryEncoder)
self.assertEqual(encoder.n, original.n)
self.assertEqual(encoder.w, original.w)
self.assertEqual(encoder.verbosity, original.verbosity)
self.assertEqual(encoder.description, original.description)
self.assertEqual(encoder.name, original.name)
self.assertDictEqual(encoder.categoryToIndex, original.categoryToIndex)
self.assertTrue(numpy.array_equal(encoder.encode("ES"), es))
self.assertEqual(original.decode(encoder.encode("ES")),
encoder.decode(original.encode("ES")))
self.assertEqual(decoded, encoder.decode(es))
# Test autogrow serialization
autogrow = SDRCategoryEncoder(n=fieldWidth,
w=bitsOn,
categoryList=None,
name="bar",
forced=True)
es = autogrow.encode("ES")
us = autogrow.encode("US")
gs = autogrow.encode("GS")
proto1 = SDRCategoryEncoderProto.new_message()
autogrow.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = SDRCategoryEncoderProto.read(f)
t = SDRCategoryEncoder.read(proto2)
self.assertTrue(numpy.array_equal(t.encode("ES"), es))
self.assertTrue(numpy.array_equal(t.encode("US"), us))
self.assertTrue(numpy.array_equal(t.encode("GS"), gs))
def testReadWrite(self):
categories = ["ES", "S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8",
"S9","S10", "S11", "S12", "S13", "S14", "S15", "S16",
"S17", "S18", "S19", "GB", "US"]
fieldWidth = 100
bitsOn = 10
original = SDRCategoryEncoder(n=fieldWidth, w=bitsOn,
categoryList=categories,
name="foo", verbosity=0, forced=True)
# internal check
self.assertEqual(original.sdrs.shape, (32, fieldWidth))
# ES
es = original.encode("ES")
self.assertEqual(es.sum(), bitsOn)
self.assertEqual(es.shape, (fieldWidth,))
self.assertEqual(es.sum(), bitsOn)
decoded = original.decode(es)
proto1 = SDRCategoryEncoderProto.new_message()
original.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = SDRCategoryEncoderProto.read(f)
encoder = SDRCategoryEncoder.read(proto2)
self.assertIsInstance(encoder, SDRCategoryEncoder)
self.assertEqual(encoder.n, original.n)
self.assertEqual(encoder.w, original.w)
self.assertEqual(encoder.verbosity, original.verbosity)
self.assertEqual(encoder.description, original.description)
self.assertEqual(encoder.name, original.name)
self.assertDictEqual(encoder.categoryToIndex, original.categoryToIndex)
self.assertTrue(numpy.array_equal(encoder.encode("ES"), es))
self.assertEqual(original.decode(encoder.encode("ES")),
encoder.decode(original.encode("ES")))
self.assertEqual(decoded, encoder.decode(es))
# Test autogrow serialization
autogrow = SDRCategoryEncoder(n=fieldWidth, w=bitsOn, categoryList = None,
name="bar", forced=True)
es = autogrow.encode("ES")
us = autogrow.encode("US")
gs = autogrow.encode("GS")
proto1 = SDRCategoryEncoderProto.new_message()
autogrow.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = SDRCategoryEncoderProto.read(f)
t = SDRCategoryEncoder.read(proto2)
self.assertTrue(numpy.array_equal(t.encode("ES"), es))
self.assertTrue(numpy.array_equal(t.encode("US"), us))
self.assertTrue(numpy.array_equal(t.encode("GS"), gs))
def testAutogrow(self):
"""testing auto-grow"""
fieldWidth = 100
bitsOn = 10
s = SDRCategoryEncoder(n=fieldWidth, w=bitsOn, name="foo", verbosity=2)
encoded = numpy.zeros(fieldWidth)
assert s.topDownCompute(encoded).value == "<UNKNOWN>"
s.encodeIntoArray("catA", encoded)
assert encoded.sum() == bitsOn
assert s.getScalars('catA') == 1
catA = encoded.copy()
s.encodeIntoArray("catB", encoded)
assert encoded.sum() == bitsOn
assert s.getScalars('catB') == 2
catB = encoded.copy()
assert s.topDownCompute(catA).value == 'catA'
assert s.topDownCompute(catB).value == 'catB'
s.encodeIntoArray(SENTINEL_VALUE_FOR_MISSING_DATA, encoded)
assert sum(encoded) == 0
assert s.topDownCompute(encoded).value == "<UNKNOWN>"
#Test Disabling Learning and autogrow
s.setLearning(False)
s.encodeIntoArray("catC", encoded)
assert encoded.sum() == bitsOn
assert s.getScalars('catC') == 0
assert s.topDownCompute(encoded).value == "<UNKNOWN>"
s.setLearning(True)
s.encodeIntoArray("catC", encoded)
assert encoded.sum() == bitsOn
assert s.getScalars('catC') == 3
assert s.topDownCompute(encoded).value == "catC"