def __init__(self, w, categoryList, name="category", verbosity=0, forced=False):
"""params:
forced (default False) : if True, skip checks for parameters' settings; see encoders/scalar.py for details
"""
self.encoders = None
self.verbosity = verbosity
# number of categories includes "unknown"
self.ncategories = len(categoryList) + 1
self.categoryToIndex = dict()
self.indexToCategory = dict()
self.indexToCategory[0] = UNKNOWN
for i in xrange(len(categoryList)):
self.categoryToIndex[categoryList[i]] = i+1
self.indexToCategory[i+1] = categoryList[i]
self.encoder = ScalarEncoder(w, minval=0, maxval=self.ncategories - 1,
radius=1, periodic=False, forced=forced)
self.width = w * self.ncategories
assert self.encoder.getWidth() == self.width
self.description = [(name, 0)]
self.name = name
# These are used to support the topDownCompute method
self._topDownMappingM = None
# This gets filled in by getBucketValues
self._bucketValues = None
def __init__(self, w, categoryList, name="category", verbosity=0):
self.encoders = None
self.verbosity = verbosity
# number of categories includes "unknown"
self.ncategories = len(categoryList) + 1
self.categoryToIndex = dict()
self.indexToCategory = dict()
self.indexToCategory[0] = "<UNKNOWN>"
for i in xrange(len(categoryList)):
self.categoryToIndex[categoryList[i]] = i + 1
self.indexToCategory[i + 1] = categoryList[i]
self.encoder = ScalarEncoder(w,
minval=0,
maxval=self.ncategories - 1,
radius=1,
periodic=False)
self.width = w * self.ncategories
assert self.encoder.getWidth() == self.width
self.description = [(name, 0)]
self.name = name
# These are used to support the topDownCompute method
self._topDownMappingM = None
# This gets filled in by getBucketValues
self._bucketValues = None
def testCreateResolution(self):
"""Test that we get the same encoder when we construct it using resolution
instead of n
"""
l = self._l
d = l.__dict__
l = ScalarEncoder(name="scalar",
resolution=0.5,
w=3,
minval=1,
maxval=8,
periodic=True,
forced=True)
self.assertEqual(l.__dict__, d)
# Test that we get the same encoder when we construct it using radius
# instead of n
l = ScalarEncoder(name="scalar",
radius=1.5,
w=3,
minval=1,
maxval=8,
periodic=True,
forced=True)
self.assertEqual(l.__dict__, d)
def testCloseness(self):
"""Test closenessScores for a periodic encoder"""
encoder = ScalarEncoder(w=7, minval=0, maxval=7, radius=1, periodic=True,
name="day of week", forced=True)
scores = encoder.closenessScores((2, 4, 7), (4, 2, 1), fractional=False)
for actual, score in itertools.izip((2, 2, 1), scores):
self.assertEqual(actual, score)
def testCloseness(self):
"""Test closenessScores for a periodic encoder"""
encoder = ScalarEncoder(w=7, minval=0, maxval=7, radius=1, periodic=True,
name="day of week", forced=True)
scores = encoder.closenessScores((2, 4, 7), (4, 2, 1), fractional=False)
for actual, score in itertools.izip((2, 2, 1), scores):
self.assertEqual(actual, score)
def __init__(self,
w=5,
minval=1e-07,
maxval=10000,
periodic=False,
n=0,
radius=0,
resolution=0,
name="log",
verbosity=0,
clipInput=True,
forced=False):
# Lower bound for log encoding near machine precision limit
lowLimit = 1e-07
# Limit minval as log10(0) is undefined.
if minval < lowLimit:
minval = lowLimit
# Check that minval is still lower than maxval
if not minval < maxval:
raise ValueError(
"Max val must be larger than min val or the lower limit "
"for this encoder %.7f" % lowLimit)
self.encoders = None
self.verbosity = verbosity
# Scale values for calculations within the class
self.minScaledValue = math.log10(minval)
self.maxScaledValue = math.log10(maxval)
if not self.maxScaledValue > self.minScaledValue:
raise ValueError(
"Max val must be larger, in log space, than min val.")
self.clipInput = clipInput
self.minval = minval
self.maxval = maxval
self.encoder = ScalarEncoder(w=w,
minval=self.minScaledValue,
maxval=self.maxScaledValue,
periodic=False,
n=n,
radius=radius,
resolution=resolution,
verbosity=self.verbosity,
clipInput=self.clipInput,
forced=forced)
self.width = self.encoder.getWidth()
self.description = [(name, 0)]
self.name = name
# This list is created by getBucketValues() the first time it is called,
# and re-created whenever our buckets would be re-arranged.
self._bucketValues = None
def testScalarEncoder(self):
"""Testing ScalarEncoder..."""
# -------------------------------------------------------------------------
# test missing values
mv = ScalarEncoder(name="mv", n=14, w=3, minval=1, maxval=8,
periodic=False, forced=True)
empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
self.assertEqual(empty.sum(), 0)
def testScalarEncoder(self):
"""Testing ScalarEncoder..."""
# -------------------------------------------------------------------------
# test missing values
mv = ScalarEncoder(name='mv', n=14, w=3, minval=1, maxval=8, periodic=False, forced=True)
empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
print "\nEncoded missing data \'None\' as %s" % empty
self.assertEqual(empty.sum(), 0)
def testScalarEncoder(self):
"""Testing ScalarEncoder..."""
# -------------------------------------------------------------------------
# test missing values
mv = ScalarEncoder(name='mv', n=14, w=3, minval=1, maxval=8, periodic=False, forced=True)
empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
print "\nEncoded missing data \'None\' as %s" % empty
self.assertEqual(empty.sum(), 0)
def testScalarEncoder(self):
"""Testing ScalarEncoder..."""
# -------------------------------------------------------------------------
# test missing values
mv = ScalarEncoder(name="mv", n=14, w=3, minval=1, maxval=8,
periodic=False, forced=True)
empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
self.assertEqual(empty.sum(), 0)
def setUp(self):
# use of forced is not recommended, but used here for readability, see scalar.py
self._l = ScalarEncoder(name='scalar',
n=14,
w=3,
minval=1,
maxval=8,
periodic=True,
forced=True)
def testEncodeInvalidInputType(self):
encoder = ScalarEncoder(name="enc",
n=14,
w=3,
minval=1,
maxval=8,
periodic=False,
forced=True)
with self.assertRaises(TypeError):
encoder.encode("String")
def testGetBucketInfoIntResolution(self):
"""Ensures that passing resolution as an int doesn't truncate values."""
encoder = ScalarEncoder(w=3,
resolution=1,
minval=1,
maxval=8,
periodic=True,
forced=True)
self.assertEqual(4.5,
encoder.topDownCompute(encoder.encode(4.5))[0].scalar)
def testNaNs(self):
"""test NaNs"""
mv = ScalarEncoder(name='mv',
n=14,
w=3,
minval=1,
maxval=8,
periodic=False)
empty = mv.encode(float("nan"))
print "\nEncoded missing data \'None\' as %s" % empty
self.assertEqual(empty.sum(), 0)
def testNaNs(self):
"""test NaNs"""
mv = ScalarEncoder(name="mv",
n=14,
w=3,
minval=1,
maxval=8,
periodic=False,
forced=True)
empty = mv.encode(float("nan"))
self.assertEqual(empty.sum(), 0)
def __init__(self):
self.lat = ScalarEncoder(name='latitude', w=3, n=100, minval=-90, maxval=90,
periodic=False)
self.long= ScalarEncoder(name='longitude', w=3, n=100, minval=-180, maxval=180,
periodic=True)
self.timeenc= DateEncoder(season=0, dayOfWeek=1, weekend=3, timeOfDay=5)
self.likes = ScalarEncoder(name='likes', w=3, n=50, minval=0, maxval=100000,
periodic=False)
self.people = ScalarEncoder(name='numpeople', w=3, n=20, minval=0, maxval=100,
periodic=False)
self.categories = SDRCategoryEncoder(n=87, w=3, categoryList = None,
name="cats", verbosity=0)
self.run()
def profileEnc(maxValue, nRuns):
minV = 0
maxV = nRuns
# generate input data
data = numpy.random.randint(minV, maxV + 1, nRuns)
# instantiate measured encoders
encScalar = ScalarEncoder(w=21, minval=minV, maxval=maxV, resolution=1)
encRDSE = RDSE(resolution=1)
# profile!
for d in data:
encScalar.encode(d)
encRDSE.encode(d)
print("Scalar n=", encScalar.n, " RDSE n=", encRDSE.n)
def profileEnc(maxValue, nRuns):
minV=0
maxV=nRuns
# generate input data
data=numpy.random.randint(minV, maxV+1, nRuns)
# instantiate measured encoders
encScalar = ScalarEncoder(w=21, minval=minV, maxval=maxV, resolution=1)
encRDSE = RDSE(resolution=1)
# profile!
for d in data:
encScalar.encode(d)
encRDSE.encode(d)
print "Scalar n=",encScalar.n," RDSE n=",encRDSE.n
def setUp(self):
self._l = ScalarEncoder(name='scalar',
n=14,
w=3,
minval=1,
maxval=8,
periodic=True)
def SE(**kwargs):
"""
SCALAR ENCODER, see definition for more info
Parameters --
.. warning:: There are three mutually exclusive parameters that determine the
overall size of of the output. Exactly one of n, radius, resolution must be
set. "0" is a special value that means "not set".
@param w: Number of ON bits which encode a single value, must be odd
@param minval: Minimum value of input signal
@param maxval: Maximum value of the input signal
@param periodic: if true, then input "wraps around" such that the
input must be strictly less than maxval. Default is False
@param n: Total number of bits in the output must be >= w
@param radius: inputs separated by more than the radius will have
non-overlapping representations
@param resolution: inputs separated by more than the resolution will have
different, but possible overlapping, representations
@param name: Optional string which will become part of the description
@param clipInput: if True, non-periodic inputs < minval or > maxval will be
clipped to minval and maxval, respectively
@param forced: if True, skip some safety checks. Default is False
"""
return ScalarEncoder(**kwargs)
def __init__(self, w, categoryList, name="category", verbosity=0, forced=False):
self.encoders = None
self.verbosity = verbosity
# number of categories includes "unknown"
self.ncategories = len(categoryList) + 1
self.categoryToIndex = dict()
self.indexToCategory = dict()
self.indexToCategory[0] = UNKNOWN
for i in xrange(len(categoryList)):
self.categoryToIndex[categoryList[i]] = i+1
self.indexToCategory[i+1] = categoryList[i]
self.encoder = ScalarEncoder(w, minval=0, maxval=self.ncategories - 1,
radius=1, periodic=False, forced=forced)
self.width = w * self.ncategories
assert self.encoder.getWidth() == self.width
self.description = [(name, 0)]
self.name = name
# These are used to support the topDownCompute method
self._topDownMappingM = None
# This gets filled in by getBucketValues
self._bucketValues = None
def __init__(self,
length,
w,
minval,
maxval,
periodic=False,
n=0,
radius=0,
resolution=0,
name=None,
verbosity=0,
clipInput=False):
"""instance of VectorEncoder using ScalarEncoder as base,
use-case: in OPF description.py files, where you cannot use VectorEncoder directly (see above);
param length: #elements in the vector,
param: rest of params is from ScalarEncoder, see scalar.py for details"""
sc = ScalarEncoder(w,
minval,
maxval,
periodic=periodic,
n=n,
radius=radius,
resolution=resolution,
name=name,
verbosity=verbosity,
clipInput=clipInput)
super(VectorEncoderOPF, self).__init__(length, sc, typeCastFn=float)
def testReadWrite(self):
"""Test ScalarEncoder Cap'n Proto serialization implementation."""
originalValue = self._l.encode(1)
proto1 = ScalarEncoderProto.new_message()
self._l.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = ScalarEncoderProto.read(f)
encoder = ScalarEncoder.read(proto2)
self.assertIsInstance(encoder, ScalarEncoder)
self.assertEqual(encoder.w, self._l.w)
self.assertEqual(encoder.minval, self._l.minval)
self.assertEqual(encoder.maxval, self._l.maxval)
self.assertEqual(encoder.periodic, self._l.periodic)
self.assertEqual(encoder.n, self._l.n)
self.assertEqual(encoder.radius, self._l.radius)
self.assertEqual(encoder.resolution, self._l.resolution)
self.assertEqual(encoder.name, self._l.name)
self.assertEqual(encoder.verbosity, self._l.verbosity)
self.assertEqual(encoder.clipInput, self._l.clipInput)
self.assertTrue(numpy.array_equal(encoder.encode(1), originalValue))
self.assertEqual(self._l.decode(encoder.encode(1)),
encoder.decode(self._l.encode(1)))
# Feed in a new value and ensure the encodings match
result1 = self._l.encode(7)
result2 = encoder.encode(7)
self.assertTrue(numpy.array_equal(result1, result2))
def __init__(self, w, categoryList, name="category", verbosity=0, forced=False):
"""params:
forced (default False) : if True, skip checks for parameters' settings; see encoders/scalar.py for details
"""
self.encoders = None
self.verbosity = verbosity
# number of categories includes "unknown"
self.ncategories = len(categoryList) + 1
self.categoryToIndex = dict() # check_later: what is the purpose of categoryToIndex and indexToCategory?
self.indexToCategory = dict()
self.indexToCategory[0] = UNKNOWN
for i in xrange(len(categoryList)):
self.categoryToIndex[categoryList[i]] = i+1
self.indexToCategory[i+1] = categoryList[i]
self.encoder = ScalarEncoder(w, minval=0, maxval=self.ncategories - 1,
radius=1, periodic=False, forced=forced)
self.width = w * self.ncategories
assert self.encoder.getWidth() == self.width
self.description = [(name, 0)]
self.name = name
# These are used to support the topDownCompute method
self._topDownMappingM = None
# This gets filled in by getBucketValues
self._bucketValues = None
def testReadWrite(self):
"""Test ScalarEncoder Cap'n Proto serialization implementation."""
originalValue = self._l.encode(1)
proto1 = ScalarEncoderProto.new_message()
self._l.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = ScalarEncoderProto.read(f)
encoder = ScalarEncoder.read(proto2)
self.assertIsInstance(encoder, ScalarEncoder)
self.assertEqual(encoder.w, self._l.w)
self.assertEqual(encoder.minval, self._l.minval)
self.assertEqual(encoder.maxval, self._l.maxval)
self.assertEqual(encoder.periodic, self._l.periodic)
self.assertEqual(encoder.n, self._l.n)
self.assertEqual(encoder.radius, self._l.radius)
self.assertEqual(encoder.resolution, self._l.resolution)
self.assertEqual(encoder.name, self._l.name)
self.assertEqual(encoder.verbosity, self._l.verbosity)
self.assertEqual(encoder.clipInput, self._l.clipInput)
self.assertTrue(numpy.array_equal(encoder.encode(1), originalValue))
self.assertEqual(self._l.decode(encoder.encode(1)),
encoder.decode(self._l.encode(1)))
# Feed in a new value and ensure the encodings match
result1 = self._l.encode(7)
result2 = encoder.encode(7)
self.assertTrue(numpy.array_equal(result1, result2))
def __init__(self):
self.encoder = CoordinateEncoder(n=1024, w=21)
self.motorEncoder = ScalarEncoder(21, -1, 1, n=1024)
self.tm = MonitoredGeneralTemporalMemory(columnDimensions=[2048],
cellsPerColumn=1,
initialPermanence=0.5,
connectedPermanence=0.6,
permanenceIncrement=0.1,
permanenceDecrement=0.02,
minThreshold=35,
activationThreshold=35,
maxNewSynapseCount=40)
self.plotter = Plotter(self.tm)
self.lastState = None
self.lastAction = None
def __init__(self,
pathToFevalFnFile,
length=5,
feedbackDelay=0,
minval=-5,
maxval=5,
resolution=1,
scoreMin=0,
scoreMax=100,
scoreResolution=1,
forced=False):
"""
@param pathToFevalFnFile path to a file that contains python \"def feval(self, other, params)\" function to be loaded; used as feval
"""
# TODO: cleanup the params
dataV = VectorEncoderOPF(length,
minval,
maxval,
resolution=resolution,
name='data')
scoreS = ScalarEncoder(21,
scoreMin,
scoreMax,
resolution=scoreResolution,
name='utility')
myDef = imp.load_source('module.name', pathToFevalFnFile)
if not isinstance(myDef.feval, FunctionType):
raise Exception("failed to load feval() function from %s" %
pathToFevalFile)
super(UtilityEncoderOPF, self).__init__(dataV,
scoreS,
feval=myDef.feval,
feedbackDelay=feedbackDelay,
name='UtilityOPF',
forced=forced)
def __init__(
self,
w=5,
minval=1e-07,
maxval=10000,
periodic=False,
n=0,
radius=0,
resolution=0,
name="log",
verbosity=0,
clipInput=True,
):
# Lower bound for log encoding near machine precision limit
lowLimit = 1e-07
# Limit minval as log10(0) is undefined.
if minval < lowLimit:
minval = lowLimit
# Check that minval is still lower than maxval
if not minval < maxval:
raise ValueError(
"Max val must be larger than min val or the lower limit " "for this encoder %.7f" % lowLimit
)
self.encoders = None
self.verbosity = verbosity
# Scale values for calculations within the class
self.minScaledValue = math.log10(minval)
self.maxScaledValue = math.log10(maxval)
if not self.maxScaledValue > self.minScaledValue:
raise ValueError("Max val must be larger, in log space, than min val.")
self.clipInput = clipInput
self.minval = minval
self.maxval = maxval
self.encoder = ScalarEncoder(
w=w,
minval=self.minScaledValue,
maxval=self.maxScaledValue,
periodic=False,
n=n,
radius=radius,
resolution=resolution,
verbosity=self.verbosity,
clipInput=self.clipInput,
)
self.width = self.encoder.getWidth()
self.description = [(name, 0)]
self.name = name
# This list is created by getBucketValues() the first time it is called,
# and re-created whenever our buckets would be re-arranged.
self._bucketValues = None
class ScalarBucketEncoder(Encoder):
def __init__(self):
self.encoder = NupicScalarEncoder(w=1, minval=0, maxval=40000, n=22, forced=True)
def encode(self, symbol):
encoding = self.encoder.encode(symbol)
return encoding
class Agent(object):
def __init__(self):
self.encoder = CoordinateEncoder(n=1024, w=21)
self.motorEncoder = ScalarEncoder(21, -1, 1, n=1024)
self.tm = MonitoredGeneralTemporalMemory(columnDimensions=[2048],
cellsPerColumn=1,
initialPermanence=0.5,
connectedPermanence=0.6,
permanenceIncrement=0.1,
permanenceDecrement=0.02,
minThreshold=35,
activationThreshold=35,
maxNewSynapseCount=40)
self.plotter = Plotter(self.tm)
self.lastState = None
self.lastAction = None
def sync(self, outputData):
if not ("location" in outputData and "steer" in outputData):
print "Warning: Missing data:", outputData
return
if outputData.get("reset"):
print "Reset."
self.tm.reset()
location = outputData["location"]
steer = outputData["steer"]
x = int(location["x"] * SCALE)
z = int(location["z"] * SCALE)
coordinate = numpy.array([x, z])
encoding = self.encoder.encode((coordinate, RADIUS))
motorEncoding = self.motorEncoder.encode(steer)
sensorPattern = set(encoding.nonzero()[0])
motorPattern = set(motorEncoding.nonzero()[0])
self.tm.compute(sensorPattern,
activeExternalCells=motorPattern,
formInternalConnections=True)
print self.tm.mmPrettyPrintMetrics(self.tm.mmGetDefaultMetrics())
overlap = 0
if self.lastState is not None:
overlap = (self.lastState & encoding).sum()
self.plotter.update(overlap)
if outputData.get("reset"):
self.plotter.render()
self.lastState = encoding
self.lastAction = steer
def __init__(self, length=5, feedbackDelay=0, minval=-5, maxval=5, resolution=1, scoreMin=0, scoreMax=100, scoreResolution=1, forced=False):
"""
initialize UtilityEncoder with default settings.
@param length size of the input vector
@param feedbackDelay integer >=0; used if you want to apply a score from the current state (T) to state in time T-1; default(0) means state(T) pairs with score(T)
@param minval minimal value of input data (from ScalarEncoder)
@param maxval max value input data can reach; eg for set of vectors {[0,0,1], [0,-1,0],[1,2,0]} it's -1 and 2
@param resolution resolution of the underliing scalar encoder
@param scoreMin
@param scoreMax same as minval, maxval but for the score
@param scoreResolution
@forced
"""
dataS = ScalarEncoder(21, minval, maxval, resolution=resolution, name='idx')
dataV = VectorEncoder(length, dataS, name='data')
scoreS = ScalarEncoder(21, scoreMin, scoreMax, resolution=scoreResolution, name='utility')
super(SimpleUtilityEncoder, self).__init__(dataV, scoreS, feedbackDelay=feedbackDelay, name='simpleUtility', forced=forced)
print "WARNING: feval not set! do not forget to def(ine) the function and set it with setEvaluationFn() "
def __init__(self,
minDx=-2.0, maxDx=2.0,
minDy=-2.0, maxDy=2.0,
minTheta1=0.0, maxTheta1=85.0,
minTheta2=0.0, maxTheta2=360.0,
):
self.dxEncoder = ScalarEncoder(5, minDx, maxDx, n=75, forced=True)
self.dyEncoder = ScalarEncoder(5, minDy, maxDy, n=75, forced=True)
self.externalSize = self.dxEncoder.getWidth()**2
self.externalOnBits = self.dxEncoder.w**2
self.theta1Encoder = ScalarEncoder(5, minTheta1, maxTheta1, n=75, forced=True)
self.theta2Encoder = ScalarEncoder(5, minTheta2, maxTheta2, n=75, forced=True)
self.bottomUpInputSize = self.theta1Encoder.getWidth()*self.theta2Encoder.getWidth()
self.bottomUpOnBits = self.theta1Encoder.w*self.theta2Encoder.w
self.minDx = 100.0
self.maxDx = -100.0
self.minTheta1 = minTheta1
self.minTheta2 = minTheta2
self.maxTheta1 = maxTheta1
self.maxTheta2 = maxTheta2
self.trainingIterations = 0
self.testIterations = 0
self.maxPredictionError = 0
self.totalPredictionError = 0
self.numMissedPredictions = 0
self.tm = TM(columnCount = self.bottomUpInputSize,
basalInputSize = self.externalSize,
cellsPerColumn=1,
initialPermanence=0.4,
connectedPermanence=0.5,
minThreshold= self.externalOnBits,
sampleSize=40,
permanenceIncrement=0.1,
permanenceDecrement=0.00,
activationThreshold=int(0.75*(self.externalOnBits+self.bottomUpOnBits)),
predictedSegmentDecrement=0.00,
checkInputs=False,
basalInputPrepend=True
)
print >>sys.stderr, "TM parameters:"
print >>sys.stderr, " num columns=",self.tm.numberOfColumns()
print >>sys.stderr, " activation threshold=",self.tm.getActivationThreshold()
print >>sys.stderr, " min threshold=",self.tm.getMinThreshold()
print >>sys.stderr, " basal input size=",self.tm.getBasalInputSize()
print >>sys.stderr
print >>sys.stderr
def testSettingRadiusWithMaxvalMinvalNone(self):
"""If radius when maxval/minval = None creates instance."""
encoder = ScalarEncoder(3,
None,
None,
name="scalar",
n=0,
radius=1.5,
resolution=0,
forced=True)
self.assertIsInstance(encoder, ScalarEncoder)
def testSettingScalarAndResolution(self):
"""Setting both scalar and resolution not allowed."""
with self.assertRaises(ValueError):
ScalarEncoder(3,
None,
None,
name="scalar",
n=0,
radius=None,
resolution=0.5,
forced=True)
def testDecoding(self):
s = ScalarEncoder(1,1,3,n=3, name='idx', forced=True)
v = VectorEncoder(3, s, typeCastFn=float)
data=[1,2,3]
enc = v.encode(data)
#decode
dec = v.decode(enc)
print "decoded=", dec
res= v.getData(dec)
self.assertEqual(data, res, "Decoded data not equal to original")
class ScalarBucketEncoder(Encoder):
def __init__(self):
self.encoder = NupicScalarEncoder(w=1,
minval=0,
maxval=40000,
n=22,
forced=True)
def encode(self, symbol):
encoding = self.encoder.encode(symbol)
return encoding
def testEncoding(self):
s = ScalarEncoder(1,1,3,n=3, name='idx', forced=True)
v = VectorEncoder(3, s, typeCastFn=float)
data=[1,2,3]
print "data=", data
# encode
enc = v.encode(data)
print "encoded=", enc
correct = [1,0,0,0,1,0,0,0,1]
self.assertTrue((enc==correct).all(), "Did not encode correctly")
def addEncoder(encoderAttr, offsetAttr):
protoVal = getattr(proto, encoderAttr)
if protoVal.n:
setattr(encoder, encoderAttr, ScalarEncoder.read(protoVal))
innerEncoder = getattr(encoder, encoderAttr)
setattr(encoder, offsetAttr, encoder.width)
innerOffset = getattr(encoder, offsetAttr)
encoder.width += innerEncoder.getWidth()
encoder.description.append((innerEncoder.name, innerOffset))
encoder.encoders.append((innerEncoder.name, innerEncoder, innerOffset))
else:
setattr(encoder, encoderAttr, None)
def __init__(self):
self.encoder = CoordinateEncoder(n=1024,
w=21)
self.motorEncoder = ScalarEncoder(21, -1, 1,
n=1024)
self.tm = MonitoredExtendedTemporalMemory(
columnDimensions=[2048],
basalInputDimensions: (999999,) # Dodge input checking.
cellsPerColumn=1,
initialPermanence=0.5,
connectedPermanence=0.6,
permanenceIncrement=0.1,
permanenceDecrement=0.02,
minThreshold=35,
activationThreshold=35,
maxNewSynapseCount=40)
self.plotter = Plotter(self.tm, showOverlaps=False, showOverlapsValues=False)
self.lastState = None
self.lastAction = None
self.prevMotorPattern = ()
def addEncoder(encoderAttr, offsetAttr):
protoVal = getattr(proto, encoderAttr)
if protoVal.n:
setattr(encoder, encoderAttr, ScalarEncoder.read(protoVal))
innerEncoder = getattr(encoder, encoderAttr)
setattr(encoder, offsetAttr, encoder.width)
innerOffset = getattr(encoder, offsetAttr)
encoder.width += innerEncoder.getWidth()
encoder.description.append((innerEncoder.name, innerOffset))
encoder.encoders.append((innerEncoder.name, innerEncoder, innerOffset))
else:
setattr(encoder, encoderAttr, None)
def __init__(self,
length=5,
minval=0,
maxval=100,
resolution=1,
name='vect'):
sc = ScalarEncoder(5,
minval,
maxval,
resolution=resolution,
name='idx')
super(SimpleVectorEncoder, self).__init__(length, sc, typeCastFn=float)
def __init__(self, filename="simple_pattern2.txt", with_classifier=True, delay=50, animate=True):
self.b = CHTMBrain(cells_per_region=self.CPR, min_overlap=1, r1_inputs=self.N_INPUTS)
self.b.initialize()
self.printer = CHTMPrinter(self.b)
self.printer.setup()
self.classifier = None
self.animate = animate
self.current_batch_target = 0
self.current_batch_counter = 0
self.delay = delay
if with_classifier:
self.classifier = CHTMClassifier(self.b, categories=self.ALPHA, region_index=len(self.CPR)-1, history_window=self.CROP_FILE/2)
if True:
self.encoder = SimpleFullWidthEncoder(n_inputs=self.N_INPUTS, n_cats=len(self.ALPHA))
else:
self.encoder = ScalarEncoder(n=self.N_INPUTS, w=5, minval=1, maxval=self.N_INPUTS, periodic=False, forced=True)
with open(self.DATA_DIR+"/"+filename, 'r') as myfile:
self.data = myfile.read()
self.cursor = 0
def read(cls, proto):
encoder = object.__new__(cls)
encoder.verbosity = proto.verbosity
encoder.minScaledValue = proto.minScaledValue
encoder.maxScaledValue = proto.maxScaledValue
encoder.clipInput = proto.clipInput
encoder.minval = proto.minval
encoder.maxval = proto.maxval
encoder.encoder = ScalarEncoder.read(proto.encoder)
encoder.name = proto.name
encoder.width = encoder.encoder.getWidth()
encoder.description = [(encoder.name, 0)]
encoder._bucketValues = None
return encoder
def read(cls, proto):
encoder = object.__new__(cls)
encoder.verbosity = proto.verbosity
encoder.encoder = ScalarEncoder.read(proto.encoder)
encoder.width = proto.width
encoder.description = [(proto.name, 0)]
encoder.name = proto.name
encoder.indexToCategory = {x.index: x.category
for x in proto.indexToCategory}
encoder.categoryToIndex = {category: index
for index, category
in encoder.indexToCategory.items()
if category != UNKNOWN}
encoder._topDownMappingM = None
encoder._bucketValues = None
return encoder
def __init__(self):
self.encoder = CoordinateEncoder(n=1024,
w=21)
self.motorEncoder = ScalarEncoder(21, -1, 1,
n=1024)
self.tm = MonitoredExtendedTemporalMemory(
columnDimensions=[2048],
cellsPerColumn=1,
initialPermanence=0.5,
connectedPermanence=0.6,
permanenceIncrement=0.1,
permanenceDecrement=0.02,
minThreshold=35,
activationThreshold=35,
maxNewSynapseCount=40)
self.plotter = Plotter(self.tm, showOverlaps=False, showOverlapsValues=False)
self.lastState = None
self.lastAction = None
def __init__(self, n, w, rate, chunk, minval=20, maxval=20000, name=None):
"""
@param n int the length of the encoded SDR
@param w int the number of 1s in the encoded SDR
@param rate int the number of sound samples per second
@param chunk int the number of samples in an input
@param minval float the lowest possible frequency detected
@param maxval float the highest possible frequency detected
@param name string the name of the encoder
"""
self.n = n
self.w = w
self.rate = rate
self.chunk = chunk
self.minval = minval
self.maxval = maxval
self.name = name
self._scalarEncoder = ScalarEncoder(name="scalar_"+str(name), n=n, w=w,
minval=minval, maxval=maxval)
def __init__(self):
self.encoder = CoordinateEncoder(n=1024,
w=21)
self.motorEncoder = ScalarEncoder(21, -1, 1,
n=1024)
self.tm = MonitoredApicalTiebreakPairMemory(
columnDimensions=[2048],
basalInputDimensions: (999999,) # Dodge input checking.
cellsPerColumn=1,
initialPermanence=0.5,
connectedPermanence=0.6,
permanenceIncrement=0.1,
permanenceDecrement=0.02,
minThreshold=35,
activationThreshold=35,
maxNewSynapseCount=40)
self.plotter = Plotter(self.tm, showOverlaps=False, showOverlapsValues=False)
self.lastState = None
self.lastAction = None
self.prevMotorPattern = ()
def __init__(self,
positions=range(36),
radius=3,
wrapAround=False,
nPerPosition=57,
wPerPosition=3,
minVal=0,
maxVal=1,
name=None,
verbosity=0):
"""
See `nupic.encoders.base.Encoder` for more information.
@param positions (list) Positions at which to encode distance
@param radius (int) Radius of positions over which to consider to get closest distance for encoding
@param wrapAround (bool) Whether radius should wrap around the sides of the input array
@param nPerPosition (int) Number of bits available for scalar encoder when encoding each position
@param wPerPosition (int) Number of bits active for scalar encoder when encoding each position
@param minVal (int) Minimum distance that can be encoded
@param maxVal (int) Maximum distance that can be encoded
"""
self.positions = positions
self.radius = radius
self.wrapAround = wrapAround
self.scalarEncoder = ScalarEncoder(wPerPosition, minVal, maxVal,
n=nPerPosition,
forced=True)
self.verbosity = verbosity
self.encoders = None
self.n = len(self.positions) * nPerPosition
self.w = len(self.positions) * wPerPosition
if name is None:
name = "[%s:%s]" % (self.n, self.w)
self.name = name
def setUp(self):
# use of forced is not recommended, but used here for readability, see
# scalar.py
self._l = ScalarEncoder(name="scalar", n=14, w=3, minval=1, maxval=8,
periodic=True, forced=True)
class ScalarEncoderTest(unittest.TestCase):
"""Unit tests for ScalarEncoder class"""
def setUp(self):
# use of forced is not recommended, but used here for readability, see
# scalar.py
self._l = ScalarEncoder(name="scalar", n=14, w=3, minval=1, maxval=8,
periodic=True, forced=True)
def testScalarEncoder(self):
"""Testing ScalarEncoder..."""
# -------------------------------------------------------------------------
# test missing values
mv = ScalarEncoder(name="mv", n=14, w=3, minval=1, maxval=8,
periodic=False, forced=True)
empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
self.assertEqual(empty.sum(), 0)
def testNaNs(self):
"""test NaNs"""
mv = ScalarEncoder(name="mv", n=14, w=3, minval=1, maxval=8,
periodic=False, forced=True)
empty = mv.encode(float("nan"))
self.assertEqual(empty.sum(), 0)
def testBottomUpEncodingPeriodicEncoder(self):
"""Test bottom-up encoding for a Periodic encoder"""
l = ScalarEncoder(n=14, w=3, minval=1, maxval=8, periodic=True,
forced=True)
self.assertEqual(l.getDescription(), [("[1:8]", 0)])
l = ScalarEncoder(name="scalar", n=14, w=3, minval=1, maxval=8,
periodic=True, forced=True)
self.assertEqual(l.getDescription(), [("scalar", 0)])
self.assertTrue(numpy.array_equal(
l.encode(3),
numpy.array([0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(l.encode(3.1), l.encode(3)))
self.assertTrue(numpy.array_equal(
l.encode(3.5),
numpy.array([0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(l.encode(3.6), l.encode(3.5)))
self.assertTrue(numpy.array_equal(l.encode(3.7), l.encode(3.5)))
self.assertTrue(numpy.array_equal(
l.encode(4),
numpy.array([0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(
l.encode(1),
numpy.array([1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(
l.encode(1.5),
numpy.array([1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(
l.encode(7),
numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(
l.encode(7.5),
numpy.array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1],
dtype=defaultDtype)))
self.assertEqual(l.resolution, 0.5)
self.assertEqual(l.radius, 1.5)
def testCreateResolution(self):
"""Test that we get the same encoder when we construct it using resolution
instead of n
"""
l = self._l
d = l.__dict__
l = ScalarEncoder(name="scalar", resolution=0.5, w=3, minval=1, maxval=8,
periodic=True, forced=True)
self.assertEqual(l.__dict__, d)
# Test that we get the same encoder when we construct it using radius
# instead of n
l = ScalarEncoder(name="scalar", radius=1.5, w=3, minval=1, maxval=8,
periodic=True, forced=True)
self.assertEqual(l.__dict__, d)
def testDecodeAndResolution(self):
"""Test the input description generation, top-down compute, and bucket
support on a periodic encoder
"""
l = self._l
v = l.minval
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
self.assertEqual(len(fieldNames), 1)
self.assertEqual(fieldNames, fieldsDict.keys())
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertLess(abs(rangeMin - v), l.resolution)
topDown = l.topDownCompute(output)[0]
self.assertTrue(numpy.array_equal(topDown.encoding, output))
self.assertLessEqual(abs(topDown.value - v), l.resolution / 2)
# Test bucket support
bucketIndices = l.getBucketIndices(v)
topDown = l.getBucketInfo(bucketIndices)[0]
self.assertLessEqual(abs(topDown.value - v), l.resolution / 2)
self.assertEqual(topDown.value, l.getBucketValues()[bucketIndices[0]])
self.assertEqual(topDown.scalar, topDown.value)
self.assertTrue(numpy.array_equal(topDown.encoding, output))
# Next value
v += l.resolution / 4
# -----------------------------------------------------------------------
# Test the input description generation on a large number, periodic encoder
l = ScalarEncoder(name='scalar', radius=1.5, w=3, minval=1, maxval=8,
periodic=True, forced=True)
# Test with a "hole"
decoded = l.decode(numpy.array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertTrue(numpy.array_equal(ranges[0], [7.5, 7.5]))
# Test with something wider than w, and with a hole, and wrapped
decoded = l.decode(numpy.array([1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 2)
self.assertTrue(numpy.array_equal(ranges[0], [7.5, 8]))
self.assertTrue(numpy.array_equal(ranges[1], [1, 1]))
# Test with something wider than w, no hole
decoded = l.decode(numpy.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertTrue(numpy.array_equal(ranges[0], [1.5, 2.5]))
# Test with 2 ranges
decoded = l.decode(numpy.array([1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 2)
self.assertTrue(numpy.array_equal(ranges[0], [1.5, 1.5]))
self.assertTrue(numpy.array_equal(ranges[1], [5.5, 6.0]))
# Test with 2 ranges, 1 of which is narrower than w
decoded = l.decode(numpy.array([0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertTrue(len(ranges), 2)
self.assertTrue(numpy.array_equal(ranges[0], [1.5, 1.5]))
self.assertTrue(numpy.array_equal(ranges[1], [5.5, 6.0]))
def testCloseness(self):
"""Test closenessScores for a periodic encoder"""
encoder = ScalarEncoder(w=7, minval=0, maxval=7, radius=1, periodic=True,
name="day of week", forced=True)
scores = encoder.closenessScores((2, 4, 7), (4, 2, 1), fractional=False)
for actual, score in itertools.izip((2, 2, 1), scores):
self.assertEqual(actual, score)
def testNonPeriodicBottomUp(self):
"""Test Non-periodic encoder bottom-up"""
l = ScalarEncoder(name="scalar", n=14, w=5, minval=1, maxval=10,
periodic=False, forced=True)
self.assertTrue(numpy.array_equal(
l.encode(1),
numpy.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(
l.encode(2),
numpy.array([0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(
l.encode(10),
numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
dtype=defaultDtype)))
# Test that we get the same encoder when we construct it using resolution
# instead of n
d = l.__dict__
l = ScalarEncoder(name="scalar", resolution=1, w=5, minval=1, maxval=10,
periodic=False, forced=True)
self.assertEqual(l.__dict__, d)
# Test that we get the same encoder when we construct it using radius
# instead of n
l = ScalarEncoder(name="scalar", radius=5, w=5, minval=1, maxval=10,
periodic=False, forced=True)
self.assertEqual(l.__dict__, d)
# -------------------------------------------------------------------------
# Test the input description generation and topDown decoding of a
# non-periodic encoder
v = l.minval
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertLess(abs(rangeMin - v), l.resolution)
topDown = l.topDownCompute(output)[0]
self.assertTrue(numpy.array_equal(topDown.encoding, output))
self.assertLessEqual(abs(topDown.value - v), l.resolution)
# Test bucket support
bucketIndices = l.getBucketIndices(v)
topDown = l.getBucketInfo(bucketIndices)[0]
self.assertLessEqual(abs(topDown.value - v), l.resolution / 2)
self.assertEqual(topDown.scalar, topDown.value)
self.assertTrue(numpy.array_equal(topDown.encoding, output))
# Next value
v += l.resolution / 4
# Make sure we can fill in holes
decoded = l.decode(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1]))
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertTrue(numpy.array_equal(ranges[0], [10, 10]))
decoded = l.decode(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1]))
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertTrue(numpy.array_equal(ranges[0], [10, 10]))
#Test min and max
l = ScalarEncoder(name="scalar", n=14, w=3, minval=1, maxval=10,
periodic=False, forced=True)
decoded = l.topDownCompute(
numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]))[0]
self.assertEqual(decoded.value, 10)
decoded = l.topDownCompute(
numpy.array([1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]))[0]
self.assertEqual(decoded.value, 1)
#Make sure only the last and first encoding encodes to max and min, and
#there is no value greater than max or min
l = ScalarEncoder(name="scalar", n=140, w=3, minval=1, maxval=141,
periodic=False, forced=True)
for i in range(137):
iterlist = [0 for _ in range(140)]
for j in range(i, i+3):
iterlist[j] =1
npar = numpy.array(iterlist)
decoded = l.topDownCompute(npar)[0]
self.assertLessEqual(decoded.value, 141)
self.assertGreaterEqual(decoded.value, 1)
self.assertTrue(decoded.value < 141 or i==137)
self.assertTrue(decoded.value > 1 or i == 0)
# -------------------------------------------------------------------------
# Test the input description generation and top-down compute on a small
# number non-periodic encoder
l = ScalarEncoder(name="scalar", n=15, w=3, minval=.001, maxval=.002,
periodic=False, forced=True)
v = l.minval
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertLess(abs(rangeMin - v), l.resolution)
topDown = l.topDownCompute(output)[0].value
self.assertLessEqual(abs(topDown - v), l.resolution / 2)
v += l.resolution / 4
# -------------------------------------------------------------------------
# Test the input description generation on a large number, non-periodic
# encoder
l = ScalarEncoder(name="scalar", n=15, w=3, minval=1, maxval=1000000000,
periodic=False, forced=True)
v = l.minval
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertLess(abs(rangeMin - v), l.resolution)
topDown = l.topDownCompute(output)[0].value
self.assertLessEqual(abs(topDown - v), l.resolution / 2)
v += l.resolution / 4
def testEncodeInvalidInputType(self):
encoder = ScalarEncoder(name="enc", n=14, w=3, minval=1, maxval=8,
periodic=False, forced=True)
with self.assertRaises(TypeError):
encoder.encode("String")
def testGetBucketInfoIntResolution(self):
"""Ensures that passing resolution as an int doesn't truncate values."""
encoder = ScalarEncoder(w=3, resolution=1, minval=1, maxval=8,
periodic=True, forced=True)
self.assertEqual(4.5,
encoder.topDownCompute(encoder.encode(4.5))[0].scalar)
@unittest.skipUnless(
capnp, "pycapnp is not installed, skipping serialization test.")
def testReadWrite(self):
"""Test ScalarEncoder Cap'n Proto serialization implementation."""
originalValue = self._l.encode(1)
proto1 = ScalarEncoderProto.new_message()
self._l.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = ScalarEncoderProto.read(f)
encoder = ScalarEncoder.read(proto2)
self.assertIsInstance(encoder, ScalarEncoder)
self.assertEqual(encoder.w, self._l.w)
self.assertEqual(encoder.minval, self._l.minval)
self.assertEqual(encoder.maxval, self._l.maxval)
self.assertEqual(encoder.periodic, self._l.periodic)
self.assertEqual(encoder.n, self._l.n)
self.assertEqual(encoder.radius, self._l.radius)
self.assertEqual(encoder.resolution, self._l.resolution)
self.assertEqual(encoder.name, self._l.name)
self.assertEqual(encoder.verbosity, self._l.verbosity)
self.assertEqual(encoder.clipInput, self._l.clipInput)
self.assertTrue(numpy.array_equal(encoder.encode(1), originalValue))
self.assertEqual(self._l.decode(encoder.encode(1)),
encoder.decode(self._l.encode(1)))
# Feed in a new value and ensure the encodings match
result1 = self._l.encode(7)
result2 = encoder.encode(7)
self.assertTrue(numpy.array_equal(result1, result2))
def testSettingNWithMaxvalMinvalNone(self):
"""Setting n when maxval/minval = None creates instance."""
encoder = ScalarEncoder(3, None, None, name="scalar",
n=14, radius=0, resolution=0, forced=True)
self.assertIsInstance(encoder, ScalarEncoder)
def testSettingScalarAndResolution(self):
"""Setting both scalar and resolution not allowed."""
with self.assertRaises(ValueError):
ScalarEncoder(3, None, None, name="scalar", n=0, radius=None,
resolution=0.5, forced=True)
def testSettingRadiusWithMaxvalMinvalNone(self):
"""If radius when maxval/minval = None creates instance."""
encoder = ScalarEncoder(3, None, None, name="scalar",
n=0, radius=1.5, resolution=0, forced=True)
self.assertIsInstance(encoder, ScalarEncoder)
class ScalarEncoderTest(unittest.TestCase):
"""Unit tests for ScalarEncoder class"""
def setUp(self):
# use of forced is not recommended, but used here for readability, see scalar.py
self._l = ScalarEncoder(name='scalar', n=14, w=3, minval=1, maxval=8, periodic=True, forced=True)
############################################################################
def testScalarEncoder(self):
"""Testing ScalarEncoder..."""
# -------------------------------------------------------------------------
# test missing values
mv = ScalarEncoder(name='mv', n=14, w=3, minval=1, maxval=8, periodic=False, forced=True)
empty = mv.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
print "\nEncoded missing data \'None\' as %s" % empty
self.assertEqual(empty.sum(), 0)
# --------------------------------------------------------------------
def testNaNs(self):
"""test NaNs"""
mv = ScalarEncoder(name='mv', n=14, w=3, minval=1, maxval=8, periodic=False, forced=True)
empty = mv.encode(float("nan"))
print "\nEncoded missing data \'None\' as %s" % empty
self.assertEqual(empty.sum(), 0)
# ------------------------------------------------------------------------
def testBottomUpEncodingPeriodicEncoder(self):
"""Test bottom-up encoding for a Periodic encoder"""
l = ScalarEncoder(n=14, w=3, minval=1, maxval=8, periodic=True, forced=True)
self.assertEqual(l.getDescription(), [("[1:8]", 0)])
l = ScalarEncoder(name='scalar', n=14, w=3, minval=1, maxval=8, periodic=True, forced=True)
self.assertEqual(l.getDescription(), [("scalar", 0)])
self.assertTrue((l.encode(3) == numpy.array([0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(3.1) == l.encode(3)).all())
self.assertTrue((l.encode(3.5) == numpy.array([0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(3.6) == l.encode(3.5)).all())
self.assertTrue((l.encode(3.7) == l.encode(3.5)).all())
self.assertTrue((l.encode(4) == numpy.array([0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(1) == numpy.array([1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(1.5) == numpy.array([1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(7) == numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(7.5) == numpy.array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1],
dtype=defaultDtype)).all())
self.assertEqual(l.resolution, 0.5)
self.assertEqual(l.radius, 1.5)
# Test that we get the same encoder when we construct it using resolution
# instead of n
def testCreateResolution(self):
"""Test that we get the same encoder when we construct it using resolution instead of n"""
l = self._l
d = l.__dict__
l = ScalarEncoder(name='scalar', resolution=0.5, w=3, minval=1, maxval=8,
periodic=True, forced=True)
self.assertEqual(l.__dict__, d)
# Test that we get the same encoder when we construct it using radius
# instead of n
l = ScalarEncoder(name='scalar', radius=1.5, w=3, minval=1, maxval=8,
periodic=True, forced=True)
self.assertEqual(l.__dict__, d)
# -------------------------------------------------------------------------
# Test the input description generation, top-down compute, and bucket
# support on a periodic encoder
def testDecodeAndResolution(self):
"""Testing periodic encoder decoding, resolution of """
l = self._l
print l.resolution
v = l.minval
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
print "decoding", output, "(%f)=>" % v, l.decodedToStr(decoded)
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertTrue(abs(rangeMin - v) < l.resolution)
topDown = l.topDownCompute(output)[0]
print "topdown =>", topDown
self.assertTrue((topDown.encoding == output).all())
self.assertTrue(abs(topDown.value - v) <= l.resolution / 2)
# Test bucket support
bucketIndices = l.getBucketIndices(v)
print "bucket index =>", bucketIndices[0]
topDown = l.getBucketInfo(bucketIndices)[0]
self.assertTrue(abs(topDown.value - v) <= l.resolution / 2)
self.assertEqual(topDown.value, l.getBucketValues()[bucketIndices[0]])
self.assertEqual(topDown.scalar, topDown.value)
self.assertTrue((topDown.encoding == output).all())
# Next value
v += l.resolution / 4
# -----------------------------------------------------------------------
# Test the input description generation on a large number, periodic encoder
l = ScalarEncoder(name='scalar', radius=1.5, w=3, minval=1, maxval=8,
periodic=True, forced=True)
print "\nTesting periodic encoder decoding, resolution of %f..." % \
l.resolution
# Test with a "hole"
decoded = l.decode(numpy.array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertTrue(len(ranges) == 1 and numpy.array_equal(ranges[0], [7.5, 7.5]))
print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded)
# Test with something wider than w, and with a hole, and wrapped
decoded = l.decode(numpy.array([1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertTrue(len(ranges) == 2 and numpy.array_equal(ranges[0], [7.5, 8]) \
and numpy.array_equal(ranges[1], [1, 1]))
print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded)
# Test with something wider than w, no hole
decoded = l.decode(numpy.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertTrue(len(ranges) == 1 and numpy.array_equal(ranges[0], [1.5, 2.5]))
print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded)
# Test with 2 ranges
decoded = l.decode(numpy.array([1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertTrue(len(ranges) == 2 and numpy.array_equal(ranges[0], [1.5, 1.5]) \
and numpy.array_equal(ranges[1], [5.5, 6.0]))
print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded)
# Test with 2 ranges, 1 of which is narrower than w
decoded = l.decode(numpy.array([0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertTrue(len(ranges) == 2 and numpy.array_equal(ranges[0], [1.5, 1.5]) \
and numpy.array_equal(ranges[1], [5.5, 6.0]))
print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded)
# ============================================================================
def testCloseness(self):
"""Test closenessScores for a periodic encoder"""
encoder = ScalarEncoder(w=7, minval=0, maxval=7, radius=1, periodic=True,
name="day of week", forced=True)
scores = encoder.closenessScores((2, 4, 7), (4, 2, 1), fractional=False)
for actual, score in itertools.izip((2, 2, 1), scores):
self.assertEqual(actual, score)
# ============================================================================
def testNonPeriodicBottomUp(self):
"""Test Non-periodic encoder bottom-up"""
l = ScalarEncoder(name='scalar', n=14, w=5, minval=1, maxval=10, periodic=False, forced=True)
print "\nTesting non-periodic encoder encoding, resolution of %f..." % \
l.resolution
self.assertTrue((l.encode(1) == numpy.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(2) == numpy.array([0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(10) == numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
dtype=defaultDtype)).all())
# Test that we get the same encoder when we construct it using resolution
# instead of n
d = l.__dict__
l = ScalarEncoder(name='scalar', resolution=1, w=5, minval=1, maxval=10,
periodic=False, forced=True)
self.assertEqual(l.__dict__, d)
# Test that we get the same encoder when we construct it using radius
# instead of n
l = ScalarEncoder(name='scalar', radius=5, w=5, minval=1, maxval=10, periodic=False, forced=True)
self.assertEqual(l.__dict__, d)
# -------------------------------------------------------------------------
# Test the input description generation and topDown decoding of a non-periodic
# encoder
v = l.minval
print "\nTesting non-periodic encoder decoding, resolution of %f..." % \
l.resolution
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
print "decoding", output, "(%f)=>" % v, l.decodedToStr(decoded)
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertTrue(abs(rangeMin - v) < l.resolution)
topDown = l.topDownCompute(output)[0]
print "topdown =>", topDown
self.assertTrue((topDown.encoding == output).all())
self.assertTrue(abs(topDown.value - v) <= l.resolution)
# Test bucket support
bucketIndices = l.getBucketIndices(v)
print "bucket index =>", bucketIndices[0]
topDown = l.getBucketInfo(bucketIndices)[0]
self.assertTrue(abs(topDown.value - v) <= l.resolution / 2)
self.assertEqual(topDown.scalar, topDown.value)
self.assertTrue((topDown.encoding == output).all())
# Next value
v += l.resolution / 4
# Make sure we can fill in holes
decoded = l.decode(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertTrue(len(ranges) == 1 and numpy.array_equal(ranges[0], [10, 10]))
print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded)
decoded = l.decode(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertTrue(len(ranges) == 1 and numpy.array_equal(ranges[0], [10, 10]))
print "decodedToStr of", ranges, "=>", l.decodedToStr(decoded)
#Test min and max
l = ScalarEncoder(name='scalar', n=14, w=3, minval=1, maxval=10, periodic=False, forced=True)
decoded = l.topDownCompute(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]))[0]
self.assertEqual(decoded.value, 10)
decoded = l.topDownCompute(numpy.array([1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]))[0]
self.assertEqual(decoded.value, 1)
#Make sure only the last and first encoding encodes to max and min, and there is no value greater than max or min
l = ScalarEncoder(name='scalar', n=140, w=3, minval=1, maxval=141, periodic=False, forced=True)
for i in range(137):
iterlist = [0 for _ in range(140)]
for j in range(i, i+3):
iterlist[j] =1
npar = numpy.array(iterlist)
decoded = l.topDownCompute(npar)[0]
self.assertTrue(decoded.value <= 141)
self.assertTrue(decoded.value >= 1)
self.assertTrue(decoded.value < 141 or i==137)
self.assertTrue(decoded.value > 1 or i == 0)
# -------------------------------------------------------------------------
# Test the input description generation and top-down compute on a small number
# non-periodic encoder
l = ScalarEncoder(name='scalar', n=15, w=3, minval=.001, maxval=.002,
periodic=False, forced=True)
print "\nTesting non-periodic encoder decoding, resolution of %f..." % \
l.resolution
v = l.minval
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
print "decoding", output, "(%f)=>" % v, l.decodedToStr(decoded)
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertTrue(abs(rangeMin - v) < l.resolution)
topDown = l.topDownCompute(output)[0].value
print "topdown =>", topDown
self.assertTrue(abs(topDown - v) <= l.resolution / 2)
v += l.resolution / 4
# -------------------------------------------------------------------------
# Test the input description generation on a large number, non-periodic encoder
l = ScalarEncoder(name='scalar', n=15, w=3, minval=1, maxval=1000000000,
periodic=False, forced=True)
print "\nTesting non-periodic encoder decoding, resolution of %f..." % \
l.resolution
v = l.minval
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
print "decoding", output, "(%f)=>" % v, l.decodedToStr(decoded)
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, desc) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertTrue(abs(rangeMin - v) < l.resolution)
topDown = l.topDownCompute(output)[0].value
print "topdown =>", topDown
self.assertTrue(abs(topDown - v) <= l.resolution / 2)
v += l.resolution / 4
# -------------------------------------------------------------------------
# Test setting fieldStats after initialization
if False:
#TODO: remove all this? (and fieldstats from ScalarEncoder (if applicable) )?
# Modified on 11/20/12 12:53 PM - setFieldStats not applicable for ScalarEncoder
l = ScalarEncoder(n=14, w=3, minval=100, maxval=800, periodic=True, forced=True)
l.setFieldStats("this", {"this":{"min":1, "max":8}})
l = ScalarEncoder(name='scalar', n=14, w=3, minval=100, maxval=800, periodic=True, forced=True)
l.setFieldStats("this", {"this":{"min":1, "max":8}})
self.assertTrue((l.encode(3) == numpy.array([0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(3.1) == l.encode(3)).all())
self.assertTrue((l.encode(3.5) == numpy.array([0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(3.6) == l.encode(3.5)).all())
self.assertTrue((l.encode(3.7) == l.encode(3.5)).all())
self.assertTrue((l.encode(4) == numpy.array([0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(1) == numpy.array([1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(1.5) == numpy.array([1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(7) == numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(7.5) == numpy.array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1],
dtype=defaultDtype)).all())
l = ScalarEncoder(name='scalar', n=14, w=5, minval=100, maxval=1000, periodic=False, forced=True)
l.setFieldStats("this", {"this":{"min":1, "max":10}})
print "\nTesting non-periodic encoding using setFieldStats, resolution of %f..." % \
l.resolution
self.assertTrue((l.encode(1) == numpy.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(2) == numpy.array([0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)).all())
self.assertTrue((l.encode(10) == numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
dtype=defaultDtype)).all())
# ============================================================================
def testEncodeInvalidInputType(self):
encoder = ScalarEncoder(name='enc', n=14, w=3, minval=1, maxval=8,
periodic=False, forced=True)
with self.assertRaises(TypeError):
encoder.encode("String")
# ============================================================================
def testGetBucketInfoIntResolution(self):
"""Ensures that passing resolution as an int doesn't truncate values."""
encoder = ScalarEncoder(w=3, resolution=1, minval=1, maxval=8,
periodic=True, forced=True)
self.assertEqual(4.5, encoder.topDownCompute(encoder.encode(4.5))[0].scalar)
def testReadWrite(self):
"""Test ScalarEncoder Cap'n Proto serialization implementation."""
originalValue = self._l.encode(1)
proto1 = ScalarEncoderProto.new_message()
self._l.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = ScalarEncoderProto.read(f)
encoder = ScalarEncoder.read(proto2)
self.assertIsInstance(encoder, ScalarEncoder)
self.assertEqual(encoder.w, self._l.w)
self.assertEqual(encoder.minval, self._l.minval)
self.assertEqual(encoder.maxval, self._l.maxval)
self.assertEqual(encoder.periodic, self._l.periodic)
self.assertEqual(encoder.n, self._l.n)
self.assertEqual(encoder.radius, self._l.radius)
self.assertEqual(encoder.resolution, self._l.resolution)
self.assertEqual(encoder.name, self._l.name)
self.assertEqual(encoder.verbosity, self._l.verbosity)
self.assertEqual(encoder.clipInput, self._l.clipInput)
self.assertTrue(numpy.array_equal(encoder.encode(1), originalValue))
self.assertEqual(self._l.decode(encoder.encode(1)),
encoder.decode(self._l.encode(1)))
# Feed in a new value and ensure the encodings match
result1 = self._l.encode(7)
result2 = encoder.encode(7)
self.assertTrue(numpy.array_equal(result1, result2))
def testGetBucketInfoIntResolution(self):
"""Ensures that passing resolution as an int doesn't truncate values."""
encoder = ScalarEncoder(w=3, resolution=1, minval=1, maxval=8,
periodic=True, forced=True)
self.assertEqual(4.5,
encoder.topDownCompute(encoder.encode(4.5))[0].scalar)
def testEncodeInvalidInputType(self):
encoder = ScalarEncoder(name="enc", n=14, w=3, minval=1, maxval=8,
periodic=False, forced=True)
with self.assertRaises(TypeError):
encoder.encode("String")
def testNonPeriodicBottomUp(self):
"""Test Non-periodic encoder bottom-up"""
l = ScalarEncoder(name="scalar", n=14, w=5, minval=1, maxval=10,
periodic=False, forced=True)
self.assertTrue(numpy.array_equal(
l.encode(1),
numpy.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(
l.encode(2),
numpy.array([0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(
l.encode(10),
numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
dtype=defaultDtype)))
# Test that we get the same encoder when we construct it using resolution
# instead of n
d = l.__dict__
l = ScalarEncoder(name="scalar", resolution=1, w=5, minval=1, maxval=10,
periodic=False, forced=True)
self.assertEqual(l.__dict__, d)
# Test that we get the same encoder when we construct it using radius
# instead of n
l = ScalarEncoder(name="scalar", radius=5, w=5, minval=1, maxval=10,
periodic=False, forced=True)
self.assertEqual(l.__dict__, d)
# -------------------------------------------------------------------------
# Test the input description generation and topDown decoding of a
# non-periodic encoder
v = l.minval
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertLess(abs(rangeMin - v), l.resolution)
topDown = l.topDownCompute(output)[0]
self.assertTrue(numpy.array_equal(topDown.encoding, output))
self.assertLessEqual(abs(topDown.value - v), l.resolution)
# Test bucket support
bucketIndices = l.getBucketIndices(v)
topDown = l.getBucketInfo(bucketIndices)[0]
self.assertLessEqual(abs(topDown.value - v), l.resolution / 2)
self.assertEqual(topDown.scalar, topDown.value)
self.assertTrue(numpy.array_equal(topDown.encoding, output))
# Next value
v += l.resolution / 4
# Make sure we can fill in holes
decoded = l.decode(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1]))
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertTrue(numpy.array_equal(ranges[0], [10, 10]))
decoded = l.decode(numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1]))
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertTrue(numpy.array_equal(ranges[0], [10, 10]))
#Test min and max
l = ScalarEncoder(name="scalar", n=14, w=3, minval=1, maxval=10,
periodic=False, forced=True)
decoded = l.topDownCompute(
numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]))[0]
self.assertEqual(decoded.value, 10)
decoded = l.topDownCompute(
numpy.array([1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]))[0]
self.assertEqual(decoded.value, 1)
#Make sure only the last and first encoding encodes to max and min, and
#there is no value greater than max or min
l = ScalarEncoder(name="scalar", n=140, w=3, minval=1, maxval=141,
periodic=False, forced=True)
for i in range(137):
iterlist = [0 for _ in range(140)]
for j in range(i, i+3):
iterlist[j] =1
npar = numpy.array(iterlist)
decoded = l.topDownCompute(npar)[0]
self.assertLessEqual(decoded.value, 141)
self.assertGreaterEqual(decoded.value, 1)
self.assertTrue(decoded.value < 141 or i==137)
self.assertTrue(decoded.value > 1 or i == 0)
# -------------------------------------------------------------------------
# Test the input description generation and top-down compute on a small
# number non-periodic encoder
l = ScalarEncoder(name="scalar", n=15, w=3, minval=.001, maxval=.002,
periodic=False, forced=True)
v = l.minval
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertLess(abs(rangeMin - v), l.resolution)
topDown = l.topDownCompute(output)[0].value
self.assertLessEqual(abs(topDown - v), l.resolution / 2)
v += l.resolution / 4
# -------------------------------------------------------------------------
# Test the input description generation on a large number, non-periodic
# encoder
l = ScalarEncoder(name="scalar", n=15, w=3, minval=1, maxval=1000000000,
periodic=False, forced=True)
v = l.minval
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
(fieldsDict, _) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertLess(abs(rangeMin - v), l.resolution)
topDown = l.topDownCompute(output)[0].value
self.assertLessEqual(abs(topDown - v), l.resolution / 2)
v += l.resolution / 4
def testNaNs(self):
"""test NaNs"""
mv = ScalarEncoder(name='mv', n=14, w=3, minval=1, maxval=8, periodic=False, forced=True)
empty = mv.encode(float("nan"))
print "\nEncoded missing data \'None\' as %s" % empty
self.assertEqual(empty.sum(), 0)
def testBottomUpEncodingPeriodicEncoder(self):
"""Test bottom-up encoding for a Periodic encoder"""
l = ScalarEncoder(n=14, w=3, minval=1, maxval=8, periodic=True,
forced=True)
self.assertEqual(l.getDescription(), [("[1:8]", 0)])
l = ScalarEncoder(name="scalar", n=14, w=3, minval=1, maxval=8,
periodic=True, forced=True)
self.assertEqual(l.getDescription(), [("scalar", 0)])
self.assertTrue(numpy.array_equal(
l.encode(3),
numpy.array([0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(l.encode(3.1), l.encode(3)))
self.assertTrue(numpy.array_equal(
l.encode(3.5),
numpy.array([0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(l.encode(3.6), l.encode(3.5)))
self.assertTrue(numpy.array_equal(l.encode(3.7), l.encode(3.5)))
self.assertTrue(numpy.array_equal(
l.encode(4),
numpy.array([0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(
l.encode(1),
numpy.array([1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(
l.encode(1.5),
numpy.array([1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(
l.encode(7),
numpy.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1],
dtype=defaultDtype)))
self.assertTrue(numpy.array_equal(
l.encode(7.5),
numpy.array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1],
dtype=defaultDtype)))
self.assertEqual(l.resolution, 0.5)
self.assertEqual(l.radius, 1.5)
def testNaNs(self):
"""test NaNs"""
mv = ScalarEncoder(name="mv", n=14, w=3, minval=1, maxval=8,
periodic=False, forced=True)
empty = mv.encode(float("nan"))
self.assertEqual(empty.sum(), 0)
def __init__(self): self.encoder = NupicScalarEncoder(w=1, minval=0, maxval=40000, n=22, forced=True)
def generateInputVectors(self, params):
if params['dataType'] == 'randomSDR':
self._inputVectors = generateRandomSDR(
params['numInputVectors'],
params['inputSize'],
params['numActiveInputBits'],
params['seed'])
elif params['dataType'] == 'denseVectors':
self._inputVectors = generateDenseVectors(
params['numInputVectors'],
params['inputSize'],
params['seed'])
elif params['dataType'] == 'randomBarPairs':
inputSize = params['nX'] * params['nY']
numInputVectors = params['numInputVectors']
self._inputVectors = np.zeros((numInputVectors, inputSize),
dtype=uintType)
for i in range(numInputVectors):
bar1 = getRandomBar((params['nX'], params['nY']),
params['barHalfLength'], 'horizontal')
bar2 = getRandomBar((params['nX'], params['nY']),
params['barHalfLength'], 'vertical')
data = bar1 + bar2
data[data > 0] = 1
self._inputVectors[i, :] = np.reshape(data, newshape=(1, inputSize))
elif params['dataType'] == 'randomBarSets':
inputSize = params['nX'] * params['nY']
numInputVectors = params['numInputVectors']
self._inputVectors = np.zeros((numInputVectors, inputSize),
dtype=uintType)
for i in range(numInputVectors):
data = 0
for barI in range(params['numBarsPerInput']):
orientation = np.random.choice(['horizontal', 'vertical'])
bar = getRandomBar((params['nX'], params['nY']),
params['barHalfLength'], orientation)
data += bar
data[data > 0] = 1
self._inputVectors[i, :] = np.reshape(data, newshape=(1, inputSize))
elif params['dataType'] == 'randomCross':
inputSize = params['nX'] * params['nY']
numInputVectors = params['numInputVectors']
self._inputVectors = np.zeros((numInputVectors, inputSize),
dtype=uintType)
for i in range(numInputVectors):
data = getCross(params['nX'], params['nY'], params['barHalfLength'])
self._inputVectors[i, :] = np.reshape(data, newshape=(1, inputSize))
elif params['dataType'] == 'correlatedSDRPairs':
(inputVectors, inputVectors1, inputVectors2, corrPairs) = \
generateCorrelatedSDRPairs(
params['numInputVectors'],
params['inputSize'],
params['numInputVectorPerSensor'],
params['numActiveInputBits'],
params['corrStrength'],
params['seed'])
self._inputVectors = inputVectors
self._additionalInfo = {"inputVectors1": inputVectors1,
"inputVectors2": inputVectors2,
"corrPairs": corrPairs}
elif params['dataType'] == 'nyc_taxi':
from nupic.encoders.scalar import ScalarEncoder
df = pd.read_csv('./data/nyc_taxi.csv', header=0, skiprows=[1, 2])
inputVectors = np.zeros((5000, params['n']))
for i in range(5000):
inputRecord = {
"passenger_count": float(df["passenger_count"][i]),
"timeofday": float(df["timeofday"][i]),
"dayofweek": float(df["dayofweek"][i]),
}
enc = ScalarEncoder(w=params['w'],
minval=params['minval'],
maxval=params['maxval'],
n=params['n'])
inputSDR = enc.encode(inputRecord["passenger_count"])
inputVectors[i, :] = inputSDR
self._inputVectors = inputVectors
def testDecodeAndResolution(self):
"""Test the input description generation, top-down compute, and bucket
support on a periodic encoder
"""
l = self._l
v = l.minval
while v < l.maxval:
output = l.encode(v)
decoded = l.decode(output)
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
self.assertEqual(len(fieldNames), 1)
self.assertEqual(fieldNames, fieldsDict.keys())
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
(rangeMin, rangeMax) = ranges[0]
self.assertEqual(rangeMin, rangeMax)
self.assertLess(abs(rangeMin - v), l.resolution)
topDown = l.topDownCompute(output)[0]
self.assertTrue(numpy.array_equal(topDown.encoding, output))
self.assertLessEqual(abs(topDown.value - v), l.resolution / 2)
# Test bucket support
bucketIndices = l.getBucketIndices(v)
topDown = l.getBucketInfo(bucketIndices)[0]
self.assertLessEqual(abs(topDown.value - v), l.resolution / 2)
self.assertEqual(topDown.value, l.getBucketValues()[bucketIndices[0]])
self.assertEqual(topDown.scalar, topDown.value)
self.assertTrue(numpy.array_equal(topDown.encoding, output))
# Next value
v += l.resolution / 4
# -----------------------------------------------------------------------
# Test the input description generation on a large number, periodic encoder
l = ScalarEncoder(name='scalar', radius=1.5, w=3, minval=1, maxval=8,
periodic=True, forced=True)
# Test with a "hole"
decoded = l.decode(numpy.array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertTrue(numpy.array_equal(ranges[0], [7.5, 7.5]))
# Test with something wider than w, and with a hole, and wrapped
decoded = l.decode(numpy.array([1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 2)
self.assertTrue(numpy.array_equal(ranges[0], [7.5, 8]))
self.assertTrue(numpy.array_equal(ranges[1], [1, 1]))
# Test with something wider than w, no hole
decoded = l.decode(numpy.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 1)
self.assertTrue(numpy.array_equal(ranges[0], [1.5, 2.5]))
# Test with 2 ranges
decoded = l.decode(numpy.array([1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertEqual(len(ranges), 2)
self.assertTrue(numpy.array_equal(ranges[0], [1.5, 1.5]))
self.assertTrue(numpy.array_equal(ranges[1], [5.5, 6.0]))
# Test with 2 ranges, 1 of which is narrower than w
decoded = l.decode(numpy.array([0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0]))
(fieldsDict, fieldNames) = decoded
self.assertEqual(len(fieldsDict), 1)
(ranges, _) = fieldsDict.values()[0]
self.assertTrue(len(ranges), 2)
self.assertTrue(numpy.array_equal(ranges[0], [1.5, 1.5]))
self.assertTrue(numpy.array_equal(ranges[1], [5.5, 6.0]))