def testEncoding(self):
"""
Test basic encoding functionality. Create encodings without crashing and
check they contain the correct number of on and off bits. Check some
encodings for expected overlap. Test that encodings for old values don't
change once we generate new buckets.
"""
# Initialize with non-default parameters and encode with a number close to
# the offset
encoder = RandomDistributedScalarEncoder(name="encoder",
resolution=1.0,
w=23,
n=500,
offset=0.0)
e0 = encoder.encode(-0.1)
self.assertEqual(e0.sum(), 23, "Number of on bits is incorrect")
self.assertEqual(e0.size, 500, "Width of the vector is incorrect")
self.assertEqual(
encoder.getBucketIndices(0.0)[0], encoder._maxBuckets / 2,
"Offset doesn't correspond to middle bucket")
self.assertEqual(len(encoder.bucketMap), 1,
"Number of buckets is not 1")
# Encode with a number that is resolution away from offset. Now we should
# have two buckets and this encoding should be one bit away from e0
e1 = encoder.encode(1.0)
self.assertEqual(len(encoder.bucketMap), 2,
"Number of buckets is not 2")
self.assertEqual(e1.sum(), 23, "Number of on bits is incorrect")
self.assertEqual(e1.size, 500, "Width of the vector is incorrect")
self.assertEqual(computeOverlap(e0, e1), 22,
"Overlap is not equal to w-1")
# Encode with a number that is resolution*w away from offset. Now we should
# have many buckets and this encoding should have very little overlap with
# e0
e25 = encoder.encode(25.0)
self.assertGreater(len(encoder.bucketMap), 23,
"Number of buckets is not 2")
self.assertEqual(e25.sum(), 23, "Number of on bits is incorrect")
self.assertEqual(e25.size, 500, "Width of the vector is incorrect")
self.assertLess(computeOverlap(e0, e25), 4, "Overlap is too high")
# Test encoding consistency. The encodings for previous numbers
# shouldn't change even though we have added additional buckets
self.assertTrue(
numpy.array_equal(e0, encoder.encode(-0.1)),
"Encodings are not consistent - they have changed after new buckets "
"have been created")
self.assertTrue(
numpy.array_equal(e1, encoder.encode(1.0)),
"Encodings are not consistent - they have changed after new buckets "
"have been created")
def testMissingValues(self):
"""
Test that missing values and NaN return all zero's.
"""
encoder = RandomDistributedScalarEncoder(name="encoder",
resolution=1.0)
empty = encoder.encode(SENTINEL_VALUE_FOR_MISSING_DATA)
self.assertEqual(empty.sum(), 0)
empty = encoder.encode(float("nan"))
self.assertEqual(empty.sum(), 0)
def calculateEncoderModelAccuracy(nBuckets, numCols, w, trainData, trainLabel):
maxValue = np.max(trainData)
minValue = np.min(trainData)
resolution = (maxValue - minValue) / nBuckets
encoder = RandomDistributedScalarEncoder(resolution, w=w, n=numCols)
activeColumnsTrain = runEncoderOverDataset(encoder, trainData)
distMatColumnTrain = calculateDistanceMatTrain(activeColumnsTrain)
meanAccuracy, outcomeColumn = calculateAccuracy(distMatColumnTrain,
trainLabel, trainLabel)
accuracyColumnOnly = np.mean(outcomeColumn)
return accuracyColumnOnly
def testOverlapOK(self):
"""
Test that the internal method _overlapOK works as expected.
"""
# Create a fake set of encodings.
encoder = RandomDistributedScalarEncoder(name="encoder",
resolution=1.0,
w=5,
n=5 * 20)
midIdx = encoder._maxBuckets / 2
encoder.bucketMap[midIdx - 3] = numpy.array(range(4, 9)) # Not ok with
# midIdx-1
encoder.bucketMap[midIdx - 2] = numpy.array(range(3, 8))
encoder.bucketMap[midIdx - 1] = numpy.array(range(4, 9))
encoder.bucketMap[midIdx] = numpy.array(range(5, 10))
encoder.bucketMap[midIdx + 1] = numpy.array(range(6, 11))
encoder.bucketMap[midIdx + 2] = numpy.array(range(7, 12))
encoder.bucketMap[midIdx + 3] = numpy.array(range(8, 13))
encoder.minIndex = midIdx - 3
encoder.maxIndex = midIdx + 3
self.assertTrue(encoder._overlapOK(midIdx, midIdx - 1),
"_overlapOK didn't work")
self.assertTrue(encoder._overlapOK(midIdx - 2, midIdx + 3),
"_overlapOK didn't work")
self.assertFalse(encoder._overlapOK(midIdx - 3, midIdx - 1),
"_overlapOK didn't work")
# We'll just use our own numbers
self.assertTrue(encoder._overlapOK(100, 50, 0),
"_overlapOK didn't work for far values")
self.assertTrue(encoder._overlapOK(100, 50, encoder._maxOverlap),
"_overlapOK didn't work for far values")
self.assertFalse(encoder._overlapOK(100, 50, encoder._maxOverlap + 1),
"_overlapOK didn't work for far values")
self.assertTrue(encoder._overlapOK(50, 50, 5),
"_overlapOK didn't work for near values")
self.assertTrue(encoder._overlapOK(48, 50, 3),
"_overlapOK didn't work for near values")
self.assertTrue(encoder._overlapOK(46, 50, 1),
"_overlapOK didn't work for near values")
self.assertTrue(encoder._overlapOK(45, 50, encoder._maxOverlap),
"_overlapOK didn't work for near values")
self.assertFalse(encoder._overlapOK(48, 50, 4),
"_overlapOK didn't work for near values")
self.assertFalse(encoder._overlapOK(48, 50, 2),
"_overlapOK didn't work for near values")
self.assertFalse(encoder._overlapOK(46, 50, 2),
"_overlapOK didn't work for near values")
self.assertFalse(encoder._overlapOK(50, 50, 6),
"_overlapOK didn't work for near values")
def testParameterChecks(self):
"""
Test that some bad construction parameters get handled.
"""
# n must be >= 6*w
with self.assertRaises(ValueError):
RandomDistributedScalarEncoder(name="mv",
resolution=1.0,
n=int(5.9 * 21))
# n must be an int
with self.assertRaises(ValueError):
RandomDistributedScalarEncoder(name="mv",
resolution=1.0,
n=5.9 * 21)
# w can't be negative
with self.assertRaises(ValueError):
RandomDistributedScalarEncoder(name="mv", resolution=1.0, w=-1)
# resolution can't be negative
with self.assertRaises(ValueError):
RandomDistributedScalarEncoder(name="mv", resolution=-2)
def testVerbosity(self):
"""
Test that nothing is printed out when verbosity=0
"""
_stdout = sys.stdout
sys.stdout = _stringio = StringIO()
encoder = RandomDistributedScalarEncoder(name="mv", resolution=1.0,
verbosity=0)
output = numpy.zeros(encoder.getWidth(), dtype=defaultDtype)
encoder.encodeIntoArray(23.0, output)
encoder.getBucketIndices(23.0)
sys.stdout = _stdout
self.assertEqual(len(_stringio.getvalue()), 0,
"zero verbosity doesn't lead to zero output")
def setUp(self):
self.tmPy = TemporalMemoryPy(columnDimensions=[2048],
cellsPerColumn=32,
initialPermanence=0.5,
connectedPermanence=0.8,
minThreshold=10,
maxNewSynapseCount=12,
permanenceIncrement=0.1,
permanenceDecrement=0.05,
activationThreshold=15)
self.tmCPP = TemporalMemoryCPP(columnDimensions=[2048],
cellsPerColumn=32,
initialPermanence=0.5,
connectedPermanence=0.8,
minThreshold=10,
maxNewSynapseCount=12,
permanenceIncrement=0.1,
permanenceDecrement=0.05,
activationThreshold=15)
self.tp = TP(numberOfCols=2048,
cellsPerColumn=32,
initialPerm=0.5,
connectedPerm=0.8,
minThreshold=10,
newSynapseCount=12,
permanenceInc=0.1,
permanenceDec=0.05,
activationThreshold=15,
globalDecay=0,
burnIn=1,
checkSynapseConsistency=False,
pamLength=1)
self.tp10x2 = TP10X2(numberOfCols=2048,
cellsPerColumn=32,
initialPerm=0.5,
connectedPerm=0.8,
minThreshold=10,
newSynapseCount=12,
permanenceInc=0.1,
permanenceDec=0.05,
activationThreshold=15,
globalDecay=0,
burnIn=1,
checkSynapseConsistency=False,
pamLength=1)
self.scalarEncoder = RandomDistributedScalarEncoder(0.88)
def _generateSequence():
scalarEncoder = RandomDistributedScalarEncoder(0.88)
sequence = []
with open(_INPUT_FILE_PATH) as fin:
reader = csv.reader(fin)
reader.next()
reader.next()
reader.next()
for _ in xrange(NUM_PATTERNS):
record = reader.next()
value = float(record[1])
encodedValue = scalarEncoder.encode(value)
activeBits = set(encodedValue.nonzero()[0])
sequence.append(activeBits)
return sequence
def testGetMethods(self):
"""
Test that the getWidth, getDescription, and getDecoderOutputFieldTypes
methods work.
"""
encoder = RandomDistributedScalarEncoder(name="theName", resolution=1.0, n=500)
self.assertEqual(encoder.getWidth(), 500,
"getWidth doesn't return the correct result")
self.assertEqual(encoder.getDescription(), [("theName", 0)],
"getDescription doesn't return the correct result")
self.assertEqual(encoder.getDecoderOutputFieldTypes(),
(FieldMetaType.float, ),
"getDecoderOutputFieldTypes doesn't return the correct"
" result")
def runSimpleSequence(self, resets, repetitions=1):
scalarEncoder = RandomDistributedScalarEncoder(0.88, n=2048, w=41)
instances = self._createInstances(cellsPerColumn=32)
times = [0.0] * len(self.contestants)
duration = 10000 * repetitions
increment = 4
sequenceLength = 25
sequence = (i % (sequenceLength * 4)
for i in xrange(0, duration * increment, increment))
t = 0
encodedValue = numpy.zeros(2048, dtype=numpy.int32)
for value in sequence:
scalarEncoder.encodeIntoArray(value, output=encodedValue)
activeBits = encodedValue.nonzero()[0]
for i in xrange(len(self.contestants)):
tmInstance = instances[i]
computeFn = self.contestants[i][2]
if resets:
if value == 0:
tmInstance.reset()
start = time.clock()
computeFn(tmInstance, encodedValue, activeBits)
times[i] += time.clock() - start
printProgressBar(t, duration, 50)
t += 1
clearProgressBar(50)
results = []
for i in xrange(len(self.contestants)):
name = self.contestants[i][3]
results.append((
name,
times[i],
))
return results
def runHotgym(self, cellsPerColumn, repetitions=1):
scalarEncoder = RandomDistributedScalarEncoder(0.88, n=2048, w=41)
instances = self._createInstances(cellsPerColumn=cellsPerColumn)
times = [0.0] * len(self.contestants)
t = 0
duration = HOTGYM_LENGTH * repetitions
for _ in xrange(repetitions):
with open(HOTGYM_PATH) as fin:
reader = csv.reader(fin)
reader.next()
reader.next()
reader.next()
encodedValue = numpy.zeros(2048, dtype=numpy.int32)
for timeStr, valueStr in reader:
value = float(valueStr)
scalarEncoder.encodeIntoArray(value, output=encodedValue)
activeBits = encodedValue.nonzero()[0]
for i in xrange(len(self.contestants)):
tmInstance = instances[i]
computeFn = self.contestants[i][2]
start = time.clock()
computeFn(tmInstance, encodedValue, activeBits)
times[i] += time.clock() - start
printProgressBar(t, duration, 50)
t += 1
clearProgressBar(50)
results = []
for i in xrange(len(self.contestants)):
name = self.contestants[i][3]
results.append((
name,
times[i],
))
return results
def testOverlapStatistics(self):
"""
Check that the overlaps for the encodings are within the expected range.
Here we ask the encoder to create a bunch of representations under somewhat
stressful conditions, and then verify they are correct. We rely on the fact
that the _overlapOK and _countOverlapIndices methods are working correctly.
"""
seed = getSeed()
# Generate about 600 encodings. Set n relatively low to increase
# chance of false overlaps
encoder = RandomDistributedScalarEncoder(resolution=1.0, w=11, n=150,
seed=seed)
encoder.encode(0.0)
encoder.encode(-300.0)
encoder.encode(300.0)
self.assertTrue(validateEncoder(encoder, subsampling=3),
"Illegal overlap encountered in encoder")
def initialize(self):
# Scalar Encoder
resolution = self.getEncoderResolution()
self.encoder = RandomDistributedScalarEncoder(resolution, seed=42)
self.encoderOutput = np.zeros(self.encoder.getWidth(), dtype=np.uint32)
# Spatial Pooler
spInputWidth = self.encoder.getWidth()
self.spParams = {
"globalInhibition": True,
"columnDimensions": [self.numColumns],
"inputDimensions": [spInputWidth],
"potentialRadius": spInputWidth,
"numActiveColumnsPerInhArea": 40,
"seed": 1956,
"potentialPct": 0.8,
"boostStrength": 0.0,
"synPermActiveInc": 0.003,
"synPermConnected": 0.2,
"synPermInactiveDec": 0.0005,
}
self.sp = SpatialPooler(**self.spParams)
self.spOutput = np.zeros(self.numColumns, dtype=np.uint32)
# Temporal Memory
self.tmParams = {
"activationThreshold": 20,
"cellsPerColumn": self.cellsPerColumn,
"columnDimensions": (self.numColumns,),
"initialPermanence": 0.24,
"maxSegmentsPerCell": 128,
"maxSynapsesPerSegment": 128,
"minThreshold": 13,
"maxNewSynapseCount": 31,
"permanenceDecrement": 0.008,
"permanenceIncrement": 0.04,
"seed": 1960,
}
self.tm = TemporalMemory(**self.tmParams)
# Sanity
if self.runSanity:
self.sanity = sanity.SPTMInstance(self.sp, self.tm)
def testMapBucketIndexToNonZeroBits(self):
"""
Test that mapBucketIndexToNonZeroBits works and that max buckets and
clipping are handled properly.
"""
encoder = RandomDistributedScalarEncoder(resolution=1.0, w=11, n=150)
# Set a low number of max buckets
encoder._initializeBucketMap(10, None)
encoder.encode(0.0)
encoder.encode(-7.0)
encoder.encode(7.0)
self.assertEqual(len(encoder.bucketMap), encoder._maxBuckets,
"_maxBuckets exceeded")
self.assertTrue(
numpy.array_equal(encoder.mapBucketIndexToNonZeroBits(-1),
encoder.bucketMap[0]),
"mapBucketIndexToNonZeroBits did not handle negative"
" index")
self.assertTrue(
numpy.array_equal(encoder.mapBucketIndexToNonZeroBits(1000),
encoder.bucketMap[9]),
"mapBucketIndexToNonZeroBits did not handle negative index")
e23 = encoder.encode(23.0)
e6 = encoder.encode(6)
self.assertEqual((e23 == e6).sum(), encoder.getWidth(),
"Values not clipped correctly during encoding")
ep8 = encoder.encode(-8)
ep7 = encoder.encode(-7)
self.assertEqual((ep8 == ep7).sum(), encoder.getWidth(),
"Values not clipped correctly during encoding")
self.assertEqual(
encoder.getBucketIndices(-8)[0], 0,
"getBucketIndices returned negative bucket index")
self.assertEqual(
encoder.getBucketIndices(23)[0], encoder._maxBuckets - 1,
"getBucketIndices returned bucket index that is too"
" large")
def smart_encode(self,data_fl):
encoder_list = []
for i in data_fl.columns:
if data_fl[i].dtype == 'M8[ns]':
time_delta = data_fl[i][1] - data_fl[i][0]
if time_delta >= pd.Timedelta(1,unit='M'):
encoder_list += [[DateEncoder(season=(5,1))]]
elif time_delta >= pd.Timedelta(1,unit='D'):
encoder_list += [[DateEncoder(season=(21)),
DateEncoder(dayOfWeek=(21,1)),
DateEncoder(weekend=5)]]
else:
encoder_list += [[DateEncoder(season=(5,1)),
DateEncoder(dayOfWeek=(5,1)),
DateEncoder(weekend=5),
DateEncoder(timeOfDay=(5,1))]]
if data_fl[i].dtype == "float":
col_range = data_fl[i].max() - data_fl[i].min()
res = col_range/(400-21)
encoder_list += [[RandomDistributedScalarEncoder(res)]]
return encoder_list
def definir_encoders():
"""
retorna o SIZE_ENCODER_, scalar_1_encoder, bits_scalar_1
"""
### A RESOLUCAO DOS 3 TINHA QUE SER 2.30 # TROCAR DEPOIS
scalar_1_encoder = RandomDistributedScalarEncoder(resolution = 0.07692307692307693,
seed = 42,
)
#two inputs separated by less than the 'resolution' will have the same encoder output.
#7 = how much bits represent one input
#0.25 = radius = if an input ir greater than the radius in comparisson with anoter ..
#they won't overlapp
bits_scalar_1 = np.zeros(scalar_1_encoder.getWidth())
SIZE_ENCODER_ = np.size(bits_scalar_1)
return SIZE_ENCODER_, scalar_1_encoder, bits_scalar_1
def runRandom(self, repetitions=1):
scalarEncoder = RandomDistributedScalarEncoder(0.88, n=2048, w=41)
instances = self._createInstances(cellsPerColumn=32)
times = [0.0] * len(self.contestants)
duration = 1000 * repetitions
t = 0
encodedValue = numpy.zeros(2048, dtype=numpy.int32)
for _ in xrange(duration):
activeBits = random.sample(xrange(2048), 40)
encodedValue = numpy.zeros(2048, dtype=numpy.int32)
encodedValue[activeBits] = 1
for i in xrange(len(self.contestants)):
tmInstance = instances[i]
computeFn = self.contestants[i][2]
start = time.clock()
computeFn(tmInstance, encodedValue, activeBits)
times[i] += time.clock() - start
printProgressBar(t, duration, 50)
t += 1
clearProgressBar(50)
results = []
for i in xrange(len(self.contestants)):
name = self.contestants[i][3]
results.append((
name,
times[i],
))
return results
def RDSE(**kwargs):
"""
RANDOM DISTRIBUTED SCALAR ENCODER, see definition for more info
Parameters --
@param resolution: inputs separated by more than the resolution will have
different, but possible overlapping, representations
@param w: Number of ON bits which encode a single value, must be odd to
to avoid centering problems
@param n: Total number of bits in the output must be >= w
@param name: Optional string which will become part of the description
@param offset: Floating point offset used to map scalar inputs to bucket
indices. If set to None, the very first input that is encoded will be
used to determine the offset.
@param seed: Seed used by numpy rnadom number generator, if set to -1, the
generator will be initialized without a fixed seed
"""
return RandomDistributedScalarEncoder(**kwargs)
def testWriteRead(self):
original = RandomDistributedScalarEncoder(name="encoder",
resolution=1.0,
w=23,
n=500,
offset=0.0)
originalValue = original.encode(1)
proto1 = RandomDistributedScalarEncoderProto.new_message()
original.write(proto1)
# Write the proto to a temp file and read it back into a new proto
with tempfile.TemporaryFile() as f:
proto1.write(f)
f.seek(0)
proto2 = RandomDistributedScalarEncoderProto.read(f)
encoder = RandomDistributedScalarEncoder.read(proto2)
self.assertIsInstance(encoder, RandomDistributedScalarEncoder)
self.assertEqual(encoder.resolution, original.resolution)
self.assertEqual(encoder.w, original.w)
self.assertEqual(encoder.n, original.n)
self.assertEqual(encoder.name, original.name)
self.assertEqual(encoder.verbosity, original.verbosity)
self.assertEqual(encoder.minIndex, original.minIndex)
self.assertEqual(encoder.maxIndex, original.maxIndex)
encodedFromOriginal = original.encode(1)
encodedFromNew = encoder.encode(1)
self.assertTrue(numpy.array_equal(encodedFromNew, originalValue))
self.assertEqual(original.decode(encodedFromNew),
encoder.decode(encodedFromOriginal))
self.assertEqual(original.random.getSeed(), encoder.random.getSeed())
for key, value in original.bucketMap.items():
self.assertTrue(numpy.array_equal(value, encoder.bucketMap[key]))
def testCountOverlap(self):
"""
Test that the internal method _countOverlap works as expected.
"""
encoder = RandomDistributedScalarEncoder(name="encoder",
resolution=1.0,
n=500)
r1 = numpy.array([1, 2, 3, 4, 5, 6])
r2 = numpy.array([1, 2, 3, 4, 5, 6])
self.assertEqual(encoder._countOverlap(r1, r2), 6,
"_countOverlap result is incorrect")
r1 = numpy.array([1, 2, 3, 4, 5, 6])
r2 = numpy.array([1, 2, 3, 4, 5, 7])
self.assertEqual(encoder._countOverlap(r1, r2), 5,
"_countOverlap result is incorrect")
r1 = numpy.array([1, 2, 3, 4, 5, 6])
r2 = numpy.array([6, 5, 4, 3, 2, 1])
self.assertEqual(encoder._countOverlap(r1, r2), 6,
"_countOverlap result is incorrect")
r1 = numpy.array([1, 2, 8, 4, 5, 6])
r2 = numpy.array([1, 2, 3, 4, 9, 6])
self.assertEqual(encoder._countOverlap(r1, r2), 4,
"_countOverlap result is incorrect")
r1 = numpy.array([1, 2, 3, 4, 5, 6])
r2 = numpy.array([1, 2, 3])
self.assertEqual(encoder._countOverlap(r1, r2), 3,
"_countOverlap result is incorrect")
r1 = numpy.array([7, 8, 9, 10, 11, 12])
r2 = numpy.array([1, 2, 3, 4, 5, 6])
self.assertEqual(encoder._countOverlap(r1, r2), 0,
"_countOverlap result is incorrect")
def runHotgym(numRecords):
with open(_PARAMS_PATH, "r") as f:
modelParams = yaml.safe_load(f)["modelParams"]
enParams = modelParams["sensorParams"]["encoders"]
spParams = modelParams["spParams"]
tmParams = modelParams["tmParams"]
scalarEncoder = RandomDistributedScalarEncoder(
enParams["consumption"]["resolution"])
scalarEncoder2 = RandomDistributedScalarEncoder(
enParams["consumption2"]["resolution"])
encodingWidth = (scalarEncoder.getWidth() + scalarEncoder2.getWidth())
sp = SpatialPooler(
inputDimensions=(encodingWidth, ),
columnDimensions=(spParams["columnCount"], ),
potentialPct=spParams["potentialPct"],
potentialRadius=encodingWidth,
globalInhibition=spParams["globalInhibition"],
localAreaDensity=spParams["localAreaDensity"],
numActiveColumnsPerInhArea=spParams["numActiveColumnsPerInhArea"],
synPermInactiveDec=spParams["synPermInactiveDec"],
synPermActiveInc=spParams["synPermActiveInc"],
synPermConnected=spParams["synPermConnected"],
boostStrength=spParams["boostStrength"],
seed=spParams["seed"],
wrapAround=True)
tm = TemporalMemory(
columnDimensions=(tmParams["columnCount"], ),
cellsPerColumn=tmParams["cellsPerColumn"],
activationThreshold=tmParams["activationThreshold"],
initialPermanence=tmParams["initialPerm"],
connectedPermanence=spParams["synPermConnected"],
minThreshold=tmParams["minThreshold"],
maxNewSynapseCount=tmParams["newSynapseCount"],
permanenceIncrement=tmParams["permanenceInc"],
permanenceDecrement=tmParams["permanenceDec"],
predictedSegmentDecrement=0.0,
maxSegmentsPerCell=tmParams["maxSegmentsPerCell"],
maxSynapsesPerSegment=tmParams["maxSynapsesPerSegment"],
seed=tmParams["seed"])
classifier = SDRClassifierFactory.create()
results = []
with open(_INPUT_FILE_PATH, "r") as fin:
reader = csv.reader(fin)
headers = reader.next()
reader.next()
reader.next()
output = output_anomaly_generic_v1.NuPICFileOutput(_FILE_NAME)
for count, record in enumerate(reader):
if count >= numRecords: break
# Convert data string into Python date object.
# dateString = datetime.datetime.strptime(record[0], "%m/%d/%y %H:%M")
# Convert data value string into float.
prediction = float(record[1])
prediction2 = float(record[2])
# To encode, we need to provide zero-filled numpy arrays for the encoders
# to populate.
consumptionBits = numpy.zeros(scalarEncoder.getWidth())
consumptionBits2 = numpy.zeros(scalarEncoder2.getWidth())
# Now we call the encoders to create bit representations for each value.
scalarEncoder.encodeIntoArray(prediction, consumptionBits)
scalarEncoder2.encodeIntoArray(prediction2, consumptionBits2)
# Concatenate all these encodings into one large encoding for Spatial
# Pooling.
encoding = numpy.concatenate([consumptionBits, consumptionBits2])
# Create an array to represent active columns, all initially zero. This
# will be populated by the compute method below. It must have the same
# dimensions as the Spatial Pooler.
activeColumns = numpy.zeros(spParams["columnCount"])
# Execute Spatial Pooling algorithm over input space.
sp.compute(encoding, True, activeColumns)
activeColumnIndices = numpy.nonzero(activeColumns)[0]
# Execute Temporal Memory algorithm over active mini-columns.
tm.compute(activeColumnIndices, learn=True)
activeCells = tm.getActiveCells()
# Get the bucket info for this input value for classification.
bucketIdx = scalarEncoder.getBucketIndices(prediction)[0]
# Run classifier to translate active cells back to scalar value.
classifierResult = classifier.compute(recordNum=count,
patternNZ=activeCells,
classification={
"bucketIdx": bucketIdx,
"actValue": prediction
},
learn=True,
infer=True)
# Print the best prediction for 1 step out.
oneStepConfidence, oneStep = sorted(zip(
classifierResult[1], classifierResult["actualValues"]),
reverse=True)[0]
# print("1-step: {:16} ({:4.4}%)".format(oneStep, oneStepConfidence * 100))
# results.append([oneStep, oneStepConfidence * 100, None, None])
results.append(
[record[0], prediction, oneStep, oneStepConfidence * 100])
output.write(record[0], prediction, oneStep,
oneStepConfidence * 100)
output.close()
return results
def runDataSet(dataName, datasetName):
trainData, trainLabel, testData, testLabel = loadDataset(
dataName, datasetName)
numTest = len(testLabel)
numTrain = len(trainLabel)
sequenceLength = len(trainData[0])
classList = np.unique(trainLabel).tolist()
numClass = len(classList)
print "Processing {}".format(dataName)
print "Train Sample # {}, Test Sample # {}".format(numTrain, numTest)
print "Sequence Length {} Class # {}".format(sequenceLength,
len(classList))
if (max(numTrain, numTest) * sequenceLength < 600 * 600):
print "skip this small dataset for now"
return
try:
unionLengthList = [1, 5, 10, 15, 20]
for unionLength in unionLengthList:
expResultTM = pickle.load(
open(
'results/modelPerformance/{}_columnOnly_union_{}'.format(
dataName, unionLength), 'r'))
return
except:
print "run data set: ", dataName
EuclideanDistanceMat = calculateEuclideanDistanceMat(testData, trainData)
outcomeEuclidean = calculateEuclideanModelAccuracy(trainData, trainLabel,
testData, testLabel)
accuracyEuclideanDist = np.mean(outcomeEuclidean)
print
print "Euclidean model accuracy: {}".format(accuracyEuclideanDist)
print
# # Use SDR overlap instead of Euclidean distance
print "Running Encoder model"
maxValue = np.max(trainData)
minValue = np.min(trainData)
numCols = 2048
w = 41
try:
searchResolution = pickle.load(
open('results/optimalEncoderResolution/{}'.format(dataName), 'r'))
nBucketList = searchResolution['nBucketList']
accuracyVsResolution = searchResolution['accuracyVsResolution']
optNumBucket = nBucketList[smoothArgMax(
np.array(accuracyVsResolution))]
optimalResolution = (maxValue - minValue) / optNumBucket
except:
return
print "optimal bucket # {}".format(
(maxValue - minValue) / optimalResolution)
encoder = RandomDistributedScalarEncoder(optimalResolution, w=w, n=numCols)
print "encoding train data ..."
activeColumnsTrain = runEncoderOverDataset(encoder, trainData)
print "encoding test data ..."
activeColumnsTest = runEncoderOverDataset(encoder, testData)
print "calculate column distance matrix ..."
# run encoder -> union model, search for the optimal union window
unionLengthList = [1, 5, 10, 15, 20]
for unionLength in unionLengthList:
activeColumnUnionTrain = runUnionStep(activeColumnsTrain, unionLength)
activeColumnUnionTest = runUnionStep(activeColumnsTest, unionLength)
distMatColumnTrain = calculateDistanceMatTrain(activeColumnUnionTrain)
distMatColumnTest = calculateDistanceMat(activeColumnUnionTest,
activeColumnUnionTrain)
trainAccuracyColumnOnly, outcomeColumn = calculateAccuracy(
distMatColumnTest, trainLabel, testLabel)
testAccuracyColumnOnly, outcomeColumn = calculateAccuracy(
distMatColumnTest, trainLabel, testLabel)
expResults = {
'distMatColumnTrain': distMatColumnTrain,
'distMatColumnTest': distMatColumnTest,
'trainAccuracyColumnOnly': trainAccuracyColumnOnly,
'testAccuracyColumnOnly': testAccuracyColumnOnly
}
outputFile = open(
'results/distanceMat/{}_columnOnly_union_{}'.format(
dataName, unionLength), 'w')
pickle.dump(expResults, outputFile)
outputFile.close()
optimalResolution = (maxValue - minValue) / optNumBucket
searchResolution = {
'nBucketList': nBucketList,
'accuracyVsResolution': accuracyVsResolution,
'optimalResolution': optimalResolution
}
# save optimal resolution for future use
outputFile = open(
'results/optimalEncoderResolution/{}'.format(dataName), 'w')
pickle.dump(searchResolution, outputFile)
outputFile.close()
print "optimal bucket # {}".format(
(maxValue - minValue) / optimalResolution)
encoder = RandomDistributedScalarEncoder(optimalResolution,
w=w,
n=numCols)
print "encoding train data ..."
activeColumnsTrain = runEncoderOverDataset(encoder, trainData)
print "encoding test data ..."
activeColumnsTest = runEncoderOverDataset(encoder, testData)
print "calculate column distance matrix ..."
distMatColumnTest = calculateDistanceMat(activeColumnsTest,
activeColumnsTrain)
testAccuracyColumnOnly, outcomeColumn = calculateAccuracy(
distMatColumnTest, trainLabel, testLabel)
print
print "Column Only model, Accuracy: {}".format(testAccuracyColumnOnly)
expResults = {
import numpy as np from nupic.encoders import ScalarEncoder ScalarEncoder? enc = ScalarEncoder(n=22, w=3, minval=2.5, maxval=97.5, clipInput=False, forced=True) print "3 =", enc.encode(3) print "4 =", enc.encode(4) print "5 =", enc.encode(5) print "1000 =", enc.encode(1000) from nupic.encoders.random_distributed_scalar import RandomDistributedScalarEncoder RandomDistributedScalarEncoder? rdse = RandomDistributedScalarEncoder(n=21, w=3, resolution=5, offset=2.5) print "3 = ", rdse.encode(3) print "4 = ", rdse.encode(4) print "5 = ", rdse.encode(5) print print "100 = ", rdse.encode(100) print "100000 =", rdse.encode(1000) import datetime from nupic.encoders.date import DateEncoder DateEncoder?
import numpy as np
from nupic.encoders.random_distributed_scalar import (
RandomDistributedScalarEncoder)
if __name__ == "__main__":
print "Testing RSDE Quality"
maxval = 100.0
minval = -100.0
Nsamples = 1000
encoder1 = RandomDistributedScalarEncoder(name="encoder",
resolution=1.0,
w=23,
n=500,
offset=0.0)
encoder2 = RandomDistributedScalarEncoder(name="encoder",
resolution=10.0,
w=23,
n=500,
offset=0.0)
distance_function = lambda x, y: abs(x - y)
sample_generator = lambda: np.random.uniform(minval, maxval)
input_pairs_source = encoder_check.InputTripleCreator(sample_generator)
err1 = encoder_check.encoderCheck(encoder1, distance_function,
def initialize(self):
# Keep track of value range for spatial anomaly detection.
self.minVal = None
self.maxVal = None
# Time of day encoder
self.timeOfDayEncoder = DateEncoder(timeOfDay=(21, 9.49),
name='time_enc')
# RDSE encoder for the time series value.
minResolution = 0.001
rangePadding = abs(self.inputMax - self.inputMin) * 0.2
minVal = self.inputMin - rangePadding
maxVal = self.inputMax + rangePadding
numBuckets = 130
resolution = max(minResolution, (maxVal - minVal) / numBuckets)
self.value_enc = RandomDistributedScalarEncoder(resolution=resolution,
name='value_rdse')
# Spatial Pooler.
encodingWidth = self.timeOfDayEncoder.getWidth(
) + self.value_enc.getWidth()
self.sp = SpatialPooler(
inputDimensions=(encodingWidth, ),
columnDimensions=(2048, ),
potentialPct=0.8,
potentialRadius=encodingWidth,
globalInhibition=1,
numActiveColumnsPerInhArea=40,
synPermInactiveDec=0.0005,
synPermActiveInc=0.003,
synPermConnected=0.2,
boostStrength=0.0,
seed=1956,
wrapAround=True,
)
self.tm = TemporalMemory(
columnDimensions=(2048, ),
cellsPerColumn=32,
activationThreshold=20,
initialPermanence=.5, # Increased to connectedPermanence.
connectedPermanence=.5,
minThreshold=13,
maxNewSynapseCount=31,
permanenceIncrement=0.04,
permanenceDecrement=0.008,
predictedSegmentDecrement=0.001,
maxSegmentsPerCell=128,
maxSynapsesPerSegment=
128, # Changed meaning. Also see connections.topology[2]
seed=1993,
)
# Initialize the anomaly likelihood object
numentaLearningPeriod = int(math.floor(self.probationaryPeriod / 2.0))
self.anomalyLikelihood = anomaly_likelihood.AnomalyLikelihood(
learningPeriod=numentaLearningPeriod,
estimationSamples=self.probationaryPeriod - numentaLearningPeriod,
reestimationPeriod=100,
)
self.age = 0
def runHotgym():
timeOfDayEncoder = DateEncoder(timeOfDay=(21,1))
weekendEncoder = DateEncoder(weekend=21)
scalarEncoder = RandomDistributedScalarEncoder(0.88)
encodingWidth = timeOfDayEncoder.getWidth() \
+ weekendEncoder.getWidth() \
+ scalarEncoder.getWidth()
sp = SpatialPooler(
# How large the input encoding will be.
inputDimensions=(encodingWidth),
# How many mini-columns will be in the Spatial Pooler.
columnDimensions=(2048),
# What percent of the columns's receptive field is available for potential
# synapses?
potentialPct=0.85,
# This means that the input space has no topology.
globalInhibition=True,
localAreaDensity=-1.0,
# Roughly 2%, giving that there is only one inhibition area because we have
# turned on globalInhibition (40 / 2048 = 0.0195)
numActiveColumnsPerInhArea=40.0,
# How quickly synapses grow and degrade.
synPermInactiveDec=0.005,
synPermActiveInc=0.04,
synPermConnected=0.1,
# boostStrength controls the strength of boosting. Boosting encourages
# efficient usage of SP columns.
boostStrength=3.0,
# Random number generator seed.
seed=1956,
# Determines if inputs at the beginning and end of an input dimension should
# be considered neighbors when mapping columns to inputs.
wrapAround=False
)
tm = TemporalMemory(
# Must be the same dimensions as the SP
columnDimensions=(2048, ),
# How many cells in each mini-column.
cellsPerColumn=32,
# A segment is active if it has >= activationThreshold connected synapses
# that are active due to infActiveState
activationThreshold=16,
initialPermanence=0.21,
connectedPermanence=0.5,
# Minimum number of active synapses for a segment to be considered during
# search for the best-matching segments.
minThreshold=12,
# The max number of synapses added to a segment during learning
maxNewSynapseCount=20,
permanenceIncrement=0.1,
permanenceDecrement=0.1,
predictedSegmentDecrement=0.0,
maxSegmentsPerCell=128,
maxSynapsesPerSegment=32,
seed=1960
)
classifier = SDRClassifierFactory.create()
with open (_INPUT_FILE_PATH) as fin:
reader = csv.reader(fin)
headers = reader.next()
reader.next()
reader.next()
for count, record in enumerate(reader):
# Convert data string into Python date object.
dateString = datetime.datetime.strptime(record[0], "%m/%d/%y %H:%M")
# Convert data value string into float.
consumption = float(record[1])
# To encode, we need to provide zero-filled numpy arrays for the encoders
# to populate.
timeOfDayBits = numpy.zeros(timeOfDayEncoder.getWidth())
weekendBits = numpy.zeros(weekendEncoder.getWidth())
consumptionBits = numpy.zeros(scalarEncoder.getWidth())
# Now we call the encoders create bit representations for each value.
timeOfDayEncoder.encodeIntoArray(dateString, timeOfDayBits)
weekendEncoder.encodeIntoArray(dateString, weekendBits)
scalarEncoder.encodeIntoArray(consumption, consumptionBits)
# Concatenate all these encodings into one large encoding for Spatial
# Pooling.
encoding = numpy.concatenate(
[timeOfDayBits, weekendBits, consumptionBits]
)
# Create an array to represent active columns, all initially zero. This
# will be populated by the compute method below. It must have the same
# dimensions as the Spatial Pooler.
activeColumns = numpy.zeros(2048)
# Execute Spatial Pooling algorithm over input space.
sp.compute(encoding, True, activeColumns)
activeColumnIndices = numpy.nonzero(activeColumns)[0]
# Execute Temporal Memory algorithm over active mini-columns.
tm.compute(activeColumnIndices, learn=True)
activeCells = tm.getActiveCells()
# Get the bucket info for this input value for classification.
bucketIdx = scalarEncoder.getBucketIndices(consumption)[0]
# Run classifier to translate active cells back to scalar value.
classifierResult = classifier.compute(
recordNum=count,
patternNZ=activeCells,
classification={
"bucketIdx": bucketIdx,
"actValue": consumption
},
learn=True,
infer=True
)
# Print the best prediction for 1 step out.
probability, value = sorted(
zip(classifierResult[1], classifierResult["actualValues"]),
reverse=True
)[0]
print("1-step: {:16} ({:4.4}%)".format(value, probability * 100))
def go():
valueEncoder = RandomDistributedScalarEncoder(resolution=0.88, seed=42)
timestampEncoder = DateEncoder(timeOfDay=(
21,
9.49,
))
inputWidth = timestampEncoder.getWidth() + valueEncoder.getWidth()
sp = SpatialPooler(
**{
"globalInhibition": True,
"columnDimensions": [2048],
"inputDimensions": [inputWidth],
"potentialRadius": inputWidth,
"numActiveColumnsPerInhArea": 40,
"seed": 1956,
"potentialPct": 0.8,
"boostStrength": 0.0,
"synPermActiveInc": 0.003,
"synPermConnected": 0.2,
"synPermInactiveDec": 0.0005,
})
tm = TemporalMemory(
**{
"activationThreshold": 20,
"cellsPerColumn": 32,
"columnDimensions": (2048, ),
"initialPermanence": 0.24,
"maxSegmentsPerCell": 128,
"maxSynapsesPerSegment": 128,
"minThreshold": 13,
"maxNewSynapseCount": 31,
"permanenceDecrement": 0.008,
"permanenceIncrement": 0.04,
"seed": 1961,
})
inputPath = os.path.join(os.path.dirname(__file__),
"data/rec-center-hourly.csv")
inputFile = open(inputPath, "rb")
csvReader = csv.reader(inputFile)
csvReader.next()
csvReader.next()
csvReader.next()
encodedValue = np.zeros(valueEncoder.getWidth(), dtype=np.uint32)
encodedTimestamp = np.zeros(timestampEncoder.getWidth(), dtype=np.uint32)
spOutput = np.zeros(2048, dtype=np.float32)
sanityInstance = sanity.SPTMInstance(sp, tm)
for timestampStr, consumptionStr in csvReader:
sanityInstance.waitForUserContinue()
timestamp = datetime.datetime.strptime(timestampStr, "%m/%d/%y %H:%M")
consumption = float(consumptionStr)
timestampEncoder.encodeIntoArray(timestamp, encodedTimestamp)
valueEncoder.encodeIntoArray(consumption, encodedValue)
sensoryInput = np.concatenate((
encodedTimestamp,
encodedValue,
))
sp.compute(sensoryInput, True, spOutput)
activeColumns = np.flatnonzero(spOutput)
predictedCells = tm.getPredictiveCells()
tm.compute(activeColumns)
activeInputBits = np.flatnonzero(sensoryInput)
displayText = {
"timestamp": timestampStr,
"consumption": consumptionStr
}
sanityInstance.appendTimestep(activeInputBits, activeColumns,
predictedCells, displayText)
def runHotgym(numRecords):
with open(_PARAMS_PATH, "r") as f:
modelParams = yaml.safe_load(f)["modelParams"]
enParams = modelParams["sensorParams"]["encoders"]
spParams = modelParams["spParams"]
tmParams = modelParams["tmParams"]
timeOfDayEncoder = DateEncoder(
timeOfDay=enParams["timestamp_timeOfDay"]["timeOfDay"])
weekendEncoder = DateEncoder(
weekend=enParams["timestamp_weekend"]["weekend"])
scalarEncoder = RandomDistributedScalarEncoder(
enParams["consumption"]["resolution"])
encodingWidth = (timeOfDayEncoder.getWidth()
+ weekendEncoder.getWidth()
+ scalarEncoder.getWidth())
sp = SpatialPooler(
# How large the input encoding will be.
inputDimensions=(encodingWidth),
# How many mini-columns will be in the Spatial Pooler.
columnDimensions=(spParams["columnCount"]),
# What percent of the columns"s receptive field is available for potential
# synapses?
potentialPct=spParams["potentialPct"],
# This means that the input space has no topology.
globalInhibition=spParams["globalInhibition"],
localAreaDensity=spParams["localAreaDensity"],
# Roughly 2%, giving that there is only one inhibition area because we have
# turned on globalInhibition (40 / 2048 = 0.0195)
numActiveColumnsPerInhArea=spParams["numActiveColumnsPerInhArea"],
# How quickly synapses grow and degrade.
synPermInactiveDec=spParams["synPermInactiveDec"],
synPermActiveInc=spParams["synPermActiveInc"],
synPermConnected=spParams["synPermConnected"],
# boostStrength controls the strength of boosting. Boosting encourages
# efficient usage of SP columns.
boostStrength=spParams["boostStrength"],
# Random number generator seed.
seed=spParams["seed"],
# TODO: is this useful?
# Determines if inputs at the beginning and end of an input dimension should
# be considered neighbors when mapping columns to inputs.
wrapAround=False
)
tm = TemporalMemory(
# Must be the same dimensions as the SP
columnDimensions=(tmParams["columnCount"],),
# How many cells in each mini-column.
cellsPerColumn=tmParams["cellsPerColumn"],
# A segment is active if it has >= activationThreshold connected synapses
# that are active due to infActiveState
activationThreshold=tmParams["activationThreshold"],
initialPermanence=tmParams["initialPerm"],
# TODO: This comes from the SP params, is this normal
connectedPermanence=spParams["synPermConnected"],
# Minimum number of active synapses for a segment to be considered during
# search for the best-matching segments.
minThreshold=tmParams["minThreshold"],
# The max number of synapses added to a segment during learning
maxNewSynapseCount=tmParams["newSynapseCount"],
permanenceIncrement=tmParams["permanenceInc"],
permanenceDecrement=tmParams["permanenceDec"],
predictedSegmentDecrement=0.0,
maxSegmentsPerCell=tmParams["maxSegmentsPerCell"],
maxSynapsesPerSegment=tmParams["maxSynapsesPerSegment"],
seed=tmParams["seed"]
)
classifier = SDRClassifierFactory.create()
results = []
with open(_INPUT_FILE_PATH, "r") as fin:
reader = csv.reader(fin)
headers = reader.next()
reader.next()
reader.next()
for count, record in enumerate(reader):
if count >= numRecords: break
# Convert data string into Python date object.
dateString = datetime.datetime.strptime(record[0], "%m/%d/%y %H:%M")
# Convert data value string into float.
consumption = float(record[1])
# To encode, we need to provide zero-filled numpy arrays for the encoders
# to populate.
timeOfDayBits = numpy.zeros(timeOfDayEncoder.getWidth())
weekendBits = numpy.zeros(weekendEncoder.getWidth())
consumptionBits = numpy.zeros(scalarEncoder.getWidth())
# Now we call the encoders create bit representations for each value.
timeOfDayEncoder.encodeIntoArray(dateString, timeOfDayBits)
weekendEncoder.encodeIntoArray(dateString, weekendBits)
scalarEncoder.encodeIntoArray(consumption, consumptionBits)
# Concatenate all these encodings into one large encoding for Spatial
# Pooling.
encoding = numpy.concatenate(
[timeOfDayBits, weekendBits, consumptionBits]
)
# Create an array to represent active columns, all initially zero. This
# will be populated by the compute method below. It must have the same
# dimensions as the Spatial Pooler.
activeColumns = numpy.zeros(spParams["columnCount"])
# Execute Spatial Pooling algorithm over input space.
sp.compute(encoding, True, activeColumns)
activeColumnIndices = numpy.nonzero(activeColumns)[0]
# Execute Temporal Memory algorithm over active mini-columns.
tm.compute(activeColumnIndices, learn=True)
activeCells = tm.getActiveCells()
# Get the bucket info for this input value for classification.
bucketIdx = scalarEncoder.getBucketIndices(consumption)[0]
# Run classifier to translate active cells back to scalar value.
classifierResult = classifier.compute(
recordNum=count,
patternNZ=activeCells,
classification={
"bucketIdx": bucketIdx,
"actValue": consumption
},
learn=True,
infer=True
)
# Print the best prediction for 1 step out.
oneStepConfidence, oneStep = sorted(
zip(classifierResult[1], classifierResult["actualValues"]),
reverse=True
)[0]
print("1-step: {:16} ({:4.4}%)".format(oneStep, oneStepConfidence * 100))
results.append([oneStep, oneStepConfidence * 100, None, None])
return results
def testEncodeInvalidInputType(self):
encoder = RandomDistributedScalarEncoder(name="encoder", resolution=1.0,
verbosity=0)
with self.assertRaises(TypeError):
encoder.encode("String")
