def testMovingAverageInstance(self): """ Test that the (internal) moving average maintains the averages correctly, even for null initial condition and when the number of values goes over windowSize. Pass in integers and floats. this is for the instantce method next() """ ma = MovingAverage(windowSize=3) newAverage = ma.next(3) self.assertEqual(newAverage, 3.0) self.assertListEqual(ma.getSlidingWindow(), [3.0]) self.assertEqual(ma.total, 3.0) newAverage = ma.next(4) self.assertEqual(newAverage, 3.5) self.assertListEqual(ma.getSlidingWindow(), [3.0, 4.0]) self.assertEqual(ma.total, 7.0) newAverage = ma.next(5) self.assertEqual(newAverage, 4.0) self.assertListEqual(ma.getSlidingWindow(), [3.0, 4.0, 5.0]) self.assertEqual(ma.total, 12.0) # Ensure the first value gets popped newAverage = ma.next(6) self.assertEqual(newAverage, 5.0) self.assertListEqual(ma.getSlidingWindow(), [4.0, 5.0, 6.0]) self.assertEqual(ma.total, 15.0)
def testMovingAverageSlidingWindowInit(self): """ Test the slidingWindow value is correctly assigned when initializing a new MovingAverage object. """ # With exisiting historical values; same values as tested in testMovingAverage() ma = MovingAverage(windowSize=3, existingHistoricalValues=[3.0, 4.0, 5.0]) self.assertListEqual(ma.getSlidingWindow(), [3.0, 4.0, 5.0]) # Withoout exisiting historical values ma = MovingAverage(windowSize=3) self.assertListEqual(ma.getSlidingWindow(), [])
def testMovingAverageReadWrite(self): ma = MovingAverage(windowSize=3) ma.next(3) ma.next(4) ma.next(5) proto1 = MovingAverageProto.new_message() ma.write(proto1) # Write the proto to a temp file and read it back into a new proto with tempfile.TemporaryFile() as f: proto1.write(f) f.seek(0) proto2 = MovingAverageProto.read(f) resurrectedMa = MovingAverage.read(proto2) newAverage = ma.next(6) self.assertEqual(newAverage, resurrectedMa.next(6)) self.assertListEqual(ma.getSlidingWindow(), resurrectedMa.getSlidingWindow()) self.assertEqual(ma.total, resurrectedMa.total)
class AdaptiveScalarEncoder(ScalarEncoder): """ This is an implementation of the scalar encoder that adapts the min and max of the scalar encoder dynamically. This is essential to the streaming model of the online prediction framework. Initialization of an adapive encoder using resolution or radius is not supported; it must be intitialized with n. This n is kept constant while the min and max of the encoder changes. The adaptive encoder must be have periodic set to false. The adaptive encoder may be initialized with a minval and maxval or with `None` for each of these. In the latter case, the min and max are set as the 1st and 99th percentile over a window of the past 100 records. **Note:** the sliding window may record duplicates of the values in the dataset, and therefore does not reflect the statistical distribution of the input data and may not be used to calculate the median, mean etc. """ ############################################################################ def __init__(self, w, minval=None, maxval=None, periodic=False, n=0, radius=0, resolution=0, name=None, verbosity=0, clipInput=True, forced=False): """ [overrides nupic.encoders.scalar.ScalarEncoder.__init__] """ self._learningEnabled = True if periodic: #Adaptive scalar encoders take non-periodic inputs only raise Exception( 'Adaptive scalar encoder does not encode periodic inputs') assert n != 0 #An adaptive encoder can only be intialized using n super(AdaptiveScalarEncoder, self).__init__(w=w, n=n, minval=minval, maxval=maxval, clipInput=True, name=name, verbosity=verbosity, forced=forced) self.recordNum = 0 #how many inputs have been sent to the encoder? self.slidingWindow = MovingAverage(300) ############################################################################ def _setEncoderParams(self): """ Set the radius, resolution and range. These values are updated when minval and/or maxval change. """ self.rangeInternal = float(self.maxval - self.minval) self.resolution = float(self.rangeInternal) / (self.n - self.w) self.radius = self.w * self.resolution self.range = self.rangeInternal + self.resolution # nInternal represents the output area excluding the possible padding on each side self.nInternal = self.n - 2 * self.padding # Invalidate the bucket values cache so that they get recomputed self._bucketValues = None ############################################################################ def setFieldStats(self, fieldName, fieldStats): """ TODO: document """ #If the stats are not fully formed, ignore. if fieldStats[fieldName]['min'] == None or \ fieldStats[fieldName]['max'] == None: return self.minval = fieldStats[fieldName]['min'] self.maxval = fieldStats[fieldName]['max'] if self.minval == self.maxval: self.maxval += 1 self._setEncoderParams() ############################################################################ def _setMinAndMax(self, input, learn): """ Potentially change the minval and maxval using input. **The learn flag is currently not supported by cla regions.** """ self.slidingWindow.next(input) if self.minval is None and self.maxval is None: self.minval = input self.maxval = input + 1 #When the min and max and unspecified and only one record has been encoded self._setEncoderParams() elif learn: sorted = self.slidingWindow.getSlidingWindow() sorted.sort() minOverWindow = sorted[0] maxOverWindow = sorted[len(sorted) - 1] if minOverWindow < self.minval: #initialBump = abs(self.minval-minOverWindow)*(1-(min(self.recordNum, 200.0)/200.0))*2 #decrement minval more aggressively in the beginning if self.verbosity >= 2: print "Input %s=%.2f smaller than minval %.2f. Adjusting minval to %.2f"\ % (self.name, input, self.minval, minOverWindow) self.minval = minOverWindow #-initialBump self._setEncoderParams() if maxOverWindow > self.maxval: #initialBump = abs(self.maxval-maxOverWindow)*(1-(min(self.recordNum, 200.0)/200.0))*2 #decrement maxval more aggressively in the beginning if self.verbosity >= 2: print "Input %s=%.2f greater than maxval %.2f. Adjusting maxval to %.2f" \ % (self.name, input, self.maxval, maxOverWindow) self.maxval = maxOverWindow #+initialBump self._setEncoderParams() ############################################################################ def getBucketIndices(self, input, learn=None): """ [overrides nupic.encoders.scalar.ScalarEncoder.getBucketIndices] """ self.recordNum += 1 if learn is None: learn = self._learningEnabled if type(input) is float and math.isnan(input): input = SENTINEL_VALUE_FOR_MISSING_DATA if input == SENTINEL_VALUE_FOR_MISSING_DATA: return [None] else: self._setMinAndMax(input, learn) return super(AdaptiveScalarEncoder, self).getBucketIndices(input) ############################################################################ def encodeIntoArray(self, input, output, learn=None): """ [overrides nupic.encoders.scalar.ScalarEncoder.encodeIntoArray] """ self.recordNum += 1 if learn is None: learn = self._learningEnabled if input == SENTINEL_VALUE_FOR_MISSING_DATA: output[0:self.n] = 0 elif not math.isnan(input): self._setMinAndMax(input, learn) super(AdaptiveScalarEncoder, self).encodeIntoArray(input, output) ############################################################################ def getBucketInfo(self, buckets): """ [overrides nupic.encoders.scalar.ScalarEncoder.getBucketInfo] """ if self.minval is None or self.maxval is None: return [ EncoderResult(value=0, scalar=0, encoding=numpy.zeros(self.n)) ] return super(AdaptiveScalarEncoder, self).getBucketInfo(buckets) ############################################################################ def topDownCompute(self, encoded): """ [overrides nupic.encoders.scalar.ScalarEncoder.topDownCompute] """ if self.minval is None or self.maxval is None: return [ EncoderResult(value=0, scalar=0, encoding=numpy.zeros(self.n)) ] return super(AdaptiveScalarEncoder, self).topDownCompute(encoded) ############################################################################ def dump(self): """ Prints details about current state to stdout. """ print "AdaptiveScalarEncoder:" print " min: %f" % self.minval print " max: %f" % self.maxval print " w: %d" % self.w print " n: %d" % self.n print " resolution: %f" % self.resolution print " radius: %f" % self.radius print " periodic: %s" % self.periodic print " nInternal: %d" % self.nInternal print " rangeInternal: %f" % self.rangeInternal print " padding: %d" % self.padding @classmethod def read(cls, proto): encoder = super(AdaptiveScalarEncoder, cls).read(proto) encoder.recordNum = proto.recordNum encoder.slidingWindow = MovingAverage.read(proto.slidingWindow) return encoder def write(self, proto): super(AdaptiveScalarEncoder, self).write(proto) proto.recordNum = self.recordNum self.slidingWindow.write(proto.slidingWindow)
class AdaptiveScalarEncoder(ScalarEncoder): """ This is an implementation of the scalar encoder that adapts the min and max of the scalar encoder dynamically. This is essential to the streaming model of the online prediction framework. Initialization of an adapive encoder using resolution or radius is not supported; it must be intitialized with n. This n is kept constant while the min and max of the encoder changes. The adaptive encoder must be have periodic set to false. The adaptive encoder may be initialized with a minval and maxval or with `None` for each of these. In the latter case, the min and max are set as the 1st and 99th percentile over a window of the past 100 records. **Note:** the sliding window may record duplicates of the values in the dataset, and therefore does not reflect the statistical distribution of the input data and may not be used to calculate the median, mean etc. """ def __init__(self, w, minval=None, maxval=None, periodic=False, n=0, radius=0, resolution=0, name=None, verbosity=0, clipInput=True, forced=False): """ [overrides nupic.encoders.scalar.ScalarEncoder.__init__] """ self._learningEnabled = True if periodic: #Adaptive scalar encoders take non-periodic inputs only raise Exception('Adaptive scalar encoder does not encode periodic inputs') assert n!=0 #An adaptive encoder can only be intialized using n super(AdaptiveScalarEncoder, self).__init__(w=w, n=n, minval=minval, maxval=maxval, clipInput=True, name=name, verbosity=verbosity, forced=forced) self.recordNum=0 #how many inputs have been sent to the encoder? self.slidingWindow = MovingAverage(300) def _setEncoderParams(self): """ Set the radius, resolution and range. These values are updated when minval and/or maxval change. """ self.rangeInternal = float(self.maxval - self.minval) self.resolution = float(self.rangeInternal) / (self.n - self.w) self.radius = self.w * self.resolution self.range = self.rangeInternal + self.resolution # nInternal represents the output area excluding the possible padding on each side self.nInternal = self.n - 2 * self.padding # Invalidate the bucket values cache so that they get recomputed self._bucketValues = None def setFieldStats(self, fieldName, fieldStats): """ TODO: document """ #If the stats are not fully formed, ignore. if fieldStats[fieldName]['min'] is None or \ fieldStats[fieldName]['max'] is None: return self.minval = fieldStats[fieldName]['min'] self.maxval = fieldStats[fieldName]['max'] if self.minval == self.maxval: self.maxval+=1 self._setEncoderParams() def _setMinAndMax(self, input, learn): """ Potentially change the minval and maxval using input. **The learn flag is currently not supported by cla regions.** """ self.slidingWindow.next(input) if self.minval is None and self.maxval is None: self.minval = input self.maxval = input+1 #When the min and max and unspecified and only one record has been encoded self._setEncoderParams() elif learn: sorted = self.slidingWindow.getSlidingWindow() sorted.sort() minOverWindow = sorted[0] maxOverWindow = sorted[len(sorted)-1] if minOverWindow < self.minval: #initialBump = abs(self.minval-minOverWindow)*(1-(min(self.recordNum, 200.0)/200.0))*2 #decrement minval more aggressively in the beginning if self.verbosity >= 2: print "Input {0!s}={1:.2f} smaller than minval {2:.2f}. Adjusting minval to {3:.2f}".format(self.name, input, self.minval, minOverWindow) self.minval = minOverWindow #-initialBump self._setEncoderParams() if maxOverWindow > self.maxval: #initialBump = abs(self.maxval-maxOverWindow)*(1-(min(self.recordNum, 200.0)/200.0))*2 #decrement maxval more aggressively in the beginning if self.verbosity >= 2: print "Input {0!s}={1:.2f} greater than maxval {2:.2f}. Adjusting maxval to {3:.2f}".format(self.name, input, self.maxval, maxOverWindow) self.maxval = maxOverWindow #+initialBump self._setEncoderParams() def getBucketIndices(self, input, learn=None): """ [overrides nupic.encoders.scalar.ScalarEncoder.getBucketIndices] """ self.recordNum +=1 if learn is None: learn = self._learningEnabled if type(input) is float and math.isnan(input): input = SENTINEL_VALUE_FOR_MISSING_DATA if input == SENTINEL_VALUE_FOR_MISSING_DATA: return [None] else: self._setMinAndMax(input, learn) return super(AdaptiveScalarEncoder, self).getBucketIndices(input) def encodeIntoArray(self, input, output,learn=None): """ [overrides nupic.encoders.scalar.ScalarEncoder.encodeIntoArray] """ self.recordNum +=1 if learn is None: learn = self._learningEnabled if input == SENTINEL_VALUE_FOR_MISSING_DATA: output[0:self.n] = 0 elif not math.isnan(input): self._setMinAndMax(input, learn) super(AdaptiveScalarEncoder, self).encodeIntoArray(input, output) def getBucketInfo(self, buckets): """ [overrides nupic.encoders.scalar.ScalarEncoder.getBucketInfo] """ if self.minval is None or self.maxval is None: return [EncoderResult(value=0, scalar=0, encoding=numpy.zeros(self.n))] return super(AdaptiveScalarEncoder, self).getBucketInfo(buckets) def topDownCompute(self, encoded): """ [overrides nupic.encoders.scalar.ScalarEncoder.topDownCompute] """ if self.minval is None or self.maxval is None: return [EncoderResult(value=0, scalar=0, encoding=numpy.zeros(self.n))] return super(AdaptiveScalarEncoder, self).topDownCompute(encoded) def dump(self): """ Prints details about current state to stdout. """ print "AdaptiveScalarEncoder:" print " min: {0:f}".format(self.minval) print " max: {0:f}".format(self.maxval) print " w: {0:d}".format(self.w) print " n: {0:d}".format(self.n) print " resolution: {0:f}".format(self.resolution) print " radius: {0:f}".format(self.radius) print " periodic: {0!s}".format(self.periodic) print " nInternal: {0:d}".format(self.nInternal) print " rangeInternal: {0:f}".format(self.rangeInternal) print " padding: {0:d}".format(self.padding) @classmethod def read(cls, proto): encoder = super(AdaptiveScalarEncoder, cls).read(proto) encoder.recordNum = proto.recordNum encoder.slidingWindow = MovingAverage.read(proto.slidingWindow) return encoder def write(self, proto): super(AdaptiveScalarEncoder, self).write(proto) proto.recordNum = self.recordNum self.slidingWindow.write(proto.slidingWindow)