def testCategories(self):
# Test two categories.
p = ScalarEncoderParameters()
p.minimum = 0
p.maximum = 1
p.activeBits = 3
p.radius = 1
enc = ScalarEncoder(p)
sdr = SDR( enc.dimensions )
zero = enc.encode( 0 )
one = enc.encode( 1 )
assert( zero.getOverlap( one ) == 0 )
# Test three categories.
p = ScalarEncoderParameters()
p.minimum = 0
p.maximum = 2
p.activeBits = 3
p.radius = 1
enc = ScalarEncoder(p)
sdr = SDR( enc.dimensions )
zero = enc.encode( 0 )
one = enc.encode( 1 )
two = enc.encode( 2 )
assert( zero.getOverlap( one ) == 0 )
assert( one.getOverlap( two ) == 0 )
assert( two.getSum() == 3 )
def testBadParameters(self):
# Start with sane parameters.
p = ScalarEncoderParameters()
p.size = 10
p.activeBits = 2
p.minimum = 0
p.maximum = 1
ScalarEncoder(p)
# Check a lot of bad parameters
p.activeBits = 12 # Can not activate more bits than are in the SDR.
with self.assertRaises(RuntimeError):
ScalarEncoder(p)
p.activeBits = 0 # not enough active bits
with self.assertRaises(RuntimeError):
ScalarEncoder(p)
p.activeBits = 1
p.size = 0 # not enough bits
with self.assertRaises(RuntimeError):
ScalarEncoder(p)
p.activeBits = 2
p.maximum = -1 # Maximum is less than the minimum
with self.assertRaises(RuntimeError):
ScalarEncoder(p)
p.maximum = 1
p.size = 0
p.activeBits = 0
p.sparsity = .1 # Specify sparsity without output size
with self.assertRaises(RuntimeError):
ScalarEncoder(p)
p.size = 10
p.activeBits = 2
p.sparsity = 0
p.sparsity = .2 # Sparsity & num activeBits specified
with self.assertRaises(RuntimeError):
ScalarEncoder(p)
p.sparsity = 0
p.clipInput = True # Incompatible features...
p.periodic = True
with self.assertRaises(RuntimeError):
ScalarEncoder(p)
p.clipInput = False
p.periodic = False
p.radius = 1 # Size specified too many times
with self.assertRaises(RuntimeError):
ScalarEncoder(p)
p.radius = 0
p.resolution = 1 # Size specified too many times
with self.assertRaises(RuntimeError):
ScalarEncoder(p)
p.resolution = 0
def testStatistics(self):
p = ScalarEncoderParameters()
p.size = 100
p.activeBits = 10
p.minimum = 0
p.maximum = 20
p.clipInput = True
enc = ScalarEncoder( p )
del p
out = SDR( enc.parameters.size )
mtr = Metrics(out, 9999)
# The activation frequency of bits near the endpoints of the range is a
# little weird, because the bits at the very end are not used as often
# as the ones in the middle of the range, unless clipInputs is enabled.
# If clipInputs is enabled then the bits 1 radius from the end get used
# twice as often as the should because they respond to inputs off
# outside of the valid range as well as inputs inside of the range.
for i in np.linspace(
enc.parameters.minimum - enc.parameters.radius / 2,
enc.parameters.maximum + enc.parameters.radius / 2,
100 + 10 ):
enc.encode( i, out )
# print( i, out.sparse )
print(str(mtr))
assert( mtr.sparsity.min() > .95 * .10 )
assert( mtr.sparsity.max() < 1.05 * .10 )
assert( mtr.activationFrequency.min() > .50 * .10 )
assert( mtr.activationFrequency.max() < 1.75 * .10 )
assert( mtr.overlap.min() > .85 )
def testRadius(self):
p = ScalarEncoderParameters()
p.activeBits = 10
p.minimum = 0
p.maximum = 100
p.radius = 10
enc = ScalarEncoder(p)
sdr1 = SDR( enc.parameters.size )
sdr2 = SDR( enc.parameters.size )
enc.encode( 77, sdr1 )
enc.encode( 77, sdr2 )
assert( sdr1.getOverlap( sdr2 ) == 10 )
enc.encode( 0, sdr1 )
enc.encode( 1, sdr2 )
assert( sdr1.getOverlap( sdr2 ) == 9 )
enc.encode( 60, sdr1 )
enc.encode( 69, sdr2 )
assert( sdr1.getOverlap( sdr2 ) == 1 )
enc.encode( 45, sdr1 )
enc.encode( 55, sdr2 )
assert( sdr1.getOverlap( sdr2 ) == 0 )
def testResolution(self):
p = ScalarEncoderParameters()
p.activeBits = 10
p.minimum = 0
p.maximum = 100
p.resolution = .5
enc = ScalarEncoder(p)
sdr1 = SDR( enc.parameters.size )
sdr2 = SDR( enc.parameters.size )
enc.encode( .0, sdr1 )
enc.encode( .1, sdr2 )
assert( sdr1 == sdr2 )
enc.encode( .0, sdr1 )
enc.encode( .6, sdr2 )
assert( sdr1.getOverlap( sdr2 ) == 9 )
enc.encode( 70, sdr1 )
enc.encode( 72.5, sdr2 )
assert( sdr1.getOverlap( sdr2 ) == 5 )
enc.encode( 70, sdr1 )
enc.encode( 75, sdr2 )
assert( sdr1.getOverlap( sdr2 ) == 0 )
enc.encode( 60, sdr1 )
enc.encode( 80, sdr2 )
assert( sdr1.getOverlap( sdr2 ) == 0 )
def testNaNs(self):
p = ScalarEncoderParameters()
p.size = 100
p.activeBits = 10
p.minimum = 0
p.maximum = 100
enc = ScalarEncoder(p)
sdr = SDR( 100 )
enc.encode( float("nan"), sdr )
assert( sdr.getSum() == 0 )
def loadThingData(dataDir="data", n=150, w=11):
"""
Load Thing sensation data. There is one file per object, each row contains one
feature, location pairs. The format is as follows:
[(-33.6705, 75.5003, 2.4207)/10] => [[list of active bits of location],
[list of active bits of feature]]
The content before "=>" is the true 3D location / sensation
We ignore the encoded values after "=>" and use :class:`ScalarEncoder` to
encode the sensation in a way that is compatible with the experiment network.
:param dataDir: The location data files
:type dataDir: str
:param n: The number of bits in the feature SDR. Usually L4 column count
:type n: int
:param w: Number of 'on' bits in the feature SDR. Usually L4 sample size
:type w: int
:return: Dictionary mapping objects to sensations that can be used directly by
class L246aNetwork 'infer' and 'learn' methods
:rtype: dict[str,list]
"""
objects = defaultdict(list)
# Thing features are scalar values ranging from 1-25 inclusive
p = ScalarEncoderParameters()
p.size = n
p.activeBits = w
p.minimum = 1
p.maximum = 25
encoder = ScalarEncoder(p)
dataPath = os.path.dirname(os.path.realpath(__file__))
dataPath = os.path.join(dataPath, dataDir)
objFiles = glob.glob1(dataPath, "*.log")
for filename in objFiles:
obj, _ = os.path.splitext(filename)
# Read raw sensations from log file. Ignore SDRs after "=>"
sensations = []
with open(os.path.join(dataPath, filename)) as f:
for line in f.readlines():
# Parse raw location/feature values
line = line.split("=>")[0]
line = ''.join(i for i in line if not i in "[,]()")
locationStr, featureStr = line.split("/")
location = list(map(float, locationStr.split()))
feature = list(encoder.encode(int(featureStr)).sparse)
sensations.append((location, feature))
# Assume single column
objects[obj] = [sensations]
return objects
def testEncode(self):
p = ScalarEncoderParameters()
p.size = 10
p.activeBits = 3
p.minimum = 0
p.maximum = 1
enc = ScalarEncoder(p)
sdr = SDR( 10 )
enc.encode( 0, sdr )
assert( list(sdr.sparse) == [0, 1, 2] )
sdr2 = enc.encode( 1 )
assert( list(sdr2.sparse) == [7, 8, 9] )
def testConstructor(self):
p = ScalarEncoderParameters()
p.size = 1000
p.activeBits = 20
p.minimum = 0
p.maximum = 345
enc = ScalarEncoder( p )
assert( enc.dimensions == [1000] )
assert( enc.size == 1000 )
assert( not enc.parameters.clipInput )
assert( not enc.parameters.periodic )
assert( abs(enc.parameters.sparsity - 20./1000) < .01 )
assert( abs(enc.parameters.radius - 7) < 1 )
assert( abs(enc.parameters.resolution - .35) < .1 )
def testClipInput(self):
p = ScalarEncoderParameters()
p.size = 345
p.sparsity = .05
p.minimum = 0
p.maximum = 1
p.clipInput = 1
enc = ScalarEncoder(p)
sdr1 = SDR( 345 )
sdr2 = SDR( 345 )
enc.encode( 0, sdr1 )
enc.encode( -1, sdr2 )
assert( sdr1 == sdr2 )
enc.encode( 1, sdr1 )
enc.encode( 10, sdr2 )
assert( sdr1 == sdr2 )
def testBadEncode(self):
# Test bad SDR
p = ScalarEncoderParameters()
p.size = 10
p.activeBits = 2
p.minimum = 0
p.maximum = 1
enc = ScalarEncoder(p)
good = SDR( 10 )
bad = SDR( 5 )
enc.encode( .25, good )
with self.assertRaises(RuntimeError):
enc.encode( .25, bad )
# Test bad inputs, out of valid range & clipping disabled.
with self.assertRaises(RuntimeError):
enc.encode( -.0001, good )
with self.assertRaises(RuntimeError):
enc.encode( 1.0001, good )
def CreateSensationStream_sensations(self, sensorDirection, n, w,
positionStream):
stream = []
# Create scalar encoder to encode features
p = ScalarEncoderParameters()
p.size = n
p.activeBits = w
p.minimum = 0
p.maximum = 1
encoder = ScalarEncoder(p)
x = positionStream[:, 0]
y = positionStream[:, 1]
for i in range(len(x)):
self.agent.move(x[i], y[i])
f = self.agent.get_feature(sensorDirection)
feature = (('X', 'Y').index(f) + 1) if f is not None else 0
stream.append(([x[i], y[i]], list(encoder.encode(feature).sparse)))
return stream
def testPeriodic(self):
p = ScalarEncoderParameters()
p.size = 100
p.activeBits = 10
p.minimum = 0
p.maximum = 20
p.periodic = True
enc = ScalarEncoder( p )
out = SDR( enc.parameters.size )
mtr = Metrics(out, 9999)
for i in range(201 * 10 + 1):
x = (i % 201) / 10.
enc.encode( x, out )
# print( x, out.sparse )
print(str(mtr))
assert( mtr.sparsity.min() > .95 * .10 )
assert( mtr.sparsity.max() < 1.05 * .10 )
assert( mtr.activationFrequency.min() > .9 * .10 )
assert( mtr.activationFrequency.max() < 1.1 * .10 )
assert( mtr.overlap.min() > .85 )
def __init__(self,
season=0,
dayOfWeek=0,
weekend=0,
holiday=0,
timeOfDay=0,
customDays=0,
holidays=((12, 25), )):
"""
Each parameter describes one attribute to encode. By default, the attribute
is not encoded.
Argument season: (int | tuple) Season of the year, where units = day.
- (int) width of attribute; default radius = 91.5 days (1 season)
- (tuple) season[0] = width; season[1] = radius
Argument dayOfWeek: (int | tuple) Day of week, where monday = 0, units = 1 day.
The timestamp is compared against day:noon, so encodings of a day switch representation on midnight.
- (int) width of attribute; default radius = 1 day
- (tuple) dayOfWeek[0] = width; dayOfWeek[1] = radius
Argument weekend: (int) Is a weekend or not. A block of bits either 0s or 1s.
Note: the implementation treats "weekend" as starting Fri 6pm, till Sun midnight.
- (int) width of attribute
- TODO remove and replace by customDays=(width, ["Saturday", "Sunday"]) ?
Argument holiday: (int) Is a holiday or not, boolean: 0, 1
- (int) width of attribute
Argument timeOfday: (int | tuple) Time of day, where midnight = 0, units = hour.
- (int) width of attribute: default radius = 4 hours
- (tuple) timeOfDay[0] = width; timeOfDay[1] = radius
Argument customDays: (tuple) A way to custom encode specific days of the week.
- [0] (int) Width of attribute
- [1] (str | list) Either a string representing a day of the week like
"Monday" or "mon", or a list of these strings.
Argument holidays: (list) a list of tuples for holidays.
- Each holiday is either (month, day) or (year, month, day).
The former will use the same month day every year eg: (12, 25) for Christmas.
The latter will be a one off holiday eg: (2018, 4, 1) for Easter Sunday 2018
- By default the only holiday is December 25.
"""
self.size = 0
self.seasonEncoder = None
if season != 0:
p = ScalarEncoderParameters()
# Ignore leapyear differences -- assume 366 days in a year
# Radius = 91.5 days = length of season
# Value is number of days since beginning of year (0 - 355)
p.minimum = 0
p.maximum = 366
p.periodic = True
try:
activeBits, radius = season
except TypeError:
p.activeBits = season
p.radius = 91.5
else:
p.activeBits = season[0]
p.radius = season[1]
self.seasonEncoder = ScalarEncoder(p)
self.size += self.seasonEncoder.size
self.dayOfWeekEncoder = None
if dayOfWeek != 0:
p = ScalarEncoderParameters()
# Value is day of week (floating point)
# Radius is 1 day
p.minimum = 0
p.maximum = 7
p.periodic = True
try:
activeBits, radius = dayOfWeek
except TypeError:
p.activeBits = dayOfWeek
p.radius = 1
else:
p.activeBits = dayOfWeek[0]
p.radius = dayOfWeek[1]
self.dayOfWeekEncoder = ScalarEncoder(p)
self.size += self.dayOfWeekEncoder.size
self.weekendEncoder = None
if weekend != 0:
p = ScalarEncoderParameters()
# Binary value.
p.minimum = 0
p.maximum = 1
p.category = True
p.activeBits = weekend
self.weekendEncoder = ScalarEncoder(p)
self.size += self.weekendEncoder.size
# Set up custom days encoder, first argument in tuple is width
# second is either a single day of the week or a list of the days
# you want encoded as ones.
self.customDaysEncoder = None
if customDays != 0:
daysToParse = []
assert len(
customDays) == 2, "Please provide a w and the desired days"
if isinstance(customDays[1], list):
daysToParse = customDays[1]
elif isinstance(customDays[1], str):
daysToParse = [customDays[1]]
else:
raise ValueError(
"You must provide either a list of days or a single day")
# Parse days
self.customDays = []
for day in daysToParse:
if (day.lower() in ["mon", "monday"]):
self.customDays += [0]
elif day.lower() in ["tue", "tuesday"]:
self.customDays += [1]
elif day.lower() in ["wed", "wednesday"]:
self.customDays += [2]
elif day.lower() in ["thu", "thursday"]:
self.customDays += [3]
elif day.lower() in ["fri", "friday"]:
self.customDays += [4]
elif day.lower() in ["sat", "saturday"]:
self.customDays += [5]
elif day.lower() in ["sun", "sunday"]:
self.customDays += [6]
else:
raise ValueError(
"Unable to understand %s as a day of week" % str(day))
p = ScalarEncoderParameters()
p.activeBits = customDays[0]
p.minimum = 0
p.maximum = 1
p.category = True
self.customDaysEncoder = ScalarEncoder(p)
self.size += self.customDaysEncoder.size
self.holidayEncoder = None
if holiday != 0:
p = ScalarEncoderParameters()
# A "continuous" binary value. = 1 on the holiday itself and smooth ramp
# 0->1 on the day before the holiday and 1->0 on the day after the
# holiday.
p.minimum = 0
p.maximum = 2
p.radius = 1
p.periodic = True
p.activeBits = holiday
self.holidayEncoder = ScalarEncoder(p)
self.size += self.holidayEncoder.size
for h in holidays:
if not (hasattr(h, "__getitem__") or len(h) not in [2, 3]):
raise ValueError(
"Holidays must be an iterable of length 2 or 3")
self.holidays = holidays
self.timeOfDayEncoder = None
if timeOfDay != 0:
p = ScalarEncoderParameters()
p.minimum = 0
p.maximum = 24
p.periodic = True
# Value is time of day in hours
# Radius = 4 hours, e.g. morning, afternoon, evening, early night, late
# night, etc.
try:
activeBits, radius = timeOfDay
except TypeError:
p.activeBits = timeOfDay
p.radius = 4
else:
p.activeBits = timeOfDay[0]
p.radius = timeOfDay[1]
self.timeOfDayEncoder = ScalarEncoder(p)
self.size += self.timeOfDayEncoder.size
self.dimensions = (self.size, )
assert (self.size > 0)