def testSensor(self):
# Create a simple network to test the sensor
rawParams = {"outputWidth": 1029}
net = Network()
rawSensor = net.addRegion("raw","py.RawSensor", json.dumps(rawParams))
vfe = net.addRegion("output","VectorFileEffector","")
net.link("raw", "output", "UniformLink", "")
self.assertEqual(rawSensor.getParameter("outputWidth"),1029,
"Incorrect outputWidth parameter")
# Add vectors to the queue using two different add methods. Later we
# will check to ensure these are actually output properly.
rawSensor.executeCommand(["addDataToQueue", "[2, 4, 6]", "0", "42"])
rawSensorPy = rawSensor.getSelf()
rawSensorPy.addDataToQueue([2, 42, 1023], 1, 43)
rawSensorPy.addDataToQueue([18, 19, 20], 0, 44)
# Set an output file before we run anything
vfe.setParameter("outputFile",os.path.join(self.tmpDir,"temp.csv"))
# Run the network and check outputs are as expected
net.run(1)
self.assertEqual(rawSensor.getOutputData("dataOut").nonzero()[0].sum(),
sum([2, 4, 6]), "Value of dataOut incorrect")
self.assertEqual(rawSensor.getOutputData("resetOut").sum(),0,
"Value of resetOut incorrect")
self.assertEqual( rawSensor.getOutputData("sequenceIdOut").sum(),42,
"Value of sequenceIdOut incorrect")
net.run(1)
self.assertEqual(rawSensor.getOutputData("dataOut").nonzero()[0].sum(),
sum([2, 42, 1023]), "Value of dataOut incorrect")
self.assertEqual(rawSensor.getOutputData("resetOut").sum(),1,
"Value of resetOut incorrect")
self.assertEqual( rawSensor.getOutputData("sequenceIdOut").sum(),43,
"Value of sequenceIdOut incorrect")
# Make sure we can save and load the network after running
net.save(os.path.join(self.tmpDir,"rawNetwork.nta"))
net2 = Network(os.path.join(self.tmpDir,"rawNetwork.nta"))
rawSensor2 = net2.regions.get("raw")
vfe2 = net2.regions.get("output")
# Ensure parameters are preserved
self.assertEqual(rawSensor2.getParameter("outputWidth"),1029,
"Incorrect outputWidth parameter")
# Ensure the queue is preserved through save/load
vfe2.setParameter("outputFile",os.path.join(self.tmpDir,"temp.csv"))
net2.run(1)
self.assertEqual(rawSensor2.getOutputData("dataOut").nonzero()[0].sum(),
sum([18, 19, 20]), "Value of dataOut incorrect")
self.assertEqual(rawSensor2.getOutputData("resetOut").sum(),0,
"Value of resetOut incorrect")
self.assertEqual( rawSensor2.getOutputData("sequenceIdOut").sum(),44,
"Value of sequenceIdOut incorrect")
def testSensor(self):
# Create a simple network to test the sensor
params = {
"activeBits": self.encoder.w,
"outputWidth": self.encoder.n,
"radius": 2,
"verbosity": self.encoder.verbosity,
}
net = Network()
region = net.addRegion("coordinate", "py.CoordinateSensorRegion", json.dumps(params))
vfe = net.addRegion("output", "VectorFileEffector", "")
net.link("coordinate", "output", "UniformLink", "")
self.assertEqual(region.getParameter("outputWidth"), self.encoder.n, "Incorrect outputWidth parameter")
# Add vectors to the queue using two different add methods. Later we
# will check to ensure these are actually output properly.
region.executeCommand(["addDataToQueue", "[2, 4, 6]", "0", "42"])
regionPy = region.getSelf()
regionPy.addDataToQueue([2, 42, 1023], 1, 43)
regionPy.addDataToQueue([18, 19, 20], 0, 44)
# Set an output file before we run anything
vfe.setParameter("outputFile", os.path.join(self.tmpDir, "temp.csv"))
# Run the network and check outputs are as expected
net.run(1)
expected = self.encoder.encode((numpy.array([2, 4, 6]), params["radius"]))
actual = region.getOutputData("dataOut")
self.assertEqual(actual.sum(), expected.sum(), "Value of dataOut incorrect")
self.assertEqual(region.getOutputData("resetOut"), 0, "Value of resetOut incorrect")
self.assertEqual(region.getOutputData("sequenceIdOut"), 42, "Value of sequenceIdOut incorrect")
net.run(1)
expected = self.encoder.encode((numpy.array([2, 42, 1023]), params["radius"]))
actual = region.getOutputData("dataOut")
self.assertEqual(actual.sum(), expected.sum(), "Value of dataOut incorrect")
self.assertEqual(region.getOutputData("resetOut"), 1, "Value of resetOut incorrect")
self.assertEqual(region.getOutputData("sequenceIdOut"), 43, "Value of sequenceIdOut incorrect")
# Make sure we can save and load the network after running
net.save(os.path.join(self.tmpDir, "coordinateNetwork.nta"))
net2 = Network(os.path.join(self.tmpDir, "coordinateNetwork.nta"))
region2 = net2.regions.get("coordinate")
vfe2 = net2.regions.get("output")
# Ensure parameters are preserved
self.assertEqual(region2.getParameter("outputWidth"), self.encoder.n, "Incorrect outputWidth parameter")
# Ensure the queue is preserved through save/load
vfe2.setParameter("outputFile", os.path.join(self.tmpDir, "temp.csv"))
net2.run(1)
expected = self.encoder.encode((numpy.array([18, 19, 20]), params["radius"]))
actual = region2.getOutputData("dataOut")
self.assertEqual(actual.sum(), expected.sum(), "Value of dataOut incorrect")
self.assertEqual(region2.getOutputData("resetOut"), 0, "Value of resetOut incorrect")
self.assertEqual(region2.getOutputData("sequenceIdOut"), 44, "Value of sequenceIdOut incorrect")
def _testNetLoad(self):
"""Test loading a network with this sensor in it."""
n = Network()
r = n.addRegion(self.nodeName, self.sensorName, '{ activeOutputCount: 11}')
r.dimensions = Dimensions([1])
n.save(self.filename)
n = Network(self.filename)
n.initialize()
self.testsPassed += 1
# Check that vectorCount parameter is zero
r = n.regions[self.nodeName]
res = r.getParameter('vectorCount')
self.assertEqual(
res, 0, "getting vectorCount:\n Expected '0', got back '%d'\n" % res)
self.sensor = r
def _testNetLoad(self):
"""Test loading a network with this sensor in it."""
n = Network()
r = n.addRegion(self.nodeName, self.sensorName, '{ activeOutputCount: 11}')
r.dimensions = Dimensions([1])
n.save(self.filename)
n = Network(self.filename)
n.initialize()
self.testsPassed += 1
# Check that vectorCount parameter is zero
r = n.regions[self.nodeName]
res = r.getParameter('vectorCount')
self.assertEqual(
res, 0, "getting vectorCount:\n Expected '0', got back '%d'\n" % res)
self.sensor = r
def testSerialization(self):
n = Network()
imageDims = (42, 38)
params = dict(width=imageDims[0],
height=imageDims[1],
mode="bw",
background=1,
invertOutput=1)
sensor = n.addRegion("sensor", "py.ImageSensor", json.dumps(params))
sensor.setDimensions(Dimensions(imageDims[0], imageDims[1]))
params = dict(inputShape=imageDims,
coincidencesShape=imageDims,
disableTemporal=1,
tpSeed=43,
spSeed=42,
nCellsPerCol=1)
l1 = n.addRegion("l1", "py.CLARegion", json.dumps(params))
params = dict(maxCategoryCount=48,
SVDSampleCount=400,
SVDDimCount=5,
distanceNorm=0.6)
_classifier = n.addRegion("classifier", "py.KNNClassifierRegion",
json.dumps(params))
# TODO: link params should not be required. Dest region dimensions are
# already specified as [1]
params = dict(mapping="in", rfSize=imageDims)
n.link("sensor", "l1", "UniformLink", json.dumps(params))
n.link("l1", "classifier", "UniformLink", "", "bottomUpOut",
"bottomUpIn")
n.link("sensor", "classifier", "UniformLink", "", "categoryOut",
"categoryIn")
n.initialize()
n.save("fdr.nta")
# Make sure the network bundle has all the expected files
self.assertTrue(os.path.exists("fdr.nta/network.yaml"))
self.assertTrue(os.path.exists("fdr.nta/R0-pkl"))
self.assertTrue(os.path.exists("fdr.nta/R1-pkl"))
self.assertTrue(os.path.exists("fdr.nta/R2-pkl"))
n2 = Network("fdr.nta")
n2.initialize() # should not fail
# Make sure the network is actually the same
sensor = n2.regions['sensor']
self.assertEqual(sensor.type, "py.ImageSensor")
# would like to directly compare, but can't -- NPC-6
self.assertEqual(str(sensor.dimensions), str(Dimensions(42, 38)))
self.assertEqual(sensor.getParameter("width"), 42)
self.assertEqual(sensor.getParameter("height"), 38)
self.assertEqual(sensor.getParameter("mode"), "bw")
self.assertEqual(sensor.getParameter("background"), 1)
self.assertEqual(sensor.getParameter("invertOutput"), 1)
l1 = n2.regions['l1']
self.assertEqual(l1.type, "py.CLARegion")
self.assertEqual(str(l1.dimensions), str(Dimensions(1)))
a = l1.getParameter("inputShape")
self.assertEqual(len(a), 2)
self.assertEqual(a[0], 42)
self.assertEqual(a[1], 38)
a = l1.getParameter("coincidencesShape")
self.assertEqual(len(a), 2)
self.assertEqual(a[0], 42)
self.assertEqual(a[1], 38)
self.assertEqual(l1.getParameter("disableTemporal"), 1)
self.assertEqual(l1.getParameter("spSeed"), 42)
self.assertEqual(l1.getParameter("tpSeed"), 43)
cl = n2.regions['classifier']
self.assertEqual(cl.type, "py.KNNClassifierRegion")
self.assertEqual(cl.getParameter("maxCategoryCount"), 48)
self.assertEqual(cl.getParameter("SVDSampleCount"), 400)
self.assertEqual(cl.getParameter("SVDDimCount"), 5)
self.assertLess((cl.getParameter("distanceNorm") - 0.6), 0.0001)
self.assertEqual(str(cl.dimensions), str(Dimensions(1)))
n2.save("fdr2.nta")
# now compare the two network bundles -- should be the same
c = filecmp.dircmp("fdr.nta", "fdr2.nta")
self.assertEqual(len(c.left_only), 0,
"fdr.nta has extra files: %s" % c.left_only)
self.assertEqual(len(c.right_only), 0,
"fdr2.nta has extra files: %s" % c.right_only)
if len(c.diff_files) > 0:
_LOGGER.warn(
"Some bundle files differ: %s\n"
"This is expected, as pickle.load() followed by "
"pickle.dump() doesn't produce the same file", c.diff_files)
class SaccadeNetwork(object):
"""
A HTM network structured as follows:
SaccadeSensor (RandomSaccade) -> SP -> TM -> Classifier (KNN)
"""
def __init__(self,
networkName,
trainingSet,
testingSet,
loggingDir=None,
validationSet=None,
detailedSaccadeWidth=IMAGE_WIDTH,
detailedSaccadeHeight=IMAGE_HEIGHT,
createNetwork=True):
"""
:param str networkName: Where the network will be serialized/saved to
:param str trainingSet: Path to set of images to train on
:param str testingSet: Path to set of images to test
:param str loggingDir: directory to store logged images in
(note: no image logging if none)
:param validationSet: (optional) Path to set of images to validate on
:param int detailedSaccadeWidth: (optional) Width of detailed saccades to
return from the runNetworkOneImage and testNetworkOneImage
:param int detailedSaccadeHeight: (optional) Height of detailed saccades to
return from the runNetworkOneImage and testNetworkOneImage
:param bool createNetwork: If false, wait until createNet is manually
called to create the network. Otherwise, create on __init__
"""
self.loggingDir = loggingDir
self.netFile = networkName
self.trainingSet = trainingSet
self.validationSet = validationSet
self.testingSet = testingSet
self.detailedSaccadeWidth = detailedSaccadeWidth
self.detailedSaccadeHeight = detailedSaccadeHeight
self.net = None
self.trainingImageIndex = None
self.networkDutyCycles = None
self.networkSensor = None
self.networkSP = None
self.networkTM = None
self.networkTP = None
self.networkClassifier = None
self.networkSensor = None
self.numTrainingImages = 0
self.numTestingImages = 0
self.trainingImageIndex = 0
self.testingImageIndex = 0
self.numCorrect = 0
if createNetwork:
self.createNet()
def createNet(self):
""" Set up the structure of the network """
net = Network()
Network.unregisterRegion(SaccadeSensor.__name__)
Network.registerRegion(SaccadeSensor)
Network.unregisterRegion(ExtendedTMRegion.__name__)
Network.registerRegion(ExtendedTMRegion)
Network.unregisterRegion(ColumnPoolerRegion.__name__)
Network.registerRegion(ColumnPoolerRegion)
imageSensorParams = copy.deepcopy(DEFAULT_IMAGESENSOR_PARAMS)
if self.loggingDir is not None:
imageSensorParams["logDir"] = "sensorImages/" + self.loggingDir
imageSensorParams["logOutputImages"] = 1
imageSensorParams["logOriginalImages"] = 1
imageSensorParams["logFilteredImages"] = 1
imageSensorParams["logLocationImages"] = 1
imageSensorParams["logLocationOnOriginalImage"] = 1
net.addRegion("sensor", "py.SaccadeSensor",
yaml.dump(imageSensorParams))
sensor = net.regions["sensor"].getSelf()
DEFAULT_SP_PARAMS["columnCount"] = sensor.getOutputElementCount(
"dataOut")
net.addRegion("SP", "py.SPRegion", yaml.dump(DEFAULT_SP_PARAMS))
sp = net.regions["SP"].getSelf()
DEFAULT_TM_PARAMS["columnDimensions"] = (
sp.getOutputElementCount("bottomUpOut"), )
DEFAULT_TM_PARAMS["basalInputWidth"] = sensor.getOutputElementCount(
"saccadeOut")
net.addRegion("TM", "py.ExtendedTMRegion",
yaml.dump(DEFAULT_TM_PARAMS))
net.addRegion("TP", "py.ColumnPoolerRegion",
yaml.dump(DEFAULT_TP_PARAMS))
net.addRegion("classifier", "py.KNNClassifierRegion",
yaml.dump(DEFAULT_CLASSIFIER_PARAMS))
net.link("sensor",
"SP",
"UniformLink",
"",
srcOutput="dataOut",
destInput="bottomUpIn")
net.link("SP",
"TM",
"UniformLink",
"",
srcOutput="bottomUpOut",
destInput="activeColumns")
net.link("sensor",
"TM",
"UniformLink",
"",
srcOutput="saccadeOut",
destInput="basalInput")
net.link("TM",
"TP",
"UniformLink",
"",
srcOutput="predictedActiveCells",
destInput="feedforwardInput")
net.link("TP",
"TM",
"UniformLink",
"",
srcOutput="feedForwardOutput",
destInput="apicalInput")
net.link("TP",
"classifier",
"UniformLink",
"",
srcOutput="feedForwardOutput",
destInput="bottomUpIn")
#net.link("TM", "classifier", "UniformLink", "",
# srcOutput="predictedActiveCells", destInput="bottomUpIn")
net.link("sensor",
"classifier",
"UniformLink",
"",
srcOutput="categoryOut",
destInput="categoryIn")
self.net = net
self.networkSensor = self.net.regions["sensor"]
self.networkSP = self.net.regions["SP"]
self.networkTM = self.net.regions["TM"]
self.networkTP = self.net.regions["TP"]
self.networkClassifier = self.net.regions["classifier"]
def loadFromFile(self, filename):
""" Load a serialized network
:param filename: Where the network should be loaded from
"""
print "Loading network from {file}...".format(file=filename)
Network.unregisterRegion(SaccadeSensor.__name__)
Network.registerRegion(SaccadeSensor)
Network.registerRegion(ExtendedTMRegion)
self.net = Network(filename)
self.networkSensor = self.net.regions["sensor"]
self.networkSensor.setParameter("numSaccades",
SACCADES_PER_IMAGE_TESTING)
self.networkSP = self.net.regions["SP"]
self.networkClassifier = self.net.regions["classifier"]
self.numCorrect = 0
def loadExperiment(self):
""" Load images into ImageSensor and set the learning mode for the SP. """
print "============= Loading training images ================="
t1 = time.time()
self.networkSensor.executeCommand(
["loadMultipleImages", self.trainingSet])
numTrainingImages = self.networkSensor.getParameter("numImages")
t2 = time.time()
print "Load time for training images:", t2 - t1
print "Number of training images", numTrainingImages
self.numTrainingImages = numTrainingImages
self.trainingImageIndex = 0
def runNetworkOneImage(self, enableViz=False):
""" Runs a single image through the network stepping through all saccades
:param bool enableViz: If true, visualizations are generated and returned
:return: If enableViz, return a tuple (saccadeImgsList, saccadeDetailList,
saccadeHistList). saccadeImgsList is a list of images with the fovea
highlighted. saccadeDetailList is a list of resized images showing the
contents of the fovea. saccadeHistList shows the fovea history.
If not enableViz, returns True
Regardless of enableViz, if False is returned, all images have been
saccaded over.
"""
if self.trainingImageIndex < self.numTrainingImages:
saccadeList = []
saccadeImgsList = []
saccadeHistList = []
saccadeDetailList = []
originalImage = None
self.networkTM.executeCommand(["reset"])
for i in range(SACCADES_PER_IMAGE_TRAINING):
self.net.run(1)
if originalImage is None:
originalImage = deserializeImage(
yaml.load(
self.networkSensor.getParameter("originalImage")))
imgCenter = (
originalImage.size[0] / 2,
originalImage.size[1] / 2,
)
saccadeList.append({
"offset1": (yaml.load(
self.networkSensor.getParameter("prevSaccadeInfo"))
["prevOffset"]),
"offset2": (yaml.load(
self.networkSensor.getParameter("prevSaccadeInfo"))
["newOffset"])
})
if enableViz:
detailImage = deserializeImage(
yaml.load(
self.networkSensor.getParameter("outputImage")))
detailImage = detailImage.resize(
(self.detailedSaccadeWidth,
self.detailedSaccadeHeight), Image.ANTIALIAS)
saccadeDetailList.append(ImageTk.PhotoImage(detailImage))
imgWithSaccade = originalImage.convert("RGB")
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)),
fill=(255, 0, 0),
width=1) # Left
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)),
fill=(255, 0, 0),
width=1) # Right
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2)),
fill=(255, 0, 0),
width=1) # Top
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)),
fill=(255, 0, 0),
width=1) # Bottom
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)),
fill=(0, 255, 0),
width=1) # Left
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)),
fill=(0, 255, 0),
width=1) # Right
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2)),
fill=(0, 255, 0),
width=1) # Top
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)),
fill=(0, 255, 0),
width=1) # Bottom
saccadeImgsList.append(ImageTk.PhotoImage(imgWithSaccade))
saccadeHist = originalImage.convert("RGB")
for i, saccade in enumerate(saccadeList):
ImageDraw.Draw(saccadeHist).rectangle(
(imgCenter[0] + saccade["offset2"][0] -
_FOVEA_SIZE / 2, imgCenter[0] +
saccade["offset2"][1] - _FOVEA_SIZE / 2,
imgCenter[0] + saccade["offset2"][0] +
_FOVEA_SIZE / 2, imgCenter[0] +
saccade["offset2"][1] + _FOVEA_SIZE / 2),
fill=(0, (255 / SACCADES_PER_IMAGE_TRAINING *
(SACCADES_PER_IMAGE_TRAINING - i)),
(255 / SACCADES_PER_IMAGE_TRAINING * i)))
saccadeHist = saccadeHist.resize(
(self.detailedSaccadeWidth,
self.detailedSaccadeHeight), Image.ANTIALIAS)
saccadeHistList.append(ImageTk.PhotoImage(saccadeHist))
self.trainingImageIndex += 1
print("Iteration: {iter}; Category: {cat}".format(
iter=self.trainingImageIndex,
cat=self.networkSensor.getOutputData("categoryOut")))
if enableViz:
return (saccadeImgsList, saccadeDetailList, saccadeHistList,
self.networkSensor.getOutputData("categoryOut"))
return True
else:
return False
def runNetworkBatch(self, batchSize):
""" Run the network in batches.
:param batchSize: Number of images to show in this batch
:return: True if there are more images left to be saccaded over.
Otherwise False.
"""
startTime = time.time()
while self.trainingImageIndex < self.numTrainingImages:
self.networkTM.executeCommand(["reset"])
for i in range(SACCADES_PER_IMAGE_TRAINING):
self.net.run(1)
self.trainingImageIndex += 1
if self.trainingImageIndex % batchSize == 0:
print(
"Iteration: {iter}; Category: {cat}; Time per batch: {t}".
format(iter=self.trainingImageIndex,
cat=self.networkSensor.getOutputData("categoryOut"),
t=time.time() - startTime))
return True
return False
def setupNetworkTest(self):
self.networkSensor.executeCommand(
["loadMultipleImages", self.testingSet])
self.numTestingImages = self.networkSensor.getParameter("numImages")
self.testingImageIndex = 0
print "NumTestingImages {test}".format(test=self.numTestingImages)
def testNetworkOneImage(self, enableViz=False):
if self.testingImageIndex < self.numTestingImages:
saccadeList = []
saccadeImgsList = []
saccadeHistList = []
saccadeDetailList = []
inferredCategoryList = []
originalImage = None
self.networkTM.executeCommand(["reset"])
for i in range(SACCADES_PER_IMAGE_TESTING):
self.net.run(1)
if originalImage is None:
originalImage = deserializeImage(
yaml.load(
self.networkSensor.getParameter("originalImage")))
imgCenter = (
originalImage.size[0] / 2,
originalImage.size[1] / 2,
)
saccadeList.append({
"offset1": (yaml.load(
self.networkSensor.getParameter("prevSaccadeInfo"))
["prevOffset"]),
"offset2": (yaml.load(
self.networkSensor.getParameter("prevSaccadeInfo"))
["newOffset"])
})
inferredCategoryList.append(
self.networkClassifier.getOutputData(
"categoriesOut").argmax())
if enableViz:
detailImage = deserializeImage(
yaml.load(
self.networkSensor.getParameter("outputImage")))
detailImage = detailImage.resize(
(self.detailedSaccadeWidth,
self.detailedSaccadeHeight), Image.ANTIALIAS)
saccadeDetailList.append(ImageTk.PhotoImage(detailImage))
imgWithSaccade = originalImage.convert("RGB")
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)),
fill=(255, 0, 0),
width=1) # Left
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)),
fill=(255, 0, 0),
width=1) # Right
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2)),
fill=(255, 0, 0),
width=1) # Top
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)),
fill=(255, 0, 0),
width=1) # Bottom
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)),
fill=(0, 255, 0),
width=1) # Left
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)),
fill=(0, 255, 0),
width=1) # Right
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2)),
fill=(0, 255, 0),
width=1) # Top
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] +
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] -
(_FOVEA_SIZE / 2), imgCenter[1] +
saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)),
fill=(0, 255, 0),
width=1) # Bottom
saccadeImgsList.append(ImageTk.PhotoImage(imgWithSaccade))
saccadeHist = originalImage.convert("RGB")
for i, saccade in enumerate(saccadeList):
ImageDraw.Draw(saccadeHist).rectangle(
(imgCenter[0] + saccade["offset2"][0] -
_FOVEA_SIZE / 2, imgCenter[0] +
saccade["offset2"][1] - _FOVEA_SIZE / 2,
imgCenter[0] + saccade["offset2"][0] +
_FOVEA_SIZE / 2, imgCenter[0] +
saccade["offset2"][1] + _FOVEA_SIZE / 2),
fill=(0, (255 / SACCADES_PER_IMAGE_TESTING *
(SACCADES_PER_IMAGE_TESTING - i)),
(255 / SACCADES_PER_IMAGE_TESTING * i)))
saccadeHist = saccadeHist.resize(
(self.detailedSaccadeWidth,
self.detailedSaccadeHeight), Image.ANTIALIAS)
saccadeHistList.append(ImageTk.PhotoImage(saccadeHist))
inferredCategory = self._getMostCommonCategory(
inferredCategoryList)
isCorrectClassification = False
if self.networkSensor.getOutputData(
"categoryOut") == inferredCategory:
isCorrectClassification = True
self.numCorrect += 1
print("Iteration: {iter}; Category: {cat}".format(
iter=self.testingImageIndex,
cat=self.networkSensor.getOutputData("categoryOut")))
self.testingImageIndex += 1
if enableViz:
return (saccadeImgsList, saccadeDetailList, saccadeHistList,
inferredCategoryList,
self.networkSensor.getOutputData("categoryOut"),
isCorrectClassification)
return (True, isCorrectClassification)
else:
return False
def testNetworkBatch(self, batchSize):
if self.testingImageIndex >= self.numTestingImages:
return False
while self.testingImageIndex < self.numTestingImages:
inferredCategoryList = []
self.networkTM.executeCommand(["reset"])
for i in range(SACCADES_PER_IMAGE_TESTING):
self.net.run(1)
inferredCategoryList.append(
self.networkClassifier.getOutputData(
"categoriesOut").argmax())
inferredCategory = self._getMostCommonCategory(
inferredCategoryList)
if self.networkSensor.getOutputData(
"categoryOut") == inferredCategory:
self.numCorrect += 1
self.testingImageIndex += 1
if self.testingImageIndex % batchSize == 0:
print("Testing iteration: {iter}".format(
iter=self.testingImageIndex))
break
return self.numCorrect
@staticmethod
def _getMostCommonCategory(categoryList):
return collections.Counter(categoryList).most_common(1)[0][0]
def setLearningMode(self,
learningSP=False,
learningTM=False,
learningTP=False,
learningClassifier=False):
if learningSP:
self.networkSP.setParameter("learningMode", 1)
self.networkSP.setParameter("inferenceMode", 0)
else:
self.networkSP.setParameter("learningMode", 0)
self.networkSP.setParameter("inferenceMode", 1)
if learningTM:
self.networkTM.setParameter("learn", 1)
else:
self.networkTM.setParameter("learn", 0)
if learningTM:
self.networkTP.setParameter("learningMode", 1)
else:
self.networkTP.setParameter("learningMode", 0)
if learningClassifier:
self.networkClassifier.setParameter("learningMode", 1)
self.networkClassifier.setParameter("inferenceMode", 0)
else:
self.networkClassifier.setParameter("learningMode", 0)
self.networkClassifier.setParameter("inferenceMode", 1)
def saveNetwork(self):
print "Saving network at {path}".format(path=self.netFile)
self.net.save(self.netFile)
def resetIndex(self):
self.trainingImageIndex = 0
class SaccadeNetwork(object):
"""
A HTM network structured as follows:
SaccadeSensor (RandomSaccade) -> SP -> TM -> Classifier (KNN)
"""
def __init__(self,
networkName,
trainingSet,
testingSet,
loggingDir = None,
validationSet=None,
detailedSaccadeWidth=IMAGE_WIDTH,
detailedSaccadeHeight=IMAGE_HEIGHT,
createNetwork=True):
"""
:param str networkName: Where the network will be serialized/saved to
:param str trainingSet: Path to set of images to train on
:param str testingSet: Path to set of images to test
:param str loggingDir: directory to store logged images in
(note: no image logging if none)
:param validationSet: (optional) Path to set of images to validate on
:param int detailedSaccadeWidth: (optional) Width of detailed saccades to
return from the runNetworkOneImage and testNetworkOneImage
:param int detailedSaccadeHeight: (optional) Height of detailed saccades to
return from the runNetworkOneImage and testNetworkOneImage
:param bool createNetwork: If false, wait until createNet is manually
called to create the network. Otherwise, create on __init__
"""
self.loggingDir = loggingDir
self.netFile = networkName
self.trainingSet = trainingSet
self.validationSet = validationSet
self.testingSet = testingSet
self.detailedSaccadeWidth = detailedSaccadeWidth
self.detailedSaccadeHeight = detailedSaccadeHeight
self.net = None
self.trainingImageIndex = None
self.networkClassifier = None
self.networkDutyCycles = None
self.networkSP = None
self.networkTM = None
self.networkSensor = None
self.numTrainingImages = 0
self.numTestingImages = 0
self.trainingImageIndex = 0
self.testingImageIndex = 0
self.numCorrect = 0
if createNetwork:
self.createNet()
def createNet(self):
""" Set up the structure of the network """
net = Network()
Network.unregisterRegion(SaccadeSensor.__name__)
Network.registerRegion(SaccadeSensor)
Network.registerRegion(TMRegion)
imageSensorParams = copy.deepcopy(DEFAULT_IMAGESENSOR_PARAMS)
if self.loggingDir is not None:
imageSensorParams["logDir"] = "sensorImages/" + self.loggingDir
imageSensorParams["logOutputImages"] = 1
imageSensorParams["logOriginalImages"] = 1
imageSensorParams["logFilteredImages"] = 1
imageSensorParams["logLocationImages"] = 1
imageSensorParams["logLocationOnOriginalImage"] = 1
net.addRegion("sensor", "py.SaccadeSensor",
yaml.dump(imageSensorParams))
sensor = net.regions["sensor"].getSelf()
DEFAULT_SP_PARAMS["columnCount"] = sensor.getOutputElementCount("dataOut")
net.addRegion("SP", "py.SPRegion", yaml.dump(DEFAULT_SP_PARAMS))
sp = net.regions["SP"].getSelf()
DEFAULT_TM_PARAMS["columnDimensions"] = (sp.getOutputElementCount("bottomUpOut"),)
net.addRegion("TM", "py.TMRegion", yaml.dump(DEFAULT_TM_PARAMS))
net.addRegion("classifier","py.KNNClassifierRegion",
yaml.dump(DEFAULT_CLASSIFIER_PARAMS))
net.link("sensor", "SP", "UniformLink", "",
srcOutput="dataOut", destInput="bottomUpIn")
net.link("SP", "TM", "UniformLink", "",
srcOutput="bottomUpOut", destInput="activeColumns")
net.link("sensor", "TM", "UniformLink", "",
srcOutput="saccadeOut", destInput="activeExternalCells")
net.link("TM", "classifier", "UniformLink", "",
srcOutput="predictedActiveCells", destInput="bottomUpIn")
net.link("sensor", "classifier", "UniformLink", "",
srcOutput="categoryOut", destInput="categoryIn")
self.net = net
self.networkSensor = self.net.regions["sensor"]
self.networkSP = self.net.regions["SP"]
self.networkTM = self.net.regions["TM"]
self.networkClassifier = self.net.regions["classifier"]
def loadFromFile(self, filename):
""" Load a serialized network
:param filename: Where the network should be loaded from
"""
print "Loading network from {file}...".format(file=filename)
Network.unregisterRegion(SaccadeSensor.__name__)
Network.registerRegion(SaccadeSensor)
Network.registerRegion(TMRegion)
self.net = Network(filename)
self.networkSensor = self.net.regions["sensor"]
self.networkSensor.setParameter("numSaccades", SACCADES_PER_IMAGE_TESTING)
self.networkSP = self.net.regions["SP"]
self.networkClassifier = self.net.regions["classifier"]
self.numCorrect = 0
def loadExperiment(self):
""" Load images into ImageSensor and set the learning mode for the SP. """
print "============= Loading training images ================="
t1 = time.time()
self.networkSensor.executeCommand(
["loadMultipleImages", self.trainingSet])
numTrainingImages = self.networkSensor.getParameter("numImages")
t2 = time.time()
print "Load time for training images:", t2-t1
print "Number of training images", numTrainingImages
self.numTrainingImages = numTrainingImages
self.trainingImageIndex = 0
def runNetworkOneImage(self, enableViz=False):
""" Runs a single image through the network stepping through all saccades
:param bool enableViz: If true, visualizations are generated and returned
:return: If enableViz, return a tuple (saccadeImgsList, saccadeDetailList,
saccadeHistList). saccadeImgsList is a list of images with the fovea
highlighted. saccadeDetailList is a list of resized images showing the
contents of the fovea. saccadeHistList shows the fovea history.
If not enableViz, returns True
Regardless of enableViz, if False is returned, all images have been
saccaded over.
"""
if self.trainingImageIndex < self.numTrainingImages:
saccadeList = []
saccadeImgsList = []
saccadeHistList = []
saccadeDetailList = []
originalImage = None
self.networkTM.executeCommand(["reset"])
for i in range(SACCADES_PER_IMAGE_TRAINING):
self.net.run(1)
if originalImage is None:
originalImage = deserializeImage(
yaml.load(self.networkSensor.getParameter("originalImage")))
imgCenter = (originalImage.size[0] / 2,
originalImage.size[1] / 2,)
saccadeList.append({
"offset1":
(yaml.load(
self.networkSensor
.getParameter("prevSaccadeInfo"))
["prevOffset"]),
"offset2":
(yaml.load(
self.networkSensor
.getParameter("prevSaccadeInfo"))
["newOffset"])})
if enableViz:
detailImage = deserializeImage(
yaml.load(self.networkSensor.getParameter("outputImage")))
detailImage = detailImage.resize((self.detailedSaccadeWidth,
self.detailedSaccadeHeight),
Image.ANTIALIAS)
saccadeDetailList.append(ImageTk.PhotoImage(detailImage))
imgWithSaccade = originalImage.convert("RGB")
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)),
fill=(255, 0, 0), width=1) # Left
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)),
fill=(255, 0, 0), width=1) # Right
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2)),
fill=(255, 0, 0), width=1) # Top
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)),
fill=(255, 0, 0), width=1) # Bottom
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)),
fill=(0, 255, 0), width=1) # Left
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)),
fill=(0, 255, 0), width=1) # Right
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2)),
fill=(0, 255, 0), width=1) # Top
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)),
fill=(0, 255, 0), width=1) # Bottom
saccadeImgsList.append(ImageTk.PhotoImage(imgWithSaccade))
saccadeHist = originalImage.convert("RGB")
for i, saccade in enumerate(saccadeList):
ImageDraw.Draw(saccadeHist).rectangle(
(imgCenter[0] + saccade["offset2"][0] - _FOVEA_SIZE/2,
imgCenter[0] + saccade["offset2"][1] - _FOVEA_SIZE/2,
imgCenter[0] + saccade["offset2"][0] + _FOVEA_SIZE/2,
imgCenter[0] + saccade["offset2"][1] + _FOVEA_SIZE/2),
fill=(0,
(255/SACCADES_PER_IMAGE_TRAINING*(SACCADES_PER_IMAGE_TRAINING-i)),
(255/SACCADES_PER_IMAGE_TRAINING*i)))
saccadeHist = saccadeHist.resize((self.detailedSaccadeWidth,
self.detailedSaccadeHeight),
Image.ANTIALIAS)
saccadeHistList.append(ImageTk.PhotoImage(saccadeHist))
self.trainingImageIndex += 1
print ("Iteration: {iter}; Category: {cat}"
.format(iter=self.trainingImageIndex,
cat=self.networkSensor.getOutputData("categoryOut")))
if enableViz:
return (saccadeImgsList, saccadeDetailList, saccadeHistList,
self.networkSensor.getOutputData("categoryOut"))
return True
else:
return False
def runNetworkBatch(self, batchSize):
""" Run the network in batches.
:param batchSize: Number of images to show in this batch
:return: True if there are more images left to be saccaded over.
Otherwise False.
"""
startTime = time.time()
while self.trainingImageIndex < self.numTrainingImages:
self.networkTM.executeCommand(["reset"])
for i in range(SACCADES_PER_IMAGE_TRAINING):
self.net.run(1)
self.trainingImageIndex += 1
if self.trainingImageIndex % batchSize == 0:
print ("Iteration: {iter}; Category: {cat}; Time per batch: {t}"
.format(iter=self.trainingImageIndex,
cat=self.networkSensor.getOutputData("categoryOut"),
t=time.time()-startTime))
return True
return False
def setupNetworkTest(self):
self.networkSensor.executeCommand(["loadMultipleImages", self.testingSet])
self.numTestingImages = self.networkSensor.getParameter("numImages")
self.testingImageIndex = 0
print "NumTestingImages {test}".format(test=self.numTestingImages)
def testNetworkOneImage(self, enableViz=False):
if self.testingImageIndex < self.numTestingImages:
saccadeList = []
saccadeImgsList = []
saccadeHistList = []
saccadeDetailList = []
inferredCategoryList = []
originalImage = None
self.networkTM.executeCommand(["reset"])
for i in range(SACCADES_PER_IMAGE_TESTING):
self.net.run(1)
if originalImage is None:
originalImage = deserializeImage(
yaml.load(self.networkSensor.getParameter("originalImage")))
imgCenter = (originalImage.size[0] / 2,
originalImage.size[1] / 2,)
saccadeList.append({
"offset1":
(yaml.load(
self.networkSensor
.getParameter("prevSaccadeInfo"))
["prevOffset"]),
"offset2":
(yaml.load(
self.networkSensor
.getParameter("prevSaccadeInfo"))
["newOffset"])})
inferredCategoryList.append(
self.networkClassifier.getOutputData("categoriesOut").argmax())
if enableViz:
detailImage = deserializeImage(
yaml.load(self.networkSensor.getParameter("outputImage")))
detailImage = detailImage.resize((self.detailedSaccadeWidth,
self.detailedSaccadeHeight),
Image.ANTIALIAS)
saccadeDetailList.append(ImageTk.PhotoImage(detailImage))
imgWithSaccade = originalImage.convert("RGB")
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)),
fill=(255, 0, 0), width=1) # Left
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)),
fill=(255, 0, 0), width=1) # Right
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2)),
fill=(255, 0, 0), width=1) # Top
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)),
fill=(255, 0, 0), width=1) # Bottom
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)),
fill=(0, 255, 0), width=1) # Left
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)),
fill=(0, 255, 0), width=1) # Right
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2)),
fill=(0, 255, 0), width=1) # Top
ImageDraw.Draw(imgWithSaccade).line(
(imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2),
imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2),
imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)),
fill=(0, 255, 0), width=1) # Bottom
saccadeImgsList.append(ImageTk.PhotoImage(imgWithSaccade))
saccadeHist = originalImage.convert("RGB")
for i, saccade in enumerate(saccadeList):
ImageDraw.Draw(saccadeHist).rectangle(
(imgCenter[0] + saccade["offset2"][0] - _FOVEA_SIZE/2,
imgCenter[0] + saccade["offset2"][1] - _FOVEA_SIZE/2,
imgCenter[0] + saccade["offset2"][0] + _FOVEA_SIZE/2,
imgCenter[0] + saccade["offset2"][1] + _FOVEA_SIZE/2),
fill=(0,
(255/SACCADES_PER_IMAGE_TESTING*(SACCADES_PER_IMAGE_TESTING-i)),
(255/SACCADES_PER_IMAGE_TESTING*i)))
saccadeHist = saccadeHist.resize((self.detailedSaccadeWidth,
self.detailedSaccadeHeight),
Image.ANTIALIAS)
saccadeHistList.append(ImageTk.PhotoImage(saccadeHist))
inferredCategory = self._getMostCommonCategory(inferredCategoryList)
isCorrectClassification = False
if self.networkSensor.getOutputData("categoryOut") == inferredCategory:
isCorrectClassification = True
self.numCorrect += 1
print ("Iteration: {iter}; Category: {cat}"
.format(iter=self.testingImageIndex,
cat=self.networkSensor.getOutputData("categoryOut")))
self.testingImageIndex += 1
if enableViz:
return (saccadeImgsList, saccadeDetailList, saccadeHistList,
inferredCategoryList,
self.networkSensor.getOutputData("categoryOut"),
isCorrectClassification)
return (True, isCorrectClassification)
else:
return False
def testNetworkBatch(self, batchSize):
if self.testingImageIndex >= self.numTestingImages:
return False
while self.testingImageIndex < self.numTestingImages:
inferredCategoryList = []
self.networkTM.executeCommand(["reset"])
for i in range(SACCADES_PER_IMAGE_TESTING):
self.net.run(1)
inferredCategoryList.append(
self.networkClassifier.getOutputData("categoriesOut").argmax())
inferredCategory = self._getMostCommonCategory(inferredCategoryList)
if self.networkSensor.getOutputData("categoryOut") == inferredCategory:
self.numCorrect += 1
self.testingImageIndex += 1
if self.testingImageIndex % batchSize == 0:
print ("Testing iteration: {iter}"
.format(iter=self.testingImageIndex))
break
return self.numCorrect
@staticmethod
def _getMostCommonCategory(categoryList):
return collections.Counter(categoryList).most_common(1)[0][0]
def setLearningMode(self,
learningSP=False,
learningTM=False,
learningClassifier=False):
if learningSP:
self.networkSP.setParameter("learningMode", 1)
self.networkSP.setParameter("inferenceMode", 0)
else:
self.networkSP.setParameter("learningMode", 0)
self.networkSP.setParameter("inferenceMode", 1)
if learningTM:
self.networkTM.setParameter("learningMode", 1)
else:
self.networkTM.setParameter("learningMode", 0)
if learningClassifier:
self.networkClassifier.setParameter("learningMode", 1)
self.networkClassifier.setParameter("inferenceMode", 0)
else:
self.networkClassifier.setParameter("learningMode", 0)
self.networkClassifier.setParameter("inferenceMode", 1)
def saveNetwork(self):
print "Saving network at {path}".format(path=self.netFile)
self.net.save(self.netFile)
def resetIndex(self):
self.trainingImageIndex = 0
def testSerialization(self):
n = Network()
imageDims = (42, 38)
params = dict(
width=imageDims[0],
height=imageDims[1],
mode="bw",
background=1,
invertOutput=1)
sensor = n.addRegion("sensor", "py.ImageSensor", json.dumps(params))
sensor.setDimensions(Dimensions(imageDims[0], imageDims[1]))
params = dict(
inputShape=imageDims,
coincidencesShape=imageDims,
disableTemporal=1,
tpSeed=43,
spSeed=42,
nCellsPerCol=1)
l1 = n.addRegion("l1", "py.CLARegion", json.dumps(params))
params = dict(
maxCategoryCount=48,
SVDSampleCount=400,
SVDDimCount=5,
distanceNorm=0.6)
_classifier = n.addRegion("classifier", "py.KNNClassifierRegion",
json.dumps(params))
# TODO: link params should not be required. Dest region dimensions are
# already specified as [1]
params = dict(
mapping="in",
rfSize=imageDims)
n.link("sensor", "l1", "UniformLink", json.dumps(params))
n.link("l1", "classifier", "UniformLink", "", "bottomUpOut", "bottomUpIn")
n.link("sensor", "classifier", "UniformLink", "", "categoryOut",
"categoryIn")
n.initialize()
n.save("fdr.nta")
# Make sure the network bundle has all the expected files
self.assertTrue(os.path.exists("fdr.nta/network.yaml"))
self.assertTrue(os.path.exists("fdr.nta/R0-pkl"))
self.assertTrue(os.path.exists("fdr.nta/R1-pkl"))
self.assertTrue(os.path.exists("fdr.nta/R2-pkl"))
n2 = Network("fdr.nta")
n2.initialize() # should not fail
# Make sure the network is actually the same
sensor = n2.regions['sensor']
self.assertEqual(sensor.type, "py.ImageSensor")
# would like to directly compare, but can't -- NPC-6
self.assertEqual(str(sensor.dimensions), str(Dimensions(42, 38)))
self.assertEqual(sensor.getParameter("width"), 42)
self.assertEqual(sensor.getParameter("height"), 38)
self.assertEqual(sensor.getParameter("mode"), "bw")
self.assertEqual(sensor.getParameter("background"), 1)
self.assertEqual(sensor.getParameter("invertOutput"), 1)
l1 = n2.regions['l1']
self.assertEqual(l1.type, "py.CLARegion")
self.assertEqual(str(l1.dimensions), str(Dimensions(1)))
a = l1.getParameter("inputShape")
self.assertEqual(len(a), 2)
self.assertEqual(a[0], 42)
self.assertEqual(a[1], 38)
a = l1.getParameter("coincidencesShape")
self.assertEqual(len(a), 2)
self.assertEqual(a[0], 42)
self.assertEqual(a[1], 38)
self.assertEqual(l1.getParameter("disableTemporal"), 1)
self.assertEqual(l1.getParameter("spSeed"), 42)
self.assertEqual(l1.getParameter("tpSeed"), 43)
cl = n2.regions['classifier']
self.assertEqual(cl.type, "py.KNNClassifierRegion")
self.assertEqual(cl.getParameter("maxCategoryCount"), 48)
self.assertEqual(cl.getParameter("SVDSampleCount"), 400)
self.assertEqual(cl.getParameter("SVDDimCount"), 5)
self.assertLess((cl.getParameter("distanceNorm") - 0.6), 0.0001)
self.assertEqual(str(cl.dimensions), str(Dimensions(1)))
n2.save("fdr2.nta")
# now compare the two network bundles -- should be the same
c = filecmp.dircmp("fdr.nta", "fdr2.nta")
self.assertEqual(len(c.left_only), 0,
"fdr.nta has extra files: %s" % c.left_only)
self.assertEqual(len(c.right_only), 0,
"fdr2.nta has extra files: %s" % c.right_only)
if len(c.diff_files) > 0:
_LOGGER.warn("Some bundle files differ: %s\n"
"This is expected, as pickle.load() followed by "
"pickle.dump() doesn't produce the same file", c.diff_files)
def testSensor(self):
# Create a simple network to test the sensor
params = {
"activeBits": self.encoder.w,
"outputWidth": self.encoder.n,
"radius": 2,
"verbosity": self.encoder.verbosity,
}
net = Network()
region = net.addRegion("coordinate", "py.CoordinateSensorRegion",
json.dumps(params))
vfe = net.addRegion("output", "VectorFileEffector", "")
net.link("coordinate", "output", "UniformLink", "")
self.assertEqual(region.getParameter("outputWidth"),
self.encoder.n, "Incorrect outputWidth parameter")
# Add vectors to the queue using two different add methods. Later we
# will check to ensure these are actually output properly.
region.executeCommand(["addDataToQueue", "[2, 4, 6]", "0", "42"])
regionPy = region.getSelf()
regionPy.addDataToQueue([2, 42, 1023], 1, 43)
regionPy.addDataToQueue([18, 19, 20], 0, 44)
# Set an output file before we run anything
vfe.setParameter("outputFile", os.path.join(self.tmpDir, "temp.csv"))
# Run the network and check outputs are as expected
net.run(1)
expected = self.encoder.encode((numpy.array([2, 4, 6]), params["radius"]))
actual = region.getOutputData("dataOut")
self.assertEqual(actual.sum(), expected.sum(), "Value of dataOut incorrect")
self.assertEqual(region.getOutputData("resetOut"), 0,
"Value of resetOut incorrect")
self.assertEqual(region.getOutputData("sequenceIdOut"), 42,
"Value of sequenceIdOut incorrect")
net.run(1)
expected = self.encoder.encode((numpy.array([2, 42, 1023]),
params["radius"]))
actual = region.getOutputData("dataOut")
self.assertEqual(actual.sum(), expected.sum(), "Value of dataOut incorrect")
self.assertEqual(region.getOutputData("resetOut"), 1,
"Value of resetOut incorrect")
self.assertEqual(region.getOutputData("sequenceIdOut"), 43,
"Value of sequenceIdOut incorrect")
# Make sure we can save and load the network after running
net.save(os.path.join(self.tmpDir, "coordinateNetwork.nta"))
net2 = Network(os.path.join(self.tmpDir, "coordinateNetwork.nta"))
region2 = net2.regions.get("coordinate")
vfe2 = net2.regions.get("output")
# Ensure parameters are preserved
self.assertEqual(region2.getParameter("outputWidth"), self.encoder.n,
"Incorrect outputWidth parameter")
# Ensure the queue is preserved through save/load
vfe2.setParameter("outputFile", os.path.join(self.tmpDir, "temp.csv"))
net2.run(1)
expected = self.encoder.encode((numpy.array([18, 19, 20]),
params["radius"]))
actual = region2.getOutputData("dataOut")
self.assertEqual(actual.sum(), expected.sum(), "Value of dataOut incorrect")
self.assertEqual(region2.getOutputData("resetOut"), 0,
"Value of resetOut incorrect")
self.assertEqual(region2.getOutputData("sequenceIdOut"), 44,
"Value of sequenceIdOut incorrect")
