def testSensor(self): # Create a simple network to test the sensor rawParams = {"outputWidth": 1029} net = Network() rawSensor = net.addRegion("raw","py.RawSensor", json.dumps(rawParams)) vfe = net.addRegion("output","VectorFileEffector","") net.link("raw", "output", "UniformLink", "") self.assertEqual(rawSensor.getParameter("outputWidth"),1029, "Incorrect outputWidth parameter") # Add vectors to the queue using two different add methods. Later we # will check to ensure these are actually output properly. rawSensor.executeCommand(["addDataToQueue", "[2, 4, 6]", "0", "42"]) rawSensorPy = rawSensor.getSelf() rawSensorPy.addDataToQueue([2, 42, 1023], 1, 43) rawSensorPy.addDataToQueue([18, 19, 20], 0, 44) # Set an output file before we run anything vfe.setParameter("outputFile",os.path.join(self.tmpDir,"temp.csv")) # Run the network and check outputs are as expected net.run(1) self.assertEqual(rawSensor.getOutputData("dataOut").nonzero()[0].sum(), sum([2, 4, 6]), "Value of dataOut incorrect") self.assertEqual(rawSensor.getOutputData("resetOut").sum(),0, "Value of resetOut incorrect") self.assertEqual( rawSensor.getOutputData("sequenceIdOut").sum(),42, "Value of sequenceIdOut incorrect") net.run(1) self.assertEqual(rawSensor.getOutputData("dataOut").nonzero()[0].sum(), sum([2, 42, 1023]), "Value of dataOut incorrect") self.assertEqual(rawSensor.getOutputData("resetOut").sum(),1, "Value of resetOut incorrect") self.assertEqual( rawSensor.getOutputData("sequenceIdOut").sum(),43, "Value of sequenceIdOut incorrect") # Make sure we can save and load the network after running net.save(os.path.join(self.tmpDir,"rawNetwork.nta")) net2 = Network(os.path.join(self.tmpDir,"rawNetwork.nta")) rawSensor2 = net2.regions.get("raw") vfe2 = net2.regions.get("output") # Ensure parameters are preserved self.assertEqual(rawSensor2.getParameter("outputWidth"),1029, "Incorrect outputWidth parameter") # Ensure the queue is preserved through save/load vfe2.setParameter("outputFile",os.path.join(self.tmpDir,"temp.csv")) net2.run(1) self.assertEqual(rawSensor2.getOutputData("dataOut").nonzero()[0].sum(), sum([18, 19, 20]), "Value of dataOut incorrect") self.assertEqual(rawSensor2.getOutputData("resetOut").sum(),0, "Value of resetOut incorrect") self.assertEqual( rawSensor2.getOutputData("sequenceIdOut").sum(),44, "Value of sequenceIdOut incorrect")
def testSensor(self): # Create a simple network to test the sensor params = { "activeBits": self.encoder.w, "outputWidth": self.encoder.n, "radius": 2, "verbosity": self.encoder.verbosity, } net = Network() region = net.addRegion("coordinate", "py.CoordinateSensorRegion", json.dumps(params)) vfe = net.addRegion("output", "VectorFileEffector", "") net.link("coordinate", "output", "UniformLink", "") self.assertEqual(region.getParameter("outputWidth"), self.encoder.n, "Incorrect outputWidth parameter") # Add vectors to the queue using two different add methods. Later we # will check to ensure these are actually output properly. region.executeCommand(["addDataToQueue", "[2, 4, 6]", "0", "42"]) regionPy = region.getSelf() regionPy.addDataToQueue([2, 42, 1023], 1, 43) regionPy.addDataToQueue([18, 19, 20], 0, 44) # Set an output file before we run anything vfe.setParameter("outputFile", os.path.join(self.tmpDir, "temp.csv")) # Run the network and check outputs are as expected net.run(1) expected = self.encoder.encode((numpy.array([2, 4, 6]), params["radius"])) actual = region.getOutputData("dataOut") self.assertEqual(actual.sum(), expected.sum(), "Value of dataOut incorrect") self.assertEqual(region.getOutputData("resetOut"), 0, "Value of resetOut incorrect") self.assertEqual(region.getOutputData("sequenceIdOut"), 42, "Value of sequenceIdOut incorrect") net.run(1) expected = self.encoder.encode((numpy.array([2, 42, 1023]), params["radius"])) actual = region.getOutputData("dataOut") self.assertEqual(actual.sum(), expected.sum(), "Value of dataOut incorrect") self.assertEqual(region.getOutputData("resetOut"), 1, "Value of resetOut incorrect") self.assertEqual(region.getOutputData("sequenceIdOut"), 43, "Value of sequenceIdOut incorrect") # Make sure we can save and load the network after running net.save(os.path.join(self.tmpDir, "coordinateNetwork.nta")) net2 = Network(os.path.join(self.tmpDir, "coordinateNetwork.nta")) region2 = net2.regions.get("coordinate") vfe2 = net2.regions.get("output") # Ensure parameters are preserved self.assertEqual(region2.getParameter("outputWidth"), self.encoder.n, "Incorrect outputWidth parameter") # Ensure the queue is preserved through save/load vfe2.setParameter("outputFile", os.path.join(self.tmpDir, "temp.csv")) net2.run(1) expected = self.encoder.encode((numpy.array([18, 19, 20]), params["radius"])) actual = region2.getOutputData("dataOut") self.assertEqual(actual.sum(), expected.sum(), "Value of dataOut incorrect") self.assertEqual(region2.getOutputData("resetOut"), 0, "Value of resetOut incorrect") self.assertEqual(region2.getOutputData("sequenceIdOut"), 44, "Value of sequenceIdOut incorrect")
def _testNetLoad(self): """Test loading a network with this sensor in it.""" n = Network() r = n.addRegion(self.nodeName, self.sensorName, '{ activeOutputCount: 11}') r.dimensions = Dimensions([1]) n.save(self.filename) n = Network(self.filename) n.initialize() self.testsPassed += 1 # Check that vectorCount parameter is zero r = n.regions[self.nodeName] res = r.getParameter('vectorCount') self.assertEqual( res, 0, "getting vectorCount:\n Expected '0', got back '%d'\n" % res) self.sensor = r
def testSerialization(self): n = Network() imageDims = (42, 38) params = dict(width=imageDims[0], height=imageDims[1], mode="bw", background=1, invertOutput=1) sensor = n.addRegion("sensor", "py.ImageSensor", json.dumps(params)) sensor.setDimensions(Dimensions(imageDims[0], imageDims[1])) params = dict(inputShape=imageDims, coincidencesShape=imageDims, disableTemporal=1, tpSeed=43, spSeed=42, nCellsPerCol=1) l1 = n.addRegion("l1", "py.CLARegion", json.dumps(params)) params = dict(maxCategoryCount=48, SVDSampleCount=400, SVDDimCount=5, distanceNorm=0.6) _classifier = n.addRegion("classifier", "py.KNNClassifierRegion", json.dumps(params)) # TODO: link params should not be required. Dest region dimensions are # already specified as [1] params = dict(mapping="in", rfSize=imageDims) n.link("sensor", "l1", "UniformLink", json.dumps(params)) n.link("l1", "classifier", "UniformLink", "", "bottomUpOut", "bottomUpIn") n.link("sensor", "classifier", "UniformLink", "", "categoryOut", "categoryIn") n.initialize() n.save("fdr.nta") # Make sure the network bundle has all the expected files self.assertTrue(os.path.exists("fdr.nta/network.yaml")) self.assertTrue(os.path.exists("fdr.nta/R0-pkl")) self.assertTrue(os.path.exists("fdr.nta/R1-pkl")) self.assertTrue(os.path.exists("fdr.nta/R2-pkl")) n2 = Network("fdr.nta") n2.initialize() # should not fail # Make sure the network is actually the same sensor = n2.regions['sensor'] self.assertEqual(sensor.type, "py.ImageSensor") # would like to directly compare, but can't -- NPC-6 self.assertEqual(str(sensor.dimensions), str(Dimensions(42, 38))) self.assertEqual(sensor.getParameter("width"), 42) self.assertEqual(sensor.getParameter("height"), 38) self.assertEqual(sensor.getParameter("mode"), "bw") self.assertEqual(sensor.getParameter("background"), 1) self.assertEqual(sensor.getParameter("invertOutput"), 1) l1 = n2.regions['l1'] self.assertEqual(l1.type, "py.CLARegion") self.assertEqual(str(l1.dimensions), str(Dimensions(1))) a = l1.getParameter("inputShape") self.assertEqual(len(a), 2) self.assertEqual(a[0], 42) self.assertEqual(a[1], 38) a = l1.getParameter("coincidencesShape") self.assertEqual(len(a), 2) self.assertEqual(a[0], 42) self.assertEqual(a[1], 38) self.assertEqual(l1.getParameter("disableTemporal"), 1) self.assertEqual(l1.getParameter("spSeed"), 42) self.assertEqual(l1.getParameter("tpSeed"), 43) cl = n2.regions['classifier'] self.assertEqual(cl.type, "py.KNNClassifierRegion") self.assertEqual(cl.getParameter("maxCategoryCount"), 48) self.assertEqual(cl.getParameter("SVDSampleCount"), 400) self.assertEqual(cl.getParameter("SVDDimCount"), 5) self.assertLess((cl.getParameter("distanceNorm") - 0.6), 0.0001) self.assertEqual(str(cl.dimensions), str(Dimensions(1))) n2.save("fdr2.nta") # now compare the two network bundles -- should be the same c = filecmp.dircmp("fdr.nta", "fdr2.nta") self.assertEqual(len(c.left_only), 0, "fdr.nta has extra files: %s" % c.left_only) self.assertEqual(len(c.right_only), 0, "fdr2.nta has extra files: %s" % c.right_only) if len(c.diff_files) > 0: _LOGGER.warn( "Some bundle files differ: %s\n" "This is expected, as pickle.load() followed by " "pickle.dump() doesn't produce the same file", c.diff_files)
class SaccadeNetwork(object): """ A HTM network structured as follows: SaccadeSensor (RandomSaccade) -> SP -> TM -> Classifier (KNN) """ def __init__(self, networkName, trainingSet, testingSet, loggingDir=None, validationSet=None, detailedSaccadeWidth=IMAGE_WIDTH, detailedSaccadeHeight=IMAGE_HEIGHT, createNetwork=True): """ :param str networkName: Where the network will be serialized/saved to :param str trainingSet: Path to set of images to train on :param str testingSet: Path to set of images to test :param str loggingDir: directory to store logged images in (note: no image logging if none) :param validationSet: (optional) Path to set of images to validate on :param int detailedSaccadeWidth: (optional) Width of detailed saccades to return from the runNetworkOneImage and testNetworkOneImage :param int detailedSaccadeHeight: (optional) Height of detailed saccades to return from the runNetworkOneImage and testNetworkOneImage :param bool createNetwork: If false, wait until createNet is manually called to create the network. Otherwise, create on __init__ """ self.loggingDir = loggingDir self.netFile = networkName self.trainingSet = trainingSet self.validationSet = validationSet self.testingSet = testingSet self.detailedSaccadeWidth = detailedSaccadeWidth self.detailedSaccadeHeight = detailedSaccadeHeight self.net = None self.trainingImageIndex = None self.networkDutyCycles = None self.networkSensor = None self.networkSP = None self.networkTM = None self.networkTP = None self.networkClassifier = None self.networkSensor = None self.numTrainingImages = 0 self.numTestingImages = 0 self.trainingImageIndex = 0 self.testingImageIndex = 0 self.numCorrect = 0 if createNetwork: self.createNet() def createNet(self): """ Set up the structure of the network """ net = Network() Network.unregisterRegion(SaccadeSensor.__name__) Network.registerRegion(SaccadeSensor) Network.unregisterRegion(ExtendedTMRegion.__name__) Network.registerRegion(ExtendedTMRegion) Network.unregisterRegion(ColumnPoolerRegion.__name__) Network.registerRegion(ColumnPoolerRegion) imageSensorParams = copy.deepcopy(DEFAULT_IMAGESENSOR_PARAMS) if self.loggingDir is not None: imageSensorParams["logDir"] = "sensorImages/" + self.loggingDir imageSensorParams["logOutputImages"] = 1 imageSensorParams["logOriginalImages"] = 1 imageSensorParams["logFilteredImages"] = 1 imageSensorParams["logLocationImages"] = 1 imageSensorParams["logLocationOnOriginalImage"] = 1 net.addRegion("sensor", "py.SaccadeSensor", yaml.dump(imageSensorParams)) sensor = net.regions["sensor"].getSelf() DEFAULT_SP_PARAMS["columnCount"] = sensor.getOutputElementCount( "dataOut") net.addRegion("SP", "py.SPRegion", yaml.dump(DEFAULT_SP_PARAMS)) sp = net.regions["SP"].getSelf() DEFAULT_TM_PARAMS["columnDimensions"] = ( sp.getOutputElementCount("bottomUpOut"), ) DEFAULT_TM_PARAMS["basalInputWidth"] = sensor.getOutputElementCount( "saccadeOut") net.addRegion("TM", "py.ExtendedTMRegion", yaml.dump(DEFAULT_TM_PARAMS)) net.addRegion("TP", "py.ColumnPoolerRegion", yaml.dump(DEFAULT_TP_PARAMS)) net.addRegion("classifier", "py.KNNClassifierRegion", yaml.dump(DEFAULT_CLASSIFIER_PARAMS)) net.link("sensor", "SP", "UniformLink", "", srcOutput="dataOut", destInput="bottomUpIn") net.link("SP", "TM", "UniformLink", "", srcOutput="bottomUpOut", destInput="activeColumns") net.link("sensor", "TM", "UniformLink", "", srcOutput="saccadeOut", destInput="basalInput") net.link("TM", "TP", "UniformLink", "", srcOutput="predictedActiveCells", destInput="feedforwardInput") net.link("TP", "TM", "UniformLink", "", srcOutput="feedForwardOutput", destInput="apicalInput") net.link("TP", "classifier", "UniformLink", "", srcOutput="feedForwardOutput", destInput="bottomUpIn") #net.link("TM", "classifier", "UniformLink", "", # srcOutput="predictedActiveCells", destInput="bottomUpIn") net.link("sensor", "classifier", "UniformLink", "", srcOutput="categoryOut", destInput="categoryIn") self.net = net self.networkSensor = self.net.regions["sensor"] self.networkSP = self.net.regions["SP"] self.networkTM = self.net.regions["TM"] self.networkTP = self.net.regions["TP"] self.networkClassifier = self.net.regions["classifier"] def loadFromFile(self, filename): """ Load a serialized network :param filename: Where the network should be loaded from """ print "Loading network from {file}...".format(file=filename) Network.unregisterRegion(SaccadeSensor.__name__) Network.registerRegion(SaccadeSensor) Network.registerRegion(ExtendedTMRegion) self.net = Network(filename) self.networkSensor = self.net.regions["sensor"] self.networkSensor.setParameter("numSaccades", SACCADES_PER_IMAGE_TESTING) self.networkSP = self.net.regions["SP"] self.networkClassifier = self.net.regions["classifier"] self.numCorrect = 0 def loadExperiment(self): """ Load images into ImageSensor and set the learning mode for the SP. """ print "============= Loading training images =================" t1 = time.time() self.networkSensor.executeCommand( ["loadMultipleImages", self.trainingSet]) numTrainingImages = self.networkSensor.getParameter("numImages") t2 = time.time() print "Load time for training images:", t2 - t1 print "Number of training images", numTrainingImages self.numTrainingImages = numTrainingImages self.trainingImageIndex = 0 def runNetworkOneImage(self, enableViz=False): """ Runs a single image through the network stepping through all saccades :param bool enableViz: If true, visualizations are generated and returned :return: If enableViz, return a tuple (saccadeImgsList, saccadeDetailList, saccadeHistList). saccadeImgsList is a list of images with the fovea highlighted. saccadeDetailList is a list of resized images showing the contents of the fovea. saccadeHistList shows the fovea history. If not enableViz, returns True Regardless of enableViz, if False is returned, all images have been saccaded over. """ if self.trainingImageIndex < self.numTrainingImages: saccadeList = [] saccadeImgsList = [] saccadeHistList = [] saccadeDetailList = [] originalImage = None self.networkTM.executeCommand(["reset"]) for i in range(SACCADES_PER_IMAGE_TRAINING): self.net.run(1) if originalImage is None: originalImage = deserializeImage( yaml.load( self.networkSensor.getParameter("originalImage"))) imgCenter = ( originalImage.size[0] / 2, originalImage.size[1] / 2, ) saccadeList.append({ "offset1": (yaml.load( self.networkSensor.getParameter("prevSaccadeInfo")) ["prevOffset"]), "offset2": (yaml.load( self.networkSensor.getParameter("prevSaccadeInfo")) ["newOffset"]) }) if enableViz: detailImage = deserializeImage( yaml.load( self.networkSensor.getParameter("outputImage"))) detailImage = detailImage.resize( (self.detailedSaccadeWidth, self.detailedSaccadeHeight), Image.ANTIALIAS) saccadeDetailList.append(ImageTk.PhotoImage(detailImage)) imgWithSaccade = originalImage.convert("RGB") ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Left ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Right ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Top ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Bottom ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Left ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Right ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Top ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Bottom saccadeImgsList.append(ImageTk.PhotoImage(imgWithSaccade)) saccadeHist = originalImage.convert("RGB") for i, saccade in enumerate(saccadeList): ImageDraw.Draw(saccadeHist).rectangle( (imgCenter[0] + saccade["offset2"][0] - _FOVEA_SIZE / 2, imgCenter[0] + saccade["offset2"][1] - _FOVEA_SIZE / 2, imgCenter[0] + saccade["offset2"][0] + _FOVEA_SIZE / 2, imgCenter[0] + saccade["offset2"][1] + _FOVEA_SIZE / 2), fill=(0, (255 / SACCADES_PER_IMAGE_TRAINING * (SACCADES_PER_IMAGE_TRAINING - i)), (255 / SACCADES_PER_IMAGE_TRAINING * i))) saccadeHist = saccadeHist.resize( (self.detailedSaccadeWidth, self.detailedSaccadeHeight), Image.ANTIALIAS) saccadeHistList.append(ImageTk.PhotoImage(saccadeHist)) self.trainingImageIndex += 1 print("Iteration: {iter}; Category: {cat}".format( iter=self.trainingImageIndex, cat=self.networkSensor.getOutputData("categoryOut"))) if enableViz: return (saccadeImgsList, saccadeDetailList, saccadeHistList, self.networkSensor.getOutputData("categoryOut")) return True else: return False def runNetworkBatch(self, batchSize): """ Run the network in batches. :param batchSize: Number of images to show in this batch :return: True if there are more images left to be saccaded over. Otherwise False. """ startTime = time.time() while self.trainingImageIndex < self.numTrainingImages: self.networkTM.executeCommand(["reset"]) for i in range(SACCADES_PER_IMAGE_TRAINING): self.net.run(1) self.trainingImageIndex += 1 if self.trainingImageIndex % batchSize == 0: print( "Iteration: {iter}; Category: {cat}; Time per batch: {t}". format(iter=self.trainingImageIndex, cat=self.networkSensor.getOutputData("categoryOut"), t=time.time() - startTime)) return True return False def setupNetworkTest(self): self.networkSensor.executeCommand( ["loadMultipleImages", self.testingSet]) self.numTestingImages = self.networkSensor.getParameter("numImages") self.testingImageIndex = 0 print "NumTestingImages {test}".format(test=self.numTestingImages) def testNetworkOneImage(self, enableViz=False): if self.testingImageIndex < self.numTestingImages: saccadeList = [] saccadeImgsList = [] saccadeHistList = [] saccadeDetailList = [] inferredCategoryList = [] originalImage = None self.networkTM.executeCommand(["reset"]) for i in range(SACCADES_PER_IMAGE_TESTING): self.net.run(1) if originalImage is None: originalImage = deserializeImage( yaml.load( self.networkSensor.getParameter("originalImage"))) imgCenter = ( originalImage.size[0] / 2, originalImage.size[1] / 2, ) saccadeList.append({ "offset1": (yaml.load( self.networkSensor.getParameter("prevSaccadeInfo")) ["prevOffset"]), "offset2": (yaml.load( self.networkSensor.getParameter("prevSaccadeInfo")) ["newOffset"]) }) inferredCategoryList.append( self.networkClassifier.getOutputData( "categoriesOut").argmax()) if enableViz: detailImage = deserializeImage( yaml.load( self.networkSensor.getParameter("outputImage"))) detailImage = detailImage.resize( (self.detailedSaccadeWidth, self.detailedSaccadeHeight), Image.ANTIALIAS) saccadeDetailList.append(ImageTk.PhotoImage(detailImage)) imgWithSaccade = originalImage.convert("RGB") ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Left ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Right ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Top ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Bottom ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Left ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Right ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Top ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Bottom saccadeImgsList.append(ImageTk.PhotoImage(imgWithSaccade)) saccadeHist = originalImage.convert("RGB") for i, saccade in enumerate(saccadeList): ImageDraw.Draw(saccadeHist).rectangle( (imgCenter[0] + saccade["offset2"][0] - _FOVEA_SIZE / 2, imgCenter[0] + saccade["offset2"][1] - _FOVEA_SIZE / 2, imgCenter[0] + saccade["offset2"][0] + _FOVEA_SIZE / 2, imgCenter[0] + saccade["offset2"][1] + _FOVEA_SIZE / 2), fill=(0, (255 / SACCADES_PER_IMAGE_TESTING * (SACCADES_PER_IMAGE_TESTING - i)), (255 / SACCADES_PER_IMAGE_TESTING * i))) saccadeHist = saccadeHist.resize( (self.detailedSaccadeWidth, self.detailedSaccadeHeight), Image.ANTIALIAS) saccadeHistList.append(ImageTk.PhotoImage(saccadeHist)) inferredCategory = self._getMostCommonCategory( inferredCategoryList) isCorrectClassification = False if self.networkSensor.getOutputData( "categoryOut") == inferredCategory: isCorrectClassification = True self.numCorrect += 1 print("Iteration: {iter}; Category: {cat}".format( iter=self.testingImageIndex, cat=self.networkSensor.getOutputData("categoryOut"))) self.testingImageIndex += 1 if enableViz: return (saccadeImgsList, saccadeDetailList, saccadeHistList, inferredCategoryList, self.networkSensor.getOutputData("categoryOut"), isCorrectClassification) return (True, isCorrectClassification) else: return False def testNetworkBatch(self, batchSize): if self.testingImageIndex >= self.numTestingImages: return False while self.testingImageIndex < self.numTestingImages: inferredCategoryList = [] self.networkTM.executeCommand(["reset"]) for i in range(SACCADES_PER_IMAGE_TESTING): self.net.run(1) inferredCategoryList.append( self.networkClassifier.getOutputData( "categoriesOut").argmax()) inferredCategory = self._getMostCommonCategory( inferredCategoryList) if self.networkSensor.getOutputData( "categoryOut") == inferredCategory: self.numCorrect += 1 self.testingImageIndex += 1 if self.testingImageIndex % batchSize == 0: print("Testing iteration: {iter}".format( iter=self.testingImageIndex)) break return self.numCorrect @staticmethod def _getMostCommonCategory(categoryList): return collections.Counter(categoryList).most_common(1)[0][0] def setLearningMode(self, learningSP=False, learningTM=False, learningTP=False, learningClassifier=False): if learningSP: self.networkSP.setParameter("learningMode", 1) self.networkSP.setParameter("inferenceMode", 0) else: self.networkSP.setParameter("learningMode", 0) self.networkSP.setParameter("inferenceMode", 1) if learningTM: self.networkTM.setParameter("learn", 1) else: self.networkTM.setParameter("learn", 0) if learningTM: self.networkTP.setParameter("learningMode", 1) else: self.networkTP.setParameter("learningMode", 0) if learningClassifier: self.networkClassifier.setParameter("learningMode", 1) self.networkClassifier.setParameter("inferenceMode", 0) else: self.networkClassifier.setParameter("learningMode", 0) self.networkClassifier.setParameter("inferenceMode", 1) def saveNetwork(self): print "Saving network at {path}".format(path=self.netFile) self.net.save(self.netFile) def resetIndex(self): self.trainingImageIndex = 0
class SaccadeNetwork(object): """ A HTM network structured as follows: SaccadeSensor (RandomSaccade) -> SP -> TM -> Classifier (KNN) """ def __init__(self, networkName, trainingSet, testingSet, loggingDir = None, validationSet=None, detailedSaccadeWidth=IMAGE_WIDTH, detailedSaccadeHeight=IMAGE_HEIGHT, createNetwork=True): """ :param str networkName: Where the network will be serialized/saved to :param str trainingSet: Path to set of images to train on :param str testingSet: Path to set of images to test :param str loggingDir: directory to store logged images in (note: no image logging if none) :param validationSet: (optional) Path to set of images to validate on :param int detailedSaccadeWidth: (optional) Width of detailed saccades to return from the runNetworkOneImage and testNetworkOneImage :param int detailedSaccadeHeight: (optional) Height of detailed saccades to return from the runNetworkOneImage and testNetworkOneImage :param bool createNetwork: If false, wait until createNet is manually called to create the network. Otherwise, create on __init__ """ self.loggingDir = loggingDir self.netFile = networkName self.trainingSet = trainingSet self.validationSet = validationSet self.testingSet = testingSet self.detailedSaccadeWidth = detailedSaccadeWidth self.detailedSaccadeHeight = detailedSaccadeHeight self.net = None self.trainingImageIndex = None self.networkClassifier = None self.networkDutyCycles = None self.networkSP = None self.networkTM = None self.networkSensor = None self.numTrainingImages = 0 self.numTestingImages = 0 self.trainingImageIndex = 0 self.testingImageIndex = 0 self.numCorrect = 0 if createNetwork: self.createNet() def createNet(self): """ Set up the structure of the network """ net = Network() Network.unregisterRegion(SaccadeSensor.__name__) Network.registerRegion(SaccadeSensor) Network.registerRegion(TMRegion) imageSensorParams = copy.deepcopy(DEFAULT_IMAGESENSOR_PARAMS) if self.loggingDir is not None: imageSensorParams["logDir"] = "sensorImages/" + self.loggingDir imageSensorParams["logOutputImages"] = 1 imageSensorParams["logOriginalImages"] = 1 imageSensorParams["logFilteredImages"] = 1 imageSensorParams["logLocationImages"] = 1 imageSensorParams["logLocationOnOriginalImage"] = 1 net.addRegion("sensor", "py.SaccadeSensor", yaml.dump(imageSensorParams)) sensor = net.regions["sensor"].getSelf() DEFAULT_SP_PARAMS["columnCount"] = sensor.getOutputElementCount("dataOut") net.addRegion("SP", "py.SPRegion", yaml.dump(DEFAULT_SP_PARAMS)) sp = net.regions["SP"].getSelf() DEFAULT_TM_PARAMS["columnDimensions"] = (sp.getOutputElementCount("bottomUpOut"),) net.addRegion("TM", "py.TMRegion", yaml.dump(DEFAULT_TM_PARAMS)) net.addRegion("classifier","py.KNNClassifierRegion", yaml.dump(DEFAULT_CLASSIFIER_PARAMS)) net.link("sensor", "SP", "UniformLink", "", srcOutput="dataOut", destInput="bottomUpIn") net.link("SP", "TM", "UniformLink", "", srcOutput="bottomUpOut", destInput="activeColumns") net.link("sensor", "TM", "UniformLink", "", srcOutput="saccadeOut", destInput="activeExternalCells") net.link("TM", "classifier", "UniformLink", "", srcOutput="predictedActiveCells", destInput="bottomUpIn") net.link("sensor", "classifier", "UniformLink", "", srcOutput="categoryOut", destInput="categoryIn") self.net = net self.networkSensor = self.net.regions["sensor"] self.networkSP = self.net.regions["SP"] self.networkTM = self.net.regions["TM"] self.networkClassifier = self.net.regions["classifier"] def loadFromFile(self, filename): """ Load a serialized network :param filename: Where the network should be loaded from """ print "Loading network from {file}...".format(file=filename) Network.unregisterRegion(SaccadeSensor.__name__) Network.registerRegion(SaccadeSensor) Network.registerRegion(TMRegion) self.net = Network(filename) self.networkSensor = self.net.regions["sensor"] self.networkSensor.setParameter("numSaccades", SACCADES_PER_IMAGE_TESTING) self.networkSP = self.net.regions["SP"] self.networkClassifier = self.net.regions["classifier"] self.numCorrect = 0 def loadExperiment(self): """ Load images into ImageSensor and set the learning mode for the SP. """ print "============= Loading training images =================" t1 = time.time() self.networkSensor.executeCommand( ["loadMultipleImages", self.trainingSet]) numTrainingImages = self.networkSensor.getParameter("numImages") t2 = time.time() print "Load time for training images:", t2-t1 print "Number of training images", numTrainingImages self.numTrainingImages = numTrainingImages self.trainingImageIndex = 0 def runNetworkOneImage(self, enableViz=False): """ Runs a single image through the network stepping through all saccades :param bool enableViz: If true, visualizations are generated and returned :return: If enableViz, return a tuple (saccadeImgsList, saccadeDetailList, saccadeHistList). saccadeImgsList is a list of images with the fovea highlighted. saccadeDetailList is a list of resized images showing the contents of the fovea. saccadeHistList shows the fovea history. If not enableViz, returns True Regardless of enableViz, if False is returned, all images have been saccaded over. """ if self.trainingImageIndex < self.numTrainingImages: saccadeList = [] saccadeImgsList = [] saccadeHistList = [] saccadeDetailList = [] originalImage = None self.networkTM.executeCommand(["reset"]) for i in range(SACCADES_PER_IMAGE_TRAINING): self.net.run(1) if originalImage is None: originalImage = deserializeImage( yaml.load(self.networkSensor.getParameter("originalImage"))) imgCenter = (originalImage.size[0] / 2, originalImage.size[1] / 2,) saccadeList.append({ "offset1": (yaml.load( self.networkSensor .getParameter("prevSaccadeInfo")) ["prevOffset"]), "offset2": (yaml.load( self.networkSensor .getParameter("prevSaccadeInfo")) ["newOffset"])}) if enableViz: detailImage = deserializeImage( yaml.load(self.networkSensor.getParameter("outputImage"))) detailImage = detailImage.resize((self.detailedSaccadeWidth, self.detailedSaccadeHeight), Image.ANTIALIAS) saccadeDetailList.append(ImageTk.PhotoImage(detailImage)) imgWithSaccade = originalImage.convert("RGB") ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Left ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Right ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Top ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Bottom ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Left ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Right ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Top ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Bottom saccadeImgsList.append(ImageTk.PhotoImage(imgWithSaccade)) saccadeHist = originalImage.convert("RGB") for i, saccade in enumerate(saccadeList): ImageDraw.Draw(saccadeHist).rectangle( (imgCenter[0] + saccade["offset2"][0] - _FOVEA_SIZE/2, imgCenter[0] + saccade["offset2"][1] - _FOVEA_SIZE/2, imgCenter[0] + saccade["offset2"][0] + _FOVEA_SIZE/2, imgCenter[0] + saccade["offset2"][1] + _FOVEA_SIZE/2), fill=(0, (255/SACCADES_PER_IMAGE_TRAINING*(SACCADES_PER_IMAGE_TRAINING-i)), (255/SACCADES_PER_IMAGE_TRAINING*i))) saccadeHist = saccadeHist.resize((self.detailedSaccadeWidth, self.detailedSaccadeHeight), Image.ANTIALIAS) saccadeHistList.append(ImageTk.PhotoImage(saccadeHist)) self.trainingImageIndex += 1 print ("Iteration: {iter}; Category: {cat}" .format(iter=self.trainingImageIndex, cat=self.networkSensor.getOutputData("categoryOut"))) if enableViz: return (saccadeImgsList, saccadeDetailList, saccadeHistList, self.networkSensor.getOutputData("categoryOut")) return True else: return False def runNetworkBatch(self, batchSize): """ Run the network in batches. :param batchSize: Number of images to show in this batch :return: True if there are more images left to be saccaded over. Otherwise False. """ startTime = time.time() while self.trainingImageIndex < self.numTrainingImages: self.networkTM.executeCommand(["reset"]) for i in range(SACCADES_PER_IMAGE_TRAINING): self.net.run(1) self.trainingImageIndex += 1 if self.trainingImageIndex % batchSize == 0: print ("Iteration: {iter}; Category: {cat}; Time per batch: {t}" .format(iter=self.trainingImageIndex, cat=self.networkSensor.getOutputData("categoryOut"), t=time.time()-startTime)) return True return False def setupNetworkTest(self): self.networkSensor.executeCommand(["loadMultipleImages", self.testingSet]) self.numTestingImages = self.networkSensor.getParameter("numImages") self.testingImageIndex = 0 print "NumTestingImages {test}".format(test=self.numTestingImages) def testNetworkOneImage(self, enableViz=False): if self.testingImageIndex < self.numTestingImages: saccadeList = [] saccadeImgsList = [] saccadeHistList = [] saccadeDetailList = [] inferredCategoryList = [] originalImage = None self.networkTM.executeCommand(["reset"]) for i in range(SACCADES_PER_IMAGE_TESTING): self.net.run(1) if originalImage is None: originalImage = deserializeImage( yaml.load(self.networkSensor.getParameter("originalImage"))) imgCenter = (originalImage.size[0] / 2, originalImage.size[1] / 2,) saccadeList.append({ "offset1": (yaml.load( self.networkSensor .getParameter("prevSaccadeInfo")) ["prevOffset"]), "offset2": (yaml.load( self.networkSensor .getParameter("prevSaccadeInfo")) ["newOffset"])}) inferredCategoryList.append( self.networkClassifier.getOutputData("categoriesOut").argmax()) if enableViz: detailImage = deserializeImage( yaml.load(self.networkSensor.getParameter("outputImage"))) detailImage = detailImage.resize((self.detailedSaccadeWidth, self.detailedSaccadeHeight), Image.ANTIALIAS) saccadeDetailList.append(ImageTk.PhotoImage(detailImage)) imgWithSaccade = originalImage.convert("RGB") ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Left ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Right ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] - (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Top ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset2"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset2"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset2"][1] + (_FOVEA_SIZE / 2)), fill=(255, 0, 0), width=1) # Bottom ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Left ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Right ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] - (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Top ImageDraw.Draw(imgWithSaccade).line( (imgCenter[0] + saccadeList[i]["offset1"][0] + (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2), imgCenter[0] + saccadeList[i]["offset1"][0] - (_FOVEA_SIZE / 2), imgCenter[1] + saccadeList[i]["offset1"][1] + (_FOVEA_SIZE / 2)), fill=(0, 255, 0), width=1) # Bottom saccadeImgsList.append(ImageTk.PhotoImage(imgWithSaccade)) saccadeHist = originalImage.convert("RGB") for i, saccade in enumerate(saccadeList): ImageDraw.Draw(saccadeHist).rectangle( (imgCenter[0] + saccade["offset2"][0] - _FOVEA_SIZE/2, imgCenter[0] + saccade["offset2"][1] - _FOVEA_SIZE/2, imgCenter[0] + saccade["offset2"][0] + _FOVEA_SIZE/2, imgCenter[0] + saccade["offset2"][1] + _FOVEA_SIZE/2), fill=(0, (255/SACCADES_PER_IMAGE_TESTING*(SACCADES_PER_IMAGE_TESTING-i)), (255/SACCADES_PER_IMAGE_TESTING*i))) saccadeHist = saccadeHist.resize((self.detailedSaccadeWidth, self.detailedSaccadeHeight), Image.ANTIALIAS) saccadeHistList.append(ImageTk.PhotoImage(saccadeHist)) inferredCategory = self._getMostCommonCategory(inferredCategoryList) isCorrectClassification = False if self.networkSensor.getOutputData("categoryOut") == inferredCategory: isCorrectClassification = True self.numCorrect += 1 print ("Iteration: {iter}; Category: {cat}" .format(iter=self.testingImageIndex, cat=self.networkSensor.getOutputData("categoryOut"))) self.testingImageIndex += 1 if enableViz: return (saccadeImgsList, saccadeDetailList, saccadeHistList, inferredCategoryList, self.networkSensor.getOutputData("categoryOut"), isCorrectClassification) return (True, isCorrectClassification) else: return False def testNetworkBatch(self, batchSize): if self.testingImageIndex >= self.numTestingImages: return False while self.testingImageIndex < self.numTestingImages: inferredCategoryList = [] self.networkTM.executeCommand(["reset"]) for i in range(SACCADES_PER_IMAGE_TESTING): self.net.run(1) inferredCategoryList.append( self.networkClassifier.getOutputData("categoriesOut").argmax()) inferredCategory = self._getMostCommonCategory(inferredCategoryList) if self.networkSensor.getOutputData("categoryOut") == inferredCategory: self.numCorrect += 1 self.testingImageIndex += 1 if self.testingImageIndex % batchSize == 0: print ("Testing iteration: {iter}" .format(iter=self.testingImageIndex)) break return self.numCorrect @staticmethod def _getMostCommonCategory(categoryList): return collections.Counter(categoryList).most_common(1)[0][0] def setLearningMode(self, learningSP=False, learningTM=False, learningClassifier=False): if learningSP: self.networkSP.setParameter("learningMode", 1) self.networkSP.setParameter("inferenceMode", 0) else: self.networkSP.setParameter("learningMode", 0) self.networkSP.setParameter("inferenceMode", 1) if learningTM: self.networkTM.setParameter("learningMode", 1) else: self.networkTM.setParameter("learningMode", 0) if learningClassifier: self.networkClassifier.setParameter("learningMode", 1) self.networkClassifier.setParameter("inferenceMode", 0) else: self.networkClassifier.setParameter("learningMode", 0) self.networkClassifier.setParameter("inferenceMode", 1) def saveNetwork(self): print "Saving network at {path}".format(path=self.netFile) self.net.save(self.netFile) def resetIndex(self): self.trainingImageIndex = 0
def testSerialization(self): n = Network() imageDims = (42, 38) params = dict( width=imageDims[0], height=imageDims[1], mode="bw", background=1, invertOutput=1) sensor = n.addRegion("sensor", "py.ImageSensor", json.dumps(params)) sensor.setDimensions(Dimensions(imageDims[0], imageDims[1])) params = dict( inputShape=imageDims, coincidencesShape=imageDims, disableTemporal=1, tpSeed=43, spSeed=42, nCellsPerCol=1) l1 = n.addRegion("l1", "py.CLARegion", json.dumps(params)) params = dict( maxCategoryCount=48, SVDSampleCount=400, SVDDimCount=5, distanceNorm=0.6) _classifier = n.addRegion("classifier", "py.KNNClassifierRegion", json.dumps(params)) # TODO: link params should not be required. Dest region dimensions are # already specified as [1] params = dict( mapping="in", rfSize=imageDims) n.link("sensor", "l1", "UniformLink", json.dumps(params)) n.link("l1", "classifier", "UniformLink", "", "bottomUpOut", "bottomUpIn") n.link("sensor", "classifier", "UniformLink", "", "categoryOut", "categoryIn") n.initialize() n.save("fdr.nta") # Make sure the network bundle has all the expected files self.assertTrue(os.path.exists("fdr.nta/network.yaml")) self.assertTrue(os.path.exists("fdr.nta/R0-pkl")) self.assertTrue(os.path.exists("fdr.nta/R1-pkl")) self.assertTrue(os.path.exists("fdr.nta/R2-pkl")) n2 = Network("fdr.nta") n2.initialize() # should not fail # Make sure the network is actually the same sensor = n2.regions['sensor'] self.assertEqual(sensor.type, "py.ImageSensor") # would like to directly compare, but can't -- NPC-6 self.assertEqual(str(sensor.dimensions), str(Dimensions(42, 38))) self.assertEqual(sensor.getParameter("width"), 42) self.assertEqual(sensor.getParameter("height"), 38) self.assertEqual(sensor.getParameter("mode"), "bw") self.assertEqual(sensor.getParameter("background"), 1) self.assertEqual(sensor.getParameter("invertOutput"), 1) l1 = n2.regions['l1'] self.assertEqual(l1.type, "py.CLARegion") self.assertEqual(str(l1.dimensions), str(Dimensions(1))) a = l1.getParameter("inputShape") self.assertEqual(len(a), 2) self.assertEqual(a[0], 42) self.assertEqual(a[1], 38) a = l1.getParameter("coincidencesShape") self.assertEqual(len(a), 2) self.assertEqual(a[0], 42) self.assertEqual(a[1], 38) self.assertEqual(l1.getParameter("disableTemporal"), 1) self.assertEqual(l1.getParameter("spSeed"), 42) self.assertEqual(l1.getParameter("tpSeed"), 43) cl = n2.regions['classifier'] self.assertEqual(cl.type, "py.KNNClassifierRegion") self.assertEqual(cl.getParameter("maxCategoryCount"), 48) self.assertEqual(cl.getParameter("SVDSampleCount"), 400) self.assertEqual(cl.getParameter("SVDDimCount"), 5) self.assertLess((cl.getParameter("distanceNorm") - 0.6), 0.0001) self.assertEqual(str(cl.dimensions), str(Dimensions(1))) n2.save("fdr2.nta") # now compare the two network bundles -- should be the same c = filecmp.dircmp("fdr.nta", "fdr2.nta") self.assertEqual(len(c.left_only), 0, "fdr.nta has extra files: %s" % c.left_only) self.assertEqual(len(c.right_only), 0, "fdr2.nta has extra files: %s" % c.right_only) if len(c.diff_files) > 0: _LOGGER.warn("Some bundle files differ: %s\n" "This is expected, as pickle.load() followed by " "pickle.dump() doesn't produce the same file", c.diff_files)