def __init__(self): # * Create application context, passing in custom arguments, and get a logger argParser = argparse.ArgumentParser(add_help=False) #argParser.add_argument('--in', type=str, default="coil-100", help="path to directory containing input images") # use input_source as directory; default to current directory argParser.add_argument('--out', type=str, default=None, help="path to output directory") # should this be a common parameter in Context? argParser.add_argument('--obj', type=str, default="1,101,1", required=False, help="object ID range, right-open interval <start>,<stop>,<step> (no spaces); default: full range") argParser.add_argument('--view', type=str, default="0,360,5", required=False, help="view angle range in degrees, right-open interval <start>,<stop>,<step> (no spaces); default: full range") self.context = Context.createInstance(description="COIL-100 image dataset processor", parent_argparsers=[argParser]) # TODO how to gather arg parsers from other interested parties? self.logger = logging.getLogger(self.__class__.__name__) # * Parse arguments self.inDir = self.context.options.input_source # should be an absolute path to a dir with COIL images; if it is a file/camera instead, it will be used as sole input # TODO also accept wildcards using glob.glob()? self.outDir = self.context.options.out # just for convenience self.outFile = None if self.outDir is not None: # TODO otherwise default to some directory? if os.path.isdir(self.outDir): now = datetime.now() outFilepath = os.path.join(self.outDir, "{}{}{}{}{}.{}".format(self.output_file_prefix, self.output_file_sep, now.strftime('%Y-%m-%d'), self.output_file_sep, now.strftime('%H-%M-%S'), self.output_file_ext)) self.logger.info("Output file: {}".format(outFilepath)) self.outFile = open(outFilepath, 'w') # open output file for storing features (TODO use with.. block instead in start()?) else: self.logger.warn("Invalid output directory \"{}\"; no output will be saved".format(self.outDir)) self.outDir = None # TODO create output directory if it doesn't exist self.objRange = xrange(*(int(x) for x in self.context.options.obj.split(','))) self.viewRange = xrange(*(int(x) for x in self.context.options.view.split(','))) # * Create visual system and manager self.context.update() # get fresh time self.visSys = VisualSystem(imageSize=self.image_size, timeNow=self.context.timeNow) self.visMan = COILManager(self.visSys)
def __init__(self): # * Create application context, passing in custom arguments, and get a logger argParser = argparse.ArgumentParser(add_help=False) argParser.add_argument('--features', type=str, default=None, help="features to look for, comma separated") self.context = Context.createInstance(description=self.__class__.__name__, parent_argparsers=[argParser]) self.logger = logging.getLogger(self.__class__.__name__) # * Parse arguments self.features = self.context.options.features.split(',') if (hasattr(self.context.options, 'features') and self.context.options.features is not None) else [] self.featureWeights = dict() for feature in self.features: if ':' in feature: # check for explicit weights, e.g. RG:0.8,BY:0.75 try: featureSpec = feature.split(':') self.featureWeights[featureSpec[0].strip()] = float(featureSpec[1].strip()) except Exception as e: self.logger.warn("Invalid feature specification '%s': %s", feature, e) else: # use default weight self.featureWeights[feature.strip()] = default_feature_weight if 'rest' not in self.featureWeights: self.featureWeights['rest'] = default_feature_weight_rest # explicitly specify rest, otherwise previous weights will remain self.logger.info("Searching with feature weights: %s", self.featureWeights) # * Create systems and associated managers self.context.update() # get fresh time self.visSys = VisualSystem(imageSize=self.image_size, timeNow=self.context.timeNow, showMonitor=False) self.visMan = VisionManager(self.visSys, screen_background=self.screen_background) # TODO: Design a better way to share systems/managers (every system has a parent/containing agent?) # * Export RPC calls, if enabled if self.context.isRPCEnabled: self.logger.info("Exporting RPC calls") rpc.export(self.visSys) rpc.export(self.visMan) rpc.refresh() # Context is expected to have started RPC server
def __init__(self, imageSize=default_image_size, timeNow=0.0): # * Initialize members, parameters self.context = Context.getInstance() self.logger = logging.getLogger(__name__) self.logger.debug("Creating simplified Retina") # to distinguish from other Retina versions self.imageSize = imageSize self.imageCenter = (self.imageSize[1] / 2, self.imageSize[0] / 2) self.timeNow = timeNow self.bounds = np.float32([[0.0, 0.0, 2.0], [self.imageSize[0] - 1, self.imageSize[1] - 1, 4.0]]) self.center = (self.bounds[0] + self.bounds[1]) / 2 self.logger.debug("Retina center: {}, image size: {}".format(self.center, self.imageSize)) self.bipolarBlurSize = (5, 5) # size of blurring kernel used when computing Bipolar cell response self.ganglionCenterSurroundKernel = np.float32( [ [ -1, -1, -1, -1, -1, -1, -1 ], [ -1, -1, -1, -1, -1, -1, -1 ], [ -1, -1, 7, 7, 7, -1, -1 ], [ -1, -1, 7, 9, 7, -1, -1 ], [ -1, -1, 7, 7, 7, -1, -1 ], [ -1, -1, -1, -1, -1, -1, -1 ], [ -1, -1, -1, -1, -1, -1, -1 ] ]) self.ganglionCenterSurroundKernel /= np.sum(self.ganglionCenterSurroundKernel) # normalize #self.logger.info("Ganglion center-surround kernel:\n{}".format(self.ganglionCenterSurroundKernel)) # [debug] self.ganglionKernelLevels = 4 self.ganglionKernels = [None] * self.ganglionKernelLevels self.ganglionKernels[0] = self.ganglionCenterSurroundKernel for i in xrange(1, self.ganglionKernelLevels): self.ganglionKernels[i] = cv2.resize(self.ganglionKernels[i - 1], dsize=None, fx=2, fy=2) self.ganglionKernels[i] /= np.sum(self.ganglionKernels[i]) # normalize #self.logger.info("Ganglion center-surround kernel sizes ({} levels): {}".format(self.ganglionKernelLevels, ", ".join("{}".format(k.shape) for k in self.ganglionKernels))) # [debug] # * Image and related members self.imageCenter = (self.imageSize[1] / 2, self.imageSize[0] / 2) self.imageShapeC3 = (self.imageSize[1], self.imageSize[0], 3) # numpy shape for 3 channel images self.imageShapeC1 = (self.imageSize[1], self.imageSize[0]) # numpy shape for single channel images # NOTE Image shapes (h, w, 1) and (h, w) are not compatible unless we use keepdims=True for numpy operations self.imageTypeInt = np.uint8 # numpy dtype for integer-valued images self.imageTypeFloat = np.float32 # numpy dtype for real-valued images self.images = OrderedDict() # ** RGB and HSV images self.images['BGR'] = np.zeros(self.imageShapeC3, dtype=self.imageTypeInt) self.images['HSV'] = np.zeros(self.imageShapeC3, dtype=self.imageTypeInt) self.images['H'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt) self.images['S'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt) self.images['V'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt) # ** Freq/hue-dependent response images for rods and different cone types self.imageRod = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) self.imagesCone = dict() # NOTE dict keys must match names of Cone.cone_types self.imagesCone['S'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) self.imagesCone['M'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) self.imagesCone['L'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) # ** Bipolar and Ganglion cell response images # TODO Add more Ganglion cell types with different receptive field properties (color-opponent cells) # 'RG' +Red -Green # 'GR' +Green -Red # 'RB' +Red -Blue # 'BR' +Blue -Red # 'BY' +Blue -Yellow # 'YB' +Yellow -Blue # 'WK' +White -Black (currently 'ON') # 'KW' +Black -White (currently 'OFF') # NOTE: R = L cones, G = M cones, B = S cones self.imagesBipolar = dict() self.imagesBipolar['ON'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) self.imagesBipolar['OFF'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) self.imagesGanglion = dict() self.imagesGanglion['ON'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) self.imagesGanglion['OFF'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) # TODO Verify why image shapes (h, w, 1) and (h, w) are not compatible (use keepdims=True for numpy operations) self.imagesGanglion['RG'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) self.imagesGanglion['GR'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) self.imagesGanglion['RB'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) self.imagesGanglion['BR'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) self.imagesGanglion['BY'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) self.imagesGanglion['YB'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) # ** Combined response (salience) image self.imageSalience = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) # ** Spatial attention map with a central (covert) spotlight (currently unused; TODO move to VisualCortex? also, use np.ogrid?) self.imageAttention = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) cv2.circle(self.imageAttention, (self.imageSize[1] / 2, self.imageSize[0] / 2), self.imageSize[0] / 3, 1.0, cv.CV_FILLED) self.imageAttention = cv2.blur(self.imageAttention, (self.imageSize[0] / 4, self.imageSize[0] / 4)) # coarse blur # ** Output image(s) if self.context.options.gui: self.imageOut = np.zeros(self.imageShapeC3, dtype=self.imageTypeInt)
# * TODO Compute feature vector of attended region # * Show output images if in GUI mode if self.context.options.gui: #cv2.imshow("Hue", self.images['H']) #cv2.imshow("Saturation", self.images['S']) #cv2.imshow("Value", self.images['V']) cv2.imshow("Rod response", self.imageRod) cv2.imshow("S-cone response", self.imagesCone['S']) cv2.imshow("M-cone response", self.imagesCone['M']) cv2.imshow("L-cone response", self.imagesCone['L']) cv2.imshow("ON Bipolar cells", self.imagesBipolar['ON']) cv2.imshow("OFF Bipolar cells", self.imagesBipolar['OFF']) #cv2.imshow("ON Ganglion cells", self.imagesGanglion['ON']) #cv2.imshow("OFF Ganglion cells", self.imagesGanglion['OFF']) for ganglionType, ganglionImage in self.imagesGanglion.iteritems(): cv2.imshow("{} Ganglion cells".format(ganglionType), ganglionImage) cv2.imshow("Salience", self.imageSalience) # Designate a representative output image self.imageOut = self.imageSalience #_, self.imageOut = cv2.threshold(self.imageOut, 0.15, 1.0, cv2.THRESH_TOZERO) # apply threshold to remove low-response regions return True, self.imageOut if __name__ == "__main__": Context.createInstance(description="Test application that uses a SimplifiedProjector to run image input through a (simplified) Retina.") run(Projector(Retina()))
#result_uint8 = cv2.normalize(result, None, 0, 255, cv2.NORM_MINMAX, cv2.CV_8U) # normalize (issue is variable scale) result_uint8 = np.uint8(result * 255.0) # scale, for display (better avoid and return None if no GUI) #return result_uint8, minMatch, maxMatch, minMatchLoc, maxMatchLoc # too many returns, generalize to *best* match value and loc if self.method == cv2.TM_SQDIFF or self.method == cv2.TM_SQDIFF_NORMED: return PatternMatch(value=minMatch, location=minMatchLoc, result=result_uint8, matcher=self) else: # TM_CCORR or TM_CCOEFF return PatternMatch(value=maxMatch, location=maxMatchLoc, result=result_uint8, matcher=self) if __name__ == "__main__": argParser = argparse.ArgumentParser(add_help=False) argParser.add_argument('--zelinsky', action='store_true', help="run a Zelinsky search agent") argParser.add_argument('--target', default='Q', choices=('Q', 'O'), help='target symbol (Q or O)') argParser.add_argument('--size', dest='num_stimuli', type=int, default=5, help='display size (no. of stimuli) to expect') argParser.add_argument('--features', type=str, default=None, help="features to look for, comma separated") # duplicated for VisualSearchAgent (TODO: Find a better way to unify args, parsers) context = Context.createInstance(description="Zelinsky search agent", parent_argparsers=[argParser]) if context.options.zelinsky: if context.options.features is None: context.options.features = 'OFF:1.0' # Zelinsky-specific ZelinksyFinder(target=context.options.target, distractors=('O' if context.options.target == 'Q' else 'Q'), numStimuli=context.options.num_stimuli).run() else: VisualSearchAgent().run() # Some example invocations #ZelinksyFinder(target='Q', distractors=['O'], numStimuli= 5).run() # target: 'Q', distractor: 'O'; size: 5 [default] #ZelinksyFinder(target='Q', distractors=['O'], numStimuli=17).run() # target: 'Q', distractor: 'O'; size: 17 #ZelinksyFinder(target='O', distractors=['Q'], numStimuli= 5).run() # target: 'O', distractor: 'Q'; size: 5 #ZelinksyFinder(target='O', distractors=['Q'], numStimuli=17).run() # target: 'O', distractor: 'Q'; size: 17
self.move(np.int_([-offset[0], -offset[1]])) # move back to center def enableEventLogging(self, filename_prefix="ocular-events", rpc_export=False, start_server=False): eventFilename = "logs/{}_{}.log".format(filename_prefix, time.strftime(EventLogger.timestamp_format, time.localtime(time.time()))) self.eventLogger = EventLogger(eventFilename, rpc_export=rpc_export, start_server=start_server) self.logEvents = True def getFocusPoint(self): return self.projector.focusPoint def getFocusOffset(self): return (self.projector.focusPoint[0] - self.projector.screenSize[0] / 2, self.projector.focusPoint[1] - self.projector.screenSize[1] / 2) def getVelocity(self): return self.v # Testing if __name__ == "__main__": context = Context.createInstance(description="Ocular motion testing") projector = Projector() ocular = EmulatedOcularMotionSystem(projector) runner = InputRunner(projector) while runner.update(): ocular.update(context.timeNow) if not ocular.isMoving: ocular.move(np.int_([np.random.uniform(-100, 100), np.random.uniform(-100, 100)])) runner.cleanUp()
def __init__(self, imageSize=default_image_size, timeNow=0.0): # * Initialize members, parameters self.context = Context.getInstance() self.logger = logging.getLogger(__name__) self.imageSize = imageSize self.timeNow = timeNow self.bounds = np.float32([[0.0, 0.0, 2.0], [self.imageSize[0] - 1, self.imageSize[1] - 1, 4.0]]) self.center = (self.bounds[0] + self.bounds[1]) / 2 self.logger.debug("Retina center: {}, image size: {}".format(self.center, self.imageSize)) self.rodDistribution = SymmetricLogNormal(mu=5.0, sigma=0.5, center=self.center) self.rodPlotColor = 'darkmagenta' self.coneDistribution = MultivariateNormal(mu=self.center, cov=(np.float32([1000.0, 1000.0, 1.0]) * np.identity(3, dtype=np.float32))) # TODO Create cone populations of different types with their respective spatial distributions (e.g. blue cones are mostly spread out) self.conePlotColor = 'darkgreen' self.conePlotColorsByType = [hsv_to_rgb(np.float32([[[coneType.hueResponse.mu / 180.0, 1.0, 1.0]]]))[0, 0] for coneType in Cone.cone_types] self.bipolarCellDistribution = MultivariateNormal(mu=self.center + np.float32([0.0, 0.0, 10.0]), cov=(np.float32([16000.0, 16000.0, 1.0]) * np.identity(3, dtype=np.float32))) self.bipolarCellPlotColor = 'orange' # * Image and related members self.imageCenter = (self.imageSize[1] / 2, self.imageSize[0] / 2) self.imageShapeC3 = (self.imageSize[1], self.imageSize[0], 3) # numpy shape for 3 channel images self.imageShapeC1 = (self.imageSize[1], self.imageSize[0]) # numpy shape for single channel images # NOTE Image shapes (h, w, 1) and (h, w) are not compatible unless we use keepdims=True for numpy operations self.imageTypeInt = np.uint8 # numpy dtype for integer-valued images self.imageTypeFloat = np.float32 # numpy dtype for real-valued images self.images = OrderedDict() # ** RGB and HSV images self.images['BGR'] = np.zeros(self.imageShapeC3, dtype=self.imageTypeInt) self.images['HSV'] = np.zeros(self.imageShapeC3, dtype=self.imageTypeInt) self.images['H'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt) self.images['S'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt) self.images['V'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt) # ** Freq/hue-dependent response images for rods and different cone types self.imageRod = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) self.imagesCone = dict() # NOTE dict keys must match names of Cone.cone_types self.imagesCone['S'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) self.imagesCone['M'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) self.imagesCone['L'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat) # ** Output image(s) self.imageOut = None if self.context.options.gui: self.imageOut = np.zeros(self.imageShapeC3, dtype=self.imageTypeInt) self.imagesBipolar = dict() self.imagesBipolar['ON'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt) self.imagesBipolar['OFF'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt) # * Create neuron populations # ** Photoreceptors self.rods = Population(numNeurons=self.num_rods, timeNow=self.timeNow, neuronTypes=[Rod], bounds=self.bounds, distribution=self.rodDistribution, retina=self) self.cones = Population(numNeurons=self.num_cones, timeNow=self.timeNow, neuronTypes=[Cone], bounds=self.bounds, distribution=self.coneDistribution, retina=self) self.coneTypeNames = [coneType.name for coneType in Cone.cone_types] # mainly for plotting # ** Bipolar cells self.bipolarCells = Population(numNeurons=self.num_bipolar_cells, timeNow=self.timeNow, neuronTypes=[BipolarCell], bounds=self.bounds, distribution=self.bipolarCellDistribution, retina=self) # * Connect neuron populations growthConeDirection = self.bipolarCells.distribution.mu - self.cones.distribution.mu # NOTE only using cone distribution center growthConeDirection /= np.linalg.norm(growthConeDirection, ord=2) # need a unit vector self.cones.connectWith(self.bipolarCells, maxConnectionsPerNeuron=10, growthCone=GrowthCone(growthConeDirection, spreadFactor=1)) self.rods.connectWith(self.bipolarCells, maxConnectionsPerNeuron=25, growthCone=GrowthCone(growthConeDirection, spreadFactor=1))
def setUp(self): Context.createInstance()
if keyChar == '.': self.testRodIdx = (self.testRodIdx + 1) % len(self.target.rods.neurons) self.testRod = self.target.rods.neurons[self.testRodIdx] self.logger.info("[>] Test rod [{}]: {}".format(self.testRodIdx, self.testRod)) elif keyChar == ',': self.testRodIdx = (self.testRodIdx - 1) % len(self.target.rods.neurons) self.testRod = self.target.rods.neurons[self.testRodIdx] self.logger.info("[<] Test rod [{}]: {}".format(self.testRodIdx, self.testRod)) else: return Projector.onKeyPress(self, key, keyChar) return True print "Running MonitoringProjector instance..." run(MonitoringProjector(Projector), description="Retina processing with monitor on a single neuron.") if __name__ == "__main__": argParser = argparse.ArgumentParser(add_help=False) argParser.add_argument('--test', default="test_projector", help="test case to run (a test_ method in TestRetina)") context = Context.createInstance(parent_argparsers=[argParser]) try: runner = TestRetina(context.options.test).run if context.options.debug: import pdb pdb.runcall(runner) else: runner() except ValueError as e: print "Invalid test: {}".format(e) print "Pick from: {}".format(", ".join(name for name, method in inspect.getmembers(TestRetina, predicate=inspect.ismethod) if name.startswith("test_")))