def __init__(self):
# * Create application context, passing in custom arguments, and get a logger
argParser = argparse.ArgumentParser(add_help=False)
#argParser.add_argument('--in', type=str, default="coil-100", help="path to directory containing input images") # use input_source as directory; default to current directory
argParser.add_argument('--out', type=str, default=None, help="path to output directory") # should this be a common parameter in Context?
argParser.add_argument('--obj', type=str, default="1,101,1", required=False, help="object ID range, right-open interval <start>,<stop>,<step> (no spaces); default: full range")
argParser.add_argument('--view', type=str, default="0,360,5", required=False, help="view angle range in degrees, right-open interval <start>,<stop>,<step> (no spaces); default: full range")
self.context = Context.createInstance(description="COIL-100 image dataset processor", parent_argparsers=[argParser]) # TODO how to gather arg parsers from other interested parties?
self.logger = logging.getLogger(self.__class__.__name__)
# * Parse arguments
self.inDir = self.context.options.input_source # should be an absolute path to a dir with COIL images; if it is a file/camera instead, it will be used as sole input
# TODO also accept wildcards using glob.glob()?
self.outDir = self.context.options.out # just for convenience
self.outFile = None
if self.outDir is not None: # TODO otherwise default to some directory?
if os.path.isdir(self.outDir):
now = datetime.now()
outFilepath = os.path.join(self.outDir, "{}{}{}{}{}.{}".format(self.output_file_prefix, self.output_file_sep, now.strftime('%Y-%m-%d'), self.output_file_sep, now.strftime('%H-%M-%S'), self.output_file_ext))
self.logger.info("Output file: {}".format(outFilepath))
self.outFile = open(outFilepath, 'w') # open output file for storing features (TODO use with.. block instead in start()?)
else:
self.logger.warn("Invalid output directory \"{}\"; no output will be saved".format(self.outDir))
self.outDir = None # TODO create output directory if it doesn't exist
self.objRange = xrange(*(int(x) for x in self.context.options.obj.split(',')))
self.viewRange = xrange(*(int(x) for x in self.context.options.view.split(',')))
# * Create visual system and manager
self.context.update() # get fresh time
self.visSys = VisualSystem(imageSize=self.image_size, timeNow=self.context.timeNow)
self.visMan = COILManager(self.visSys)
def __init__(self):
# * Create application context, passing in custom arguments, and get a logger
argParser = argparse.ArgumentParser(add_help=False)
argParser.add_argument('--features', type=str, default=None, help="features to look for, comma separated")
self.context = Context.createInstance(description=self.__class__.__name__, parent_argparsers=[argParser])
self.logger = logging.getLogger(self.__class__.__name__)
# * Parse arguments
self.features = self.context.options.features.split(',') if (hasattr(self.context.options, 'features') and self.context.options.features is not None) else []
self.featureWeights = dict()
for feature in self.features:
if ':' in feature: # check for explicit weights, e.g. RG:0.8,BY:0.75
try:
featureSpec = feature.split(':')
self.featureWeights[featureSpec[0].strip()] = float(featureSpec[1].strip())
except Exception as e:
self.logger.warn("Invalid feature specification '%s': %s", feature, e)
else: # use default weight
self.featureWeights[feature.strip()] = default_feature_weight
if 'rest' not in self.featureWeights:
self.featureWeights['rest'] = default_feature_weight_rest # explicitly specify rest, otherwise previous weights will remain
self.logger.info("Searching with feature weights: %s", self.featureWeights)
# * Create systems and associated managers
self.context.update() # get fresh time
self.visSys = VisualSystem(imageSize=self.image_size, timeNow=self.context.timeNow, showMonitor=False)
self.visMan = VisionManager(self.visSys, screen_background=self.screen_background)
# TODO: Design a better way to share systems/managers (every system has a parent/containing agent?)
# * Export RPC calls, if enabled
if self.context.isRPCEnabled:
self.logger.info("Exporting RPC calls")
rpc.export(self.visSys)
rpc.export(self.visMan)
rpc.refresh() # Context is expected to have started RPC server
def __init__(self, imageSize=default_image_size, timeNow=0.0):
# * Initialize members, parameters
self.context = Context.getInstance()
self.logger = logging.getLogger(__name__)
self.logger.debug("Creating simplified Retina") # to distinguish from other Retina versions
self.imageSize = imageSize
self.imageCenter = (self.imageSize[1] / 2, self.imageSize[0] / 2)
self.timeNow = timeNow
self.bounds = np.float32([[0.0, 0.0, 2.0], [self.imageSize[0] - 1, self.imageSize[1] - 1, 4.0]])
self.center = (self.bounds[0] + self.bounds[1]) / 2
self.logger.debug("Retina center: {}, image size: {}".format(self.center, self.imageSize))
self.bipolarBlurSize = (5, 5) # size of blurring kernel used when computing Bipolar cell response
self.ganglionCenterSurroundKernel = np.float32(
[ [ -1, -1, -1, -1, -1, -1, -1 ],
[ -1, -1, -1, -1, -1, -1, -1 ],
[ -1, -1, 7, 7, 7, -1, -1 ],
[ -1, -1, 7, 9, 7, -1, -1 ],
[ -1, -1, 7, 7, 7, -1, -1 ],
[ -1, -1, -1, -1, -1, -1, -1 ],
[ -1, -1, -1, -1, -1, -1, -1 ] ])
self.ganglionCenterSurroundKernel /= np.sum(self.ganglionCenterSurroundKernel) # normalize
#self.logger.info("Ganglion center-surround kernel:\n{}".format(self.ganglionCenterSurroundKernel)) # [debug]
self.ganglionKernelLevels = 4
self.ganglionKernels = [None] * self.ganglionKernelLevels
self.ganglionKernels[0] = self.ganglionCenterSurroundKernel
for i in xrange(1, self.ganglionKernelLevels):
self.ganglionKernels[i] = cv2.resize(self.ganglionKernels[i - 1], dsize=None, fx=2, fy=2)
self.ganglionKernels[i] /= np.sum(self.ganglionKernels[i]) # normalize
#self.logger.info("Ganglion center-surround kernel sizes ({} levels): {}".format(self.ganglionKernelLevels, ", ".join("{}".format(k.shape) for k in self.ganglionKernels))) # [debug]
# * Image and related members
self.imageCenter = (self.imageSize[1] / 2, self.imageSize[0] / 2)
self.imageShapeC3 = (self.imageSize[1], self.imageSize[0], 3) # numpy shape for 3 channel images
self.imageShapeC1 = (self.imageSize[1], self.imageSize[0]) # numpy shape for single channel images
# NOTE Image shapes (h, w, 1) and (h, w) are not compatible unless we use keepdims=True for numpy operations
self.imageTypeInt = np.uint8 # numpy dtype for integer-valued images
self.imageTypeFloat = np.float32 # numpy dtype for real-valued images
self.images = OrderedDict()
# ** RGB and HSV images
self.images['BGR'] = np.zeros(self.imageShapeC3, dtype=self.imageTypeInt)
self.images['HSV'] = np.zeros(self.imageShapeC3, dtype=self.imageTypeInt)
self.images['H'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt)
self.images['S'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt)
self.images['V'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt)
# ** Freq/hue-dependent response images for rods and different cone types
self.imageRod = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
self.imagesCone = dict() # NOTE dict keys must match names of Cone.cone_types
self.imagesCone['S'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
self.imagesCone['M'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
self.imagesCone['L'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
# ** Bipolar and Ganglion cell response images
# TODO Add more Ganglion cell types with different receptive field properties (color-opponent cells)
# 'RG' +Red -Green
# 'GR' +Green -Red
# 'RB' +Red -Blue
# 'BR' +Blue -Red
# 'BY' +Blue -Yellow
# 'YB' +Yellow -Blue
# 'WK' +White -Black (currently 'ON')
# 'KW' +Black -White (currently 'OFF')
# NOTE: R = L cones, G = M cones, B = S cones
self.imagesBipolar = dict()
self.imagesBipolar['ON'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
self.imagesBipolar['OFF'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
self.imagesGanglion = dict()
self.imagesGanglion['ON'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
self.imagesGanglion['OFF'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
# TODO Verify why image shapes (h, w, 1) and (h, w) are not compatible (use keepdims=True for numpy operations)
self.imagesGanglion['RG'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
self.imagesGanglion['GR'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
self.imagesGanglion['RB'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
self.imagesGanglion['BR'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
self.imagesGanglion['BY'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
self.imagesGanglion['YB'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
# ** Combined response (salience) image
self.imageSalience = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
# ** Spatial attention map with a central (covert) spotlight (currently unused; TODO move to VisualCortex? also, use np.ogrid?)
self.imageAttention = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
cv2.circle(self.imageAttention, (self.imageSize[1] / 2, self.imageSize[0] / 2), self.imageSize[0] / 3, 1.0, cv.CV_FILLED)
self.imageAttention = cv2.blur(self.imageAttention, (self.imageSize[0] / 4, self.imageSize[0] / 4)) # coarse blur
# ** Output image(s)
if self.context.options.gui:
self.imageOut = np.zeros(self.imageShapeC3, dtype=self.imageTypeInt)
# * TODO Compute feature vector of attended region
# * Show output images if in GUI mode
if self.context.options.gui:
#cv2.imshow("Hue", self.images['H'])
#cv2.imshow("Saturation", self.images['S'])
#cv2.imshow("Value", self.images['V'])
cv2.imshow("Rod response", self.imageRod)
cv2.imshow("S-cone response", self.imagesCone['S'])
cv2.imshow("M-cone response", self.imagesCone['M'])
cv2.imshow("L-cone response", self.imagesCone['L'])
cv2.imshow("ON Bipolar cells", self.imagesBipolar['ON'])
cv2.imshow("OFF Bipolar cells", self.imagesBipolar['OFF'])
#cv2.imshow("ON Ganglion cells", self.imagesGanglion['ON'])
#cv2.imshow("OFF Ganglion cells", self.imagesGanglion['OFF'])
for ganglionType, ganglionImage in self.imagesGanglion.iteritems():
cv2.imshow("{} Ganglion cells".format(ganglionType), ganglionImage)
cv2.imshow("Salience", self.imageSalience)
# Designate a representative output image
self.imageOut = self.imageSalience
#_, self.imageOut = cv2.threshold(self.imageOut, 0.15, 1.0, cv2.THRESH_TOZERO) # apply threshold to remove low-response regions
return True, self.imageOut
if __name__ == "__main__":
Context.createInstance(description="Test application that uses a SimplifiedProjector to run image input through a (simplified) Retina.")
run(Projector(Retina()))
#result_uint8 = cv2.normalize(result, None, 0, 255, cv2.NORM_MINMAX, cv2.CV_8U) # normalize (issue is variable scale)
result_uint8 = np.uint8(result * 255.0) # scale, for display (better avoid and return None if no GUI)
#return result_uint8, minMatch, maxMatch, minMatchLoc, maxMatchLoc # too many returns, generalize to *best* match value and loc
if self.method == cv2.TM_SQDIFF or self.method == cv2.TM_SQDIFF_NORMED:
return PatternMatch(value=minMatch, location=minMatchLoc, result=result_uint8, matcher=self)
else: # TM_CCORR or TM_CCOEFF
return PatternMatch(value=maxMatch, location=maxMatchLoc, result=result_uint8, matcher=self)
if __name__ == "__main__":
argParser = argparse.ArgumentParser(add_help=False)
argParser.add_argument('--zelinsky', action='store_true', help="run a Zelinsky search agent")
argParser.add_argument('--target', default='Q', choices=('Q', 'O'), help='target symbol (Q or O)')
argParser.add_argument('--size', dest='num_stimuli', type=int, default=5, help='display size (no. of stimuli) to expect')
argParser.add_argument('--features', type=str, default=None, help="features to look for, comma separated") # duplicated for VisualSearchAgent (TODO: Find a better way to unify args, parsers)
context = Context.createInstance(description="Zelinsky search agent", parent_argparsers=[argParser])
if context.options.zelinsky:
if context.options.features is None:
context.options.features = 'OFF:1.0' # Zelinsky-specific
ZelinksyFinder(target=context.options.target, distractors=('O' if context.options.target == 'Q' else 'Q'), numStimuli=context.options.num_stimuli).run()
else:
VisualSearchAgent().run()
# Some example invocations
#ZelinksyFinder(target='Q', distractors=['O'], numStimuli= 5).run() # target: 'Q', distractor: 'O'; size: 5 [default]
#ZelinksyFinder(target='Q', distractors=['O'], numStimuli=17).run() # target: 'Q', distractor: 'O'; size: 17
#ZelinksyFinder(target='O', distractors=['Q'], numStimuli= 5).run() # target: 'O', distractor: 'Q'; size: 5
#ZelinksyFinder(target='O', distractors=['Q'], numStimuli=17).run() # target: 'O', distractor: 'Q'; size: 17
self.move(np.int_([-offset[0], -offset[1]])) # move back to center
def enableEventLogging(self, filename_prefix="ocular-events", rpc_export=False, start_server=False):
eventFilename = "logs/{}_{}.log".format(filename_prefix, time.strftime(EventLogger.timestamp_format, time.localtime(time.time())))
self.eventLogger = EventLogger(eventFilename, rpc_export=rpc_export, start_server=start_server)
self.logEvents = True
def getFocusPoint(self):
return self.projector.focusPoint
def getFocusOffset(self):
return (self.projector.focusPoint[0] - self.projector.screenSize[0] / 2, self.projector.focusPoint[1] - self.projector.screenSize[1] / 2)
def getVelocity(self):
return self.v
# Testing
if __name__ == "__main__":
context = Context.createInstance(description="Ocular motion testing")
projector = Projector()
ocular = EmulatedOcularMotionSystem(projector)
runner = InputRunner(projector)
while runner.update():
ocular.update(context.timeNow)
if not ocular.isMoving:
ocular.move(np.int_([np.random.uniform(-100, 100), np.random.uniform(-100, 100)]))
runner.cleanUp()
def __init__(self, imageSize=default_image_size, timeNow=0.0):
# * Initialize members, parameters
self.context = Context.getInstance()
self.logger = logging.getLogger(__name__)
self.imageSize = imageSize
self.timeNow = timeNow
self.bounds = np.float32([[0.0, 0.0, 2.0], [self.imageSize[0] - 1, self.imageSize[1] - 1, 4.0]])
self.center = (self.bounds[0] + self.bounds[1]) / 2
self.logger.debug("Retina center: {}, image size: {}".format(self.center, self.imageSize))
self.rodDistribution = SymmetricLogNormal(mu=5.0, sigma=0.5, center=self.center)
self.rodPlotColor = 'darkmagenta'
self.coneDistribution = MultivariateNormal(mu=self.center, cov=(np.float32([1000.0, 1000.0, 1.0]) * np.identity(3, dtype=np.float32)))
# TODO Create cone populations of different types with their respective spatial distributions (e.g. blue cones are mostly spread out)
self.conePlotColor = 'darkgreen'
self.conePlotColorsByType = [hsv_to_rgb(np.float32([[[coneType.hueResponse.mu / 180.0, 1.0, 1.0]]]))[0, 0] for coneType in Cone.cone_types]
self.bipolarCellDistribution = MultivariateNormal(mu=self.center + np.float32([0.0, 0.0, 10.0]), cov=(np.float32([16000.0, 16000.0, 1.0]) * np.identity(3, dtype=np.float32)))
self.bipolarCellPlotColor = 'orange'
# * Image and related members
self.imageCenter = (self.imageSize[1] / 2, self.imageSize[0] / 2)
self.imageShapeC3 = (self.imageSize[1], self.imageSize[0], 3) # numpy shape for 3 channel images
self.imageShapeC1 = (self.imageSize[1], self.imageSize[0]) # numpy shape for single channel images
# NOTE Image shapes (h, w, 1) and (h, w) are not compatible unless we use keepdims=True for numpy operations
self.imageTypeInt = np.uint8 # numpy dtype for integer-valued images
self.imageTypeFloat = np.float32 # numpy dtype for real-valued images
self.images = OrderedDict()
# ** RGB and HSV images
self.images['BGR'] = np.zeros(self.imageShapeC3, dtype=self.imageTypeInt)
self.images['HSV'] = np.zeros(self.imageShapeC3, dtype=self.imageTypeInt)
self.images['H'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt)
self.images['S'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt)
self.images['V'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt)
# ** Freq/hue-dependent response images for rods and different cone types
self.imageRod = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
self.imagesCone = dict() # NOTE dict keys must match names of Cone.cone_types
self.imagesCone['S'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
self.imagesCone['M'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
self.imagesCone['L'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeFloat)
# ** Output image(s)
self.imageOut = None
if self.context.options.gui:
self.imageOut = np.zeros(self.imageShapeC3, dtype=self.imageTypeInt)
self.imagesBipolar = dict()
self.imagesBipolar['ON'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt)
self.imagesBipolar['OFF'] = np.zeros(self.imageShapeC1, dtype=self.imageTypeInt)
# * Create neuron populations
# ** Photoreceptors
self.rods = Population(numNeurons=self.num_rods, timeNow=self.timeNow, neuronTypes=[Rod], bounds=self.bounds, distribution=self.rodDistribution, retina=self)
self.cones = Population(numNeurons=self.num_cones, timeNow=self.timeNow, neuronTypes=[Cone], bounds=self.bounds, distribution=self.coneDistribution, retina=self)
self.coneTypeNames = [coneType.name for coneType in Cone.cone_types] # mainly for plotting
# ** Bipolar cells
self.bipolarCells = Population(numNeurons=self.num_bipolar_cells, timeNow=self.timeNow, neuronTypes=[BipolarCell], bounds=self.bounds, distribution=self.bipolarCellDistribution, retina=self)
# * Connect neuron populations
growthConeDirection = self.bipolarCells.distribution.mu - self.cones.distribution.mu # NOTE only using cone distribution center
growthConeDirection /= np.linalg.norm(growthConeDirection, ord=2) # need a unit vector
self.cones.connectWith(self.bipolarCells, maxConnectionsPerNeuron=10, growthCone=GrowthCone(growthConeDirection, spreadFactor=1))
self.rods.connectWith(self.bipolarCells, maxConnectionsPerNeuron=25, growthCone=GrowthCone(growthConeDirection, spreadFactor=1))
def setUp(self): Context.createInstance()
if keyChar == '.':
self.testRodIdx = (self.testRodIdx + 1) % len(self.target.rods.neurons)
self.testRod = self.target.rods.neurons[self.testRodIdx]
self.logger.info("[>] Test rod [{}]: {}".format(self.testRodIdx, self.testRod))
elif keyChar == ',':
self.testRodIdx = (self.testRodIdx - 1) % len(self.target.rods.neurons)
self.testRod = self.target.rods.neurons[self.testRodIdx]
self.logger.info("[<] Test rod [{}]: {}".format(self.testRodIdx, self.testRod))
else:
return Projector.onKeyPress(self, key, keyChar)
return True
print "Running MonitoringProjector instance..."
run(MonitoringProjector(Projector), description="Retina processing with monitor on a single neuron.")
if __name__ == "__main__":
argParser = argparse.ArgumentParser(add_help=False)
argParser.add_argument('--test', default="test_projector", help="test case to run (a test_ method in TestRetina)")
context = Context.createInstance(parent_argparsers=[argParser])
try:
runner = TestRetina(context.options.test).run
if context.options.debug:
import pdb
pdb.runcall(runner)
else:
runner()
except ValueError as e:
print "Invalid test: {}".format(e)
print "Pick from: {}".format(", ".join(name for name, method in inspect.getmembers(TestRetina, predicate=inspect.ismethod) if name.startswith("test_")))
