def spatialCorrelationAlgorithm_ZScore(data, radius, printProcess=False, eventsKey='ZScore', spontKey='SpontZScore', threshold=1.645):
## check that data has 'xPos', 'yPos' and 'numOfPostEvents'
#SpatialCorrelator.checkArrayInput(data)
#prof = Profiler("bendelsSpatialCorrelationAlgorithm", disabled=True)
fields = data.dtype.names
if 'xPos' not in fields or 'yPos' not in fields or eventsKey not in fields or spontKey not in fields:
raise HelpfulException("Array input needs to have the following fields: 'xPos', 'yPos', the fields specified in *eventsKey* and *spontKey*. Current fields are: %s" %str(fields))
#prof.mark("checked fields")
## add 'prob' field to data array
if 'prob' not in data.dtype.names:
arr = utilFn.concatenateColumns([data, np.zeros(len(data), dtype=[('prob', float)])])
#arr[:] = data
data = arr
else:
data['prob']=0
#prof.mark("set 'prob' field")
table = np.zeros((200, 200)) ## add a lookup table so that we don't have to calculate the same probabilities over and over...saves a bit of time
## spatial correlation algorithm from :
## Bendels, MHK; Beed, P; Schmitz, D; Johenning, FW; and Leibold C. Detection of input sites in
## scanning photostimulation data based on spatial correlations. 2010. Journal of Neuroscience Methods.
## calculate probability of seeing a spontaneous event in time window -- for ZScore method, calculate probability that ZScore is spontaneously high
p = len(data[data[spontKey] < -threshold])/float(len(data))
#p = 1-np.exp(-spontRate*timeWindow)
#if printProcess:
# print "====== Spontaneous Probability: %f =======" % p
#prof.mark('calculated spontaneous probability')
## for each spot, calculate the probability of having the events in nearby spots occur randomly
for x in data:
spots = data[(np.sqrt((data['xPos']-x['xPos'])**2+(data['yPos']-x['yPos'])**2)) < radius]
nSpots = len(spots)
nEventSpots = len(spots[spots[eventsKey] < -threshold])
prob = 0
if table[nEventSpots, nSpots] != 0: ## try looking up value in table (it was stored there if we calculated it before), otherwise calculate it now
prob = table[nEventSpots, nSpots]
#prof.mark('look-up')
else:
for j in range(nEventSpots, nSpots+1):
a = ((p**j)*((1-p)**(nSpots-j))*math.factorial(nSpots))/(math.factorial(j)*math.factorial(nSpots-j))
if printProcess:
print " Prob for %i events: %f Total: %f" %(j, a, prob+a)
prob += a
table[nEventSpots, nSpots] = prob
#prof.mark('calculate')
if printProcess: ## for debugging
print " %i out of %i spots had events. Probability: %f" %(nEventSpots, nSpots, prob)
x['prob'] = prob
#prof.mark("calculated probabilities")
#prof.finish()
return data
def loadCell(cell, reloadData=False):
global events
if reloadData:
events.pop(cell, None)
if cell in events:
return
db = man.getModule('Data Manager').currentDatabase()
mod = man.dataModel
allEvents = []
hvals = {}
nEv = 0
positionCache = {}
tcache = {}
print "Loading all events for cell", cell
tot = db.select(eventView, 'count()', where={'CellDir': cell})[0]['count()']
print tot, "total events.."
with pg.ProgressDialog('Loading event data...', maximum=tot, wait=0) as dlg:
for ev in db.iterSelect(eventView, ['ProtocolSequenceDir', 'SourceFile', 'fitAmplitude', 'fitTime', 'fitDecayTau', 'fitRiseTau', 'fitTimeToPeak', 'fitLengthOverDecay', 'fitFractionalError', 'userTransform', 'CellType', 'CellDir', 'ProtocolDir'], where={'CellDir': cell}, toArray=True, chunkSize=200):
extra = np.empty(ev.shape, dtype=[('right', float), ('anterior', float), ('dorsal', float), ('holding', float)])
## insert holding levels
for i in range(len(ev)):
sd = ev[i]['ProtocolSequenceDir']
if sd not in hvals:
cf = ev[i]['SourceFile']
hvals[sd] = mod.getClampHoldingLevel(cf)
#print hvals[sd], cf
extra[i]['holding'] = hvals[sd]
## insert positions
for i in range(len(ev)):
protoDir = ev[i]['SourceFile'].parent()
key = protoDir
#key = (ev[i]['ProtocolSequenceDir'], ev[i]['SourceFile'])
if key not in positionCache:
#try:
#dh = ev[i]['ProtocolDir']
#p1 = pg.Point(dh.info()['Scanner']['position'])
#if key[0] not in tcache:
#tr = pg.SRTTransform()
#tr.restoreState(dh.parent().info()['userTransform'])
#tcache[key[0]] = tr
#trans = tcache[key[0]]
#p2 = trans.map(p1)
#pcache[key] = (p2.x(),p2.y())
#except:
#print key
#raise
rec = db.select('CochlearNucleus_Protocol', where={'ProtocolDir': protoDir})
if len(rec) == 0:
pos = (None, None, None)
elif len(rec) == 1:
pos = (rec[0]['right'], rec[0]['anterior'], rec[0]['dorsal'])
elif len(rec) == 2:
raise Exception("Multiple position records for %s!" % str(protoDir))
positionCache[key] = pos
extra[i]['right'] = positionCache[key][0]
extra[i]['anterior'] = positionCache[key][1]
extra[i]['dorsal'] = positionCache[key][2]
ev = fn.concatenateColumns([ev, extra])
allEvents.append(ev)
nEv += len(ev)
dlg.setValue(nEv)
if dlg.wasCanceled():
raise Exception('Canceled by user.')
ev = np.concatenate(allEvents)
numExSites = 0
numInSites = 0
for site in db.select(siteView, 'ProtocolSequenceDir', where={'CellDir': cell}):
h = hvals.get(site['ProtocolSequenceDir'],None)
if h is None:
continue
if h > -0.02:
numInSites += 1
elif h < -0.04:
numExSites += 1
events[cell] = (ev, numExSites, numInSites)
def loadCell(cell, reloadData=False):
global events
if reloadData:
events.pop(cell, None)
if cell in events:
return
db = man.getModule('Data Manager').currentDatabase()
mod = man.dataModel
allEvents = []
hvals = {}
nEv = 0
positionCache = {}
tcache = {}
print("Loading all events for cell", cell)
tot = db.select(eventView, 'count()', where={'CellDir':
cell})[0]['count()']
print(tot, "total events..")
with pg.ProgressDialog('Loading event data...', maximum=tot,
wait=0) as dlg:
for ev in db.iterSelect(eventView, [
'ProtocolSequenceDir', 'SourceFile', 'fitAmplitude', 'fitTime',
'fitDecayTau', 'fitRiseTau', 'fitTimeToPeak',
'fitLengthOverDecay', 'fitFractionalError', 'userTransform',
'CellType', 'CellDir', 'ProtocolDir'
],
where={'CellDir': cell},
toArray=True,
chunkSize=200):
extra = np.empty(ev.shape,
dtype=[('right', float), ('anterior', float),
('dorsal', float), ('holding', float)])
## insert holding levels
for i in range(len(ev)):
sd = ev[i]['ProtocolSequenceDir']
if sd not in hvals:
cf = ev[i]['SourceFile']
hvals[sd] = mod.getClampHoldingLevel(cf)
#print hvals[sd], cf
extra[i]['holding'] = hvals[sd]
## insert positions
for i in range(len(ev)):
protoDir = ev[i]['SourceFile'].parent()
key = protoDir
#key = (ev[i]['ProtocolSequenceDir'], ev[i]['SourceFile'])
if key not in positionCache:
#try:
#dh = ev[i]['ProtocolDir']
#p1 = pg.Point(dh.info()['Scanner']['position'])
#if key[0] not in tcache:
#tr = pg.SRTTransform()
#tr.restoreState(dh.parent().info()['userTransform'])
#tcache[key[0]] = tr
#trans = tcache[key[0]]
#p2 = trans.map(p1)
#pcache[key] = (p2.x(),p2.y())
#except:
#print key
#raise
rec = db.select('CochlearNucleus_Protocol',
where={'ProtocolDir': protoDir})
if len(rec) == 0:
pos = (None, None, None)
elif len(rec) == 1:
pos = (rec[0]['right'], rec[0]['anterior'],
rec[0]['dorsal'])
elif len(rec) == 2:
raise Exception("Multiple position records for %s!" %
str(protoDir))
positionCache[key] = pos
extra[i]['right'] = positionCache[key][0]
extra[i]['anterior'] = positionCache[key][1]
extra[i]['dorsal'] = positionCache[key][2]
ev = fn.concatenateColumns([ev, extra])
allEvents.append(ev)
nEv += len(ev)
dlg.setValue(nEv)
if dlg.wasCanceled():
raise Exception('Canceled by user.')
ev = np.concatenate(allEvents)
numExSites = 0
numInSites = 0
for site in db.select(siteView,
'ProtocolSequenceDir',
where={'CellDir': cell}):
h = hvals.get(site['ProtocolSequenceDir'], None)
if h is None:
continue
if h > -0.02:
numInSites += 1
elif h < -0.04:
numExSites += 1
events[cell] = (ev, numExSites, numInSites)
