# grab all the data we will need
#
################################################
data = {}
for typ in ['AML32', 'Special']:
for condition in ['immobilized',
'transition']: # ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)
try:
# load multiple datasets
dataSets = dh.loadDictFromHDF(outLocData)
keyList = np.sort(dataSets.keys())
results = dh.loadDictFromHDF(outLoc)
# store in dictionary by typ and condition
key = '{}_{}'.format(typ, condition)
data[key] = {}
data[key]['dsets'] = keyList
data[key]['input'] = dataSets
data[key]['analysis'] = results
except IOError:
print typ, condition, 'not found.'
pass
print 'Done reading data.'
################################################
#
def main():
################################################
#
# grab all the data we will need
#
################################################
data = {}
for typ in ['AML32', 'AML70']:
for condition in ['moving',
'chip']: # ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(
typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)
try:
# load multiple datasets
dataSets = dh.loadDictFromHDF(outLocData)
keyList = np.sort(dataSets.keys())
results = dh.loadDictFromHDF(outLoc)
# store in dictionary by typ and condition
key = '{}_{}'.format(typ, condition)
data[key] = {}
data[key]['dsets'] = keyList
data[key]['input'] = dataSets
data[key]['analysis'] = results
except IOError:
print typ, condition, 'not found.'
pass
print 'Done reading data.'
################################################
#
# create figure 1: This is twice the normal size
#
################################################
# we will select a 'special' dataset here, which will have all the individual plots
# select a special dataset - moving AML32. Should be the same as in fig 2
movingAML32 = 'BrainScanner20170613_134800'
#
moving = data['AML32_moving']['input'][movingAML32]
movingAnalysis = data['AML32_moving']['analysis'][movingAML32]
fig = plt.figure('Fig - 4 : Recreating postures', figsize=(9.5, 9.5))
# this gridspec makes one example plot of a heatmap with its PCA
gs1 = gridspec.GridSpec(3, 1, height_ratios=[1, 1, 0.6])
gs1.update(left=0.058,
right=0.99,
wspace=0.25,
bottom=0.075,
top=0.95,
hspace=0.25)
# third row
gsNL = gridspec.GridSpecFromSubplotSpec(
1, 3, subplot_spec=gs1[2, :], wspace=0.4,
hspace=0.5) #, width_ratios = [1,1,1,1])
#ax3= plt.subplot(gsNL[0,0])
#moveAxes(ax3, 'scalex', 0.05)
#moveAxes(ax3, 'scaley', -0.1)
#moveAxes(ax3, 'right', 0.025)
ax1 = plt.subplot(gs1[0, 0], aspect='equal')
ax2 = plt.subplot(gs1[1, 0], aspect='equal')
moveAxes(ax1, 'left', 0.01)
moveAxes(ax2, 'left', 0.01)
for ax in [ax1, ax2]:
moveAxes(ax, 'scale', 0.03)
moveAxes(ax, 'right', 0.03)
moveAxes(ax2, 'up', 0.025)
moveAxes(ax1, 'up', 0.025)
#moveAxes(ax, 'left', 0.03)
ax5 = plt.subplot(gsNL[0])
ax6 = plt.subplot(gsNL[1])
#ax8 = plt.subplot(gsNL[2,0])
ax9 = plt.subplot(gsNL[2])
for ax in [ax5, ax6, ax9]:
moveAxes(ax, 'scale', -0.02)
#moveAxes(ax, 'scalex', -0.02)
moveAxes(ax, 'left', -0.03)
#for ax in [ ax9]:
# moveAxes(ax, 'scale', 0.02)
# moveAxes(ax, 'left', 0.055)
# moveAxes(ax, 'down', 0.02)
#
#moveAxes(ax6, 'down', 0.05)
#add a,b,c letters, 9 pt final size = 18pt in this case
letters = ['A', 'B']
x0 = 0.0
locations = [(x0, 0.99), (x0, 0.65)]
for letter, loc in zip(letters, locations):
plt.figtext(loc[0], loc[1], letter, weight='bold', size=18,\
horizontalalignment='left',verticalalignment='top',)
letters = ['C', 'D', 'E']
y0 = 0.28
locations = [(0, y0), (0.36, y0), (0.69, y0), (0.84, 0.31)]
for letter, loc in zip(letters, locations):
plt.figtext(loc[0], loc[1], letter, weight='bold', size=18,\
horizontalalignment='left',verticalalignment='top',)
# recreate worms
#=============================================================================#
# # load eigenworms
#=============================================================================#
eigenwormsFull = dh.loadEigenBasis(filename='../../utility/Eigenworms.dat',
nComp=7,
new=True)
eigenwormsFull = dh.resize(eigenwormsFull, (7, 100))
eigenworms = dh.loadEigenBasis(filename='../../utility/Eigenworms.dat',
nComp=3,
new=True)
pc1, pc2, pc3, avTrue, thetaTrue = moving['Behavior']['Eigenworm1'],moving['Behavior']['Eigenworm2'],\
moving['Behavior']['Eigenworm3'], moving['Behavior']['AngleVelocity'], moving['Behavior']['Theta']
#thetaTrue2 = np.unwrap(np.arctan2(pc2, pc1))
#plt.plot(thetaTrue2)
#plt.show(block=True)
pcs = np.vstack([pc3, pc2, pc1])
# actual centerline
cl = moving['CL']
#cl = dh.resize(cl, (cl.shape[0], 101, cl.shape[2]))
#pcsNew, meanAngle, lengths, refPoint = dh.calculateEigenwormsFromCL(cl,eigenwormsFull)
#cl = dh.calculateCLfromEW(pcsNew, eigenwormsFull, meanAngle, lengths, refPoint)
#=============================================================================#
# load predicted worm
#=============================================================================#
avP = movingAnalysis['ElasticNet']['AngleVelocity']['output'][:len(pcs[0])]
tP = movingAnalysis['ElasticNet']['Eigenworm3']['output'][:len(pcs[0])]
print 'R2'
print movingAnalysis['ElasticNet']['AngleVelocity'][
'score'], movingAnalysis['ElasticNet']['AngleVelocity'][
'scorepredicted']
print movingAnalysis['ElasticNet']['Eigenworm3']['score'], movingAnalysis[
'ElasticNet']['Eigenworm3']['scorepredicted']
################################################
#
#get data
################################################
# weights
flag = 'ElasticNet'
# one example
time = moving['Neurons']['Time']
label = 'AngleVelocity'
splits = movingAnalysis['Training']
train, test = splits[label]['Train'], splits[label]['Test']
t = moving['Neurons']['Time'][test]
# calculate non-linear fits for many datasets
#=============================================================================#
# correct undershoots in turns
#=============================================================================#
r2s = []
dataAll = []
for key in ['AML32_moving', 'AML70_chip']:
dset = data[key]['input']
res = data[key]['analysis']
for idn in dset.keys():
xdata1 = res[idn]['ElasticNet']['Eigenworm3']['output']
ydata1 = dset[idn]['Behavior']['Eigenworm3']
testL = res[idn]['Training']['Eigenworm3']['Test']
trainL = res[idn]['Training']['Eigenworm3']['Train']
#dataAll.append([xdata1[trainL], ydata1[trainL]])
dataAll.append([xdata1, ydata1])
# fit all datasets jointly
xD, yD = np.concatenate(dataAll, axis=1)
# non-linearity corrected
#=============================================================================#
# correct undershoots in turns
#=============================================================================#
indices = np.argsort(tP)
xdata, ydata = tP[indices], pc3[indices]
# bin the data
def fitfun(x, A, m, s):
# taper off towards the ends, otherwise overshoot horribly
#return x + A*erf(x/s) -m
#return (-5)*(-0.5*(np.sign(x+5)-1)) +x
#return 1.2*(x)#**3 +m*x +s*x**2
#return 2*x#A*x + s*x**3 + m
#return x +(2*erf(x/5))#*x
#return A*(x +np.abs(erf(x/5)))
#return A*(np.abs(erf(x/s)) +m)#*x
return A * erf(x / s) - m
#return x + x[np.abs(x)>10]*1.5
#return A*(x/s-m)**3
#return A/(1+np.exp(s*x))+m#(A*(1-np.exp(-(x-m)**2/s**2))-1)#*x
p0 = [10, 100, 100]
popt, pcov = curve_fit(fitfun, xD, yD, p0, sigma=1. / (np.abs(
(yD)))) #,bounds=[[-2,-10, -10], [15,10,10]])
# correct turning amplitude
tPnew = fitfun(tP, *popt)
cl2, clApprox, clPred, cl3 = compareReconstructedWorms(
cl, eigenworms, avTrue, thetaTrue, pc3, avP, tP, tPnew)
#plt.show(block=True)
mse = np.mean(np.sqrt(np.sum((clPred - cl2)**2, axis=2)), axis=1)
mse2 = np.mean(np.sqrt(np.sum((clApprox - cl2)**2, axis=2)), axis=1)
#plt.figure()
#yN = fitfun(xD, *popt)
#plt.scatter(yD, yD-xD*1.5, alpha=0.05, marker='.')
#plt.scatter(yD, yD-xD, alpha=0.05, marker='.')
#plt.show()
#plt.plot(mse[test])
#plt.plot(pc3[test])
#plt.show()
msePosture = []
r2s = []
dataAll = []
for key in ['AML32_moving', 'AML70_chip']:
dset = data[key]['input']
res = data[key]['analysis']
for idn in dset.keys():
xdataL = res[idn]['ElasticNet']['Eigenworm3']['output']
ydataL = dset[idn]['Behavior']['Eigenworm3']
testL = res[idn]['Training']['Eigenworm3']['Test']
trainL = res[idn]['Training']['Eigenworm3']['Train']
indicesL = np.argsort(xdataL[trainL])
xdataS = xdataL[indicesL]
ydataS = ydataL[indicesL]
# correct with fitted nonlinearit
tNew = fitfun(xdataL, *popt)
#use joined fit fun
#r2s.append([r2_score( ydataL[testL],xdataL[testL]), r2_score(ydataL[testL],tNew[testL])])
#r2s.append([r2_score( ydataL[testL],xdataL[testL]), r2_score(ydataL[testL],tNew[testL])])
r2s.append([
pearsonr(ydataL[testL], xdataL[testL])[0]**2,
pearsonr(ydataL[testL], tNew[testL])[0]**2
])
# use mse - reconstruct and calculate mse for each worm
clTmp = dset[idn]['CL']
avPTmp = res[idn]['ElasticNet']['AngleVelocity']['output']
avTmp = dset[idn]['Behavior']['AngleVelocity']
thetaTmp = dset[idn]['Behavior']['Theta']
_, clApproxTmp, clPredTmp, cl3Tmp = compareReconstructedWorms(
clTmp, eigenworms, avTmp, thetaTmp, ydataL, avPTmp, xdataL,
tNew)
mseLTmp = np.mean(np.sqrt(
np.sum((clPredTmp - clApproxTmp)**2, axis=2)),
axis=1)
mseNLTmp = np.mean(np.sqrt(
np.sum((cl3Tmp - clApproxTmp)**2, axis=2)),
axis=1)
print 'MSE_L:', np.mean(mseLTmp[testL])
print 'MSE_NL:', np.mean(mseNLTmp[testL])
#msePosture.append([np.mean(mseLTmp[testL]), np.mean(mseNLTmp[testL])])
turns = np.where(np.abs(ydataL[testL]) > 10)
msePosture.append([
np.mean(mseLTmp[testL][turns]),
np.mean(mseNLTmp[testL][turns])
])
#cl3 = dh.calculateCLfromEW(pcsP, eigenworms, meanAngle, lengths, refPoint)
#r2s.append([mean_squared_error( ydataL[testL],xdataL[testL]), mean_squared_error(ydataL[testL],fitfun(xdataL[testL], *popt))])
################################################
#
#first row - posture prediction
#
################################################
# find samples that are starting with a reset
#loc1, loc2 = 1096, 196
k = 1
loc1, loc2 = 1086, 6
#loc1, loc2 = 1096, 976
offsetReset = 50
resets = np.where((test - offsetReset) % 60 == 0)[0]
#print t[np.where((test-offsetReset)%60==0)[0]]
print resets
# plt.figure()
# offsetx = 450
# for li, l in enumerate(resets):
# if np.mean(mse[test][l:l+30]) <50:
# print l, np.mean(mse[test][l:l+30]), np.mean(mse2[test][l:l+30])
# samplePost =test[l:l+66:6]
#
# for cindex, cline in enumerate(cl2[samplePost]):
# x,y = cline[:,0], cline[:,1]
# x -=np.mean(x)-offsetx*cindex
# y -=np.mean(y)+li*offsetx
# plt.plot(x, y, 'r')
# for cindex, cline in enumerate(clPred[samplePost]):
# x,y = cline[:,0], cline[:,1]
# x -=np.mean(x)-offsetx*cindex
# y -=np.mean(y)+li*offsetx
# plt.plot(x, y, 'k')
# plt.text(0, li*offsetx, li)
# plt.show()
#ax.plot(x, y)
# plot predicted behaviors and location of postures
#ybeh = [10, 15]
##print -time[samplePost[0]]+time[samplePost[-1]]
#for behavior, color, cpred, yl, label, align in zip(['AngleVelocity','Eigenworm3' ], \
# [N1, N1], [R1, B1], ybeh, ['Velocity', 'Turn'], ['center', 'center']):
# beh = moving['Behavior'][behavior][test]
#
# meanb, maxb = np.mean(beh),np.std(beh)
# beh = (beh-meanb)/maxb
#
# behPred = movingAnalysis[flag][behavior]['output'][test]
# behPred = (behPred-meanb)/maxb
#
# #ax3.plot(t, beh+yl, color=color)
# ax3.plot(t, behPred+yl, color=cpred)
# #ax1.text(t[test[-1]], yl, \
# #r'$R^2 = {:.2f}$'.format(np.float(movingAnalysis[flag][behavior]['scorepredicted'])), horizontalalignment = 'right')
# ax3.text(t[0]*0.6, yl, label, rotation=90, color=cpred, verticalalignment=align)
## add scalebar
#l =120
#y = ybeh[0]*0.7
#ax3.plot([t[0], t[0]+l],[y, y], 'k', lw=2)
#ax3.text(t[0]+l*0.5,y*0.5, '2 min', horizontalalignment='center')
#
#ax3.spines['left'].set_visible(False)
#ax3.spines['bottom'].set_visible(False)
#ax3.set_yticks([])
#ax3.set_xticks([])
testloc1, testloc2 = test[loc1:loc1 + 66:6], test[loc2:loc2 + 66:6]
for ax, samplePost, color, timestamp in zip(
[ax1, ax2], [testloc2, testloc1], [N0, N0],
[time[test[loc2]], time[test[loc1]]]):
#=============================================================================#
# calculate mse
#=============================================================================#
# plt.figure()
# mse = np.mean(np.sum((clPred - clApprox)**2, axis=2), axis=1)
# #plt.plot(mse)
# plt.plot(np.abs(beh)*np.max(mse)/10)
# print np.where(mse<1)
# plt.show()
#=============================================================================#
# create finite-width worms
#=============================================================================#
# offsets to create aspect ratio and spacing in equal proportion plots
ax.tick_params(axis='both', which='both', length=0)
offsetx = 450
offsety = 450
# original worms
patches1 = []
for cindex, cline in enumerate(cl2[samplePost]):
x, y = cline[:, 0], cline[:, 1]
x -= np.mean(x) - offsetx * cindex
y -= np.mean(y)
vertices = createWorm(x, y)
patches1.append(mpl.patches.Polygon(vertices, closed=True))
#ax.plot(x, y)
p = mpl.collections.PatchCollection(patches1,
alpha=1,
color=color,
edgecolor='none')
ax.add_collection(p)
#p = mpl.collections.PatchCollection(patches1, alpha=1, color=color, edgecolor='none', offsets=(0,offsety), transOffset=ax.transData)
#ax.add_collection(p)
# # 3 eigenworm approximate worms
# patches = []
# for cindex, cline in enumerate(clApprox[samplePost]):
# x,y = cline[:,0], cline[:,1]
# x -=np.mean(x)-offsetx*cindex
# y -=np.mean(y)+offsety
# vertices = createWorm(x,y)
# patches.append(mpl.patches.Polygon(vertices, closed=True))
# #ax.plot(x, y)
# p = mpl.collections.PatchCollection(patches, alpha=1, color=N0, edgecolor='none')
# ax.add_collection(p)
# predicted worms
patches = []
for cindex, cline in enumerate(clPred[samplePost]):
x, y = cline[:, 0], cline[:, 1]
x -= np.mean(x) - offsetx * cindex
y -= np.mean(y) + 1.5 * offsety
vertices = createWorm(x, y)
patches.append(mpl.patches.Polygon(vertices, closed=True))
#ax.plot(x, y)
p = mpl.collections.PatchCollection(patches,
alpha=1,
color=L1,
edgecolor='none')
ax.add_collection(p)
# # 3 eigenworm approximate worms
# patches = []
# for cindex, cline in enumerate(clApprox[samplePost]):
# x,y = cline[:,0], cline[:,1]
# x -=np.mean(x)-offsetx*cindex
# y -=np.mean(y)+offsety
# vertices = createWorm(x,y)
# patches.append(mpl.patches.Polygon(vertices, closed=True))
# #ax.plot(x, y)
# p = mpl.collections.PatchCollection(patches, alpha=1, color=N1, edgecolor='none')
# ax.add_collection(p)
# predicted worms - with nonlinearity
patches = []
for cindex, cline in enumerate(cl3[samplePost]):
x, y = cline[:, 0], cline[:, 1]
x -= np.mean(x) - offsetx * cindex
y -= np.mean(y) + 2.5 * offsety
vertices = createWorm(x, y)
patches.append(mpl.patches.Polygon(vertices, closed=True))
#ax.plot(x, y)
p = mpl.collections.PatchCollection(patches,
alpha=1,
color=L3,
edgecolor='none')
ax.add_collection(p)
# add original to all lines
for off in [1.5 * offsety, 2.5 * offsety]:
patches1 = []
for cindex, cline in enumerate(cl2[samplePost]):
x, y = cline[:, 0], cline[:, 1]
x -= np.mean(x) - offsetx * cindex
y -= np.mean(y) + off
vertices = createWorm(x, y)
patches1.append(mpl.patches.Polygon(vertices, closed=True))
#ax.plot(x, y)
p = mpl.collections.PatchCollection(patches1,
alpha=1,
color=color,
edgecolor='none',
zorder=-1)
ax.add_collection(p)
# add horizontal line and text
#ax.axhline(-2*offsetx,color = 'k', linestyle =':')
ax.text(0.5,
0.65,
'predicted from neural activity',
transform=ax.transAxes,
horizontalalignment='center')
ax.text(0.5,
0.95,
'measured postures (t = {} s)'.format(int(timestamp)),
transform=ax.transAxes,
horizontalalignment='center')
# general plot style
ax.set_xticks(np.arange(0, len(samplePost + 1) * offsetx, 2 * offsetx))
ax.set_xticklabels(
['{} s'.format(i) for i in range(0,
len(samplePost) + 1, 2)])
ax.set_xlim([-300, 4700])
ax.set_ylim([-2.5 * offsety - 350, 350])
ax.set_yticks([])
ax.spines['left'].set_visible(False)
ax.spines['bottom'].set_visible(False)
# just left plot
ax1.set_yticks([0, -1.5 * offsety, -2.5 * offsety])
ax1.set_yticklabels(['original', 'SLM', 'SLM+NL'])
ax2.set_yticks([0, -1.5 * offsety, -2.5 * offsety])
ax2.set_yticklabels(['original', 'SLM', 'SLM+NL'])
# move to right
#moveAxes(ax1, 'right', -0.07)
#moveAxes(ax2, 'right', -0.07)
# remove xticks from first axes
cleanAxes(ax1, 'x')
ax2.spines['bottom'].set_visible(True)
################################################
#
#third row - non-linearity
#
################################################
#ax4 = plt.subplot(gsNL[0,1])
# show non-linearity for one neuron
#ax4.scatter(xdata, ydata, color='k', alpha=0.01)
#ax4.plot(xdata, tPnew[indices], color=R1, linestyle='--', label='fit')
#ax4.set_xlabel('Predicted turn')
#ax4.set_ylabel('True turn', labelpad=-10)
#moveAxes(ax4, 'right', 0.02)
ax6.axvspan(time[test[loc1]],
time[test[loc1 + 60]],
color=N2,
zorder=-10,
alpha=1,
ymax=0.75,
edgecolor='None')
ax6.axvspan(time[test[loc2]],
time[test[loc2 + 60]],
color='k',
zorder=-10,
alpha=1,
ymax=0.75,
fill=False,
edgecolor='k',
hatch='\\\\\\\\')
# compare to linear
#ax6.plot(time[test], pc3[test], color=N1, label='true', zorder=-1, linestyle=':')
ax6.plot(time[test],
tP[test],
color=L1,
label='linear',
linestyle='--',
zorder=20)
#ax7.plot(time[test], tP[test], color=R1, label='linear', zorder=10)
# compare to nonlinear
ax6.plot(time[test],
pc3[test],
color=N0,
label='true',
zorder=-1,
linestyle=':')
ax6.plot(time[test],
tPnew[test],
color=L3,
linestyle='-',
label='non-linear')
r2orig = r2_score(pc3[test], tP[test])
r2nl = r2_score(pc3[test], tPnew[test])
#r2orig = pearsonr(pc3[test], tP[test])[0]**2
#r2nl = pearsonr(pc3[test], tPnew[test])[0]**2
ax6.text(1,
1.1,
r'$r_L^2 = {:.2f}$'.format(r2orig),
horizontalalignment='right',
transform=ax6.transAxes)
#ax6.text(-0.1,0.5,r'Turn', horizontalalignment = 'center', transform = ax6.transAxes, rotation =90)
ax6.text(1,
1,
r'$r_{{NL}}^2 = {:.2f}$'.format(r2nl),
horizontalalignment='right',
verticalalignment='top',
transform=ax6.transAxes)
#ax6.legend(loc=8, bbox_to_anchor=(-0.1, 0.5))
#ax7.spines['left'].set_visible(False)
#ax6.spines['left'].set_visible(False)
#ax7.set_yticks([])
#ax6.set_yticks([])
#ax6.plot([time[test][0], time[test][0]], [-15,-5], color=B1, lw=2)
#ax6.text(time[test][0]*0.99, -10,'10', color=B1, verticalalignment='center', horizontalalignment='right')
ax6.set_ylabel('Body curvature')
ax6.set_xlabel('Time (s)')
ax6.set_ylim([-20, 20])
#ax7.set_xlabel('Time (s)')
# move
#moveAxes(ax6, 'right', 0.015)
#moveAxes(ax7, 'left', 0.025)
fitx = np.arange(-15, 15, 0.1)
ax5.plot(fitx, fitfun(fitx, *popt), color='r', alpha=0.75)
ax5.scatter(xD, yD, s=3, c='k', alpha=0.01)
ax5.set_xlabel('Predicted \n body curvature')
ax5.set_ylabel('Actual \n body curvature')
#mkStyledBoxplot(ax8, [0,1], np.reshape(fitdata, (1,-1)),[L1], ['A'])
#mkStyledBoxplot(ax8, [0,1], np.array(r2s).T,[L1, L3], ['L', 'NL'])
#print 'R2_scores', np.mean(r2s, axis=0)
#ax8.set_xlim([-0.5,1.5])
#ax8.set_ylabel(r'$r^2$')
mkStyledBoxplot(ax9, [0, 1],
np.array(msePosture).T, [L1, L3], ['SLM', 'SLM+NL'],
rotate=False)
print 'MSE_scores', np.mean(msePosture, axis=0)
ax9.set_xlim([-0.5, 1.5])
ax9.set_ylabel(r'RMSE (BC > 10)')
plt.show()