def separate_channels(filename):
filename_red = filename[:-4]+'_red.hdf5'
filename_green = filename[:-4]+'_green.hdf5'
frameRate=30
# load movie
# for loading only a portion of the movie or only some channels
# you can use the option: subindices=range(0,1500,10)
m = cb.load(filename, fr=frameRate, subindices=slice(1,None,2))
print m.shape
m.save(filename_red)
m = cb.load(filename, fr=frameRate, subindices=slice(0,None,2))
m.save(filename_green)
return((filename_green,filename_red))
def __init__(self, *args, **kwargs):
wx.Window.__init__(self, *args, **kwargs)
self.lines = []
self.imgs = []
self.bkgrnd = []
self.mean = []
self.figure = Figure()
self.canvas = FigureCanvasWxAgg(self, -1, self.figure)
self.canvas.callbacks.connect('button_press_event', self.mouseDown)
self.canvas.callbacks.connect('motion_notify_event', self.mouseMotion)
self.canvas.callbacks.connect('button_release_event', self.mouseUp)
self.state = ''
self.mov=cb.load('movies/demoMovie.tif',fr=20)
with np.load('results_analysis.npz') as ld:
self.comps=np.transpose(np.reshape(ld['A2'],(ld['d1'],ld['d2'],-1),order='F'),[2,0,1])
self.mean = np.mean(self.mov,0)
self.mouseInfo = (None, None, None, None)
self.f0 = Param(np.percentile(self.mean,95), minimum=np.min(self.mean), maximum=np.max(self.mean))
self.A = Param(1, minimum=0, maximum=self.mov.shape[0],step=1)
self.toggle_background = BoolParam(True)
self.draw()
# Not sure I like having two params attached to the same Knob,
# but that is what we have here... it works but feels kludgy -
# although maybe it's not too bad since the knob changes both params
# at the same time (both f0 and A are affected during a drag)
self.f0.attach(self)
self.A.attach(self)
self.toggle_background.attach(self)
self.Bind(wx.EVT_SIZE, self.sizeHandler)
def place_holder(fl):
import calblitz as cb
import ca_source_extraction as cse
import numpy as np
m=cb.load(fl)
Cn = m.local_correlations()
cnmf=cse.CNMF(1, k=4,gSig=[8,8],merge_thresh=0.8,p=2,dview=None,Ain=None)
cnmf=cnmf.fit(m)
A,C,b,f,YrA=cnmf.A,cnmf.C,cnmf.b,cnmf.f,cnmf.YrA
np.savez(fl[:-5]+'_result.npz',A=A,C=C,b=b,f=f,YrA=YrA,Cn=Cn)
return fl[:-5]+'_result.npz'
def create_images_for_labeling(pars):
import scipy.stats as st
import os
import numpy as np
import calblitz as cb
from glob import glob
try:
f_name = pars
cdir = os.path.dirname(f_name)
print('loading')
m = cb.load(f_name)
print('corr image')
img = m.local_correlations(eight_neighbours=True)
im = cb.movie(img, fr=1)
im.save(os.path.join(cdir, 'correlation_image.tif'))
print('std image')
img = np.std(m, 0)
im = cb.movie(np.array(img), fr=1)
im.save(os.path.join(cdir, 'std_projection.tif'))
m1 = m.resize(1, 1, old_div(1., m.fr))
print('median image')
img = np.median(m1, 0)
im = cb.movie(np.array(img), fr=1)
im.save(os.path.join(cdir, 'median_projection.tif'))
print('save BL')
m1 = m1 - img
m1.save(os.path.join(cdir, 'MOV_BL.tif'))
m1 = m1.bilateral_blur_2D()
m1.save(os.path.join(cdir, 'MOV_BL_BIL.tif'))
m = np.array(m1)
print('max image')
img = np.max(m, 0)
im = cb.movie(np.array(img), fr=1)
im.save(os.path.join(cdir, 'max_projection.tif'))
print('skew image')
img = st.skew(m, 0)
im = cb.movie(img, fr=1)
im.save(os.path.join(cdir, 'skew_projection.tif'))
del m
del m1
except Exception as e:
return e
return f_name
def create_images_for_labeling(pars):
import scipy.stats as st
import os
import numpy as np
import calblitz as cb
from glob import glob
try:
f_name = pars
cdir = os.path.dirname(f_name)
print('loading')
m = cb.load(f_name)
print('corr image')
img = m.local_correlations(eight_neighbours=True)
im = cb.movie(img, fr=1)
im.save(os.path.join(cdir, 'correlation_image.tif'))
print('std image')
img = np.std(m, 0)
im = cb.movie(np.array(img), fr=1)
im.save(os.path.join(cdir, 'std_projection.tif'))
m1 = m.resize(1, 1, old_div(1., m.fr))
print('median image')
img = np.median(m1, 0)
im = cb.movie(np.array(img), fr=1)
im.save(os.path.join(cdir, 'median_projection.tif'))
print('save BL')
m1 = m1 - img
m1.save(os.path.join(cdir, 'MOV_BL.tif'))
m1 = m1.bilateral_blur_2D()
m1.save(os.path.join(cdir, 'MOV_BL_BIL.tif'))
m = np.array(m1)
print('max image')
img = np.max(m, 0)
im = cb.movie(np.array(img), fr=1)
im.save(os.path.join(cdir, 'max_projection.tif'))
print('skew image')
img = st.skew(m, 0)
im = cb.movie(img, fr=1)
im.save(os.path.join(cdir, 'skew_projection.tif'))
del m
del m1
except Exception as e:
return e
return f_name
def place_holder(prs):
fl,diam=prs
import calblitz as cb
import ca_source_extraction as cse
import numpy as np
m=cb.load(fl)
Cn = m.local_correlations()
cnmf=cse.CNMF(1, k=6,gSig=[diam/2,diam/2],merge_thresh=0.8,p=2,dview=None,Ain=None)
cnmf=cnmf.fit(m)
A,C,b,f,YrA=cnmf.A,cnmf.C,cnmf.b,cnmf.f,cnmf.YrA
np.savez(fl[:-5]+'_result.npz',A=A,C=C,b=b,f=f,YrA=YrA,Cn=Cn)
return fl[:-5]+'_result.npz'
def process_movie_parallel(arg_in):
# import calblitz
# import calblitz.movies
import ca_source_extraction as cse
import calblitz as cb
import numpy as np
import sys
fname,fr,margins_out,template,max_shift_w, max_shift_h,remove_blanks,apply_smooth=arg_in
with open(fname[:-4]+'.stout', "a") as log:
# sys.stdout = log
# import pdb
# pdb.set_trace()
Yr=cb.load(fname,fr=fr)
print 'loaded'
if apply_smooth:
print 'applying smoothing'
Yr=Yr.bilateral_blur_2D(diameter=10,sigmaColor=10000,sigmaSpace=0)
Yr=Yr-np.float32(np.percentile(Yr,1)) # needed to remove baseline
print 'Remove BL'
if margins_out!=0:
Yr=Yr[:,margins_out:-margins_out,margins_out:-margins_out] # borders create troubles
print 'motion correcting'
Yr,shifts,xcorrs,template=Yr.motion_correct(max_shift_w=max_shift_w, max_shift_h=max_shift_h, method='opencv',template=template,remove_blanks=remove_blanks)
print 'median computing'
template=Yr.bin_median()
print 'saving'
Yr.save(fname[:-3]+'hdf5')
print 'saving 2'
np.savez(fname[:-3]+'npz',shifts=shifts,xcorrs=xcorrs,template=template)
print 'deleting'
del Yr
print 'done!'
#sys.stdout = sys.__stdout__
return fname[:-3]
def pre_process_handle(args):
# import calblitz as cb
from scipy.ndimage import filters as ft
import logging
fil, resize_factors, diameter_bilateral_blur, median_filter_size = args
name_log = fil[:-4] + '_LOG'
logger = logging.getLogger(name_log)
hdlr = logging.FileHandler(name_log)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.INFO)
logger.info('START')
logger.info(fil)
mov = cb.load(fil, fr=30)
logger.info('Read file')
mov = mov.resize(1, 1, resize_factors[0])
logger.info('Resize')
mov = mov.bilateral_blur_2D(diameter=diameter_bilateral_blur)
logger.info('Bilateral')
mov1 = cb.movie(ft.median_filter(mov, median_filter_size), fr=30)
logger.info('Median filter')
#mov1=mov1-np.median(mov1,0)
mov1 = mov1.resize(1, 1, resize_factors[1])
logger.info('Resize 2')
mov1 = mov1 - cb.utils.mode_robust(mov1, 0)
logger.info('Mode')
mov = mov.resize(1, 1, resize_factors[1])
logger.info('Resize')
# mov=mov-np.percentile(mov,1)
mov.save(fil[:-4] + '_compress_.tif')
logger.info('Save 1')
mov1.save(fil[:-4] + '_BL_compress_.tif')
logger.info('Save 2')
return 1
def pre_process_handle(args):
# import calblitz as cb
from scipy.ndimage import filters as ft
import logging
fil, resize_factors, diameter_bilateral_blur,median_filter_size=args
name_log=fil[:-4]+ '_LOG'
logger = logging.getLogger(name_log)
hdlr = logging.FileHandler(name_log)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.INFO)
logger.info('START')
logger.info(fil)
mov=cb.load(fil,fr=30)
logger.info('Read file')
mov=mov.resize(1,1,resize_factors[0])
logger.info('Resize')
mov=mov.bilateral_blur_2D(diameter=diameter_bilateral_blur)
logger.info('Bilateral')
mov1=cb.movie(ft.median_filter(mov,median_filter_size),fr=30)
logger.info('Median filter')
#mov1=mov1-np.median(mov1,0)
mov1=mov1.resize(1,1,resize_factors[1])
logger.info('Resize 2')
mov1=mov1-cb.utils.mode_robust(mov1,0)
logger.info('Mode')
mov=mov.resize(1,1,resize_factors[1])
logger.info('Resize')
# mov=mov-np.percentile(mov,1)
mov.save(fil[:-4] + '_compress_.tif')
logger.info('Save 1')
mov1.save(fil[:-4] + '_BL_compress_.tif')
logger.info('Save 2')
return 1
def __init__(self, *args, **kwargs):
wx.Window.__init__(self, *args, **kwargs)
self.lines = []
self.imgs = []
self.bkgrnd = []
self.mean = []
self.figure = Figure()
self.canvas = FigureCanvasWxAgg(self, -1, self.figure)
self.canvas.callbacks.connect('button_press_event', self.mouseDown)
self.canvas.callbacks.connect('motion_notify_event', self.mouseMotion)
self.canvas.callbacks.connect('button_release_event', self.mouseUp)
self.state = ''
self.mov = cb.load('movies/demoMovie.tif', fr=20)
with np.load('results_analysis.npz') as ld:
self.comps = np.transpose(
np.reshape(ld['A2'], (ld['d1'], ld['d2'], -1), order='F'),
[2, 0, 1])
self.mean = np.mean(self.mov, 0)
self.mouseInfo = (None, None, None, None)
self.f0 = Param(np.percentile(self.mean, 95),
minimum=np.min(self.mean),
maximum=np.max(self.mean))
self.A = Param(1, minimum=0, maximum=self.mov.shape[0], step=1)
self.toggle_background = BoolParam(True)
self.draw()
# Not sure I like having two params attached to the same Knob,
# but that is what we have here... it works but feels kludgy -
# although maybe it's not too bad since the knob changes both params
# at the same time (both f0 and A are affected during a drag)
self.f0.attach(self)
self.A.attach(self)
self.toggle_background.attach(self)
self.Bind(wx.EVT_SIZE, self.sizeHandler)
all_movs.save(base_folder + 'avg_movies.tif')
template = np.median(all_movs[:], axis=0)
np.save(base_folder + 'template_total', template)
pl.imshow(template, cmap=pl.cm.gray, vmax=100)
#%%
all_movs.play(backend='opencv', gain=10, fr=5)
#%%
big_mov = []
big_shifts = []
fr_remove_init = 30
for f in fnames:
with np.load(f[:-3] + 'npz') as fl:
big_shifts.append(fl['shifts'])
print(f)
Yr = cb.load(f[:-3] + 'hdf5')[fr_remove_init:]
Yr = Yr.resize(fx=1, fy=1, fz=.2)
Yr = np.transpose(Yr, (1, 2, 0))
d1, d2, T = Yr.shape
Yr = np.reshape(Yr, (d1 * d2, T), order='F')
print((Yr.shape))
# np.save(fname[:-3]+'npy',np.asarray(Yr))
big_mov.append(np.asarray(Yr))
#%%
big_mov = np.concatenate(big_mov, axis=-1)
big_shifts = np.concatenate(big_shifts, axis=0)
#%%
np.save('Yr_DS.npy', big_mov)
np.save('big_shifts.npy', big_shifts)
#%%
@author: agiovann
"""
from glob import glob
import scipy.stats as st
import calblitz as cb
import numpy as np
#%%
for fl in glob('k36*compress_.tif'):
print(fl)
m = cb.load(fl, fr=3)
img = m.local_correlations(eight_neighbours=True)
im = cb.movie(img, fr=1)
im.save(fl[:-4] + 'correlation_image.tif')
m = np.array(m)
img = st.skew(m, 0)
im = cb.movie(img, fr=1)
im.save(fl[:-4] + 'skew.tif')
img = st.kurtosis(m, 0)
im = cb.movie(img, fr=1)
im.save(fl[:-4] + 'kurtosis.tif')
pl.ion()
%load_ext autoreload
%autoreload 2
except:
print "Probably not a Ipython interactive environment"
#%% define movie
filename='movies/demoMovie_PC.tif'
frameRate=15.62;
start_time=0;
#%%
filename_py=filename[:-4]+'.npz'
filename_hdf5=filename[:-4]+'.hdf5'
filename_mc=filename[:-4]+'_mc.npz'
#%% load movie
m=cb.load(filename, fr=frameRate,start_time=start_time);
#%% load portion of the movie or only some channels
#m=cb.load(filename, fr=frameRate,start_time=start_time,subindices=range(0,1500,10));
#%% save movie hdf5 format. fastest
m.save(filename_hdf5)
#%% backend='opencv' is much faster
m.play(fr=100,gain=15.0,magnification=1,backend='pylab')
#%%
low_SNR=True
if low_SNR:
N=1000000
mn1=m.copy().bilateral_blur_2D(diameter=5,sigmaColor=10000,sigmaSpace=0)
mn1,shifts,xcorrs, template=mn1.motion_correct()
mn2=mn1.apply_shifts(shifts)
#mn1=cb.movie(np.transpose(np.array(Y_n),[2,0,1]),fr=30)
pl.ion()
%load_ext autoreload
%autoreload 2
except:
print "Probably not a Ipython interactive environment"
#%%
tif_files=glob.glob('*.tif')
num_frames_median=1000;
movs=[];
fr=30;
start_time=0;
templates=[];
for tif_file in tif_files:
print(tif_file)
m=cb.load(tif_file,fr=30,start_time=0,subindices=range(0,1500,20))
min_val_add=np.percentile(m,.01)
m=m-min_val_add
movs.append(m)
templ=np.nanmedian(m,axis=0);
m,template,shifts,xcorrs=m.motion_correct(max_shift_w=5, max_shift_h=5, show_movie=False, template=templ, method='opencv')
templates.append(np.median(m,axis=0))
all_movs=cb.concatenate(movs)
m=cb.movie(np.array(templates),fr=1)
m=m.motion_correct(template=m[0])[0]
template=np.median(m,axis=0)
cb.matrixMontage(m,cmap=pl.cm.gray,vmin=0,vmax=1000)
#%%
all_shifts=[];
def process_movie_parallel(arg_in):
# import calblitz
# import calblitz.movies
import calblitz as cb
import numpy as np
import sys
fname,fr,margins_out,template,max_shift_w, max_shift_h,remove_blanks,apply_smooth,save_hdf5=arg_in
if template is not None:
if type(template) is str:
if os.path.exists(template):
template=cb.load(template,fr=1)
else:
raise Exception('Path to template does not exist:'+template)
# with open(fname[:-4]+'.stout', "a") as log:
# print fname
# sys.stdout = log
# import pdb
# pdb.set_trace()
if type(fname) is cb.movie or type(fname) is cb.movies.movie:
print type(fname)
Yr=fname
else:
Yr=cb.load(fname,fr=fr)
if Yr.ndim>1:
print 'loaded'
if apply_smooth:
print 'applying smoothing'
Yr=Yr.bilateral_blur_2D(diameter=10,sigmaColor=10000,sigmaSpace=0)
# bl_yr=np.float32(np.percentile(Yr,8))
# Yr=Yr-bl_yr # needed to remove baseline
print 'Remove BL'
if margins_out!=0:
Yr=Yr[:,margins_out:-margins_out,margins_out:-margins_out] # borders create troubles
print 'motion correcting'
Yr,shifts,xcorrs,template=Yr.motion_correct(max_shift_w=max_shift_w, max_shift_h=max_shift_h, method='opencv',template=template,remove_blanks=remove_blanks)
# Yr = Yr + bl_yr
if type(fname) is cb.movie:
return Yr
else:
print 'median computing'
template=Yr.bin_median()
print 'saving'
idx_dot=len(fname.split('.')[-1])
if save_hdf5:
Yr.save(fname[:-idx_dot]+'hdf5')
print 'saving 2'
np.savez(fname[:-idx_dot]+'npz',shifts=shifts,xcorrs=xcorrs,template=template)
print 'deleting'
del Yr
print 'done!'
return fname[:-idx_dot]
#sys.stdout = sys.__stdout__
else:
return None
all_movs.save(base_folder+'avg_movies.tif')
template=np.median(all_movs[:],axis=0)
np.save(base_folder+'template_total',template)
pl.imshow(template,cmap=pl.cm.gray,vmax=100)
#%%
all_movs.play(backend='opencv',gain=10,fr=5)
#%%
big_mov=[];
big_shifts=[]
fr_remove_init=30
for f in fnames:
with np.load(f[:-3]+'npz') as fl:
big_shifts.append(fl['shifts'])
print f
Yr=cb.load(f[:-3]+'hdf5')[fr_remove_init:]
Yr=Yr.resize(fx=1,fy=1,fz=.2)
Yr = np.transpose(Yr,(1,2,0))
d1,d2,T=Yr.shape
Yr=np.reshape(Yr,(d1*d2,T),order='F')
print Yr.shape
# np.save(fname[:-3]+'npy',np.asarray(Yr))
big_mov.append(np.asarray(Yr))
#%%
big_mov=np.concatenate(big_mov,axis=-1)
big_shifts=np.concatenate(big_shifts,axis=0)
#%%
np.save('Yr_DS.npy',big_mov)
np.save('big_shifts.npy',big_shifts)
#%%
pl.subplot(1,2,2)
pl.plot(fl['shifts'])
pl.pause(0.001)
pl.cla()
#%%
import shutil
names_new_each=[]
for mov_names_each in movie_names:
movie_names_hdf5=[]
for mov_name in mov_names_each:
movie_names_hdf5.append(mov_name[:-3]+'hdf5')
#idx_x=slice(12,500,None)
#idx_y=slice(12,500,None)
#idx_xy=(idx_x,idx_y)
idx_xy=None
8 name_new.sort()
names_new_each.append(name_new)
print name_new
#%%
fnames_new_each=dview.map_sync(cse.utilities.save_memmap_join,names_new_each)
#%%
#for name_new in names_new_each:
# fnames_new_each.append(cse.utilities.save_memmap_join(name_new, n_chunks=2, dview=None))
#%%
m=cb.load(fnames_new_each[-1],fr=5)
m.play(backend='opencv',gain=2.,fr=30)
#%%
def generate_linked_traces(mov_names, chunk_sizes, A, b, f):
"""
Generate traces (DFF,BL and DF) for a group of movies that share the same A,b and f,
by applying the same transformation over a set of movies. This removes
the contamination of neuropil and then masks the components.
Parameters:
-----------
mov_names: list of path to movies associated with the same A,b,and f
chunk_sizes:list containing the number of frames in each movie
A,b and f: from CNMF
Returns:
--------
"""
num_chunks = np.sum(chunk_sizes)
# A = A_s[idx][:,neurons[idx]]
nA = (A.power(2)).sum(0)
# bckg=cb.movie(cb.to_3D(b.dot(f).T,(-1,shape[0],shape[1])),fr=1)
f = np.array(f).squeeze()
# bckg=bckg.resize(1,1,1.*num_chunks/b_size)
b_size = f.shape[0]
# if num_chunks != b_size:
# raise Exception('The number of frames are not matching')
#
counter = 0
f_in = np.atleast_2d(scipy.signal.resample(f, num_chunks))
# C,f,S,bl,c1,neurons_sn,g,YrA = cse.temporal.update_temporal_components(Yr,A,b,C_in,f_in,p=0)
traces = []
traces_BL = []
traces_DFF = []
for jj, mv in enumerate(mov_names):
mov_chunk_name = os.path.splitext(os.path.split(mv)[-1])[0] + '.hdf5'
mov_chunk_name = os.path.join(os.path.dirname(mv), mov_chunk_name)
print mov_chunk_name
m = cb.load(mov_chunk_name).to_2D().T
bckg_1 = b.dot(f_in[:, counter:counter + chunk_sizes[jj]])
m = m - bckg_1
# (m).play(backend='opencv',gain=10.,fr=33)
# m=np.reshape(m,(-1,np.prod(shape)),order='F').T
# bckg_1=np.reshape(bckg_1,(-1,np.prod(shape)),order='F').T
counter += chunk_sizes[jj]
Y_r_sig = A.T.dot(m)
Y_r_sig = scipy.sparse.linalg.spsolve(
scipy.sparse.spdiags(np.sqrt(nA), 0, nA.size, nA.size), Y_r_sig)
traces.append(Y_r_sig)
Y_r_bl = A.T.dot(bckg_1)
Y_r_bl = scipy.sparse.linalg.spsolve(
scipy.sparse.spdiags(np.sqrt(nA), 0, nA.size, nA.size), Y_r_bl)
traces_BL.append(Y_r_bl)
Y_r_bl = cse.utilities.mode_robust(Y_r_bl, 1)
traces_DFF.append(Y_r_sig / Y_r_bl[:, np.newaxis])
return traces, traces_DFF, traces_BL
def save_memmap(filenames,base_name='Yr',resize_fact=(1,1,1),remove_init=0,idx_xy=None):
""" Saves efficiently a list of tif files into a memory mappable file
Parameters
----------
filenames: list
list of tif files
base_name: str
the base yused to build the file name. IT MUST NOT CONTAIN "_"
resize_fact: tuple
x,y, and z downampling factors (0.5 means downsampled by a factor 2)
remove_init: int
number iof frames to remove at the begining of each tif file (used for resonant scanning images if laser in rutned on trial by trial)
idx_xy: tuple size 2
for selecting slices of the original FOV, for instance idx_xy=(slice(150,350,None),slice(150,350,None))
Return
-------
fname_new: the name of the mapped file, the format is such that the name will contain the frame dimensions and the number of f
"""
order='F'
Ttot=0;
for idx,f in enumerate(filenames):
print f
if os.path.splitext(f)[-1] == '.hdf5':
import calblitz as cb
if idx_xy is None:
Yr=np.array(cb.load(f))[remove_init:]
else:
Yr=np.array(cb.load(f))[remove_init:,idx_xy[0],idx_xy[1]]
else:
if idx_xy is None:
Yr=imread(f)[remove_init:]
else:
Yr=imread(f)[remove_init:,idx_xy[0],idx_xy[1]]
fx,fy,fz=resize_fact
if fx!=1 or fy!=1 or fz!=1:
try:
import calblitz as cb
Yr=cb.movie(Yr,fr=1)
Yr=Yr.resize(fx=fx,fy=fy,fz=fz)
except:
print('You need to install the CalBlitz package to resize the movie')
raise
[T,d1,d2]=Yr.shape;
Yr=np.transpose(Yr,(1,2,0))
Yr=np.reshape(Yr,(d1*d2,T),order=order)
if idx==0:
fname_tot=base_name+'_d1_'+str(d1)+'_d2_'+str(d2)+'_order_'+str(order)
big_mov=np.memmap(fname_tot,mode='w+',dtype=np.float32,shape=(d1*d2,T),order=order);
else:
big_mov=np.memmap(fname_tot,dtype=np.float32,mode='r+',shape=(d1*d2,Ttot+T),order=order)
# np.save(fname[:-3]+'npy',np.asarray(Yr))
big_mov[:,Ttot:Ttot+T]=np.asarray(Yr,dtype=np.float32)+1e-10
big_mov.flush()
Ttot=Ttot+T;
fname_new=fname_tot+'_frames_' +str(Ttot) + '_.mmap'
os.rename(fname_tot,fname_new)
return fname_new
import glob
import os
import scipy
from ipyparallel import Client
import calblitz as cb
#%% download example
import urllib.request, urllib.parse, urllib.error
import shutil
if not os.path.exists('PPC.tif'):
url = "https://www.dropbox.com/s/z8rj5dekrl6yxfr/PPC.tif?dl=1" # dl=1 is important
u = urllib.request.urlretrieve(url)
shutil.move(u[0],'./PPC.tif')
#%% load and motion correct movie
m=cb.load('PPC.tif',fr=30)
m -= np.min(m)
play_movie = False
if play_movie:
m.play(backend='opencv',magnification=2,fr=60,gain=5.)
#%%
max_w=25 # max pixel shift in width direction
max_h=25
mc,shifts,corrs, template = m.motion_correct(max_shift_w=max_w,max_shift_h=max_h,remove_blanks=True)
if play_movie:
mc.play(backend='opencv',magnification=2,fr=60,gain=5.)
#%%
mc.save('PPC_mc.tif')
fnames=['PPC_mc.tif']
import calblitz as cb
try:
pl.ion()
%load_ext autoreload
%autoreload 2
except:
print "Probably not a Ipython interactive environment"
#%% many movies
import glob
file_list=glob.glob('CHR L10 503 539 -82_2X_LOBV or VI_08*.tif')
for f in file_list:
print f
m=cb.load_movie_chain(file_list,fr=15.625,subindices=range(3,54,3))
#%% q
m=cb.load('M_FLUO.tif',fr=8);
#%%
m=cb.load('M_FLUO.tif',fr=8);
#
T,h,w=np.shape(m)
m1=m[:,:h/2,:]
m2=m[:,h/2:,:]
m=m2;
#%%
m,shifts,xcorrs,template=m.motion_correct(max_shift_w=5,max_shift_h=5, num_frames_template=None, template = None,method='opencv')
max_h,max_w=np.ceil(np.nanmax(shifts,axis=0))
min_h,min_w=np.floor(np.nanmin(shifts,axis=0))
m=m[:,-min_h:-max_h,-min_w:-max_w]
t = tifffile.TiffFile('movies/demoMovie.tif')
#t = libtiff.TiffFile('/Users/eftychios/Documents/_code/calcium_paper_code/datasets/clay/2014-04-05-003.tif')
Y = t.asarray()
Y = np.transpose(Y,(1,2,0))*1.
d1,d2,T=Y.shape
#%%
#m=cb.load('demoMovie.tif',fr=8);
#T,h,w=np.shape(m)
#Y=np.asarray(m)
#Y = np.transpose(Y,(1,2,0))
##Ymat = sio.loadmat('Y.mat')
##Y = Ymat['Y']*1.
#d1,d2,T = np.shape(Y)
#%%
m=cb.load('movies/demoMovie.tif',fr=8);
T,h,w=np.shape(m)
Y=np.asarray(m)
Y = np.transpose(Y,(1,2,0))
#Ymat = sio.loadmat('Y.mat')
#Y = Ymat['Y']*1.
d1,d2,T = np.shape(Y)
#%%
t = tifffile.TiffFile('movies/demoMovie.tif')
Y = t.asarray()
Y = np.transpose(Y,(1,2,0))*1.
d1,d2,T=Y.shape
a = sio.loadmat('movies/demo_movie_test.mat')
Y_test=a['Y']
#sys.path.append('../SPGL1_python_port')
#%
from time import time
from scipy.sparse import coo_matrix
import tifffile
import subprocess
import time as tm
from time import time
import pylab as pl
import psutil
import calblitz as cb
#%% load and motion correct movie
preprocess=1
if preprocess:
Yr=cb.load('movies/demo_mc.tif',fr=30)
Yr=Yr-np.min(Yr) # needed to remove
t1 = time()
Yr,shifts,xcorrs,template=Yr.motion_correct(max_shift_w=10, max_shift_h=10, method='opencv')
max_h,max_w= np.max(shifts,axis=0)
min_h,min_w= np.min(shifts,axis=0)
Yr=Yr.crop(crop_top=max_h,crop_bottom=-min_h+1,crop_left=max_w,crop_right=-min_w,crop_begin=0,crop_end=0)
Yr.save('demo_mc.hdf5')
Yr = np.transpose(Yr,(1,2,0))
d1,d2,T=Yr.shape
Yr=np.reshape(Yr,(d1*d2,T),order='F')
np.save('Yr',np.asarray(Yr))
print((time() - t1))
_,d1,d2=np.shape(cb.load('demo_mc.hdf5',subindices=list(range(3))))
Yr=np.load('Yr.npy',mmap_mode='r')
import calblitz as cb
import time
import pylab as pl
import numpy as np
import cv2
#% set basic ipython functionalities
#pl.ion()
#%load_ext autoreload
#%autoreload 2
#%%
filename = 'ac_001_001.tif'
frameRate = 30
start_time = 0
#%% load and motion correct movie (see other Demo for more details)
m = cb.load(filename, fr=frameRate, start_time=start_time)
template_before = np.mean(m, 0)
m = m - np.min(np.mean(m, axis=0))
m = m.gaussian_blur_2D(kernel_size_x=5,
kernel_size_y=5,
kernel_std_x=1,
kernel_std_y=1,
borderType=cv2.BORDER_REPLICATE)
#%% automatic parameters motion correction
max_shift_h = 15
max_shift_w = 15
m, shifts, xcorrs, template = m.motion_correct(max_shift_w=max_shift_w,
max_shift_h=max_shift_h,
num_frames_template=None,
template=None,
method='opencv')
def process_movie_parallel(arg_in):
# import calblitz
# import calblitz.movies
import ca_source_extraction as cse
import calblitz as cb
import numpy as np
import sys
fname, fr, margins_out, template, max_shift_w, max_shift_h, remove_blanks, apply_smooth, save_hdf5 = arg_in
if template is not None:
if type(template) is str:
if os.path.exists(template):
template = cb.load(template, fr=1)
else:
raise Exception('Path to template does not exist:' + template)
# with open(fname[:-4]+'.stout', "a") as log:
# print fname
# sys.stdout = log
# import pdb
# pdb.set_trace()
if type(fname) is cb.movie or type(fname) is cb.movies.movie:
print type(fname)
Yr = fname
else:
Yr = cb.load(fname, fr=fr)
if Yr.ndim > 1:
print 'loaded'
if apply_smooth:
print 'applying smoothing'
Yr = Yr.bilateral_blur_2D(diameter=10,
sigmaColor=10000,
sigmaSpace=0)
# bl_yr=np.float32(np.percentile(Yr,8))
# Yr=Yr-bl_yr # needed to remove baseline
print 'Remove BL'
if margins_out != 0:
Yr = Yr[:, margins_out:-margins_out,
margins_out:-margins_out] # borders create troubles
print 'motion correcting'
Yr, shifts, xcorrs, template = Yr.motion_correct(
max_shift_w=max_shift_w,
max_shift_h=max_shift_h,
method='opencv',
template=template,
remove_blanks=remove_blanks)
# Yr = Yr + bl_yr
if type(fname) is cb.movie:
return Yr
else:
print 'median computing'
template = Yr.bin_median()
print 'saving'
idx_dot = len(fname.split('.')[-1])
if save_hdf5:
Yr.save(fname[:-idx_dot] + 'hdf5')
print 'saving 2'
np.savez(fname[:-idx_dot] + 'npz',
shifts=shifts,
xcorrs=xcorrs,
template=template)
print 'deleting'
del Yr
print 'done!'
return fname[:-idx_dot]
#sys.stdout = sys.__stdout__
else:
return None
#filename='img002.tif'
filename='M_FLUO.tif'
frameRate=15.62;
start_time=0;
type_='granule'
#%%
filename_py=filename[:-4]+'.npz'
filename_hdf5=filename[:-4]+'.hdf5'
filename_mc=filename[:-4]+'_mc.npz'
filename_analysis=filename[:-4]+'_analysis.npz'
filename_traces=filename[:-4]+'_traces.npz'
#%% load movie
m=cb.load(filename, fr=frameRate,start_time=start_time);
if type_ is 'MLI':
warn('Removing Channel 2')
m=m[::2]
#%% load submovie
load_sub_movie=False
if load_sub_movie:
m=cb.load(filename, fr=frameRate,start_time=start_time,subindices=range(0,1500,10));
#% example plot a frame
pl.imshow(m[100],cmap=pl.cm.Greys_r)
#% example play movie
m.play(fr=20,gain=1.0,magnification=1)
#%% save movie python format. Less efficient than hdf5 below
if False:
# define movie
filename = '/mnt/ceph/users/epnevmatikakis/Ca_datasets/Seidemann/ax3_002_000.sbx'
filename_hdf5 = '/mnt/ceph/users/agiovann/ax3_002_000_ag'+'.hdf5'
filename_mc = '/mnt/ceph/users/agiovann/ax3_002_000_ag'+'_mc.tif'
frameRate = 50
start_time = 0
num_frames=1500
#%%
# load movie
# for loading only a portion of the movie or only some channels
# you can use the option: subindices=range(0,1500,10)
m = cb.load(filename, fr=frameRate, start_time=start_time,num_frames_sub_idx=num_frames)[0]
# save movie hdf5 format. fastest
m.save(filename_hdf5)
#%%
m=cb.load(filename_hdf5)
#%%
# backend='opencv' is much faster
m.play(fr=100, gain=1.0, magnification=1, backend='opencv')
#%%
# automatic parameters motion correction
max_shift_h = 30 # maximum allowed shifts in y
max_shift_w = 30 # maximum allowed shifts in x
m = cb.load(filename_hdf5)
m_mc, shifts, xcorrs, template = m.motion_correct(max_shift_w=max_shift_w,
max_shift_h=max_shift_h,
if '__IPYTHON__' in globals():
ipython.magic('load_ext autoreload')
ipython.magic('autoreload 2')
#%%
# define movie
filename = '09_09_2016_Running2_RGB.tif'
filename_hdf5 = filename[:-4]+'.hdf5'
filename_mc = filename[:-4]+'_mc.npz'
frameRate = 30
start_time = 0
# load movie
# for loading only a portion of the movie or only some channels
# you can use the option: subindices=range(0,1500,10)
m = cb.load(filename, fr=frameRate, start_time=start_time)
# red and green channels
m_r=m[:,:,:,0]
m=m[:,:,:,1]
# backend='opencv' is much faster
cb.concatenate([m,m_r],axis=1).resize(.5,.5,.1).play(fr=100, gain=1.0, magnification=1,backend='opencv')
# automatic parameters motion correction
max_shift_h = 20 # maximum allowed shifts in y
max_shift_w = 20 # maximum allowed shifts in x
m_r_mc, shifts, xcorrs, template = m_r.motion_correct(max_shift_w=max_shift_w,
max_shift_h=max_shift_h,
num_frames_template=None,
#%%
import glob
binary_mask=True
fls=glob.glob('*.hdf5')
pars=[]
for fl in fls:
with np.load(fl[:-5]+'_result.npz') as ld:
A=ld['A'][()]
C=ld['C']
b=ld['b']
f=ld['f']
YrA=ld['YrA']
Cn=ld['Cn']
# traces=C+YrA
m=cb.load(fl)
m_fl=m.to_2D().T
if binary_mask:
masks=A.toarray()
masks=masks/np.max(masks,0)
traces=(masks>0.5).T.dot(m_fl)
else:
nA = (A.power(2)).sum(0)
f=np.array(f).squeeze()
b_size=f.shape[0]
f_in=np.atleast_2d(f)
bckg_1=b.dot(f_in)
m_fl=m_fl-bckg_1
Y_r_sig = A.T.dot(m_fl)
Y_r_sig = scipy.sparse.linalg.spsolve(scipy.sparse.spdiags(np.sqrt(nA),0,nA.size,nA.size),Y_r_sig)
import psutil
import calblitz as cb
import cv2
import scipy
from sklearn.decomposition import MiniBatchDictionaryLearning,PCA,NMF,IncrementalPCA
import h5py
#%% CREATE PATCH OF DATA
import os
fnames=[]
for file in os.listdir("./"):
if file.startswith("k37_") and file.endswith(".tif"):
fnames.append(file)
fnames.sort()
print fnames
#%%
Yr=cb.load(fnames[0],fr=30)
Yr=Yr[:,80:130,360:410]
pl.imshow(np.mean(Yr,0),cmap=pl.cm.gray)
#%%
big_mov=[];
big_shifts=[]
for f in fnames[:]:
print f
Yr=cb.load(f,fr=30)
Yr=Yr[:,80:130,360:410]
# pl.imshow(np.mean(Yr,0))
#Yr=Yr.resize(fx=1,fy=1,fz=1)
Yr = np.transpose(Yr,(1,2,0))
d1,d2,T=Yr.shape
Yr=np.reshape(Yr,(d1*d2,T),order='F')
# load movie
# for loading only a portion of the movie or only some channels
# you can use the option: subindices=range(0,1500,10)
m = cb.load(filename, fr=frameRate, subindices=slice(1,None,2))
print m.shape
m.save(filename_red)
m = cb.load(filename, fr=frameRate, subindices=slice(0,None,2))
m.save(filename_green)
return((filename_green,filename_red))
#%%
file_names=glob.glob('*.tif')
file_names.sort()
print file_names
#%%
res=map(separate_channels,file_names)
#%%
greens=[a for a,b in res]
#%%
file_res=cb.motion_correct_parallel(greens,fr=30,template=None,margins_out=0,max_shift_w=45, max_shift_h=45,dview=None,apply_smooth=True,save_hdf5=True)
#%% run a second time since it gives better results
file_res=cb.motion_correct_parallel(greens,fr=30,template=None,margins_out=0,max_shift_w=25, max_shift_h=25,dview=None,apply_smooth=True,save_hdf5=True)
#%%
m=cb.load(file_res[2]+'hdf5',fr=30)
m.resize(1,1,.2).play(backend='opencv',gain=3.,fr=100)
#%%
for f in file_res:
m=cb.load(f+'hdf5',fr=30)
m.save(f[:-1]+'_mc.tif')
from merge_rois import mergeROIS
import pylab as pl
#import libtiff
from utilities import *
from scipy.sparse import coo_matrix
import calblitz as cb
try:
pl.ion()
%load_ext autoreload
%autoreload 2
except:
print "Probably not a Ipython interactive environment"
#%%
m=cb.load('SinglePlane.mat',fr=.5);
T,h,w=np.shape(m)
#%%
#m,shifts,xcorrs,template=m.motion_correct(max_shift_w=5,max_shift_h=5, num_frames_template=None, template = None,method='opencv')
#max_h,max_w=np.ceil(np.nanmax(shifts,axis=0))
#min_h,min_w=np.floor(np.nanmin(shifts,axis=0))
#m=m[:,-min_h:-max_h,-min_w:-max_w]
#%%
Cn2=np.nanmedian(m,axis=0)
#%%
#if np.min(m) <=0:
# print 'Removing baseline'
def save_memmap(filenames,
base_name='Yr',
resize_fact=(1, 1, 1),
remove_init=0,
idx_xy=None):
""" Saves efficiently a list of tif files into a memory mappable file
Parameters
----------
filenames: list
list of tif files
base_name: str
the base yused to build the file name. IT MUST NOT CONTAIN "_"
resize_fact: tuple
x,y, and z downampling factors (0.5 means downsampled by a factor 2)
remove_init: int
number iof frames to remove at the begining of each tif file (used for resonant scanning images if laser in rutned on trial by trial)
idx_xy: tuple size 2
for selecting slices of the original FOV, for instance idx_xy=(slice(150,350,None),slice(150,350,None))
Return
-------
fname_new: the name of the mapped file, the format is such that the name will contain the frame dimensions and the number of f
"""
order = 'F'
Ttot = 0
for idx, f in enumerate(filenames):
print f
if os.path.splitext(f)[-1] == '.hdf5':
import calblitz as cb
if idx_xy is None:
Yr = np.array(cb.load(f))[remove_init:]
else:
Yr = np.array(cb.load(f))[remove_init:, idx_xy[0], idx_xy[1]]
else:
if idx_xy is None:
Yr = imread(f)[remove_init:]
else:
Yr = imread(f)[remove_init:, idx_xy[0], idx_xy[1]]
fx, fy, fz = resize_fact
if fx != 1 or fy != 1 or fz != 1:
try:
import calblitz as cb
Yr = cb.movie(Yr, fr=1)
Yr = Yr.resize(fx=fx, fy=fy, fz=fz)
except:
print(
'You need to install the CalBlitz package to resize the movie'
)
raise
[T, d1, d2] = Yr.shape
Yr = np.transpose(Yr, (1, 2, 0))
Yr = np.reshape(Yr, (d1 * d2, T), order=order)
if idx == 0:
fname_tot = base_name + '_d1_' + str(d1) + '_d2_' + str(
d2) + '_order_' + str(order)
big_mov = np.memmap(fname_tot,
mode='w+',
dtype=np.float32,
shape=(d1 * d2, T),
order=order)
else:
big_mov = np.memmap(fname_tot,
dtype=np.float32,
mode='r+',
shape=(d1 * d2, Ttot + T),
order=order)
# np.save(fname[:-3]+'npy',np.asarray(Yr))
big_mov[:, Ttot:Ttot + T] = np.asarray(Yr, dtype=np.float32)
big_mov.flush()
Ttot = Ttot + T
fname_new = fname_tot + '_frames_' + str(Ttot) + '_.mmap'
os.rename(fname_tot, fname_new)
return fname_new
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 3 14:23:22 2016
@author: agiovann
"""
import calblitz as cb
import pylab as pl
import cv2
m = cb.load('M_FLUO_1.tif', fr=15)
m, _, _, _ = m.motion_correct(10, 10)
im1 = np.median(m, axis=0)
pl.imshow(im1, cmap=pl.cm.gray, vmax=np.percentile(m, 90))
#%%
# Read the images to be aligned
newm = []
# Find size of image1
sz = im1.shape
for idx, im2 in enumerate(m):
print(idx)
# Define the motion model
warp_mode = cv2.MOTION_TRANSLATION
warp_mode = cv2.MOTION_AFFINE
# -*- coding: utf-8 -*-
"""
Created on Mon Jul 11 10:09:09 2016
@author: agiovann
"""
from __future__ import print_function
from glob import glob
import scipy.stats as st
import calblitz as cb
import numpy as np
#%%
for fl in glob('k36*compress_.tif'):
print(fl)
m = cb.load(fl, fr=3)
img = m.local_correlations(eight_neighbours=True)
im = cb.movie(img, fr=1)
im.save(fl[:-4] + 'correlation_image.tif')
m = np.array(m)
img = st.skew(m, 0)
im = cb.movie(img, fr=1)
im.save(fl[:-4] + 'skew.tif')
img = st.kurtosis(m, 0)
im = cb.movie(img, fr=1)
im.save(fl[:-4] + 'kurtosis.tif')
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 3 14:23:22 2016
@author: agiovann
"""
import calblitz as cb
import pylab as pl
import cv2
m=cb.load('M_FLUO_1.tif',fr=15)
m,_,_,_=m.motion_correct(10,10)
im1=np.median(m,axis=0)
pl.imshow(im1,cmap=pl.cm.gray,vmax=np.percentile(m,90))
#%%
# Read the images to be aligned
newm=[];
# Find size of image1
sz = im1.shape
for idx,im2 in enumerate(m):
print(idx)
# Define the motion model
warp_mode = cv2.MOTION_TRANSLATION
warp_mode = cv2.MOTION_AFFINE
#%%
with open('file_list.txt') as f:
for ln in f:
ln1=ln[:ln.find('summary')]
print(ln1)
ffinal=ln1[:-3] + 'summary._correlation_image_full.tif'
shutil.copyfile(ffinal,os.path.join('./correlation_images/',os.path.split(ffinal)[-1]))
#%% Create images for Labeling
from glob import glob
import scipy.stats as st
import calblitz as cb
import numpy as np
for fl in res:
print(fl)
m=cb.load(fl,fr=3)
img=m.local_correlations(eight_neighbours=True)
im=cb.movie(img,fr=1)
im.save(fl[:-4]+'correlation_image.tif')
m=np.array(m)
img=st.skew(m,0)
im=cb.movie(img,fr=1)
im.save(fl[:-4]+'skew.tif')
img=st.kurtosis(m,0)
im=cb.movie(img,fr=1)
im.save(fl[:-4]+'kurtosis.tif')
counter=0
images=[os.path.join(el,'images') for el in base_folders]
regions=[os.path.join(el,'regions') for el in base_folders]
projections=[os.path.join(el,'projections') for el in base_folders]
counter=0
masks_all=[];
masks_all_nf=[];
templates=[]
templates_path=[]
corr_images=[]
for reg,img,proj,rot,self_mot,do_mot in zip(regions,images,projections,do_rotate_template,do_self_motion_correct,do_motion_correct):
print(counter)
counter+=1
m=cb.load(proj+'/median_projection.tif',fr=1)
templates_path.append(proj+'/median_projection.tif')
m1=cb.load(proj+'/correlation_image.tif',fr=1)
masks=cse.utilities.nf_read_roi_zip(reg+'/ben_regions.zip',m.shape)
masks_nf=None
# masks_nf=cse.utilities.nf_load_masks(reg+'/regions.json',m.shape)
if rot:
m=m.T
m1=m1.T
masks=np.transpose(masks,[0,2,1])
# masks_nf=np.transpose(masks_nf,[0,2,1])
if self_mot and do_mot:
m=None
m1=None
## -*- coding: utf-8 -*-
"""
Created on Fri Mar 4 21:02:12 2016
@author: agiovann, adapted from motion correction algorithm from Selmaan Chettih
"""
#%%
import calblitz as cb
import numpy as np
import pylab as pl
#%%
import scipy
chone=cb.load('/Users/agiovann/Documents/MATLAB/Motion_Correction/M_FLUO_1.tif',fr=15)
chone=chone[:,8:-8,8:-8]
T=np.median(chone,axis=0)
Nbasis = 8
minIters = 5
#linear b-splines
knots = np.linspace(1,np.shape(T)[0],Nbasis+1);
knots = np.hstack([knots[0]-(knots[1]-knots[0]),knots,knots[-1]+(knots[-1]-knots[-2])]);
weights=knots[:-2]
order=len(knots)-len(weights)-1
x=range(T.shape[0])
B = np.zeros((len(x),len(weights)))
for ii in range(len(knots)-order-1):
B[:,ii] = bin(this.knots,ii,this.order,x);
end
# sys.path.append('../SPGL1_python_port')
#%
from time import time
from scipy.sparse import coo_matrix
import tifffile
import subprocess
import time as tm
from time import time
import pylab as pl
import psutil
import calblitz as cb
#%% load and motion correct movie
preprocess = 1
if preprocess:
Yr = cb.load('movies/demo_mc.tif', fr=30)
Yr = Yr - np.min(Yr) # needed to remove
t1 = time()
Yr, shifts, xcorrs, template = Yr.motion_correct(max_shift_w=10,
max_shift_h=10,
method='opencv')
max_h, max_w = np.max(shifts, axis=0)
min_h, min_w = np.min(shifts, axis=0)
Yr = Yr.crop(crop_top=max_h,
crop_bottom=-min_h + 1,
crop_left=max_w,
crop_right=-min_w,
crop_begin=0,
crop_end=0)
Yr.save('demo_mc.hdf5')
Yr = np.transpose(Yr, (1, 2, 0))
# define movie
filename = '/mnt/ceph/users/epnevmatikakis/Ca_datasets/Seidemann/ax3_002_000.sbx'
filename_hdf5 = '/mnt/ceph/users/agiovann/ax3_002_000_ag' + '.hdf5'
filename_mc = '/mnt/ceph/users/agiovann/ax3_002_000_ag' + '_mc.tif'
frameRate = 50
start_time = 0
num_frames = 1500
#%%
# load movie
# for loading only a portion of the movie or only some channels
# you can use the option: subindices=range(0,1500,10)
m = cb.load(filename,
fr=frameRate,
start_time=start_time,
num_frames_sub_idx=num_frames)[0]
# save movie hdf5 format. fastest
m.save(filename_hdf5)
#%%
m = cb.load(filename_hdf5)
#%%
# backend='opencv' is much faster
m.play(fr=100, gain=1.0, magnification=1, backend='opencv')
#%%
# automatic parameters motion correction
max_shift_h = 30 # maximum allowed shifts in y
max_shift_w = 30 # maximum allowed shifts in x
m = cb.load(filename_hdf5)
m_mc, shifts, xcorrs, template = m.motion_correct(max_shift_w=max_shift_w,
pl.subplot(1,2,2)
pl.plot(fl['shifts'])
pl.pause(0.1)
pl.cla()
print time() - t1 - 200
#%%
big_mov=[];
big_shifts=[]
fr_remove_init=30
for f in fnames:
with np.load(f[:-3]+'npz') as fl:
big_shifts.append(fl['shifts'])
print f
Yr=cb.load(f[:-3]+'hdf5')[fr_remove_init:]
Yr=Yr.resize(fx=1,fy=1,fz=.2)
Yr = np.transpose(Yr,(1,2,0))
d1,d2,T=Yr.shape
Yr=np.reshape(Yr,(d1*d2,T),order='F')
print Yr.shape
# np.save(fname[:-3]+'npy',np.asarray(Yr))
big_mov.append(np.asarray(Yr))
#%%
big_mov=np.concatenate(big_mov,axis=-1)
big_shifts=np.concatenate(big_shifts,axis=0)
#%%
np.save('Yr_DS.npy',big_mov)
np.save('big_shifts.npy',big_shifts)
#%%
import calblitz as cb
import cv2
import scipy
from sklearn.decomposition import MiniBatchDictionaryLearning, PCA, NMF, IncrementalPCA
import h5py
from glob import glob
#%% CREATE PATCH OF DATA
import os
fnames = []
for file in os.listdir("./"):
if file.startswith("dendr") and file.endswith(".tif"):
fnames.append(file)
fnames.sort()
print fnames
#%%
Yr = cb.load(fnames[0], fr=30)
# Yr=Yr[:,80:130,360:410]
pl.imshow(np.mean(Yr, 0), cmap=pl.cm.gray)
#%%
big_mov = []
big_shifts = []
for f in fnames[:]:
print f
Yr = cb.load(f, fr=30)
Yr = Yr[:, 80:130, 360:410]
# pl.imshow(np.mean(Yr,0))
# Yr=Yr.resize(fx=1,fy=1,fz=1)
Yr = np.transpose(Yr, (1, 2, 0))
d1, d2, T = Yr.shape
Yr = np.reshape(Yr, (d1 * d2, T), order='F')
import glob
import os
import scipy
from ipyparallel import Client
import calblitz as cb
#%% download example
import urllib.request, urllib.parse, urllib.error
import shutil
if not os.path.exists('PPC.tif'):
url = "https://www.dropbox.com/s/z8rj5dekrl6yxfr/PPC.tif?dl=1" # dl=1 is important
u = urllib.request.urlretrieve(url)
shutil.move(u[0], './PPC.tif')
#%% load and motion correct movie
m = cb.load('PPC.tif', fr=30)
m -= np.min(m)
play_movie = False
if play_movie:
m.play(backend='opencv', magnification=2, fr=60, gain=5.)
#%%
max_w = 25 # max pixel shift in width direction
max_h = 25
mc, shifts, corrs, template = m.motion_correct(max_shift_w=max_w,
max_shift_h=max_h,
remove_blanks=True)
if play_movie:
mc.play(backend='opencv', magnification=2, fr=60, gain=5.)
#%%
idx = counter
A = matvar['A_s'][0][idx]
A_init = matvar['A_ins'][0][idx]
C = matvar['C_s'][0][idx]
template = matvar['templates'][0][idx]
idx_shapes = matvar['init'][0][idx]
idx_global = matvar['globalId'][0][idx]
if idx < 0:
raise Exception('Matching name not found!')
return A, C, template, idx_shapes, A_init
#%%
img_corr = cb.load(base_folder + 'projections/correlation_image.tif', fr=1)
img_median = cb.load(base_folder + 'projections/median_projection.tif', fr=1)
shape = np.shape(img_median)
pl.close()
if os.path.exist(base_folder + 'regions/princeton_regions.mat'):
a = loadmat(base_folder + 'regions/princeton_regions.mat')
try:
rois_1 = a['allROIs']
except:
rois_1 = a['M']
elif os.path.exist(base_folder + 'regions/sue_ann_regions.mat'):
A, C, template, idx_shapes, A_in = extract_sue_ann_info(
base_folder + 'regions/sue_ann_regions.mat', base_folder)
rois_1 = np.reshape(A.todense(), (shape[0], shape[1], -1), order='F')
#frameRate=30; #filename='M_FLUO_1.tif' #frameRate=15.62; #start_time=0; #%% filename_py=filename[:-4]+'.npz' filename_hdf5=filename[:-4]+'.hdf5' filename_mc=filename[:-4]+'_mc.npz' filename_analysis=filename[:-4]+'_analysis.npz' filename_traces=filename[:-4]+'_traces.npz' #%% reload and apply shifts m=cb.load(filename_hdf5) meta_data=m.meta_data[0]; shifts,min_val_add,xcorrs=[meta_data[x] for x in ['shifts','min_val_add','xcorrs']] m=m.apply_shifts(shifts,interpolation='cubic') #%% crop borders created by motion correction max_h,max_w= np.max(shifts,axis=0) min_h,min_w= np.min(shifts,axis=0) m=m.crop(crop_top=max_h,crop_bottom=-min_h+1,crop_left=max_w,crop_right=-min_w,crop_begin=0,crop_end=0) #%% play movie m.play(fr=50,gain=10.0,magnification=.5) #%% visualize average meanMov=(np.mean(m,axis=0)) pl.imshow(meanMov,cmap=pl.cm.gray,vmax=200)
return fl[:-5]+'_result.npz'
#%%
import glob
fls=glob.glob('*.hdf5')
pars=[]
for fl in fls:
with np.load(fl[:-5]+'_result.npz') as ld:
A=ld['A'][()]
C=ld['C']
b=ld['b']
f=ld['f']
YrA=ld['YrA']
Cn=ld['Cn']
traces=C+YrA
m=cb.load(fl)
T,d1,d2=m.shape
with np.load(fl[:-4]+'npz') as ld:
pars=ld['pars']
if T%len(m.file_name[0]):
raise exception('Issue with the number of components!')
num_trials=len(m.file_name[0])
traces_f=[]
traces_dff=[]
time=range(T/len(m.file_name[0]))/m.fr
for tr in traces:
tr_tmp=np.reshape(tr,(num_trials,-1)).T
traces_f.append(tr_tmp)
idx = counter
A = matvar['A_s'][0][idx]
A_init = matvar['A_ins'][0][idx]
C = matvar['C_s'][0][idx]
template = matvar['templates'][0][idx]
idx_shapes = matvar['init'][0][idx]
idx_global = matvar['globalId'][0][idx]
if idx < 0:
raise Exception('Matching name not found!')
return A, C, template, idx_shapes, A_init
#%%
img_corr = cb.load(base_folder + 'projections/correlation_image.tif', fr=1)
img_median = cb.load(base_folder + 'projections/median_projection.tif', fr=1)
shape = np.shape(img_median)
pl.close()
if os.path.exist(base_folder + 'regions/princeton_regions.mat'):
a = loadmat(base_folder + 'regions/princeton_regions.mat')
try:
rois_1 = a['allROIs']
except:
rois_1 = a['M']
elif os.path.exist(base_folder + 'regions/sue_ann_regions.mat'):
A, C, template, idx_shapes, A_in = extract_sue_ann_info(
base_folder + 'regions/sue_ann_regions.mat', base_folder)
rois_1 = np.reshape(A.todense(), (shape[0], shape[1], -1), order='F')
from scipy.sparse import lil_matrix,coo_matrix
import os
import ca_source_extraction as cse
import calblitz as cb
import time
import psutil
import sys
fnames=[]
for file in os.listdir("./"):
if file.endswith(".tif"):
fnames.append(file)
fnames.sort()
print fnames
_,d1,d2=np.shape(cb.load(fnames[0][:-3]+'hdf5',subindices=range(3),fr=10))
st=time.time()
n_processes = np.maximum(psutil.cpu_count() - 2,1) # roughly number of cores on your machine minus 1
#print 'using ' + str(n_processes) + ' processes'
p=2 # order of the AR model (in general 1 or 2)
print "Stopping cluster to avoid unnencessary use of memory...."
sys.stdout.flush()
cse.utilities.stop_server()
cse.utilities.start_server(n_processes)
file_name='/media/agiovann/10c9a792-5dbd-4bb9-b569-6c9d2435640e/andrea/Jeff/challenge/mov_stable/Yr.npy'
#%%
def nmf_patches(args_in):
import numpy as np
from sklearn.decomposition import NMF,PCA
file_name, idx_,shapes,perctl,n_components,tol,max_iter=args_in
