def populatevolpy_modified():
#%%
volpy_basedir = str(pathlib.Path.home()) + '/Data/Voltage_imaging/VolPy/'
volpy_modified_basedir = str(
pathlib.Path.home()) + '/Data/Voltage_imaging/VolPy_modified/'
usematlabroi = True
movies = imaging.Movie().fetch(as_dict=True)
for movie in movies: #[::-1]:
moviefiles = imaging.MovieFile() & movie
filenames, dirs, basedirs = moviefiles.fetch('movie_file_name',
'movie_file_directory',
'movie_file_repository')
fnames = list()
for filename, dir_now, basedir in zip(filenames, dirs, basedirs):
fnames.append(
os.path.join(dj.config['locations.' + basedir], dir_now,
filename))
fr = movie['movie_frame_rate']
loaddir = os.path.join(
volpy_basedir, dir_now[dir_now.find('raw') + len('raw') + 1:]
) #,movie['movie_name'] # there is an additional directory here!
savedir = os.path.join(
volpy_modified_basedir,
dir_now[dir_now.find('raw') + len('raw') + 1:]
) #,movie['movie_name'] # there is an additional directory here!
pathlib.Path(savedir).mkdir(parents=True, exist_ok=True)
print(movie)
#time.sleep(5)
if len(os.listdir(savedir)) > 0:
print('already done.. skipped')
else:
roidir = loaddir[:loaddir.find(
'VolPy')] + 'Spikepursuit' + loaddir[loaddir.find('VolPy') +
len('Volpy'):]
try:
files = os.listdir(roidir)
except:
files = []
if usematlabroi and 'ROIs.mat' not in files:
print('no matlab ROIs found')
else:
#%
original_files = os.listdir(loaddir)
for file_now in original_files:
if 'mmap' in file_now:
registered_movie_file = os.path.join(loaddir, file_now)
#%
print('waiting')
time.sleep(1000)
if movie['movie_frame_num'] > 500:
run_modified_caiman_pipeline(
movie, fr, fnames, savedir,
registered_movie_file, usematlabroi)
def populatevolpy():
volpy_basedir = str(pathlib.Path.home()) + '/Data/Voltage_imaging/VolPy/'
usematlabroi = True
movies = imaging.Movie().fetch(as_dict=True)
for movie in movies: #[::-1]:
moviefiles = imaging.MovieFile() & movie
filenames, dirs, basedirs = moviefiles.fetch('movie_file_name',
'movie_file_directory',
'movie_file_repository')
fnames = list()
for filename, dir_now, basedir in zip(filenames, dirs, basedirs):
fnames.append(
os.path.join(dj.config['locations.' + basedir], dir_now,
filename))
fr = movie['movie_frame_rate']
savedir = os.path.join(
volpy_basedir, dir_now[dir_now.find('Voltage_imaging') +
len('Voltage_imaging') + 1:],
movie['movie_name'])
pathlib.Path(savedir).mkdir(parents=True, exist_ok=True)
print(movie)
#time.sleep(5)
if len(os.listdir(savedir)) > 0:
print('already done.. skipped')
else:
roidir = savedir[:savedir.find(
'VolPy')] + 'Spikepursuit' + savedir[savedir.find('VolPy') +
len('Volpy'):]
try:
files = os.listdir(roidir)
except:
files = []
if usematlabroi and 'ROIs.mat' not in files:
print('no matlab ROIs found')
else:
#print('waiting')
#time.sleep(1000)
if movie['movie_frame_num'] > 500:
run_caiman_pipeline(movie, fr, fnames, savedir,
usematlabroi)
# #print('waiting')
# #time.sleep(1000)
# if movie['movie_frame_num']>500:
# #print('starting')
# #time.sleep(1000)
# merge_denoised_tiff_files(movie,loaddir,savedir)
# #run_denoised_caiman_pipeline(movie,fr,loaddir,savedir,usematlabroi)
# =============================================================================
#%% running spikepursuit
denoised_caiman_basedir = str(
pathlib.Path.home()) + '/Data/Voltage_imaging/denoised_volpy/'
usematlabroi = True
movies = imaging.Movie().fetch(as_dict=True)
for movie in movies: #[::-1]:
moviefiles = imaging.MovieFile() & movie
filenames, dirs, basedirs = moviefiles.fetch('movie_file_name',
'movie_file_directory',
'movie_file_repository')
fnames = list()
for filename, dir_now, basedir in zip(filenames, dirs, basedirs):
fnames.append(
os.path.join(dj.config['locations.' + basedir], dir_now, filename))
fr = movie['movie_frame_rate']
loaddir = os.path.join(
denoised_caiman_basedir,
dir_now[dir_now.find('Voltage_imaging') + len('Voltage_imaging') + 1:],
movie['movie_name'])
savedir = loaddir
print(movie)
#time.sleep(5)
#%%
#%% save movie - motion correction, denoising, etc
key = {'session': 1,
'subject_id': 462149,
'cell_number': 1,
'motion_correction_method': 'VolPy',
'roi_type': 'VolPy'}
key = {'session': 1,
'subject_id': 456462,
'cell_number': 3,
'motion_correction_method': 'Matlab',
'roi_type': 'SpikePursuit'}
movie_nums = (imaging_gt.GroundTruthROI()*imaging.Movie()&key).fetch('movie_number')
movie_name = (imaging_gt.GroundTruthROI()*imaging.Movie()&key &'movie_number = {}'.format(min(movie_nums))).fetch1('movie_name')
session_date = str((experiment.Session()*imaging_gt.GroundTruthROI()*imaging.Movie()&key &'movie_number = {}'.format(min(movie_nums))).fetch1('session_date'))
fnames,dirs = (imaging.MovieFile()*imaging_gt.GroundTruthROI()*imaging.Movie()&key &'movie_number = {}'.format(min(movie_nums))).fetch('movie_file_name','movie_file_directory')
allfnames = list()
for fname,directory in zip(fnames,dirs):
allfnames.append(os.path.join(homefolder,directory,fname))
originaldir = os.path.join(homefolder,directory)
puthere = originaldir.find('Voltage_imaging')+len('Voltage_imaging')
denoiseddir = os.path.join(originaldir[:puthere],'denoised_volpy',originaldir[puthere+1:],movie_name)
volpydir = os.path.join(originaldir[:puthere],'VolPy',originaldir[puthere+1:],movie_name)
files = os.listdir(denoiseddir)
for file in files:
if movie_name in file and file[-4:] == 'mmap':
denoised_file = file
if 'memmap_' in file and file[-4:] == 'mmap':
motioncorrected_denoised_file = file
def populatevolpy():
#%%
volpy_basedir = str(pathlib.Path.home()) + '/Data/Voltage_imaging/VolPy/'
parameters = {
'usematlabroi': False,
'dospikepursuit': True,
'usevolpyroi': True
}
movies = imaging.Movie().fetch(as_dict=True)
for movie in movies: #[::-1]:
moviefiles = imaging.MovieFile() & movie
filenames, dirs, basedirs = moviefiles.fetch('movie_file_name',
'movie_file_directory',
'movie_file_repository')
fnames = list()
for filename, dir_now, basedir in zip(filenames, dirs, basedirs):
fnames.append(
os.path.join(dj.config['locations.' + basedir], dir_now,
filename))
fr = movie['movie_frame_rate']
savedir = os.path.join(
volpy_basedir, dir_now[dir_now.find('raw') + len('raw') + 1:]
) #,movie['movie_name'] # there is an additional directory here!
pathlib.Path(savedir).mkdir(parents=True, exist_ok=True)
print(movie)
#time.sleep(5)
if (len(os.listdir(savedir)) > 0 and not parameters['dospikepursuit']
) or ('spikepursuit.pickle' in os.listdir(savedir)
and parameters['dospikepursuit']):
print('already done.. skipped')
else:
if parameters['usematlabroi']:
roidir = savedir[:savedir.
find('VolPy')] + 'Spikepursuit' + savedir[
savedir.find('VolPy') + len('Volpy'):]
try:
files = os.listdir(roidir)
except:
files = []
if parameters['usematlabroi'] and 'ROIs.mat' not in files:
print('no matlab ROIs found')
continue
if parameters['usevolpyroi']:
roidir = savedir[:savedir.find('VolPy')] + 'ROI' + savedir[
savedir.find('VolPy') + len('Volpy'):]
try:
files = os.listdir(roidir)
except:
files = []
if parameters['usevolpyroi'] and 'VolPy.npy' not in files:
print('no VolPy ROIs found')
continue
if len(os.listdir(savedir)) > 0:
parameters['motion_correction_already_done'] = True
else:
parameters['motion_correction_already_done'] = False
# =============================================================================
#
# print('waiting')
# time.sleep(1000)
# =============================================================================
if movie['movie_frame_num'] > 500:
run_caiman_pipeline(movie, fr, fnames, savedir, parameters)
movie_dict[keynow] = str(movie_dict[keynow])
elif type(movie_dict[keynow]) == np.ndarray:
movie_dict[keynow] = movie_dict[keynow].tolist()
save_dir_base = os.path.join(gt_package_directory,str(key['subject_id']),'Cell_{}'.format(key['cell_number']),movie_dict['movie_name'])
if ((os.path.isdir(save_dir_base) and (len(os.listdir(save_dir_base))>3 and not overwrite))):
print('already exported, skipped')
else:
session_time, cell_recording_start = (experiment.Session()*ephys_patch.Cell()&key).fetch1('session_time','cell_recording_start')
first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key).fetch('movie_start_time'),float))
first_movie_start_time_real = first_movie_start_time + session_time.total_seconds()
frame_times = (imaging.MovieFrameTimes()&movie).fetch1('frame_times') - cell_recording_start.total_seconds() + session_time.total_seconds()
motion_corr_vectors = np.asarray((imaging.MotionCorrection()*imaging.RegisteredMovie()&movie&'motion_correction_method = "VolPy"'&'motion_corr_description= "rigid motion correction done with VolPy"').fetch1('motion_corr_vectors'))
movie_dict['movie_start_time'] = frame_times[0]
movie_files = list()
repositories , directories , fnames = (imaging.MovieFile() & movie).fetch('movie_file_repository','movie_file_directory','movie_file_name')
for repository,directory,fname in zip(repositories,directories,fnames):
movie_files.append(os.path.join(dj.config['locations.{}'.format(repository)],directory,fname))
sweepdata_out = list()
sweepmetadata_out = list()
for sweep_number in sweep_numbers:
#%
key_sweep = key.copy()
key_sweep['sweep_number'] = sweep_number
neutralizationenable,e_sr= (ephys_patch.SweepMetadata()&key_sweep).fetch1('neutralizationenable','sample_rate')
try:
uncompensatedRS = float((ephysanal.SweepSeriesResistance()&key_sweep).fetch1('series_resistance_residual'))
except:
uncompensatedRS = 0
v = (ephys_patch.SweepResponse()&key_sweep).fetch1('response_trace')
key = {
'session': 1,
'subject_id': 456462,
'cell_number': 3,
'motion_correction_method': 'Matlab',
'roi_type': 'SpikePursuit'
}
movie_nums = (imaging_gt.GroundTruthROI() * imaging.Movie()
& key).fetch('movie_number')
movie_name = (imaging_gt.GroundTruthROI() * imaging.Movie() & key &
'movie_number = {}'.format(min(movie_nums))).fetch1('movie_name')
session_date = str(
(experiment.Session() * imaging_gt.GroundTruthROI() * imaging.Movie() & key
& 'movie_number = {}'.format(min(movie_nums))).fetch1('session_date'))
fnames, dirs = (
imaging.MovieFile() * imaging_gt.GroundTruthROI() * imaging.Movie() & key
& 'movie_number = {}'.format(min(movie_nums))).fetch(
'movie_file_name', 'movie_file_directory')
allfnames = list()
for fname, directory in zip(fnames, dirs):
allfnames.append(os.path.join(homefolder, directory, fname))
originaldir = os.path.join(homefolder, directory)
puthere = originaldir.find('Voltage_imaging') + len('Voltage_imaging')
denoiseddir = os.path.join(originaldir[:puthere], 'denoised_volpy',
originaldir[puthere + 1:], movie_name)
volpydir = os.path.join(originaldir[:puthere], 'VolPy',
originaldir[puthere + 1:], movie_name)
files = os.listdir(denoiseddir)
for file in files: