def mesoscan_to_binary(ops):
# copy ops to list where each element is ops for each plane
# load json file with line start stops
if 'lines' not in ops:
fpath = os.path.join(ops['data_path'][0], '*json')
fs = glob.glob(fpath)
with open(fs[0], 'r') as f:
opsj = json.load(f)
ops['nplanes'] = len(opsj)
else:
ops['nplanes'] = len(ops['lines'])
ops1 = utils.init_ops(ops)
if 'lines' not in ops:
for j in range(len(ops1)):
ops1[j] = {**ops1[j], **opsj[j]}.copy()
nplanes = ops['nplanes']
print(nplanes)
nchannels = ops1[0]['nchannels']
# open all binary files for writing
reg_file = []
reg_file_chan2 = []
for ops in ops1:
reg_file.append(open(ops['reg_file'], 'wb'))
if nchannels > 1:
reg_file_chan2.append(open(ops['reg_file_chan2'], 'wb'))
fs, ops = get_tif_list(ops1[0]) # look for tiffs in all requested folders
if nchannels > 1:
nfunc = ops['functional_chan'] - 1
else:
nfunc = 0
batch_size = 500
# loop over all tiffs
for ik, file in enumerate(fs):
ix = 0
while 1:
im = imread(file, pages=range(ix, ix + batch_size))
if im.size == 0:
break
#im = io.imread(file)
if len(im.shape) < 3:
im = np.expand_dims(im, axis=0)
nframes = im.shape[0]
for j in range(0, nplanes):
if ik == 0:
ops1[j]['meanImg'] = np.zeros(
(len(ops1[j]['lines']), im.shape[2]), np.float32)
if nchannels > 1:
ops1[j]['meanImg_chan2'] = np.zeros(
(len(ops1[j]['lines']), im.shape[2]), np.float32)
ops1[j]['nframes'] = 0
im2write = im[:, ops1[j]['lines'], :].astype(np.int16)
ops1[j]['meanImg'] += im2write.astype(np.float32).sum(axis=0)
reg_file[j].write(bytearray(im2write))
if nchannels > 1:
#im2write = im[np.arange(1-nfunc, nframes, nplanes*nchannels),:,:].astype(np.int16)
reg_file_chan2[j].write(bytearray(im2write))
ops1[j]['meanImg_chan2'] += im2write.astype(
np.float32).sum(axis=0)
ops1[j]['nframes'] += im2write.shape[0]
ix += nframes
# write ops files
do_registration = ops['do_registration']
do_nonrigid = ops1[0]['nonrigid']
for ops in ops1:
ops['Ly'], ops['Lx'] = ops['meanImg'].shape
if not do_registration:
ops['yrange'] = np.array([0, ops['Ly']])
ops['xrange'] = np.array([0, ops['Lx']])
ops['meanImg'] /= ops['nframes']
if nchannels > 1:
ops['meanImg_chan2'] /= ops['nframes']
np.save(ops['ops_path'], ops)
# close all binary files and write ops files
for j in range(0, nplanes):
reg_file[j].close()
if nchannels > 1:
reg_file_chan2[j].close()
return ops1
def tiff_to_binary(ops):
# copy ops to list where each element is ops for each plane
ops1 = utils.init_ops(ops)
nplanes = ops1[0]['nplanes']
nchannels = ops1[0]['nchannels']
# open all binary files for writing
reg_file = []
reg_file_chan2 = []
for ops in ops1:
reg_file.append(open(ops['reg_file'], 'wb'))
if nchannels > 1:
reg_file_chan2.append(open(ops['reg_file_chan2'], 'wb'))
fs, ops = get_tif_list(ops1[0]) # look for tiffs in all requested folders
batch_size = 2000
batch_size = nplanes * nchannels * math.ceil(batch_size /
(nplanes * nchannels))
# loop over all tiffs
for ik, file in enumerate(fs):
# size of tiff
tif = TiffFile(file)
Ltif = len(tif)
# keep track of the plane identity of the first frame (channel identity is assumed always 0)
if ops['first_tiffs'][ik]:
iplane = 0
ix = 0
while 1:
if ix >= Ltif:
break
nfr = min(Ltif - ix, batch_size)
im = imread(file, pages=range(ix, ix + nfr))
if len(im.shape) < 3:
im = np.expand_dims(im, axis=0)
if im.shape[0] > nfr:
im = im[:nfr, :, :]
nframes = im.shape[0]
for j in range(0, nplanes):
if ik == 0 and ix == 0:
ops1[j]['meanImg'] = np.zeros((im.shape[1], im.shape[2]),
np.float32)
if nchannels > 1:
ops1[j]['meanImg_chan2'] = np.zeros(
(im.shape[1], im.shape[2]), np.float32)
ops1[j]['nframes'] = 0
i0 = nchannels * ((iplane + j) % nplanes)
if nchannels > 1:
nfunc = ops['functional_chan'] - 1
else:
nfunc = 0
im2write = im[
np.arange(int(i0) + nfunc, nframes, nplanes *
nchannels), :, :].astype(np.int16)
ops1[j]['meanImg'] += im2write.astype(np.float32).sum(axis=0)
reg_file[j].write(bytearray(im2write))
ops1[j]['nframes'] += im2write.shape[0]
if nchannels > 1:
im2write = im[np.arange(
int(i0) + 1 - nfunc, nframes, nplanes *
nchannels), :, :].astype(np.int16)
reg_file_chan2[j].write(bytearray(im2write))
ops1[j]['meanImg_chan2'] += im2write.astype(
np.float32).sum(axis=0)
iplane = (iplane - nframes / nchannels) % nplanes
ix += nframes
print(ops1[0]['nframes'])
# write ops files
do_registration = ops['do_registration']
do_nonrigid = ops1[0]['nonrigid']
for ops in ops1:
ops['Ly'], ops['Lx'] = ops['meanImg'].shape
ops['filelist'] = fs
if not do_registration:
ops['yrange'] = np.array([0, ops['Ly']])
ops['xrange'] = np.array([0, ops['Lx']])
ops['meanImg'] /= ops['nframes']
if nchannels > 1:
ops['meanImg_chan2'] /= ops['nframes']
np.save(ops['ops_path'], ops)
# close all binary files and write ops files
for j in range(0, nplanes):
reg_file[j].close()
if nchannels > 1:
reg_file_chan2[j].close()
return ops1
def tiff_to_binary(ops):
''' converts tiff to *.bin file '''
''' requires ops keys: nplanes, nchannels, data_path, look_one_level_down, reg_file '''
''' assigns ops keys: tiffreader, first_tiffs, frames_per_folder, nframes, meanImg, meanImg_chan2'''
# copy ops to list where each element is ops for each plane
ops1 = utils.init_ops(ops)
nplanes = ops1[0]['nplanes']
nchannels = ops1[0]['nchannels']
# open all binary files for writing
# look for tiffs in all requested folders
fs, ops2 = get_tif_list(ops1[0])
reg_file = []
reg_file_chan2=[]
for ops in ops1:
reg_file.append(open(ops['reg_file'], 'wb'))
if nchannels>1:
reg_file_chan2.append(open(ops['reg_file_chan2'], 'wb'))
ops['first_tiffs'] = ops2['first_tiffs']
ops['frames_per_folder'] = np.zeros((ops2['first_tiffs'].sum(),), np.int32)
ops['filelist'] = fs
# try tiff readers
try:
tif = ScanImageTiffReader(fs[0])
im = tif.data(beg=0, end=np.minimum(500, tif.shape()[0]-1))
tif.close()
sktiff=False
except:
sktiff = True
print('ScanImageTiffReader not working for this tiff type, using scikit-image')
batch_size = 500
batch_size = nplanes*nchannels*math.ceil(batch_size/(nplanes*nchannels))
# loop over all tiffs
which_folder = -1
for ik, file in enumerate(fs):
# open tiff
tif, Ltif = open_tiff(file, sktiff)
# keep track of the plane identity of the first frame (channel identity is assumed always 0)
if ops['first_tiffs'][ik]:
which_folder += 1
iplane = 0
ix = 0
while 1:
if ix >= Ltif:
break
nfr = min(Ltif - ix, batch_size)
# tiff reading
if sktiff:
im = imread(file, pages = range(ix, ix + nfr), fastij = False)
else:
im = tif.data(beg=ix, end=ix+nfr)
# for single-page tiffs, add 1st dim
if len(im.shape) < 3:
im = np.expand_dims(im, axis=0)
# check if uint16
if type(im[0,0,0]) == np.uint16:
im = im / 2
if im.shape[0] > nfr:
im = im[:nfr, :, :]
nframes = im.shape[0]
for j in range(0,nplanes):
if ik==0 and ix==0:
ops1[j]['meanImg'] = np.zeros((im.shape[1],im.shape[2]),np.float32)
if nchannels>1:
ops1[j]['meanImg_chan2'] = np.zeros((im.shape[1],im.shape[2]),np.float32)
ops1[j]['nframes'] = 0
i0 = nchannels * ((iplane+j)%nplanes)
if nchannels>1:
nfunc = ops['functional_chan']-1
else:
nfunc = 0
im2write = im[np.arange(int(i0)+nfunc, nframes, nplanes*nchannels),:,:].astype(np.int16)
ops1[j]['meanImg'] += im2write.astype(np.float32).sum(axis=0)
reg_file[j].write(bytearray(im2write))
ops1[j]['nframes'] += im2write.shape[0]
ops1[j]['frames_per_folder'][which_folder] += im2write.shape[0]
#print(ops1[j]['frames_per_folder'][which_folder])
if nchannels>1:
im2write = im[np.arange(int(i0)+1-nfunc, nframes, nplanes*nchannels),:,:].astype(np.int16)
reg_file_chan2[j].write(bytearray(im2write))
ops1[j]['meanImg_chan2'] += im2write.astype(np.float32).sum(axis=0)
iplane = (iplane-nframes/nchannels)%nplanes
ix+=nframes
print(ops1[0]['nframes'])
# write ops files
do_registration = ops['do_registration']
do_nonrigid = ops1[0]['nonrigid']
for ops in ops1:
ops['Ly'],ops['Lx'] = ops['meanImg'].shape
if not do_registration:
ops['yrange'] = np.array([0,ops['Ly']])
ops['xrange'] = np.array([0,ops['Lx']])
ops['meanImg'] /= ops['nframes']
if nchannels>1:
ops['meanImg_chan2'] /= ops['nframes']
np.save(ops['ops_path'], ops)
# close all binary files and write ops files
for j in range(0,nplanes):
reg_file[j].close()
if nchannels>1:
reg_file_chan2[j].close()
return ops1
def h5py_to_binary(ops):
# copy ops to list where each element is ops for each plane
ops1 = utils.init_ops(ops)
nplanes = ops1[0]['nplanes']
nchannels = ops1[0]['nchannels']
# open all binary files for writing
reg_file = []
reg_file_chan2 = []
for ops in ops1:
reg_file.append(open(ops['reg_file'], 'wb'))
if nchannels > 1:
reg_file_chan2.append(open(ops['reg_file_chan2'], 'wb'))
# open h5py file for reading
key = ops1[0]['h5py_key']
if ops1[0]['look_one_level_down']:
h5list = list_h5(ops1[0])
print('using a list of h5 files:')
print(h5list)
else:
h5list = [ops1[0]['h5py']]
iall = 0
for h5 in h5list:
with h5py.File(h5, 'r') as f:
# if h5py data is 4D instead of 3D, assume that
# data = nframes x nplanes x pixels x pixels
hdims = f[key].ndim
# keep track of the plane identity of the first frame (channel identity is assumed always 0)
nbatch = nplanes * nchannels * math.ceil(ops1[0]['batch_size'] /
(nplanes * nchannels))
if hdims == 3:
nframes_all = f[key].shape[0]
else:
nframes_all = f[key].shape[0] * f[key].shape[1]
nbatch = min(nbatch, nframes_all)
if nchannels > 1:
nfunc = ops['functional_chan'] - 1
else:
nfunc = 0
# loop over all tiffs
ik = 0
while 1:
if hdims == 3:
irange = np.arange(ik, min(ik + nbatch, nframes_all), 1)
if irange.size == 0:
break
im = f[key][irange, :, :]
else:
irange = np.arange(
ik / nplanes,
min(ik / nplanes + nbatch / nplanes,
nframes_all / nplanes), 1)
if irange.size == 0:
break
im = f[key][irange, :, :, :]
im = np.reshape(
im, (im.shape[0] * nplanes, im.shape[2], im.shape[3]))
nframes = im.shape[0]
for j in range(0, nplanes):
if iall == 0:
ops1[j]['meanImg'] = np.zeros(
(im.shape[1], im.shape[2]), np.float32)
if nchannels > 1:
ops1[j]['meanImg_chan2'] = np.zeros(
(im.shape[1], im.shape[2]), np.float32)
ops1[j]['nframes'] = 0
i0 = nchannels * ((j) % nplanes)
im2write = im[np.arange(
int(i0) + nfunc, nframes, nplanes *
nchannels), :, :].astype(np.int16)
reg_file[j].write(bytearray(im2write))
ops1[j]['meanImg'] += im2write.astype(
np.float32).sum(axis=0)
if nchannels > 1:
im2write = im[np.arange(
int(i0) + 1 - nfunc, nframes, nplanes *
nchannels), :, :].astype(np.int16)
reg_file_chan2[j].write(bytearray(im2write))
ops1[j]['meanImg_chan2'] += im2write.astype(
np.float32).sum(axis=0)
ops1[j]['nframes'] += im2write.shape[0]
ik += nframes
iall += nframes
# write ops files
do_registration = ops1[0]['do_registration']
do_nonrigid = ops1[0]['nonrigid']
for ops in ops1:
ops['Ly'] = im2write.shape[1]
ops['Lx'] = im2write.shape[2]
if not do_registration:
ops['yrange'] = np.array([0, ops['Ly']])
ops['xrange'] = np.array([0, ops['Lx']])
ops['meanImg'] /= ops['nframes']
if nchannels > 1:
ops['meanImg_chan2'] /= ops['nframes']
np.save(ops['ops_path'], ops)
# close all binary files and write ops files
for j in range(0, nplanes):
reg_file[j].close()
if nchannels > 1:
reg_file_chan2[j].close()
return ops1
def run_s2p(ops={}, db={}):
i0 = tic()
ops = {**ops, **db}
if 'save_path0' not in ops or len(ops['save_path0']) == 0:
if ('h5py' in ops) and len(ops['h5py']) > 0:
ops['save_path0'], tail = os.path.split(ops['h5py'])
else:
ops['save_path0'] = ops['data_path'][0]
# check if there are files already registered
fpathops1 = os.path.join(ops['save_path0'], 'suite2p', 'ops1.npy')
if os.path.isfile(fpathops1):
files_found_flag = True
flag_binreg = True
ops1 = np.load(fpathops1)
for i, op in enumerate(ops1):
# default behavior is to look in the ops
flag_reg = os.path.isfile(op['reg_file'])
if not flag_reg:
# otherwise look in the user defined save_path0
op['save_path'] = os.path.join(ops['save_path0'], 'suite2p',
'plane%d' % i)
op['ops_path'] = os.path.join(op['save_path'], 'ops.npy')
op['reg_file'] = os.path.join(op['save_path'], 'data.bin')
flag_reg = os.path.isfile(op['reg_file'])
files_found_flag &= flag_reg
if 'refImg' not in op:
flag_binreg = False
# use the new options
ops1[i] = {**op, **ops}
ops1[i] = ops1[i].copy()
print(ops1[i]['save_path'])
# except for registration results
ops1[i]['xrange'] = op['xrange']
ops1[i]['yrange'] = op['yrange']
else:
files_found_flag = False
flag_binreg = False
######### REGISTRATION #########
if not files_found_flag:
# get default options
ops0 = default_ops()
# combine with user options
ops = {**ops0, **ops}
# copy ops to list where each element is ops for each plane
ops1 = utils.init_ops(ops)
# copy tiff to a binary
if len(ops['h5py']):
ops1 = utils.h5py_to_binary(ops1)
print('time %4.4f. Wrote h5py to binaries for %d planes' %
(toc(i0), len(ops1)))
else:
try:
ops1 = utils.tiff_to_binary(ops1)
print('time %4.4f. Wrote tifs to binaries for %d planes' %
(toc(i0), len(ops1)))
except Exception as e:
if HAS_HAUS:
dataset = haussio.load_haussio(ops['data_path'][0])
ops1 = dataset.tosuite2p(ops)
print('time %4.4f. Wrote data to binaries for %d planes' %
(toc(i0), len(ops1)))
else:
print('Unsupported file format: ' + str(e))
return
# save ops1
np.save(fpathops1, ops1)
if not ops['do_registration']:
flag_binreg = True
if not flag_binreg:
ops1 = register.register_binary(ops1) # register tiff
np.save(fpathops1, ops1) # save ops1
print('time %4.4f. Registration complete' % toc(i0))
elif files_found_flag:
print('found ops1 and pre-registered binaries')
print(ops1[0]['reg_file'])
print('overwriting ops1 with new ops')
print('skipping registration...')
# registration is off, but previous binary was not found
# need to initialize ops
else:
print('binary file created, but registration not performed')
######### CELL DETECTION #########
if len(ops1) > 1 and ops['num_workers_roi'] >= 0:
if ops['num_workers_roi'] == 0:
ops['num_workers_roi'] = len(ops1)
with Pool(ops['num_workers_roi']) as p:
ops1 = p.map(utils.get_cells, ops1)
else:
for k in range(len(ops1)):
ops1[k] = utils.get_cells(ops1[k])
######### SPIKE DECONVOLUTION AND CLASSIFIER #########
for ops in ops1:
fpath = ops['save_path']
F = np.load(os.path.join(fpath, 'F.npy'))
Fneu = np.load(os.path.join(fpath, 'Fneu.npy'))
dF = F - ops['neucoeff'] * Fneu
spks = dcnv.oasis(dF, ops)
np.save(os.path.join(ops['save_path'], 'spks.npy'), spks)
print('time %4.4f. Detected spikes in %d ROIs' % (toc(i0), F.shape[0]))
stat = np.load(os.path.join(fpath, 'stat.npy'))
# apply default classifier
classfile = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'classifiers/classifier_user.npy')
print(classfile)
iscell = classifier.run(classfile, stat)
np.save(os.path.join(ops['save_path'], 'iscell.npy'), iscell)
# save as matlab file
if ('save_mat' in ops) and ops['save_mat']:
matpath = os.path.join(ops['save_path'], 'Fall.mat')
io.savemat(
matpath, {
'stat': stat,
'ops': ops,
'F': F,
'Fneu': Fneu,
'spks': spks,
'iscell': iscell
})
# save final ops1 with all planes
np.save(fpathops1, ops1)
#### COMBINE PLANES or FIELDS OF VIEW ####
if len(ops1) > 1 and ops1[0]['combined']:
utils.combined(ops1)
for ops in ops1:
if ('delete_bin' in ops) and ops['delete_bin']:
os.remove(ops['reg_file'])
if ops['nchannels'] > 1:
os.remove(ops['reg_file_chan2'])
print('finished all tasks in total time %4.4f sec' % toc(i0))
return ops1