def getTifFRnFrames(tifDir):
"""Gets frame rate and number of frames from all .tifs in tifDir."""
from ScanImageTiffReader import ScanImageTiffReader
tifFiles = glob.glob(os.path.join(tifDir, '*.tif'))
tifFrameRate = []
nFrames = []
with ScanImageTiffReader(tifFiles[0]) as reader:
if not reader.metadata():
SIv = 3
frRegex = regex.compile(
r'frameRate=(?P<frameRate>\d.\d+)') #for ScanImage v3
else:
SIv = 5
frRegex = regex.compile(
r'hRoiManager.scanFrameRate = (?P<frameRate>\d+)'
) #for ScanImage v5+
for tifFile in tifFiles:
with ScanImageTiffReader(tifFile) as reader:
if SIv == 3:
tifFrameRate.append(
regex.search(frRegex,
reader.description(0)).group('frameRate'))
elif SIv == 5:
tifFrameRate.append(
regex.search(frRegex,
reader.metadata()).group('frameRate'))
nFrames.append(reader.shape()[0])
return list(
zip(tifFiles,
np.array(tifFrameRate, dtype=float).round(1), nFrames))
def loading_tiff_movie(tiff_file_name):
# loading the movie
try:
start_time = time.time()
tiff_movie = ScanImageTiffReader(tiff_file_name).data()
stop_time = time.time()
print(f"tiff_movie.dtype.name {tiff_movie.dtype.name}")
raise Exception("titi")
print(
f"Time for loading movie {value['tiff_file']} with scan_image_tiff: "
f"{np.round(stop_time - start_time, 3)} s")
return tiff_movie
except:
start_time = time.time()
im = PIL.Image.open(tiff_file_name)
n_frames = len(list(ImageSequence.Iterator(im)))
dim_x, dim_y = np.array(im).shape
print(f"n_frames {n_frames}, dim_x {dim_x}, dim_y {dim_y}")
tiff_movie = np.zeros((n_frames, dim_x, dim_y), dtype="uint16")
for frame, page in enumerate(ImageSequence.Iterator(im)):
tiff_movie[frame] = np.array(page)
stop_time = time.time()
print(f"Time for loading movie: "
f"{np.round(stop_time - start_time, 3)} s")
print(f"PIL tiff_movie.dtype.name {tiff_movie.dtype.name}")
return tiff_movie
def load_tiff_movie_in_memory(self, tif_movie_file_name):
"""
Load tiff movie from filename using Scan Image Tiff
Args:
tif_movie_file_name (str) : Absolute path to tiff movie
Returns:
tiff_movie (array) : Tiff movie as 3D-array
"""
if tif_movie_file_name is not None:
print(f"Loading movie")
try:
if 'ci_recording_on_pause' in self.nwb_file.intervals:
return self.load_tiff_movie_in_memory_using_pil(tif_movie_file_name)
except AttributeError:
try:
start_time = time.time()
tiff_movie = ScanImageTiffReader(tif_movie_file_name).data()
stop_time = time.time()
print(f"Time for loading movie with scan_image_tiff: "
f"{np.round(stop_time - start_time, 3)} s")
except Exception as e:
return self.load_tiff_movie_in_memory_using_pil(tif_movie_file_name)
return tiff_movie
def _validate_tiffs(self, bad_tiff_size=5):
"""
Finds the weird small tiffs and removes them. Arbitrarily set to <5 frame because it's not too
small and not too big. Also gets the lengths of all good tiffs.
Args:
bad_tiff_size (int, optional): Size tiffs must be to not be trashed. Defaults to 5.
"""
crap = []
lengths = []
for tiff in self.files:
with ScanImageTiffReader(str(tiff)) as reader:
data = reader.data()
if data.shape[0] < bad_tiff_size:
# remove them from the list of tiffs
self.files.remove(tiff)
# add them to the bad tiff list for removal from HD
crap.append(tiff)
else:
# otherwise we append the length of tiff to the lengths list
lengths.append(data.shape[0])
for crap_tiff in crap:
os.remove(crap_tiff)
self.splits = (np.array(lengths) /
(self.nchannels * self.nplanes)).astype(np.int)
def append_to_queue(q, tiff_folder, tslice, add_rate=1):
tiff_list = Path(tiff_folder).glob('*.tif*')
lengths = []
for i,t in enumerate(tiff_list):
# open data
logger.debug(f'Adding tiff {i}.')
with ScanImageTiffReader(str(t)) as reader:
data = reader.data()
# check if valid tiff
if data.shape[0] > 15:
# slice movie for this plane
mov = data[tslice, :, :]
lengths.append(mov.shape[0])
# add frames to the queue
for f in mov:
q.put(f.squeeze())
else:
continue
# so we don't overload memory
time.sleep(add_rate)
fname = Path(tiff_folder, 'file_lengths.json')
data = dict(lengths=lengths)
with open(fname, 'w') as f:
json.dump(data, f)
q.put('STOP')
def load_tiff_movie(movie_file_name):
if movie_file_name is None:
raise Exception("movie_file_name is None")
print(f"Loading movie {movie_file_name}")
# using_scan_image_tiff = False
# if using_scan_image_tiff:
try:
start_time = time.time()
tiff_movie = ScanImageTiffReader(movie_file_name).data()
stop_time = time.time()
print(f"Time for loading movie with scan_image_tiff: "
f"{np.round(stop_time - start_time, 3)} s")
except Exception as e:
start_time = time.time()
im = Image.open(movie_file_name)
n_frames = len(list(ImageSequence.Iterator(im)))
dim_x, dim_y = np.array(im).shape
print(f"n_frames {n_frames}, dim_x {dim_x}, dim_y {dim_y}")
tiff_movie = np.zeros((n_frames, dim_x, dim_y), dtype="uint16")
for frame, page in enumerate(ImageSequence.Iterator(im)):
tiff_movie[frame] = np.array(page)
stop_time = time.time()
print(f"Time for loading movie: "
f"{np.round(stop_time - start_time, 3)} s")
return tiff_movie
def open_tiff(file, sktiff):
if sktiff:
tif = TiffFile(file, fastij = False)
Ltif = len(tif)
else:
tif = ScanImageTiffReader(file)
Ltif = tif.shape()[0]
return tif, Ltif
def get_nvols(file):
with ScanImageTiffReader(file) as reader:
metadata = reader.metadata()
if metadata.split('hStackManager.zs = ')[1][0] == '0':
return 1
nvols = len(
metadata.split('hStackManager.zs = [')[1].split(']')[0].split(' '))
return nvols
def use_sktiff_reader(tiff_filename, batch_size: Optional[int] = None) -> bool:
"""Returns False if ScanImageTiffReader works on the tiff file, else True (in which case use tifffile)."""
try:
with ScanImageTiffReader(tiff_filename) as tif:
tif.data() if len(tif.shape()) < 3 else tif.data(beg=0, end=np.minimum(batch_size, tif.shape()[0] - 1))
return False
except:
print('NOTE: ScanImageTiffReader not working for this tiff type, using tifffile')
return True
def open_tiff(file: str, sktiff: bool) -> Tuple[Union[TiffFile, ScanImageTiffReader], int]:
""" Returns image and its length from tiff file with either ScanImageTiffReader or tifffile, based on 'sktiff'"""
if sktiff:
tif = TiffFile(file)
Ltif = len(tif.pages)
else:
tif = ScanImageTiffReader(file)
Ltif = 1 if len(tif.shape()) < 3 else tif.shape()[0] # single page tiffs
return tif, Ltif
def __getitem__(self, indices):
""" Loads and returns the image data directly from disk """
# Use the provided slice object to get the requested frames
if isinstance(indices, slice):
frames = np.arange(self.nframes)[indices]
elif isinstance(indices, list) or isinstance(indices, tuple):
frames = np.array(indices)
else:
frames = np.array([
indices,
])
# Define the indices of the requested frames
# tiffs are stored as [ch0-sl0, ch1-sl0, ch0-sl1, ch2-sl1, ch0-sl2 etc]
start_frame = (self._plane * self.nchannels) + self._channel
frame_jump = self.nchannels * self.nplanes
frame_ixs = start_frame + (frames * frame_jump)
n_frame_ixs = len(frame_ixs)
frame_ids = np.arange(n_frame_ixs)
# Identify the block files to open, and which frames to load
block_ixs_per_frame = np.floor(frame_ixs /
self._nframesperblock).astype(np.int)
frame_ixs_in_block = np.mod(frame_ixs, self._nframesperblock)
block_numbers, block_inverse = np.unique(block_ixs_per_frame,
return_inverse=True)
block_indexes = list(range(len(block_numbers)))
# Split the list of frames into a per-block list
frame_ixs_per_block = []
frame_ids_per_block = []
for b in block_indexes:
frame_ixs_per_block.append(
list(frame_ixs_in_block[block_inverse == b]))
frame_ids_per_block.append(list(frame_ids[block_inverse == b]))
# Loop block files, and frame indices to load all requested frames
imagedata = np.zeros((self.yres, self.xres, n_frame_ixs),
dtype=self._datatype)
with tqdm(total=n_frame_ixs, desc="Reading", unit="Fr") as bar:
for bnr, bix in zip(block_numbers, block_indexes):
with ScanImageTiffReader(self._block_files[bnr]) as tifffile:
for ix, id_ in zip(frame_ixs_per_block[bix],
frame_ids_per_block[bix]):
imagedata[:, :, id_] = tifffile.data(beg=ix,
end=ix + 1)
bar.update(1)
# Register the stack and return
if self._do_register:
imagedata = self._imregfunc(imagedata, self._plane, frames,
*self._imregparams)
# Return the stack
return imagedata
def get_nchannels(file):
with ScanImageTiffReader(file) as reader:
metadata = reader.metadata()
channel_pass_1 = metadata.split('channelSave = [')
if len(channel_pass_1) == 1:
nchannels = 1
else:
nchannels = len(
metadata.split('channelSave = [')[1].split(']')[0].split(';'))
return nchannels
def crop_tiffs(self):
self.images = []
self.file_list = []
for plane in list(range(self.planes)):
time_slice = slice(plane*self.channels+self.channel2use, -1, self.channels*self.planes)
with ScanImageTiffReader(self.tiff) as reader:
data = reader.data()
data = data[time_slice, : , self.xslice]
self.images.append(data)
tif_name = self.tiff.split('.')[0] + '_template_plane' + str(plane) + '.tif'
self.file_list.append(tif_name)
tifffile.imsave(tif_name, data)
def get_last_tiff(self):
"""Get the last tiff and make sure it's the correct size."""
last_tiffs = list(Path(self.folder).glob('*.tif*'))[-4:-2]
# pull the last few tiffs to make sure none are weirdos and get trial lengths
for tiff in last_tiffs:
with ScanImageTiffReader(str(tiff)) as reader:
data = reader.data()
# check for bad tiffs
if data.shape[0] < 10:
last_tiffs.remove(tiff)
return str(last_tiffs[-1])
def open_tiff(file, sktiff):
''' opens tiff with either ScanImageTiffReader or skimage
returns tiff and its length '''
if sktiff:
tif = TiffFile(file, fastij=False)
Ltif = len(tif)
else:
tif = ScanImageTiffReader(file)
tsize = tif.shape()
if len(tsize) < 3:
# single page tiffs
Ltif = 1
else:
Ltif = tif.shape()[0]
return tif, Ltif
def choose_tiff_reader(fs0, ops):
try:
tif = ScanImageTiffReader(fs0)
tsize = tif.shape()
if len(tsize) < 3:
# single page tiffs
im = tif.data()
else:
im = tif.data(beg=0, end=np.minimum(ops['batch_size'], tif.shape()[0]-1))
tif.close()
sktiff=False
except:
sktiff = True
print('NOTE: ScanImageTiffReader not working for this tiff type, using scikit-image')
if 'force_sktiff' in ops and ops['force_sktiff']:
sktiff=True
print('NOTE: user chose scikit-image for tiff reading')
return sktiff
def validate_tiffs(self, bad_tiff_size=5):
"""
Finds the weird small tiffs and removes them. Arbitrarily set to <5 frame because it's not too
small and not too big.
Args:
bad_tiff_size (int, optional): Size tiffs must be to not be trashed. Defaults to 5.
"""
crap = []
for tiff in self.tiffs:
with ScanImageTiffReader(tiff) as reader:
data = reader.data()
if data.shape[0] < self.bad_tiff_size:
crap.append(tiff)
for crap_tiff in crap:
os.remove(crap_tiff)
def open_tiff(file, sktiff):
""" opens tiff with either ScanImageTiffReader or tifffile
returns tiff and its length
"""
if sktiff:
tif = TiffFile(file)
Ltif = len(tif.pages)
else:
tif = ScanImageTiffReader(file)
tsize = tif.shape()
if len(tsize) < 3:
# single page tiffs
Ltif = 1
else:
Ltif = tif.shape()[0]
return tif, Ltif
def choose_tiff_reader(fs0, ops):
""" chooses tiff reader (ScanImageTiffReader is default)
tries to open tiff with ScanImageTiffReader and if it fails sets sktiff to True
Parameters
----------
fs0 : string
path to first tiff in list
ops : dictionary
'batch_size'
Returns
-------
sktiff : bool
whether or not to use tifffile tiff reader
"""
try:
tif = ScanImageTiffReader(fs0)
tsize = tif.shape()
if len(tsize) < 3:
# single page tiffs
im = tif.data()
else:
im = tif.data(beg=0,
end=np.minimum(ops['batch_size'],
tif.shape()[0] - 1))
tif.close()
sktiff = False
except:
sktiff = True
print(
'NOTE: ScanImageTiffReader not working for this tiff type, using tifffile'
)
if 'force_sktiff' in ops and ops['force_sktiff']:
sktiff = True
print('NOTE: user chose tifffile for tiff reading')
return sktiff
async def put_tiff_frames_in_queue(self, tiff_name=None):
# added sleep because last tiff isn't closed in time I think
await asyncio.sleep(0.5)
short_tiff_threshold = 15
try:
# TODO: fold this into below so there is less opening and closing of tiffs
if tiff_name is None:
tiff_name = self.get_last_tiff()
# open data
with ScanImageTiffReader(str(tiff_name)) as reader:
data = reader.data()
# check if valid tiff
if data.shape[0] > short_tiff_threshold:
# iterate through planes to get lengths and add to queue
for p in range(self.nplanes):
# slice movie for this plane
t_slice = slice(p * self.nchannels, -1,
self.nchannels * self.nplanes)
mov = data[t_slice, :, :]
# get lengths for one plane only/once per tiff
if p == 0:
self.lengths.append(mov.shape[0])
# add frames to the queue
for f in mov:
self.qs[p].put_nowait(f.squeeze())
else:
logger.warning(
f'A tiff that was too short (<{short_tiff_threshold} frames total) was attempted to be added to the queue and was skipped.'
)
return
except: # ScanImage can't open file is a generic exception
# this will skip the last file since we can't open it until ScanImage aborts
logger.warning(
'Failed to add tiff to queue. If this was the last acq, this is expected. Otherwise something is wrong.'
)
def readTimeStampOfRecording(tiffFile, nFrame):
desc = ScanImageTiffReader(tiffFile).description(nFrame)
keyWordIdx = desc.find('epoch')
dateString = re.split('\[|\]', desc[keyWordIdx:])
dateIdv = dateString[1].split()
#print('dateString',dateString)
#print('dateIdv',dateIdv)
unixStartTime = int(
datetime.datetime(int(dateIdv[0]), int(dateIdv[1]), int(dateIdv[2]),
int(dateIdv[3]), int(dateIdv[4]),
int(float(dateIdv[5]))).strftime('%s'))
#
keyWordIdx = desc.find('frameTimestamps_sec')
splitString = re.split('=|\n', desc[keyWordIdx:])
frameTimestamps = float(splitString[1])
keyWordIdx = desc.find('acqTriggerTimestamps_sec')
splitString = re.split('=|\n', desc[keyWordIdx:])
acqTriggerTimestamps = float(splitString[1])
keyWordIdx = desc.find('frameNumberAcquisition')
splitString = re.split('=|\n', desc[keyWordIdx:])
frameNumberAcquisition = int(splitString[1])
keyWordIdx = desc.find('acquisitionNumbers')
splitString = re.split('=|\n', desc[keyWordIdx:])
acquisitionNumbers = int(splitString[1])
unixFrameTime = unixStartTime + frameTimestamps
local_time = datetime.datetime.fromtimestamp(unixFrameTime)
#print(type(dt))
print(local_time.strftime("%Y-%m-%d %H:%M:%S.%f%z"))
DDate = local_time.strftime("%Y-%m-%d")
TTime = local_time.strftime("%H:%M:%S.%f%z")
#return ([frameNumberAcquisition,frameTimestamps-acqTriggerTimestamps,acquisitionNumbers,unixStartTime,unixFrameTime,frameTimestamps,acqTriggerTimestamps])
return ([
frameNumberAcquisition,
int(local_time.strftime("%Y")),
int(local_time.strftime("%m")),
int(local_time.strftime("%d")),
int(local_time.strftime("%H")),
int(local_time.strftime("%M")),
float(local_time.strftime("%S.%f%z"))
])
def get_image_from_tiff(file_name):
"""
Args:
file_name:
Returns:
"""
try:
layer_data = ScanImageTiffReader(file_name).data()
except Exception as e:
im = PIL.Image.open(file_name)
print(
f"np.array(im).shape {np.array(im).shape}, np.max {np.max(np.array(im))}"
)
layer_data = ImageSequence.Iterator(im)[0]
n_frames = len(list(ImageSequence.Iterator(im)))
dim_y, dim_x = np.array(im).shape
# print(f"layer_data.shape {layer_data.shape}, np.max {np.max(layer_data)}, np.min{np.max(layer_data)}")
return layer_data
def load_tiff_movie(tiff_file_name):
"""
Load a tiff movie from tiff file name.
Args:
tiff_file_name:
Returns: a 3d array: n_frames * width_FOV * height_FOV
"""
try:
# start_time = time.time()
tiff_movie = ScanImageTiffReader(tiff_file_name).data()
# stop_time = time.time()
# print(f"Time for loading movie with ScanImageTiffReader: "
# f"{np.round(stop_time - start_time, 3)} s")
except Exception as e:
im = PIL.Image.open(tiff_file_name)
n_frames = len(list(ImageSequence.Iterator(im)))
dim_y, dim_x = np.array(im).shape
tiff_movie = np.zeros((n_frames, dim_y, dim_x), dtype="uint16")
for frame, page in enumerate(ImageSequence.Iterator(im)):
tiff_movie[frame] = np.array(page)
return tiff_movie
def tiff_to_binary(ops):
''' converts tiff to *.bin file '''
''' requires ops keys: nplanes, nchannels, data_path, look_one_level_down, reg_file '''
''' assigns ops keys: tiffreader, first_tiffs, frames_per_folder, nframes, meanImg, meanImg_chan2'''
# copy ops to list where each element is ops for each plane
ops1 = utils.init_ops(ops)
nplanes = ops1[0]['nplanes']
nchannels = ops1[0]['nchannels']
# open all binary files for writing
# look for tiffs in all requested folders
fs, ops2 = get_tif_list(ops1[0])
reg_file = []
reg_file_chan2=[]
for ops in ops1:
reg_file.append(open(ops['reg_file'], 'wb'))
if nchannels>1:
reg_file_chan2.append(open(ops['reg_file_chan2'], 'wb'))
ops['first_tiffs'] = ops2['first_tiffs']
ops['frames_per_folder'] = np.zeros((ops2['first_tiffs'].sum(),), np.int32)
ops['filelist'] = fs
# try tiff readers
try:
tif = ScanImageTiffReader(fs[0])
im = tif.data(beg=0, end=np.minimum(500, tif.shape()[0]-1))
tif.close()
sktiff=False
except:
sktiff = True
print('ScanImageTiffReader not working for this tiff type, using scikit-image')
batch_size = 500
batch_size = nplanes*nchannels*math.ceil(batch_size/(nplanes*nchannels))
# loop over all tiffs
which_folder = -1
for ik, file in enumerate(fs):
# open tiff
tif, Ltif = open_tiff(file, sktiff)
# keep track of the plane identity of the first frame (channel identity is assumed always 0)
if ops['first_tiffs'][ik]:
which_folder += 1
iplane = 0
ix = 0
while 1:
if ix >= Ltif:
break
nfr = min(Ltif - ix, batch_size)
# tiff reading
if sktiff:
im = imread(file, pages = range(ix, ix + nfr), fastij = False)
else:
im = tif.data(beg=ix, end=ix+nfr)
# for single-page tiffs, add 1st dim
if len(im.shape) < 3:
im = np.expand_dims(im, axis=0)
# check if uint16
if type(im[0,0,0]) == np.uint16:
im = im / 2
if im.shape[0] > nfr:
im = im[:nfr, :, :]
nframes = im.shape[0]
for j in range(0,nplanes):
if ik==0 and ix==0:
ops1[j]['meanImg'] = np.zeros((im.shape[1],im.shape[2]),np.float32)
if nchannels>1:
ops1[j]['meanImg_chan2'] = np.zeros((im.shape[1],im.shape[2]),np.float32)
ops1[j]['nframes'] = 0
i0 = nchannels * ((iplane+j)%nplanes)
if nchannels>1:
nfunc = ops['functional_chan']-1
else:
nfunc = 0
im2write = im[np.arange(int(i0)+nfunc, nframes, nplanes*nchannels),:,:].astype(np.int16)
ops1[j]['meanImg'] += im2write.astype(np.float32).sum(axis=0)
reg_file[j].write(bytearray(im2write))
ops1[j]['nframes'] += im2write.shape[0]
ops1[j]['frames_per_folder'][which_folder] += im2write.shape[0]
#print(ops1[j]['frames_per_folder'][which_folder])
if nchannels>1:
im2write = im[np.arange(int(i0)+1-nfunc, nframes, nplanes*nchannels),:,:].astype(np.int16)
reg_file_chan2[j].write(bytearray(im2write))
ops1[j]['meanImg_chan2'] += im2write.astype(np.float32).sum(axis=0)
iplane = (iplane-nframes/nchannels)%nplanes
ix+=nframes
print(ops1[0]['nframes'])
# write ops files
do_registration = ops['do_registration']
do_nonrigid = ops1[0]['nonrigid']
for ops in ops1:
ops['Ly'],ops['Lx'] = ops['meanImg'].shape
if not do_registration:
ops['yrange'] = np.array([0,ops['Ly']])
ops['xrange'] = np.array([0,ops['Lx']])
ops['meanImg'] /= ops['nframes']
if nchannels>1:
ops['meanImg_chan2'] /= ops['nframes']
np.save(ops['ops_path'], ops)
# close all binary files and write ops files
for j in range(0,nplanes):
reg_file[j].close()
if nchannels>1:
reg_file_chan2[j].close()
return ops1
def align_files(root, imaging, verbose=False, enc_unit='speed'):
"""
Wrapper for that calls align_behavior_files for the proper files. Works for both imaging and non-imaging structure.
This function is called separately for one session.
:param root: str, path to the session folder (includes behavioral .txt files)
:param imaging: boolean flag whether this was an imaging trial (.tif files exist)
:param verbose: boolean flag whether unnecessary status updates should be printed to the console
:param enc_unit: str, if 'speed', encoder data is translated into cm/s; otherwise raw encoder data in
rotation [degrees] / sample window (8 ms) is saved
:return: saves merged_behavior_timestamp.txt for each aligned trial
"""
def find_file(tstamp, file_list):
"""
Finds a file with the same timestamp from a list of files.
"""
time_format = '%H%M%S'
time_stamp = datetime.strptime(str(tstamp), time_format)
matched_file = []
for filename in file_list:
curr_stamp = datetime.strptime(filename.split('_')[-1][:-4], time_format)
diff = time_stamp-curr_stamp
if abs(diff.total_seconds()) < 3:
matched_file.append(filename)
if len(matched_file) == 0:
print(f'No files with timestamp {tstamp} found in {root}!')
return
elif len(matched_file) > 1:
print(f'More than one file with timestamp {tstamp} found in {root}!')
return
else:
return matched_file[0]
print(f'\nStart processing session {root}...')
if imaging:
enc_files = glob(root + r'\\**\\Encoder*.txt')
pos_files = glob(root + r'\\**\\TCP*.txt')
trig_files = glob(root + r'\\**\\TDT TASK*.txt')
else:
enc_files = glob(root + r'\\Encoder*.txt')
pos_files = glob(root + r'\\TCP*.txt')
trig_files = glob(root + r'\\TDT TASK*.txt')
if not len(enc_files) == len(pos_files) & len(enc_files) == len(trig_files):
print(f'Uneven numbers of encoder, position and trigger files in folder {root}!')
return
counter = 1
for file in enc_files:
timestamp = int(os.path.basename(file).split('_')[1][:-4]) # here the 0 in front of early times is removed
pos_file = find_file(timestamp, pos_files)
trig_file = find_file(timestamp, trig_files)
if pos_file is None or trig_file is None:
return
if verbose:
print(f'\nNow processing trial {counter} of {len(enc_files)}, time stamp {timestamp}...')
frame_count = None
if imaging:
old_file = glob(os.path.dirname(file) + r'\\merged_behavior*.txt')
if len(old_file) == 1:
# use the sum of frame triggers from previous merged behavior file (loads faster than raw movie)
frame_count = int(np.sum(np.loadtxt(old_file[0])[:, 3]))
else:
# if trial has not been aligned yet, get frame count from raw file (either one has to be present)
movie_path = glob(str(Path(file).parents[0]) + r'\\*.tif')
if len(movie_path) == 1:
with ScanImageTiffReader(movie_path[0]) as tif:
frame_count = tif.shape()[0]
elif len(movie_path) > 1:
print(f'More than one TIFF file at {root}! Skipping trial, please check!')
continue
else:
print(f'No TIFF files or previous merged_behavior files at {root}! Skipping trial, please check!')
continue
# Get path of LOG file
log_files = glob(root + r'\\TDT LOG*.txt')
if len(log_files) == 1:
log_file = log_files[0]
elif len(log_files) > 1:
print(f'More than one LOG file in {root}. Try to merge them before processing!')
log_file = None
else:
log_file = None
merge = align_behavior_files(file, pos_file, trig_file, log_file, imaging=imaging,
frame_count=frame_count, verbose=verbose, enc_unit=enc_unit)
if merge is not None:
# save file (4 decimal places for time (0.5 ms), 2 dec for position, ints for lick, trigger, encoder)
if imaging:
file_path = os.path.join(str(Path(file).parents[0]), f'merged_behavior_{str(timestamp)}.txt')
else:
file_path = os.path.join(root, f'merged_behavior_{str(timestamp)}.txt')
np.savetxt(file_path, merge, delimiter='\t',
fmt=['%.5f', '%.3f', '%1i', '%1i', '%1i', '%.2f', '%1i'],
header='Time\tVR pos\tlicks\tframe\tencoder\tcm/s\treward')
if verbose:
print(f'Done! \nSaving merged file to {file_path}...\n')
else:
print(f'Skipped trial {file}, please check!')
counter += 1
print('Done!\n')
def ome_to_binary(ops):
"""
converts ome.tiff to *.bin file for non-interleaved red channel recordings
assumes SINGLE-PAGE tiffs where first channel has string 'Ch1'
and also SINGLE FOLDER
Parameters
----------
ops : dictionary
keys nplanes, nchannels, data_path, look_one_level_down, reg_file
Returns
-------
ops1 : list of dictionaries
creates binaries ops1[j]['reg_file']
assigns keys: tiffreader, first_tiffs, frames_per_folder, nframes, meanImg, meanImg_chan2
"""
t0 = time.time()
# copy ops to list where each element is ops for each plane
ops1 = utils.init_ops(ops)
nplanes = ops1[0]['nplanes']
# open all binary files for writing
# look for tiffs in all requested folders
ops1, fs, reg_file, reg_file_chan2 = utils.find_files_open_binaries(
ops1, False)
ops = ops1[0]
fs_Ch1, fs_Ch2 = [], []
for f in fs:
if f.find('Ch1') > -1:
if ops['functional_chan'] == 1:
fs_Ch1.append(f)
else:
fs_Ch2.append(f)
else:
if ops['functional_chan'] == 1:
fs_Ch2.append(f)
else:
fs_Ch1.append(f)
if len(fs_Ch2) == 0:
ops1[0]['nchannels'] = 1
nchannels = ops1[0]['nchannels']
# loop over all tiffs
which_folder = 0
tif = ScanImageTiffReader(fs_Ch1[0])
im0 = tif.data()
Ly, Lx = im0.shape
j = 0
for j in range(nplanes):
ops1[j]['nframes'] = 0
ops1[j]['frames_per_folder'][0] = 0
ops1[j]['meanImg'] = np.zeros((Ly, Lx))
if nchannels > 1:
ops1[j]['meanImg_chan2'] = np.zeros((Ly, Lx))
ix = 0
j = 0
for ik, file in enumerate(fs_Ch1):
# open tiff
tif = ScanImageTiffReader(file)
im = tif.data()
tif.close()
if type(im[0, 0]) == np.uint16:
im = im // 2
im = im.astype(np.int16)
ops1[j]['meanImg'] += im
ops1[j]['nframes'] += 1
ops1[j]['frames_per_folder'][0] += 1
reg_file[j].write(bytearray(im))
ix += 1
if ix % (1000) == 0:
print('%d frames of binary, time %0.2f sec.' %
(ix, time.time() - t0))
gc.collect()
j += 1
j = j % nplanes
if nchannels > 1:
ix = 0
j = 0
for ik, file in enumerate(fs_Ch2):
# open tiff
tif = ScanImageTiffReader(file)
im = tif.data()
if type(im[0, 0]) == np.uint16:
im = im // 2
im = im.astype(np.int16)
ops1[j]['meanImg_chan2'] += im
reg_file_chan2[j].write(bytearray(im))
ix += 1
if ix % (1000) == 0:
print('%d frames of binary, time %0.2f sec.' %
(ix, time.time() - t0))
gc.collect()
j += 1
j = j % nplanes
# write ops files
do_registration = ops['do_registration']
do_nonrigid = ops1[0]['nonrigid']
for ops in ops1:
ops['Ly'], ops['Lx'] = Ly, Lx
if not do_registration:
ops['yrange'] = np.array([0, ops['Ly']])
ops['xrange'] = np.array([0, ops['Lx']])
ops['meanImg'] /= ops['nframes']
if nchannels > 1:
ops['meanImg_chan2'] /= ops['nframes']
np.save(ops['ops_path'], ops)
# close all binary files and write ops files
for j in range(0, nplanes):
reg_file[j].close()
if nchannels > 1:
reg_file_chan2[j].close()
return ops1
def __init__(self,
filestem='',
filepath='.',
extention="tif",
imagesettingsfile=None,
do_reg=False,
imregfunc=None,
imregparams=[],
verbose=False):
""" Initializes the image stack and gathers the meta data
Inputs
- filestem: Part of the file name that is shared among all tiffs belonging to the stack (optional, if left out all tiffs in filepath will be included)
- filepath: Full directory path to the tiffs
- extention: file extention of the stack
- imagesettingsfile: manually stored image settings
- do_reg: Whether or not to perform registration on the images
- imregfunc: Function to use for image registration
- imregparams: List of parameters to supply to imregfunc
- verbose: print warnings
"""
super(XYT, self).__init__()
# Set the filepath
self._filepath = filepath
self._extention = extention
self._verbose = verbose
# Find the tiff files
self._block_files = sorted(
glob.glob(os.path.join(self._filepath,
filestem + '*.' + extention)))
self._nblocks = len(self._block_files)
# Load and parse the header
with ScanImageTiffReader(self._block_files[0]) as tifffile:
header = (tifffile.description(0))
self.si_info = parseheader(header)
self._nframesperblock = self.si_info["loggingFramesPerFile"]
# correct number of frames if discrepancy detected between header and block files
self._nframes = int(self.si_info["fastZNumVolumes"])
n_frames_from_blocks = int((self._nframesperblock * self._nblocks) /
(self.nplanes * self.nchannels))
if n_frames_from_blocks != self.nframes:
if self._verbose:
print(
"#frames/volumes in header ({}); inferred different number from tiff blocks ({})"
.format(self.nframes, n_frames_from_blocks))
self._nframes = n_frames_from_blocks
# Load default settings and internal variables
if imagesettingsfile is None:
self_path = os.path.dirname(os.path.realpath(__file__))
settings_path = os.path.join(
os.path.sep.join(self_path.split(os.path.sep)[:-1]),
"settings")
imagesettingsfile = os.path.join(settings_path,
"default.imagesettings.py")
self._imagesettingsfile = imagesettingsfile
settings = {}
with open(imagesettingsfile) as f:
exec(f.read(), settings)
self._fovsize_for_zoom = settings["fovsize_for_zoom"]
self._laserpowers_for_wavelength = settings[
"laserpowers_for_wavelength"]
self.register = do_reg
self._imregfunc = None
if imregfunc is not None:
self.imregfunc = imregfunc
self.imregparams = imregparams
self._datatype = np.int16
self.channel = 0
self.plane = 0
def ome_to_binary(ops):
'''
converts ome.tiff to *.bin file for non-interleaved red channel recordings
assumes single-page tiffs
also assumes only single-plane recordings - will fail otherwise
requires ops keys: nplanes, nchannels, data_path, look_one_level_down, reg_file
assigns ops keys: tiffreader, first_tiffs, frames_per_folder, nframes, meanImg, meanImg_chan2
'''
t0 = tic()
# copy ops to list where each element is ops for each plane
ops1 = init_ops(ops)
nplanes = ops1[0]['nplanes']
# open all binary files for writing
# look for tiffs in all requested folders
ops1, fs, reg_file, reg_file_chan2 = find_files_open_binaries(ops1, False)
ops = ops1[0]
fs_Ch1, fs_Ch2 = [], []
for f in fs:
if f.find('Ch1') > -1:
if ops['functional_chan'] == 1:
fs_Ch1.append(f)
else:
fs_Ch2.append(f)
else:
if ops['functional_chan'] == 1:
fs_Ch2.append(f)
else:
fs_Ch1.append(f)
if len(fs_Ch2) == 0:
ops1[0]['nchannels'] = 1
nchannels = ops1[0]['nchannels']
# loop over all tiffs
which_folder = 0
tif = ScanImageTiffReader(fs_Ch1[0])
im0 = tif.data()
Ly, Lx = im0.shape
j = 0
nframes = len(fs_Ch1)
ops1[j]['nframes'] = nframes
ops1[j]['frames_per_folder'][0] = nframes
ops1[j]['meanImg'] = np.zeros((Ly, Lx))
ix = 0
for ik, file in enumerate(fs_Ch1):
# open tiff
tif = ScanImageTiffReader(file)
im = tif.data()
tif.close()
if type(im[0, 0]) == np.uint16:
im = im // 2
im = im.astype(np.int16)
if ik == 0:
ops['meanImg'] += im
reg_file[j].write(bytearray(im))
ix += 1
if ix % (1000) == 0:
print('%d frames of binary, time %0.2f sec.' % (ix, toc(t0)))
gc.collect()
if nchannels > 1:
ops1[j]['meanImg_chan2'] = np.zeros((Ly, Lx))
ix = 0
for ik, file in enumerate(fs_Ch2):
# open tiff
tif = ScanImageTiffReader(file)
im = tif.data()
if type(im[0, 0]) == np.uint16:
im = im // 2
im = im.astype(np.int16)
if ik == 0:
ops['meanImg_chan2'] += im
reg_file_chan2[j].write(bytearray(im))
ix += 1
if ix % (1000) == 0:
print('%d frames of binary, time %0.2f sec.' % (ix, toc(t0)))
gc.collect()
# write ops files
do_registration = ops['do_registration']
do_nonrigid = ops1[0]['nonrigid']
for ops in ops1:
ops['Ly'], ops['Lx'] = Ly, Lx
if not do_registration:
ops['yrange'] = np.array([0, ops['Ly']])
ops['xrange'] = np.array([0, ops['Lx']])
ops['meanImg'] /= ops['nframes']
if nchannels > 1:
ops['meanImg_chan2'] /= ops['nframes']
np.save(ops['ops_path'], ops)
# close all binary files and write ops files
for j in range(0, nplanes):
reg_file[j].close()
if nchannels > 1:
reg_file_chan2[j].close()
return ops1
''' Created on May 14, 2017 @author: Surf32 ''' from ScanImageTiffReader import ScanImageTiffReader import os file = "C:\\Users\\Surf32\Desktop\\ResearchDSKTOP\\DataJ\\A\\A30_FastROI\\SampleDataComplete\\20161117_r3_nsyb_LC4_RNAi_flyd_L_00001\\20161117_r3_nsyb_LC4_RNAi_flyd_L_00001_00001.tif" vol=ScanImageTiffReader(file).data(); dir1 = 'C:\\Users\\Surf32\Desktop\\ResearchDSKTOP\\DataJ\\A\\A30_FastROI\\SampleDataComplete' file1 = '20161117_r3_nsyb_LC4_RNAi_flyd_L_00001_00001.tif' newpath = os.path.join(dir1, file1) vol=ScanImageTiffReader(file1).data();
def crop_movie(mov_path, x_slice, t_slice):
with ScanImageTiffReader(mov_path) as reader:
data = reader.data()
data = data[t_slice, :, x_slice]
return data
