def test_save_load(tmpdir):
import os
print(f'tempdir = {tmpdir}')
print(f'{os.path.exists(tmpdir)}')
arr_in = zeros((10, 5))
str_in = 'this is string'
dict_in = {'1': 1, '2': 2}
list_in = [1, 2, 3, 4]
compound_in = {}
compound_in['arr'] = arr_in
compound_in['str'] = str_in
compound_in['dict'] = dict_in
compound_in['list'] = list_in
# save to a file
save_to_file(tmpdir + '/arr.pkl', arr_in)
save_to_file(tmpdir + '/str.pkl', str_in)
save_to_file(tmpdir + '/dict.pkl', dict_in)
save_to_file(tmpdir + '/list.pkl', list_in)
save_to_file(tmpdir + '/compound.pkl', compound_in)
# Load to a file
arr_out = load_from_file(tmpdir + '/arr.pkl')
str_out = load_from_file(tmpdir + '/str.pkl')
dict_out = load_from_file(tmpdir + '/dict.pkl')
list_out = load_from_file(tmpdir + '/list.pkl')
compound_out = load_from_file(tmpdir + '/compound.pkl')
#run tests
assert array_equal(arr_in, arr_out)
assert str_in == str_out
assert dict_in == dict_out
assert list_in == list_out
assert compound_in.keys() == compound_out.keys()
def generate_video_from_list_of_stats(root, lst, key='dmax_frame'):
"""
Generates a movie clip from a collection of stats files.
image modification:
- sqrt (just take a square root of images)
- log10 (take log base 10 of images)
Parameters
----------
root :: (string)
lst :: (list of filenames)
key :: (string)
Returns
-------
Examples
--------
>>> from ubcs_auxiliary.multiprocessing import new_process
>>> new_process(generate_video_from_list_of_stats,'/mnt/data/dmout_breathing_1.stats.pkl')
"""
from numpy import sqrt, log10, zeros, where
import cv2
import h5py
import os
from time import time, ctime
from ubcs_auxiliary.save_load_object import load_from_file
roottemp, tail = os.path.split(lst[0])
basename, extension = tail.split('.')[0], '.'.join(tail.split('.')[1:])
vbasename = '_'.join(basename.split('_')[:-1])
fstats = load_from_file(os.path.join(root, tail))
width = fstats[key].shape[1]
height = fstats[key].shape[0]
size = (width, height)
fps = 24
video_filename = os.path.join(root, vbasename + f'.{key}.mp4')
print(f'the video filename is: {video_filename}')
video_out = cv2.VideoWriter(video_filename,
cv2.VideoWriter_fourcc(*'mp4v'), fps, size)
for item in lst:
print(f'{ctime(time())} processing file {item}')
fstats = load_from_file(os.path.join(root, item))
for i in range(int(fstats[key].max() + 1)):
# writing to a image array
img = zeros((height, width, 3), dtype='uint8')
img[:, :, 1] = (fstats[key] == i)
if i - 1 >= 0:
img[:, :, 0] = (fstats[key] == i - 1)
if i + 1 < 600:
img[:, :, 2] = (fstats[key] == i + 1)
img = img * 254
video_out.write(img)
video_out.release()
def test_mono12p_to_image():
"""
"""
from lcp_video import analysis
from ubcs_auxiliary.save_load_object import load_from_file
height = 2048
width = 2448
raw = load_from_file('lcp_video/test_data/flir_mono12p_rawdata.pkl')
image_original = load_from_file(
'lcp_video/test_data/flir_mono12p_image.pkl')
image_reconstruct = analysis.mono12p_to_image(raw, height, width).reshape(
(2048, 2448))
assert (image_reconstruct == image_original).all()
def get_spots_mask(root,data,stats_ref,stats_dark):
"""
"""
from h5py import File
from ubcs_auxiliary.save_load_object import load_from_file
from skimage import measure
from numpy import where, zeros_like, bool, zeros,sqrt, ones
from time import time
stats_ref = load_from_file(root+'camera_8mm_Reference.stats.pkl')
stats_dark = load_from_file(root+'/camera_8mm_Dark.stats.pkl')
data = root + 'camera_8mm_Spray.gzip.hdf5'
mask_filename = root + 'camera_8mm_Spray.mask.gzip.hdf5'
M1 = stats_ref['M1']
M2 = stats_ref['M2']
M3 = stats_ref['M3']
row = 3000
col = 4096
N_images = 600
sigma = 6
slit_mask = ones((row,col))
slit_mask[:,3950:] = 0
N_count = zeros(N_images)
t1 = time()
with File(data,'r') as f_data:
with File(mask_filename,'a') as f_mask:
N_images,row,col = (600,3000,4096)
f_mask.create_dataset('mask', (N_images,3000,4096), dtype = 'int8', chunks = (1,3000,4096), compression = 'gzip')
f_mask.create_dataset('blobs', (N_images,3000,4096), dtype = 'int16', chunks = (1,3000,4096), compression = 'gzip')
images = f_data['images']
skew_threshold = 0.3
normal_mask = (M3 > -skew_threshold) & (M3 < skew_threshold)
threshold = []
for i in range(N_images):
t0 = time()
mask = zeros((3000,4096))
for j in [2,3,4,5,6]:
if j == 2:
c = 2
else:
c = 1
mask += c*(((images[i] - M1) > j*sqrt(M2))*normal_mask*slit_mask)
blobs = measure.label(mask==6).astype('int16')
f_mask['blobs'][i] = blobs
f_mask['mask'][i] = mask
print(i,time()-t0, blobs.max())
t2 = time()
print(t2-t1)
def benchmark_u12_to_16():
#mono12packed_to_image(rawdata, height, width, mask)
from ubcs_auxiliary.save_load_object import load_from_file
from time import time
import timeit
from numpy import array, ones, uint8
preparation = ""
testcode = 'data = u12_to_16(rawdata, height=height, width = width)'
preparation += "from ubcs_auxiliary.save_load_object import load_from_file;\n"
preparation += 'from numpy import vstack, tile, hstack, arange,reshape, vstack, tile, hstack, arange,reshape, packbits, reshape, int16, concatenate, array, ones, uint8;\n'
preparation += "from lcp_video.analysis import u12_to_16;\n"
preparation += f"height = 3000; width = 4096;\n"
preparation += "rawdata = load_from_file('lcp_video/test_data/mono12p_dataset_12Mpixels.pkl');\n "
t_full = timeit.Timer(testcode, preparation)
print("Preparation")
print(preparation)
print('tested code')
print(
" arr = rawdata.reshape(-1,3) \n byte_even = arr[:,0]+256*(bitwise_and(arr[:,1],15)) \n \
byte_odd = right_shift(bitwise_and(arr[:,1],240),4) + right_shift(256*arr[:,2],4) \n \
img = empty(height*width,dtype='int16') \n \
img[0::2] = byte_even\n \
img[1::2] = byte_odd\n \
return img.reshape(height,width) \n")
temp = array(t_full.repeat(4, 100)) / 100
print(
f"Full function: {round(temp.mean(),3)} +- {round(temp.std(),3)}, with min {round(temp.min(),3)} and max {round(temp.max(),3)}"
)
from numpy import vstack, tile, hstack, arange, reshape, vstack, tile, hstack, arange, reshape, packbits, reshape, int16, concatenate, array, ones, uint8
from ubcs_auxiliary.save_load_object import load_from_file
height = 2048
width = 2448
from lcp_video.analysis import mono12p_to_image
rawdata = load_from_file('lcp_video/test_data/flir_mono12p_rawdata.pkl')
img = load_from_file('lcp_video/test_data/flir_mono12p_image.pkl')
data = mono12p_to_image(rawdata, height=height, width=width)
print((data.reshape(height, width) == img).any())
from matplotlib import pyplot as plt
plt.figure()
plt.imshow(img)
plt.figure()
plt.imshow(data.reshape(height, width))
def profile_mono12p_to_image():
from ubcs_auxiliary.save_load_object import load_from_file
from lcp_video.analysis import mono12p_to_image
mono12p = load_from_file('lcp_video/test_data/flir_rawdata_mono12p.pkl')
height = 2048
width = 2448
data = mono12p_to_image(mono12p, height, width)
def generate_video_from_list_of_hdf5files(root, lst, stats_finame):
"""
Generates a movie clip from a collection of hdf5 files. The input is a list of hdf5 filenames in correct order
image modification:
- sqrt (just take a square root of images)
- log10 (take log base 10 of images)
Parameters
----------
filename :: (string)
Returns
-------
Examples
--------
>>> from ubcs_auxiliary.multiprocessing import new_process
>>> new_process(generate_video_from_hdf5file,['/mnt/data/dmout_breathing_1.data.hdf5'])
"""
from numpy import sqrt, log10, zeros, where
import cv2
import h5py
import os
from time import time, ctime
from ubcs_auxiliary.save_load_object import load_from_file
roottemp, tail = os.path.split(lst[0])
basename, extension = tail.split('.')[0], '.'.join(tail.split('.')[1:])
vbasename = '_'.join(basename.split('_')[:-1])
fdata = h5py.File(os.path.join(root, tail), 'r')
width = fdata['image width'][()]
height = fdata['image height'][()]
size = (width, height)
fps = 1 / 0.032
stats = load_from_file(stats_finame)
img_mean = stats['S1'] / stats['N']
video_filename = os.path.join(root, vbasename + '.mp4')
print(f'the video filename is: {video_filename}')
video_out = cv2.VideoWriter(video_filename,
cv2.VideoWriter_fourcc(*'mp4v'), fps, size)
for item in lst:
print(f'{ctime(time())} processing file {item}')
fdata = h5py.File(os.path.join(root, item), 'r')
for i in range(fdata['images'].shape[0]):
# writing to a image array
img = zeros((height, width, 3))
img[:, :, 1] = fdata['images'][i] - img_mean
if i - 1 >= 0:
img[:, :, 0] = fdata['images'][i - 1] - img_mean
if i + 1 < fdata['images'].shape[0]:
img[:, :, 2] = fdata['images'][i + 1] - img_mean
idx = where(img < 0)
img[idx] = 0
img = img.astype('uint8')
video_out.write(img)
video_out.release()
def stats_from_chunk(reference, sigma=6):
"""Returns mean, var, and threshold (in counts) for reference. The mean
and var are calculated after omitting the largest and smallest values
found for each pixel, which assumes few particles in the laser
sheet. The threshold statistic corresponds to the specified sigma level.
Adding 0.5 to var helps compensate for digitization error so that false
positives in the light sheet approximately match that outside the light
sheet. The threshold for pixels defined by 'mask' are reset to 4095, which
ensures they won't contribute to hits."""
from lcp_video.procedures.analysis_functions import poisson_array, dm16_mask, images_from_file, save_to_file
from numpy import sqrt, ceil, cast, array, zeros_like
from time import time
from os import path
t0 = time()
if '.data.' in reference:
stats_name = reference.replace('.data.hdf5', '.stats.pkl')
if '.raw.' in reference:
stats_name = reference.replace('.raw.hdf5', '.stats.pkl')
if path.exists(stats_name):
stats = load_from_file(stats_name)
median = stats['median']
mean = stats['mean']
var = stats['var']
threshold = stats['threshold']
else:
print('Processing {} ... please wait'.format(reference))
# Load images and sort in place to minmimze memory footprint
images = images_from_file(reference)
images.sort(axis=0)
mask = zeros_like(images[0])
if 'dm16' in reference: mask = dm16_mask()
# Compute median, then mean and var after omitting smallest and largest values.
N = len(images)
M = int(N / 2)
median = images[M]
mean = images[1:-1].mean(axis=0, dtype='float32')
var = images[1:-1].var(axis=0, dtype='float32')
# Compute std_ratio; used to rescale stdev.
std_ratio = []
for i in range(10000):
dist = poisson_array(3, N, sort=True)
std_ratio.append(dist.std() / dist[1:-1].std())
std_ratio_mean = array(std_ratio).mean()
# Compute threshold to nearest larger integer; recast as int16.
threshold = ceil(mean + sigma * std_ratio_mean * sqrt(var + 0.5))
threshold = cast['int16'](threshold) - median
threshold[mask] = 4095
save_to_file(stats_name, {
'median': median,
'mean': mean,
'var': var,
'threshold': threshold
})
print('time to execute stats_from_chunk [s]: {:0.1f}'.format(time() - t0))
return median, mean, var, threshold
def find_spots_sigma(filename_source,
filename_destination=None,
ref_dic=None,
roi_mask=None):
"""
"""
from numpy import zeros_like, rint, sqrt, maximum, where, ones
from h5py import File
from time import ctime, time
from skimage import measure
from ubcs_auxiliary.save_load_object import load_from_file
ref_dic = load_from_file(ref_dic)
if roi_mask is None:
roi_mask = ones((3000, 4096))
with File(filename_source, 'r') as fs:
if filename_destination == None:
filename_destination = filename_source.replace(
'.gzip.hdf5', '.mask1.hdf5')
with File(filename_destination, 'a') as fd:
length, width, height = fs['images'].shape
fd.create_dataset('pixel_mask', (length, 3000, 4096),
dtype='int8',
chunks=(1, 3000, 4096),
compression='gzip')
fd.create_dataset('pixel_hits', (length, ),
dtype='int32',
compression='gzip')
fd.create_dataset('particle_mask', (length, 3000, 4096),
dtype='int16',
chunks=(1, 3000, 4096),
compression='gzip')
fd.create_dataset('particle_hits', (length, ),
dtype='int16',
compression='gzip')
M1 = (ref_dic['S1'] - ref_dic['Imax1'] -
ref_dic['Imin']) / (ref_dic['N'] - 2)
sigma = sqrt(ref_dic['S2'] / ref_dic['N'] - M1**2)
for i in range(length):
image = fs['images'][i]
mask = zeros_like(image)
for j in range(2, 7):
if j == 2:
coeff = 2
else:
coeff = 1
mask += (((image - M1) * roi_mask) > j * sigma) * coeff
idx = where(mask == 6)
fd['pixel_mask'][i] = mask
enummask = measure.label(mask == 6)
fd['particle_mask'][i] = enummask
fd['particle_hits'][i] = enummask.max()
fd['pixel_hits'][i] = mask[idx].sum()
def generate_mask_from_stats(root, filename):
"""
Generates a binary mask from a .stats. file
image modification:
- sqrt (just take a square root of images)
- log10 (take log base 10 of images)
Parameters
----------
root :: (string)
filename :: (string)
Returns
-------
Examples
--------
>>> from ubcs_auxiliary.multiprocessing import new_child_process
>>> new_child_process(generate_mask_from_stats,'/mnt/data/', dmout_breathing_1.stats.pkl')
"""
from ubcs_auxiliary.save_load_object import load_from_file
from h5py import File
from numpy import zeros, where
from time import time, ctime
print(
f"{ctime(time())} Start creating {root+filename.split('.')[0]+'.mask.hdf5'} file"
)
stats = load_from_file(root + filename)
fdata = File(root + filename.split('.')[0] + '.data.hdf5', 'r')
mask = zeros(fdata['images'].shape, dtype='bool')
zhit_dic = Zhit(stats)
hit_mask = zhit_dic[4] * stats['dmax_frame']
for i in range(600):
mask[i] = (hit_mask == i)
print(f'{ctime(time())} writing .mask.hdf5')
with File(root + filename.split('.')[0] + '.mask.hdf5', 'a') as fmask:
fmask.create_dataset('masks',
data=mask,
dtype='bool',
compression='gzip',
compression_opts=9)
print(f'{ctime(time())} Done')
def zfs_light_data_hdf5(dark_zfs_filename, light_filename, clip=2):
"""
"""
from lcp_video.analysis import zinger_free_statistics
from ubcs_auxiliary.save_load_object import load_from_file, save_to_file
from numpy import ndarray
from h5py import File
print(f'Getting DarkData from {dark_zfs_filename}')
print(f'Analysing LightData from {light_filename}')
filename = light_filename.split(',')[0] + '.light_zfs'
print(f'ZFS LightData will be saved to {filename}')
dmean = load_from_file(dark_zfs_filename)['M1']
with File(light_filename, 'r') as f:
dic = zinger_free_statistics(f['images'], Dmean=dmean, clip=clip)
save_to_file(filename, dic)
filename_zfs = filename.split('.hdf5')[0] + '.light_zfs.hdf5'
with File(filename_zfs, 'a') as f_new:
for key in list(dic.keys()):
data = dic[key]
if type(data) is ndarray:
f_new.create_dataset(key, data=data, compression='gzip')
else:
f_new.create_dataset(key, data=data)
print(f'analysis of {light_filename} is done')
def calculate_spots_from_mask(data_filename, mask_filename, stats_filename,
ffcorr_filename):
"""
creates a catalog(table) of unique particles according to the supplied mask. The
"""
from time import time, ctime
import h5py
from ubcs_auxiliary.save_load_object import load_from_file
import numpy as np
import cv2
import os
from lcp_video.analysis import grow_mask, get_array_piece
temp_arr = np.zeros((21, ))
d_file = h5py.File(data_filename, 'r')
m_file = h5py.File(mask_filename, 'r')
stats_data = load_from_file(stats_filename)
M1 = (stats_data['S1'] - stats_data['Imax1'] -
stats_data['Imin']) / (stats_data['N'] - 2)
FFcorr = load_from_file(ffcorr_filename)['FF_corr']
N_spots = np.sum(m_file['particle_hits'])
head, tail = os.path.split(data_filename)
destination_filename = os.path.join(
head,
tail.split('.')[0] + '.spots.gzip.hdf5')
with h5py.File(destination_filename, 'a') as f_destination:
f_destination.create_dataset('spots', (N_spots, 21),
compression='gzip')
f_destination.create_dataset('spots_images', (N_spots, 31, 31),
compression='gzip')
f_destination.create_dataset('stats file', data=stats_filename)
f_destination.create_dataset('FFcorr file', data=ffcorr_filename)
f_destination.create_dataset('time', data=ctime(time()))
spot_i = 0
for i in range(600):
print(
f'{ctime(time())}: processing image {i} and saving into file {f_destination}'
)
spots_N = m_file['particle_hits'][i]
enummask = m_file['particle_mask'][i]
image = d_file['images'][i] - M1
t = d_file['timestamp'][i]
for j in range(spots_N):
mask = grow_mask((enummask == (j + 1)), 2)
img = image * mask / FFcorr
temp_arr[0] = i
temp_arr[1] = spot_i
temp_arr[2] = t
temp_arr[3] = np.max(img)
temp_arr[4] = np.where(img == temp_arr[3])[1][0]
temp_arr[5] = np.where(img == temp_arr[3])[0][0]
temp_arr[6] = np.sum(mask)
m = cv2.moments(img)
temp_arr[7] = m['m00']
temp_arr[8] = m['m10']
temp_arr[9] = m['m01']
temp_arr[10] = m['m11']
temp_arr[11] = m['m20']
temp_arr[12] = m['m02']
temp_arr[13] = m['m21']
temp_arr[14] = m['m12']
temp_arr[15] = m['m30']
temp_arr[16] = m['m03']
x_mean = m['m10'] / m['m00']
y_mean = m['m01'] / m['m00']
x_var = (m['m20'] / m['m00']) - (m['m10'] / m['m00'])**2
y_var = (m['m02'] / m['m00']) - (m['m01'] / m['m00'])**2
temp_arr[17] = x_mean
temp_arr[18] = y_mean
temp_arr[19] = x_var
temp_arr[20] = y_var
spots_image = get_array_piece(img,
center=(int(x_mean),
int(y_mean)),
radius=15)
f_destination['spots'][spot_i] = temp_arr
f_destination['spots_images'][spot_i] = spots_image
spot_i += 1
def get_test_images(i=0):
from ubcs_auxiliary.save_load_object import load_from_file
filename = './laue_crystallography/tests/images/holes_example1.imgpkl'
img = load_from_file(filename)
return img
from ubcs_auxiliary.save_load_object import load_from_file
from lcp_video.analysis import mono12p_to_image
mono12p = load_from_file('lcp_video/test_data/flir_rawdata_mono12p.pkl')
height = 2048
width = 2448
data = mono12p_to_image(mono12p, height, width)
step1:
convert to (N,8) 1bit arrays
Step2:
flatten the (N,8) array to (N*8,)
Step3:
construct (M,12) array from (N*8,)
Step4. Sum axis 1.
"""
import numpy as np
from ubcs_auxiliary.save_load_object import load_from_file
from matplotlib import pyplot as plt
image = load_from_file('./lcp_video/test_data/flir_image.pkl')
rawdata = load_from_file('./lcp_video/test_data/flir_rawdata.pkl')
def get_mask(length):
from numpy import vstack, tile, hstack, arange
b0 = 2**hstack((arange(4,12,1),arange(4)))
b1 = 2**arange(12)
b = vstack((b0,b1))
bt = tile(b,(int((length/(2*1.5))),1)).astype('uint16')
return bt
def raw_to_image(rawdata, height, witdh, mask):
from numpy import vstack, tile, hstack, arange,reshape
data_Nx8 = ((rawdata.reshape((-1,1)) & (2**arange(8))) != 0)
data_N8x1 = data_Nx8.flatten()
data_Mx12 = data_N8x1.reshape((int(rawdata.shape[0]/1.5),12))
def read_dict(filename):
from ubcs_auxiliary.save_load_object import load_from_file
return load_from_file(filename)
def load_buffer(filename):
from ubcs_auxiliary.save_load_object import load_from_file
data = load_from_file(filename)
if __name__ == '__main__':
# The Record Name is specified by prefix
prefix = 'microfludics'
from pdb import pm
from tempfile import gettempdir
import logging
logging.basicConfig(filename=gettempdir() + '/{}.log'.format(prefix),
level=logging.DEBUG,
format="%(asctime)s %(levelname)s: %(message)s")
from tempfile import gettempdir
#For testing
from numpy import arange
from matplotlib import pyplot as plt
from ubcs_auxiliary.save_load_object import load_from_file
root = '/Users/femto-13/microfluidics_data/puck5/'
bckg = load_from_file(root + 'bck_19.imgpkl').astype('float64')
for i in range(19):
bckg += load_from_file(root + f'bck_{i}.imgpkl').astype('float64')
bckg = bckg.T / 20.0
img1 = load_from_file(root + 'image_15.imgpkl').astype('float64').T
data = -1 * (img1 - bckg)
mask = analyse_array(data[:, :, 0] + data[:, :, 1] + data[:, :, 2],
threshold=10,
N=[1, 2, 3])
plot_image_with_sum(mask[0][0].astype('int16'))
plot_image_with_sum(mask[3][0])