def once(lst):
import os
from time import time, ctime
from h5py import File
from numpy import copy
with File(source + lst[0], 'r') as f:
width = f['image width'][()]
height = f['image height'][()]
length = 1
pixel_format = f['pixel format'][()]
for filename in lst:
prefix = filename.split('.raw.hdf5')[0]
print(ctime(time()), f'moving file....')
print(f'from{source+filename}')
print(f'to {destination+prefix}.tmpdata.hdf5')
t1 = time()
with File(source + filename, 'r') as f:
width = f['image width'][()]
height = f['image height'][()]
length = 1
with File(destination + prefix + '.tmpdata.hdf5', 'a') as fnew:
for key in list(f.keys()):
data = f[key]
if key == 'images':
if compression:
fnew.create_dataset(
key, (data.shape[0], height, width),
compression='gzip',
chunks=(1, height / 8, width / 8),
dtype='int16')
else:
fnew.create_dataset(
key, (data.shape[0], height, width),
chunks=(1, height / 8, width / 8),
dtype='int16')
for i in range(data.shape[0]):
if pixel_format == 'mono12p':
fnew['images'][i] = mono12p_to_image(
data[i], height, width).reshape(
(height, width))
elif pixel_format == 'mono12p_16':
fnew['images'][i] = data[i].reshape(
(height, width))
else:
fnew.create_dataset(key, data=data)
timestamp = f['timestamps_lab'][0]
t2 = time()
print(
ctime(time()),
f'time: {t2-t1} with size {os.path.getsize(source+filename)/(1024*1024)}, speed {os.path.getsize(source+filename)/((t2-t1)*1024*1024)} MB/s'
)
print(f'removing file {source+filename}')
print(f'changing {destination+prefix}.tmpdata.hdf5')
os.utime(destination + prefix + '.tmpdata.hdf5',
(timestamp, timestamp))
os.rename(destination + prefix + '.tmpdata.hdf5',
destination + prefix + '.data.hdf5')
os.remove(source + filename)
def profile_mono12p_to_image():
from ubcs_auxiliary.save_load_object import load_from_file
from lcp_video.analysis import mono12p_to_image
mono12p = load_from_file('lcp_video/test_data/flir_rawdata_mono12p.pkl')
height = 2048
width = 2448
data = mono12p_to_image(mono12p, height, width)
def convert_raw_to_img(src_filename, dst_filename, verbose=False, numba=False):
"""
converts .raw. data files to .img. data files preserving all original fields. The function will create a new file with the same name but different suffix
"""
if verbose:
print(f'source filename: {src_filename}')
print(f'destination filename: {dst_filename}')
from h5py import File
from lcp_video.analysis import mono12p_to_image, mono12p_to_image_numba
if numba:
mono12p_to_image = mono12p_to_image_numba
src = File(src_filename, 'r')
width = src['image width'][()]
height = src['image height'][()]
length = src['images'].shape[0]
with File(dst_filename, 'w') as dst:
for key in src.keys():
if "images" not in key:
dst.create_dataset(key, data=src[key][()])
else:
dst.create_dataset('images', (length, height, width),
dtype='int16',
chunks=(1, height, width))
for i in range(length):
raw = src['images'][i]
dst['images'][i] = mono12p_to_image(raw, height, width).reshape(
(height, width))
def convert_raw_to_image(self, rawdata):
"""
"""
from lcp_video.analysis import mono12p_to_image, mono12packed_to_image
from numpy import right_shift
rawdata = rawdata[:self.img_len]
print(f'rawdata = {rawdata.dtype}')
if self.pixel_format == 'mono12p':
image = mono12p_to_image(rawdata=rawdata,
height=self.height,
width=self.width)
elif self.pixel_format == 'mono12packed':
mask = self.conversion_mask
mask8 = self.conversion_mask8
image = mono12packe_to_image(rawdata=rawdata,
height=self.height,
width=self.width,
mask=mask,
mask8=mask8)
elif self.pixel_format == 'mono16':
image = right_shift(rawdata, 4).reshape((self.height, self.width))
elif self.pixel_format == 'mono12p_16':
image = rawdata.reshape((self.height, self.width))
return image
def convert_raw_to_image(raw, pixel_format, height, width):
"""
"""
from lcp_video.analysis import mono12p_to_image
from numpy import zeros, empty
if pixel_format == 'mono12p_16':
images = raw.reshape((height, width))
elif pixel_format == 'mono12p':
images = mono12p_to_image(raw, width=width, height=height).reshape(
(height, width))
return image
def test_mono12p_to_image():
"""
"""
from lcp_video import analysis
from ubcs_auxiliary.save_load_object import load_from_file
height = 2048
width = 2448
raw = load_from_file('lcp_video/test_data/flir_mono12p_rawdata.pkl')
image_original = load_from_file(
'lcp_video/test_data/flir_mono12p_image.pkl')
image_reconstruct = analysis.mono12p_to_image(raw, height, width).reshape(
(2048, 2448))
assert (image_reconstruct == image_original).all()
def convert_raw_to_images(raw, pixel_format, length, height, width):
"""
"""
from lcp_video.analysis import mono12p_to_image
from numpy import zeros, empty
if pixel_format == 'mono12p_16':
images = raw[()].reshape((length, height, width))
elif pixel_format == 'mono12p':
images = empty((length, height, width))
for i in range(length):
images[i] = mono12p_to_image(raw[i], width=width,
height=height).reshape(
(height, width))
return images
def benchmark_u12_to_16():
#mono12packed_to_image(rawdata, height, width, mask)
from ubcs_auxiliary.save_load_object import load_from_file
from time import time
import timeit
from numpy import array, ones, uint8
preparation = ""
testcode = 'data = u12_to_16(rawdata, height=height, width = width)'
preparation += "from ubcs_auxiliary.save_load_object import load_from_file;\n"
preparation += 'from numpy import vstack, tile, hstack, arange,reshape, vstack, tile, hstack, arange,reshape, packbits, reshape, int16, concatenate, array, ones, uint8;\n'
preparation += "from lcp_video.analysis import u12_to_16;\n"
preparation += f"height = 3000; width = 4096;\n"
preparation += "rawdata = load_from_file('lcp_video/test_data/mono12p_dataset_12Mpixels.pkl');\n "
t_full = timeit.Timer(testcode, preparation)
print("Preparation")
print(preparation)
print('tested code')
print(
" arr = rawdata.reshape(-1,3) \n byte_even = arr[:,0]+256*(bitwise_and(arr[:,1],15)) \n \
byte_odd = right_shift(bitwise_and(arr[:,1],240),4) + right_shift(256*arr[:,2],4) \n \
img = empty(height*width,dtype='int16') \n \
img[0::2] = byte_even\n \
img[1::2] = byte_odd\n \
return img.reshape(height,width) \n")
temp = array(t_full.repeat(4, 100)) / 100
print(
f"Full function: {round(temp.mean(),3)} +- {round(temp.std(),3)}, with min {round(temp.min(),3)} and max {round(temp.max(),3)}"
)
from numpy import vstack, tile, hstack, arange, reshape, vstack, tile, hstack, arange, reshape, packbits, reshape, int16, concatenate, array, ones, uint8
from ubcs_auxiliary.save_load_object import load_from_file
height = 2048
width = 2448
from lcp_video.analysis import mono12p_to_image
rawdata = load_from_file('lcp_video/test_data/flir_mono12p_rawdata.pkl')
img = load_from_file('lcp_video/test_data/flir_mono12p_image.pkl')
data = mono12p_to_image(rawdata, height=height, width=width)
print((data.reshape(height, width) == img).any())
from matplotlib import pyplot as plt
plt.figure()
plt.imshow(img)
plt.figure()
plt.imshow(data.reshape(height, width))
def run_once(self):
from time import time
from numpy import zeros, array
from lcp_video.analysis import mono12p_to_image
if self.acquiring:
raw = self.get_image().reshape(1, self.img_len + 4096)
self.queue.enqueue(raw)
if not self.recording and self.calc_on_the_fly:
self.last_reshaped_image = mono12p_to_image(
raw[0, :self.img_len], self.height, self.width).reshape(
(self.height, self.width))
hits = ((self.last_reshaped_image >
(self.image_threshold + self.image_median)) *
~self.mask).sum()
arr = zeros((1, 2))
arr[0, 0] = time()
arr[0, 1] = hits
from EPICS_CA.CAServer import casput
casput(f'{self.name.upper()}_CAMERA:HITS.RBV', hits)
self.hits_buffer.append(arr)
def get_image(self):
from lcp_video.analysis import mono12p_to_image
self.last_raw_image *= 0
if self.acquiring:
image_result = self.cam.GetNextImage()
timestamp = image_result.GetTimeStamp()
frameid = image_result.GetFrameID()
info(f'get : {timestamp}, {frameid}')
if (self.last_frameID != -1) and (
(frameid - self.last_frameID) != 1):
missed = frameid - self.last_frameID
self.num_of_missed_frames += missed
info(
f'missed {missed} frames. {self.queue.global_rear}. Current frame ID {frameid}, last frame ID {self.last_frameID} '
)
self.last_frameID = frameid
# Getting the image data as a numpy array
image_res = image_result.GetData()
if self.pixel_format == 'mono12p_16':
image_data = mono12p_to_image(image_res, self.height,
self.width)
else:
image_data = image_res
image_result.Release()
else:
info('No Data in get image')
image_data = zeros((self.height * self.width, ))
pointer = self.img_len
self.last_raw_image[:pointer] = image_data
self.last_raw_image[pointer:pointer + 64] = self.get_image_header(
value=int(time() * 1000000), length=64)
self.last_raw_image[pointer + 64:pointer +
128] = self.get_image_header(value=timestamp,
length=64)
self.last_raw_image[pointer + 128:pointer +
192] = self.get_image_header(value=frameid,
length=64)
return self.last_raw_image
from ubcs_auxiliary.save_load_object import load_from_file
from lcp_video.analysis import mono12p_to_image
mono12p = load_from_file('lcp_video/test_data/flir_rawdata_mono12p.pkl')
height = 2048
width = 2448
data = mono12p_to_image(mono12p, height, width)
def generate_mono_video(filename,
verbose=False,
scale='log',
rotate=0,
fliplr=False,
N=3e9):
"""
generates video from a set of raw.hdf5 files specified by filename.
"""
import h5py
import cv2
import numpy as np
from time import ctime, time
import os
root, camera_name, dataset_name, chunk, extension = split_raw_filename(
filename)
if scale == 'linear':
video_pathname = os.path.join(
root, camera_name + '_' + dataset_name + '.linear_mono.mp4')
elif scale == 'log':
video_pathname = os.path.join(
root, camera_name + '_' + dataset_name + '.log_mono.mp4')
fourcc = cv2.VideoWriter_fourcc(*'DIVX')
f = h5py.File(filename, 'r')
fps = 10**6 / f['exposure time'][()]
width = f['image width'][()]
height = f['image height'][()]
if rotate % 2 == 0:
video = cv2.VideoWriter(video_pathname, fourcc, fps, (width, height))
img = np.rot90(np.zeros((height, width, 3), dtype='uint8'), rotate)
else:
video = cv2.VideoWriter(video_pathname, fourcc, fps, (height, width))
img = np.zeros((width, height, 3), dtype='uint8')
from lcp_video.analysis import mono12p_to_image
pathnames = list_dataset(filename)
length = len(pathnames) * 256
print(f"number of frames to analyze {length}")
j = 0
t1 = time()
for pathname in pathnames:
with h5py.File(pathname, 'r') as f:
i = 0
while i < f['images'].shape[0]:
if verbose:
print(
f"{ctime(time())}: converting frame with ID = {f['frameIDs'][i]}"
)
if scale == 'linear':
green = ((np.rot90(
mono12p_to_image(
f['images'][i], height, width, reshape=True),
rotate) / 4095) * 255).astype('uint8')
elif scale == 'log':
green = (np.log2(
np.rot90(
mono12p_to_image(
f['images'][i], height, width, reshape=True),
rotate)) * 255 / 12).astype('uint8')
frameID = f['frameIDs'][i]
img = img * 0
img[:, :, 1] = green
# font
font = cv2.FONT_HERSHEY_SIMPLEX
org = (50, 50)
fontScale = 1
color = (255, 0, 0)
thickness = 2
image = cv2.putText(
img,
f'{dataset_name} {ctime(f["timestamps_lab"][()][i])} frame# = {j}',
org, font, fontScale, color, thickness, cv2.LINE_AA)
video.write(img)
j += 1
i += 1
if j > N:
break
if j % 10000 == 0:
t2 = time()
dt = t2 - t1
print(ctime(time()))
print(
f'last 10000 frames took {dt} seconds, with {dt/10000} seconds per frame'
)
print(f'-----------------------------------')
t1 = time()
if j > N:
break
cv2.destroyAllWindows()
video.release()
if verbose:
print('done')
def generate_rgb_video(filename, verbose=False, scale='log', N=3e9):
"""
generates video from raw files with "RGB" encoding of "previous/current/next frames"
"""
import h5py
import cv2
import numpy as np
from time import ctime, time
root, camera_name, dataset_name, chunk, extension = split_raw_filename(
filename)
video_pathname = os.path.join(
root, camera_name + '_' + dataset_name + '.rgb.mp4')
fourcc = cv2.VideoWriter_fourcc(*'DIVX')
fps = 15.67
f = h5py.File(filename, 'r')
width = f['image width'][()]
height = f['image height'][()]
video = cv2.VideoWriter(video_pathname, fourcc, fps, (height, width))
img = np.zeros((width, height, 3), dtype='uint8')
from lcp_video.analysis import mono12p_to_image
pathnames = list_dataset(filename)
for pathname in pathnames:
with h5py.File(pathname, 'r') as f:
for i in range(f['images'].shape[0]):
if verbose:
print(
f"{ctime(time())}: converting frame with ID = {f['frameIDs'][i]}"
)
green = ((np.fliplr(
np.rot90(
mono12p_to_image(
f['images'][i], height, width, reshape=True), 1)) /
4095) * 255).astype('uint8')
if i > 0:
red = ((np.fliplr(
np.rot90(
mono12p_to_image(f['images'][i - 1],
height,
width,
reshape=True), 1)) / 4095) *
255).astype('uint8')
else:
red = green * 0
if i < 255:
blue = ((np.fliplr(
np.rot90(
mono12p_to_image(f['images'][i - 1],
height,
width,
reshape=True), 1)) / 4095) *
255).astype('uint8')
else:
blue = green * 0
img[:, :, 0] = red
img[:, :, 1] = green
img[:, :, 2] = blue
video.write(img)
if i > N:
break
cv2.destroyAllWindows()
video.release()
if verbose:
print('done')
def plot_image(self):
import io
from matplotlib.figure import Figure
from matplotlib import pyplot as plt
from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg
from scipy import stats
from numpy import nonzero, zeros,nan, ones, argwhere, mean, nanmean
from lcp_video.analysis import mono12p_to_image
fig = Figure(figsize=(6,6))
grid = plt.GridSpec(3, 3, hspace=0.025, wspace=0.025)
t1 = time()
if self.camera is not None:
print('background corrected')
if self.camera.pixel_format == 'mono16':
img = self.camera.convert_raw_to_image(self.get_image())
elif self.camera.pixel_format == 'mono12':
img = mono12p_to_image(self.get_image()[:self.camera.img_len], camera.height, camera.width)
img = img.astype('float64')
image = img-self.camera.image_median
else:
image = self.get_image()
roi_row_s = self.roi_row_s
roi_row_e = self.roi_row_e
roi_col_s = self.roi_col_s
roi_col_e = self.roi_col_e
img = image[roi_row_s:roi_row_e,roi_col_s:roi_col_e]
bckg = (img[:,0:5].mean() + img[:,-5:-1].mean() + img[0:5,:].mean() + img[-5:-1,:].mean())/4
ax1 = fig.add_subplot(grid[0:2,0:2])
ax1.imshow(img.astype('int16'))
vrow = img.sum(axis = 1)
y = arange(0,vrow.shape[0])
axv = fig.add_subplot(grid[0:2,2], sharey = ax1)
axv.plot(vrow,y)
axh = fig.add_subplot(grid[2,0:2], sharex = ax1 )
vcol = img.sum(axis = 0)
x = arange(0,vcol.shape[0])
axh.plot(x,vcol)
if (img > 4050).any():
satur_flag = True
else:
satur_flag = False
s2 = (img*img).sum()
import numpy as np
self.time_series.append(s2)
if len(self.time_series) > 100:
self.time_series.pop(0)
self.time_series_min = np.min(np.array(self.time_series))
self.time_series_max = np.max(np.array(self.time_series))
ax3 = fig.add_subplot(grid[2,2] )
ax3.plot(self.time_series)
ax3.axhline(self.time_series_max)
ax3.axhline(self.time_series_min)
from lcp_video.analysis import get_moments
img = img.astype('float64')
m = get_moments(img-bckg)
idx = where(img == img.max())
m['max_x'] = idx[1][0] + roi_col_s
m['max_y'] = idx[0][0] + roi_row_s
m['max_pixel'] = img.max()
self.update_moments(m = m)
fig.tight_layout()
buf = io.BytesIO()
fig.savefig(buf, format='jpg')
buf.seek(0)
return buf