def once(lst): import os from time import time, ctime from h5py import File from numpy import copy with File(source + lst[0], 'r') as f: width = f['image width'][()] height = f['image height'][()] length = 1 pixel_format = f['pixel format'][()] for filename in lst: prefix = filename.split('.raw.hdf5')[0] print(ctime(time()), f'moving file....') print(f'from{source+filename}') print(f'to {destination+prefix}.tmpdata.hdf5') t1 = time() with File(source + filename, 'r') as f: width = f['image width'][()] height = f['image height'][()] length = 1 with File(destination + prefix + '.tmpdata.hdf5', 'a') as fnew: for key in list(f.keys()): data = f[key] if key == 'images': if compression: fnew.create_dataset( key, (data.shape[0], height, width), compression='gzip', chunks=(1, height / 8, width / 8), dtype='int16') else: fnew.create_dataset( key, (data.shape[0], height, width), chunks=(1, height / 8, width / 8), dtype='int16') for i in range(data.shape[0]): if pixel_format == 'mono12p': fnew['images'][i] = mono12p_to_image( data[i], height, width).reshape( (height, width)) elif pixel_format == 'mono12p_16': fnew['images'][i] = data[i].reshape( (height, width)) else: fnew.create_dataset(key, data=data) timestamp = f['timestamps_lab'][0] t2 = time() print( ctime(time()), f'time: {t2-t1} with size {os.path.getsize(source+filename)/(1024*1024)}, speed {os.path.getsize(source+filename)/((t2-t1)*1024*1024)} MB/s' ) print(f'removing file {source+filename}') print(f'changing {destination+prefix}.tmpdata.hdf5') os.utime(destination + prefix + '.tmpdata.hdf5', (timestamp, timestamp)) os.rename(destination + prefix + '.tmpdata.hdf5', destination + prefix + '.data.hdf5') os.remove(source + filename)
def profile_mono12p_to_image(): from ubcs_auxiliary.save_load_object import load_from_file from lcp_video.analysis import mono12p_to_image mono12p = load_from_file('lcp_video/test_data/flir_rawdata_mono12p.pkl') height = 2048 width = 2448 data = mono12p_to_image(mono12p, height, width)
def convert_raw_to_img(src_filename, dst_filename, verbose=False, numba=False): """ converts .raw. data files to .img. data files preserving all original fields. The function will create a new file with the same name but different suffix """ if verbose: print(f'source filename: {src_filename}') print(f'destination filename: {dst_filename}') from h5py import File from lcp_video.analysis import mono12p_to_image, mono12p_to_image_numba if numba: mono12p_to_image = mono12p_to_image_numba src = File(src_filename, 'r') width = src['image width'][()] height = src['image height'][()] length = src['images'].shape[0] with File(dst_filename, 'w') as dst: for key in src.keys(): if "images" not in key: dst.create_dataset(key, data=src[key][()]) else: dst.create_dataset('images', (length, height, width), dtype='int16', chunks=(1, height, width)) for i in range(length): raw = src['images'][i] dst['images'][i] = mono12p_to_image(raw, height, width).reshape( (height, width))
def convert_raw_to_image(self, rawdata): """ """ from lcp_video.analysis import mono12p_to_image, mono12packed_to_image from numpy import right_shift rawdata = rawdata[:self.img_len] print(f'rawdata = {rawdata.dtype}') if self.pixel_format == 'mono12p': image = mono12p_to_image(rawdata=rawdata, height=self.height, width=self.width) elif self.pixel_format == 'mono12packed': mask = self.conversion_mask mask8 = self.conversion_mask8 image = mono12packe_to_image(rawdata=rawdata, height=self.height, width=self.width, mask=mask, mask8=mask8) elif self.pixel_format == 'mono16': image = right_shift(rawdata, 4).reshape((self.height, self.width)) elif self.pixel_format == 'mono12p_16': image = rawdata.reshape((self.height, self.width)) return image
def convert_raw_to_image(raw, pixel_format, height, width): """ """ from lcp_video.analysis import mono12p_to_image from numpy import zeros, empty if pixel_format == 'mono12p_16': images = raw.reshape((height, width)) elif pixel_format == 'mono12p': images = mono12p_to_image(raw, width=width, height=height).reshape( (height, width)) return image
def test_mono12p_to_image(): """ """ from lcp_video import analysis from ubcs_auxiliary.save_load_object import load_from_file height = 2048 width = 2448 raw = load_from_file('lcp_video/test_data/flir_mono12p_rawdata.pkl') image_original = load_from_file( 'lcp_video/test_data/flir_mono12p_image.pkl') image_reconstruct = analysis.mono12p_to_image(raw, height, width).reshape( (2048, 2448)) assert (image_reconstruct == image_original).all()
def convert_raw_to_images(raw, pixel_format, length, height, width): """ """ from lcp_video.analysis import mono12p_to_image from numpy import zeros, empty if pixel_format == 'mono12p_16': images = raw[()].reshape((length, height, width)) elif pixel_format == 'mono12p': images = empty((length, height, width)) for i in range(length): images[i] = mono12p_to_image(raw[i], width=width, height=height).reshape( (height, width)) return images
def benchmark_u12_to_16(): #mono12packed_to_image(rawdata, height, width, mask) from ubcs_auxiliary.save_load_object import load_from_file from time import time import timeit from numpy import array, ones, uint8 preparation = "" testcode = 'data = u12_to_16(rawdata, height=height, width = width)' preparation += "from ubcs_auxiliary.save_load_object import load_from_file;\n" preparation += 'from numpy import vstack, tile, hstack, arange,reshape, vstack, tile, hstack, arange,reshape, packbits, reshape, int16, concatenate, array, ones, uint8;\n' preparation += "from lcp_video.analysis import u12_to_16;\n" preparation += f"height = 3000; width = 4096;\n" preparation += "rawdata = load_from_file('lcp_video/test_data/mono12p_dataset_12Mpixels.pkl');\n " t_full = timeit.Timer(testcode, preparation) print("Preparation") print(preparation) print('tested code') print( " arr = rawdata.reshape(-1,3) \n byte_even = arr[:,0]+256*(bitwise_and(arr[:,1],15)) \n \ byte_odd = right_shift(bitwise_and(arr[:,1],240),4) + right_shift(256*arr[:,2],4) \n \ img = empty(height*width,dtype='int16') \n \ img[0::2] = byte_even\n \ img[1::2] = byte_odd\n \ return img.reshape(height,width) \n") temp = array(t_full.repeat(4, 100)) / 100 print( f"Full function: {round(temp.mean(),3)} +- {round(temp.std(),3)}, with min {round(temp.min(),3)} and max {round(temp.max(),3)}" ) from numpy import vstack, tile, hstack, arange, reshape, vstack, tile, hstack, arange, reshape, packbits, reshape, int16, concatenate, array, ones, uint8 from ubcs_auxiliary.save_load_object import load_from_file height = 2048 width = 2448 from lcp_video.analysis import mono12p_to_image rawdata = load_from_file('lcp_video/test_data/flir_mono12p_rawdata.pkl') img = load_from_file('lcp_video/test_data/flir_mono12p_image.pkl') data = mono12p_to_image(rawdata, height=height, width=width) print((data.reshape(height, width) == img).any()) from matplotlib import pyplot as plt plt.figure() plt.imshow(img) plt.figure() plt.imshow(data.reshape(height, width))
def run_once(self): from time import time from numpy import zeros, array from lcp_video.analysis import mono12p_to_image if self.acquiring: raw = self.get_image().reshape(1, self.img_len + 4096) self.queue.enqueue(raw) if not self.recording and self.calc_on_the_fly: self.last_reshaped_image = mono12p_to_image( raw[0, :self.img_len], self.height, self.width).reshape( (self.height, self.width)) hits = ((self.last_reshaped_image > (self.image_threshold + self.image_median)) * ~self.mask).sum() arr = zeros((1, 2)) arr[0, 0] = time() arr[0, 1] = hits from EPICS_CA.CAServer import casput casput(f'{self.name.upper()}_CAMERA:HITS.RBV', hits) self.hits_buffer.append(arr)
def get_image(self): from lcp_video.analysis import mono12p_to_image self.last_raw_image *= 0 if self.acquiring: image_result = self.cam.GetNextImage() timestamp = image_result.GetTimeStamp() frameid = image_result.GetFrameID() info(f'get : {timestamp}, {frameid}') if (self.last_frameID != -1) and ( (frameid - self.last_frameID) != 1): missed = frameid - self.last_frameID self.num_of_missed_frames += missed info( f'missed {missed} frames. {self.queue.global_rear}. Current frame ID {frameid}, last frame ID {self.last_frameID} ' ) self.last_frameID = frameid # Getting the image data as a numpy array image_res = image_result.GetData() if self.pixel_format == 'mono12p_16': image_data = mono12p_to_image(image_res, self.height, self.width) else: image_data = image_res image_result.Release() else: info('No Data in get image') image_data = zeros((self.height * self.width, )) pointer = self.img_len self.last_raw_image[:pointer] = image_data self.last_raw_image[pointer:pointer + 64] = self.get_image_header( value=int(time() * 1000000), length=64) self.last_raw_image[pointer + 64:pointer + 128] = self.get_image_header(value=timestamp, length=64) self.last_raw_image[pointer + 128:pointer + 192] = self.get_image_header(value=frameid, length=64) return self.last_raw_image
from ubcs_auxiliary.save_load_object import load_from_file from lcp_video.analysis import mono12p_to_image mono12p = load_from_file('lcp_video/test_data/flir_rawdata_mono12p.pkl') height = 2048 width = 2448 data = mono12p_to_image(mono12p, height, width)
def generate_mono_video(filename, verbose=False, scale='log', rotate=0, fliplr=False, N=3e9): """ generates video from a set of raw.hdf5 files specified by filename. """ import h5py import cv2 import numpy as np from time import ctime, time import os root, camera_name, dataset_name, chunk, extension = split_raw_filename( filename) if scale == 'linear': video_pathname = os.path.join( root, camera_name + '_' + dataset_name + '.linear_mono.mp4') elif scale == 'log': video_pathname = os.path.join( root, camera_name + '_' + dataset_name + '.log_mono.mp4') fourcc = cv2.VideoWriter_fourcc(*'DIVX') f = h5py.File(filename, 'r') fps = 10**6 / f['exposure time'][()] width = f['image width'][()] height = f['image height'][()] if rotate % 2 == 0: video = cv2.VideoWriter(video_pathname, fourcc, fps, (width, height)) img = np.rot90(np.zeros((height, width, 3), dtype='uint8'), rotate) else: video = cv2.VideoWriter(video_pathname, fourcc, fps, (height, width)) img = np.zeros((width, height, 3), dtype='uint8') from lcp_video.analysis import mono12p_to_image pathnames = list_dataset(filename) length = len(pathnames) * 256 print(f"number of frames to analyze {length}") j = 0 t1 = time() for pathname in pathnames: with h5py.File(pathname, 'r') as f: i = 0 while i < f['images'].shape[0]: if verbose: print( f"{ctime(time())}: converting frame with ID = {f['frameIDs'][i]}" ) if scale == 'linear': green = ((np.rot90( mono12p_to_image( f['images'][i], height, width, reshape=True), rotate) / 4095) * 255).astype('uint8') elif scale == 'log': green = (np.log2( np.rot90( mono12p_to_image( f['images'][i], height, width, reshape=True), rotate)) * 255 / 12).astype('uint8') frameID = f['frameIDs'][i] img = img * 0 img[:, :, 1] = green # font font = cv2.FONT_HERSHEY_SIMPLEX org = (50, 50) fontScale = 1 color = (255, 0, 0) thickness = 2 image = cv2.putText( img, f'{dataset_name} {ctime(f["timestamps_lab"][()][i])} frame# = {j}', org, font, fontScale, color, thickness, cv2.LINE_AA) video.write(img) j += 1 i += 1 if j > N: break if j % 10000 == 0: t2 = time() dt = t2 - t1 print(ctime(time())) print( f'last 10000 frames took {dt} seconds, with {dt/10000} seconds per frame' ) print(f'-----------------------------------') t1 = time() if j > N: break cv2.destroyAllWindows() video.release() if verbose: print('done')
def generate_rgb_video(filename, verbose=False, scale='log', N=3e9): """ generates video from raw files with "RGB" encoding of "previous/current/next frames" """ import h5py import cv2 import numpy as np from time import ctime, time root, camera_name, dataset_name, chunk, extension = split_raw_filename( filename) video_pathname = os.path.join( root, camera_name + '_' + dataset_name + '.rgb.mp4') fourcc = cv2.VideoWriter_fourcc(*'DIVX') fps = 15.67 f = h5py.File(filename, 'r') width = f['image width'][()] height = f['image height'][()] video = cv2.VideoWriter(video_pathname, fourcc, fps, (height, width)) img = np.zeros((width, height, 3), dtype='uint8') from lcp_video.analysis import mono12p_to_image pathnames = list_dataset(filename) for pathname in pathnames: with h5py.File(pathname, 'r') as f: for i in range(f['images'].shape[0]): if verbose: print( f"{ctime(time())}: converting frame with ID = {f['frameIDs'][i]}" ) green = ((np.fliplr( np.rot90( mono12p_to_image( f['images'][i], height, width, reshape=True), 1)) / 4095) * 255).astype('uint8') if i > 0: red = ((np.fliplr( np.rot90( mono12p_to_image(f['images'][i - 1], height, width, reshape=True), 1)) / 4095) * 255).astype('uint8') else: red = green * 0 if i < 255: blue = ((np.fliplr( np.rot90( mono12p_to_image(f['images'][i - 1], height, width, reshape=True), 1)) / 4095) * 255).astype('uint8') else: blue = green * 0 img[:, :, 0] = red img[:, :, 1] = green img[:, :, 2] = blue video.write(img) if i > N: break cv2.destroyAllWindows() video.release() if verbose: print('done')
def plot_image(self): import io from matplotlib.figure import Figure from matplotlib import pyplot as plt from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg from scipy import stats from numpy import nonzero, zeros,nan, ones, argwhere, mean, nanmean from lcp_video.analysis import mono12p_to_image fig = Figure(figsize=(6,6)) grid = plt.GridSpec(3, 3, hspace=0.025, wspace=0.025) t1 = time() if self.camera is not None: print('background corrected') if self.camera.pixel_format == 'mono16': img = self.camera.convert_raw_to_image(self.get_image()) elif self.camera.pixel_format == 'mono12': img = mono12p_to_image(self.get_image()[:self.camera.img_len], camera.height, camera.width) img = img.astype('float64') image = img-self.camera.image_median else: image = self.get_image() roi_row_s = self.roi_row_s roi_row_e = self.roi_row_e roi_col_s = self.roi_col_s roi_col_e = self.roi_col_e img = image[roi_row_s:roi_row_e,roi_col_s:roi_col_e] bckg = (img[:,0:5].mean() + img[:,-5:-1].mean() + img[0:5,:].mean() + img[-5:-1,:].mean())/4 ax1 = fig.add_subplot(grid[0:2,0:2]) ax1.imshow(img.astype('int16')) vrow = img.sum(axis = 1) y = arange(0,vrow.shape[0]) axv = fig.add_subplot(grid[0:2,2], sharey = ax1) axv.plot(vrow,y) axh = fig.add_subplot(grid[2,0:2], sharex = ax1 ) vcol = img.sum(axis = 0) x = arange(0,vcol.shape[0]) axh.plot(x,vcol) if (img > 4050).any(): satur_flag = True else: satur_flag = False s2 = (img*img).sum() import numpy as np self.time_series.append(s2) if len(self.time_series) > 100: self.time_series.pop(0) self.time_series_min = np.min(np.array(self.time_series)) self.time_series_max = np.max(np.array(self.time_series)) ax3 = fig.add_subplot(grid[2,2] ) ax3.plot(self.time_series) ax3.axhline(self.time_series_max) ax3.axhline(self.time_series_min) from lcp_video.analysis import get_moments img = img.astype('float64') m = get_moments(img-bckg) idx = where(img == img.max()) m['max_x'] = idx[1][0] + roi_col_s m['max_y'] = idx[0][0] + roi_row_s m['max_pixel'] = img.max() self.update_moments(m = m) fig.tight_layout() buf = io.BytesIO() fig.savefig(buf, format='jpg') buf.seek(0) return buf