def get_avg_imgc( FD, beg=None,end=None,sampling = 100, plot_ = False, bad_frame_list=None,
show_progress=True, *argv,**kwargs):
'''Get average imagef from a data_series by every sampling number to save time'''
#avg_img = np.average(data_series[:: sampling], axis=0)
if beg is None:
beg = FD.beg
if end is None:
end = FD.end
avg_img = FD.rdframe(beg)
n=1
flag=True
if show_progress:
#print( sampling-1 + beg , end, sampling )
if bad_frame_list is None:
bad_frame_list =[]
fra_num = int( (end - beg )/sampling ) - len( bad_frame_list )
for i in tqdm(range( sampling-1 + beg , end, sampling ), desc= 'Averaging %s images'% fra_num):
if bad_frame_list is not None:
if i in bad_frame_list:
flag= False
else:
flag=True
#print(i, flag)
if flag:
(p,v) = FD.rdrawframe(i)
if len(p)>0:
np.ravel(avg_img )[p] += v
n += 1
else:
for i in range( sampling-1 + beg , end, sampling ):
if bad_frame_list is not None:
if i in bad_frame_list:
flag= False
else:
flag=True
if flag:
(p,v) = FD.rdrawframe(i)
if len(p)>0:
np.ravel(avg_img )[p] += v
n += 1
avg_img /= n
if plot_:
if RUN_GUI:
fig = Figure()
ax = fig.add_subplot(111)
else:
fig, ax = plt.subplots()
uid = 'uid'
if 'uid' in kwargs.keys():
uid = kwargs['uid']
im = ax.imshow(avg_img , cmap='viridis',origin='lower',
norm= LogNorm(vmin=0.001, vmax=1e2))
#ax.set_title("Masked Averaged Image")
ax.set_title('uid= %s--Masked-Averaged-Image-'%uid)
fig.colorbar(im)
if save:
#dt =datetime.now()
#CurTime = '%s%02d%02d-%02d%02d-' % (dt.year, dt.month, dt.day,dt.hour,dt.minute)
path = kwargs['path']
if 'uid' in kwargs:
uid = kwargs['uid']
else:
uid = 'uid'
#fp = path + "uid= %s--Waterfall-"%uid + CurTime + '.png'
fp = path + "uid=%s--avg-img-"%uid + '.png'
plt.savefig( fp, dpi=fig.dpi)
#plt.show()
return avg_img
def _readImage(self):
(p,v)=self._readImageRaw()
img = np.zeros( ( self.md['ncols'], self.md['nrows'] ) )
np.put( np.ravel(img), p, v )
return(img)
def rdrawframe(self,n):
x_= np.ravel( self.rdframe(n) )
p= np.where( x_ ) [0]
v = np.array( x_[ p ])
return ( np.array(p, dtype=np.int32), v)
def segment_compress_eigerdata( images, mask, md, filename,
bad_pixel_threshold=1e15, hot_pixel_threshold=2**30,
bad_pixel_low_threshold=0, nobytes=4, bins=1,
N1=None, N2=None, dtypes='images',reverse =True ):
'''
Create a compressed eiger data without header, this function is for parallel compress
for parallel compress don't pass any non-scalar parameters
'''
if dtypes=='uid':
uid= md['uid'] #images
detector = get_detector( db[uid ] )
images = load_data( uid, detector, reverse= reverse )[N1:N2]
print(N1,N2)
Nimg_ = len( images)
M,N = images[0].shape
avg_img = np.zeros( [M,N], dtype= np.float )
Nopix = float( avg_img.size )
n=0
good_count = 0
#frac = 0.0
if nobytes==2:
dtype= np.int16
elif nobytes==4:
dtype= np.int32
elif nobytes==8:
dtype=np.float64
else:
print ( "Wrong type of nobytes, only support 2 [np.int16] or 4 [np.int32]")
dtype= np.int32
#Nimg = Nimg_//bins
Nimg = int( np.ceil( Nimg_ / bins ) )
time_edge = np.array(create_time_slice( N= Nimg_,
slice_num= Nimg, slice_width= bins ))
#print( time_edge, Nimg_, Nimg, bins, N1, N2 )
imgsum = np.zeros( Nimg )
if bins!=1:
print('The frames will be binned by %s'%bins)
fp = open( filename,'wb' )
for n in range(Nimg):
t1,t2 = time_edge[n]
if bins!=1:
img = np.array( np.average( images[t1:t2], axis=0 ) , dtype=np.float64) #dtype=np.int32)
else:
img = np.array( images[t1], dtype=np.int32)
mask &= img < hot_pixel_threshold
p = np.where( (np.ravel(img)>0) * np.ravel(mask) )[0] #don't use masked data
v = np.ravel( np.array( img, dtype= dtype )) [p]
dlen = len(p)
imgsum[n] = v.sum()
if (dlen==0) or (imgsum[n] > bad_pixel_threshold) or (imgsum[n] <=bad_pixel_low_threshold):
dlen = 0
fp.write( struct.pack( '@I', dlen ))
else:
np.ravel( avg_img )[p] += v
good_count +=1
fp.write( struct.pack( '@I', dlen ))
fp.write( struct.pack( '@{}i'.format( dlen), *p))
if bins==1:
fp.write( struct.pack( '@{}{}'.format( dlen,'ih'[nobytes==2]), *v))
else:
fp.write( struct.pack( '@{}{}'.format( dlen,'dd'[nobytes==2] ), *v)) #n +=1
del p,v, img
fp.flush()
fp.close()
avg_img /= good_count
bad_frame_list = (np.array(imgsum) > bad_pixel_threshold) | (np.array(imgsum) <= bad_pixel_low_threshold)
sys.stdout.write('#')
sys.stdout.flush()
#del images, mask, avg_img, imgsum, bad_frame_list
#print( 'Should release memory here')
return mask, avg_img, imgsum, bad_frame_list
def init_compress_eigerdata( images, mask, md, filename,
bad_pixel_threshold=1e15, hot_pixel_threshold=2**30,
bad_pixel_low_threshold=0,nobytes=4, bins=1, with_pickle=True ):
'''
Compress the eiger data
Create a new mask by remove hot_pixel
Do image average
Do each image sum
Find badframe_list for where image sum above bad_pixel_threshold
Generate a compressed data with filename
if bins!=1, will bin the images with bin number as bins
Header contains 1024 bytes ['Magic value', 'beam_center_x', 'beam_center_y', 'count_time', 'detector_distance',
'frame_time', 'incident_wavelength', 'x_pixel_size', 'y_pixel_size',
bytes per pixel (either 2 or 4 (Default)),
Nrows, Ncols, Rows_Begin, Rows_End, Cols_Begin, Cols_End ]
Return
mask
avg_img
imsum
bad_frame_list
'''
fp = open( filename,'wb' )
#Make Header 1024 bytes
#md = images.md
if bins!=1:
nobytes=8
Header = struct.pack('@16s8d7I916x',b'Version-COMP0001',
md['beam_center_x'],md['beam_center_y'], md['count_time'], md['detector_distance'],
md['frame_time'],md['incident_wavelength'], md['x_pixel_size'],md['y_pixel_size'],
nobytes, md['pixel_mask'].shape[1], md['pixel_mask'].shape[0],
0, md['pixel_mask'].shape[1],
0, md['pixel_mask'].shape[0]
)
fp.write( Header)
Nimg_ = len( images)
avg_img = np.zeros_like( images[0], dtype= np.float )
Nopix = float( avg_img.size )
n=0
good_count = 0
frac = 0.0
if nobytes==2:
dtype= np.int16
elif nobytes==4:
dtype= np.int32
elif nobytes==8:
dtype=np.float64
else:
print ( "Wrong type of nobytes, only support 2 [np.int16] or 4 [np.int32]")
dtype= np.int32
Nimg = Nimg_//bins
time_edge = np.array(create_time_slice( N= Nimg_,
slice_num= Nimg, slice_width= bins ))
imgsum = np.zeros( Nimg )
if bins!=1:
print('The frames will be binned by %s'%bins)
for n in tqdm( range(Nimg) ):
t1,t2 = time_edge[n]
img = np.average( images[t1:t2], axis=0 )
mask &= img < hot_pixel_threshold
p = np.where( (np.ravel(img)>0) & np.ravel(mask) )[0] #don't use masked data
v = np.ravel( np.array( img, dtype= dtype )) [p]
dlen = len(p)
imgsum[n] = v.sum()
if (imgsum[n] >bad_pixel_threshold) or (imgsum[n] <=bad_pixel_low_threshold):
#if imgsum[n] >=bad_pixel_threshold :
dlen = 0
fp.write( struct.pack( '@I', dlen ))
else:
np.ravel(avg_img )[p] += v
good_count +=1
frac += dlen/Nopix
#s_fmt ='@I{}i{}{}'.format( dlen,dlen,'ih'[nobytes==2])
fp.write( struct.pack( '@I', dlen ))
fp.write( struct.pack( '@{}i'.format( dlen), *p))
if bins==1:
fp.write( struct.pack( '@{}{}'.format( dlen,'ih'[nobytes==2]), *v))
else:
fp.write( struct.pack( '@{}{}'.format( dlen,'dd'[nobytes==2] ), *v))
#n +=1
fp.close()
frac /=good_count
print( "The fraction of pixel occupied by photon is %6.3f%% "%(100*frac) )
avg_img /= good_count
bad_frame_list = np.where( (np.array(imgsum) > bad_pixel_threshold) | (np.array(imgsum) <= bad_pixel_low_threshold) )[0]
#bad_frame_list1 = np.where( np.array(imgsum) > bad_pixel_threshold )[0]
#bad_frame_list2 = np.where( np.array(imgsum) < bad_pixel_low_threshold )[0]
#bad_frame_list = np.unique( np.concatenate( [bad_frame_list1, bad_frame_list2]) )
if len(bad_frame_list):
print ('Bad frame list are: %s' %bad_frame_list)
else:
print ('No bad frames are involved.')
if with_pickle:
pkl.dump( [mask, avg_img, imgsum, bad_frame_list], open(filename + '.pkl', 'wb' ) )
return mask, avg_img, imgsum, bad_frame_list
def get_avg_imgc(FD,
beg=None,
end=None,
sampling=100,
plot_=False,
bad_frame_list=None,
show_progress=True,
*argv,
**kwargs):
'''Get average imagef from a data_series by every sampling number to save time'''
#avg_img = np.average(data_series[:: sampling], axis=0)
if beg is None:
beg = FD.beg
if end is None:
end = FD.end
avg_img = FD.rdframe(beg)
n = 1
flag = True
if show_progress:
#print( sampling-1 + beg , end, sampling )
if bad_frame_list is None:
bad_frame_list = []
fra_num = int((end - beg) / sampling) - len(bad_frame_list)
for i in tqdm(range(sampling - 1 + beg, end, sampling),
desc='Averaging %s images' % fra_num):
if bad_frame_list is not None:
if i in bad_frame_list:
flag = False
else:
flag = True
#print(i, flag)
if flag:
(p, v) = FD.rdrawframe(i)
if len(p) > 0:
np.ravel(avg_img)[p] += v
n += 1
else:
for i in range(sampling - 1 + beg, end, sampling):
if bad_frame_list is not None:
if i in bad_frame_list:
flag = False
else:
flag = True
if flag:
(p, v) = FD.rdrawframe(i)
if len(p) > 0:
np.ravel(avg_img)[p] += v
n += 1
avg_img /= n
if plot_:
if RUN_GUI:
fig = Figure()
ax = fig.add_subplot(111)
else:
fig, ax = plt.subplots()
uid = 'uid'
if 'uid' in kwargs.keys():
uid = kwargs['uid']
im = ax.imshow(avg_img,
cmap='viridis',
origin='lower',
norm=LogNorm(vmin=0.001, vmax=1e2))
#ax.set_title("Masked Averaged Image")
ax.set_title('uid= %s--Masked-Averaged-Image-' % uid)
fig.colorbar(im)
if save:
#dt =datetime.now()
#CurTime = '%s%02d%02d-%02d%02d-' % (dt.year, dt.month, dt.day,dt.hour,dt.minute)
path = kwargs['path']
if 'uid' in kwargs:
uid = kwargs['uid']
else:
uid = 'uid'
#fp = path + "uid= %s--Waterfall-"%uid + CurTime + '.png'
fp = path + "uid=%s--avg-img-" % uid + '.png'
plt.savefig(fp, dpi=fig.dpi)
#plt.show()
return avg_img
def rdrawframe(self, n):
x_ = np.ravel(self.rdframe(n))
p = np.where(x_)[0]
v = np.array(x_[p])
return (np.array(p, dtype=np.int32), v)
def _readImage(self):
(p, v) = self._readImageRaw()
img = np.zeros((self.md['ncols'], self.md['nrows']))
np.put(np.ravel(img), p, v)
return (img)
def init_compress_eigerdata(images,
mask,
md,
filename,
bad_pixel_threshold=1e15,
hot_pixel_threshold=2**30,
bad_pixel_low_threshold=0,
nobytes=4,
bins=1,
with_pickle=True,
direct_load_data=False,
data_path=None):
'''
Compress the eiger data
Create a new mask by remove hot_pixel
Do image average
Do each image sum
Find badframe_list for where image sum above bad_pixel_threshold
Generate a compressed data with filename
if bins!=1, will bin the images with bin number as bins
Header contains 1024 bytes ['Magic value', 'beam_center_x', 'beam_center_y', 'count_time', 'detector_distance',
'frame_time', 'incident_wavelength', 'x_pixel_size', 'y_pixel_size',
bytes per pixel (either 2 or 4 (Default)),
Nrows, Ncols, Rows_Begin, Rows_End, Cols_Begin, Cols_End ]
Return
mask
avg_img
imsum
bad_frame_list
'''
fp = open(filename, 'wb')
#Make Header 1024 bytes
#md = images.md
if bins != 1:
nobytes = 8
Header = struct.pack('@16s8d7I916x', b'Version-COMP0001',
md['beam_center_x'], md['beam_center_y'],
md['count_time'], md['detector_distance'],
md['frame_time'], md['incident_wavelength'],
md['x_pixel_size'], md['y_pixel_size'], nobytes,
md['pixel_mask'].shape[1], md['pixel_mask'].shape[0],
0, md['pixel_mask'].shape[1], 0,
md['pixel_mask'].shape[0])
fp.write(Header)
Nimg_ = len(images)
avg_img = np.zeros_like(images[0], dtype=np.float)
Nopix = float(avg_img.size)
n = 0
good_count = 0
frac = 0.0
if nobytes == 2:
dtype = np.int16
elif nobytes == 4:
dtype = np.int32
elif nobytes == 8:
dtype = np.float64
else:
print(
"Wrong type of nobytes, only support 2 [np.int16] or 4 [np.int32]")
dtype = np.int32
Nimg = Nimg_ // bins
time_edge = np.array(
create_time_slice(N=Nimg_, slice_num=Nimg, slice_width=bins))
imgsum = np.zeros(Nimg)
if bins != 1:
print('The frames will be binned by %s' % bins)
for n in tqdm(range(Nimg)):
t1, t2 = time_edge[n]
img = np.average(images[t1:t2], axis=0)
mask &= img < hot_pixel_threshold
p = np.where((np.ravel(img) > 0)
& np.ravel(mask))[0] #don't use masked data
v = np.ravel(np.array(img, dtype=dtype))[p]
dlen = len(p)
imgsum[n] = v.sum()
if (imgsum[n] > bad_pixel_threshold) or (imgsum[n] <=
bad_pixel_low_threshold):
#if imgsum[n] >=bad_pixel_threshold :
dlen = 0
fp.write(struct.pack('@I', dlen))
else:
np.ravel(avg_img)[p] += v
good_count += 1
frac += dlen / Nopix
#s_fmt ='@I{}i{}{}'.format( dlen,dlen,'ih'[nobytes==2])
fp.write(struct.pack('@I', dlen))
fp.write(struct.pack('@{}i'.format(dlen), *p))
if bins == 1:
fp.write(
struct.pack('@{}{}'.format(dlen, 'ih'[nobytes == 2]), *v))
else:
fp.write(
struct.pack('@{}{}'.format(dlen, 'dd'[nobytes == 2]), *v))
#n +=1
fp.close()
frac /= good_count
print("The fraction of pixel occupied by photon is %6.3f%% " %
(100 * frac))
avg_img /= good_count
bad_frame_list = np.where((np.array(imgsum) > bad_pixel_threshold) | (
np.array(imgsum) <= bad_pixel_low_threshold))[0]
#bad_frame_list1 = np.where( np.array(imgsum) > bad_pixel_threshold )[0]
#bad_frame_list2 = np.where( np.array(imgsum) < bad_pixel_low_threshold )[0]
#bad_frame_list = np.unique( np.concatenate( [bad_frame_list1, bad_frame_list2]) )
if len(bad_frame_list):
print('Bad frame list are: %s' % bad_frame_list)
else:
print('No bad frames are involved.')
if with_pickle:
pkl.dump([mask, avg_img, imgsum, bad_frame_list],
open(filename + '.pkl', 'wb'))
return mask, avg_img, imgsum, bad_frame_list
def segment_compress_eigerdata(images,
mask,
md,
filename,
bad_pixel_threshold=1e15,
hot_pixel_threshold=2**30,
bad_pixel_low_threshold=0,
nobytes=4,
bins=1,
N1=None,
N2=None,
dtypes='images',
reverse=True,
direct_load_data=False,
data_path=None):
'''
Create a compressed eiger data without header, this function is for parallel compress
for parallel compress don't pass any non-scalar parameters
'''
if dtypes == 'uid':
uid = md['uid'] #images
if not direct_load_data:
detector = get_detector(db[uid])
images = load_data(uid, detector, reverse=reverse)[N1:N2]
else:
images = EigerImages(data_path, md)[N1:N2]
Nimg_ = len(images)
M, N = images[0].shape
avg_img = np.zeros([M, N], dtype=np.float)
Nopix = float(avg_img.size)
n = 0
good_count = 0
#frac = 0.0
if nobytes == 2:
dtype = np.int16
elif nobytes == 4:
dtype = np.int32
elif nobytes == 8:
dtype = np.float64
else:
print(
"Wrong type of nobytes, only support 2 [np.int16] or 4 [np.int32]")
dtype = np.int32
#Nimg = Nimg_//bins
Nimg = int(np.ceil(Nimg_ / bins))
time_edge = np.array(
create_time_slice(N=Nimg_, slice_num=Nimg, slice_width=bins))
#print( time_edge, Nimg_, Nimg, bins, N1, N2 )
imgsum = np.zeros(Nimg)
if bins != 1:
#print('The frames will be binned by %s'%bins)
dtype = np.float64
fp = open(filename, 'wb')
for n in range(Nimg):
t1, t2 = time_edge[n]
if bins != 1:
img = np.array(np.average(images[t1:t2], axis=0), dtype=dtype)
else:
img = np.array(images[t1], dtype=dtype)
mask &= img < hot_pixel_threshold
p = np.where(
(np.ravel(img) > 0) * np.ravel(mask))[0] #don't use masked data
v = np.ravel(np.array(img, dtype=dtype))[p]
dlen = len(p)
imgsum[n] = v.sum()
if (dlen == 0) or (imgsum[n] > bad_pixel_threshold) or (
imgsum[n] <= bad_pixel_low_threshold):
dlen = 0
fp.write(struct.pack('@I', dlen))
else:
np.ravel(avg_img)[p] += v
good_count += 1
fp.write(struct.pack('@I', dlen))
fp.write(struct.pack('@{}i'.format(dlen), *p))
if bins == 1:
fp.write(
struct.pack('@{}{}'.format(dlen, 'ih'[nobytes == 2]), *v))
else:
fp.write(
struct.pack('@{}{}'.format(dlen, 'dd'[nobytes == 2]),
*v)) #n +=1
del p, v, img
fp.flush()
fp.close()
avg_img /= good_count
bad_frame_list = (np.array(imgsum) > bad_pixel_threshold) | (
np.array(imgsum) <= bad_pixel_low_threshold)
sys.stdout.write('#')
sys.stdout.flush()
#del images, mask, avg_img, imgsum, bad_frame_list
#print( 'Should release memory here')
return mask, avg_img, imgsum, bad_frame_list