def cal_c12p( FD, ring_mask, bad_frame_list=None, good_start=0, num_buf = 8, num_lev = None, imgsum=None, norm=None ): '''calculation g2 by using a multi-tau algorithm for a compressed file with parallel calculation ''' FD.beg = max(FD.beg, good_start) noframes = FD.end - FD.beg #+1 # number of frames, not "no frames" for i in range(FD.beg, FD.end): pass_FD(FD,i) if num_lev is None: num_lev = int(np.log( noframes/(num_buf-1))/np.log(2) +1) +1 print ('In this g2 calculation, the buf and lev number are: %s--%s--'%(num_buf,num_lev)) if bad_frame_list is not None: if len(bad_frame_list)!=0: print ('Bad frame involved and will be precessed!') noframes -= len(np.where(np.in1d( bad_frame_list, range(good_start, FD.end)))[0]) print ('%s frames will be processed...'%(noframes)) ring_masks = [ np.array(ring_mask==i, dtype = np.int64) for i in np.unique( ring_mask )[1:] ] qind, pixelist = roi.extract_label_indices( ring_mask ) if norm is not None: norms = [ norm[ np.in1d( pixelist, extract_label_indices( np.array(ring_mask==i, dtype = np.int64))[1])] for i in np.unique( ring_mask )[1:] ] inputs = range( len(ring_masks) ) pool = Pool(processes= len(inputs) ) internal_state = None print( 'Starting assign the tasks...') results = {} if norm is not None: for i in tqdm( inputs ): #for i in inputs: results[i] = apply_async( pool, lazy_two_timep, ( FD, num_lev, num_buf, ring_masks[i], internal_state, bad_frame_list, imgsum, norms[i], ) ) else: #print ('for norm is None') for i in tqdm ( inputs ): #for i in inputs: results[i] = apply_async( pool, lazy_two_timep, ( FD, num_lev, num_buf, ring_masks[i], internal_state, bad_frame_list,imgsum, None, ) ) pool.close() print( 'Starting running the tasks...') res = [ results[k].get() for k in tqdm( list(sorted(results.keys())) ) ] c12 = np.zeros( [ noframes, noframes, len(ring_masks)] ) for i in inputs: #print( res[i][0][:,0].shape, g2.shape ) c12[:,:,i] = res[i][0][0] #[:len_lag, :len_lag] if i==0: lag_steps = res[0][1] print( 'G2 calculation DONE!') del results del res return c12, lag_steps[ lag_steps < noframes ]
def test_rectangles(): shape = (15, 26) roi_data = np.array(([2, 2, 6, 3], [6, 7, 8, 5], [8, 18, 5, 10]), dtype=np.int64) all_roi_inds = roi.rectangles(roi_data, shape) roi_inds, pixel_list = roi.extract_label_indices(all_roi_inds) ty = np.zeros(shape).ravel() ty[pixel_list] = roi_inds num_pixels_m = (np.bincount(ty.astype(int)))[1:] re_mesh = ty.reshape(*shape) for i, (col_coor, row_coor, col_val, row_val) in enumerate(roi_data, 0): ind_co = np.column_stack(np.where(re_mesh == i + 1)) left, right = np.max([col_coor, 0]), np.min([col_coor + col_val, shape[0]]) top, bottom = np.max([row_coor, 0]), np.min([row_coor + row_val, shape[1]]) assert_almost_equal(left, ind_co[0][0]) assert_almost_equal(right-1, ind_co[-1][0]) assert_almost_equal(top, ind_co[0][1]) assert_almost_equal(bottom-1, ind_co[-1][-1])
def auto_two_Array(data, rois, data_pixel=None): ''' Dec 16, 2015, Y.G.@CHX a numpy operation method to get two-time correlation function Parameters: data: images sequence, shape as [img[0], img[1], imgs_length] rois: 2-D array, the interested roi, has the same shape as image, can be rings for saxs, boxes for gisaxs Options: data_pixel: if not None, 2-D array, shape as (len(images), len(qind)), use function Get_Pixel_Array( ).get_data( ) to get Return: g12: a 3-D array, shape as ( imgs_length, imgs_length, q) One example: g12 = auto_two_Array( imgsr, ring_mask, data_pixel = data_pixel ) ''' start_time = time.time() qind, pixelist = roi.extract_label_indices(rois) noqs = len(np.unique(qind)) nopr = np.bincount(qind, minlength=(noqs + 1))[1:] if data_pixel is None: data_pixel = Get_Pixel_Array(data, pixelist).get_data() #print (data_pixel.shape) noframes = data_pixel.shape[0] g12b = np.zeros([noframes, noframes, noqs]) Unitq = (noqs / 10) proi = 0 for qi in tqdm(range(1, noqs + 1)): pixelist_qi = np.where(qind == qi)[0] #print (pixelist_qi.shape, data_pixel[qi].shape) data_pixel_qi = data_pixel[:, pixelist_qi] sum1 = (np.average(data_pixel_qi, axis=1)).reshape(1, noframes) sum2 = sum1.T g12b[:, :, qi - 1] = np.dot( data_pixel_qi, data_pixel_qi.T) / sum1 / sum2 / nopr[qi - 1] #print ( proi, int( qi //( Unitq) ) ) # if int( qi //( Unitq) ) == proi: # sys.stdout.write("#") # sys.stdout.flush() # proi += 1 elapsed_time = time.time() - start_time print('Total time: %.2f min' % (elapsed_time / 60.)) return g12b
def get_pixelist_interp_iq(qp, iq, ring_mask, center): qind, pixelist = roi.extract_label_indices(ring_mask) #pixely = pixelist%FD.md['nrows'] -center[1] #pixelx = pixelist//FD.md['nrows'] - center[0] pixely = pixelist % ring_mask.shape[1] - center[1] pixelx = pixelist // ring_mask.shape[1] - center[0] r = np.hypot(pixelx, pixely) #leave as float. #r= np.int_( np.hypot(pixelx, pixely) +0.5 ) + 0.5 return np.interp(r, qp, iq)
def get_pixelist_interp_iq( qp, iq, ring_mask, center): qind, pixelist = roi.extract_label_indices( ring_mask ) #pixely = pixelist%FD.md['nrows'] -center[1] #pixelx = pixelist//FD.md['nrows'] - center[0] pixely = pixelist%ring_mask.shape[1] -center[1] pixelx = pixelist//ring_mask.shape[1] - center[0] r= np.hypot(pixelx, pixely) #leave as float. #r= np.int_( np.hypot(pixelx, pixely) +0.5 ) + 0.5 return np.interp( r, qp, iq )
def get_max_countc(FD, labeled_array): """Compute the max intensity of ROIs in the compressed file (FD) Parameters ---------- FD: Multifile class compressed file labeled_array : array labeled array; 0 is background. Each ROI is represented by a nonzero integer. It is not required that the ROI labels are contiguous index : int, list, optional The ROI's to use. If None, this function will extract averages for all ROIs Returns ------- max_intensity : a float index : list The labels for each element of the `mean_intensity` list """ qind, pixelist = roi.extract_label_indices(labeled_array) timg = np.zeros(FD.md['ncols'] * FD.md['nrows'], dtype=np.int32) timg[pixelist] = np.arange(1, len(pixelist) + 1) if labeled_array.shape != (FD.md['ncols'], FD.md['nrows']): raise ValueError( " `image` shape (%d, %d) in FD is not equal to the labeled_array shape (%d, %d)" % (FD.md['ncols'], FD.md['nrows'], labeled_array.shape[0], labeled_array.shape[1])) max_inten = 0 for i in tqdm(range(FD.beg, FD.end, 1), desc='Get max intensity of ROIs in all frames'): (p, v) = FD.rdrawframe(i) w = np.where(timg[p])[0] max_inten = max(max_inten, np.max(v[w])) return max_inten
def get_max_countc(FD, labeled_array ): """Compute the max intensity of ROIs in the compressed file (FD) Parameters ---------- FD: Multifile class compressed file labeled_array : array labeled array; 0 is background. Each ROI is represented by a nonzero integer. It is not required that the ROI labels are contiguous index : int, list, optional The ROI's to use. If None, this function will extract averages for all ROIs Returns ------- max_intensity : a float index : list The labels for each element of the `mean_intensity` list """ qind, pixelist = roi.extract_label_indices( labeled_array ) timg = np.zeros( FD.md['ncols'] * FD.md['nrows'] , dtype=np.int32 ) timg[pixelist] = np.arange( 1, len(pixelist) + 1 ) if labeled_array.shape != ( FD.md['ncols'],FD.md['nrows']): raise ValueError( " `image` shape (%d, %d) in FD is not equal to the labeled_array shape (%d, %d)" %( FD.md['ncols'],FD.md['nrows'], labeled_array.shape[0], labeled_array.shape[1]) ) max_inten =0 for i in tqdm(range( FD.beg, FD.end, 1 ), desc= 'Get max intensity of ROIs in all frames' ): (p,v) = FD.rdrawframe(i) w = np.where( timg[p] )[0] max_inten = max( max_inten, np.max(v[w]) ) return max_inten
def test_rectangles(): shape = (15, 26) roi_data = np.array(([2, 2, 6, 3], [6, 7, 8, 5], [8, 18, 5, 10]), dtype=np.int64) all_roi_inds = roi.rectangles(roi_data, shape) roi_inds, pixel_list = roi.extract_label_indices(all_roi_inds) ty = np.zeros(shape).ravel() ty[pixel_list] = roi_inds re_mesh = ty.reshape(*shape) for i, (col_coor, row_coor, col_val, row_val) in enumerate(roi_data, 0): ind_co = np.column_stack(np.where(re_mesh == i + 1)) left, right = np.max([col_coor, 0]), np.min([col_coor + col_val, shape[0]]) top, bottom = np.max([row_coor, 0]), np.min([row_coor + row_val, shape[1]]) assert_almost_equal(left, ind_co[0][0]) assert_almost_equal(right - 1, ind_co[-1][0]) assert_almost_equal(top, ind_co[0][1]) assert_almost_equal(bottom - 1, ind_co[-1][-1])
def auto_two_Arrayc( data_pixel, rois, index=None): ''' Dec 16, 2015, Y.G.@CHX a numpy operation method to get two-time correlation function Parameters: data: images sequence, shape as [img[0], img[1], imgs_length] rois: 2-D array, the interested roi, has the same shape as image, can be rings for saxs, boxes for gisaxs Options: data_pixel: if not None, 2-D array, shape as (len(images), len(qind)), use function Get_Pixel_Array( ).get_data( ) to get Return: g12: a 3-D array, shape as ( imgs_length, imgs_length, q) One example: g12 = auto_two_Array( imgsr, ring_mask, data_pixel = data_pixel ) ''' qind, pixelist = roi.extract_label_indices( rois ) noqs = len( np.unique(qind) ) nopr = np.bincount(qind, minlength=(noqs+1))[1:] noframes = data_pixel.shape[0] if index is None: index = np.arange( 1, noqs + 1 ) else: try: len(index) index = np.array( index ) except TypeError: index = np.array( [index] ) #print( index ) qlist = np.arange( 1, noqs + 1 )[ index -1 ] #print( qlist ) try: g12b = np.zeros( [noframes, noframes, len(qlist) ] ) DO = True except: print("The array is too large. The Sever can't handle such big array. Will calulate different Q sequencely") '''TO be done here ''' DO = False if DO: i = 0 for qi in tqdm(qlist ): #print (qi-1) pixelist_qi = np.where( qind == qi)[0] #print (pixelist_qi.shape, data_pixel[qi].shape) data_pixel_qi = data_pixel[:,pixelist_qi] sum1 = (np.average( data_pixel_qi, axis=1)).reshape( 1, noframes ) sum2 = sum1.T #print( qi, qlist, ) #print( g12b[:,:,qi -1 ] ) g12b[:,:, i ] = np.dot( data_pixel_qi, data_pixel_qi.T) /sum1 / sum2 / nopr[qi -1] i +=1 return g12b
def auto_two_Arrayp( data_pixel, rois, index=None): ''' TODO list will try to use dask Dec 16, 2015, Y.G.@CHX a numpy operation method to get two-time correlation function using parallel computation Parameters: data: images sequence, shape as [img[0], img[1], imgs_length] rois: 2-D array, the interested roi, has the same shape as image, can be rings for saxs, boxes for gisaxs Options: data_pixel: if not None, 2-D array, shape as (len(images), len(qind)), use function Get_Pixel_Array( ).get_data( ) to get Return: g12: a 3-D array, shape as ( imgs_length, imgs_length, q) One example: g12 = auto_two_Array( imgsr, ring_mask, data_pixel = data_pixel ) ''' qind, pixelist = roi.extract_label_indices( rois ) noqs = len( np.unique(qind) ) nopr = np.bincount(qind, minlength=(noqs+1))[1:] noframes = data_pixel.shape[0] g12b = np.zeros( [noframes, noframes, noqs] ) if index is None: index = np.arange( 1, noqs + 1 ) else: try: len(index) index = np.array( index ) except TypeError: index = np.array( [index] ) qlist = np.arange( 1, noqs + 1 )[ index -1 ] inputs = range( len(qlist) ) data_pixel_qis = [0]* len(qlist) for i in inputs: pixelist_qi = np.where( qind == qlist[i] )[0] data_pixel_qis[i] = data_pixel[:,pixelist_qi] #pool = Pool(processes= len(inputs) ) #results = [ apply_async( pool, _get_two_time_for_one_q, ( qlist[i], # data_pixel_qis[i], nopr, noframes ) ) for i in tqdm( inputs ) ] #res = [r.get() for r in results] pool = Pool(processes= len(inputs) ) results = {} for i in inputs: results[i] = pool.apply_async( _get_two_time_for_one_q, [ qlist[i], data_pixel_qis[i], nopr, noframes ] ) pool.close() pool.join() res = np.array( [ results[k].get() for k in list(sorted(results.keys())) ] ) #print('here') for i in inputs: qi=qlist[i] g12b[:,:,qi -1 ] = res[i] print( 'G12 calculation DONE!') return g12b #g12b
def lazy_one_timep(FD, num_levels, num_bufs, labels, internal_state=None, bad_frame_list=None, imgsum=None, norm = None ): if internal_state is None: internal_state = _internal_statep(num_levels, num_bufs, labels) # create a shorthand reference to the results and state named tuple s = internal_state qind, pixelist = roi.extract_label_indices( labels ) # iterate over the images to compute multi-tau correlation fra_pix = np.zeros_like( pixelist, dtype=np.float64) timg = np.zeros( FD.md['ncols'] * FD.md['nrows'] , dtype=np.int32 ) timg[pixelist] = np.arange( 1, len(pixelist) + 1 ) if bad_frame_list is None: bad_frame_list=[] #for i in tqdm(range( FD.beg , FD.end )): for i in range( FD.beg , FD.end ): if i in bad_frame_list: fra_pix[:]= np.nan else: (p,v) = FD.rdrawframe(i) w = np.where( timg[p] )[0] pxlist = timg[ p[w] ] -1 if imgsum is None: if norm is None: #print ('here') fra_pix[ pxlist] = v[w] else: fra_pix[ pxlist] = v[w]/ norm[pxlist] #-1.0 else: if norm is None: fra_pix[ pxlist] = v[w] / imgsum[i] else: fra_pix[ pxlist] = v[w]/ imgsum[i]/ norm[pxlist] level = 0 # increment buffer s.cur[0] = (1 + s.cur[0]) % num_bufs # Put the ROI pixels into the ring buffer. s.buf[0, s.cur[0] - 1] = fra_pix fra_pix[:]=0 #print( i, len(p), len(w), len( pixelist)) #print ('i= %s init fra_pix'%i ) buf_no = s.cur[0] - 1 # Compute the correlations between the first level # (undownsampled) frames. This modifies G, # past_intensity, future_intensity, # and img_per_level in place! #print (s.G) _one_time_processp(s.buf, s.G, s.past_intensity, s.future_intensity, s.label_array, num_bufs, s.num_pixels, s.img_per_level, level, buf_no, s.norm, s.lev_len) #print (s.G) # check whether the number of levels is one, otherwise # continue processing the next level processing = num_levels > 1 level = 1 while processing: if not s.track_level[level]: s.track_level[level] = True processing = False else: prev = (1 + (s.cur[level - 1] - 2) % num_bufs) s.cur[level] = ( 1 + s.cur[level] % num_bufs) s.buf[level, s.cur[level] - 1] = (( s.buf[level - 1, prev - 1] + s.buf[level - 1, s.cur[level - 1] - 1]) / 2) # make the track_level zero once that level is processed s.track_level[level] = False # call processing_func for each multi-tau level greater # than one. This is modifying things in place. See comment # on previous call above. buf_no = s.cur[level] - 1 _one_time_processp(s.buf, s.G, s.past_intensity, s.future_intensity, s.label_array, num_bufs, s.num_pixels, s.img_per_level, level, buf_no, s.norm, s.lev_len) level += 1 # Checking whether there is next level for processing processing = level < num_levels # If any past intensities are zero, then g2 cannot be normalized at # those levels. This if/else code block is basically preventing # divide-by-zero errors. if len(np.where(s.past_intensity == 0)[0]) != 0: g_max = np.where(s.past_intensity == 0)[0][0] else: g_max = s.past_intensity.shape[0] g2 = (s.G[:g_max] / (s.past_intensity[:g_max] * s.future_intensity[:g_max])) #print (FD) #sys.stdout.write('#') #del FD #sys.stdout.flush() #print (g2) #return results(g2, s.lag_steps[:g_max], s) return g2, s.lag_steps[:g_max] #, s
def lazy_one_time(FD, num_levels, num_bufs, labels, internal_state=None, bad_frame_list=None, imgsum=None, norm=None): """Generator implementation of 1-time multi-tau correlation If you do not want multi-tau correlation, set num_levels to 1 and num_bufs to the number of images you wish to correlate The number of bins (of size 1) is one larger than the largest value in `x`. If `minlength` is specified, there will be at least this number of bins in the output array (though it will be longer if necessary, depending on the contents of `x`). Each bin gives the number of occurrences of its index value in `x`. If `weights` is specified the input array is weighted by it, i.e. if a value ``n`` is found at position ``i``, ``out[n] += weight[i]`` instead of ``out[n] += 1``. Parameters ---------- image_iterable : FD, a compressed eiger file by Multifile class num_levels : int how many generations of downsampling to perform, i.e., the depth of the binomial tree of averaged frames num_bufs : int, must be even maximum lag step to compute in each generation of downsampling labels : array Labeled array of the same shape as the image stack. Each ROI is represented by sequential integers starting at one. For example, if you have four ROIs, they must be labeled 1, 2, 3, 4. Background is labeled as 0 internal_state : namedtuple, optional internal_state is a bucket for all of the internal state of the generator. It is part of the `results` object that is yielded from this generator For the sake of normalization: imgsum: a list with the same length as FD, sum of each frame qp, iq: the circular average radius (in pixel) and intensity center: beam center Yields ------ Returns ------- A `results` object is yielded after every image has been processed. This `reults` object contains, in this order: - `g2`: the normalized correlation shape is (len(lag_steps), num_rois) - `lag_steps`: the times at which the correlation was computed - `_internal_state`: all of the internal state. Can be passed back in to `lazy_one_time` as the `internal_state` parameter Notes ----- The normalized intensity-intensity time-autocorrelation function is defined as .. math:: g_2(q, t') = \\frac{<I(q, t)I(q, t + t')> }{<I(q, t)>^2} t' > 0 Here, ``I(q, t)`` refers to the scattering strength at the momentum transfer vector ``q`` in reciprocal space at time ``t``, and the brackets ``<...>`` refer to averages over time ``t``. The quantity ``t'`` denotes the delay time This implementation is based on published work. [1]_ References ---------- .. [1] D. Lumma, L. B. Lurio, S. G. J. Mochrie and M. Sutton, "Area detector based photon correlation in the regime of short data batches: Data reduction for dynamic x-ray scattering," Rev. Sci. Instrum., vol 71, p 3274-3289, 2000. """ if internal_state is None: internal_state = _init_state_one_time(num_levels, num_bufs, labels) # create a shorthand reference to the results and state named tuple s = internal_state qind, pixelist = roi.extract_label_indices(labels) # iterate over the images to compute multi-tau correlation fra_pix = np.zeros_like(pixelist, dtype=np.float64) timg = np.zeros(FD.md['ncols'] * FD.md['nrows'], dtype=np.int32) timg[pixelist] = np.arange(1, len(pixelist) + 1) if bad_frame_list is None: bad_frame_list = [] for i in tqdm(range(FD.beg, FD.end)): if i in bad_frame_list: fra_pix[:] = np.nan else: (p, v) = FD.rdrawframe(i) w = np.where(timg[p])[0] pxlist = timg[p[w]] - 1 if imgsum is None: if norm is None: fra_pix[pxlist] = v[w] else: fra_pix[pxlist] = v[w] / norm[pxlist] #-1.0 else: if norm is None: fra_pix[pxlist] = v[w] / imgsum[i] else: fra_pix[pxlist] = v[w] / imgsum[i] / norm[pxlist] level = 0 # increment buffer s.cur[0] = (1 + s.cur[0]) % num_bufs # Put the ROI pixels into the ring buffer. s.buf[0, s.cur[0] - 1] = fra_pix fra_pix[:] = 0 #print( i, len(p), len(w), len( pixelist)) #print ('i= %s init fra_pix'%i ) buf_no = s.cur[0] - 1 # Compute the correlations between the first level # (undownsampled) frames. This modifies G, # past_intensity, future_intensity, # and img_per_level in place! _one_time_process(s.buf, s.G, s.past_intensity, s.future_intensity, s.label_array, num_bufs, s.num_pixels, s.img_per_level, level, buf_no, s.norm, s.lev_len) # check whether the number of levels is one, otherwise # continue processing the next level processing = num_levels > 1 level = 1 while processing: if not s.track_level[level]: s.track_level[level] = True processing = False else: prev = (1 + (s.cur[level - 1] - 2) % num_bufs) s.cur[level] = (1 + s.cur[level] % num_bufs) s.buf[level, s.cur[level] - 1] = ((s.buf[level - 1, prev - 1] + s.buf[level - 1, s.cur[level - 1] - 1]) / 2) # make the track_level zero once that level is processed s.track_level[level] = False # call processing_func for each multi-tau level greater # than one. This is modifying things in place. See comment # on previous call above. buf_no = s.cur[level] - 1 _one_time_process(s.buf, s.G, s.past_intensity, s.future_intensity, s.label_array, num_bufs, s.num_pixels, s.img_per_level, level, buf_no, s.norm, s.lev_len) level += 1 # Checking whether there is next level for processing processing = level < num_levels # If any past intensities are zero, then g2 cannot be normalized at # those levels. This if/else code block is basically preventing # divide-by-zero errors. if len(np.where(s.past_intensity == 0)[0]) != 0: g_max = np.where(s.past_intensity == 0)[0][0] else: g_max = s.past_intensity.shape[0] g2 = (s.G[:g_max] / (s.past_intensity[:g_max] * s.future_intensity[:g_max])) yield results(g2, s.lag_steps[:g_max], s)
def auto_two_Arrayc(data_pixel, rois, index=None): ''' Dec 16, 2015, Y.G.@CHX a numpy operation method to get two-time correlation function Parameters: data: images sequence, shape as [img[0], img[1], imgs_length] rois: 2-D array, the interested roi, has the same shape as image, can be rings for saxs, boxes for gisaxs Options: data_pixel: if not None, 2-D array, shape as (len(images), len(qind)), use function Get_Pixel_Array( ).get_data( ) to get Return: g12: a 3-D array, shape as ( imgs_length, imgs_length, q) One example: g12 = auto_two_Array( imgsr, ring_mask, data_pixel = data_pixel ) ''' qind, pixelist = roi.extract_label_indices(rois) noqs = len(np.unique(qind)) nopr = np.bincount(qind, minlength=(noqs + 1))[1:] noframes = data_pixel.shape[0] if index is None: index = np.arange(1, noqs + 1) else: try: len(index) index = np.array(index) except TypeError: index = np.array([index]) #print( index ) qlist = np.arange(1, noqs + 1)[index - 1] #print( qlist ) try: g12b = np.zeros([noframes, noframes, len(qlist)]) DO = True except: print( "The array is too large. The Sever can't handle such big array. Will calulate different Q sequencely" ) '''TO be done here ''' DO = False if DO: i = 0 for qi in tqdm(qlist): #print (qi-1) pixelist_qi = np.where(qind == qi)[0] #print (pixelist_qi.shape, data_pixel[qi].shape) data_pixel_qi = data_pixel[:, pixelist_qi] sum1 = (np.average(data_pixel_qi, axis=1)).reshape(1, noframes) sum2 = sum1.T #print( qi, qlist, ) #print( g12b[:,:,qi -1 ] ) g12b[:, :, i] = np.dot( data_pixel_qi, data_pixel_qi.T) / sum1 / sum2 / nopr[qi - 1] i += 1 return g12b
def lazy_two_time(FD, num_levels, num_bufs, labels, two_time_internal_state=None, bad_frame_list=None, imgsum= None, norm = None ): #def lazy_two_time(labels, images, num_frames, num_bufs, num_levels=1, # two_time_internal_state=None): """ Generator implementation of two-time correlation If you do not want multi-tau correlation, set num_levels to 1 and num_bufs to the number of images you wish to correlate Multi-tau correlation uses a scheme to achieve long-time correlations inexpensively by downsampling the data, iteratively combining successive frames. The longest lag time computed is num_levels * num_bufs. ** see comments on multi_tau_auto_corr Parameters ---------- FD: the handler of compressed data num_levels : int, optional how many generations of downsampling to perform, i.e., the depth of the binomial tree of averaged frames default is one num_bufs : int, must be even maximum lag step to compute in each generation of downsampling labels : array labeled array of the same shape as the image stack; each ROI is represented by a distinct label (i.e., integer) two_time_internal_state: None Yields ------ namedtuple A ``results`` object is yielded after every image has been processed. This `reults` object contains, in this order: - ``g2``: the normalized correlation shape is (num_rois, len(lag_steps), len(lag_steps)) - ``lag_steps``: the times at which the correlation was computed - ``_internal_state``: all of the internal state. Can be passed back in to ``lazy_one_time`` as the ``internal_state`` parameter Notes ----- The two-time correlation function is defined as .. math:: C(q,t_1,t_2) = \\frac{<I(q,t_1)I(q,t_2)>}{<I(q, t_1)><I(q,t_2)>} Here, the ensemble averages are performed over many pixels of detector, all having the same ``q`` value. The average time or age is equal to ``(t1+t2)/2``, measured by the distance along the ``t1 = t2`` diagonal. The time difference ``t = |t1 - t2|``, with is distance from the ``t1 = t2`` diagonal in the perpendicular direction. In the equilibrium system, the two-time correlation functions depend only on the time difference ``t``, and hence the two-time correlation contour lines are parallel. References ---------- .. [1] A. Fluerasu, A. Moussaid, A. Mandsen and A. Schofield, "Slow dynamics and aging in collodial gels studied by x-ray photon correlation spectroscopy," Phys. Rev. E., vol 76, p 010401(1-4), 2007. """ num_frames = FD.end - FD.beg if two_time_internal_state is None: two_time_internal_state = _init_state_two_time(num_levels, num_bufs,labels, num_frames) # create a shorthand reference to the results and state named tuple s = two_time_internal_state qind, pixelist = roi.extract_label_indices( labels ) # iterate over the images to compute multi-tau correlation fra_pix = np.zeros_like( pixelist, dtype=np.float64) timg = np.zeros( FD.md['ncols'] * FD.md['nrows'] , dtype=np.int32 ) timg[pixelist] = np.arange( 1, len(pixelist) + 1 ) if bad_frame_list is None: bad_frame_list=[] for i in tqdm(range( FD.beg , FD.end )): if i in bad_frame_list: fra_pix[:]= np.nan else: (p,v) = FD.rdrawframe(i) w = np.where( timg[p] )[0] pxlist = timg[ p[w] ] -1 if imgsum is None: if norm is None: fra_pix[ pxlist] = v[w] else: fra_pix[ pxlist] = v[w]/ norm[pxlist] #-1.0 else: if norm is None: fra_pix[ pxlist] = v[w] / imgsum[i] else: fra_pix[ pxlist] = v[w]/ imgsum[i]/ norm[pxlist] level = 0 # increment buffer s.cur[0] = (1 + s.cur[0]) % num_bufs s.count_level[0] = 1 + s.count_level[0] # get the current image time s = s._replace(current_img_time=(s.current_img_time + 1)) # Put the ROI pixels into the ring buffer. s.buf[0, s.cur[0] - 1] = fra_pix fra_pix[:]=0 _two_time_process(s.buf, s.g2, s.label_array, num_bufs, s.num_pixels, s.img_per_level, s.lag_steps, s.current_img_time, level=0, buf_no=s.cur[0] - 1) # time frame for each level s.time_ind[0].append(s.current_img_time) # check whether the number of levels is one, otherwise # continue processing the next level processing = num_levels > 1 # Compute the correlations for all higher levels. level = 1 while processing: if not s.track_level[level]: s.track_level[level] = 1 processing = False else: prev = 1 + (s.cur[level - 1] - 2) % num_bufs s.cur[level] = 1 + s.cur[level] % num_bufs s.count_level[level] = 1 + s.count_level[level] s.buf[level, s.cur[level] - 1] = ( s.buf[level - 1, prev - 1] + s.buf[level - 1, s.cur[level - 1] - 1] )/2 t1_idx = (s.count_level[level] - 1) * 2 current_img_time = ((s.time_ind[level - 1])[t1_idx] + (s.time_ind[level - 1])[t1_idx + 1])/2. # time frame for each level s.time_ind[level].append(current_img_time) # make the track_level zero once that level is processed s.track_level[level] = 0 # call the _two_time_process function for each multi-tau level # for multi-tau levels greater than one # Again, this is modifying things in place. See comment # on previous call above. _two_time_process(s.buf, s.g2, s.label_array, num_bufs, s.num_pixels, s.img_per_level, s.lag_steps, current_img_time, level=level, buf_no=s.cur[level]-1) level += 1 # Checking whether there is next level for processing processing = level < num_levels #print (s.g2[1,:,1] ) yield s
def test_rings(): center = (100., 100.) img_dim = (200, 205) first_q = 10. delta_q = 5. num_rings = 7 # number of Q rings one_step_q = 5.0 step_q = [2.5, 3.0, 5.8] # test when there is same spacing between rings edges = roi.ring_edges(first_q, width=delta_q, spacing=one_step_q, num_rings=num_rings) print("edges there is same spacing between rings ", edges) label_array = roi.rings(edges, center, img_dim) print("label_array there is same spacing between rings", label_array) label_mask, pixel_list = roi.extract_label_indices(label_array) # number of pixels per ROI num_pixels = np.bincount(label_mask, minlength=(np.max(label_mask)+1)) num_pixels = num_pixels[1:] # test when there is same spacing between rings edges = roi.ring_edges(first_q, width=delta_q, spacing=2.5, num_rings=num_rings) print("edges there is same spacing between rings ", edges) label_array = roi.rings(edges, center, img_dim) print("label_array there is same spacing between rings", label_array) label_mask, pixel_list = roi.extract_label_indices(label_array) # number of pixels per ROI num_pixels = np.bincount(label_mask, minlength=(np.max(label_mask)+1)) num_pixels = num_pixels[1:] # test when there is different spacing between rings edges = roi.ring_edges(first_q, width=delta_q, spacing=step_q, num_rings=4) print("edges when there is different spacing between rings", edges) label_array = roi.rings(edges, center, img_dim) print("label_array there is different spacing between rings", label_array) label_mask, pixel_list = roi.extract_label_indices(label_array) # number of pixels per ROI num_pixels = np.bincount(label_mask, minlength=(np.max(label_mask)+1)) num_pixels = num_pixels[1:] # test when there is no spacing between rings edges = roi.ring_edges(first_q, width=delta_q, num_rings=num_rings) print("edges", edges) label_array = roi.rings(edges, center, img_dim) print("label_array", label_array) label_mask, pixel_list = roi.extract_label_indices(label_array) # number of pixels per ROI num_pixels = np.bincount(label_mask, minlength=(np.max(label_mask)+1)) num_pixels = num_pixels[1:] # Did we draw the right number of rings? print(np.unique(label_array)) actual_num_rings = len(np.unique(label_array)) - 1 assert_equal(actual_num_rings, num_rings) # Does each ring have more pixels than the last, being larger? ring_areas = np.bincount(label_array.ravel())[1:] area_comparison = np.diff(ring_areas) print(area_comparison) areas_monotonically_increasing = np.all(area_comparison > 0) assert_true(areas_monotonically_increasing) # Test various illegal inputs assert_raises(ValueError, lambda: roi.ring_edges(1, 2)) # need num_rings # width incompatible with num_rings assert_raises(ValueError, lambda: roi.ring_edges(1, [1, 2, 3], num_rings=2)) # too few spacings assert_raises(ValueError, lambda: roi.ring_edges(1, [1, 2, 3], [1])) # too many spacings assert_raises(ValueError, lambda: roi.ring_edges(1, [1, 2, 3], [1, 2, 3])) # num_rings conflicts with width, spacing assert_raises(ValueError, lambda: roi.ring_edges(1, [1, 2, 3], [1, 2], 5)) w_edges = [[5, 7], [1, 2]] assert_raises(ValueError, roi.rings, w_edges, center=(4, 4), shape=(20, 20))
def xsvs(image_sets, label_array, number_of_img, timebin_num=2, time_bin=None,only_first_level=False, max_cts=None, bad_images = None, threshold=None): """ This function will provide the probability density of detecting photons for different integration times. The experimental probability density P(K) of detecting photons K is obtained by histogramming the speckle counts over an ensemble of equivalent pixels and over a number of speckle patterns recorded with the same integration time T under the same condition. Parameters ---------- image_sets : array sets of images label_array : array labeled array; 0 is background. Each ROI is represented by a distinct label (i.e., integer). number_of_img : int number of images (how far to go with integration times when finding the time_bin, using skxray.utils.geometric function) timebin_num : int, optional integration time; default is 2 max_cts : int, optional the brightest pixel in any ROI in any image in the image set. defaults to using skxray.core.roi.roi_max_counts to determine the brightest pixel in any of the ROIs bad_images: array, optional the bad images number list, the XSVS will not analyze the binning image groups which involve any bad images threshold: float, optional If one image involves a pixel with intensity above threshold, such image will be considered as a bad image. Returns ------- prob_k_all : array probability density of detecting photons prob_k_std_dev : array standard deviation of probability density of detecting photons Notes ----- These implementation is based on following references References: text [1]_, text [2]_ .. [1] L. Li, P. Kwasniewski, D. Oris, L Wiegart, L. Cristofolini, C. Carona and A. Fluerasu , "Photon statistics and speckle visibility spectroscopy with partially coherent x-rays" J. Synchrotron Rad., vol 21, p 1288-1295, 2014. .. [2] R. Bandyopadhyay, A. S. Gittings, S. S. Suh, P.K. Dixon and D.J. Durian "Speckle-visibilty Spectroscopy: A tool to study time-varying dynamics" Rev. Sci. Instrum. vol 76, p 093110, 2005. There is an example in https://github.com/scikit-xray/scikit-xray-examples It will demonstrate the use of these functions in this module for experimental data. """ if max_cts is None: max_cts = roi.roi_max_counts(image_sets, label_array) # find the label's and pixel indices for ROI's labels, indices = roi.extract_label_indices(label_array) nopixels = len(indices) # number of ROI's u_labels = list(np.unique(labels)) num_roi = len(u_labels) # create integration times if time_bin is None:time_bin = geometric_series(timebin_num, number_of_img) if only_first_level: time_bin = [1] # number of times in the time bin num_times = len(time_bin) # number of pixels per ROI num_pixels = np.bincount(labels, minlength=(num_roi+1))[1:] # probability density of detecting photons prob_k_all = np.zeros([num_times, num_roi], dtype=np.object) # square of probability density of detecting photons prob_k_pow_all = np.zeros_like(prob_k_all) # standard deviation of probability density of detecting photons prob_k_std_dev = np.zeros_like(prob_k_all) # get the bin edges for each time bin for each ROI bin_edges = np.zeros(prob_k_all.shape[0], dtype=prob_k_all.dtype) for i in range(num_times): bin_edges[i] = np.arange(max_cts*2**i) start_time = time.time() # used to log the computation time (optionally) for i, images in enumerate(image_sets): #print( i, images ) # Ring buffer, a buffer with periodic boundary conditions. # Images must be keep for up to maximum delay in buf. #buf = np.zeros([num_times, timebin_num], dtype=np.object) # matrix of buffers buf = np.ma.zeros([num_times, timebin_num, nopixels]) buf.mask = True # to track processing each time level track_level = np.zeros( num_times ) track_bad_level = np.zeros( num_times ) # to increment buffer cur = np.int_( np.full(num_times, timebin_num) ) # to track how many images processed in each level img_per_level = np.zeros(num_times, dtype=np.int64) prob_k = np.zeros_like(prob_k_all) prob_k_pow = np.zeros_like(prob_k_all) try: noframes= len(images) except: noframes= images.length #Num= { key: [0]* len( dict_dly[key] ) for key in list(dict_dly.keys()) } for n, img in enumerate(images): cur[0] = 1 + cur[0]% timebin_num # read each frame # Put the image into the ring buffer. img_ = (np.ravel(img))[indices] if threshold is not None: if img_.max() >= threshold: print ('bad image: %s here!'%n ) img_ = np.ma.zeros( len(img_) ) img_.mask = True if bad_images is not None: if n in bad_images: print ('bad image: %s here!'%n) img_ = np.ma.zeros( len(img_) ) img_.mask = True buf[0, cur[0] - 1] = img_ #print( n, np.sum(buf[0, cur[0] - 1] ), np.sum( img ) ) _process(num_roi, 0, cur[0] - 1, buf, img_per_level, labels, max_cts, bin_edges[0], prob_k, prob_k_pow,track_bad_level) #print (0, img_per_level) # check whether the number of levels is one, otherwise # continue processing the next level level = 1 if number_of_img>1: processing=1 else: processing=0 #print ('track_level: %s'%track_level) #while level < num_times: #if not track_level[level]: #track_level[level] = 1 if only_first_level: processing = 0 while processing: if track_level[level]: prev = 1 + (cur[level - 1] - 2) % timebin_num cur[level] = 1 + cur[level] % timebin_num bufa = buf[level-1,prev-1] bufb= buf[level-1,cur[level-1]-1] if (bufa.data==0).all(): buf[level,cur[level]-1] = bufa elif (bufb.data==0).all(): buf[level,cur[level]-1] = bufb else: buf[level, cur[level]-1] = bufa + bufb #print (level, cur[level]-1) track_level[level] = 0 _process(num_roi, level, cur[level]-1, buf, img_per_level, labels, max_cts, bin_edges[level], prob_k, prob_k_pow,track_bad_level) level += 1 if level < num_times:processing = 1 else:processing = 0 else: track_level[level] = 1 processing = 0 #print ('track_level: %s'%track_level) if noframes>=10 and n %( int(noframes/10) ) ==0: sys.stdout.write("#") sys.stdout.flush() prob_k_all += (prob_k - prob_k_all)/(i + 1) prob_k_pow_all += (prob_k_pow - prob_k_pow_all)/(i + 1) prob_k_std_dev = np.power((prob_k_pow_all - np.power(prob_k_all, 2)), .5) for i in range(num_times): if isinstance(prob_k_all[i,0], float ): for j in range( len(u_labels)): prob_k_all[i,j] = np.array( [0] * (len(bin_edges[i]) -1 ) ) prob_k_std_dev[i,j] = np.array( [0] * (len(bin_edges[i]) -1 ) ) logger.info("Processing time for XSVS took %s seconds." "", (time.time() - start_time)) elapsed_time = time.time() - start_time #print (Num) print ('Total time: %.2f min' %(elapsed_time/60.)) #print (img_per_level - track_bad_level) #print (buf) return bin_edges, prob_k_all, prob_k_std_dev
def lazy_two_timep(FD, num_levels, num_bufs, labels, internal_state=None, bad_frame_list=None, imgsum= None, norm = None ): """ Generator implementation of two-time correlation If you do not want multi-tau correlation, set num_levels to 1 and num_bufs to the number of images you wish to correlate Multi-tau correlation uses a scheme to achieve long-time correlations inexpensively by downsampling the data, iteratively combining successive frames. The longest lag time computed is num_levels * num_bufs. ** see comments on multi_tau_auto_corr Parameters ---------- FD: the handler of compressed data num_levels : int, optional how many generations of downsampling to perform, i.e., the depth of the binomial tree of averaged frames default is one num_bufs : int, must be even maximum lag step to compute in each generation of downsampling labels : array labeled array of the same shape as the image stack; each ROI is represented by a distinct label (i.e., integer) two_time_internal_state: None Yields ------ namedtuple A ``results`` object is yielded after every image has been processed. This `reults` object contains, in this order: - ``g2``: the normalized correlation shape is (num_rois, len(lag_steps), len(lag_steps)) - ``lag_steps``: the times at which the correlation was computed - ``_internal_state``: all of the internal state. Can be passed back in to ``lazy_one_time`` as the ``internal_state`` parameter Notes ----- The two-time correlation function is defined as .. math:: C(q,t_1,t_2) = \\frac{<I(q,t_1)I(q,t_2)>}{<I(q, t_1)><I(q,t_2)>} Here, the ensemble averages are performed over many pixels of detector, all having the same ``q`` value. The average time or age is equal to ``(t1+t2)/2``, measured by the distance along the ``t1 = t2`` diagonal. The time difference ``t = |t1 - t2|``, with is distance from the ``t1 = t2`` diagonal in the perpendicular direction. In the equilibrium system, the two-time correlation functions depend only on the time difference ``t``, and hence the two-time correlation contour lines are parallel. References ---------- .. [1] A. Fluerasu, A. Moussaid, A. Mandsen and A. Schofield, "Slow dynamics and aging in collodial gels studied by x-ray photon correlation spectroscopy," Phys. Rev. E., vol 76, p 010401(1-4), 2007. """ num_frames = FD.end - FD.beg if internal_state is None: internal_state = _init_state_two_timep(num_levels, num_bufs, labels, num_frames ) # create a shorthand reference to the results and state named tuple s = internal_state qind, pixelist = roi.extract_label_indices( labels ) # iterate over the images to compute multi-tau correlation fra_pix = np.zeros_like( pixelist, dtype=np.float64) timg = np.zeros( FD.md['ncols'] * FD.md['nrows'] , dtype=np.int32 ) timg[pixelist] = np.arange( 1, len(pixelist) + 1 ) if bad_frame_list is None: bad_frame_list=[] for i in range( FD.beg , FD.end ): if i in bad_frame_list: fra_pix[:]= np.nan else: (p,v) = FD.rdrawframe(i) w = np.where( timg[p] )[0] pxlist = timg[ p[w] ] -1 if imgsum is None: if norm is None: fra_pix[ pxlist] = v[w] else: fra_pix[ pxlist] = v[w]/ norm[pxlist] #-1.0 else: if norm is None: fra_pix[ pxlist] = v[w] / imgsum[i] else: fra_pix[ pxlist] = v[w]/ imgsum[i]/ norm[pxlist] level = 0 # increment buffer s.cur[0] = (1 + s.cur[0]) % num_bufs s.count_level[0] = 1 + s.count_level[0] # get the current image time #s = s._replace(current_img_time=(s.current_img_time + 1)) s.current_img_time += 1 # Put the ROI pixels into the ring buffer. s.buf[0, s.cur[0] - 1] = fra_pix fra_pix[:]=0 _two_time_processp(s.buf, s.g2, s.label_array, num_bufs, s.num_pixels, s.img_per_level, s.lag_steps, s.current_img_time, level=0, buf_no=s.cur[0] - 1) # time frame for each level s.time_ind[0].append(s.current_img_time) # check whether the number of levels is one, otherwise # continue processing the next level processing = num_levels > 1 # Compute the correlations for all higher levels. level = 1 while processing: if not s.track_level[level]: s.track_level[level] = 1 processing = False else: prev = 1 + (s.cur[level - 1] - 2) % num_bufs s.cur[level] = 1 + s.cur[level] % num_bufs s.count_level[level] = 1 + s.count_level[level] s.buf[level, s.cur[level] - 1] = ( s.buf[level - 1, prev - 1] + s.buf[level - 1, s.cur[level - 1] - 1] )/2 t1_idx = (s.count_level[level] - 1) * 2 current_img_time = ((s.time_ind[level - 1])[t1_idx] + (s.time_ind[level - 1])[t1_idx + 1])/2. # time frame for each level s.time_ind[level].append(current_img_time) # make the track_level zero once that level is processed s.track_level[level] = 0 # call the _two_time_process function for each multi-tau level # for multi-tau levels greater than one # Again, this is modifying things in place. See comment # on previous call above. _two_time_processp(s.buf, s.g2, s.label_array, num_bufs, s.num_pixels, s.img_per_level, s.lag_steps, current_img_time, level=level, buf_no=s.cur[level]-1) level += 1 # Checking whether there is next level for processing processing = level < num_levels #print (s.g2[1,:,1] ) #yield s for q in range(np.max(s.label_array)): x0 = (s.g2)[q, :, :] (s.g2)[q, :, :] = (np.tril(x0) + np.tril(x0).T - np.diag(np.diag(x0))) return s.g2, s.lag_steps
def _validate_and_transform_inputs(num_bufs, num_levels, labels): """ This is a helper function to validate inputs and create initial state inputs for both one time and two time correlation Parameters ---------- num_bufs : int num_levels : int labels : array labeled array of the same shape as the image stack; each ROI is represented by a distinct label (i.e., integer) Returns ------- label_array : array labels of the required region of interests(ROI's) pixel_list : array 1D array of indices into the raveled image for all foreground pixels (labeled nonzero) e.g., [5, 6, 7, 8, 14, 15, 21, 22] num_rois : int number of region of interests (ROI) num_pixels : array number of pixels in each ROI lag_steps : array the times at which the correlation was computed buf : array image data for correlation img_per_level : array to track how many images processed in each level track_level : array to track processing each level cur : array to increment the buffer norm : dict to track bad images lev_len : array length of each levels """ if num_bufs % 2 != 0: raise ValueError("There must be an even number of `num_bufs`. You " "provided %s" % num_bufs) label_array, pixel_list = extract_label_indices(labels) # map the indices onto a sequential list of integers starting at 1 label_mapping = {label: n+1 for n, label in enumerate(np.unique(label_array))} # remap the label array to go from 1 -> max(_labels) for label, n in label_mapping.items(): label_array[label_array == label] = n # number of ROI's num_rois = len(label_mapping) # stash the number of pixels in the mask num_pixels = np.bincount(label_array)[1:] # Convert from num_levels, num_bufs to lag frames. tot_channels, lag_steps, dict_lag = multi_tau_lags(num_levels, num_bufs) # these norm and lev_len will help to find the one time correlation # normalization norm will updated when there is a bad image norm = {key: [0] * len(dict_lag[key]) for key in (dict_lag.keys())} lev_len = np.array([len(dict_lag[i]) for i in (dict_lag.keys())]) # Ring buffer, a buffer with periodic boundary conditions. # Images must be keep for up to maximum delay in buf. buf = np.zeros((num_levels, num_bufs, len(pixel_list)), dtype=np.float64) # to track how many images processed in each level img_per_level = np.zeros(num_levels, dtype=np.int64) # to track which levels have already been processed track_level = np.zeros(num_levels, dtype=bool) # to increment buffer cur = np.ones(num_levels, dtype=np.int64) return (label_array, pixel_list, num_rois, num_pixels, lag_steps, buf, img_per_level, track_level, cur, norm, lev_len)
def autocor_two_time( num_buf, rois, imgs, num_lev=None, start_img=None, end_img=None ): ''' Dec 16, 2015, Y.G.@CHX a multi-tau code for two-time correlation function Parameters: num_buf: int, number of buffer rois: 2-D array, the interested roi, has the same shape as image, can be rings for saxs, boxes for gisaxs imgs: pims sequences, image stack Options: num_lev: int, number of level, if None: = int(np.log( noframes/(num_buf-1))/np.log(2) +1) +1 start_img: int, None, =0 end_img: int, None, = len(imgs) #to be done to deal with bad frames #bad_images: list, None,bad_images list #threshold: float, None, intensity max threshold, above which is considered as bad images Return: g12, 3D-array, shape as ( len(imgs), len(imgs), q) One example: g12 = autocor_two_time( num_buf, ring_mask, imgsr, num_lev=None ) ''' #print (dly) if start_img is None:start_img=0 if end_img is None: try: end_img= len(imgs) except: end_img= imgs.length #print (start_img, end_img) noframes = end_img - start_img #+ 1 #print (noframes) ring_mask = rois if num_lev is None:num_lev = int(np.log( noframes/(num_buf-1))/np.log(2) +1) +1 print ( 'The lev number is %s'%num_lev) dly, dict_dly = delays( num_lev, num_buf, time=1 ) #print (dly.max()) qind, pixelist = roi.extract_label_indices( ring_mask ) noqs = np.max(qind) nopr = np.bincount(qind, minlength=(noqs+1))[1:] nopixels = nopr.sum() start_time = time.time() buf=np.zeros([num_lev,num_buf,nopixels]) #// matrix of buffers, for store img cts=np.zeros(num_lev) cur=np.ones(num_lev) * num_buf countl = np.array( np.zeros( num_lev ),dtype='int') g12 = np.zeros( [ noframes, noframes, noqs] ) num= np.array( np.zeros( num_lev ),dtype='int') time_ind ={key: [] for key in range(num_lev)} ttx=0 for n in range( start_img, end_img ): ##do the work here cur[0]=1+cur[0]%num_buf # increment buffer img = imgs[n] #print ( 'The insert image is %s' %(n) ) buf[0, cur[0]-1 ]= (np.ravel(img))[pixelist] img=[] #//save space countl[0] = 1+ countl[0] current_img_time = n - start_img +1 process_two_time(lev=0, bufno=cur[0]-1,n=current_img_time, g12=g12, buf=buf, num=num, num_buf=num_buf, noqs=noqs, qind=qind, nopr=nopr, dly=dly) time_ind[0].append( current_img_time ) processing=1 lev=1 while processing: if cts[lev]: prev= 1+ (cur[lev-1]-1-1+num_buf)%num_buf cur[lev]= 1+ cur[lev]%num_buf countl[lev] = 1+ countl[lev] buf[lev,cur[lev]-1] = ( buf[lev-1,prev-1] + buf[lev-1,cur[lev-1]-1] ) /2. cts[lev]=0 t1_idx= (countl[lev]-1) *2 current_img_time = ((time_ind[lev-1])[t1_idx ] + (time_ind[lev-1])[t1_idx +1 ] )/2. time_ind[lev].append( current_img_time ) process_two_time(lev=lev, bufno=cur[lev]-1,n=current_img_time, g12=g12, buf=buf, num=num, num_buf=num_buf, noqs=noqs, qind=qind, nopr=nopr, dly=dly) lev+=1 #//Since this level finished, test if there is a next level for processing if lev<num_lev:processing = 1 else:processing = 0 else: cts[lev]=1 #// set flag to process next time processing=0 #// can stop until more images are accumulated if n %(noframes/10) ==0: sys.stdout.write("#") sys.stdout.flush() for q in range(noqs): x0 = g12[:,:,q] g12[:,:,q] = np.tril(x0) + np.tril(x0).T - np.diag( np.diag(x0) ) elapsed_time = time.time() - start_time print ('Total time: %.2f min' %(elapsed_time/60.)) return g12, elapsed_time/60.
def lazy_one_time(FD, num_levels, num_bufs, labels, internal_state=None, bad_frame_list=None, imgsum=None, norm = None ): """Generator implementation of 1-time multi-tau correlation If you do not want multi-tau correlation, set num_levels to 1 and num_bufs to the number of images you wish to correlate The number of bins (of size 1) is one larger than the largest value in `x`. If `minlength` is specified, there will be at least this number of bins in the output array (though it will be longer if necessary, depending on the contents of `x`). Each bin gives the number of occurrences of its index value in `x`. If `weights` is specified the input array is weighted by it, i.e. if a value ``n`` is found at position ``i``, ``out[n] += weight[i]`` instead of ``out[n] += 1``. Parameters ---------- image_iterable : FD, a compressed eiger file by Multifile class num_levels : int how many generations of downsampling to perform, i.e., the depth of the binomial tree of averaged frames num_bufs : int, must be even maximum lag step to compute in each generation of downsampling labels : array Labeled array of the same shape as the image stack. Each ROI is represented by sequential integers starting at one. For example, if you have four ROIs, they must be labeled 1, 2, 3, 4. Background is labeled as 0 internal_state : namedtuple, optional internal_state is a bucket for all of the internal state of the generator. It is part of the `results` object that is yielded from this generator For the sake of normalization: imgsum: a list with the same length as FD, sum of each frame qp, iq: the circular average radius (in pixel) and intensity center: beam center Yields ------ Returns ------- A `results` object is yielded after every image has been processed. This `reults` object contains, in this order: - `g2`: the normalized correlation shape is (len(lag_steps), num_rois) - `lag_steps`: the times at which the correlation was computed - `_internal_state`: all of the internal state. Can be passed back in to `lazy_one_time` as the `internal_state` parameter Notes ----- The normalized intensity-intensity time-autocorrelation function is defined as .. math:: g_2(q, t') = \\frac{<I(q, t)I(q, t + t')> }{<I(q, t)>^2} t' > 0 Here, ``I(q, t)`` refers to the scattering strength at the momentum transfer vector ``q`` in reciprocal space at time ``t``, and the brackets ``<...>`` refer to averages over time ``t``. The quantity ``t'`` denotes the delay time This implementation is based on published work. [1]_ References ---------- .. [1] D. Lumma, L. B. Lurio, S. G. J. Mochrie and M. Sutton, "Area detector based photon correlation in the regime of short data batches: Data reduction for dynamic x-ray scattering," Rev. Sci. Instrum., vol 71, p 3274-3289, 2000. """ if internal_state is None: internal_state = _init_state_one_time(num_levels, num_bufs, labels) # create a shorthand reference to the results and state named tuple s = internal_state qind, pixelist = roi.extract_label_indices( labels ) # iterate over the images to compute multi-tau correlation fra_pix = np.zeros_like( pixelist, dtype=np.float64) timg = np.zeros( FD.md['ncols'] * FD.md['nrows'] , dtype=np.int32 ) timg[pixelist] = np.arange( 1, len(pixelist) + 1 ) if bad_frame_list is None: bad_frame_list=[] for i in tqdm(range( FD.beg , FD.end )): if i in bad_frame_list: fra_pix[:]= np.nan else: (p,v) = FD.rdrawframe(i) w = np.where( timg[p] )[0] pxlist = timg[ p[w] ] -1 if imgsum is None: if norm is None: fra_pix[ pxlist] = v[w] else: fra_pix[ pxlist] = v[w]/ norm[pxlist] #-1.0 else: if norm is None: fra_pix[ pxlist] = v[w] / imgsum[i] else: fra_pix[ pxlist] = v[w]/ imgsum[i]/ norm[pxlist] level = 0 # increment buffer s.cur[0] = (1 + s.cur[0]) % num_bufs # Put the ROI pixels into the ring buffer. s.buf[0, s.cur[0] - 1] = fra_pix fra_pix[:]=0 #print( i, len(p), len(w), len( pixelist)) #print ('i= %s init fra_pix'%i ) buf_no = s.cur[0] - 1 # Compute the correlations between the first level # (undownsampled) frames. This modifies G, # past_intensity, future_intensity, # and img_per_level in place! _one_time_process(s.buf, s.G, s.past_intensity, s.future_intensity, s.label_array, num_bufs, s.num_pixels, s.img_per_level, level, buf_no, s.norm, s.lev_len) # check whether the number of levels is one, otherwise # continue processing the next level processing = num_levels > 1 level = 1 while processing: if not s.track_level[level]: s.track_level[level] = True processing = False else: prev = (1 + (s.cur[level - 1] - 2) % num_bufs) s.cur[level] = ( 1 + s.cur[level] % num_bufs) s.buf[level, s.cur[level] - 1] = (( s.buf[level - 1, prev - 1] + s.buf[level - 1, s.cur[level - 1] - 1]) / 2) # make the track_level zero once that level is processed s.track_level[level] = False # call processing_func for each multi-tau level greater # than one. This is modifying things in place. See comment # on previous call above. buf_no = s.cur[level] - 1 _one_time_process(s.buf, s.G, s.past_intensity, s.future_intensity, s.label_array, num_bufs, s.num_pixels, s.img_per_level, level, buf_no, s.norm, s.lev_len) level += 1 # Checking whether there is next level for processing processing = level < num_levels # If any past intensities are zero, then g2 cannot be normalized at # those levels. This if/else code block is basically preventing # divide-by-zero errors. if len(np.where(s.past_intensity == 0)[0]) != 0: g_max = np.where(s.past_intensity == 0)[0][0] else: g_max = s.past_intensity.shape[0] g2 = (s.G[:g_max] / (s.past_intensity[:g_max] * s.future_intensity[:g_max])) yield results(g2, s.lag_steps[:g_max], s)
def auto_two_Array( data, rois, data_pixel=None ): ''' Dec 16, 2015, Y.G.@CHX a numpy operation method to get two-time correlation function Parameters: data: images sequence, shape as [img[0], img[1], imgs_length] rois: 2-D array, the interested roi, has the same shape as image, can be rings for saxs, boxes for gisaxs Options: data_pixel: if not None, 2-D array, shape as (len(images), len(qind)), use function Get_Pixel_Array( ).get_data( ) to get Return: g12: a 3-D array, shape as ( imgs_length, imgs_length, q) One example: g12 = auto_two_Array( imgsr, ring_mask, data_pixel = data_pixel ) ''' start_time = time.time() qind, pixelist = roi.extract_label_indices( rois ) noqs = len( np.unique(qind) ) nopr = np.bincount(qind, minlength=(noqs+1))[1:] if data_pixel is None: data_pixel = Get_Pixel_Array( data, pixelist).get_data() #print (data_pixel.shape) try: noframes = len(data) except: noframes = data.length g12b = np.zeros( [noframes, noframes, noqs] ) Unitq = (noqs/10) proi=0 for qi in range(1, noqs + 1 ): pixelist_qi = np.where( qind == qi)[0] #print (pixelist_qi.shape, data_pixel[qi].shape) data_pixel_qi = data_pixel[:,pixelist_qi] sum1 = (np.average( data_pixel_qi, axis=1)).reshape( 1, noframes ) sum2 = sum1.T g12b[:,:,qi -1 ] = np.dot( data_pixel_qi, data_pixel_qi.T) /sum1 / sum2 / nopr[qi -1] #print ( proi, int( qi //( Unitq) ) ) if int( qi //( Unitq) ) == proi: sys.stdout.write("#") sys.stdout.flush() proi += 1 elapsed_time = time.time() - start_time print ('Total time: %.2f min' %(elapsed_time/60.)) return g12b
def xsvs(image_sets, label_array, number_of_img, timebin_num=2, time_bin=None, only_first_level=False, max_cts=None, bad_images=None, threshold=None): """ This function will provide the probability density of detecting photons for different integration times. The experimental probability density P(K) of detecting photons K is obtained by histogramming the speckle counts over an ensemble of equivalent pixels and over a number of speckle patterns recorded with the same integration time T under the same condition. Parameters ---------- image_sets : array sets of images label_array : array labeled array; 0 is background. Each ROI is represented by a distinct label (i.e., integer). number_of_img : int number of images (how far to go with integration times when finding the time_bin, using skxray.utils.geometric function) timebin_num : int, optional integration time; default is 2 max_cts : int, optional the brightest pixel in any ROI in any image in the image set. defaults to using skxray.core.roi.roi_max_counts to determine the brightest pixel in any of the ROIs bad_images: array, optional the bad images number list, the XSVS will not analyze the binning image groups which involve any bad images threshold: float, optional If one image involves a pixel with intensity above threshold, such image will be considered as a bad image. Returns ------- prob_k_all : array probability density of detecting photons prob_k_std_dev : array standard deviation of probability density of detecting photons Notes ----- These implementation is based on following references References: text [1]_, text [2]_ .. [1] L. Li, P. Kwasniewski, D. Oris, L Wiegart, L. Cristofolini, C. Carona and A. Fluerasu , "Photon statistics and speckle visibility spectroscopy with partially coherent x-rays" J. Synchrotron Rad., vol 21, p 1288-1295, 2014. .. [2] R. Bandyopadhyay, A. S. Gittings, S. S. Suh, P.K. Dixon and D.J. Durian "Speckle-visibilty Spectroscopy: A tool to study time-varying dynamics" Rev. Sci. Instrum. vol 76, p 093110, 2005. There is an example in https://github.com/scikit-xray/scikit-xray-examples It will demonstrate the use of these functions in this module for experimental data. """ if max_cts is None: max_cts = roi.roi_max_counts(image_sets, label_array) # find the label's and pixel indices for ROI's labels, indices = roi.extract_label_indices(label_array) nopixels = len(indices) # number of ROI's u_labels = list(np.unique(labels)) num_roi = len(u_labels) # create integration times if time_bin is None: time_bin = geometric_series(timebin_num, number_of_img) if only_first_level: time_bin = [1] # number of times in the time bin num_times = len(time_bin) # number of pixels per ROI num_pixels = np.bincount(labels, minlength=(num_roi + 1))[1:] # probability density of detecting photons prob_k_all = np.zeros([num_times, num_roi], dtype=np.object) # square of probability density of detecting photons prob_k_pow_all = np.zeros_like(prob_k_all) # standard deviation of probability density of detecting photons prob_k_std_dev = np.zeros_like(prob_k_all) # get the bin edges for each time bin for each ROI bin_edges = np.zeros(prob_k_all.shape[0], dtype=prob_k_all.dtype) for i in range(num_times): bin_edges[i] = np.arange(max_cts * 2**i) start_time = time.time() # used to log the computation time (optionally) for i, images in enumerate(image_sets): # Ring buffer, a buffer with periodic boundary conditions. # Images must be keep for up to maximum delay in buf. #buf = np.zeros([num_times, timebin_num], dtype=np.object) # matrix of buffers buf = np.ma.zeros([num_times, timebin_num, nopixels]) buf.mask = True # to track processing each time level track_level = np.zeros(num_times) track_bad_level = np.zeros(num_times) # to increment buffer cur = np.int_(np.full(num_times, timebin_num)) # to track how many images processed in each level img_per_level = np.zeros(num_times, dtype=np.int64) prob_k = np.zeros_like(prob_k_all) prob_k_pow = np.zeros_like(prob_k_all) try: noframes = len(images) except: noframes = images.length #Num= { key: [0]* len( dict_dly[key] ) for key in list(dict_dly.keys()) } for n, img in enumerate(images): cur[0] = 1 + cur[0] % timebin_num # read each frame # Put the image into the ring buffer. img_ = (np.ravel(img))[indices] if threshold is not None: if img_.max() >= threshold: print('bad image: %s here!' % n) img_ = np.ma.zeros(len(img_)) img_.mask = True if bad_images is not None: if n in bad_images: print('bad image: %s here!' % n) img_ = np.ma.zeros(len(img_)) img_.mask = True buf[0, cur[0] - 1] = img_ #print( n, np.sum(buf[0, cur[0] - 1] ), np.sum( img ) ) _process(num_roi, 0, cur[0] - 1, buf, img_per_level, labels, max_cts, bin_edges[0], prob_k, prob_k_pow, track_bad_level) #print (0, img_per_level) # check whether the number of levels is one, otherwise # continue processing the next level level = 1 if number_of_img > 1: processing = 1 else: processing = 0 #print ('track_level: %s'%track_level) #while level < num_times: #if not track_level[level]: #track_level[level] = 1 if only_first_level: processing = 0 while processing: if track_level[level]: prev = 1 + (cur[level - 1] - 2) % timebin_num cur[level] = 1 + cur[level] % timebin_num bufa = buf[level - 1, prev - 1] bufb = buf[level - 1, cur[level - 1] - 1] if (bufa.data == 0).all(): buf[level, cur[level] - 1] = bufa elif (bufb.data == 0).all(): buf[level, cur[level] - 1] = bufb else: buf[level, cur[level] - 1] = bufa + bufb #print (level, cur[level]-1) track_level[level] = 0 _process(num_roi, level, cur[level] - 1, buf, img_per_level, labels, max_cts, bin_edges[level], prob_k, prob_k_pow, track_bad_level) level += 1 if level < num_times: processing = 1 else: processing = 0 else: track_level[level] = 1 processing = 0 #print ('track_level: %s'%track_level) if noframes >= 10 and n % (int(noframes / 10)) == 0: sys.stdout.write("#") sys.stdout.flush() prob_k_all += (prob_k - prob_k_all) / (i + 1) prob_k_pow_all += (prob_k_pow - prob_k_pow_all) / (i + 1) prob_k_std_dev = np.power((prob_k_pow_all - np.power(prob_k_all, 2)), .5) for i in range(num_times): if isinstance(prob_k_all[i, 0], float): for j in range(len(u_labels)): prob_k_all[i, j] = np.array([0] * (len(bin_edges[i]) - 1)) prob_k_std_dev[i, j] = np.array([0] * (len(bin_edges[i]) - 1)) logger.info("Processing time for XSVS took %s seconds." "", (time.time() - start_time)) elapsed_time = time.time() - start_time #print (Num) print('Total time: %.2f min' % (elapsed_time / 60.)) #print (img_per_level - track_bad_level) #print (buf) return prob_k_all, prob_k_std_dev
def _validate_and_transform_inputs(num_bufs, num_levels, labels): """ This is a helper function to validate inputs and create initial state inputs for both one time and two time correlation Parameters ---------- num_bufs : int num_levels : int labels : array labeled array of the same shape as the image stack; each ROI is represented by a distinct label (i.e., integer) Returns ------- label_array : array labels of the required region of interests(ROI's) pixel_list : array 1D array of indices into the raveled image for all foreground pixels (labeled nonzero) e.g., [5, 6, 7, 8, 14, 15, 21, 22] num_rois : int number of region of interests (ROI) num_pixels : array number of pixels in each ROI lag_steps : array the times at which the correlation was computed buf : array image data for correlation img_per_level : array to track how many images processed in each level track_level : array to track processing each level cur : array to increment the buffer norm : dict to track bad images lev_len : array length of each levels """ if num_bufs % 2 != 0: raise ValueError("There must be an even number of `num_bufs`. You " "provided %s" % num_bufs) label_array, pixel_list = extract_label_indices(labels) # map the indices onto a sequential list of integers starting at 1 label_mapping = { label: n + 1 for n, label in enumerate(np.unique(label_array)) } # remap the label array to go from 1 -> max(_labels) for label, n in label_mapping.items(): label_array[label_array == label] = n # number of ROI's num_rois = len(label_mapping) # stash the number of pixels in the mask num_pixels = np.bincount(label_array)[1:] # Convert from num_levels, num_bufs to lag frames. tot_channels, lag_steps, dict_lag = multi_tau_lags(num_levels, num_bufs) # these norm and lev_len will help to find the one time correlation # normalization norm will updated when there is a bad image norm = {key: [0] * len(dict_lag[key]) for key in (dict_lag.keys())} lev_len = np.array([len(dict_lag[i]) for i in (dict_lag.keys())]) # Ring buffer, a buffer with periodic boundary conditions. # Images must be keep for up to maximum delay in buf. buf = np.zeros((num_levels, num_bufs, len(pixel_list)), dtype=np.float64) # to track how many images processed in each level img_per_level = np.zeros(num_levels, dtype=np.int64) # to track which levels have already been processed track_level = np.zeros(num_levels, dtype=bool) # to increment buffer cur = np.ones(num_levels, dtype=np.int64) return (label_array, pixel_list, num_rois, num_pixels, lag_steps, buf, img_per_level, track_level, cur, norm, lev_len)
def test_rings(): center = (100., 100.) img_dim = (200, 205) first_q = 10. delta_q = 5. num_rings = 7 # number of Q rings one_step_q = 5.0 step_q = [2.5, 3.0, 5.8] # test when there is same spacing between rings edges = roi.ring_edges(first_q, width=delta_q, spacing=one_step_q, num_rings=num_rings) print("edges there is same spacing between rings ", edges) label_array = roi.rings(edges, center, img_dim) print("label_array there is same spacing between rings", label_array) label_mask, pixel_list = roi.extract_label_indices(label_array) # number of pixels per ROI num_pixels = np.bincount(label_mask, minlength=(np.max(label_mask) + 1)) num_pixels = num_pixels[1:] # test when there is same spacing between rings edges = roi.ring_edges(first_q, width=delta_q, spacing=2.5, num_rings=num_rings) print("edges there is same spacing between rings ", edges) label_array = roi.rings(edges, center, img_dim) print("label_array there is same spacing between rings", label_array) label_mask, pixel_list = roi.extract_label_indices(label_array) # number of pixels per ROI num_pixels = np.bincount(label_mask, minlength=(np.max(label_mask) + 1)) num_pixels = num_pixels[1:] # test when there is different spacing between rings edges = roi.ring_edges(first_q, width=delta_q, spacing=step_q, num_rings=4) print("edges when there is different spacing between rings", edges) label_array = roi.rings(edges, center, img_dim) print("label_array there is different spacing between rings", label_array) label_mask, pixel_list = roi.extract_label_indices(label_array) # number of pixels per ROI num_pixels = np.bincount(label_mask, minlength=(np.max(label_mask) + 1)) num_pixels = num_pixels[1:] # test when there is no spacing between rings edges = roi.ring_edges(first_q, width=delta_q, num_rings=num_rings) print("edges", edges) label_array = roi.rings(edges, center, img_dim) print("label_array", label_array) label_mask, pixel_list = roi.extract_label_indices(label_array) # number of pixels per ROI num_pixels = np.bincount(label_mask, minlength=(np.max(label_mask) + 1)) num_pixels = num_pixels[1:] # Did we draw the right number of rings? print(np.unique(label_array)) actual_num_rings = len(np.unique(label_array)) - 1 assert_equal(actual_num_rings, num_rings) # Does each ring have more pixels than the last, being larger? ring_areas = np.bincount(label_array.ravel())[1:] area_comparison = np.diff(ring_areas) print(area_comparison) areas_monotonically_increasing = np.all(area_comparison > 0) assert areas_monotonically_increasing # Test various illegal inputs assert_raises(ValueError, lambda: roi.ring_edges(1, 2)) # need num_rings # width incompatible with num_rings assert_raises(ValueError, lambda: roi.ring_edges(1, [1, 2, 3], num_rings=2)) # too few spacings assert_raises(ValueError, lambda: roi.ring_edges(1, [1, 2, 3], [1])) # too many spacings assert_raises(ValueError, lambda: roi.ring_edges(1, [1, 2, 3], [1, 2, 3])) # num_rings conflicts with width, spacing assert_raises(ValueError, lambda: roi.ring_edges(1, [1, 2, 3], [1, 2], 5)) w_edges = [[5, 7], [1, 2]] assert_raises(ValueError, roi.rings, w_edges, center=(4, 4), shape=(20, 20))
def autocor_two_time(num_buf, rois, imgs, num_lev=None, start_img=None, end_img=None): ''' Dec 16, 2015, Y.G.@CHX a multi-tau code for two-time correlation function Parameters: num_buf: int, number of buffer rois: 2-D array, the interested roi, has the same shape as image, can be rings for saxs, boxes for gisaxs imgs: pims sequences, image stack Options: num_lev: int, number of level, if None: = int(np.log( noframes/(num_buf-1))/np.log(2) +1) +1 start_img: int, None, =0 end_img: int, None, = len(imgs) #to be done to deal with bad frames #bad_images: list, None,bad_images list #threshold: float, None, intensity max threshold, above which is considered as bad images Return: g12, 3D-array, shape as ( len(imgs), len(imgs), q) One example: g12 = autocor_two_time( num_buf, ring_mask, imgsr, num_lev=None ) ''' #print (dly) if start_img is None: start_img = 0 if end_img is None: try: end_img = len(imgs) except: end_img = imgs.length #print (start_img, end_img) noframes = end_img - start_img #+ 1 #print (noframes) ring_mask = rois if num_lev is None: num_lev = int(np.log(noframes / (num_buf - 1)) / np.log(2) + 1) + 1 print('The lev number is %s' % num_lev) dly, dict_dly = delays(num_lev, num_buf, time=1) #print (dly.max()) qind, pixelist = roi.extract_label_indices(ring_mask) noqs = np.max(qind) nopr = np.bincount(qind, minlength=(noqs + 1))[1:] nopixels = nopr.sum() start_time = time.time() buf = np.zeros([num_lev, num_buf, nopixels]) #// matrix of buffers, for store img cts = np.zeros(num_lev) cur = np.ones(num_lev) * num_buf countl = np.array(np.zeros(num_lev), dtype='int') g12 = np.zeros([noframes, noframes, noqs]) num = np.array(np.zeros(num_lev), dtype='int') time_ind = {key: [] for key in range(num_lev)} ttx = 0 for n in range(start_img, end_img): ##do the work here cur[0] = 1 + cur[0] % num_buf # increment buffer img = imgs[n] #print ( 'The insert image is %s' %(n) ) buf[0, cur[0] - 1] = (np.ravel(img))[pixelist] img = [] #//save space countl[0] = 1 + countl[0] current_img_time = n - start_img + 1 process_two_time(lev=0, bufno=cur[0] - 1, n=current_img_time, g12=g12, buf=buf, num=num, num_buf=num_buf, noqs=noqs, qind=qind, nopr=nopr, dly=dly) time_ind[0].append(current_img_time) processing = 1 lev = 1 while processing: if cts[lev]: prev = 1 + (cur[lev - 1] - 1 - 1 + num_buf) % num_buf cur[lev] = 1 + cur[lev] % num_buf countl[lev] = 1 + countl[lev] buf[lev, cur[lev] - 1] = (buf[lev - 1, prev - 1] + buf[lev - 1, cur[lev - 1] - 1]) / 2. cts[lev] = 0 t1_idx = (countl[lev] - 1) * 2 current_img_time = ((time_ind[lev - 1])[t1_idx] + (time_ind[lev - 1])[t1_idx + 1]) / 2. time_ind[lev].append(current_img_time) process_two_time(lev=lev, bufno=cur[lev] - 1, n=current_img_time, g12=g12, buf=buf, num=num, num_buf=num_buf, noqs=noqs, qind=qind, nopr=nopr, dly=dly) lev += 1 #//Since this level finished, test if there is a next level for processing if lev < num_lev: processing = 1 else: processing = 0 else: cts[lev] = 1 #// set flag to process next time processing = 0 #// can stop until more images are accumulated if n % (noframes / 10) == 0: sys.stdout.write("#") sys.stdout.flush() for q in range(noqs): x0 = g12[:, :, q] g12[:, :, q] = np.tril(x0) + np.tril(x0).T - np.diag(np.diag(x0)) elapsed_time = time.time() - start_time print('Total time: %.2f min' % (elapsed_time / 60.)) return g12, elapsed_time / 60.
roi_width = 9 # in pixels roi_spacing = (5.0, 4.0) x_center = 7. # in pixels y_center = (129.) # in pixels num_rings = 3 # get the edges of the rings edges = roi.ring_edges(roi_start, width=roi_width, spacing=roi_spacing, num_rings=num_rings) # get the label array from the ring shaped 3 region of interests(ROI's) labeled_roi_array = roi.rings( edges, (y_center, x_center), img_stack.shape[1:]) # extarct the ROI's lables and pixel indices corresponding to those labels roi_indices, pixel_list = roi.extract_label_indices(labeled_roi_array) # define the ROIs roi_start = 65 # in pixels roi_width = 9 # in pixels roi_spacing = (5.0, 4.0) x_center = 7. # in pixels y_center = (129.) # in pixels num_rings = 3 # get the edges of the rings edges = roi.ring_edges(roi_start, width=roi_width, spacing=roi_spacing, num_rings=num_rings) # get the label array from the ring shaped 3 region of interests(ROI's)
def autocor_one_time(num_buf, rois, imgs, num_lev=None, start_img=None, end_img=None, bad_images=None, threshold=None): ''' Dec 16, 2015, Y.G.@CHX a multi-tau code for one-time correlation function, add new funciton to deal with bad images, which masked intensities are still large than threshold Parameters: num_buf: int, number of buffer rois: 2-D array, the interested roi, has the same shape as image, can be rings for saxs, boxes for gisaxs imgs: pims sequences, image stack Options: num_lev: int, number of level, if None: = int(np.log( noframes/(num_buf-1))/np.log(2) +1) +1 start_img: int, None, =0 end_img: int, None, = len(imgs) bad_images: list, None,bad_images list threshold: float, None, intensity max threshold, above which is considered as bad images Return: g2, 2D-array, shape as (tau, q) tau, 1D-array One example: g2, tau = autocor_one_time( num_buf, ring_mask, imgsr, num_lev=None, bad_images=None, threshold= 65500 ) ''' start_time = time.time() #print (dly) if start_img is None: start_img = 0 if end_img is None: try: end_img = len(imgs) except: end_img = imgs.length #print (start_img, end_img) noframes = end_img - start_img #+ 1 #print (noframes) ring_mask = rois if num_lev is None: num_lev = int(np.log(noframes / (num_buf - 1)) / np.log(2) + 1) + 1 nolev = num_lev nobuf = num_buf print('The lev number is %s' % num_lev) dly, dict_dly = delays(num_lev, num_buf, time=1) #print (dly.max()) lev_leng = np.array([len(dict_dly[i]) for i in list(dict_dly.keys())]) qind, pixelist = roi.extract_label_indices(ring_mask) noqs = np.max(qind) nopr = np.bincount(qind, minlength=(noqs + 1))[1:] nopixels = nopr.sum() start_time = time.time() buf = np.ma.zeros([num_lev, num_buf, nopixels]) buf.mask = True cts = np.zeros(num_lev) cur = np.ones(num_lev) * num_buf countl = np.array(np.zeros(num_lev), dtype='int') g2 = np.zeros([noframes, noframes, noqs]) G = np.zeros([(nolev + 1) * int(nobuf / 2), noqs]) IAP = np.zeros([(nolev + 1) * int(nobuf / 2), noqs]) IAF = np.zeros([(nolev + 1) * int(nobuf / 2), noqs]) num = np.array(np.zeros(num_lev), dtype='int') Num = {key: [0] * len(dict_dly[key]) for key in list(dict_dly.keys())} print('Doing g2 caculation of %s frames---' % (noframes)) ttx = 0 #if bad_images is None:bad_images=[] for n in range(start_img, end_img): ##do the work here img = imgs[n] #for n, img in enumerate( imgs): img_ = (np.ravel(img))[pixelist] #print ( img_.max() ) if threshold is not None: if img_.max() >= threshold: print('bad image: %s here!' % n) img_ = np.ma.zeros(len(img_)) img_.mask = True if bad_images is not None: if n in bad_images: print('bad image: %s here!' % n) img_ = np.ma.zeros(len(img_)) img_.mask = True cur[0] = 1 + cur[0] % num_buf # increment buffer buf[0, cur[0] - 1] = img_ img = [] #//save space img_ = [] countl[0] = 1 + countl[0] process_one_time(lev=0, bufno=cur[0] - 1, G=G, IAP=IAP, IAF=IAF, buf=buf, num=num, num_buf=num_buf, noqs=noqs, qind=qind, nopr=nopr, dly=dly, Num=Num, lev_leng=lev_leng) #time_ind[0].append( current_img_time ) processing = 1 lev = 1 while processing: if cts[lev]: prev = 1 + (cur[lev - 1] - 1 - 1 + num_buf) % num_buf cur[lev] = 1 + cur[lev] % num_buf countl[lev] = 1 + countl[lev] bufa = buf[lev - 1, prev - 1] bufb = buf[lev - 1, cur[lev - 1] - 1] if (bufa.data == 0).all(): buf[lev, cur[lev] - 1] = bufa elif (bufb.data == 0).all(): buf[lev, cur[lev] - 1] = bufb else: buf[lev, cur[lev] - 1] = (bufa + bufb) / 2. cts[lev] = 0 t1_idx = (countl[lev] - 1) * 2 process_one_time(lev=lev, bufno=cur[lev] - 1, G=G, IAP=IAP, IAF=IAF, buf=buf, num=num, num_buf=num_buf, noqs=noqs, qind=qind, nopr=nopr, dly=dly, Num=Num, lev_leng=lev_leng) lev += 1 #//Since this level finished, test if there is a next level for processing if lev < num_lev: processing = 1 else: processing = 0 else: cts[lev] = 1 #// set flag to process next time processing = 0 #// can stop until more images are accumulated if n % (int(noframes / 10)) == 0: sys.stdout.write("#") sys.stdout.flush() #print G.shape if (len(np.where(IAP == 0)[0]) != 0) and (0 not in nopr): gmax = np.where(IAP == 0)[0][0] else: gmax = IAP.shape[0] #g2=G/(IAP*IAF) #print G g2 = (G[:gmax] / (IAP[:gmax] * IAF[:gmax])) elapsed_time = time.time() - start_time #print (Num) print('Total time: %.2f min' % (elapsed_time / 60.)) return g2, dly[:gmax] #, elapsed_time/60.
def autocor_one_time( num_buf, rois, imgs, num_lev=None, start_img=None, end_img=None, bad_images = None, threshold=None): ''' Dec 16, 2015, Y.G.@CHX a multi-tau code for one-time correlation function, add new funciton to deal with bad images, which masked intensities are still large than threshold Parameters: num_buf: int, number of buffer rois: 2-D array, the interested roi, has the same shape as image, can be rings for saxs, boxes for gisaxs imgs: pims sequences, image stack Options: num_lev: int, number of level, if None: = int(np.log( noframes/(num_buf-1))/np.log(2) +1) +1 start_img: int, None, =0 end_img: int, None, = len(imgs) bad_images: list, None,bad_images list threshold: float, None, intensity max threshold, above which is considered as bad images Return: g2, 2D-array, shape as (tau, q) tau, 1D-array One example: g2, tau = autocor_one_time( num_buf, ring_mask, imgsr, num_lev=None, bad_images=None, threshold= 65500 ) ''' start_time = time.time() #print (dly) if start_img is None: start_img=0 if end_img is None: try: end_img= len(imgs) except: end_img= imgs.length #print (start_img, end_img) noframes = end_img - start_img #+ 1 #print (noframes) ring_mask = rois if num_lev is None:num_lev = int(np.log( noframes/(num_buf-1))/np.log(2) +1) +1 nolev = num_lev nobuf =num_buf print ( 'The lev number is %s'%num_lev) dly, dict_dly = delays( num_lev, num_buf, time=1 ) #print (dly.max()) lev_leng = np.array( [ len( dict_dly[i] ) for i in list(dict_dly.keys()) ]) qind, pixelist = roi.extract_label_indices( ring_mask ) noqs = np.max(qind) nopr = np.bincount(qind, minlength=(noqs+1))[1:] nopixels = nopr.sum() start_time = time.time() buf = np.ma.zeros([num_lev,num_buf,nopixels]) buf.mask = True cts=np.zeros(num_lev) cur=np.ones(num_lev) * num_buf countl = np.array( np.zeros( num_lev ),dtype='int') g2 = np.zeros( [ noframes, noframes, noqs] ) G=np.zeros( [(nolev+1)*int(nobuf/2),noqs]) IAP=np.zeros( [(nolev+1)*int(nobuf/2),noqs]) IAF=np.zeros( [(nolev+1)*int(nobuf/2),noqs]) num= np.array( np.zeros( num_lev ),dtype='int') Num= { key: [0]* len( dict_dly[key] ) for key in list(dict_dly.keys()) } print ('Doing g2 caculation of %s frames---'%(noframes )) ttx=0 #if bad_images is None:bad_images=[] for n in range( start_img, end_img ): ##do the work here img = imgs[n] #for n, img in enumerate( imgs): img_ = (np.ravel(img))[pixelist] #print ( img_.max() ) if threshold is not None: if img_.max() >= threshold: print ('bad image: %s here!'%n) img_ = np.ma.zeros( len(img_) ) img_.mask = True if bad_images is not None: if n in bad_images: print ('bad image: %s here!'%n) img_ = np.ma.zeros( len(img_) ) img_.mask = True cur[0]=1+cur[0]%num_buf # increment buffer buf[0, cur[0]-1 ]= img_ img=[] #//save space img_=[] countl[0] = 1+ countl[0] process_one_time(lev=0, bufno=cur[0]-1, G=G,IAP=IAP,IAF=IAF, buf=buf, num=num, num_buf=num_buf, noqs=noqs, qind=qind, nopr=nopr, dly=dly, Num=Num, lev_leng=lev_leng ) #time_ind[0].append( current_img_time ) processing=1 lev=1 while processing: if cts[lev]: prev= 1+ (cur[lev-1]-1-1+num_buf)%num_buf cur[lev]= 1+ cur[lev]%num_buf countl[lev] = 1+ countl[lev] bufa = buf[lev-1,prev-1] bufb= buf[lev-1,cur[lev-1]-1] if (bufa.data==0).all(): buf[lev,cur[lev]-1] = bufa elif (bufb.data==0).all(): buf[lev,cur[lev]-1] = bufb else: buf[lev,cur[lev]-1] = ( bufa + bufb ) /2. cts[lev]=0 t1_idx= (countl[lev]-1) *2 process_one_time(lev=lev, bufno=cur[lev]-1, G=G,IAP=IAP,IAF=IAF, buf=buf, num=num, num_buf=num_buf, noqs=noqs, qind=qind, nopr=nopr, dly=dly,Num =Num, lev_leng=lev_leng ) lev+=1 #//Since this level finished, test if there is a next level for processing if lev<num_lev:processing = 1 else:processing = 0 else: cts[lev]=1 #// set flag to process next time processing=0 #// can stop until more images are accumulated if n %( int(noframes/10) ) ==0: sys.stdout.write("#") sys.stdout.flush() #print G.shape if (len(np.where(IAP==0)[0])!=0) and ( 0 not in nopr): gmax = np.where(IAP==0)[0][0] else: gmax=IAP.shape[0] #g2=G/(IAP*IAF) #print G g2=(G[:gmax]/(IAP[:gmax]*IAF[:gmax])) elapsed_time = time.time() - start_time #print (Num) print ('Total time: %.2f min' %(elapsed_time/60.)) return g2,dly[:gmax] #, elapsed_time/60.
def auto_two_Array_g1_norm(data, rois, data_pixel=None): ''' Dec 16, 2015, Y.G.@CHX a numpy operation method to get two-time correlation function with a normalization the purpose for this fucntion is to get a exactly same result as one-time correlation function Parameters: data: images sequence, shape as [img[0], img[1], imgs_length] rois: 2-D array, the interested roi, has the same shape as image, can be rings for saxs, boxes for gisaxs Options: data_pixel: if not None, 2-D array, shape as (len(images), len(qind)), use function Get_Pixel_Array( ).get_data( ) to get Return: g12b_norm: a 3-D array, shape as ( imgs_length, imgs_length, q), a convention two-time correlation functio same as obtained by auto_two_Array( data, roi, data_pixel=None ) g12b: a 3-D array, shape as ( imgs_length, imgs_length, q), a non-normlized two-time correlation function norms: a 2-D array, shape as ( imgs_length, q), a normalization for further get one-time from two time One example: g12b_norm, g12b_not_norm, norms = auto_two_Array_g1_norm( imgsr, ring_mask, data_pixel = data_pixel ) ''' start_time = time.time() qind, pixelist = roi.extract_label_indices(rois) noqs = len(np.unique(qind)) nopr = np.bincount(qind, minlength=(noqs + 1))[1:] if data_pixel is None: data_pixel = Get_Pixel_Array(data, pixelist).get_data() #print (data_pixel.shape) try: noframes = len(data) except: noframes = data.length g12b_norm = np.zeros([noframes, noframes, noqs]) g12b = np.zeros([noframes, noframes, noqs]) norms = np.zeros([noframes, noqs]) Unitq = (noqs / 10) proi = 0 for qi in range(1, noqs + 1): pixelist_qi = np.where(qind == qi)[0] #print (pixelist_qi.shape, data_pixel[qi].shape) data_pixel_qi = data_pixel[:, pixelist_qi] sum1 = (np.average(data_pixel_qi, axis=1)).reshape(1, noframes) sum2 = sum1.T #norms_g12 = sum1 * sum2 * nopr[qi -1] norms[:, qi - 1] = sum1 g12b[:, :, qi - 1] = np.dot(data_pixel_qi, data_pixel_qi.T) g12b_norm[:, :, qi - 1] = g12b[:, :, qi - 1] / sum1 / sum2 / nopr[qi - 1] #print ( proi, int( qi //( Unitq) ) ) if int(qi // (Unitq)) == proi: sys.stdout.write("#") sys.stdout.flush() proi += 1 elapsed_time = time.time() - start_time print('Total time: %.2f min' % (elapsed_time / 60.)) return g12b_norm, g12b, norms
def auto_two_Array_g1_norm( data, rois, data_pixel=None ): ''' Dec 16, 2015, Y.G.@CHX a numpy operation method to get two-time correlation function with a normalization the purpose for this fucntion is to get a exactly same result as one-time correlation function Parameters: data: images sequence, shape as [img[0], img[1], imgs_length] rois: 2-D array, the interested roi, has the same shape as image, can be rings for saxs, boxes for gisaxs Options: data_pixel: if not None, 2-D array, shape as (len(images), len(qind)), use function Get_Pixel_Array( ).get_data( ) to get Return: g12b_norm: a 3-D array, shape as ( imgs_length, imgs_length, q), a convention two-time correlation functio same as obtained by auto_two_Array( data, roi, data_pixel=None ) g12b: a 3-D array, shape as ( imgs_length, imgs_length, q), a non-normlized two-time correlation function norms: a 2-D array, shape as ( imgs_length, q), a normalization for further get one-time from two time One example: g12b_norm, g12b_not_norm, norms = auto_two_Array_g1_norm( imgsr, ring_mask, data_pixel = data_pixel ) ''' start_time = time.time() qind, pixelist = roi.extract_label_indices( rois ) noqs = len( np.unique(qind) ) nopr = np.bincount(qind, minlength=(noqs+1))[1:] if data_pixel is None: data_pixel = Get_Pixel_Array( data, pixelist).get_data() #print (data_pixel.shape) try: noframes = len(data) except: noframes = data.length g12b_norm = np.zeros( [noframes, noframes, noqs] ) g12b = np.zeros( [noframes, noframes, noqs] ) norms = np.zeros( [noframes, noqs] ) Unitq = (noqs/10) proi=0 for qi in range(1, noqs + 1 ): pixelist_qi = np.where( qind == qi)[0] #print (pixelist_qi.shape, data_pixel[qi].shape) data_pixel_qi = data_pixel[:,pixelist_qi] sum1 = (np.average( data_pixel_qi, axis=1)).reshape( 1, noframes ) sum2 = sum1.T #norms_g12 = sum1 * sum2 * nopr[qi -1] norms[:,qi -1 ] = sum1 g12b[:,:,qi -1 ] = np.dot( data_pixel_qi, data_pixel_qi.T) g12b_norm[:,:,qi -1 ] = g12b[:,:,qi -1 ]/ sum1 / sum2 / nopr[qi -1] #print ( proi, int( qi //( Unitq) ) ) if int( qi //( Unitq) ) == proi: sys.stdout.write("#") sys.stdout.flush() proi += 1 elapsed_time = time.time() - start_time print ('Total time: %.2f min' %(elapsed_time/60.)) return g12b_norm, g12b, norms
def cal_g2p(FD, ring_mask, bad_frame_list=None, good_start=0, num_buf=8, num_lev=None, imgsum=None, norm=None, cal_error=False): '''calculation g2 by using a multi-tau algorithm for a compressed file with parallel calculation if return_g2_details: return g2 with g2_denomitor, g2_past, g2_future ''' FD.beg = max(FD.beg, good_start) noframes = FD.end - FD.beg + 1 # number of frames, not "no frames" for i in range(FD.beg, FD.end): pass_FD(FD, i) if num_lev is None: num_lev = int(np.log(noframes / (num_buf - 1)) / np.log(2) + 1) + 1 print('In this g2 calculation, the buf and lev number are: %s--%s--' % (num_buf, num_lev)) if bad_frame_list is not None: if len(bad_frame_list) != 0: print('%s Bad frames involved and will be discarded!' % len(bad_frame_list)) noframes -= len( np.where(np.in1d(bad_frame_list, range(good_start, FD.end)))[0]) print('%s frames will be processed...' % (noframes - 1)) ring_masks = [ np.array(ring_mask == i, dtype=np.int64) for i in np.unique(ring_mask)[1:] ] qind, pixelist = roi.extract_label_indices(ring_mask) noqs = len(np.unique(qind)) nopr = np.bincount(qind, minlength=(noqs + 1))[1:] if norm is not None: norms = [ norm[np.in1d( pixelist, extract_label_indices(np.array(ring_mask == i, dtype=np.int64))[1])] for i in np.unique(ring_mask)[1:] ] inputs = range(len(ring_masks)) pool = Pool(processes=len(inputs)) internal_state = None print('Starting assign the tasks...') results = {} if norm is not None: for i in tqdm(inputs): results[i] = apply_async( pool, lazy_one_timep, (FD, num_lev, num_buf, ring_masks[i], internal_state, bad_frame_list, imgsum, norms[i], cal_error)) else: #print ('for norm is None') for i in tqdm(inputs): results[i] = apply_async( pool, lazy_one_timep, (FD, num_lev, num_buf, ring_masks[i], internal_state, bad_frame_list, imgsum, None, cal_error)) pool.close() print('Starting running the tasks...') res = [results[k].get() for k in tqdm(list(sorted(results.keys())))] len_lag = 10**10 for i in inputs: #to get the smallest length of lag_step, ##***************************** ##Here could result in problem for significantly cut useful data if some Q have very short tau list ##**************************** if len_lag > len(res[i][1]): lag_steps = res[i][1] len_lag = len(lag_steps) #lag_steps = res[0][1] if not cal_error: g2 = np.zeros([len(lag_steps), len(ring_masks)]) else: g2 = np.zeros([int((num_lev + 1) * num_buf / 2), len(ring_masks)]) g2_err = np.zeros_like(g2) g2_G = np.zeros((int((num_lev + 1) * num_buf / 2), len(pixelist))) g2_P = np.zeros_like(g2_G) g2_F = np.zeros_like(g2_G) Gmax = 0 lag_steps_err = res[0][1] nopr_ = np.lib.pad(np.cumsum(nopr), [1], mode='constant', constant_values=(0))[:-1] for i in inputs: #print( res[i][0][:,0].shape, g2.shape ) if not cal_error: g2[:, i] = res[i][0][:, 0][:len_lag] else: s_Gall_qi = res[i][2] #[:len_lag] s_Pall_qi = res[i][3] #[:len_lag] s_Fall_qi = res[i][4] #[:len_lag] #print( s_Gall_qi.shape,s_Pall_qi.shape,s_Fall_qi.shape ) avgGi = (np.average(s_Gall_qi, axis=1)) devGi = (np.std(s_Gall_qi, axis=1)) avgPi = (np.average(s_Pall_qi, axis=1)) devPi = (np.std(s_Pall_qi, axis=1)) avgFi = (np.average(s_Fall_qi, axis=1)) devFi = (np.std(s_Fall_qi, axis=1)) if len(np.where(avgPi == 0)[0]) != 0: g_max1 = np.where(avgPi == 0)[0][0] else: g_max1 = avgPi.shape[0] if len(np.where(avgFi == 0)[0]) != 0: g_max2 = np.where(avgFi == 0)[0][0] else: g_max2 = avgFi.shape[0] g_max = min(g_max1, g_max2) g2[:g_max, i] = avgGi[:g_max] / (avgPi[:g_max] * avgFi[:g_max]) g2_err[:g_max, i] = np.sqrt( (1 / (avgFi[:g_max] * avgPi[:g_max]))**2 * devGi[:g_max]**2 + (avgGi[:g_max] / (avgFi[:g_max]**2 * avgPi[:g_max]))**2 * devFi[:g_max]**2 + (avgGi[:g_max] / (avgFi[:g_max] * avgPi[:g_max]**2))**2 * devPi[:g_max]**2) Gmax = max(g_max, Gmax) lag_stepsi = res[i][1] if len(lag_steps_err) < len(lag_stepsi): lag_steps_err = lag_stepsi g2_G[:, nopr_[i]:nopr_[i + 1]] = s_Gall_qi g2_P[:, nopr_[i]:nopr_[i + 1]] = s_Pall_qi g2_F[:, nopr_[i]:nopr_[i + 1]] = s_Fall_qi del results del res if cal_error: print('G2 with error bar calculation DONE!') return g2[:Gmax, :], lag_steps_err[:Gmax], g2_err[:Gmax, :] / np.sqrt( nopr), g2_G, g2_P, g2_F else: print('G2 calculation DONE!') return g2, lag_steps