def _make_assembled_frames_worker(self, i, numproc): for j in range(i, self.chunk_size_current, numproc): self.img_data[j] = geom.apply_geom_ij_yx((self.det_x, self.det_y), self.modules_data[j]) self.msk_data[j] = geom.apply_geom_ij_yx( (self.det_x, self.det_y), self.mask_data[j] ) # output mask for xcca -0 =bad pixel, 1 - good pixel np.multiply(self.img_data[j], self.msk_data[j], out=self.img_data[j])
def plot_bg_full(b, gx, gy): fig = plt.figure(figsize=[20,20]) ax = fig.gca() lt = 5 mm = apply_geom_ij_yx( (gy,gx), ~b.bl.mask ) bb = apply_geom_ij_yx( (gy,gx), b.bl) bb = np.ma.masked_array(bb, ~mm) ax.matshow(bb, vmin=-0.15*64, norm=SymLogNorm(0.15*64,lt, base=10)) ax.set_xticks([]) ax.set_yticks([])
def assembled(self, i): """Returns assembled frame for given event.""" if self.geom is None: print( "Cannot provide assembled image, no geometry has been provided" ) return geom.apply_geom_ij_yx((self.y, self.x), self._modules_for_geom(i))[::-1, ::-1]
def goodpixels(self): """assembled boolean image with good pixels = True.""" if self.geom is None: print( "Cannot provide good-pixel mask, no geometry has been provided" ) return geom.apply_geom_ij_yx((self.y, self.x), np.transpose(self.goodmask, axes=(0, 2, 1)))[::-1, ::-1]
def activepixels(self): """assembled boolean image with active pixels = True.""" if self.geom is None: print( "Cannot provide active-pixel mask, no geometry has been provided" ) return geom.apply_geom_ij_yx((self.y, self.x), np.ones((4, 128, 128), dtype=np.bool))[::-1, ::-1]
def _get_frame(self, num, type='frame', calibrate=False, threshold=False, sync=True, assemble=True): if num > self.nframes or num < 0: print('Out of range') return if not sync: shift = 0 cell_ind = num % len(self.good_cells) train_ind = num // len(self.good_cells) if type == 'frame': ind = self.good_cells[cell_ind] + train_ind * self.num_h5cells elif type == 'gain': ind = self.good_cells[cell_ind] + train_ind * self.num_h5cells + 1 else: raise ValueError file_num = np.where(ind < self.nframes_list)[0][0] if file_num == 0: frame_num = ind else: frame_num = ind - self.nframes_list[file_num - 1] for i in range(16): if len(self.flist[i]) == 0: self.frame[i] = np.zeros_like(self.frame[0]) continue with h5py.File(self.flist[i][file_num], 'r') as f: dset_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/%dCH0:xtdf/image/data' % i cell_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/%dCH0:xtdf/image/cellId' % i train_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/%dCH0:xtdf/image/trainId' % i if sync: if i == self.first_module: trainid = f[train_name][frame_num].astype('i8')[0] shift = 0 else: shift = (trainid - f[train_name][frame_num].astype('i8')[0] ) * self.num_h5cells data = f[dset_name][frame_num + shift, 0] if calibrate: data = self._calibrate( data, f[dset_name][frame_num + shift + 1, 0], i, self.good_cells[cell_ind] // 2) if threshold: data = self._threshold(data, i, cell_ind) self.frame[i] = data if not assemble or self.geom_fname is None: return np.copy(self.frame) else: return geom.apply_geom_ij_yx((self.x, self.y), self.frame)
def _get_frames(self, num, type='frame', calibrate=False, threshold=False, assemble=True): assert len(num.shape) == 1, "Must contain a 1D array of integers" for n in num: if n > self.nframes or n < 0: print('Out of range: %d, skipping event..' % n) num = np.delete(num, np.where(num == n)) elif (self.vds[self.cell_name][n] == 65535): print('Missing cell for event: %d, skipping event..' % n) num = np.delete(num, np.where(num == n)) # Any frames left? if not num.shape[0]: return # frame or frames? if num.shape[0] > 1: self.frame = np.empty((num.shape[0], 16, 512, 128)) # these will not be the same as the IDs in the VDS file, depends on the selection of good_cells #cell_ind = num % len(self.good_cells) #train_ind = num // len(self.good_cells) #ind = self.good_cells[cell_ind] + train_ind * self.num_h5cells # stick to VDS file for IDs self.cell_ids = self.vds[self.cell_name][:][num] self.pulse_ids = self.vds[self.pulse_name][:][num] self.train_ids = self.vds[self.train_name][:][num] if type == 'frame': type_ind = 0 threshold = False elif type == 'gain': type_ind = 1 calibrate = False else: print('Unknown type string: %s' % type) return data = self.vds[self.dset_name][:, num] for n in range(num.shape[0]): if self.verbose: print( 'Getting frame with cellId=%d, pulseId=%d and trainId=%d' % (self.cell_ids[n], self.pulse_ids[n], self.train_ids[n])) if calibrate: if self.verbose: print('Calibrating frame %d' % num[n]) for i in range(16): if num.shape[0] > 1: self.frame[n, i] = self._calibrate_module( data[i, n, 0, :, :], data[i, n, 1, :, :], i, self.cell_ids[n]) else: self.frame[i] = self._calibrate_module( data[i, n, 0, :, :], data[i, n, 1, :, :], i, self.cell_ids[n]) else: if self.is_raw_data: if num.shape[0] > 1: self.frame[n] = data[:, n, 0, :, :] else: self.frame = data[:, n, 0, :, :] else: if num.shape[0] > 1: self.frame[n] = data[:] else: self.frame = data[:] if not assemble or self.geom_fname is None: return self.frame else: if num.shape[0] > 1: output = [] for n in range(num.shape[0]): if self.verbose: print('Assembling frame %d' % num[n]) output.append( geom.apply_geom_ij_yx((self.y, self.x), self.frame[n])) return np.array(output) else: return geom.apply_geom_ij_yx((self.y, self.x), self.frame)
def assemble(modules, geomfile): x, y = geom.pixel_maps_from_geometry_file(geomfile) image = geom.apply_geom_ij_yx( (y, x), np.transpose(modules, axes=(0, 2, 1)))[::-1, ::-1] return image
def analysis_finalize(self): if self.averimage: img_average = np.zeros(self.detshape) img_average = np.divide(self.img_sum, self.pix_sum, where=(self.pix_sum > 0)) img_assembled = geom.apply_geom_ij_yx((self.det_x, self.det_y), img_average) print('\nAssembled image shape={}'.format(img_assembled.shape)) out_fname = self.dir_save + os.path.basename( self.vds_file).split('_')[0] + '_powder_average_2d.bin' helper.io.write_binary_2D_arr(out_fname, img_assembled) """ img_unassembled=self.agipd_stack(img_average) out_fname = self.dir_save+os.path.basename(self.vds_file).split('_')[0] + '_powder_average_2d_unassembled.bin' helper.io.write_binary_2D_arr(out_fname, img_unassembled) """ if (self.intensity_histogram[0] == True) & (self.intensity_histogram[1] == 'cell'): if self.verbose >= 1: print( '\nNumber of histograms averaged for each specified cellID:\n cellID N\n{}\n' .format( np.vstack((self.hclist, self.hist1D_num)).T.astype(np.int32))) for i in range(len(self.hclist)): if self.hist1D_num[i] > 0: # plot current average hitogram fig_hist1 = plt.plot(self.bin_centers, np.divide(self.hist1D_aver[i], self.hist1D_num[i]), linewidth=2, marker="o", color='black') plt.yscale('log') plt.xlabel('ADU') plt.ylabel('Frequency') if self.intensity_histogram[6] == 'image': plt.savefig( '{}histogram_1d_cell_{}_{}_range_{}_{}.png'.format( self.dir_hist, self.hclist[i], self.intensity_histogram[6], self.intensity_histogram[3][0], self.intensity_histogram[3][1])) plt.clf() # output results to h5 file out_fname = self.dir_save + os.path.basename( self.vds_file).split('.')[0] + '_results.h5' if os.path.exists(out_fname): os.remove(out_fname) with h5py.File(out_fname, 'a') as f: if self.litpixels: if 'litpixels' in f: del f['litpixels'] f['litpixels'] = self.litpix.sum(0) if self.avpixi: if 'averpixintensity' in f: del f['averpixintensity'] iav = self.pixintens.sum(0) iavp = self.pixcnts.sum(0) np.divide(iav, iavp, where=(iavp != 0), out=iav) f['averpixintensity'] = iav if 'is_good' in f: del f['is_good'] f['is_good'] = self.is_good_data_all self._copy_ids(f) print( 'Total number of frames read from the input file: {}, effective analysis performance: {:.2f} fps \n' .format(self.num_imgs_read, self.num_imgs_read / (time.time() - self.start_time)))
def __init__(self, vds_file, dir_calib, geo_file, dir_save, nproc, chunks_to_read, chunk_size, refmod, exclude_modules, filterbadcells, intensity_threshold, cmode, custommask, litpixels, litpixels_threshold, averimage, avpixi, intensity_histogram, hclist, verbose): self.start_time = time.time() self.vds_file = vds_file self.dir_calib = dir_calib self.geo_file = geo_file self.dir_save = dir_save self.chunk_size = chunk_size self.chunks_to_read = chunks_to_read.copy() self.refmod = refmod self.exclude_modules = exclude_modules self.filterbadcells = filterbadcells self.intensity_threshold = intensity_threshold self.cmode = cmode self.custommask = custommask self.litpixels = litpixels self.lithr = litpixels_threshold self.averimage = averimage self.avpixi = avpixi self.intensity_histogram = intensity_histogram self.hclist = hclist self.xcca_perform = xcca_perform self.verbose = verbose self.detshape = (16, 512, 128 ) # data structure for a single frame of AGIPD1M if self.chunk_size % 32 != 0: print( '\nWARNING: Performance is best with a multiple of 32 chunk_size' ) if nproc == 0: self.nproc = int(subprocess.check_output('nproc').decode().strip()) else: self.nproc = nproc with h5py.File(vds_file, 'r') as f: self.dset_name = 'INSTRUMENT/' + list( f['INSTRUMENT'])[0] + '/DET/image/data' self.mask_name = 'INSTRUMENT/' + list( f['INSTRUMENT'])[0] + '/DET/image/mask' self.cellid_name = 'INSTRUMENT/' + list( f['INSTRUMENT'])[0] + '/DET/image/cellId' self.trainid_name = 'INSTRUMENT/' + list( f['INSTRUMENT'])[0] + '/DET/image/trainId' self.pulseid_name = 'INSTRUMENT/' + list( f['INSTRUMENT'])[0] + '/DET/image/pulseId' self.dshape = f[self.dset_name].shape print('\nResults output directory: {}'.format(self.dir_save)) print('Input vds file: {}'.format(self.vds_file)) if len(self.dshape) == 4: self.is_raw = False print('Processing calibrated data, found records for {} frames'. format(self.dshape[1])) elif len(self.dshape) == 5: self.is_raw = True print('Processing raw data, found records for {} frames'.format( self.dshape[1])) # chunk reading scheme related stuff num_chunks_max = int(math.ceil( self.dshape[1] / self.chunk_size)) # maximum possible number of data chunks if self.verbose >= 1: print( 'Maximum number of data chunks ({} frames each) is {} '.format( self.chunk_size, num_chunks_max)) self.chunks_to_read[1] = min(self.chunks_to_read[1], num_chunks_max) self.num_chunks = int( math.ceil((self.chunks_to_read[1] - self.chunks_to_read[0]) / self.chunks_to_read[2])) if self.num_chunks <= 0: print( 'Specified scheme ({}) for reading data in chunks cannot be used. Exiting...' .format(chunks_to_read)) sys.exit(1) self.num_imgs_good = 0 # current number of good images self.num_imgs_read = 0 # current number of images read from the input file self.num_chk_read = 0 # current number of chunks read from the input file if self.is_raw: # open calibration files calib_str = self.dir_calib + 'Cheetah*.h5' self.calib = [ h5py.File(f, 'r') for f in sorted(glob.glob(calib_str)) ] print('AGIPD detector geometry file: {}'.format(self.geo_file)) try: print('Allocating shared memory arrays...') self.modules_shape = ( self.chunk_size, ) + self.detshape # data format specificly for AGIPD1M detector self.modules_size = int( self.modules_shape[0] * self.modules_shape[1] * self.modules_shape[2] * self.modules_shape[3]) self.modules_data = np.frombuffer( mp.Array(ctypes.c_double, self.modules_size).get_obj()).reshape( self.modules_shape) # scattering data self.mask_data = np.frombuffer( mp.Array(ctypes.c_uint, self.modules_size).get_obj(), dtype=np.uint32).reshape(self.modules_shape) # mask self.cellid_data = np.frombuffer( mp.Array(ctypes.c_uint, self.modules_shape[0]).get_obj(), dtype=np.uint32).reshape(self.modules_shape[0]) # cell IDs self.pulseid_data = np.frombuffer( mp.Array(ctypes.c_uint, self.modules_shape[0]).get_obj(), dtype=np.uint32).reshape(self.modules_shape[0]) # pulse IDs self.trainid_data = np.frombuffer( mp.Array(ctypes.c_uint, self.modules_shape[0]).get_obj(), dtype=np.uint32).reshape(self.modules_shape[0]) # train IDs self.is_good_data = np.frombuffer( mp.Array(ctypes.c_uint, self.modules_shape[0]).get_obj(), dtype=np.uint32 ).reshape( self.modules_shape[0] ) # image quality identifier: 1 - good image, 0 - bad image; used to label bad images self.is_good_data_all = np.frombuffer(mp.Array( ctypes.c_uint, self.dshape[1]).get_obj(), dtype=np.uint32) # litpixels if self.litpixels: self.litpix = np.frombuffer( mp.Array(ctypes.c_ulong, self.dshape[0] * self.dshape[1]).get_obj(), dtype='u8').reshape(16, -1) # litpixels if self.avpixi: self.pixintens = np.frombuffer( mp.Array(ctypes.c_double, self.dshape[0] * self.dshape[1]).get_obj()).reshape( 16, -1) # average intensity/pixel self.pixcnts = np.frombuffer( mp.Array(ctypes.c_ulong, self.dshape[0] * self.dshape[1]).get_obj(), dtype='u8').reshape(16, -1) # counts pixels if self.averimage: self.img_sum = np.frombuffer( mp.Array( ctypes.c_double, self.detshape[0] * self.detshape[1] * self.detshape[2]).get_obj()).reshape( self.detshape) # powder pattern self.pix_sum = np.frombuffer( mp.Array( ctypes.c_uint, self.detshape[0] * self.detshape[1] * self.detshape[2]).get_obj(), dtype=np.uint32).reshape( self.detshape) # sum of contributions if self.intensity_histogram[0] == True: self.hist_shape = (self.chunk_size, self.intensity_histogram[2]) self.hist_vals = np.frombuffer( mp.Array(ctypes.c_double, self.hist_shape[0] * self.hist_shape[1]).get_obj()).reshape( self.hist_shape) # arrays for assembled images processing self.det_y, self.det_x = geom.pixel_maps_from_geometry_file( self.geo_file ) # pixel coordinates of the assembled detector image self.img_shape_assembled = geom.apply_geom_ij_yx( (self.det_x, self.det_y), np.zeros(self.detshape)).shape #print('\nAssembled image shape={}'.format(self.img_shape_assembled)) self.img_shape = ( self.chunk_size, ) + self.img_shape_assembled # for assembled images self.img_size = int(self.img_shape[0] * self.img_shape[1] * self.img_shape[2]) self.img_data = None # assembled image of scattering data, will be allocated later if required self.msk_data = None # assembled image of mask print('Finished with memory allocation') except: print('Problems with memory allocation. Exiting...') sys.exit(1) # create custom detector mask if self.custommask: self._mask_custom_agipd() if self.filterbadcells: self.define_bad_cells() # list of bad memroy cells # intensity histogram related stuff if self.intensity_histogram[0] == True: self.dir_hist = self.dir_save + './histograms/' if os.path.exists(self.dir_hist) is not True: os.makedirs(self.dir_hist)
def _get_frame(self, num, type='frame', calibrate=False, threshold=False, sync=True, assemble=True): if num > self.nframes or num < 0: print('Out of range') return if not sync: shift = 0 cell_ind = num % len(self.good_cells) train_ind = num // len(self.good_cells) ind = self.good_cells[cell_ind] + train_ind * self.num_h5cells if type == 'frame': type_ind = 0 threshold = False elif type == 'gain': type_ind = 1 calibrate = False else: print('Unknown type string: %s' % type) return file_num = np.where(ind < self.nframes_list)[0][0] if file_num == 0: frame_num = ind else: frame_num = ind - self.nframes_list[file_num-1] for i in range(16): if len(self.flist[i]) == 0: self.frame[i] = np.zeros_like(self.frame[0]) continue with h5py.File(self.flist[i][file_num], 'r') as f: dset_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/%dCH0:xtdf/image/data'%i if self.raw_frame: cell_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/%dCH0:xtdf/image/cellId'%i train_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/%dCH0:xtdf/image/trainId'%i if sync: if i == self.first_module: trainid = f[train_name][frame_num].astype('i8')[0] cellid = f[cell_name][frame_num].astype('i8')[0] shift = 0 else: shift = (trainid - f[train_name][frame_num].astype('i8')[0]) * self.num_h5cells shift += (cellid - f[cell_name][frame_num].astype('i8')[0]) if frame_num+shift > f[dset_name].shape[0]: print('Not syncing in last train') shift = 0 data = f[dset_name][frame_num+shift, type_ind] if calibrate: data = self._calibrate(data, f[dset_name][frame_num+shift,1], i, self.good_cells[cell_ind]) if threshold: data = self._threshold(data, i, cell_ind) else: data = f[dset_name][frame_num] data[data>1.e9] = 0 data[data<-1.e6] = 0 self.frame[i] = data if not assemble or self.geom_fname is None: return np.copy(self.frame) else: return geom.apply_geom_ij_yx((self.x, self.y), self.frame)