def _make_assembled_frames_worker(self, i, numproc):
for j in range(i, self.chunk_size_current, numproc):
self.img_data[j] = geom.apply_geom_ij_yx((self.det_x, self.det_y),
self.modules_data[j])
self.msk_data[j] = geom.apply_geom_ij_yx(
(self.det_x, self.det_y), self.mask_data[j]
) # output mask for xcca -0 =bad pixel, 1 - good pixel
np.multiply(self.img_data[j],
self.msk_data[j],
out=self.img_data[j])
def plot_bg_full(b, gx, gy):
fig = plt.figure(figsize=[20,20])
ax = fig.gca()
lt = 5
mm = apply_geom_ij_yx( (gy,gx), ~b.bl.mask )
bb = apply_geom_ij_yx( (gy,gx), b.bl)
bb = np.ma.masked_array(bb, ~mm)
ax.matshow(bb, vmin=-0.15*64, norm=SymLogNorm(0.15*64,lt, base=10))
ax.set_xticks([])
ax.set_yticks([])
def assembled(self, i):
"""Returns assembled frame for given event."""
if self.geom is None:
print(
"Cannot provide assembled image, no geometry has been provided"
)
return geom.apply_geom_ij_yx((self.y, self.x),
self._modules_for_geom(i))[::-1, ::-1]
def goodpixels(self):
"""assembled boolean image with good pixels = True."""
if self.geom is None:
print(
"Cannot provide good-pixel mask, no geometry has been provided"
)
return geom.apply_geom_ij_yx((self.y, self.x),
np.transpose(self.goodmask,
axes=(0, 2, 1)))[::-1, ::-1]
def activepixels(self):
"""assembled boolean image with active pixels = True."""
if self.geom is None:
print(
"Cannot provide active-pixel mask, no geometry has been provided"
)
return geom.apply_geom_ij_yx((self.y, self.x),
np.ones((4, 128, 128),
dtype=np.bool))[::-1, ::-1]
def _get_frame(self,
num,
type='frame',
calibrate=False,
threshold=False,
sync=True,
assemble=True):
if num > self.nframes or num < 0:
print('Out of range')
return
if not sync:
shift = 0
cell_ind = num % len(self.good_cells)
train_ind = num // len(self.good_cells)
if type == 'frame':
ind = self.good_cells[cell_ind] + train_ind * self.num_h5cells
elif type == 'gain':
ind = self.good_cells[cell_ind] + train_ind * self.num_h5cells + 1
else:
raise ValueError
file_num = np.where(ind < self.nframes_list)[0][0]
if file_num == 0:
frame_num = ind
else:
frame_num = ind - self.nframes_list[file_num - 1]
for i in range(16):
if len(self.flist[i]) == 0:
self.frame[i] = np.zeros_like(self.frame[0])
continue
with h5py.File(self.flist[i][file_num], 'r') as f:
dset_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/%dCH0:xtdf/image/data' % i
cell_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/%dCH0:xtdf/image/cellId' % i
train_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/%dCH0:xtdf/image/trainId' % i
if sync:
if i == self.first_module:
trainid = f[train_name][frame_num].astype('i8')[0]
shift = 0
else:
shift = (trainid -
f[train_name][frame_num].astype('i8')[0]
) * self.num_h5cells
data = f[dset_name][frame_num + shift, 0]
if calibrate:
data = self._calibrate(
data, f[dset_name][frame_num + shift + 1, 0], i,
self.good_cells[cell_ind] // 2)
if threshold:
data = self._threshold(data, i, cell_ind)
self.frame[i] = data
if not assemble or self.geom_fname is None:
return np.copy(self.frame)
else:
return geom.apply_geom_ij_yx((self.x, self.y), self.frame)
def _get_frames(self,
num,
type='frame',
calibrate=False,
threshold=False,
assemble=True):
assert len(num.shape) == 1, "Must contain a 1D array of integers"
for n in num:
if n > self.nframes or n < 0:
print('Out of range: %d, skipping event..' % n)
num = np.delete(num, np.where(num == n))
elif (self.vds[self.cell_name][n] == 65535):
print('Missing cell for event: %d, skipping event..' % n)
num = np.delete(num, np.where(num == n))
# Any frames left?
if not num.shape[0]:
return
# frame or frames?
if num.shape[0] > 1:
self.frame = np.empty((num.shape[0], 16, 512, 128))
# these will not be the same as the IDs in the VDS file, depends on the selection of good_cells
#cell_ind = num % len(self.good_cells)
#train_ind = num // len(self.good_cells)
#ind = self.good_cells[cell_ind] + train_ind * self.num_h5cells
# stick to VDS file for IDs
self.cell_ids = self.vds[self.cell_name][:][num]
self.pulse_ids = self.vds[self.pulse_name][:][num]
self.train_ids = self.vds[self.train_name][:][num]
if type == 'frame':
type_ind = 0
threshold = False
elif type == 'gain':
type_ind = 1
calibrate = False
else:
print('Unknown type string: %s' % type)
return
data = self.vds[self.dset_name][:, num]
for n in range(num.shape[0]):
if self.verbose:
print(
'Getting frame with cellId=%d, pulseId=%d and trainId=%d' %
(self.cell_ids[n], self.pulse_ids[n], self.train_ids[n]))
if calibrate:
if self.verbose:
print('Calibrating frame %d' % num[n])
for i in range(16):
if num.shape[0] > 1:
self.frame[n, i] = self._calibrate_module(
data[i, n, 0, :, :], data[i, n, 1, :, :], i,
self.cell_ids[n])
else:
self.frame[i] = self._calibrate_module(
data[i, n, 0, :, :], data[i, n, 1, :, :], i,
self.cell_ids[n])
else:
if self.is_raw_data:
if num.shape[0] > 1:
self.frame[n] = data[:, n, 0, :, :]
else:
self.frame = data[:, n, 0, :, :]
else:
if num.shape[0] > 1:
self.frame[n] = data[:]
else:
self.frame = data[:]
if not assemble or self.geom_fname is None:
return self.frame
else:
if num.shape[0] > 1:
output = []
for n in range(num.shape[0]):
if self.verbose:
print('Assembling frame %d' % num[n])
output.append(
geom.apply_geom_ij_yx((self.y, self.x), self.frame[n]))
return np.array(output)
else:
return geom.apply_geom_ij_yx((self.y, self.x), self.frame)
def assemble(modules, geomfile):
x, y = geom.pixel_maps_from_geometry_file(geomfile)
image = geom.apply_geom_ij_yx(
(y, x), np.transpose(modules, axes=(0, 2, 1)))[::-1, ::-1]
return image
def analysis_finalize(self):
if self.averimage:
img_average = np.zeros(self.detshape)
img_average = np.divide(self.img_sum,
self.pix_sum,
where=(self.pix_sum > 0))
img_assembled = geom.apply_geom_ij_yx((self.det_x, self.det_y),
img_average)
print('\nAssembled image shape={}'.format(img_assembled.shape))
out_fname = self.dir_save + os.path.basename(
self.vds_file).split('_')[0] + '_powder_average_2d.bin'
helper.io.write_binary_2D_arr(out_fname, img_assembled)
"""
img_unassembled=self.agipd_stack(img_average)
out_fname = self.dir_save+os.path.basename(self.vds_file).split('_')[0] + '_powder_average_2d_unassembled.bin'
helper.io.write_binary_2D_arr(out_fname, img_unassembled)
"""
if (self.intensity_histogram[0] == True) & (self.intensity_histogram[1]
== 'cell'):
if self.verbose >= 1:
print(
'\nNumber of histograms averaged for each specified cellID:\n cellID N\n{}\n'
.format(
np.vstack((self.hclist,
self.hist1D_num)).T.astype(np.int32)))
for i in range(len(self.hclist)):
if self.hist1D_num[i] > 0:
# plot current average hitogram
fig_hist1 = plt.plot(self.bin_centers,
np.divide(self.hist1D_aver[i],
self.hist1D_num[i]),
linewidth=2,
marker="o",
color='black')
plt.yscale('log')
plt.xlabel('ADU')
plt.ylabel('Frequency')
if self.intensity_histogram[6] == 'image':
plt.savefig(
'{}histogram_1d_cell_{}_{}_range_{}_{}.png'.format(
self.dir_hist, self.hclist[i],
self.intensity_histogram[6],
self.intensity_histogram[3][0],
self.intensity_histogram[3][1]))
plt.clf()
# output results to h5 file
out_fname = self.dir_save + os.path.basename(
self.vds_file).split('.')[0] + '_results.h5'
if os.path.exists(out_fname):
os.remove(out_fname)
with h5py.File(out_fname, 'a') as f:
if self.litpixels:
if 'litpixels' in f: del f['litpixels']
f['litpixels'] = self.litpix.sum(0)
if self.avpixi:
if 'averpixintensity' in f: del f['averpixintensity']
iav = self.pixintens.sum(0)
iavp = self.pixcnts.sum(0)
np.divide(iav, iavp, where=(iavp != 0), out=iav)
f['averpixintensity'] = iav
if 'is_good' in f: del f['is_good']
f['is_good'] = self.is_good_data_all
self._copy_ids(f)
print(
'Total number of frames read from the input file: {}, effective analysis performance: {:.2f} fps \n'
.format(self.num_imgs_read,
self.num_imgs_read / (time.time() - self.start_time)))
def __init__(self, vds_file, dir_calib, geo_file, dir_save, nproc,
chunks_to_read, chunk_size, refmod, exclude_modules,
filterbadcells, intensity_threshold, cmode, custommask,
litpixels, litpixels_threshold, averimage, avpixi,
intensity_histogram, hclist, verbose):
self.start_time = time.time()
self.vds_file = vds_file
self.dir_calib = dir_calib
self.geo_file = geo_file
self.dir_save = dir_save
self.chunk_size = chunk_size
self.chunks_to_read = chunks_to_read.copy()
self.refmod = refmod
self.exclude_modules = exclude_modules
self.filterbadcells = filterbadcells
self.intensity_threshold = intensity_threshold
self.cmode = cmode
self.custommask = custommask
self.litpixels = litpixels
self.lithr = litpixels_threshold
self.averimage = averimage
self.avpixi = avpixi
self.intensity_histogram = intensity_histogram
self.hclist = hclist
self.xcca_perform = xcca_perform
self.verbose = verbose
self.detshape = (16, 512, 128
) # data structure for a single frame of AGIPD1M
if self.chunk_size % 32 != 0:
print(
'\nWARNING: Performance is best with a multiple of 32 chunk_size'
)
if nproc == 0:
self.nproc = int(subprocess.check_output('nproc').decode().strip())
else:
self.nproc = nproc
with h5py.File(vds_file, 'r') as f:
self.dset_name = 'INSTRUMENT/' + list(
f['INSTRUMENT'])[0] + '/DET/image/data'
self.mask_name = 'INSTRUMENT/' + list(
f['INSTRUMENT'])[0] + '/DET/image/mask'
self.cellid_name = 'INSTRUMENT/' + list(
f['INSTRUMENT'])[0] + '/DET/image/cellId'
self.trainid_name = 'INSTRUMENT/' + list(
f['INSTRUMENT'])[0] + '/DET/image/trainId'
self.pulseid_name = 'INSTRUMENT/' + list(
f['INSTRUMENT'])[0] + '/DET/image/pulseId'
self.dshape = f[self.dset_name].shape
print('\nResults output directory: {}'.format(self.dir_save))
print('Input vds file: {}'.format(self.vds_file))
if len(self.dshape) == 4:
self.is_raw = False
print('Processing calibrated data, found records for {} frames'.
format(self.dshape[1]))
elif len(self.dshape) == 5:
self.is_raw = True
print('Processing raw data, found records for {} frames'.format(
self.dshape[1]))
# chunk reading scheme related stuff
num_chunks_max = int(math.ceil(
self.dshape[1] /
self.chunk_size)) # maximum possible number of data chunks
if self.verbose >= 1:
print(
'Maximum number of data chunks ({} frames each) is {} '.format(
self.chunk_size, num_chunks_max))
self.chunks_to_read[1] = min(self.chunks_to_read[1], num_chunks_max)
self.num_chunks = int(
math.ceil((self.chunks_to_read[1] - self.chunks_to_read[0]) /
self.chunks_to_read[2]))
if self.num_chunks <= 0:
print(
'Specified scheme ({}) for reading data in chunks cannot be used. Exiting...'
.format(chunks_to_read))
sys.exit(1)
self.num_imgs_good = 0 # current number of good images
self.num_imgs_read = 0 # current number of images read from the input file
self.num_chk_read = 0 # current number of chunks read from the input file
if self.is_raw: # open calibration files
calib_str = self.dir_calib + 'Cheetah*.h5'
self.calib = [
h5py.File(f, 'r') for f in sorted(glob.glob(calib_str))
]
print('AGIPD detector geometry file: {}'.format(self.geo_file))
try:
print('Allocating shared memory arrays...')
self.modules_shape = (
self.chunk_size,
) + self.detshape # data format specificly for AGIPD1M detector
self.modules_size = int(
self.modules_shape[0] * self.modules_shape[1] *
self.modules_shape[2] * self.modules_shape[3])
self.modules_data = np.frombuffer(
mp.Array(ctypes.c_double,
self.modules_size).get_obj()).reshape(
self.modules_shape) # scattering data
self.mask_data = np.frombuffer(
mp.Array(ctypes.c_uint, self.modules_size).get_obj(),
dtype=np.uint32).reshape(self.modules_shape) # mask
self.cellid_data = np.frombuffer(
mp.Array(ctypes.c_uint, self.modules_shape[0]).get_obj(),
dtype=np.uint32).reshape(self.modules_shape[0]) # cell IDs
self.pulseid_data = np.frombuffer(
mp.Array(ctypes.c_uint, self.modules_shape[0]).get_obj(),
dtype=np.uint32).reshape(self.modules_shape[0]) # pulse IDs
self.trainid_data = np.frombuffer(
mp.Array(ctypes.c_uint, self.modules_shape[0]).get_obj(),
dtype=np.uint32).reshape(self.modules_shape[0]) # train IDs
self.is_good_data = np.frombuffer(
mp.Array(ctypes.c_uint, self.modules_shape[0]).get_obj(),
dtype=np.uint32
).reshape(
self.modules_shape[0]
) # image quality identifier: 1 - good image, 0 - bad image; used to label bad images
self.is_good_data_all = np.frombuffer(mp.Array(
ctypes.c_uint, self.dshape[1]).get_obj(),
dtype=np.uint32) # litpixels
if self.litpixels:
self.litpix = np.frombuffer(
mp.Array(ctypes.c_ulong,
self.dshape[0] * self.dshape[1]).get_obj(),
dtype='u8').reshape(16, -1) # litpixels
if self.avpixi:
self.pixintens = np.frombuffer(
mp.Array(ctypes.c_double, self.dshape[0] *
self.dshape[1]).get_obj()).reshape(
16, -1) # average intensity/pixel
self.pixcnts = np.frombuffer(
mp.Array(ctypes.c_ulong,
self.dshape[0] * self.dshape[1]).get_obj(),
dtype='u8').reshape(16, -1) # counts pixels
if self.averimage:
self.img_sum = np.frombuffer(
mp.Array(
ctypes.c_double, self.detshape[0] * self.detshape[1] *
self.detshape[2]).get_obj()).reshape(
self.detshape) # powder pattern
self.pix_sum = np.frombuffer(
mp.Array(
ctypes.c_uint, self.detshape[0] * self.detshape[1] *
self.detshape[2]).get_obj(),
dtype=np.uint32).reshape(
self.detshape) # sum of contributions
if self.intensity_histogram[0] == True:
self.hist_shape = (self.chunk_size,
self.intensity_histogram[2])
self.hist_vals = np.frombuffer(
mp.Array(ctypes.c_double, self.hist_shape[0] *
self.hist_shape[1]).get_obj()).reshape(
self.hist_shape)
# arrays for assembled images processing
self.det_y, self.det_x = geom.pixel_maps_from_geometry_file(
self.geo_file
) # pixel coordinates of the assembled detector image
self.img_shape_assembled = geom.apply_geom_ij_yx(
(self.det_x, self.det_y), np.zeros(self.detshape)).shape
#print('\nAssembled image shape={}'.format(self.img_shape_assembled))
self.img_shape = (
self.chunk_size,
) + self.img_shape_assembled # for assembled images
self.img_size = int(self.img_shape[0] * self.img_shape[1] *
self.img_shape[2])
self.img_data = None # assembled image of scattering data, will be allocated later if required
self.msk_data = None # assembled image of mask
print('Finished with memory allocation')
except:
print('Problems with memory allocation. Exiting...')
sys.exit(1)
# create custom detector mask
if self.custommask:
self._mask_custom_agipd()
if self.filterbadcells:
self.define_bad_cells() # list of bad memroy cells
# intensity histogram related stuff
if self.intensity_histogram[0] == True:
self.dir_hist = self.dir_save + './histograms/'
if os.path.exists(self.dir_hist) is not True:
os.makedirs(self.dir_hist)
def _get_frame(self, num, type='frame', calibrate=False, threshold=False, sync=True, assemble=True):
if num > self.nframes or num < 0:
print('Out of range')
return
if not sync:
shift = 0
cell_ind = num % len(self.good_cells)
train_ind = num // len(self.good_cells)
ind = self.good_cells[cell_ind] + train_ind * self.num_h5cells
if type == 'frame':
type_ind = 0
threshold = False
elif type == 'gain':
type_ind = 1
calibrate = False
else:
print('Unknown type string: %s' % type)
return
file_num = np.where(ind < self.nframes_list)[0][0]
if file_num == 0:
frame_num = ind
else:
frame_num = ind - self.nframes_list[file_num-1]
for i in range(16):
if len(self.flist[i]) == 0:
self.frame[i] = np.zeros_like(self.frame[0])
continue
with h5py.File(self.flist[i][file_num], 'r') as f:
dset_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/%dCH0:xtdf/image/data'%i
if self.raw_frame:
cell_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/%dCH0:xtdf/image/cellId'%i
train_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/%dCH0:xtdf/image/trainId'%i
if sync:
if i == self.first_module:
trainid = f[train_name][frame_num].astype('i8')[0]
cellid = f[cell_name][frame_num].astype('i8')[0]
shift = 0
else:
shift = (trainid - f[train_name][frame_num].astype('i8')[0]) * self.num_h5cells
shift += (cellid - f[cell_name][frame_num].astype('i8')[0])
if frame_num+shift > f[dset_name].shape[0]:
print('Not syncing in last train')
shift = 0
data = f[dset_name][frame_num+shift, type_ind]
if calibrate:
data = self._calibrate(data,
f[dset_name][frame_num+shift,1],
i, self.good_cells[cell_ind])
if threshold:
data = self._threshold(data, i, cell_ind)
else:
data = f[dset_name][frame_num]
data[data>1.e9] = 0
data[data<-1.e6] = 0
self.frame[i] = data
if not assemble or self.geom_fname is None:
return np.copy(self.frame)
else:
return geom.apply_geom_ij_yx((self.x, self.y), self.frame)
