def __init__(
self,
run,
raw=True,
calib_run=None,
good_cells=range(176),
verbose=0,
geom_fname='/gpfs/exfel/exp/SPB/201802/p002145/scratch/ayyerkar/ana/geometry/b1.geom'
):
self.num_h5cells = 176
self.verbose = verbose
self.good_cells = np.array(good_cells)
self.geom_fname = geom_fname
self.is_raw_data = raw
if self.geom_fname is not None:
self.x, self.y = geom.pixel_maps_from_geometry_file(geom_fname)
calib_glob = None
if self.is_raw_data and calib_run is None:
calib_glob = '/gpfs/exfel/exp/SPB/201802/p002145/usr/Shared/calib/latest/Cheetah*.h5'
elif self.is_raw_data:
calib_glob = '/gpfs/exfel/exp/SPB/201802/p002145/usr/Shared/calib/r%.4d/Cheetah*.h5' % calib_run
self._make_flist(run, calib_glob)
self._get_nframes_list()
self.frame = np.empty((16, 512, 128))
self.powder = None
self.train_ids = None
def __init__(
self,
fname,
raw=True,
calib_run=None,
good_cells=range(176),
chunk_size=None,
verbose=0,
geom_fname='/gpfs/exfel/exp/SPB/201802/p002145/scratch/geom/b1.geom'
):
self.num_h5cells = 176
self.chunk_size = chunk_size
self.verbose = verbose
self.good_cells = np.array(good_cells)
self.geom_fname = geom_fname
self.is_raw_data = raw
if self.geom_fname is not None:
self.x, self.y = geom.pixel_maps_from_geometry_file(geom_fname)
calib_glob = None
if self.is_raw_data and calib_run is None:
calib_glob = '/gpfs/exfel/exp/SPB/201802/p002145/usr/Shared/calib/latest/Cheetah*.h5'
elif self.is_raw_data:
calib_glob = '/gpfs/exfel/exp/SPB/201802/p002145/scratch/calib/r%.4d/Cheetah*.h5' % calib_run
self.dset_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/image/data'
self.train_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/image/trainId'
self.pulse_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/image/pulseId'
self.cell_name = '/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/image/cellId'
self._load_vds(fname, calib_glob)
self.frame = np.empty((16, 512, 128))
self.powder = None
self.train_ids = None
self.cell_ids = None
self.pulse_ids = None
def __init__(self, run, calib_run=None, good_cells=range(2,62,2), verbose=0,
geom_fname='/gpfs/exfel/exp/SPB/201701/p002013/scratch/geom/agipd_taw9_oy2_1050addu_hmg5.geom'):
self.num_h5cells = 64
self.verbose = verbose
self.good_cells = np.array(good_cells)
self.geom_fname = geom_fname
if self.geom_fname is not None:
self.x, self.y = geom.pixel_maps_from_geometry_file(geom_fname)
if calib_run is None:
calib_glob='/gpfs/exfel/exp/SPB/201701/p002013/usr/Shared/calib/latest/Cheetah*.h5'
else:
calib_glob='/gpfs/exfel/exp/SPB/201701/p002013/scratch/calib/r%.4d/Cheetah*.h5'%calib_run
self._make_flist(run, calib_glob)
self._get_nframes_list()
self.frame = np.empty((16,512,128))
self.powder = None
self.train_ids = None
def __init__(
self,
folder_path,
verbose=0,
good_cells=[4, 8, 12, 16, 20, 24, 28],
geom_fname=None,
calib_file='/gpfs/exfel/exp/SPB/201701/p002012/scratch/filipe/offset_and_threshold.h5'
):
self.num_h5cells = 64
self.verbose = verbose
self.good_cells = np.array(good_cells) * 2
self.geom_fname = geom_fname
if self.geom_fname is not None:
self.x, self.y = geom.pixel_maps_from_geometry_file(geom_fname)
self._make_flist(folder_path)
self._get_nframes_list()
self.frame = np.empty((16, 512, 128))
self.powder = None
self.train_ids = None
self.calib = h5py.File(calib_file, 'r')
def __init__(self, filename, geometry=None, goodmask=None):
"""
Parameters
----------
filename : str
Path to sparse HDF5 file
geometry : str
Path to Cheetah-style geometry file
goodmask : str
Path to HDF5 good-pixel mask file
must have a dataset 'data/data' with shape (16,128,512)
"""
Run.__init__(self, filename)
self.geom = geometry
if geometry is not None:
self.x, self.y = geom.pixel_maps_from_geometry_file(self.geom)
if goodmask is None:
self.goodmask = np.ones((16, 128, 512), dtype=np.bool)
else:
self.goodmask = goodmask
assert self.goodmask.shape == (
16, 128, 512), "Mask should have shape (16,128,512)"
def assemble(modules, geomfile):
x, y = geom.pixel_maps_from_geometry_file(geomfile)
image = geom.apply_geom_ij_yx(
(y, x), np.transpose(modules, axes=(0, 2, 1)))[::-1, ::-1]
return image
def __init__(self, vds_file, dir_calib, geo_file, dir_save, nproc,
chunks_to_read, chunk_size, refmod, exclude_modules,
filterbadcells, intensity_threshold, cmode, custommask,
litpixels, litpixels_threshold, averimage, avpixi,
intensity_histogram, hclist, verbose):
self.start_time = time.time()
self.vds_file = vds_file
self.dir_calib = dir_calib
self.geo_file = geo_file
self.dir_save = dir_save
self.chunk_size = chunk_size
self.chunks_to_read = chunks_to_read.copy()
self.refmod = refmod
self.exclude_modules = exclude_modules
self.filterbadcells = filterbadcells
self.intensity_threshold = intensity_threshold
self.cmode = cmode
self.custommask = custommask
self.litpixels = litpixels
self.lithr = litpixels_threshold
self.averimage = averimage
self.avpixi = avpixi
self.intensity_histogram = intensity_histogram
self.hclist = hclist
self.xcca_perform = xcca_perform
self.verbose = verbose
self.detshape = (16, 512, 128
) # data structure for a single frame of AGIPD1M
if self.chunk_size % 32 != 0:
print(
'\nWARNING: Performance is best with a multiple of 32 chunk_size'
)
if nproc == 0:
self.nproc = int(subprocess.check_output('nproc').decode().strip())
else:
self.nproc = nproc
with h5py.File(vds_file, 'r') as f:
self.dset_name = 'INSTRUMENT/' + list(
f['INSTRUMENT'])[0] + '/DET/image/data'
self.mask_name = 'INSTRUMENT/' + list(
f['INSTRUMENT'])[0] + '/DET/image/mask'
self.cellid_name = 'INSTRUMENT/' + list(
f['INSTRUMENT'])[0] + '/DET/image/cellId'
self.trainid_name = 'INSTRUMENT/' + list(
f['INSTRUMENT'])[0] + '/DET/image/trainId'
self.pulseid_name = 'INSTRUMENT/' + list(
f['INSTRUMENT'])[0] + '/DET/image/pulseId'
self.dshape = f[self.dset_name].shape
print('\nResults output directory: {}'.format(self.dir_save))
print('Input vds file: {}'.format(self.vds_file))
if len(self.dshape) == 4:
self.is_raw = False
print('Processing calibrated data, found records for {} frames'.
format(self.dshape[1]))
elif len(self.dshape) == 5:
self.is_raw = True
print('Processing raw data, found records for {} frames'.format(
self.dshape[1]))
# chunk reading scheme related stuff
num_chunks_max = int(math.ceil(
self.dshape[1] /
self.chunk_size)) # maximum possible number of data chunks
if self.verbose >= 1:
print(
'Maximum number of data chunks ({} frames each) is {} '.format(
self.chunk_size, num_chunks_max))
self.chunks_to_read[1] = min(self.chunks_to_read[1], num_chunks_max)
self.num_chunks = int(
math.ceil((self.chunks_to_read[1] - self.chunks_to_read[0]) /
self.chunks_to_read[2]))
if self.num_chunks <= 0:
print(
'Specified scheme ({}) for reading data in chunks cannot be used. Exiting...'
.format(chunks_to_read))
sys.exit(1)
self.num_imgs_good = 0 # current number of good images
self.num_imgs_read = 0 # current number of images read from the input file
self.num_chk_read = 0 # current number of chunks read from the input file
if self.is_raw: # open calibration files
calib_str = self.dir_calib + 'Cheetah*.h5'
self.calib = [
h5py.File(f, 'r') for f in sorted(glob.glob(calib_str))
]
print('AGIPD detector geometry file: {}'.format(self.geo_file))
try:
print('Allocating shared memory arrays...')
self.modules_shape = (
self.chunk_size,
) + self.detshape # data format specificly for AGIPD1M detector
self.modules_size = int(
self.modules_shape[0] * self.modules_shape[1] *
self.modules_shape[2] * self.modules_shape[3])
self.modules_data = np.frombuffer(
mp.Array(ctypes.c_double,
self.modules_size).get_obj()).reshape(
self.modules_shape) # scattering data
self.mask_data = np.frombuffer(
mp.Array(ctypes.c_uint, self.modules_size).get_obj(),
dtype=np.uint32).reshape(self.modules_shape) # mask
self.cellid_data = np.frombuffer(
mp.Array(ctypes.c_uint, self.modules_shape[0]).get_obj(),
dtype=np.uint32).reshape(self.modules_shape[0]) # cell IDs
self.pulseid_data = np.frombuffer(
mp.Array(ctypes.c_uint, self.modules_shape[0]).get_obj(),
dtype=np.uint32).reshape(self.modules_shape[0]) # pulse IDs
self.trainid_data = np.frombuffer(
mp.Array(ctypes.c_uint, self.modules_shape[0]).get_obj(),
dtype=np.uint32).reshape(self.modules_shape[0]) # train IDs
self.is_good_data = np.frombuffer(
mp.Array(ctypes.c_uint, self.modules_shape[0]).get_obj(),
dtype=np.uint32
).reshape(
self.modules_shape[0]
) # image quality identifier: 1 - good image, 0 - bad image; used to label bad images
self.is_good_data_all = np.frombuffer(mp.Array(
ctypes.c_uint, self.dshape[1]).get_obj(),
dtype=np.uint32) # litpixels
if self.litpixels:
self.litpix = np.frombuffer(
mp.Array(ctypes.c_ulong,
self.dshape[0] * self.dshape[1]).get_obj(),
dtype='u8').reshape(16, -1) # litpixels
if self.avpixi:
self.pixintens = np.frombuffer(
mp.Array(ctypes.c_double, self.dshape[0] *
self.dshape[1]).get_obj()).reshape(
16, -1) # average intensity/pixel
self.pixcnts = np.frombuffer(
mp.Array(ctypes.c_ulong,
self.dshape[0] * self.dshape[1]).get_obj(),
dtype='u8').reshape(16, -1) # counts pixels
if self.averimage:
self.img_sum = np.frombuffer(
mp.Array(
ctypes.c_double, self.detshape[0] * self.detshape[1] *
self.detshape[2]).get_obj()).reshape(
self.detshape) # powder pattern
self.pix_sum = np.frombuffer(
mp.Array(
ctypes.c_uint, self.detshape[0] * self.detshape[1] *
self.detshape[2]).get_obj(),
dtype=np.uint32).reshape(
self.detshape) # sum of contributions
if self.intensity_histogram[0] == True:
self.hist_shape = (self.chunk_size,
self.intensity_histogram[2])
self.hist_vals = np.frombuffer(
mp.Array(ctypes.c_double, self.hist_shape[0] *
self.hist_shape[1]).get_obj()).reshape(
self.hist_shape)
# arrays for assembled images processing
self.det_y, self.det_x = geom.pixel_maps_from_geometry_file(
self.geo_file
) # pixel coordinates of the assembled detector image
self.img_shape_assembled = geom.apply_geom_ij_yx(
(self.det_x, self.det_y), np.zeros(self.detshape)).shape
#print('\nAssembled image shape={}'.format(self.img_shape_assembled))
self.img_shape = (
self.chunk_size,
) + self.img_shape_assembled # for assembled images
self.img_size = int(self.img_shape[0] * self.img_shape[1] *
self.img_shape[2])
self.img_data = None # assembled image of scattering data, will be allocated later if required
self.msk_data = None # assembled image of mask
print('Finished with memory allocation')
except:
print('Problems with memory allocation. Exiting...')
sys.exit(1)
# create custom detector mask
if self.custommask:
self._mask_custom_agipd()
if self.filterbadcells:
self.define_bad_cells() # list of bad memroy cells
# intensity histogram related stuff
if self.intensity_histogram[0] == True:
self.dir_hist = self.dir_save + './histograms/'
if os.path.exists(self.dir_hist) is not True:
os.makedirs(self.dir_hist)