def load_crop_JF_batches(fname, roi1, roi2,roi3,roi4,
gain_file=None, pedestal_file=None, nshots=None, batch_size = 1000):
with ju.File(fname, gain_file=gain_file, pedestal_file=pedestal_file) as juf:
pulse_ids = juf["pulse_id"][:nshots].T[0]
n_images = pulse_ids.shape[0]
images_roi1 = []
images_roi2 = []
images_roi3 = []
images_roi4 = []
print ('Total images = {}, load them in batches of {}'.format(n_images, batch_size))
for ind in range(0, n_images, batch_size):
batch_slice = slice(ind, min(ind + batch_size, n_images))
print ('Load batch = {}'.format(batch_slice))
batch_images = juf[batch_slice, :, :]
images_roi1.extend(crop_roi(batch_images, roi1))
images_roi2.extend(crop_roi(batch_images, roi2))
images_roi3.extend(crop_roi(batch_images, roi3))
images_roi4.extend(crop_roi(batch_images, roi4))
del batch_images
images_roi1 = np.asarray(images_roi1)
images_roi2 = np.asarray(images_roi2)
images_roi3 = np.asarray(images_roi3)
images_roi4 = np.asarray(images_roi4)
return images_roi1, images_roi2, images_roi3, images_roi4, pulse_ids
def load_crop_JF_data_on_off(fname,
roi1,
roi2,
reprate_FEL,
reprate_laser,
gain_file=None,
pedestal_file=None,
nshots=None):
with ju.File(fname, gain_file=gain_file,
pedestal_file=pedestal_file) as juf:
images = juf[:nshots]
pulse_ids = juf["pulse_id"][:nshots].T[0]
images_roi1 = crop_roi(images, roi1)
images_roi2 = crop_roi(images, roi2)
reprate_on, reprate_off = _make_reprates_on_off(pulse_ids, reprate_FEL,
reprate_laser)
images_on_roi1 = images_roi1[reprate_on]
images_on_roi2 = images_roi2[reprate_on]
images_off_roi1 = images_roi1[reprate_off]
images_off_roi2 = images_roi2[reprate_off]
pulse_ids_on = pulse_ids[reprate_on]
pulse_ids_off = pulse_ids[reprate_off]
return images_on_roi1, images_on_roi2, pulse_ids_on, images_off_roi1, images_off_roi2, pulse_ids_off
def load_JF_data_4rois_on_off(fname, index_light, index_dark, roi1, roi2,roi3,roi4,
gain_file=None, pedestal_file=None, nshots=None):
with ju.File(fname, gain_file=gain_file, pedestal_file=pedestal_file) as juf:
images = juf[:nshots]
pulse_ids = juf["pulse_id"][:nshots].T[0]
images_roi1 = crop_roi(images, roi1)
images_on_roi1 = images_roi1[index_light]
images_off_roi1 = images_roi1[index_dark]
images_roi2 = crop_roi(images, roi2)
images_on_roi2 = images_roi2[index_light]
images_off_roi2 = images_roi2[index_dark]
images_roi3 = crop_roi(images, roi3)
images_on_roi3 = images_roi3[index_light]
images_off_roi3 = images_roi3[index_dark]
images_roi4 = crop_roi(images, roi4)
images_on_roi4 = images_roi4[index_light]
images_off_roi4 = images_roi4[index_dark]
pulse_ids_on = pulse_ids[index_light]
pulse_ids_off = pulse_ids[index_dark]
return images_on_roi1, images_off_roi1, images_on_roi2, images_off_roi2, images_on_roi3, images_off_roi3, images_on_roi4, images_off_roi4, pulse_ids_on, pulse_ids_off
def load_crop_JF_data(fname, roi1, roi2,roi3,roi4,
gain_file=None, pedestal_file=None, nshots=None):
with ju.File(fname, gain_file=gain_file, pedestal_file=pedestal_file) as juf:
images = juf[:nshots]
pulse_ids = juf["pulse_id"][:nshots].T[0]
images_roi1 = crop_roi(images, roi1)
images_roi2 = crop_roi(images, roi2)
images_roi3 = crop_roi(images, roi3)
images_roi4 = crop_roi(images, roi4)
return images_roi1, images_roi2, images_roi3, images_roi4, pulse_ids
def load_crop_JF_batches_on_off_2rois(fname, roi1, roi2, reprate_FEL, reprate_laser,
gain_file=None, pedestal_file=None, nshots=None, batch_size = 1000):
with ju.File(fname, gain_file=gain_file, pedestal_file=pedestal_file) as juf:
pulse_ids = juf["pulse_id"][:nshots].T[0]
n_images = pulse_ids.shape[0]
images_roi1 = []
images_roi2 = []
print ('Total images = {}, load them in batches of {}'.format(n_images, batch_size))
for ind in range(0, n_images, batch_size):
batch_slice = slice(ind, min(ind + batch_size, n_images))
print ('Load batch = {}'.format(batch_slice))
batch_images = juf[batch_slice, :, :]
images_roi1.extend(crop_roi(batch_images, roi1))
images_roi2.extend(crop_roi(batch_images, roi2))
del batch_images
images_roi1 = np.asarray(images_roi1)
images_roi2 = np.asarray(images_roi2)
reprate_on, reprate_off = _make_reprates_on_off(pulse_ids, reprate_FEL, reprate_laser)
images_on_roi1 = images_roi1[reprate_on]
images_on_roi2 = images_roi2[reprate_on]
images_off_roi1 = images_roi1[reprate_off]
images_off_roi2 = images_roi2[reprate_off]
pulse_ids_on = pulse_ids[reprate_on]
pulse_ids_off = pulse_ids[reprate_off]
return images_on_roi1, images_on_roi2, pulse_ids_on, images_off_roi1, images_off_roi2, pulse_ids_off
#TODO
#import numpy as np
#output_folder = "/".join(ofname.split("/")[:-2])
#roi1_file = output_folder + "/roi1.txt"
#roi2_file = output_folder + "/roi2.txt"
#roi1 = np.genfromtxt(roi1_file).astype(int)
#roi2 = np.genfromtxt(roi2_file).astype(int)
clock = Clock()
print("It took", clock.tick(), "seconds to set things up.")
print("Processing", ifname)
with ju.File(ifname, gain_file=fname_gain, pedestal_file=fname_pede) as juf:
print("Detector name:", juf.detector_name)
print("> Load jungfrau data")
pulse_ids = juf["pulse_id"][:]
images = juf[:]
print("> Crop")
images_roi1 = crop_roi(images, roi1)
images_roi2 = crop_roi(images, roi2)
images_roi3 = crop_roi(images, roi3)
images_roi4 = crop_roi(images, roi4)
if 0 in images_roi1.shape:
print("ROI1 seems to have removed too much. cropped images shape:",
images_roi1.shape)
def load_from_ju_file(fname):
juf = ju.File(fname)
name = juf.detector_name
chan = SFChannelJF(name, juf)
return juf, {name: chan}
def showdata(self, runno, step,exclude_files, maxshots = None, clim=(0,20), jf=None, hist_lineout = False, autolocate_pedestal=True):
if jf is None:
jf = list(self.cfg['JFs'].keys())[0]
if autolocate_pedestal:
for j in self.cfg['JFs'].keys():
jfcfg = self.cfg['JFs'][j]
ped_file = self.autolocate_closest_pedestal(j, runno)
jfcfg['pedestal_file'] = ped_file
jf_cfg = self.cfg['JFs'][jf]
pgroup = self.cfg['Exp_config']['pgroup']
pedestal_file = jf_cfg['pedestal_file']
json_dir = Path(f'/sf/bernina/data/{pgroup}/res/scan_info/')
print(json_dir)
json_file = list(json_dir.glob(f'run{runno:04}*'))[0]
data = {}
p = Path(json_file)
with p.open(mode="r") as f:
s = json.load(f)
f = [f for f in s['scan_files'][step] if jf_cfg['id'] in f][0]
with ju.File(file_path= f, pedestal_file=pedestal_file, geometry=False, gap_pixels=False) as juf:
if maxshots:
imgs_ju = juf[:maxshots]
else:
imgs_ju = juf[:]
threshold = jf_cfg['threshold']
if not threshold == None:
imgs_ju[imgs_ju<threshold]=0
energy_limits = jf_cfg['energy_limits']
if not energy_limits == None:
imgs_ju[imgs_ju<np.min(energy_limits)]=0
imgs_ju[imgs_ju>np.max(energy_limits)]=0
nshots = len(imgs_ju)
img = np.nansum(imgs_ju,axis=0)
def hist_only_y(*args,**kwargs):
y, bins = np.histogram(*args,**kwargs)
return y
bins = np.linspace(-5,50,400)
if hist_lineout:
hists_vertical = np.apply_along_axis(hist_only_y, 1,np.hstack(imgs_ju), bins=bins)
data['hists_vertical']=hists_vertical
hist_av = np.histogram(imgs_ju[:100], bins=bins)
data['hist_av']=hist_av
figname = 'Run_{}, Step_{}. Nbr_shots = {}'.format(runno,step, nshots)
plt.close(figname)
fig = plt.figure(num=figname, figsize=(9,9))
ax0= fig.add_subplot(1,2,1)
ax0.imshow(img,clim=clim,origin='lower')
ax1=[]
rois = jf_cfg['rois']
rois_img = jf_cfg['rois_img']
print(rois)
print(rois_img)
for i in range(1,len(rois)+len(rois_img)+1):
ax1.append(fig.add_subplot(len(rois)+len(rois_img),2,2*i))
for i,(r,r_rng) in enumerate(rois.items()):
if type(r_rng)==dict:
r_rng = r_rng['range']
ax1[i].imshow(self.mkroi(img, r_rng),clim=clim)
ax1[i].set_title('{}'.format(r))
rr =r_rng
ax0.add_patch(Rectangle((rr[2], rr[0]), abs(rr[2]-rr[3]),abs(rr[1]-rr[0]),fill=None,
alpha=1,linewidth=2, edgecolor='r'))
ax0.text(rr[2],rr[1],r,color='white')
for i,(r,r_rng) in enumerate(rois_img.items()):
i +=len(rois.keys())
ax1[i].imshow(self.mkroi(img, r_rng),clim=clim)
ax1[i].set_title('Img {}'.format(r))
rr =r_rng
ax0.add_patch(Rectangle((rr[2], rr[0]), abs(rr[2]-rr[3]),abs(rr[1]-rr[0]),fill=None,
alpha=1,linewidth=2, edgecolor='g'))
plt.tight_layout()
hist = plt.figure('histogram', figsize=(9,3))
plt.plot(hist_av[1][:-1],hist_av[0])
plt.xlabel('energy (keV)')
plt.ylabel('px/bin')
plt.yscale('log')
plt.ylim(hist_av[0][-1],np.max(hist_av[0]))
plt.tight_layout()
data['img']=img
data['fig']=fig
data['hist_bins']=bins
return data
def convert_file(file_in, file_out, json_run_file, detector_config_file):
with open(detector_config_file, "r") as detector_file:
data = json.load(detector_file)
detector_name = data["detector_name"]
gain_file = data["gain_file"]
pedestal_file = data["pedestal_file"]
with open(json_run_file, "r") as run_file:
data = json.load(run_file)
detector_params = data["detectors"][detector_name]
compression = detector_params.get("compression", False)
conversion = detector_params.get("adc_to_energy", False)
disabled_modules = detector_params.get("disabled_modules", [])
remove_raw_files = detector_params.get("remove_raw_files", False)
if conversion:
mask = detector_params.get("mask", True)
double_pixels_action = detector_params.get("double_pixels_action", "mask")
geometry = detector_params.get("geometry", False)
gap_pixels = detector_params.get("gap_pixels", True)
factor = detector_params.get("factor", None)
else:
mask = False
double_pixels_action = "keep"
geometry = False
gap_pixels = False
factor = None
files_to_remove = set()
file_tmp = file_in
if len(disabled_modules)>0:
files_to_remove.add(file_in)
_logger.info(f"Will reduce data file, disabled_modules: {disabled_modules}")
if conversion:
file_tmp = file_out+".tmp"
files_to_remove.add(file_tmp)
else:
file_tmp = file_out
postprocess_raw(file_in, file_tmp, compression=compression, disabled_modules=disabled_modules)
if conversion:
files_to_remove.add(file_in)
with ju.File(
file_tmp,
gain_file=gain_file,
pedestal_file=pedestal_file,
conversion=conversion,
mask=mask,
double_pixels=double_pixels_action,
gap_pixels=gap_pixels,
geometry=geometry,
parallel=True,
) as juf:
n_input_frames = len(juf["data"])
good_frames = np.nonzero(juf["is_good_frame"])[0]
n_output_frames = len(good_frames)
juf.export(
file_out,
index=good_frames,
roi=None,
compression=compression,
factor=factor,
dtype=None,
batch_size=35,
)
else:
with h5py.File(file_tmp, "r") as juf:
n_input_frames = len(juf[f"data/{detector_name}/data"])
good_frames = np.nonzero(juf[f"data/{detector_name}/is_good_frame"])[0]
n_output_frames = len(good_frames)
# Utility info
with h5py.File(file_out, "r") as h5f:
_logger.info("daq_rec : %s" % h5f[f"/data/{detector_name}/daq_rec"][0, 0])
frame_index = h5f[f"/data/{detector_name}/frame_index"][:]
_logger.info("frame_index range: (%d - %d)" % (np.min(frame_index), np.max(frame_index)))
_logger.info(f"input frames : {n_input_frames}")
_logger.info(f"bad frames : {n_input_frames - n_output_frames}")
_logger.info(f"output frames: {n_output_frames}")
_logger.info(f"gain_file : {gain_file}")
_logger.info(f"pedestal_file : {pedestal_file}")
_logger.info(f"conversion : {conversion}")
_logger.info(f"mask : {mask}")
_logger.info(f"double_pixels_action: {double_pixels_action}")
_logger.info(f"geometry : {geometry}")
_logger.info(f"gap_pixels : {gap_pixels}")
_logger.info(f"compression : {compression}")
_logger.info(f"factor : {factor}")
if remove_raw_files:
_logger.info(f'removing raw and temporary files {files_to_remove}')
for file_remove in files_to_remove:
os.remove(file_remove)
if conversion:
file_tmp = args.file_out + ".tmp"
else:
file_tmp = args.file_out
postprocess_raw.postprocess_raw(args.file_in,
file_tmp,
compression=compression,
disabled_modules=disabled_modules)
if conversion:
with ju.File(
file_tmp,
gain_file=gain_file,
pedestal_file=pedestal_file,
conversion=conversion,
mask=mask,
gap_pixels=gap_pixels,
geometry=geometry,
parallel=False,
) as juf:
n_input_frames = len(juf["data"])
good_frames = np.nonzero(juf["is_good_frame"])[0]
n_output_frames = len(good_frames)
juf.handler.mask_double_pixels = mask_double_pixels
juf.export(
args.file_out,
index=good_frames,
roi=None,
compression=compression,
factor=factor,