def test_scrip_runs_and_produces_correct_outputs(folder_test_dst,
test_dst_file,
output_maps_tmdir,
test_map_file):
"""
Run map creation script and check if an ASectormap is the output.
"""
map_file_out = os.path.join(output_maps_tmdir, 'test_out_map.h5')
histo_file_out = os.path.join(output_maps_tmdir, 'test_out_histo.h5')
default_n_bins = 15
run_number = 7517
config = configure(
'maps $ICARO/krcal/map_builder/config_LBphys.conf'.split())
map_params_new = copy.copy(config.as_namespace.map_params)
map_params_new['nmin'] = 100
map_params_new['nStimeprofile'] = 1200
config.update(
dict(folder=folder_test_dst,
file_in=test_dst_file,
file_out_map=map_file_out,
file_out_hists=histo_file_out,
default_n_bins=default_n_bins,
run_number=run_number,
map_params=map_params_new))
map_builder(config.as_namespace)
maps = read_maps(map_file_out)
assert type(maps) == ASectorMap
old_maps = read_maps(test_map_file)
assert_dataframes_close(maps.e0, old_maps.e0, rtol=1e-5)
assert_dataframes_close(maps.e0u, old_maps.e0u, rtol=1e-5)
assert_dataframes_close(maps.lt, old_maps.lt, rtol=1e-5)
assert_dataframes_close(maps.ltu, old_maps.ltu, rtol=1e-5)
def test_correct_map_with_unsorted_dst(folder_test_dst, test_dst_file,
output_maps_tmdir):
"""
This test shuffles the input dst, and checks that the map is the same
as the one created with the same sorted dst.
"""
map_file_sort = os.path.join(output_maps_tmdir, 'test_out_map.h5')
map_file_unsort = os.path.join(output_maps_tmdir, 'test_out_unsort.h5')
histo_file_out = os.path.join(output_maps_tmdir, 'test_out_histo.h5')
dst = load_dst(folder_test_dst + test_dst_file, 'DST', 'Events')
if "index" in dst: del dst["index"]
dst = dst.sort_values(by=['S2e'])
tmp_unsorted_dst = 'unsorted_dst.h5'
dst.to_hdf(output_maps_tmdir + tmp_unsorted_dst,
key="DST",
mode="w",
format="table",
data_columns=True,
complib="zlib",
complevel=4)
with tb.open_file(output_maps_tmdir + tmp_unsorted_dst, "r+") as file:
file.rename_node(file.root.DST.table, "Events")
file.root.DST.Events.title = "Events"
default_n_bins = 15
run_number = 7517
config = configure(
'maps $ICARO/krcal/map_builder/config_LBphys.conf'.split())
map_params_new = config.as_namespace.map_params
map_params_new['nmin'] = 100
config.update(
dict(folder=output_maps_tmdir,
file_in=tmp_unsorted_dst,
file_out_map=map_file_unsort,
file_out_hists=histo_file_out,
default_n_bins=default_n_bins,
run_number=run_number,
map_params=map_params_new))
map_builder(config.as_namespace)
unsorted_maps = read_maps(map_file_unsort)
sorted_maps = read_maps(map_file_sort)
assert_dataframes_close(unsorted_maps.e0, sorted_maps.e0, rtol=1e-5)
assert_dataframes_close(unsorted_maps.e0u, sorted_maps.e0u, rtol=1e-5)
assert_dataframes_close(unsorted_maps.lt, sorted_maps.lt, rtol=1e-5)
assert_dataframes_close(unsorted_maps.ltu, sorted_maps.ltu, rtol=1e-5)
def test_time_evol_table_correct_elements(output_maps_tmdir):
map_file_out = os.path.join(output_maps_tmdir, 'test_out_map.h5')
emaps = read_maps(map_file_out)
time_table = emaps.t_evol
columns = time_table.columns
elements = [
'ts', 'e0', 'e0u', 'lt', 'ltu', 'dv', 'dvu', 'resol', 'resolu', 's1w',
's1wu', 's1h', 's1hu', 's1e', 's1eu', 's2w', 's2wu', 's2h', 's2hu',
's2e', 's2eu', 's2q', 's2qu', 'Nsipm', 'Nsipmu', 'Xrms', 'Xrmsu',
'Yrms', 'Yrmsu', 'S1eff', 'S2eff', 'Bandeff'
]
for element in elements:
assert element in columns
def load_data(
input_path: str, input_dsts: str, file_bootstrap_map: str,
ref_histo_file: str, key_Z_histo: str, quality_ranges: dict
) -> Tuple[pd.DataFrame, ASectorMap, ref_hist_container]:
"""
Reads kdst files and applies basic R cut. Outputs kdst as pd.DataFrame,
bootstrap map, and reference histograms
Parameters
----------
input_path : str
Path to the input map_folder
input_dsts : str
Name criteria for the dst to be read
file_bootstrap_map : str
Path to the bootstrap map file
ref_Z_histo_file : str
Path to the reference histogram file
quality_ranges : dict
Dictionary containing ranges for the quality cuts
Returns
----------
dst_filtered : pd.DataFrame
Dst containing all the events once filtered
bootstrap_map : ASectorMap
Bootstrap map
ref_hist_container : ref_hist_container
To be completed
"""
input_path = os.path.expandvars(input_path)
dst_files = glob.glob(input_path + input_dsts)
dst_full = load_dsts(dst_files, "DST", "Events")
dst_full = dst_full.sort_values(by=['time'])
mask_quality = quality_cut(dst_full, **quality_ranges)
dst_filtered = dst_full[mask_quality]
file_bootstrap_map = os.path.expandvars(file_bootstrap_map)
bootstrap_map = read_maps(file_bootstrap_map)
ref_histo_file = os.path.expandvars(ref_histo_file)
z_pd = pd.read_hdf(ref_histo_file, key=key_Z_histo)
z_histo = ref_hist(bin_centres=z_pd.bin_centres,
bin_entries=z_pd.bin_entries,
err_bin_entries=z_pd.err_bin_entries)
ref_histos = ref_hist_container(Z_dist_hist=z_histo)
return dst_filtered, bootstrap_map, ref_histos
def test_maps_nans_outside_rmax(xy_pos, output_maps_tmdir):
map_file_out = os.path.join(output_maps_tmdir, 'test_out_map.h5')
xs, ys = xy_pos
emaps = read_maps(map_file_out)
get_coef = maps_coefficient_getter(emaps.mapinfo, emaps.e0)
r_max = emaps.mapinfo.xmax
nbins = emaps.mapinfo.nx
bins = np.linspace(-200, 200, nbins)
xs = bins[np.digitize(xs, bins, right=True)]
ys = bins[np.digitize(ys, bins, right=True)]
coefs = get_coef(xs, ys)
maskin = np.sqrt(xs**2 + ys**2) < r_max
maskout = np.sqrt(xs**2 + ys**2) >= r_max + 2 * r_max / nbins
assert all(np.isnan(coefs[maskout]))
assert all(np.isfinite(coefs[maskin]))
def get_psf_pmt_from_krmap(filename, factor=1.):
""" reads KrMap and generate a psf function with the E0-map
"""
maps = read_maps(filename)
xmin = maps.mapinfo.xmin
xmax = maps.mapinfo.xmax
ymin = maps.mapinfo.ymin
ymax = maps.mapinfo.ymax
nx = maps.mapinfo.nx
ny = maps.mapinfo.ny
dx = (xmax - xmin) / float(nx)
dy = (ymax - ymin) / float(ny)
xymap = factor * np.nan_to_num(np.array(maps.e0), 0.)
return _xyfun((xmin, xmax, dx), (ymin, ymax, dy), xymap)
def fast_prod(s1emin, s1wmin, pmt_ids,\
run, files_in, nfin, file_out, n_baseline,\
n_mau, thr_mau, thr_csum_s1, thr_csum_s2, sipm_thr,\
s1_tmin, s1_tmax, s1_stride, s1_lmin, s1_lmax, s1_rebin_stride,\
s2_tmin, s2_tmax, s2_stride, s2_lmin, s2_lmax ,s2_rebin_stride,\
thr_sipm_s2, detector_db,\
qth_penth, rebin,\
qth_esmer, map_file, apply_temp):
# FILE OUT
h5file = tb.open_file(file_out, mode="w", title="Fast Prod")
group = h5file.create_group("/", "Summary", "Summary")
h5file.create_earray(group, "Z", tb.Float64Atom(), shape=(0, ))
h5file.create_earray(group, "DZ", tb.Float64Atom(), shape=(0, ))
h5file.create_earray(group, "E", tb.Float64Atom(), shape=(0, ))
h5file.create_earray(group, "Q", tb.Float64Atom(), shape=(0, ))
h5file.create_earray(group, "Ec", tb.Float64Atom(), shape=(0, ))
group = h5file.create_group("/", f"Event_Info", "Info")
class Event_Info(tb.IsDescription):
event = tb.Int32Col()
time = tb.UInt64Col()
Event_Info_table = h5file.create_table(group, "Event_Time", Event_Info,
"Event_Time")
EI = Event_Info_table.row
files_in = glob.glob(files_in)
files_in.sort()
files_in = files_in[:nfin]
# DATASIPMs AND MAPS
datasipm = load_db.DataSiPM("new", run)
sipm_xs = datasipm.X.values
sipm_ys = datasipm.Y.values
maps = read_maps(map_file)
total_correction = apply_all_correction(maps,
apply_temp=apply_temp,
norm_strat=norm_strategy.kr)
# FAST PROD
_Z, _DZ, _E, _Q, _Ec = [], [], [], [], []
for file in files_in:
print(file)
RWFs_file = tb.open_file(file)
pmt_rwfs_all = RWFs_file.root.RD.pmtrwf
sipm_rwfs_all = RWFs_file.root.RD.sipmrwf
time_stamps = RWFs_file.root.Run.events.read()
for event_time, pmt_rwfs, sipm_rwfs in zip(time_stamps, pmt_rwfs_all,
sipm_rwfs_all):
################################
############ IRENE #############
################################
#pmt processing
rwf_to_cwf = deconv_pmt(detector_db, run, n_baseline)
pmt_cwfs = rwf_to_cwf(pmt_rwfs)
cwf_to_ccwf = calibrate_pmts(detector_db, run, n_mau, thr_mau)
pmt_ccwfs, ccwfs_mau, cwf_sum, cwf_sum_mau = cwf_to_ccwf(pmt_cwfs)
#sipm processing
sipm_rwf_to_cal = calibrate_sipms(detector_db, run, sipm_thr)
sipm_cwfs = sipm_rwf_to_cal(sipm_rwfs)
#Find signals
zero_suppress = zero_suppress_wfs(thr_csum_s1, thr_csum_s2)
s1_indices, s2_indices = zero_suppress(cwf_sum, cwf_sum_mau)
s1_selected_splits,\
s2_selected_splits = utils.signals_selected_splits(s1_indices, s2_indices,
s1_stride , s2_stride,
s1_tmin , s1_tmax , s1_lmin, s1_lmax,
s2_tmin , s2_tmax , s2_lmin, s2_lmax)
######## 1S1 1S2 CUT ##########
S1_time = utils._1s1_1s2(pmt_ccwfs, s2_selected_splits,
s1_selected_splits, s1emin, s1wmin)
if not S1_time: continue
# Rebin S2_pmts
times, rebinned_widths, s2_pmts = pick_slice_and_rebin(
s2_selected_splits[0],
np.arange(pmt_ccwfs.shape[1]) * 25 * units.ns,
np.full(pmt_ccwfs.shape[1], 25 * units.ns),
pmt_ccwfs,
rebin_stride=s2_rebin_stride,
pad_zeros=True)
#select and thr_sipm_s2
s2_sipms = sipm_cwfs[:, s2_selected_splits[0][0] //
40:s2_selected_splits[0][-1] // 40 + 1]
sipm_ids, s2_sipms = select_wfs_above_time_integrated_thr(
s2_sipms, thr_sipm_s2)
######## IRENE FINAL S2 WFS #######
s2_pmts = np.float32(s2_pmts)
s2_sipms = np.float32(s2_sipms)
times = np.float32(times)
#select pmt_ids wfs
c = np.zeros(s2_pmts.shape[0])
c[pmt_ids] = 1
s2_pmts = np.multiply(c, s2_pmts.T).T
################################
######## PENTHESILEA ###########
################################
########## Rebin ############
_, _, s2_sipms = rebin_times_and_waveforms(times,
rebinned_widths,
s2_sipms,
rebin_stride=rebin,
slices=None)
times, _, s2_pmts = rebin_times_and_waveforms(times,
rebinned_widths,
s2_pmts,
rebin_stride=rebin,
slices=None)
######### Charge cut #########
s2_pmts_penth = np.copy(s2_pmts)
s2_sipms_penth = np.where(s2_sipms >= qth_penth, s2_sipms, 0)
###### create penthesilea hits ########
hits = utils.create_penthesilea_hits(s2_pmts_penth, s2_sipms_penth,
sipm_xs, sipm_ys, sipm_ids,
times, S1_time)
################################
######### ESMERALDA ############
################################
#### Charge cut ####
hits = utils.esmeralda_charge_cut(hits, qth_esmer)
###### join NN hits ######
hits = utils.join_NN_hits(hits)
#### Corrections ######
X, Y, Z = hits["X"], hits["Y"], hits["Z"]
E, Q = hits["E"], hits["Q"]
T = np.full(len(hits), event_time[-1] / 1000)
correction_factor = total_correction(X, Y, Z, T)
Ec = correction_factor * E
hits["Ec"] = Ec
hits["Z"] = Z * maps.t_evol.dv.mean()
###########################
####### APPEND DATA #######
###########################
# Event Info
EI["event"] = event_time[0]
EI["time"] = event_time[1]
EI.append()
## Z, DZ, E, Q, Ec
Z, E, Q, Ec = hits["Z"], hits["E"], hits["Q"], hits["Ec"]
Ec[np.isnan(Ec)] = 0
_Z.append(np.sum(Ec * Z) / np.sum(Ec))
_DZ.append(np.max(Z) - np.min(Z))
_E.append(np.sum(E))
_Q.append(np.sum(Q))
_Ec.append(np.sum(Ec))
# close RWF file
RWFs_file.close()
h5file.root.Summary.Z.append(_Z)
h5file.root.Summary.DZ.append(_DZ)
h5file.root.Summary.E.append(_E)
h5file.root.Summary.Q.append(_Q)
h5file.root.Summary.Ec.append(_Ec)
#write to disk
h5file.flush()
h5file.close()
def test_time_evol_eff_less_one(output_maps_tmdir):
map_file_out = os.path.join(output_maps_tmdir, 'test_out_map.h5')
emaps = read_maps(map_file_out)
assert np.all(emaps.t_evol.S1eff <= 1.)
assert np.all(emaps.t_evol.S2eff <= 1.)
assert np.all(emaps.t_evol.Bandeff <= 1.)
def test_time_evol_table_exact_numbers(t_evol_table, output_maps_tmdir):
map_file_out = os.path.join(output_maps_tmdir, 'test_out_map.h5')
emaps = read_maps(map_file_out)
t_evol = pd.pandas.read_hdf(t_evol_table, 't_evol')
assert_dataframes_close(emaps.t_evol, t_evol, rtol=1e-5)
def MAPS(MAPSDIR):
map_fn = os.path.join(MAPSDIR, "kr_emap_xy_100_100_r_6346.h5")
maps = read_maps(filename=map_fn)
return maps
