def load_sensors(file_names : List[str],
db_file : str ,
run_no : int ) -> Generator[Tuple, None, None]:
pmt_ids = DB.DataPMT (db_file, run_no).SensorID
sipm_ids = DB.DataSiPM(db_file, run_no).SensorID
for file_name in file_names:
(all_evt ,
pmt_binwid ,
sipm_binwid,
all_wf ) = load_mcsensor_response_df(file_name, db_file, run_no)
with tb.open_file(file_name, 'r') as h5in:
mc_info = tbl.get_mc_info(h5in)
timestamps = event_timestamp(h5in)
for evt in all_evt:
pmt_wfs = all_wf.loc[evt].loc[ pmt_ids]
sipm_wfs = all_wf.loc[evt].loc[sipm_ids]
yield dict(evt = evt ,
mc = mc_info ,
timestamp = timestamps(),
pmt_binwid = pmt_binwid ,
sipm_binwid = sipm_binwid ,
pmt_wfs = pmt_wfs ,
sipm_wfs = sipm_wfs )
def tools(correction_filename, run_number):
calibrate = corrections.Calibration(correction_filename, 'scale')
datasipm = db.DataSiPM(run_number)
xpos, ypos = datasipm.X.values, datasipm.Y.values
return calibrate, xpos, ypos
def clarice(run_number,
input_filenames,
output_filename,
read_fun=hpfun.get_pmaps_gd):
# initalize
correction_filename = f"$IC_DATA/maps/kr_corrections_run{run_number}.h5"
correction_filename = os.path.expandvars(correction_filename)
calibrate = corrections.Calibration(correction_filename, 'scale')
datasipm = db.DataSiPM(run_number)
sipms_xs, sipms_ys = datasipm.X.values, datasipm.Y.values
ntotal, naccepted = 0, 0
for file in input_filenames:
print('processing ', file)
try:
pmaps = read_fun(file)
except:
continue
dfevent, dfslice, _ = hpfun.create_event_dfs(pmaps, sipms_xs, sipms_ys,
calibrate)
dfevent.to_hdf(output_filename, key='pkevent', append=True)
dfslice.to_hdf(output_filename, key='pkslice', append=True)
ntotal += len(set(pmaps.s1.event))
naccepted += len(set(dfevent.event))
f = 100. * naccepted / (1. * ntotal) if ntotal > 0 else 0.
print('total events ', ntotal, ', accepted ', naccepted, 'fraction (%)',
f)
def __init__(self):
super(NEW_meta, self).__init__()
# Load the database objects
self._det_geo = load_db.DetectorGeo()
self._pmt_data = load_db.DataPMT()
self._sipm_data = load_db.DataSiPM()
self.skim_det_info(self._det_geo)
def __init__(self):
super(io_manager, self).__init__()
# Load the database objects
self._det_geo = load_db.DetectorGeo()
self._pmt_data = load_db.DataPMT()
self._sipm_data = load_db.DataSiPM()
self._events = []
self._entry = 0
self._max_entry = 0
self._run = 5
def compare_runs(run_no, infiles):
#run_no = int(sys.argv[1]) # Only for sensor positions and mapping
#infiles = sys.argv[2:]
sensor_x = DB.DataSiPM(det_db, run_no).X.values
sensor_y = DB.DataSiPM(det_db, run_no).Y.values
chi_vals = []
mu_vals = []
for ifile in infiles:
chis, mus = pde(run_no, ifile)
chi_vals.append(chis)
mu_vals.append(mus)
fig, axes = plt.subplots(nrows=2, ncols=len(infiles), figsize=(20, 6))
for chis, mus, ax, fl in zip(chi_vals, mu_vals, axes[0], infiles):
plt_info = ax.scatter(sensor_x[chis < 5],
sensor_y[chis < 5],
c=mus[chis < 5])
ax.set_title("Poisson mu map file " + fl)
ax.set_xlabel("X (mm)")
ax.set_ylabel("Y (mm)")
plt.colorbar(plt_info, ax=ax)
plt.tight_layout()
conditions = (chi_vals[0] < 5) & (chi_vals[1] < 5) & (mu_vals[0] > 0.9) & (
mu_vals[1] > 0.9)
dif_plt = axes[1][0].scatter(sensor_x[conditions],
sensor_y[conditions],
c=mu_vals[0][conditions] -
mu_vals[1][conditions])
plt.colorbar(dif_plt, ax=axes[1][0])
rat_plt = axes[1][1].scatter(
sensor_x[conditions],
sensor_y[conditions],
c=(mu_vals[0][conditions] / mu_vals[1][conditions]) /
(mu_vals[0][1024] / mu_vals[1][1024]))
plt.colorbar(rat_plt, ax=axes[1][1])
fig.show()
plt.show()
def test_fill_pmap_var_2d(dbnew, pmaps):
var_dict = defaultdict(list)
(s1s, s2s), _ = pmaps
data_sipm = dbf.DataSiPM(dbnew, 4670)
monf.fill_pmap_var_1d(s1s, var_dict, "S1", DataSiPM=None)
monf.fill_pmap_var_1d(s2s, var_dict, "S2", DataSiPM=data_sipm)
monf.fill_pmap_var_2d(var_dict, 'S1')
monf.fill_pmap_var_2d(var_dict, 'S2')
for i, speak in enumerate(s1s):
assert var_dict['S1_Energy_S1_Width'][0, i] == speak.total_energy
assert var_dict['S1_Energy_S1_Width'][1, i] == speak.width / units.mus
assert var_dict['S1_Energy_S1_Height'][0, i] == speak.total_energy
assert var_dict['S1_Energy_S1_Height'][1, i] == speak.height
assert var_dict['S1_Energy_S1_Charge'][0, i] == speak.total_energy
assert var_dict['S1_Energy_S1_Charge'][1, i] == speak.total_charge
assert var_dict['S1_Time_S1_Energy'][
0, i] == speak.time_at_max_energy / units.mus
assert var_dict['S1_Time_S1_Energy'][1, i] == speak.total_energy
counter = 0
for i, speak in enumerate(s2s):
assert var_dict['S2_Energy_S2_Width'][0, i] == speak.total_energy
assert var_dict['S2_Energy_S2_Width'][1, i] == speak.width / units.mus
assert var_dict['S2_Energy_S2_Height'][0, i] == speak.total_energy
assert var_dict['S2_Energy_S2_Height'][1, i] == speak.height
assert var_dict['S2_Energy_S2_Charge'][0, i] == speak.total_energy
assert var_dict['S2_Energy_S2_Charge'][1, i] == speak.total_charge
assert var_dict['S2_Time_S2_Energy'][
0, i] == speak.time_at_max_energy / units.mus
assert var_dict['S2_Time_S2_Energy'][1, i] == speak.total_energy
if len(s1s) == 1:
assert var_dict['S2_Energy_S1_Energy'][0, i] == speak.total_energy
assert var_dict['S2_Energy_S1_Energy'][1, i] == s1s[0].total_energy
sipm_ids = speak.sipms.ids
assert np.allclose(
var_dict['S2_XYSiPM'][0, counter:counter + len(sipm_ids)],
data_sipm.X.values[speak.sipms.ids].tolist())
assert np.allclose(
var_dict['S2_XYSiPM'][1, counter:counter + len(sipm_ids)],
data_sipm.Y.values[speak.sipms.ids].tolist())
counter += len(sipm_ids)
def load_sensors(file_names: List[str], db_file: str,
run_no: int) -> Generator:
"""
Loads the nexus MC sensor information into
a pandas DataFrame using the IC function
load_mcsensor_response_df.
Returns info event by event in as a
generator in the structure expected by
the dataflow.
file_names : List of strings
List of input file names to be read
db_file : string
Name of detector database to be used
run_no : int
Run number for database
"""
pmt_ids = DB.DataPMT(db_file, run_no).SensorID
sipm_ids = DB.DataSiPM(db_file, run_no).SensorID
for file_name in file_names:
(all_evt, pmt_binwid, sipm_binwid,
all_wf) = load_mcsensor_response_df(file_name, db_file, run_no)
with tb.open_file(file_name, 'r') as h5in:
mc_info = tbl.get_mc_info(h5in)
timestamps = event_timestamp(h5in)
for evt in all_evt:
pmt_wfs = all_wf.loc[evt].loc[pmt_ids]
sipm_wfs = all_wf.loc[evt].loc[sipm_ids]
yield dict(evt=evt,
mc=mc_info,
timestamp=timestamps(),
pmt_binwid=pmt_binwid,
sipm_binwid=sipm_binwid,
pmt_wfs=pmt_wfs,
sipm_wfs=sipm_wfs)
def fill_pmap_histos(in_path, run_number, config_dict):
"""
Creates and returns an HistoManager object with the pmap histograms.
Arguments:
in_path = String with the path to the file(s) to be monitored.
run_number = Run number of the dataset (used to obtain the SiPM database).
config_dict = Dictionary with the configuration parameters (bins, labels).
"""
var_bins, var_labels = pmap_bins(config_dict)
histo_manager = histf.create_histomanager_from_dicts(var_bins, var_labels)
SiPM_db = dbf.DataSiPM(run_number)
for in_file in glob.glob(in_path):
pmaps = load_pmaps(in_file)
for ti, pi in enumerate(pmaps):
var = fill_pmap_var(pmaps[pi], SiPM_db)
histo_manager.fill_histograms(var)
return histo_manager
def test_fill_pmt_var(dbnew, pmaps):
var_dict = defaultdict(list)
(_, s2s), _ = pmaps
data_sipm = dbf.DataSiPM(dbnew, 4670)
monf.fill_pmt_var(s2s, var_dict)
for i, speak in enumerate(s2s):
pmts = speak.pmts
times = speak.times
energies = pmts.sum_over_times
heights = np.max(pmts.all_waveforms, axis=1)
times = np.apply_along_axis(lambda wf: times[np.argmax(wf)],
axis=1,
arr=pmts.all_waveforms) / units.mus
npmts = len(pmts.all_waveforms)
for j in range(npmts):
assert var_dict[f'PMT{j}_S2_Energy'][i] == energies[j]
assert var_dict[f'PMT{j}_S2_Height'][i] == heights[j]
assert var_dict[f'PMT{j}_S2_Time'][i] == times[j]
def test_fill_pmap_var_1d(dbnew, pmaps):
var_dict = defaultdict(list)
(s1s, s2s), _ = pmaps
data_sipm = dbf.DataSiPM(dbnew, 4670)
monf.fill_pmap_var_1d(s1s, var_dict, "S1", DataSiPM=None)
monf.fill_pmap_var_1d(s2s, var_dict, "S2", DataSiPM=data_sipm)
assert var_dict['S1_Number'][-1] == len(s1s)
assert var_dict['S2_Number'][-1] == len(s2s)
for i, speak in enumerate(s1s):
assert var_dict['S1_Energy'][i] == speak.total_energy
assert var_dict['S1_Width'][i] == speak.width / units.mus
assert var_dict['S1_Height'][i] == speak.height
assert var_dict['S1_Charge'][i] == speak.total_charge
assert var_dict['S1_Time'][i] == speak.time_at_max_energy / units.mus
counter = 0
for i, speak in enumerate(s2s):
nsipm = len(speak.sipms.ids)
assert var_dict['S2_Energy'][i] == speak.total_energy
assert var_dict['S2_Width'][i] == speak.width / units.mus
assert var_dict['S2_Height'][i] == speak.height
assert var_dict['S2_Charge'][i] == speak.total_charge
assert var_dict['S2_Time'][i] == speak.time_at_max_energy / units.mus
assert var_dict['S2_SingleS1'][i] == len(s1s)
assert var_dict['S2_NSiPM'][i] == nsipm
if len(s1s) == 1:
assert var_dict['S2_SingleS1_Energy'][i] == s1s[0].total_energy
assert np.allclose(var_dict['S2_QSiPM'][counter:counter + nsipm],
speak.sipms.sum_over_times)
assert np.allclose(var_dict['S2_IdSiPM'][counter:counter + nsipm],
speak.sipms.ids)
assert np.allclose(var_dict['S2_XSiPM'][counter:counter + nsipm],
data_sipm.X.values[speak.sipms.ids].tolist())
assert np.allclose(var_dict['S2_YSiPM'][counter:counter + nsipm],
data_sipm.Y.values[speak.sipms.ids].tolist())
counter += nsipm
def fill_pmap_histos(in_path, detector_db, run_number, config_dict):
"""Creates and returns a HistoManager object with the pmap histograms.
Parameters
----------
in_path : str
Path to the file(s) to be monitored.
run_number : int
Run number of the dataset (used to obtain the SiPM database).
config_dict : dict
Contains the configuration parameters (bins, labels).
"""
var_bins, var_labels, var_scales = pmap_bins(config_dict)
histo_manager = histf.create_histomanager_from_dicts(var_bins ,
var_labels,
var_scales)
SiPM_db = dbf.DataSiPM(detector_db, run_number)
for in_file in glob.glob(in_path):
pmaps = load_pmaps(in_file)
for pmap in pmaps.values():
var = fill_pmap_var(pmap, SiPM_db)
histo_manager.fill_histograms(var)
return histo_manager
def plot_pdfs():
file_name = sys.argv[1]
if '.h5' in file_name:
print('Single file mode')
all_files = [file_name]
runs = [file_name[-7:-3]]
else:
all_files = sorted(glob(file_name + '*.h5'), key=file_sorter)
runs = [fn[-7:-3] for fn in all_files]
pdf_type = 'adc'
if len(sys.argv) > 2:
pdf_type = sys.argv[2]
sipmIn = [tb.open_file(fn) for fn in all_files]
#with tb.open_file(file_name, 'r') as sipmIn:
bins = np.array(sipmIn[0].get_node('/HIST/', pdf_type + '_bins'))
pdfs = [
np.array(siIn.get_node('/HIST/', pdf_type)).sum(axis=0)
for siIn in sipmIn
]
if 'Sensors' in sipmIn[0].root:
atcaNos = np.fromiter(
(x['channel'] for x in sipmIn[0].root.Sensors.DataSiPM), np.int)
chNos = np.fromiter(
(x['sensorID'] for x in sipmIn[0].root.Sensors.DataSiPM), np.int)
if np.all(chNos == -1):
dats = DB.DataSiPM(5166)
chns = dats.ChannelID
chNos = np.fromiter(
(dats.loc[chns == x, 'SensorID'] for x in atcaNos), np.int)
else:
## print('No sensor info in file, assuming DB ordering for run 5166')
## atcaNos = DB.DataSiPM(5166).ChannelID.values
## chNos = DB.DataSiPM(5166).SensorID.values
print('No sensor info in file, hardwired')
dices = [14, 15, 17, 21]
atcaNos = np.fromiter((d * 1000 + i for d in dices for i in range(64)),
np.int)
chNos = atcaNos
means = []
rmss = []
peak1 = []
grads = []
cable = []
n_cable = [1, 2, 3, 4, 5, 6, 7]
for ich, pdf in enumerate(zip(*pdfs)):
if chNos[ich] < 21000: continue
#plt.cla()
#plt.ion()
#plt.show()
for j, pf in enumerate(pdf):
mean, rms = weighted_mean_and_std(bins, pf, True)
means.append(mean)
rmss.append(rms)
cable.append(n_cable[j])
peaks = find_peaks(pf, 100, distance=15)
try:
peak1.append(bins[peaks[0][1]])
except IndexError:
## Fake to save hassle
peak1.append(-1)
#plt.bar(bins, pf, width=1, log=True, label='Run '+runs[j])
grads.append(
(peak1[-1] - peak1[-len(pdf)]) / (cable[-1] - cable[-len(pdf)]))
#plt.title('Zero suppressed PDF for sensor '+str(chNos[ich])+', atca '+str(atcaNos[ich]))
#plt.xlabel('ADC')
#plt.ylabel('AU')
#plt.legend()
#plt.draw()
#plt.pause(0.001)
#plt.show()
#status = input('next?')
#if 'q' in status:
# exit()
plt.scatter(cable, means)
plt.title('Scatter of distribution mean with number of cables')
plt.xlabel('Number of cables')
plt.ylabel('PDF mean')
plt.show()
plt.scatter(cable, rmss)
plt.title('Scatter of distribution rms with number of cables')
plt.xlabel('Number of cables')
plt.ylabel('PDF rms')
plt.show()
plt.scatter(cable, peak1)
plt.xlabel('Number of cables')
plt.ylabel('Approximate 1 pe peak position')
plt.show()
plt.hist(grads)
plt.title('histogram of 1 pe peak pos / no. cables gradient')
plt.xlabel('gradient')
plt.ylabel('AU')
plt.show()
def fit_dataset():
file_name = sys.argv[1]
min_stat = 0
if len(sys.argv) > 2:
min_stat = int(sys.argv[2])
run_no = file_name[file_name.find('R') + 1:file_name.find('R') + 5]
sipmIn = tb.open_file(file_name, 'r')
## Bins are the same for dark and light, just use light for now
bins = np.array(sipmIn.root.HIST.sipm_spe_bins)
## LED correlated and anticorrelated spectra:
specsL = np.array(sipmIn.root.HIST.sipm_spe).sum(axis=0)
specsD = np.array(sipmIn.root.HIST.sipm_dark).sum(axis=0)
ffunc = partial(speR.scaled_dark_pedestal, min_integral=100)
## This needs to be from a DB or some tempory mapping!!
if 'Sensors' in sipmIn.root:
atcaNos = np.fromiter(
(x['channel'] for x in sipmIn.root.Sensors.DataSiPM), np.int)
chNos = np.fromiter(
(x['sensorID'] for x in sipmIn.root.Sensors.DataSiPM), np.int)
if np.all(chNos == -1):
dats = DB.DataSiPM(5166)
chns = dats.ChannelID
chNos = np.fromiter(
(dats.loc[chns == x, 'SensorID'] for x in atcaNos), np.int)
else:
print('No sensor info in file, assuming DB ordering for run 5166')
atcaNos = DB.DataSiPM(5166).ChannelID.values
chNos = DB.DataSiPM(5166).SensorID.values
n0 = [
'Normalization', 'norm error', 'poisson mu', 'poisson error',
'Pedestal', 'pedestal error', 'Pedestal sigma', 'ped sig error',
'gain', 'gain error', 'gain sigma', 'gain sig error', 'chi2'
]
outData = []
for ich, (led, dar) in enumerate(zip(specsL, specsD)):
print('channel index = ', ich, ', sensor: ', chNos[ich], ', atca: ',
atcaNos[ich])
channelRes = [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]
plt.cla()
plt.ion()
plt.show()
#protections
avL, _ = weighted_mean_and_var(bins, led, True)
avD, _ = weighted_mean_and_var(bins, dar, True)
print(avL - avD)
if avL - avD < 1.0:
plt.bar(bins, led, width=1)
plt.bar(bins, dar, width=1)
plt.title('Spe dark and led spectra for sensor ' +
str(chNos[ich]) + ', atca ' + str(atcaNos[ich]))
plt.xlabel('ADC')
plt.ylabel('AU')
plt.draw()
plt.pause(0.001)
status = input(
'Potential dead or zero light channel, do fit? [yes/no] ')
if 'no' in status:
outData.append(channelRes)
continue
## Limits for safe fit
b1 = 0
b2 = len(dar)
if min_stat != 0:
valid_bins = np.argwhere(led >= min_stat)
b1 = valid_bins[0][0]
b2 = valid_bins[-1][0]
# Seed finding
pD = find_peaks_cwt(dar, np.arange(2, 20), min_snr=2)
if len(pD) == 0:
## Try to salvage in case not a masked channel
## Masked channels have al entries in one bin.
if led[led > 0].size == 1:
outData.append(channelRes)
print('no peaks in dark spectrum, spec ', ich)
continue
else:
pD = np.array([dar.argmax()])
## Fit the dark spectrum with a Gaussian
gb0 = [(0, -100, 0), (1e99, 100, 10000)]
sd0 = (dar.sum(), 0, 2)
errs = np.sqrt(dar[pD[0] - 5:pD[0] + 5])
errs[errs == 0] = 0.1
gfitRes = fitf.fit(fitf.gauss,
bins[pD[0] - 5:pD[0] + 5],
dar[pD[0] - 5:pD[0] + 5],
sd0,
sigma=errs,
bounds=gb0)
channelRes[4] = gfitRes.values[1]
channelRes[5] = gfitRes.errors[1]
channelRes[6] = gfitRes.values[2]
channelRes[7] = gfitRes.errors[2]
respF = ffunc(dark_spectrum=dar[b1:b2],
pedestal_mean=gfitRes.values[1],
pedestal_sigma=gfitRes.values[2])
ped_vals = np.array(
[gfitRes.values[0], gfitRes.values[1], gfitRes.values[2]])
binR = bins[b1:b2]
global darr
darr = dar[b1:b2]
darr = darr[binR < 5]
seeds, bounds = seeds_and_bounds(bins[b1:b2], led[b1:b2], ped_vals)
## The fit
errs = np.sqrt(led + np.exp(-2 * seeds[1]) * dar)
errs[errs == 0] = 0.001
rfit = fitf.fit(respF,
bins[b1:b2],
led[b1:b2],
seeds,
sigma=errs[b1:b2],
bounds=bounds)
chi = rfit.chi2
if chi >= 7 or rfit.values[3] >= 2.5 or rfit.values[3] <= 1:
## The offending parameter seems to be the sigma in most cases
nseed = rfit.values
nseed[3] = 1.7
nbound = [(bounds[0][0], bounds[0][1], bounds[0][2], 1),
(bounds[1][0], bounds[1][1], bounds[1][2], 2.5)]
rfit = fitf.fit(respF,
bins[b1:b2],
led[b1:b2],
nseed,
sigma=errs[b1:b2],
bounds=nbound)
chi = rfit.chi2
if chi >= 10 or rfit.values[2] < 12 or rfit.values[3] > 3:
plt.errorbar(bins,
led,
xerr=0.5 * np.diff(bins)[0],
yerr=errs,
fmt='b.')
plt.plot(bins[b1:b2], respF(bins[b1:b2], *rfit.values), 'r')
plt.plot(bins[b1:b2], respF(bins[b1:b2], *seeds), 'g')
plt.title('Spe response fit to sensor ' + str(chNos[ich]) +
', atca ' + str(atcaNos[ich]))
plt.xlabel('ADC')
plt.ylabel('AU')
plt.draw()
plt.pause(0.001)
input('bad fit, just so you know')
channelRes[0] = rfit.values[0]
channelRes[1] = rfit.errors[0]
channelRes[2] = rfit.values[1]
channelRes[3] = rfit.errors[1]
channelRes[8] = rfit.values[2]
channelRes[9] = rfit.errors[2]
channelRes[10] = rfit.values[3]
channelRes[11] = rfit.errors[3]
channelRes[12] = chi
outData.append(channelRes)
dfDat = DataFrame(outData, columns=n0, index=chNos)
dfDat.chi2.plot.hist(bins=250)
input('Chi^2 dist ok?')
plt.cla()
dfDat.gain.plot.hist(bins=250)
input('Gain dist ok?')
dfDat.to_csv('SiPMFits_file-' + file_name.split('/')[-1] + '.txt')
help='In case the spectrum is from the bb',
default=False)
args = parser.parse_args()
#db_file = '/Users/carmenromoluque/IC/invisible_cities/database/localdb.DEMOPPDB.sqlite3'
db_file = 'demopp'
file_name = args.file_in
func_name = args.func_name
use_db_gain_seeds = args.use_db_gain_seeds
min_stat = args.min_stat
black_box = args.black_box
h5in = tb.open_file(file_name, 'r')
run_no = get_run_number(h5in)
channs = db.DataSiPM(db_file, run_no).SensorID.values
sens_type = SensorType.SIPM if 'sipm' in file_name else SensorType.PMT
## Bins are the same for dark and light, just use light for now
bins = np.array(h5in.root.HIST.sipm_spe_bins)
## LED correlated and anticorrelated spectra:
lspecs = np.array(h5in.root.HIST.sipm_spe).sum(axis=0)
dspecs = np.array(h5in.root.HIST.sipm_dark).sum(axis=0)
ffuncs = {
'ngau': sper.poisson_scaled_gaussians(n_gaussians=7),
'intgau': sper.poisson_scaled_gaussians(min_integral=100),
'dfunc': partial(sper.scaled_dark_pedestal, min_integral=100),
'conv': partial(sper.dark_convolution, min_integral=100)
}
def fast_prod(s1emin, s1wmin, pmt_ids,\
run, files_in, nfin, file_out, n_baseline,\
n_mau, thr_mau, thr_csum_s1, thr_csum_s2, sipm_thr,\
s1_tmin, s1_tmax, s1_stride, s1_lmin, s1_lmax, s1_rebin_stride,\
s2_tmin, s2_tmax, s2_stride, s2_lmin, s2_lmax ,s2_rebin_stride,\
thr_sipm_s2, detector_db,\
qth_penth, rebin,\
qth_esmer, map_file, apply_temp):
# FILE OUT
h5file = tb.open_file(file_out, mode="w", title="Fast Prod")
group = h5file.create_group("/", "Summary", "Summary")
h5file.create_earray(group, "Z", tb.Float64Atom(), shape=(0, ))
h5file.create_earray(group, "DZ", tb.Float64Atom(), shape=(0, ))
h5file.create_earray(group, "E", tb.Float64Atom(), shape=(0, ))
h5file.create_earray(group, "Q", tb.Float64Atom(), shape=(0, ))
h5file.create_earray(group, "Ec", tb.Float64Atom(), shape=(0, ))
group = h5file.create_group("/", f"Event_Info", "Info")
class Event_Info(tb.IsDescription):
event = tb.Int32Col()
time = tb.UInt64Col()
Event_Info_table = h5file.create_table(group, "Event_Time", Event_Info,
"Event_Time")
EI = Event_Info_table.row
files_in = glob.glob(files_in)
files_in.sort()
files_in = files_in[:nfin]
# DATASIPMs AND MAPS
datasipm = load_db.DataSiPM("new", run)
sipm_xs = datasipm.X.values
sipm_ys = datasipm.Y.values
maps = read_maps(map_file)
total_correction = apply_all_correction(maps,
apply_temp=apply_temp,
norm_strat=norm_strategy.kr)
# FAST PROD
_Z, _DZ, _E, _Q, _Ec = [], [], [], [], []
for file in files_in:
print(file)
RWFs_file = tb.open_file(file)
pmt_rwfs_all = RWFs_file.root.RD.pmtrwf
sipm_rwfs_all = RWFs_file.root.RD.sipmrwf
time_stamps = RWFs_file.root.Run.events.read()
for event_time, pmt_rwfs, sipm_rwfs in zip(time_stamps, pmt_rwfs_all,
sipm_rwfs_all):
################################
############ IRENE #############
################################
#pmt processing
rwf_to_cwf = deconv_pmt(detector_db, run, n_baseline)
pmt_cwfs = rwf_to_cwf(pmt_rwfs)
cwf_to_ccwf = calibrate_pmts(detector_db, run, n_mau, thr_mau)
pmt_ccwfs, ccwfs_mau, cwf_sum, cwf_sum_mau = cwf_to_ccwf(pmt_cwfs)
#sipm processing
sipm_rwf_to_cal = calibrate_sipms(detector_db, run, sipm_thr)
sipm_cwfs = sipm_rwf_to_cal(sipm_rwfs)
#Find signals
zero_suppress = zero_suppress_wfs(thr_csum_s1, thr_csum_s2)
s1_indices, s2_indices = zero_suppress(cwf_sum, cwf_sum_mau)
s1_selected_splits,\
s2_selected_splits = utils.signals_selected_splits(s1_indices, s2_indices,
s1_stride , s2_stride,
s1_tmin , s1_tmax , s1_lmin, s1_lmax,
s2_tmin , s2_tmax , s2_lmin, s2_lmax)
######## 1S1 1S2 CUT ##########
S1_time = utils._1s1_1s2(pmt_ccwfs, s2_selected_splits,
s1_selected_splits, s1emin, s1wmin)
if not S1_time: continue
# Rebin S2_pmts
times, rebinned_widths, s2_pmts = pick_slice_and_rebin(
s2_selected_splits[0],
np.arange(pmt_ccwfs.shape[1]) * 25 * units.ns,
np.full(pmt_ccwfs.shape[1], 25 * units.ns),
pmt_ccwfs,
rebin_stride=s2_rebin_stride,
pad_zeros=True)
#select and thr_sipm_s2
s2_sipms = sipm_cwfs[:, s2_selected_splits[0][0] //
40:s2_selected_splits[0][-1] // 40 + 1]
sipm_ids, s2_sipms = select_wfs_above_time_integrated_thr(
s2_sipms, thr_sipm_s2)
######## IRENE FINAL S2 WFS #######
s2_pmts = np.float32(s2_pmts)
s2_sipms = np.float32(s2_sipms)
times = np.float32(times)
#select pmt_ids wfs
c = np.zeros(s2_pmts.shape[0])
c[pmt_ids] = 1
s2_pmts = np.multiply(c, s2_pmts.T).T
################################
######## PENTHESILEA ###########
################################
########## Rebin ############
_, _, s2_sipms = rebin_times_and_waveforms(times,
rebinned_widths,
s2_sipms,
rebin_stride=rebin,
slices=None)
times, _, s2_pmts = rebin_times_and_waveforms(times,
rebinned_widths,
s2_pmts,
rebin_stride=rebin,
slices=None)
######### Charge cut #########
s2_pmts_penth = np.copy(s2_pmts)
s2_sipms_penth = np.where(s2_sipms >= qth_penth, s2_sipms, 0)
###### create penthesilea hits ########
hits = utils.create_penthesilea_hits(s2_pmts_penth, s2_sipms_penth,
sipm_xs, sipm_ys, sipm_ids,
times, S1_time)
################################
######### ESMERALDA ############
################################
#### Charge cut ####
hits = utils.esmeralda_charge_cut(hits, qth_esmer)
###### join NN hits ######
hits = utils.join_NN_hits(hits)
#### Corrections ######
X, Y, Z = hits["X"], hits["Y"], hits["Z"]
E, Q = hits["E"], hits["Q"]
T = np.full(len(hits), event_time[-1] / 1000)
correction_factor = total_correction(X, Y, Z, T)
Ec = correction_factor * E
hits["Ec"] = Ec
hits["Z"] = Z * maps.t_evol.dv.mean()
###########################
####### APPEND DATA #######
###########################
# Event Info
EI["event"] = event_time[0]
EI["time"] = event_time[1]
EI.append()
## Z, DZ, E, Q, Ec
Z, E, Q, Ec = hits["Z"], hits["E"], hits["Q"], hits["Ec"]
Ec[np.isnan(Ec)] = 0
_Z.append(np.sum(Ec * Z) / np.sum(Ec))
_DZ.append(np.max(Z) - np.min(Z))
_E.append(np.sum(E))
_Q.append(np.sum(Q))
_Ec.append(np.sum(Ec))
# close RWF file
RWFs_file.close()
h5file.root.Summary.Z.append(_Z)
h5file.root.Summary.DZ.append(_DZ)
h5file.root.Summary.E.append(_E)
h5file.root.Summary.Q.append(_Q)
h5file.root.Summary.Ec.append(_Ec)
#write to disk
h5file.flush()
h5file.close()
def main():
""" Plot some pdfs """
old_pdfs = sys.argv[1]
new_pdfs = sys.argv[2]
db1 = DB.DataSiPM(6499)
db2 = DB.DataSiPM(6562)
active = (db1.Active & db2.Active) == 1
changed = (db1.SensorID >= 5000) & (db1.SensorID < 8064)
sensor_x = db1.X.values
sensor_y = db1.Y.values
with tb.open_file(new_pdfs) as newfile, tb.open_file(old_pdfs) as oldfile:
## obins = np.array(oldfile.root.HIST.sipm_mode_bins)
## ospecs = np.array(oldfile.root.HIST.sipm_mode).sum(0)
## nbins = np.array(newfile.root.HIST.sipm_mode_bins)
## nspecs = np.array(newfile.root.HIST.sipm_mode).sum(0)
obins = np.array(oldfile.root.HIST.sipm_adc_bins)
ospecs = np.array(oldfile.root.HIST.sipm_adc).sum(0)
nbins = np.array(newfile.root.HIST.sipm_adc_bins)
nspecs = np.array(newfile.root.HIST.sipm_adc).sum(0)
omerms = [weighted_mean_and_std(obins, spec, True) for spec in ospecs]
nmerms = [weighted_mean_and_std(nbins, spec, True) for spec in nspecs]
norm_old = np.apply_along_axis(normalize_distribution, 1, ospecs)
thr_old = general_thresholds(obins, norm_old, 0.99)
norm_new = np.apply_along_axis(normalize_distribution, 1, nspecs)
thr_new = general_thresholds(nbins, norm_new, 0.99)
fig, axes = plt.subplots(nrows=3, ncols=3, figsize=(20, 6))
ptitles = [
'MEAN new', 'MEAN old', 'Mean new - Mean old', 'RMS new',
'RMS old', 'RMS new - RMS old', 'Thr new', 'Thr old',
'Thr new - Thr old'
]
vals = [
np.fromiter((v[0] for v in nmerms), np.float),
np.fromiter((v[0] for v in omerms), np.float),
np.zeros(2),
np.fromiter((v[1] for v in nmerms), np.float),
np.fromiter((v[1] for v in omerms), np.float),
np.zeros(2), thr_new, thr_old,
np.zeros(2)
]
for i, (ax, val) in enumerate(zip(axes.flatten(), vals)):
if not np.any(val):
diff = vals[i - 2] - vals[i - 1]
pinfo = ax.scatter(sensor_x[active & changed],
sensor_y[active & changed],
c=diff[active & changed],
s=1.5)
else:
pinfo = ax.scatter(sensor_x[active & changed],
sensor_y[active & changed],
c=val[active & changed],
s=1.5)
ax.set_title(ptitles[i])
ax.set_xlabel('X (mm)')
ax.set_ylabel('Y (mm)')
plt.colorbar(pinfo, ax=ax)
plt.tight_layout()
fig.show()
plt.show()
def generate_pdfs():
"""
Generate SiPM PDFs using Kr RAW data.
Multiple types of PDF are generated:
Full spectrum PDFs : using full buffer
Z vetoed PDFs : Vetoing all sipms for regions in z where there is an identified s1 or s2
1 ring vetoed PDFs : Vetoing sipms within 1 ring distance of a hit.
2 ring vetoed PDFs : As above for 2 rings
...
"""
pmap_file_base = sys.argv[1]
hit_file_base = sys.argv[2]
raw_file_base = sys.argv[3]
pmap_sorter = sorter_func(pmap_file_base)
pmap_files = sorted(glob(pmap_file_base + '*.h5'), key=pmap_sorter)
hit_sorter = sorter_func(hit_file_base)
hit_files = sorted(glob(hit_file_base + '*.h5'), key=hit_sorter)
raw_sorter = sorter_func(raw_file_base)
raw_files = sorted(glob(raw_file_base + '*.h5'), key=raw_sorter)
## Details of raw waveforms to aid vetoing (assumes all same in run)
with tb.open_file(raw_files[0]) as rwf_in:
sipmrwf = rwf_in.root.RD.sipmrwf[0][0]
wf_range = np.arange(len(sipmrwf))
run_no = int(sys.argv[4])
## Gains and sensor positions
sipm_gains = DB.DataSiPM(run_no).adc_to_pes.values
sipm_xy = DB.DataSiPM(run_no)[['X', 'Y']].values
## output
histbins = np.arange(-10, 300, 0.1)
bin_centres = shift_to_bin_centers(histbins)
with tb.open_file('vetoedPDFs_R' + str(run_no) + '.h5', 'w') as pdf_out:
HIST = partial(hist_writer,
pdf_out,
group_name='HIST',
n_sensors=1792,
n_bins=len(bin_centres),
bin_centres=bin_centres)
full_spec = HIST(table_name='full_spec')
z_vetoed = HIST(table_name='z_vetoed')
one_ring = HIST(table_name='one_ring_vetoed')
two_ring = HIST(table_name='two_ring_vetoed')
thr_ring = HIST(table_name='thr_ring_vetoed')
inv_thre = HIST(table_name='thr_ring_avetoed')
## Hit info
hit_positions = load_dsts(hit_files, 'DST',
'Events')[['event', 'X', 'Y']].values
## Start assuming KR data and Kdst
## For each event [evt_no, list tuples start and end veto areas]
reduced_pulse_info = []
for pmf in pmap_files:
#print(pmf)
sys.stdout.write(pmf + '\n')
sys.stdout.flush()
try:
## pmap_dict = load_pmaps(pmf)
s1s, s2s, _, _, _ = load_pmaps_as_df(pmf)
except (ValueError, tb.exceptions.NoSuchNodeError):
print("Empty file. Skipping.")
continue
## for key, pmap in pmap_dict.items():
for evtNo in s1s['event'].unique():
evtS1 = s1s[s1s['event'] == evtNo]
evtS2 = s2s[s2s['event'] == evtNo]
mask_list = []
## for s1 in pmap.s1s:
for is1 in evtS1['peak'].unique():
s1 = evtS1[evtS1['peak'] == is1]
mask_list.append(
(wf_range < s1['time'].iloc[0] / units.mus - 1)
| (wf_range > s1['time'].iloc[-1] / units.mus + 1))
for is2 in evtS2['peak'].unique():
s2 = evtS2[evtS2['peak'] == is2]
mask_list.append(
(wf_range < s2['time'].iloc[0] / units.mus - 2)
| (wf_range > s2['time'].iloc[-1] / units.mus + 2))
reduced_pulse_info.append(
[evtNo, np.logical_and.reduce(mask_list)])
print('masking info stored')
mask_counter = 0
pmap_evts = np.fromiter((x[0] for x in reduced_pulse_info), np.int)
for rawf in raw_files:
#print(rawf)
sys.stdout.write(rawf + '\n')
sys.stdout.flush()
if mask_counter >= len(reduced_pulse_info):
continue
try:
## empty arrays for histograms
shape = 1792, len(bin_centres)
hist_full_spec = np.zeros(shape, dtype=np.int)
hist_z_vetoed = np.zeros(shape, dtype=np.int)
hist_1_vetoed = np.zeros(shape, dtype=np.int)
hist_2_vetoed = np.zeros(shape, dtype=np.int)
hist_3_vetoed = np.zeros(shape, dtype=np.int)
hist_3_aveto = np.zeros(shape, dtype=np.int)
with tb.open_file(rawf) as raw_in:
revent_nos = np.fromiter(
(x[0] for x in raw_in.root.Run.events), np.int)
#evt_no = reduced_pulse_info[mask_counter][0]
#indx = np.argwhere(revent_nos==evt_no)
#print(reduced_pulse_info)
#pmap_evts = np.array(reduced_pulse_info)[:, 0]
pmap_overlap_indx = np.arange(revent_nos.shape[0])[np.in1d(
revent_nos, pmap_evts)]
hit_overlap_indx = np.arange(revent_nos.shape[0])[np.in1d(
revent_nos, hit_positions[:, 0])]
hit_indcs = np.arange(hit_positions[:,
0].shape[0])[np.in1d(
hit_positions[:,
0],
revent_nos)]
hindx = 0
#print(indx, indx[0][0])
#while indx.shape[0] != 0:
for indx in pmap_overlap_indx:
#print(indx[0][0])
#rwf = raw_in.root.RD.sipmrwf[indx[0][0]]
## cwf = csf.sipm_processing["subtract_mode_calibrate"](raw_in.root.RD.sipmrwf[indx[0][0]], sipm_gains)
cwf = csf.sipm_processing["subtract_mode_calibrate"](
raw_in.root.RD.sipmrwf[indx], sipm_gains)
hist_full_spec += cf.bin_waveforms(cwf, histbins)
z_veto = reduced_pulse_info[mask_counter][1]
hist_z_vetoed += cf.bin_waveforms(
cwf[:, z_veto], histbins)
#dst_indx = np.argwhere(hit_positions[:, 0]==evt_no)
#if dst_indx.shape[0] != 0:
if indx in hit_overlap_indx:
## hit_p = hit_positions[dst_indx[0][0], 1:]
hit_p = hit_positions[hit_indcs[hindx], 1:]
hindx += 1
hist_1_vetoed += cf.bin_waveforms(
ring_veto(cwf, 1, z_veto, hit_p, sipm_xy),
histbins)
hist_2_vetoed += cf.bin_waveforms(
ring_veto(cwf, 2, z_veto, hit_p, sipm_xy),
histbins)
thrVeto = ring_veto(cwf, 3, z_veto, hit_p, sipm_xy)
hist_3_vetoed += cf.bin_waveforms(
thrVeto, histbins)
hist_3_aveto += cf.bin_waveforms(
thrVeto[:, np.invert(z_veto)], histbins)
mask_counter += 1
#if mask_counter < len(reduced_pulse_info):
# evt_no = reduced_pulse_info[mask_counter][0]
# indx = np.argwhere(revent_nos==evt_no)
#else:
## dummy evt_no to definitely give no info
# indx = np.argwhere(revent_nos==-100)
#print(indx, indx[0][0])
full_spec(hist_full_spec)
z_vetoed(hist_z_vetoed)
one_ring(hist_1_vetoed)
two_ring(hist_2_vetoed)
thr_ring(hist_3_vetoed)
inv_thre(hist_3_aveto)
except tb.HDF5ExtError:
print('corrupt file')
continue
parser.add_argument('func_name', type=str, help='function that will be used to fit', default='dfunc')
parser.add_argument('use_db_gain_seeds', type=str2bool, help='option to take gain values from db', default=False)
parser.add_argument('min_stat', type=int, help='min statistics for the peaks', default=10)
args = parser.parse_args()
#db_file = '/Users/carmenromoluque/IC/invisible_cities/database/localdb.NEWDB.sqlite3'
db_file = 'new'
file_name = args.file_in
func_name = args.func_name
use_db_gain_seeds = args.use_db_gain_seeds
min_stat = args.min_stat
sipmIn = tb.open_file(file_name, 'r')
run_no = get_run_number(sipmIn)
channs = DB.DataSiPM(db_file, run_no).SensorID.values
sens_type = SensorType.SIPM if 'sipm' in file_name else SensorType.PMT
masked_ch = DB.DataSiPM(db_file, run_no).index[DB.DataSiPM(db_file, run_no).Active==0].values
if use_db_gain_seeds:
GainSeeds = DB.DataSiPM(db_file, run_no).adc_to_pes.values
SigSeeds = DB.DataSiPM(db_file, run_no).Sigma .values
## Give generic values to previously dead or dodgy channels
GainSeeds[masked_ch] = 15
SigSeeds [masked_ch] = 2
## Bins are the same for dark and light, just use light for now
bins = np.array(sipmIn.root.HIST.sipm_spe_bins)
def compare_sims():
"""
Just a copy (more or less) of Paola's notebook for
simulation low level plots adjusted to compare simulation
with and without the low frequency noise.
"""
run_number = -4735
DataSiPM = DB.DataSiPM(run_number)
data_xs = DataSiPM.X.values
data_ys = DataSiPM.Y.values
## conf parameters
## n_rebin = 1
n_files = 10
drift_v = 1.
## Containers for plots ##
s1Q_NO = []
s1H_NO = []
s1W_NO = []
s1per_evt_NO = []
s1Q_O = []
s1H_O = []
s1W_O = []
s1per_evt_O = []
#maxPMTtot_ratio = []
s2per_evt_NO = []
s2Q_NO = []
s2H_NO = []
s2W_NO = []
s2per_evt_O = []
s2Q_O = []
s2H_O = []
s2W_O = []
no_iter = gl("/Users/laingandrew/Documents/NEXT/IC_stuff/simdats/noOsc_paola/Tl_DISK_CLOSE_TO_ANODE_7bar_noiseSiPM_pmaps.*.h5")
o_iter = gl("/Users/laingandrew/Documents/NEXT/IC_stuff/simdats/Tl_DISK_CLOSE_TO_ANODE_7bar_noiseSiPM_pmaps.*.h5")
for no_file, o_file in zip(no_iter, o_iter):
##for ifile in range(n_files):
##PATH_IN_NOOSC = "/Users/laingandrew/Documents/NEXT/IC_stuff/simdats/noOsc_paola/Tl_DISK_CLOSE_TO_ANODE_7bar_noiseSiPM_pmaps.{}.h5".format(ifile)
##PATH_IN_OSC = "/Users/laingandrew/Documents/NEXT/IC_stuff/simdats/Tl_DISK_CLOSE_TO_ANODE_7bar_noiseSiPM_pmaps.{}.h5".format(ifile)
print(no_file)
#pmapNO_dict = load_pmaps(no_file)
s1sNO, s2sNO, _, _, _ = load_pmaps_as_df(no_file)
print('NO pmaps got')
#pmapO_dict = load_pmaps(o_file)
s1sO, s2sO, _, _, _ = load_pmaps_as_df(o_file)
print('O pmaps got')
## event_numbers_NO, timestamps_NO = get_event_numbers_and_timestamps_from_file_name(PATH_IN_NOOSC)
## event_numbers_O, timestamps_O = get_event_numbers_and_timestamps_from_file_name(PATH_IN_OSC)
event_numbers_NO = s2sNO['event'].unique()
event_numbers_O = s2sO['event'].unique()
print(len(event_numbers_NO), len(event_numbers_NO))
for evtNO, evtO in zip(event_numbers_NO, event_numbers_O):
## pmapNO = pmapNO_dict.get(evtNO, None)
## pmapO = pmapO_dict.get(evtO, None)
s1s_NO = s1sNO[s1sNO['event']==evtNO]
s1s_O = s1sO[s1sO['event']==evtO]
s2s_NO = s2sNO[s2sNO['event']==evtNO]
s2s_O = s2sO[s2sO['event']==evtO]
#print(len(s1s_NO), len(s1s_O), len(s2s_NO), len(s2s_O))
#print('evtNO = ', evtNO, ' evtO = ', evtO,' pmaps got')
if len(s1s_NO) == 0 or len(s1s_O) == 0 or len(s2s_NO) == 0 or len(s2s_O) == 0:
continue
## if pmapNO:
## s1s_NO = pmapNO.s1s
## s2sraw_NO = pmapNO.s2s
## else:
## continue
## if pmapO:
## s1s_O = pmapO.s1s
## s2sraw_O = pmapO.s2s
## else:
## continue
#if not s1s_NO or not s2sraw_NO or not s1s_O or not s2sraw_O: continue
s1per_evt_NO.append(s1s_NO['peak'].nunique())
s1per_evt_O .append(s1s_O['peak'].nunique())
#print('n_s1_nosc = ', s1s_NO['peak'].nunique(), ', n_s1_osc = ', s1s_O['peak'].nunique())
#if s1s_NO['peak'].nunique() != 1: continue
#if s1s_O['peak'].nunique() != 1: continue
## s2sNO = [rebin_peak(s2raw_NO, n_rebin) for s2raw_NO in s2sraw_NO]
## s2sO = [rebin_peak(s2raw_O, n_rebin) for s2raw_O in s2sraw_O]
## s1tNO = s1s_NO[0].times
## s1eNO = s1s_NO[0].pmts
## S1tNO = s1t_NO[np.argmax(s1eNO)]
## s1tO = s1s_O[0].times
## s1eO = s1s_O[0].pmts
## S1tO = s1t_O[np.argmax(s1eO)]
#print('Aqui?')
#fill histograms
for is1 in range(s1s_NO['peak'].nunique()):
s1 = s1s_NO[s1s_NO['peak'] == is1]
s1H_NO.append(s1['ene'].max())
s1W_NO.append(s1['time'].iloc[-1] - s1['time'].iloc[0])
s1Q_NO.append(s1['ene'].sum())
for js1 in range(s1s_O['peak'].nunique()):
s1 = s1s_O[s1s_O['peak'] == js1]
s1H_O.append(s1['ene'].max())
s1W_O.append(s1['time'].iloc[-1] - s1['time'].iloc[0])
s1Q_O.append(s1['ene'].sum())
s2per_evt_NO.append(s2s_NO['peak'].nunique())
s2per_evt_O.append(s2s_O['peak'].nunique())
for is2 in range(s2s_NO['peak'].nunique()):
s2 = s2s_NO[s2s_NO['peak'] == is2]
s2Q_NO.append(s2['ene'].sum())
## if s2['time'].iloc[-1] - s2['time'].iloc[0] < 0:
## print(evtNO, ' has -ve width S2 in NO')
s2W_NO.append(s2['time'].iloc[-1] - s2['time'].iloc[0])
s2H_NO.append(s2['ene'].max())
for js2 in range(s2s_O['peak'].nunique()):
s2 = s2s_O[s2s_O['peak'] == js2]
s2Q_O.append(s2['ene'].sum())
## if s2['time'].iloc[-1] - s2['time'].iloc[0] < 0:
## print(evtO, ' has -ve width S2 in O')
s2W_O.append(s2['time'].iloc[-1] - s2['time'].iloc[0])
s2H_O.append(s2['ene'].max())
print(min(s1per_evt_NO), max(s1per_evt_NO), min(s1per_evt_O), max(s1per_evt_O))
print(min(s2per_evt_NO), max(s2per_evt_NO), min(s2per_evt_O), max(s2per_evt_O))
print(min(s1Q_NO), max(s1Q_NO), min(s2Q_O), max(s2Q_O))
print(min(s1H_NO), max(s1H_NO), min(s2H_O), max(s2H_O))
print(min(s1W_NO), max(s1W_NO), min(s2W_NO), max(s2W_NO))
print(min(s1W_O), max(s1W_O), min(s2W_O), max(s2W_O))
fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(20,6))
axes[0][0].hist(s1per_evt_NO, bins=10, range=(0, 10), label='No. S1s no low frequency noise', log=True, histtype='step')
axes[0][0].hist(s1per_evt_O , bins=10, range=(0, 10), label='No. S1s with low frequency noise', log=True, histtype='step')
axes[0][0].legend()
axes[0][0].set_xlabel('No. s1 per event')
axes[0][0].set_ylabel('AU')
axes[0][1].hist(s1H_NO, bins=200, range=(0, 200), label='S1 height no low frequency noise', log=True, histtype='step')
axes[0][1].hist(s1H_O , bins=200, range=(0, 200), label='S1 height with low frequency noise', log=True, histtype='step')
axes[0][1].legend()
axes[0][1].set_xlabel('S1 height (pes)')
axes[0][1].set_ylabel('AU')
axes[1][0].hist(s1Q_NO, bins=100, range=(0, 1500), label='S1 charge no low frequency noise', log=True, histtype='step')
axes[1][0].hist(s1Q_O , bins=100, range=(0, 1500), label='S1 charge with low frequency noise', log=True, histtype='step')
axes[1][0].legend()
axes[1][0].set_xlabel('S1 charge (pes)')
axes[1][0].set_ylabel('AU')
axes[1][1].hist(s1W_NO, bins=40, range=(0, 1000), label='S1 width no low frequency noise', log=True, histtype='step')
axes[1][1].hist(s1W_O , bins=40, range=(0, 1000), label='S1 width with low frequency noise', log=True, histtype='step')
axes[1][1].legend()
axes[1][1].set_xlabel('S1 width (ns)')
axes[1][1].set_ylabel('AU')
fig.show()
fig.savefig('s1_plots_pmaps_oscNoosc.png')
fig2, axes2 = plt.subplots(nrows=2, ncols=2, figsize=(20,6))
axes2[0][0].hist(s2per_evt_NO, bins=10, range=(0, 10), label='No. S2s no low frequency noise', log=True, histtype='step')
axes2[0][0].hist(s2per_evt_O , bins=10, range=(0, 10), label='No. S2s with low frequency noise', log=True, histtype='step')
axes2[0][0].legend()
axes2[0][0].set_xlabel('No. s2 per event')
axes2[0][0].set_ylabel('AU')
axes2[0][1].hist(s2H_NO, bins=1000, range=(10, 65000), label='S2 height no low frequency noise', log=True, histtype='step')
axes2[0][1].hist(s2H_O , bins=1000, range=(10, 65000), label='S2 height with low frequency noise', log=True, histtype='step')
axes2[0][1].legend()
axes2[0][1].set_xlabel('S2 height (pes)')
axes2[0][1].set_ylabel('AU')
axes2[1][0].hist(s2Q_NO, bins=1000, range=(10, 750000), label='S2 charge no low frequency noise', log=True, histtype='step')
axes2[1][0].hist(s2Q_O , bins=1000, range=(10, 750000), label='S2 charge with low frequency noise', log=True, histtype='step')
axes2[1][0].legend()
axes2[1][0].set_xlabel('S2 charge (pes)')
axes2[1][0].set_ylabel('AU')
axes2[1][1].hist(s2W_NO, bins=235, range=(1500, 350000), label='S2 width no low frequency noise', log=True, histtype='step')
axes2[1][1].hist(s2W_O , bins=235, range=(1500, 350000), label='S2 width with low frequency noise', log=True, histtype='step')
axes2[1][1].legend()
axes2[1][1].set_xlabel('S2 width (ns)')
axes2[1][1].set_ylabel('AU')
fig2.show()
fig2.savefig('s2_plots_pmaps_oscNoosc.png')
input('Done with plots?')
# In[21]:
e40rd = tb.open_file(MCRD_file,'r+')
NEVENTS_DST, NSIPM, SIPMWL = e40rd.root.sipmrd.shape
# In[22]:
print('number of SiPM = {}, waveform length = {}'.format(NSIPM,SIPMWL))
# In[23]:
DataSiPM = load_db.DataSiPM(0)
sipm_adc_to_pes = DataSiPM.adc_to_pes.values.astype(np.double)
# Conversion constants (number of adc counts per PES)
# In[24]:
plt.plot(sipm_adc_to_pes)
# ### Nb
# A few constants have values equal to zero, corresponding to dead SiPMs in the data. The average number of adc counts per pes is very flat (e.g, about the same for all SiPMs) and near 16.
# In[25]:
def main():
""" Check new fit function on SiPM spectra """
saved_g = np.array(DB.DataSiPM(4832).adc_to_pes, np.float)
chNos = np.array(DB.DataSiPM(4832).SensorID)
## sipmIn = tb.open_file('../outdats/sipmGainTest_R4832.h5', 'r')
sipmIn = tb.open_file('../outdats/sipmSpeSpecs_gainmode_R5354.h5', 'r')
## Bins are the same for dark and light, just use light for now
bins = np.array(sipmIn.root.HIST.sipm_spe_bins)
## LED correlated and anticorrelated spectra:
specsL = np.array(sipmIn.root.HIST.sipm_spe).sum(axis=0)
specsD = np.array(sipmIn.root.HIST.sipm_dark).sum(axis=0)
## General definition of fit class.
#respF = SensorSpeResponse(bins, specsL[0].sum())
## The bounds on the parameters (norm, 1pe mean, poisson mean, 1pe sig)
## Are loose and general
b0 = [(0., 0., 0., 0.001), (1e20, 100., 100., 10000.)]
## Names for output
n0 = [
'poisson', 'gain', 'sigma', 'chi2', 'saved gain', 'pull_gain', 'sigPe'
]
## Loop over the specra:
outData = []
dbDat = np.zeros((1792, 4), dtype=np.float)
## Extra protection since 3065 is weird
knownDead = [3056, 8056, 14010, 25049]
specialCheck = [1006, 1007, 3000, 3001, 7000, 22029, 28056, 28057]
for ich, (led, dar, gn) in enumerate(zip(specsL, specsD, saved_g)):
if chNos[ich] in knownDead:
outData.append([0., 0., 0., 0., 0., 0., 0.])
print('no peaks in dark spectrum, spec ', ich)
continue
## Limits for safe fit
b1 = np.argwhere(led >= 50)[0][0]
b2 = np.argwhere(led >= 50)[-1][0]
# Seed finding
## Dark:
pD = find_peaks_cwt(dar, np.arange(2, 20), min_snr=2)
if len(pD) == 0:
## Try to salvage in case not a masked channel
## Masked channels have al entries in one bin.
if led[led > 0].size == 1:
outData.append([0., 0., 0., 0., 0., 0., 0.])
print('no peaks in dark spectrum, spec ', ich)
continue
else:
pD = np.array([dar.argmax()])
## ped, pE = curve_fit(gauFit, bins[pD[0]-5:pD[0]+5], dar[pD[0]-5:pD[0]+5], p0=[dar[bins==0][0], 0., 2.])
gfunc = fitf.fit(fitf.gauss,
bins[pD[0] - 5:pD[0] + 5],
dar[pD[0] - 5:pD[0] + 5],
seed=(dar[bins == 0][0], 0., 2.))
## Set the dark values to the fit function (still not ideal)
#respF.set_dark_func(dar[b1:b2], ped[1], np.abs(ped[2]))
respF = speR.scaled_dark_pedestal(dar[b1:b2], gfunc.values[1],
np.abs(gfunc.values[2]), 100)
global darr
binR = bins[b1:b2]
darr = dar[b1:b2]
darr = darr[binR < 0]
## LED
pDL = find_peaks_cwt(led, np.arange(4, 20), min_snr=1, noise_perc=5)
if len(pDL) <= 1 or abs(pDL[1] - pDL[0]) > 25:
## default values
ledR = led[b1:b2]
fscaler = fitf.fit(scaler, binR[binR < 0], ledR[binR < 0],
(led[bins < 0].max() / dar[bins < 10].max()))
muSeed = -np.log(fscaler.values[0])
if muSeed < 0: muSeed = 0.001
## sd0 = (led.sum(), 15., muSeed, 2.)
sd0 = (led.sum(), muSeed, 15., 2.)
else:
z0 = pDL[bins[pDL] < 10]
print('z0: ', z0)
muSeed = 0.
if z0.size == 0:
print('check1: ', led[bins == 0])
ledR = led[b1:b2]
fscaler = fitf.fit(
scaler, binR[binR < 0], ledR[binR < 0],
(led[bins < 0].max() / dar[bins < 10].max()))
muSeed = -np.log(fscaler.values[0])
if muSeed < 0: muSeed = 0.001
else:
## ped, pE = curve_fit(gauFit, bins[z0[0]-5:z0[0]+5], led[z0[0]-5:z0[0]+5], p0=[led[z0[0]], ped[1], np.abs(ped[2])])
vals = fitf.fit(fitf.gauss,
bins[z0[0] - 5:z0[0] + 5],
led[z0[0] - 5:z0[0] + 5],
seed=(led[z0[0]], gfunc.values[1],
np.abs(gfunc.values[2])))
ped = vals.values
print('Ped sees: ', ped)
z1 = pDL[(bins[pDL] > 10) & (bins[pDL] < 30)]
print('z1: ', z1)
if z1.size == 0:
print('check2: ', led[(bins > 10) & (bins < 30)].max())
p1 = [led[(bins > 10) & (bins < 30)].max(), 15., 2.]
if muSeed == 0.:
ledR = led[b1:b2]
fscaler = fitf.fit(
scaler, binR[binR < 0], ledR[binR < 0],
(led[bins < 0].max() / dar[bins < 10].max()))
muSeed = -np.log(fscaler.values[0])
if muSeed < 0: muSeed = 0.001
else:
z1 = z1[led[z1].argmax()]
## p1, p1E = curve_fit(gauFit, bins[z1-6:z1+6], led[z1-6:z1+6], p0=[led[z1], bins[z1], 3.])
p1 = fitf.fit(fitf.gauss,
bins[z1 - 6:z1 + 6],
led[z1 - 6:z1 + 6],
seed=(led[z1], bins[z1], 3.))
p1 = p1.values
if muSeed == 0.:
ledR = led[b1:b2]
fscaler = fitf.fit(
scaler, binR[binR < 0], ledR[binR < 0],
(led[bins < 0].max() / dar[bins < 10].max()))
muSeed = -np.log(fscaler.values[0])
if muSeed < 0: muSeed = 0.001
print('1pe seed fit: ', p1)
print(p1[0], ped[0], muSeed)
# improve 1pe sigma seed?
sigSeed = np.abs(p1[2])
sd0 = (led.sum(), muSeed, p1[1], sigSeed)
print('Pre fit seed check: ', sd0)
##
## Fit function
## vals, errs = curve_fit(respF.scaled_dark_pedestal,
## bins[b1:b2], led[b1:b2],
## sigma=np.sqrt(led[b1:b2]+dar[b1:b2]),
## p0=sd0, bounds=b0)
rfun = fitf.fit(respF,
bins[b1:b2],
led[b1:b2],
seed=sd0,
sigma=np.sqrt(led[b1:b2] + dar[b1:b2]),
bounds=b0)
chDic = {}
e1 = rfun.errors #np.sqrt(np.diagonal(rfun.errors))
#chi = np.sum(((led[b1:b2]-respF.scaled_dark_pedestal(bins[b1:b2], *vals))/np.sqrt(led[b1:b2]+dar[b1:b2]))**2)
#chi = chi/(len(bins[b1:b2])-len(vals))
chi = rfun.chi2
for i in range(1, len(rfun.values)):
chDic[n0[i - 1]] = (rfun.values[i], e1[i])
#print(ich, chDic)
print('indx = ', ich, 'channel = ', chNos[ich], ' seeds=', sd0)
if chNos[ich] in specialCheck:
## if chi > 10:
plt.bar(bins, led, width=1)
## plt.plot(bins[b1:b2], respF.scaled_dark_pedestal(bins[b1:b2], *vals), 'r')
plt.plot(bins[b1:b2], respF(bins[b1:b2], *rfun.values), 'r')
plt.show()
print('nGau = ', respF.n_gaussians)
dbDat[ich][0] = chDic['gain'][0]
dbDat[ich][1] = chDic['gain'][1]
dbDat[ich][2] = chDic['sigma'][0]
dbDat[ich][3] = chDic['sigma'][1]
outData.append([
chDic['poisson'], chDic['gain'], chDic['sigma'], chi, gn,
(gn - chDic['gain'][0]) / chDic['gain'][1],
chDic['sigma'][0] / chDic['gain'][0]
])
dfDat = DataFrame(outData, columns=n0, index=chNos)
#print('Check: ', dfDat[(dfDat.chi2<10.) & (dfDat.chi2!=0)].pull_gain)
dfDat.chi2.plot.hist(bins=250)
plt.show()
#dfDat[(dfDat.chi2!=0) & (abs(dfDat.pull_gain)<50)].pull_gain.hist(bins=500)
ents, bins = np.histogram(np.array(
dfDat[(dfDat.chi2 != 0) & (abs(dfDat.pull_gain) < 50)].pull_gain),
bins=np.arange(-10, 10., 0.1))
#print(len(ents), len(bins))
print('Check entries: ', ents.sum())
plt.bar(bins[:-1], ents, width=0.1)
b1 = np.argwhere(ents >= 10)[0][0]
b2 = np.argwhere(ents >= 10)[-1][0]
pv, pvE = curve_fit(gauFit,
bins[b1:b2],
ents[b1:b2],
sigma=np.sqrt(ents[b1:b2]),
p0=[ents.max(), -1, 1])
print('pull fit: ', pv, pvE)
plt.plot(bins[:-1], gauFit(bins[:-1], *pv), 'r')
print(dfDat[(dfDat.chi2 != 0) & (abs(dfDat.pull_gain) >= 50)].pull_gain)
plt.show()
dfDat[(dfDat.chi2 < 10.) & (dfDat.chi2 != 0) &
(dfDat.sigPe < 10)].sigPe.plot.hist(bins=100)
plt.show()
#print(dfDat.sigma)
dfDat.to_csv('SiPMDatTest.dat')
with open('SiPMGain_database.dat', 'w') as dbF:
## for i in range(len(n0)):
## dbF.write('5166, 100000, '+str(n0[i])+', '+str(dbDat[i][0])+', '+str(dbDat[i][1])+', '+str(dbDat[i][2])+', '+str(dbDat[i][3])+'\n')
for ch, (gain, gE, sig, sigE) in zip(chNos, dbDat):
dbF.write('5166, 100000, ' + str(ch) + ', ' + str(gain) + ', ' +
str(gE) + ', ' + str(sig) + ', ' + str(sigE) + '\n')
def fit_dataset(dataF=None, funcName=None, minStat=None, limitPed=None):
""" Check new fit function on SiPM spectra """
global useSavedSeeds, GainSeeds, SigSeeds
file_name = dataF
func_name = funcName
min_stat = minStat
limit_ped = limitPed
optimise = True
if not file_name:
optimise = False
file_name = sys.argv[1]
func_name = sys.argv[2]
min_stat = 0
limit_ped = 10000.
if len(sys.argv) > 3:
useSavedSeeds = True if 'true' in sys.argv[3] else False
min_stat = int(sys.argv[4])
limit_ped = int(sys.argv[5])
run_no = file_name[file_name.find('R')+1:file_name.find('R')+5]
run_no = int(run_no)
chNos = DB.DataSiPM(run_no).SensorID.values
if useSavedSeeds:
dodgy = DB.DataSiPM(run_no).index[DB.DataSiPM(run_no).Active==0].values
GainSeeds = DB.DataSiPM(run_no).adc_to_pes.values
SigSeeds = DB.DataSiPM(run_no).Sigma.values
## Give generic values to previously dead or dodgy channels
GainSeeds[dodgy] = 15
SigSeeds[dodgy] = 2
sipmIn = tb.open_file(file_name, 'r')
## Bins are the same for dark and light, just use light for now
bins = np.array(sipmIn.root.HIST.sipm_spe_bins)
## LED correlated and anticorrelated spectra:
specsL = np.array(sipmIn.root.HIST.sipm_spe).sum(axis=0)
specsD = np.array(sipmIn.root.HIST.sipm_dark).sum(axis=0)
ffuncs = {'ngau':speR.poisson_scaled_gaussians(n_gaussians=7),
'intgau':speR.poisson_scaled_gaussians(min_integral=100),
'dfunc':partial(speR.scaled_dark_pedestal, min_integral=100),
'conv':partial(speR.dark_convolution, min_integral=100)}
## Loop over the specra:
outData = []
outDict = {}
llchans = []
if not optimise:
fnam = {'ngau':'poisson_scaled_gaussians_ngau', 'intgau':'poisson_scaled_gaussians_min', 'dfunc':'scaled_dark_pedestal', 'conv':'dark_convolution'}
pOut = tb.open_file('sipmCalParOut_R'+str(run_no)+'_F'+func_name+'.h5', 'w')
param_writer = pIO.channel_param_writer(pOut,
sensor_type='sipm',
func_name=fnam[func_name],
param_names=pIO.generic_params)
## Extra protection since 3065 is weird
knownDead = [ 3056, 11009, 12058, 14010, 22028, 22029, 25049 ]
specialCheck = [1006, 1007, 3000, 3001, 5010, 7000, 22029, 28056, 28057]
for ich, (led, dar) in enumerate(zip(specsL, specsD)):
if chNos[ich] in knownDead:
outData.append([chNos[ich], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], 0, 0])
if not optimise:
for kname in pIO.generic_params:
outDict[kname] = (0, 0)
param_writer(chNos[ich], outDict)
print('no peaks in dark spectrum, spec ', ich)
continue
## Limits for safe fit
b1 = 0
b2 = len(dar)
if min_stat != 0:
valid_bins = np.argwhere(led>=min_stat)
b1 = valid_bins[0][0]
b2 = valid_bins[-1][0]
outDict[pIO.generic_params[-2]] = (bins[b1], bins[min(len(bins)-1, b2)])
# Seed finding
pD = find_peaks_cwt(dar, np.arange(2, 20), min_snr=2)
if len(pD) == 0:
## Try to salvage in case not a masked channel
## Masked channels have al entries in one bin.
if led[led>0].size == 1:
outData.append([0., 0., 0., 0., 0., 0., 0.])
print('no peaks in dark spectrum, spec ', ich)
continue
else:
pD = np.array([dar.argmax()])
## Fit the dark spectrum with a Gaussian (not really necessary for the conv option)
gb0 = [(0, -100, 0), (1e99, 100, 10000)]
sd0 = (dar.sum(), 0, 2)
errs = np.sqrt(dar[pD[0]-5:pD[0]+5])
errs[errs==0] = 0.1
gfitRes = fitf.fit(fitf.gauss, bins[pD[0]-5:pD[0]+5], dar[pD[0]-5:pD[0]+5], sd0, sigma=errs, bounds=gb0)
outDict[pIO.generic_params[2]] = (gfitRes.values[1], gfitRes.errors[1])
outDict[pIO.generic_params[3]] = (gfitRes.values[2], gfitRes.errors[2])
## Scale just in case we lost a different amount of integrals in dark and led
## scale = led.sum() / dar.sum()
scale = 1
## Take into account the scale in seed finding (could affect Poisson mu)????
ped_vals = np.array([gfitRes.values[0] * scale, gfitRes.values[1], gfitRes.values[2]])
binR = bins[b1:b2]
global darr
darr = dar[b1:b2] * scale
darr = darr[binR<5]
seeds, bounds = seeds_and_bounds(ich, func_name, bins[b1:b2], led[b1:b2],
ped_vals, gfitRes.errors, limit_ped)
## Protect low light channels
if seeds[1] < 0.2:
llchans.append(chNos[ich])
## Dodgy setting of high charge dark bins to zero
dar[bins>gfitRes.values[1] + 3*gfitRes.values[2]] = 0
##
if 'dfunc' in func_name:
respF = ffuncs[func_name](dark_spectrum=dar[b1:b2] * scale,
pedestal_mean=gfitRes.values[1],
pedestal_sigma=gfitRes.values[2])
elif 'conv' in func_name:
respF = ffuncs[func_name](dark_spectrum=dar[b1:b2] * scale,
bins=bins[b1:b2])
else:
respF = ffuncs[func_name]
## The fit
errs = np.sqrt(led)
if not 'gau' in func_name:
errs = np.sqrt(errs**2 + np.exp(-2 * seeds[1]) * dar)
errs[errs==0] = 0.001
print('About to fit channel ', chNos[ich])
rfit = fitf.fit(respF, bins[b1:b2], led[b1:b2], seeds, sigma=errs[b1:b2], bounds=bounds)
chi = rfit.chi2
## Attempt to catch bad fits and refit (currently only valid for dfunc and conv)
if chi >= 7 or rfit.values[3] >= 2.5 or rfit.values[3] <= 1:
## The offending parameter seems to be the sigma in most cases
nseed = rfit.values
nseed[3] = 1.7
nbound = [(bounds[0][0], bounds[0][1], bounds[0][2], 1),
(bounds[1][0], bounds[1][1], bounds[1][2], 2.5)]
rfit = fitf.fit(respF, bins[b1:b2], led[b1:b2], nseed, sigma=errs[b1:b2], bounds=nbound)
chi = rfit.chi2
if not optimise:
if chNos[ich] in specialCheck or chi >= 10 or rfit.values[2] < 12 or rfit.values[2] > 19 or rfit.values[3] > 3:
if chNos[ich] in specialCheck: print('Special check channel '+str(chNos[ich]))
print('Channel fit: ', rfit.values, chi)
plt.errorbar(bins, led, xerr=0.5*np.diff(bins)[0], yerr=errs, fmt='b.')
plt.plot(bins[b1:b2], respF(bins[b1:b2], *rfit.values), 'r')
plt.plot(bins[b1:b2], respF(bins[b1:b2], *seeds), 'g')
plt.title('Spe response fit to channel '+str(chNos[ich]))
plt.xlabel('ADC')
plt.ylabel('AU')
plt.show()
outData.append([chNos[ich], rfit.values, rfit.errors, respF.n_gaussians, chi])
outDict[pIO.generic_params[0]] = (rfit.values[0], rfit.errors[0])
outDict[pIO.generic_params[1]] = (rfit.values[1], rfit.errors[1])
gIndx = 2
if 'gau' in func_name:
gaIndx = 4
outDict[pIO.generic_params[4]] = (rfit.values[gIndx], rfit.errors[gIndx])
outDict[pIO.generic_params[5]] = (rfit.values[gIndx+1], rfit.errors[gIndx+1])
outDict[pIO.generic_params[-1]] = (respF.n_gaussians, rfit.chi2)
if not optimise:
param_writer(chNos[ich], outDict)
## Couple of plots
gainIndx = 2
if 'gau' in func_name:
gainIndx = 4
plot_names = ["Gain", "1pe sigma", "Poisson mu", "chi2"]
pVals = [np.fromiter((ch[1][gainIndx] for ch in outData), np.float),
np.fromiter((ch[1][gainIndx+1] for ch in outData), np.float),
np.fromiter((ch[1][1] for ch in outData), np.float),
np.fromiter((ch[4] for ch in outData), np.float)]
if optimise:
sipmIn.close()
return pVals
pOut.close()
#global scalerChis
pos_x = DB.DataSiPM(run_no).X.values
pos_y = DB.DataSiPM(run_no).Y.values
chNos = DB.DataSiPM(run_no).SensorID.values
## vals2D = np.zeros((int((pos_x.max()-pos_x.min())/10)+1, int((pos_y.max()-pos_y.min())/10)+1))
## print('shape: ', vals2D.shape)
## *_, chis = display_matrix(pos_x, pos_y, pVals[3])
## Trampa
#pVals[3][pVals[3]>10] = 0
plt.scatter(pos_x, pos_y, c=pVals[3])
plt.title("Fit chi^2 map")
plt.xlabel("X (mm)")
plt.ylabel("Y (mm)")
plt.colorbar()
plt.show()
#mask = np.argwhere((pVals[2]>=2) & (pVals[2]<8))
#mask = np.argwhere((chNos<9000) | (chNos>=11000))
#plt.scatter(pos_x[mask], pos_y[mask], c=pVals[2][mask])
plt.scatter(pos_x, pos_y, c=pVals[2])
plt.title("Fit poisson mu")
plt.xlabel("X (mm)")
plt.ylabel("Y (mm)")
plt.colorbar()
plt.show()
## fg2, ax2 = plt.subplots()
## p = ax2.pcolor(pos_x, pos_y, scalerChis, cmap=cm.Spectral, vmin=np.abs(scalerChis).min(), vmax=np.abs(scalerChis).max())
## plt.colorbar(p, ax=ax2)
## fg2.show()
#plt.hist(scalerChis, bins=1000)
#plt.show()
fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(20,6))
chiVs = pVals[3]
for ax, val, nm in zip(axes.flatten(), pVals, plot_names):
ax.hist(val[(chiVs<10) & (chiVs!=0)], bins=100)
ax.set_title(nm)
fig.show()
input('finished with plots?')
print('Low light chans:', llchans)
from invisible_cities.reco.psf_functions import add_empty_sensors_and_normalize_q
from invisible_cities.reco.psf_functions import add_variable_weighted_mean
import invisible_cities.core.core_functions as coref
import invisible_cities.io.dst_io as dstio
from invisible_cities.database import load_db
from invisible_cities.io.kdst_io import psf_writer
"""
This script takes the penthesilea output of Kr as an input
and returns files to be combined to generate the full point spread functions.
"""
# Database
run = -1
the_db = load_db.DataSiPM('next100', run)
z_max = 1210
z_step = 20
zrange = []
zrange.extend(np.arange(0, z_max, z_step))
### PSF binning and range
bin_size = 1
Xrange = [-100, 100]
Yrange = [-100, 100]
ranges = [Xrange, Yrange]
nbinX = int(np.diff(Xrange) / bin_size)
nbinY = int(np.diff(Yrange) / bin_size)
xstep = [-100, 100]
def sipm_ids():
return DB.DataSiPM('new', 6400).SensorID.values
type=int,
required=False,
default=800)
ns = parser.parse_args()
in_files = ns.in_files
out_path = ns.out_path
detector = ns.detector
min_sample = ns.min_sample
max_sample = ns.max_sample
run_no = int(in_files[0][in_files[0].find('run_') +
4:in_files[0].find('run_') + 8])
num_wf = int(in_files[0][in_files[0].find('run_') +
9:in_files[0].find('run_') + 13])
out_file = f'{out_path}/dark_rate_comp_q_values_R{run_no}_{num_wf}'
conv_gain = load_db.DataSiPM(detector, run_no).adc_to_pes.values[:, np.newaxis]
conv_gain[conv_gain == 0] = 17
conv_gain = conv_gain.reshape(1792)
dark_qs_all = []
for file in in_files:
dark_rwf = tb.open_file(file)
for evt_sipm in dark_rwf.root.RD.sipmrwf:
cwf = subtract_mode(evt_sipm[:, min_sample:max_sample])
charges_cd = cwf.sum(1)
dark_qs_all.append(charges_cd / conv_gain)
tqs = np.array(dark_qs_all).T
np.savez(out_file, tqs=tqs)
def s2_peak():
times = np.arange(20) * units.mus
bin_widths = np.full_like(times, units.mus)
pmt_ids = DB.DataPMT('new', 6400).SensorID.values
sipm_ids = DB.DataSiPM('new', 6400).index.values
np.random.seed(22)
pmts = PMTResponses(pmt_ids,
np.random.uniform(0, 100, (len(pmt_ids), len(times))))
sipms = SiPMResponses(
sipm_ids, np.random.uniform(0, 10, (len(sipm_ids), len(times))))
## Values for comparison in tests
chan_slice = slice(3, 9)
raw_arr = [
[
7.51759755, 5.60509619, 6.42950506, 9.20429423, 9.70312128,
2.02362046
],
[
7.05972334, 2.22846472, 5.46749176, 7.31111882, 3.40960906,
4.56294121
],
[
6.70472842, 3.59272265, 5.16309049, 1.56536978, 9.81538459,
6.82850322
],
[
4.15689334, 3.41990664, 5.79051761, 0.81210519, 1.0304272,
0.40297561
],
[4.73786661, 0.6207543, 1.6485871, 2.52693314, 8.91532864, 1.52840296],
[
5.92994762, 3.13080909, 2.56228467, 3.18354286, 1.55118873,
0.60703854
],
[
3.51052921, 6.40861592, 4.52017217, 1.08859183, 5.93003437,
7.38501235
],
[
8.57725314, 5.32329684, 4.3160815, 6.77629621, 7.49288644,
9.25124645
],
[8.0475847, 6.4163836, 3.62396899, 5.04696816, 8.86944591, 0.08356193],
[3.0936211, 3.95499815, 5.96360799, 0.6605126, 3.86466816, 9.6719478],
[
1.96541611, 5.36489548, 3.70399066, 1.12084339, 7.96734339,
3.24741959
],
[6.14385189, 1.83113282, 2.0741162, 4.83746891, 3.2362641, 0.17227037],
[
1.67604863, 5.59126381, 2.09007632, 5.46945561, 9.71342408,
1.6268169
],
[
9.61221953, 5.27643349, 2.7304223, 7.30143289, 2.37011787,
7.83264795
],
[
9.90983548, 5.79429284, 3.23532663, 9.17294685, 5.97553658,
4.4452637
],
[
0.35683435, 4.30303963, 1.90266857, 4.17023551, 4.06186979,
2.7965879
],
[0.83376644, 9.7300586, 6.05654181, 5.09992868, 8.97882656, 4.6280693],
[
8.48222031, 0.12398542, 7.06665329, 7.90802805, 8.97576213,
1.98215824
],
[0.45055814, 9.90187098, 0.7664873, 4.74533329, 8.9772245, 6.48808184],
[
0.88080893, 5.19986295, 6.27365681, 4.46247949, 4.42179112,
0.3344096
]
]
sn_arr = [[
2.55346379, 2.1727307, 2.35093766, 2.86042287, 2.97285019, 1.15426079
], [
2.46390195, 1.23075187, 2.14164701, 2.51344824, 1.6329798, 1.92597048
], [
2.3922998, 1.6673905, 2.07136149, 0.94995374, 2.99152438, 2.43278461
], [
1.80446599, 1.61757723, 2.21397598, 0.57976386, 0.73221374, 0.33431018
], [
1.95255757, 0.48024565, 1.00356077, 1.31780419, 2.83845215, 0.95183676
], [
2.22861903, 1.5311806, 1.34814435, 1.52928755, 0.98082474, 0.47157412
], [
1.62612892, 2.34657869, 1.91522882, 0.72757474, 2.26079876, 2.54277753
], [
2.75012934, 2.10862141, 1.86320656, 2.40689549, 2.57871696, 2.88240738
], [
2.65355845, 2.34820047, 1.67626069, 2.02740696, 2.83043378, 0.07848485
], [
1.50175408, 1.76784557, 2.25184422, 0.49088564, 1.76128254, 2.95377289
], [
1.11351605, 2.11819761, 1.69879401, 0.74379173, 2.66806122, 1.56651489
], [
2.27489977, 1.07879322, 1.17346325, 1.97695394, 1.58172378, 0.15582134
], [
0.99700377, 2.16962471, 1.17948397, 2.1258517, 2.97456874, 0.99432725
], [
2.93003466, 2.09778509, 1.40429925, 2.51155634, 1.30074808, 2.62807274
], [
2.97983427, 2.21480833, 1.56288192, 2.85500604, 2.27064999, 1.89628347
], [
0.29069255, 1.859838, 1.10727576, 1.80798637, 1.81437105, 1.42590517
], [
0.59196924, 2.96349975, 2.27193322, 2.03998311, 2.84951285, 1.94222486
], [
2.73304224, 0.11442983, 2.48042035, 2.62755231, 2.84898002, 1.13831482
], [0.35613537, 2.99207933, 0.56870727, 1.95439588, 2.8492343, 2.36312065],
[
0.61808354, 2.07996798, 2.3182313, 1.8836441, 1.90782557,
0.28421464
]]
expected_array = {
SiPMCharge.raw: raw_arr,
SiPMCharge.signal_to_noise: sn_arr
}
expected_single = {
SiPMCharge.raw:
[99.6473, 93.8179, 81.3852, 92.4639, 125.2603, 75.8990],
SiPMCharge.signal_to_noise:
[9.6651, 9.3927, 8.7087, 9.3396, 10.9941, 8.4203]
}
return (S2(times, bin_widths, pmts,
sipms), chan_slice, expected_single, expected_array)
def __init__(
self,
detector: str = 'new', # detector name in data-base
run_number: int = -1, # run number -1 for MC
krmap_filename: str = '', # KrMap file name to generate PMT-PSF
psfsipm_filename:
str = '', # PSF-SiPM file name to generate SiPM-PSF
**kargs):
self.ws = 60.0 * units.eV
self.wi = 22.4 * units.eV
self.fano_factor = 0.15
self.conde_policarpo_factor = 0.10
self.drift_velocity = 1.00 * units.mm / units.mus
self.lifetime = 7.00 * units.ms
self.transverse_diffusion = 1.00 * units.mm / units.cm**0.5
self.longitudinal_diffusion = 0.30 * units.mm / units.cm**0.5
self.voxel_sizes = np.array(
(2.00 * units.mm, 2.00 * units.mm, 0.2 * units.mm), dtype=float)
self.el_gain = 1e3
self.wf_buffer_time = 1300 * units.mus
self.wf_pmt_bin_time = 25 * units.ns
self.wf_sipm_bin_time = 1 * units.mus
self.EP_z = 632 * units.mm
self.EL_dz = 5 * units.mm
self.TP_z = -self.EL_dz
self.EL_dtime = self.EL_dz / self.drift_velocity
self.EL_pmt_time_sample = self.wf_pmt_bin_time
self.EL_sipm_time_sample = self.wf_sipm_bin_time
self.s1_dtime = 200 * units.ns
db_pmts = db.DataPMT(detector, run_number)
self.x_pmts = db_pmts['X'].values
self.y_pmts = db_pmts['Y'].values
self.adc_to_pes_pmts = db_pmts['adc_to_pes'].values
self.npmts = len(self.x_pmts)
db_sipms = db.DataSiPM(detector, run_number)
self.x_sipms = db_sipms['X'].values
self.y_sipms = db_sipms['Y'].values
self.adc_to_pes_sipms = db_sipms['adc_to_pes'].values
self.nsipms = len(self.x_sipms)
## TODO
nphotons = (41.5 * units.keV / self.wi) * self.el_gain * self.npmts
if (krmap_filename != ''):
print('load psf pmt from file : ', krmap_filename)
if (psfsipm_filename != ''):
print('load psf sipm from file : ', psfsipm_filename)
psf_pmt = partial(_psf, dz=self.EP_z, factor=25e3)
psf_sipm = partial(_psf, dz=self.TP_z, factor=0.15)
psf_s1 = partial(_psf, factor=1e3)
factor = 1. / nphotons
self.psf_pmt = psf_pmt if krmap_filename == '' else get_psf_pmt_from_krmap(
krmap_filename, factor)
self.psf_sipm = psf_sipm if psfsipm_filename == '' else get_psf_sipm_from_file(
psfsipm_filename, factor / 0.7e-4)
self.psf_s1 = psf_s1
self._configure(**kargs)
self._update()
def sipm_connectivity_check(elec_name, dark_name, run_no):
"""
Compare basic parameters of electronics only
and dark current spectra and have the user classify
channels with little difference.
input:
elec_name : name of file with electronics only spectra
dark_name : name of file with dark counts only spectra
run_no : run number for classification file
"""
sensors = DB.DataSiPM(db_file, int(run_no)).SensorID.values
with tb.open_file(elec_name, 'r') as eFile, \
tb.open_file(dark_name, 'r') as dFile:
min_incr = 0.5
try:
binsE = np.array(eFile.root.HIST.sipm_dark_bins)
binsD = np.array(dFile.root.HIST.sipm_dark_bins)
specsE = np.array(eFile.root.HIST.sipm_dark).sum(axis=0)
specsD = np.array(dFile.root.HIST.sipm_dark).sum(axis=0)
except tb.NoSuchNodeError:
print('Node not found in files, spectra in HIST node required')
raise
## Bin half widths as x errors
xerrE = 0.5 * np.diff(binsE)[0]
xerrD = 0.5 * np.diff(binsD)[0]
with open('bad_channels_'+str(run_no)+'.txt', 'w') as dChan:
dChan.write('Run \t Channel \t classification \n')
for sipm, espec, dspec in zip(sensors, specsE, specsD):
avE, rmsE = weighted_mean_and_var(binsE, espec, unbiased=True)
avD, rmsD = weighted_mean_and_var(binsD, dspec, unbiased=True)
mean_low = avE + min_incr >= avD
rms_low = rmsE + min_incr >= rmsD
if mean_low or rms_low:
print("Possible bad channel: "+str(sipm))
print("identified as having: ")
if mean_low:
print('low mean: meanE='+str(avE)+', meanD='+str(avD))
if rms_low:
print('low rms: rmsE='+str(rmsE)+', rmsD='+str(rmsD))
plot_title = 'Elec/dark current comparison ch'+str(sipm)
plot_spectra([binsE, binsD],
[espec, dspec],
[xerrE, xerrD],
['Electronics only', 'Dark current'],
title=plot_title)
clif = input("How do you classify this channel? [dead/noisy/suspect/ok]")
dChan.write(run_no+' \t '+str(sipm)+' \t '+clif+' \n');
plt.clf()
plt.close()
chan = input("Want to check specific channels? [y/n]")
if chan == 'y':
chan = input("Which channel number? [num/stop]")
while chan != 'stop':
indx = np.argwhere(sensors==int(chan))
if len(indx) == 0:
print('Channel not found')
continue
indx = indx[0][0]
plot_title = 'Elec/dark current comparison ch'+chan
plot_spectra([binsE, binsD],
[specsE[indx], specsD[indx]],
[xerrE, xerrD],
['Electronics only', 'Dark current'],
title=plot_title)
clif = input("How do you classify this channel? [dead/noisy/suspect/ok]")
dChan.write(run_no+' \t '+str(chan)+' \t '+clif+' \n');
plt.clf()
plt.close()
chan = input("Another channel? [num/stop]")
return
def sipm_connectivity_check(elec_name, dark_name, RUN_NO):
""" Compare basic parameters of electronics only
and dark current runs and have the user classify
channels with little difference """
sensors = DB.DataSiPM(int(RUN_NO)).SensorID.values
with tb.open_file(elec_name, 'r') as eFile, \
tb.open_file(dark_name, 'r') as dFile:
min_incr = 0.5
if 'HIST' not in eFile.root:
print('Error, this check requires spectra')
exit()
binsE = np.array(eFile.root.HIST.sipm_dark_bins)
binsD = np.array(dFile.root.HIST.sipm_dark_bins)
specsE = np.array(eFile.root.HIST.sipm_dark).sum(axis=0)
specsD = np.array(dFile.root.HIST.sipm_dark).sum(axis=0)
## Bin half widths as x errors
xerrE = 0.5 * np.diff(binsE)[0]
xerrD = 0.5 * np.diff(binsD)[0]
with open('dodgy_channels.txt', 'a') as dChan:
dChan.write('Run \t Channel \t classification \n')
for ich, (espec, dspec) in enumerate(zip(specsE, specsD)):
## Maybe should be replaced with
## database access.
chan_no = sensors[ich]
avE, rmsE = weighted_mean_and_std(binsE, espec, unbiased=True)
avD, rmsD = weighted_mean_and_std(binsD, dspec, unbiased=True)
mean_low = avE + min_incr >= avD
rms_low = rmsE + min_incr >= rmsD
#stats_diff = espec.sum() != dspec.sum()
if mean_low or rms_low:# or stats_diff:
print("Possible dodgy channel: "+str(chan_no))
print("identified as having: ")
if mean_low: print('low mean: meanE='+str(avE)+', meanD='+str(avD))
if rms_low: print('low rms: rmsE='+str(rmsE)+', rmsD='+str(rmsD))
#if stats_diff: print('missing entries: sumE/sumD='+str(espec.sum()/dspec.sum()))
plt.yscale('log')
plt.errorbar(binsE, espec, xerr=xerrE,
yerr=np.sqrt(espec), fmt='b.',
label='Electronics only')
plt.errorbar(binsD, dspec, xerr=xerrD,
yerr=np.sqrt(dspec), fmt='r.', ecolor='r',
label='Dark current')
plt.title('Elec/dark current comparison ch'+str(chan_no))
plt.xlabel('ADC')
plt.ylabel('AU')
plt.legend()
plt.show(block=False)
clif = input("How do you classify this channel? [dead/noisy/suspect/ok]")
dChan.write(RUN_NO+' \t '+str(chan_no)+' \t '+clif+' \n');
plt.clf()
plt.close()
check_chan = input("Want to check specific channels? [y/n]")
if check_chan == 'y':
check_chan = input("Which channel number? [num/stop]")
while check_chan != 'stop':
indx = np.argwhere(sensors==int(check_chan))
if len(indx) == 0:
print('Channel not found')
continue
indx = indx[0][0]
plt.yscale('log')
plt.errorbar(binsE, specsE[indx], xerr=xerrE,
yerr=np.sqrt(specsE[indx]), fmt='b.',
label='Electronics only')
plt.errorbar(binsD, specsD[indx], xerr=xerrD,
yerr=np.sqrt(specsD[indx]), fmt='r.',
ecolor='r',
label='Dark current')
plt.title('Elec/dark current comparison ch'+str(check_chan))
plt.xlabel('ADC')
plt.ylabel('AU')
plt.legend()
plt.show(block=False)
clif = input("How do you classify this channel? [dead/noisy/suspect/ok]")
dChan.write(RUN_NO+' \t '+str(check_chan)+' \t '+clif+' \n');
plt.clf()
plt.close()
check_chan = input("Another channel? [num/stop]")
return
