def test_simple_pass():
# This test exposes the half-open vs full-open histogram code difference
# between np.histogram and skbeam's histogram.
h = Histogram((5, 0, 3.1))
a = np.arange(1, 6)
b = np.asarray([1, 1, 2, 3, 2])
h.fill(a, weights=b)
np_res = np.histogram(a, h.edges[0], weights=b)[0]
assert_array_equal(h.values, np_res)
def _1d_histogram_tester(binlowhighs, x, weights=1):
h = Histogram(binlowhighs)
h.fill(x, weights=weights)
if np.isscalar(weights):
ynp = np.histogram(x, h.edges[0])[0]
else:
ynp = np.histogram(x, h.edges[0], weights=weights)[0]
assert_array_almost_equal(ynp, h.values)
h.reset()
h._always_use_fillnd = True
h.fill(x, weights=weights)
assert_array_almost_equal(ynp, h.values)
def histogram_plot(file_usage, log_yaxis=False):
min_size = 0
max_size = 1e9 # 10 GB here
Nbins = 10000
for plan in file_usage:
fig, ax = plt.subplots()
h = Histogram((Nbins, min_size, max_size))
h.fill(file_usage[plan])
ax.set_xlabel('FILE USAGE')
ax.set_ylabel('FREQUENCY')
ax.set_title('CHX | Plan:Detector')
if log_yaxis:
ax.set_yscale('log')
plt.plot(h.centers[0] * 1e-6, h.values, label=plan)
plt.legend()
plt.savefig('hist_{}'.format(plan))
def _1d_histogram_tester(binlowhighs, x, weights=1):
h = Histogram(binlowhighs)
h.fill(x, weights=weights)
if np.isscalar(weights):
ynp = np.histogram(x, h.edges[0])[0]
else:
ynp = np.histogram(x, h.edges[0], weights=weights)[0]
assert_array_almost_equal(ynp, h.values)
h.reset()
h._always_use_fillnd = True
h.fill(x, weights=weights)
assert_array_almost_equal(ynp, h.values)
def _2d_histogram_tester(binlowhighs, x, y, weights=1):
h = Histogram(*binlowhighs)
h.fill(x, y, weights=weights)
if np.isscalar(weights):
if np.isscalar(x):
assert np.isscalar(y), 'If x is a scalar, y must be also'
ynp = np.histogram2d([x], [y], bins=h.edges)[0]
else:
ynp = np.histogram2d(x, y, bins=h.edges)[0]
else:
ynp = np.histogram2d(x, y, bins=h.edges, weights=weights)[0]
assert_array_almost_equal(ynp, h.values)
h.reset()
h._always_use_fillnd = True
h.fill(x, y, weights=weights)
assert_array_almost_equal(ynp, h.values)
def histogram_plot(dfs, log_yscale=False):
min_size = 0
max_size = 1e9 # 10 GB here
Nbins = 10000
for df in dfs:
fig, ax = plt.subplots()
h = Histogram((Nbins, min_size, max_size))
col_name = df.columns.values[0]
col = np.array(df[col_name])
h.fill(col)
ax.set_xlabel('FILE USAGE')
ax.set_ylabel('FREQUENCY')
ax.set_title('CHX | Plan:Detector')
if log_yscale:
ax.set_yscale('log')
plt.plot(h.centers[0] * 1e-6, h.values, label=col_name)
plt.legend(loc=1)
print(col_name.replace(':', '_').replace('(fileusage)', ''))
plt.savefig('chx_{}'.format(
col_name.replace(':', '_').replace('(fileusage)', '')))
def plot_histogram(dfs):
min_size = 0
max_size = 5e8 # 10 GB here
Nbins = 10000
labels = [num for num in range(11000) if num % 2000 == 0]
minor_labels = [num for num in range(11000) if num % 1000 == 0]
minor_2_labels = [num for num in range(11000) if num % 500 == 0]
#y_minor_labels = [num for num in range()]
x_labels = [i for i in range(10)]
plt.ion()
fig, axs = plt.subplots(2, 2)
plans = ['count', 'rel_scan']
detectors = ['eiger1m_single_image', 'eiger4m_single_image']
for i, plan in enumerate(plans):
for j, detector in enumerate(detectors):
ax = axs[i, j]
df = dfs.get(plan, dict()).get(detector, None)
if df is not None:
h = Histogram((Nbins, min_size, max_size))
col_name = df.columns[0]
col = np.array(df[col_name])
h.fill(col)
ax.semilogy(h.centers[0][1:] * 1e-9,
h.values[1:],
color='darkgreen')
ax.set_ylim(ymin=1.1)
ax.set_xlim(xmin=1e-7)
if i == 1 and j == 1:
ax.set_ylim(ymax=10)
if (i == 0 and j == 0 or i == 1 and j == 0):
ax.set_ylabel(
col_name.split(':')[0] + ' - # of occurences')
if (i == 0 and j == 0 or i == 0 and j == 1):
ax.set_title(
col_name.replace('(fileusage)', '').split(':')[1])
plt.suptitle('CHX Plans + Detectors')
plt.title('Histogram Semi-Log Plot', position=(-.14, -0.30))
from psana import *
import numpy as np
from ImgAlgos.PyAlgos import PyAlgos
from skbeam.core.accumulators.histogram import Histogram
dsource = MPIDataSource('exp=sxr07416:run=28:smd')
det = Detector('OPAL1')
alg = PyAlgos()
alg.set_peak_selection_pars(npix_min=9,
npix_max=100,
amax_thr=40,
atot_thr=300,
son_min=0)
hist_row = Histogram((1024, 0., 1024.))
hist_col = Histogram((1024, 0., 1024.))
hist_amp = Histogram((1024, 0., 3000.))
smldata = dsource.small_data('run28.h5', gather_interval=100)
peakrow = np.zeros((10), dtype=int)
peakcol = np.zeros((10), dtype=int)
peakamp = np.zeros((10), dtype=float)
for nevt, evt in enumerate(dsource.events()):
calib = det.calib(evt)
if calib is None: continue
peaks = alg.peak_finder_v1(calib,
thr_low=40,
thr_high=40,
if __name__ == '__main__':
import itertools
x = [1000, 0, 10.01]
y = [1000, 0, 9.01]
xf = np.random.random(1000000)*10*4
yf = np.random.random(1000000)*9*15
xi = xf.astype(int)
yi = yf.astype(int)
wf = np.linspace(1, 10, len(xf))
wi = wf.copy()
times = []
print("Testing 2D histogram timings")
for xvals, yvals, weights in itertools.product([xf, xi], [yf, yi],
[wf, wi]):
t0 = time()
h = Histogram(x, y)
h.fill(xvals, yvals, weights=weights)
skbeam_time = time() - t0
edges = h.edges
t0 = time()
ynp = np.histogram2d(xvals, yvals, bins=edges, weights=weights)[0]
numpy_time = time() - t0
times.append(numpy_time / skbeam_time)
assert_almost_equal(np.sum(h.values), np.sum(ynp))
print('skbeam is %s times faster than numpy, on average' %
np.average(times))
# test_1d_histogram()
# test_2d_histogram()
# TODO do a better job sampling the variable space
##################################################################
########################### Histograms & stuff ###################
##################################################################
runs = np.array([args.runs], dtype='int').flatten()
xLength = 839 # rotated camera 798
yLength = 591 # rotated camera 647
photonE = 508. # 1050. for runs 134 - ~150 |||| 508. for 156, 157 |||| 530 for 158+
cutoff = 0.2 # gas detector cutoff
xtcav = True
binsOfHist = xLength * len(runs)
spectraCombined = Histogram((binsOfHist, 0., float(binsOfHist)))
delayhist2d = Histogram((50, 0., 50.), (binsOfHist, 0., float(binsOfHist)))
delayCountHist = Histogram((50, 0., 50.))
feegashist2d = Histogram((100, 0., 2.0), (binsOfHist, 0., float(binsOfHist)))
feeCountHist = Histogram((100, 0., 2.0))
photonhist2d = Histogram((200, photonE * (1. - 0.02), photonE * (1. + 0.02)),
(binsOfHist, 0., float(binsOfHist)))
photonCountHist = Histogram(
(200, photonE * (1. - 0.02), photonE * (1. + 0.02)))
RatiosXTCAV = Histogram((50, -5., 5.), (binsOfHist, 0., float(binsOfHist)))
TotalPowXTCAV = Histogram((100, 600., 1200.),
(binsOfHist, 0., float(binsOfHist)))
maxhistlim_L3_elec1mom_L3=3400
numbins_L3_elec1mom_elec1mom=100
minhistlim_L3_elec1mom_elec1mom=0
maxhistlim_L3_elec1mom_elec1mom=714
numbins_L3Energy=100
minhistlim_L3Energy=3300
maxhistlim_L3Energy=3400
numbins_L3EnergyWeighted=100
minhistlim_L3EnergyWeighted=3300
maxhistlim_L3EnergyWeighted=3400
hist_L3_elec1mom = Histogram((numbins_L3_elec1mom_L3,minhistlim_L3_elec1mom_L3,\
mmaxhistlim_L3_elec1mom_L3),(numbins_L3_elec1mom_elec1mom,\
minhistlim_L3_elec1mom_elec1mom,maxhistlim_L3_elec1mom_elec1mom))
hist_L3Energy=Histogram((numbins_L3Energy,minhistlim_L3Energy,maxhistlim_L3Energy))
hist_L3EnergyWeighted=Histogram((numbins_L3EnergyWeighted,minhistlim_L3EnergyWeighted,maxhistlim_L3EnergyWeighted))
for nevt, evt in enumerate(ds.events()):
print nevt
energyL3=procL3Energy.L3Energy(evt)
moments, numhits=procSHES.CalibProcess(evt, inPix=True)
mom1=moments[0]
hist_L3Energy.fill(energyL3)
hist_L3EnergyWeighted.fill([energyL3], weights=[numhits])
if nevt>100:
opal_hit_avg_buff = collections.deque(maxlen=history_len_long) opal_circ_buff = collections.deque(maxlen=history_len) xproj_int_buff = collections.deque(maxlen=history_len) xproj_int_avg_buff = collections.deque(maxlen=history_len) moments_buff = collections.deque(maxlen=history_len_long) #moments_avg_buff = collections.deque(maxlen=history_len_long) #xhistogram_buff = collections.deque(maxlen=history_len) hitxprojhist_buff = collections.deque(maxlen=history_len) hitxprojjitter_buff = collections.deque(maxlen=history_len) hitxprojseeded_buff = collections.deque(maxlen=history_len) delayhist2d_buff = collections.deque(maxlen=history_len) minitof_volts_buff = collections.deque(maxlen=history_len) hitrate_roi_buff = collections.deque(maxlen=history_len_long) # Histograms hithist = Histogram((100, 0., 1023.)) hitjitter = Histogram((100, 0., 1023.)) hitseeded = Histogram((100, 0., 1023.)) delayhist2d = Histogram((100, 0., 1023.), (20, -100., 100.)) # ion yield array for SXRSS scan ion_yield = np.zeros(102) ##choose appropriate range # For perspective transformation and warp pts1 = np.float32([[96, 248], [935, 193], [96, 762], [935, 785]]) xLength = 839 yLength = 591 pts2 = np.float32([[0, 0], [xLength, 0], [0, yLength], [xLength, yLength]]) M = cv2.getPerspectiveTransform(pts1, pts2) print "DONE"
#energy1_mean = np.nanmean(energies_pos[0]) #energy2_mean = np.nanmean(energies_pos[1]) e1_int_bins=5 e1_int_min=10000 e1_int_max=90000 e2_int_bins=5 e2_int_min=500 e2_int_max=80000 #energy_threshold = 50000 #bin: (GD),(energy_1),(energy_2),(e1_int),(e2_int),(SHS_pix) His2d = Histogram((GD_bins,GD_min,GD_max),(energy1_bins,energy1_min,energy1_max),\ (energy2_bins,energy2_min,energy2_max),(e1_int_bins,e1_int_min,e1_int_max),\ (e2_int_bins,e2_int_min,e2_int_max),(SHESpix_bins,SHES_pix_min,SHES_pix_max)) His2d_w = Histogram((GD_bins,GD_min,GD_max),(energy1_bins,energy1_min,energy1_max),\ (energy2_bins,energy2_min,energy2_max),(e1_int_bins,e1_int_min,e1_int_max),\ (e2_int_bins,e2_int_min,e2_int_max),(SHESpix_bins,SHES_pix_min,SHES_pix_max)) commhist=np.zeros((GD_bins,energy1_bins,energy2_bins,e1_int_bins,e2_int_bins,SHESpix_bins)) commhist_w=np.zeros((GD_bins,energy1_bins,energy2_bins,e1_int_bins,e2_int_bins,SHESpix_bins)) commtest=np.zeros(1) for nfi,fi in enumerate(files): if nfi%size!=rank: continue data=np.load(os.path.join(folder,fi)) evtnr=len(data['GasDetector'][:,0])
# Extract shape of arrays which SHES processor will return
_, j_len, i_len = processor.pers_trans_params # for specified x_len_param/y_len_param in SHESPreProcessor,
# PerspectiveTransform() returns array of shape (y_len_param,x_len_param)
# Other initialisation
image_buff = np.zeros(
(refresh_rate, i_len, j_len)) # this gets reset to 0 after each reduction
x_proj_buff = np.zeros(
(refresh_rate, j_len)) # this gets reset to 0 after each reduction
counts_buff = np.zeros(
(refresh_rate, 2)) # this gets reset to 0 after each reduction
counts_buff_roi = np.zeros(
(refresh_rate, 2)) # this gets reset to 0 after each reduction
hist_L3PhotEnergy = Histogram(
(numbins_L3PhotEnergy, minhistlim_L3PhotEnergy, maxhistlim_L3PhotEnergy))
hist_FeeGasEnergy = Histogram(
(numbins_FeeGasEnergy, minhistlim_FeeGasEnergy, maxhistlim_FeeGasEnergy))
hist_FeeGasEnergy_CountsROI = Histogram((numbins_FEE_CountsROI_FEE,minhistlim_FEE_CountsROI_FEE,\
maxhistlim_FEE_CountsROI_FEE),(numbins_FEE_CountsROI_CountsROI,\
minhistlim_FEE_CountsROI_CountsROI,maxhistlim_FEE_CountsROI_CountsROI))
# Initialise variables which are only used by root core
image_sum = np.zeros(
(i_len, j_len)) # accumulated sum of OPAL image over history_len shots
x_proj_sum = np.zeros(
j_len
) # accumulated sum of projected electron spectrum over history_len shots
good_shot_count_all = 0
image_sum_slice = np.zeros(
from psana import *
from skbeam.core.accumulators.histogram import Histogram
import numpy as np
# 1D histogram: 11 bins going from -5.5 to 5.5
hist = Histogram((11, -5.5, 5.5))
hist.fill(4)
hist.fill(0, weights=2)
x = np.array((-4, -4, -4))
weights = np.array((1, 1, 0.5))
hist.fill(x, weights=weights)
import matplotlib.pyplot as plt
plt.plot(hist.centers[0], hist.values)
plt.title('1D Histogram Results')
plt.show()
# 2D histogram: 3 bins in x going from -0.5 to 2.5
# and 2 bins going from -0.5 to 1.5
hist2d = Histogram((3, -0.5, 2.5), (2, -0.5, 1.5))
hist2d.fill(1, 1)
x = np.array((0, 1, 2))
y = np.array((0, 1, 2))
weights = np.array((5.5, 2.5, 9999.))
hist2d.fill(x, y, weights=weights)
print '2D histogram values:\n', hist2d.values
import timeit
import time
import numpy as np
from skbeam.core.accumulators.histogram import Histogram
h = Histogram((10, 0, 10.1), (7, 0, 7.1));
x = np.random.random(1000000)*40
y = np.random.random(1000000)*10
w = np.ones_like(x)
xi = x.astype(int)
xi = xi.astype(float)
wi = np.ones_like(xi)
gg = globals()
def timethis(stmt):
return np.mean(timeit.repeat(stmt, number=10, repeat=5, globals=gg))
def histfromzero(h, fncname, x, w):
h.data[:] = 0
getattr(h, fncname)(x, w)
return h.data.copy()
print("Timing h.fill",
timethis('h.fill(x, y, weights=w)'))
h._always_use_fillnd = True
print("Timing h.fill with _always_use_fillnd",
timethis('h.fill(x, y, weights=w)'))
import itertools
x = [1000, 0, 10.01]
y = [1000, 0, 9.01]
xf = np.random.random(1000000) * 10 * 4
yf = np.random.random(1000000) * 9 * 15
xi = xf.astype(int)
yi = yf.astype(int)
wf = np.linspace(1, 10, len(xf))
wi = wf.copy()
times = []
print("Testing 2D histogram timings")
for xvals, yvals, weights in itertools.product([xf, xi], [yf, yi],
[wf, wi]):
t0 = time()
h = Histogram(x, y)
h.fill(xvals, yvals, weights=weights)
skbeam_time = time() - t0
edges = h.edges
t0 = time()
ynp = np.histogram2d(xvals, yvals, bins=edges, weights=weights)[0]
numpy_time = time() - t0
times.append(numpy_time / skbeam_time)
assert_almost_equal(np.sum(h.values), np.sum(ynp))
print('skbeam is %s times faster than numpy, on average' %
np.average(times))
# test_1d_histogram()
# test_2d_histogram()
# TODO do a better job sampling the variable space
# Other initialisation
image_sum_buff = deque(
maxlen=1 +
history_len / plot_every) # These keep the most recent one NOT to
x_proj_sum_buff = deque(
maxlen=1 +
history_len / plot_every) # be plotted so that it can be taken away
# from the rolling sum
image_buff = np.zeros((plot_every, i_len, j_len)) # this gets reset to 0
x_proj_buff = np.zeros((plot_every, j_len)) # this gets reset to 0
counts_buff = deque(maxlen=history_len_counts) # this doesn't get reset to 0
counts_buff_regint = deque(maxlen=history_len_counts
) # this doesn't get reset to 0. regint = region of
# interest, specified above
hist_L3PhotEnergy = Histogram(
(numbins_L3PhotEnergy, minhistlim_L3PhotEnergy, maxhistlim_L3PhotEnergy))
hist_FeeGasEnergy = Histogram(
(numbins_FeeGasEnergy, minhistlim_FeeGasEnergy, maxhistlim_FeeGasEnergy))
# hist_FeeGasEnergy_Counts = Histogram((numbins_FeeGasEnergy,minhistlim_FeeGasEnergy,maxhistlim_FeeGasEnergy, more, more, more))
image_sum = np.zeros((i_len, j_len))
x_proj_sum = np.zeros(j_len)
arcing_freeze = False
rolling_count = 0
arc_time_ref = 0.0 # will be set at certain number of seconds if required
# Now being looping over events
for nevt, evt in enumerate(ds.events()):
fee_gas_energy = feeGas.ShotEnergy(evt)
cent_pe = l3Proc.CentPE(evt)
vals_std = np.asarray(vals).std()
vals_mean = np.asarray(vals).mean()
threshold = vals_mean - 2 * vals_std
l3_mean = int(np.asarray(vals_l3).mean())
print "threshold: %f, mean: %f, std: %f, l3: %d" % (
threshold, vals_mean, vals_std, EBEAM_beam_energy)
numbins_L3Energy = 100
minhistlim_L3Energy = l3_mean - 50
maxhistlim_L3Energy = l3_mean + 50
numbins_L3EnergyWeighted = 100
minhistlim_L3EnergyWeighted = minhistlim_L3Energy
maxhistlim_L3EnergyWeighted = maxhistlim_L3Energy
hist_L3Energy = Histogram(
(numbins_L3Energy, minhistlim_L3Energy, maxhistlim_L3Energy))
hist_L3EnergyWeighted = Histogram(
(numbins_L3EnergyWeighted, minhistlim_L3EnergyWeighted,
maxhistlim_L3EnergyWeighted))
hist_L3EnergyWeighted_unnorm = Histogram(
(numbins_L3EnergyWeighted, minhistlim_L3EnergyWeighted,
maxhistlim_L3EnergyWeighted))
ds = DataSource(
"exp=AMO/amolr2516:run=%d:smd:dir=/reg/d/psdm/amo/amolr2516/xtc:live" %
run)
for nevt, evt in enumerate(ds.events()):
wf = MINITOF_det.waveform(evt)
#wf_time = MINITOF_det.wftime(evt)
EBEAM = EBEAM_det.get(evt)
import numpy as np
import os
from skbeam.core.accumulators.histogram import Histogram
run = 66
folder = "/reg/d/psdm/AMO/amolr2516/results/npzfiles/run00%d" % run
files = os.listdir(folder)
for nfi, fi in enumerate(files):
f = np.load(os.path.join(folder, fi))
if nfi == 0:
print "start"
gas_mean = np.nanmean(f['GasDetector'][:, 0])
gas_std = np.nanstd(f['GasDetector'][:, 0])
gas_max = np.nanmax(f['GasDetector'][:, 0])
His2d = Histogram((100 / 10, 0, gas_max), (714 / 10, 0, 713))
His2d_weights = Histogram((100 / 10, 0, gas_max), (714 / 10, 0, 713))
His2d_weights_not_norm = Histogram((100 / 10, 0, gas_max),
(714 / 10, 0, 713))
for evt in range(len(f['GasDetector'][:, 0])):
His2d.fill(list(f['GasDetector'][evt, 0] * np.ones(714)), range(714))
values = list(np.nan_to_num(np.asarray(f['SHESwf'][evt, :])))
His2d_weights.fill(list(f['GasDetector'][evt, 0] * np.ones(714)),
range(714),
weights=values / f['GasDetector'][evt, 0])
His2d_weights_not_norm.fill(list(f['GasDetector'][evt, 0] *
np.ones(714)),
range(714),
weights=values)
print "%.2f done" % (float(nfi) / len(files))
if __name__ == "__main__":
import timeit
import numpy as np
from skbeam.core.accumulators.histogram import Histogram
h = Histogram((10, 0, 10.1), (7, 0, 7.1))
x = np.random.random(1000000) * 40
y = np.random.random(1000000) * 10
w = np.ones_like(x)
xi = x.astype(int)
xi = xi.astype(float)
wi = np.ones_like(xi)
gg = globals()
def timethis(stmt):
return np.mean(timeit.repeat(stmt, number=10, repeat=5, globals=gg))
def histfromzero(h, fncname, x, w):
h.data[:] = 0
getattr(h, fncname)(x, w)
return h.data.copy()
print("Timing h.fill", timethis('h.fill(x, y, weights=w)'))
h._always_use_fillnd = True
print("Timing h.fill with _always_use_fillnd",
timethis('h.fill(x, y, weights=w)'))
