def test_simple_pass():
# This test exposes the half-open vs full-open histogram code difference
# between np.histogram and skbeam's histogram.
h = Histogram((5, 0, 3.1))
a = np.arange(1, 6)
b = np.asarray([1, 1, 2, 3, 2])
h.fill(a, weights=b)
np_res = np.histogram(a, h.edges[0], weights=b)[0]
assert_array_equal(h.values, np_res)
def _1d_histogram_tester(binlowhighs, x, weights=1):
h = Histogram(binlowhighs)
h.fill(x, weights=weights)
if np.isscalar(weights):
ynp = np.histogram(x, h.edges[0])[0]
else:
ynp = np.histogram(x, h.edges[0], weights=weights)[0]
assert_array_almost_equal(ynp, h.values)
h.reset()
h._always_use_fillnd = True
h.fill(x, weights=weights)
assert_array_almost_equal(ynp, h.values)
def _1d_histogram_tester(binlowhighs, x, weights=1):
h = Histogram(binlowhighs)
h.fill(x, weights=weights)
if np.isscalar(weights):
ynp = np.histogram(x, h.edges[0])[0]
else:
ynp = np.histogram(x, h.edges[0], weights=weights)[0]
assert_array_almost_equal(ynp, h.values)
h.reset()
h._always_use_fillnd = True
h.fill(x, weights=weights)
assert_array_almost_equal(ynp, h.values)
def _2d_histogram_tester(binlowhighs, x, y, weights=1):
h = Histogram(*binlowhighs)
h.fill(x, y, weights=weights)
if np.isscalar(weights):
if np.isscalar(x):
assert np.isscalar(y), 'If x is a scalar, y must be also'
ynp = np.histogram2d([x], [y], bins=h.edges)[0]
else:
ynp = np.histogram2d(x, y, bins=h.edges)[0]
else:
ynp = np.histogram2d(x, y, bins=h.edges, weights=weights)[0]
assert_array_almost_equal(ynp, h.values)
h.reset()
h._always_use_fillnd = True
h.fill(x, y, weights=weights)
assert_array_almost_equal(ynp, h.values)
def histogram_plot(file_usage, log_yaxis=False):
min_size = 0
max_size = 1e9 # 10 GB here
Nbins = 10000
for plan in file_usage:
fig, ax = plt.subplots()
h = Histogram((Nbins, min_size, max_size))
h.fill(file_usage[plan])
ax.set_xlabel('FILE USAGE')
ax.set_ylabel('FREQUENCY')
ax.set_title('CHX | Plan:Detector')
if log_yaxis:
ax.set_yscale('log')
plt.plot(h.centers[0] * 1e-6, h.values, label=plan)
plt.legend()
plt.savefig('hist_{}'.format(plan))
def histogram_plot(dfs, log_yscale=False):
min_size = 0
max_size = 1e9 # 10 GB here
Nbins = 10000
for df in dfs:
fig, ax = plt.subplots()
h = Histogram((Nbins, min_size, max_size))
col_name = df.columns.values[0]
col = np.array(df[col_name])
h.fill(col)
ax.set_xlabel('FILE USAGE')
ax.set_ylabel('FREQUENCY')
ax.set_title('CHX | Plan:Detector')
if log_yscale:
ax.set_yscale('log')
plt.plot(h.centers[0] * 1e-6, h.values, label=col_name)
plt.legend(loc=1)
print(col_name.replace(':', '_').replace('(fileusage)', ''))
plt.savefig('chx_{}'.format(
col_name.replace(':', '_').replace('(fileusage)', '')))
def plot_histogram(dfs):
min_size = 0
max_size = 5e8 # 10 GB here
Nbins = 10000
labels = [num for num in range(11000) if num % 2000 == 0]
minor_labels = [num for num in range(11000) if num % 1000 == 0]
minor_2_labels = [num for num in range(11000) if num % 500 == 0]
#y_minor_labels = [num for num in range()]
x_labels = [i for i in range(10)]
plt.ion()
fig, axs = plt.subplots(2, 2)
plans = ['count', 'rel_scan']
detectors = ['eiger1m_single_image', 'eiger4m_single_image']
for i, plan in enumerate(plans):
for j, detector in enumerate(detectors):
ax = axs[i, j]
df = dfs.get(plan, dict()).get(detector, None)
if df is not None:
h = Histogram((Nbins, min_size, max_size))
col_name = df.columns[0]
col = np.array(df[col_name])
h.fill(col)
ax.semilogy(h.centers[0][1:] * 1e-9,
h.values[1:],
color='darkgreen')
ax.set_ylim(ymin=1.1)
ax.set_xlim(xmin=1e-7)
if i == 1 and j == 1:
ax.set_ylim(ymax=10)
if (i == 0 and j == 0 or i == 1 and j == 0):
ax.set_ylabel(
col_name.split(':')[0] + ' - # of occurences')
if (i == 0 and j == 0 or i == 0 and j == 1):
ax.set_title(
col_name.replace('(fileusage)', '').split(':')[1])
plt.suptitle('CHX Plans + Detectors')
plt.title('Histogram Semi-Log Plot', position=(-.14, -0.30))
#################################### eBeam & FEE gas det#######################
L3Energy = ebeam.ebeamL3Energy()
bla = L3Energy / 13720.
photonEnergy = 8330. * bla**2
fee_1 = (feegas.f_11_ENRC() + feegas.f_12_ENRC()) / 2.
fee_2 = (feegas.f_21_ENRC() + feegas.f_22_ENRC()) / 2.
#fee_6 = (feegas.f_64_ENRC()+feegas.f_63_ENRC())/2. # New R&D gas dets likely not calibrated.
###############################################################################
################################ Populate histograms ##########################
if fee_2 > cutoff and iTOF_yield_CO < -2.:
for hit in c:
count = [hit[1] + xLength * n]
spectraCombined.fill(count)
delayhist2d.fill([pulse_separation], count)
feegashist2d.fill([fee_2], count)
photonhist2d.fill([photonEnergy], count)
TotalPowiTOF.fill([iTOF_yield_Ne + iTOF_yield_CO], count)
RelYieldsiTOF.fill([iTOF_yield_Ne], [iTOF_yield_CO])
if iTOF_yield_Ne > iTOF_yield_CO:
RatiosiTOF.fill([iTOF_yield_Ne / iTOF_yield_CO], count)
if iTOF_yield_Ne < iTOF_yield_CO:
RatiosiTOF.fill([-iTOF_yield_CO / iTOF_yield_Ne], count)
if iyieldXTCAV == True:
TotalPowXTCAV.fill([yieldXTCAV1 + yieldXTCAV2], count)
RelYieldsXTCAV.fill([yieldXTCAV1], [yieldXTCAV2])
Correlation_iTOF_XTCAV_pow.fill(
[iTOF_yield_Ne + iTOF_yield_CO],
numbins_L3EnergyWeighted=100
minhistlim_L3EnergyWeighted=3300
maxhistlim_L3EnergyWeighted=3400
hist_L3_elec1mom = Histogram((numbins_L3_elec1mom_L3,minhistlim_L3_elec1mom_L3,\
mmaxhistlim_L3_elec1mom_L3),(numbins_L3_elec1mom_elec1mom,\
minhistlim_L3_elec1mom_elec1mom,maxhistlim_L3_elec1mom_elec1mom))
hist_L3Energy=Histogram((numbins_L3Energy,minhistlim_L3Energy,maxhistlim_L3Energy))
hist_L3EnergyWeighted=Histogram((numbins_L3EnergyWeighted,minhistlim_L3EnergyWeighted,maxhistlim_L3EnergyWeighted))
for nevt, evt in enumerate(ds.events()):
print nevt
energyL3=procL3Energy.L3Energy(evt)
moments, numhits=procSHES.CalibProcess(evt, inPix=True)
mom1=moments[0]
hist_L3Energy.fill(energyL3)
hist_L3EnergyWeighted.fill([energyL3], weights=[numhits])
if nevt>100:
break
avShotsPerL3Energy=hist_L3EnergyWeighted/np.float(hist_L3Energy.values)
L3_weighted_plot = XYPlot(nevt, 'L3_weighted', hist_L3EnergyWeighted.centers[0], hist_L3EnergyWeighted.values)
publish.send('L3 weighted', L3_weighted_plot)
#mu,sig = moments(opal_thresh_xproj[bot:top])
#shift = 250 - int(mu) - bot
#index = int(mu/10)
else:
shift = 0
index = 0
#print shift
# Hit Histogram
hithist.reset()
hitjitter.reset()
hitseeded.reset()
delayhist2d.reset()
for hit in c:
hithist.fill(float(hit[1] + shift))
hitjitter.fill(float(hit[1]))
###############################################################################
############################ MINI TOF ANALYSIS ################################
# find yield of waveform over certain interval
wf_yield = np.sum(minitof_volts[1:401] -
minitof_volts[5300:5700]) #choose proper window later
# get maximum value of x-ray beam
#max_hithist = np.amax(hithist.data,axis=0) # might have to change this to x2.data?
max_hithist = np.amax(opal_thresh_xproj,
axis=0) # might have to change this to x2.data?
#print max_hithist
if nfi%size!=rank: continue data=np.load(os.path.join(folder,fi)) evtnr=len(data['GasDetector'][:,0]) for evt in range(evtnr): #check for threshold of intensity in both pulses UXSpc=np.asarray(data['UXSpc'][evt,:]) Pressure = data['Pressure'][evt]*1e5 #if UXSpc[2]<energy_threshold or UXSpc[5]<energy_threshold or data['GasDetector'][evt,0]<GD_min: if data['GasDetector'][evt,0:2].mean()<GD_min: continue if UXSpc[0]>UXSpc[3]: UXSpc = np.roll(UXSpc,3) values= list(np.nan_to_num(np.asarray(data['SHESwf'][evt,:]/Pressure))) His2d.fill(list(data['GasDetector'][evt,0:2].mean()*np.ones(714)),list(UXSpc[0]*np.ones(714)),\ list(UXSpc[3]*np.ones(714)),list(UXSpc[2]*np.ones(714)),\ list(UXSpc[5]*np.ones(714)),range(714)) His2d_w.fill(list(data['GasDetector'][evt,0:2].mean()*np.ones(714)),list(UXSpc[0]*np.ones(714)),\ list(UXSpc[3]*np.ones(714)),list(UXSpc[2]*np.ones(714)), \ list(UXSpc[5]*np.ones(714)),range(714),weights=values) #print "%f, %f" %(UXSpc[2],UXSpc[5]) print "rank %d done file %d" %(rank,nfi) print "rank %d done" if rank==0: time.sleep(3) if 'His2d' in locals(): comm.Reduce(His2d.values,commhist) comm.Reduce(His2d_w.values,commhist_w) comm.Reduce(np.ones(1),commtest)
if nevt % size != rank: continue # each core only processes its own runs
gather_electrons = True
fee_gas_energy = feeGas.ShotEnergy(evt)
cent_pe = l3Proc.CentPE(evt)
# Check data exists
if fee_gas_energy is None:
print 'FEE gas energy is None, continuing to next event'
continue
if cent_pe is None:
print 'L3 e-beam energy is None, continuing to next event'
continue
# If data exists, fill histograms
hist_L3PhotEnergy.fill(cent_pe)
hist_FeeGasEnergy.fill(fee_gas_energy)
opal_image, x_proj, arced = processor.OnlineProcess(evt)
if opal_image is None:
print 'SHES OPAL image is None, continuing to next event'
continue
if arced:
print '***WARNING - ARC DETECTED!!!***'
opal_image_copy = np.copy(opal_image)
cv2.putText(opal_image_copy,'ARCING DETECTED!!!', \
(50,int(i_len/2)), cv2.FONT_HERSHEY_SIMPLEX, 2, (255,0,0), 10)
# Just send the single shot so you can see
plotshot = Image(nevt, 'Single shot', opal_image_copy)
from psana import *
from skbeam.core.accumulators.histogram import Histogram
import numpy as np
# 1D histogram: 11 bins going from -5.5 to 5.5
hist = Histogram((11, -5.5, 5.5))
hist.fill(4)
hist.fill(0, weights=2)
x = np.array((-4, -4, -4))
weights = np.array((1, 1, 0.5))
hist.fill(x, weights=weights)
import matplotlib.pyplot as plt
plt.plot(hist.centers[0], hist.values)
plt.title('1D Histogram Results')
plt.show()
# 2D histogram: 3 bins in x going from -0.5 to 2.5
# and 2 bins going from -0.5 to 1.5
hist2d = Histogram((3, -0.5, 2.5), (2, -0.5, 1.5))
hist2d.fill(1, 1)
x = np.array((0, 1, 2))
y = np.array((0, 1, 2))
weights = np.array((5.5, 2.5, 9999.))
hist2d.fill(x, y, weights=weights)
print '2D histogram values:\n', hist2d.values
folder = "/reg/d/psdm/AMO/amolr2516/results/npzfiles/run00%d" % run
files = os.listdir(folder)
for nfi, fi in enumerate(files):
f = np.load(os.path.join(folder, fi))
if nfi == 0:
print "start"
gas_mean = np.nanmean(f['GasDetector'][:, 0])
gas_std = np.nanstd(f['GasDetector'][:, 0])
gas_max = np.nanmax(f['GasDetector'][:, 0])
His2d = Histogram((100 / 10, 0, gas_max), (714 / 10, 0, 713))
His2d_weights = Histogram((100 / 10, 0, gas_max), (714 / 10, 0, 713))
His2d_weights_not_norm = Histogram((100 / 10, 0, gas_max),
(714 / 10, 0, 713))
for evt in range(len(f['GasDetector'][:, 0])):
His2d.fill(list(f['GasDetector'][evt, 0] * np.ones(714)), range(714))
values = list(np.nan_to_num(np.asarray(f['SHESwf'][evt, :])))
His2d_weights.fill(list(f['GasDetector'][evt, 0] * np.ones(714)),
range(714),
weights=values / f['GasDetector'][evt, 0])
His2d_weights_not_norm.fill(list(f['GasDetector'][evt, 0] *
np.ones(714)),
range(714),
weights=values)
print "%.2f done" % (float(nfi) / len(files))
filename = "/reg/neh/home5/sjarosch/lr25_code/npz/ana_intensity_scan_run_%d" % run
np.savez(filename,
His2d_centers=His2d.centers,
His2d_val=His2d.values,
His2d_weights_val=His2d_weights.values,
# Now being looping over events
for nevt, evt in enumerate(ds.events()):
fee_gas_energy = feeGas.ShotEnergy(evt)
cent_pe = l3Proc.CentPE(evt)
# Check data exists
if fee_gas_energy is None:
print 'No FEE gas energy, continuing to next event'
continue
if cent_pe is None:
print 'No L3 e-beam energy, continuing to next event'
continue
# If data exists, fill histograms
hist_L3PhotEnergy.fill(cent_pe)
hist_FeeGasEnergy.fill(fee_gas_energy)
#Check data falls within thresholds
if fee_gas_energy < fee_gas_threshold:
print 'FEE gas energy = ' + str(
fee_gas_energy) + ' mJ -> continuing to next event'
continue
if not (cent_pe < max_cent_pe and cent_pe > min_cent_pe):
print '\'Central\' photon energy = '+str(np.round(cent_pe,2))+\
'-> outside specified range, skipping event'
opal_image, x_proj, arced = processor.OnlineProcess(evt)
if opal_image is None:
import itertools
x = [1000, 0, 10.01]
y = [1000, 0, 9.01]
xf = np.random.random(1000000)*10*4
yf = np.random.random(1000000)*9*15
xi = xf.astype(int)
yi = yf.astype(int)
wf = np.linspace(1, 10, len(xf))
wi = wf.copy()
times = []
print("Testing 2D histogram timings")
for xvals, yvals, weights in itertools.product([xf, xi], [yf, yi],
[wf, wi]):
t0 = time()
h = Histogram(x, y)
h.fill(xvals, yvals, weights=weights)
skbeam_time = time() - t0
edges = h.edges
t0 = time()
ynp = np.histogram2d(xvals, yvals, bins=edges, weights=weights)[0]
numpy_time = time() - t0
times.append(numpy_time / skbeam_time)
assert_almost_equal(np.sum(h.values), np.sum(ynp))
print('skbeam is %s times faster than numpy, on average' %
np.average(times))
# test_1d_histogram()
# test_2d_histogram()
# TODO do a better job sampling the variable space
"exp=AMO/amolr2516:run=%d:smd:dir=/reg/d/psdm/amo/amolr2516/xtc:live" %
run)
for nevt, evt in enumerate(ds.events()):
wf = MINITOF_det.waveform(evt)
#wf_time = MINITOF_det.wftime(evt)
EBEAM = EBEAM_det.get(evt)
if wf is None or EBEAM is None:
continue
EBEAM_beam_energy = EBEAM.ebeamL3Energy() # beam energy in MeV
FEE_shot_energy = procFEE.ShotEnergy(evt)
#print FEE_shot_energy
if FEE_shot_energy is None or FEE_shot_energy < threshold:
continue
hist_L3Energy.fill(EBEAM_beam_energy)
hist_L3EnergyWeighted.fill(
[EBEAM_beam_energy],
weights=[np.abs(wf[1][:10000].sum()) / FEE_shot_energy])
hist_L3EnergyWeighted_unnorm.fill([EBEAM_beam_energy],
weights=[np.abs(wf[1][:10000].sum())])
if nevt % 500 == 0:
print nevt
multi = MultiPlot(nevt, 'MULTI')
vShotsPerL3Energy = np.nan_to_num(hist_L3EnergyWeighted.values /
hist_L3Energy.values)
L3_weighted_plot = XYPlot(nevt, 'L3', hist_L3EnergyWeighted.centers[0],
vShotsPerL3Energy)
#publish.send('L3', L3_weighted_plot)
multi.add(L3_weighted_plot)
vShotsPerL3Energy_un = np.nan_to_num(
x = [1000, 0, 10.01]
y = [1000, 0, 9.01]
xf = np.random.random(1000000) * 10 * 4
yf = np.random.random(1000000) * 9 * 15
xi = xf.astype(int)
yi = yf.astype(int)
wf = np.linspace(1, 10, len(xf))
wi = wf.copy()
times = []
print("Testing 2D histogram timings")
for xvals, yvals, weights in itertools.product([xf, xi], [yf, yi],
[wf, wi]):
t0 = time()
h = Histogram(x, y)
h.fill(xvals, yvals, weights=weights)
skbeam_time = time() - t0
edges = h.edges
t0 = time()
ynp = np.histogram2d(xvals, yvals, bins=edges, weights=weights)[0]
numpy_time = time() - t0
times.append(numpy_time / skbeam_time)
assert_almost_equal(np.sum(h.values), np.sum(ynp))
print('skbeam is %s times faster than numpy, on average' %
np.average(times))
# test_1d_histogram()
# test_2d_histogram()
# TODO do a better job sampling the variable space
if npeaks == 0: continue
print nevt, len(peaks)
if npeaks > 10:
print 'too many peaks'
continue
# save per-event data
peakrow.fill(-1)
peakcol.fill(-1)
peakamp.fill(0.0)
for pnum, peak in enumerate(peaks):
peakrow[pnum] = peak[1]
peakcol[pnum] = peak[2]
peakamp[pnum] = peak[5]
print peakamp[pnum]
hist_row.fill(float(peakrow[pnum]))
hist_col.fill(float(peakcol[pnum]))
hist_amp.fill(float(peakamp[pnum]))
print("peak value = " + str(peak[1]))
smldata.event(npeaks=len(peaks),
peakrow=peakrow,
peakcol=peakcol,
peakamp=peakamp)
if nevt > 35: break
#import matplotlib.pyplot as plt
#plt.plot(hist_row.values)
#plt.show()
hist_row_tot = smldata.sum(hist_row.values)
