def test_simple_pass(): # This test exposes the half-open vs full-open histogram code difference # between np.histogram and skbeam's histogram. h = Histogram((5, 0, 3.1)) a = np.arange(1, 6) b = np.asarray([1, 1, 2, 3, 2]) h.fill(a, weights=b) np_res = np.histogram(a, h.edges[0], weights=b)[0] assert_array_equal(h.values, np_res)
def _1d_histogram_tester(binlowhighs, x, weights=1): h = Histogram(binlowhighs) h.fill(x, weights=weights) if np.isscalar(weights): ynp = np.histogram(x, h.edges[0])[0] else: ynp = np.histogram(x, h.edges[0], weights=weights)[0] assert_array_almost_equal(ynp, h.values) h.reset() h._always_use_fillnd = True h.fill(x, weights=weights) assert_array_almost_equal(ynp, h.values)
def _2d_histogram_tester(binlowhighs, x, y, weights=1): h = Histogram(*binlowhighs) h.fill(x, y, weights=weights) if np.isscalar(weights): if np.isscalar(x): assert np.isscalar(y), 'If x is a scalar, y must be also' ynp = np.histogram2d([x], [y], bins=h.edges)[0] else: ynp = np.histogram2d(x, y, bins=h.edges)[0] else: ynp = np.histogram2d(x, y, bins=h.edges, weights=weights)[0] assert_array_almost_equal(ynp, h.values) h.reset() h._always_use_fillnd = True h.fill(x, y, weights=weights) assert_array_almost_equal(ynp, h.values)
def histogram_plot(file_usage, log_yaxis=False): min_size = 0 max_size = 1e9 # 10 GB here Nbins = 10000 for plan in file_usage: fig, ax = plt.subplots() h = Histogram((Nbins, min_size, max_size)) h.fill(file_usage[plan]) ax.set_xlabel('FILE USAGE') ax.set_ylabel('FREQUENCY') ax.set_title('CHX | Plan:Detector') if log_yaxis: ax.set_yscale('log') plt.plot(h.centers[0] * 1e-6, h.values, label=plan) plt.legend() plt.savefig('hist_{}'.format(plan))
def histogram_plot(dfs, log_yscale=False): min_size = 0 max_size = 1e9 # 10 GB here Nbins = 10000 for df in dfs: fig, ax = plt.subplots() h = Histogram((Nbins, min_size, max_size)) col_name = df.columns.values[0] col = np.array(df[col_name]) h.fill(col) ax.set_xlabel('FILE USAGE') ax.set_ylabel('FREQUENCY') ax.set_title('CHX | Plan:Detector') if log_yscale: ax.set_yscale('log') plt.plot(h.centers[0] * 1e-6, h.values, label=col_name) plt.legend(loc=1) print(col_name.replace(':', '_').replace('(fileusage)', '')) plt.savefig('chx_{}'.format( col_name.replace(':', '_').replace('(fileusage)', '')))
def plot_histogram(dfs): min_size = 0 max_size = 5e8 # 10 GB here Nbins = 10000 labels = [num for num in range(11000) if num % 2000 == 0] minor_labels = [num for num in range(11000) if num % 1000 == 0] minor_2_labels = [num for num in range(11000) if num % 500 == 0] #y_minor_labels = [num for num in range()] x_labels = [i for i in range(10)] plt.ion() fig, axs = plt.subplots(2, 2) plans = ['count', 'rel_scan'] detectors = ['eiger1m_single_image', 'eiger4m_single_image'] for i, plan in enumerate(plans): for j, detector in enumerate(detectors): ax = axs[i, j] df = dfs.get(plan, dict()).get(detector, None) if df is not None: h = Histogram((Nbins, min_size, max_size)) col_name = df.columns[0] col = np.array(df[col_name]) h.fill(col) ax.semilogy(h.centers[0][1:] * 1e-9, h.values[1:], color='darkgreen') ax.set_ylim(ymin=1.1) ax.set_xlim(xmin=1e-7) if i == 1 and j == 1: ax.set_ylim(ymax=10) if (i == 0 and j == 0 or i == 1 and j == 0): ax.set_ylabel( col_name.split(':')[0] + ' - # of occurences') if (i == 0 and j == 0 or i == 0 and j == 1): ax.set_title( col_name.replace('(fileusage)', '').split(':')[1]) plt.suptitle('CHX Plans + Detectors') plt.title('Histogram Semi-Log Plot', position=(-.14, -0.30))
opal_hit_avg_buff = collections.deque(maxlen=history_len_long) opal_circ_buff = collections.deque(maxlen=history_len) xproj_int_buff = collections.deque(maxlen=history_len) xproj_int_avg_buff = collections.deque(maxlen=history_len) moments_buff = collections.deque(maxlen=history_len_long) #moments_avg_buff = collections.deque(maxlen=history_len_long) #xhistogram_buff = collections.deque(maxlen=history_len) hitxprojhist_buff = collections.deque(maxlen=history_len) hitxprojjitter_buff = collections.deque(maxlen=history_len) hitxprojseeded_buff = collections.deque(maxlen=history_len) delayhist2d_buff = collections.deque(maxlen=history_len) minitof_volts_buff = collections.deque(maxlen=history_len) hitrate_roi_buff = collections.deque(maxlen=history_len_long) # Histograms hithist = Histogram((100, 0., 1023.)) hitjitter = Histogram((100, 0., 1023.)) hitseeded = Histogram((100, 0., 1023.)) delayhist2d = Histogram((100, 0., 1023.), (20, -100., 100.)) # ion yield array for SXRSS scan ion_yield = np.zeros(102) ##choose appropriate range # For perspective transformation and warp pts1 = np.float32([[96, 248], [935, 193], [96, 762], [935, 785]]) xLength = 839 yLength = 591 pts2 = np.float32([[0, 0], [xLength, 0], [0, yLength], [xLength, yLength]]) M = cv2.getPerspectiveTransform(pts1, pts2) print "DONE"
#energy1_mean = np.nanmean(energies_pos[0]) #energy2_mean = np.nanmean(energies_pos[1]) e1_int_bins=5 e1_int_min=10000 e1_int_max=90000 e2_int_bins=5 e2_int_min=500 e2_int_max=80000 #energy_threshold = 50000 #bin: (GD),(energy_1),(energy_2),(e1_int),(e2_int),(SHS_pix) His2d = Histogram((GD_bins,GD_min,GD_max),(energy1_bins,energy1_min,energy1_max),\ (energy2_bins,energy2_min,energy2_max),(e1_int_bins,e1_int_min,e1_int_max),\ (e2_int_bins,e2_int_min,e2_int_max),(SHESpix_bins,SHES_pix_min,SHES_pix_max)) His2d_w = Histogram((GD_bins,GD_min,GD_max),(energy1_bins,energy1_min,energy1_max),\ (energy2_bins,energy2_min,energy2_max),(e1_int_bins,e1_int_min,e1_int_max),\ (e2_int_bins,e2_int_min,e2_int_max),(SHESpix_bins,SHES_pix_min,SHES_pix_max)) commhist=np.zeros((GD_bins,energy1_bins,energy2_bins,e1_int_bins,e2_int_bins,SHESpix_bins)) commhist_w=np.zeros((GD_bins,energy1_bins,energy2_bins,e1_int_bins,e2_int_bins,SHESpix_bins)) commtest=np.zeros(1) for nfi,fi in enumerate(files): if nfi%size!=rank: continue data=np.load(os.path.join(folder,fi)) evtnr=len(data['GasDetector'][:,0])
import itertools x = [1000, 0, 10.01] y = [1000, 0, 9.01] xf = np.random.random(1000000) * 10 * 4 yf = np.random.random(1000000) * 9 * 15 xi = xf.astype(int) yi = yf.astype(int) wf = np.linspace(1, 10, len(xf)) wi = wf.copy() times = [] print("Testing 2D histogram timings") for xvals, yvals, weights in itertools.product([xf, xi], [yf, yi], [wf, wi]): t0 = time() h = Histogram(x, y) h.fill(xvals, yvals, weights=weights) skbeam_time = time() - t0 edges = h.edges t0 = time() ynp = np.histogram2d(xvals, yvals, bins=edges, weights=weights)[0] numpy_time = time() - t0 times.append(numpy_time / skbeam_time) assert_almost_equal(np.sum(h.values), np.sum(ynp)) print('skbeam is %s times faster than numpy, on average' % np.average(times)) # test_1d_histogram() # test_2d_histogram() # TODO do a better job sampling the variable space
# Other initialisation image_sum_buff = deque( maxlen=1 + history_len / plot_every) # These keep the most recent one NOT to x_proj_sum_buff = deque( maxlen=1 + history_len / plot_every) # be plotted so that it can be taken away # from the rolling sum image_buff = np.zeros((plot_every, i_len, j_len)) # this gets reset to 0 x_proj_buff = np.zeros((plot_every, j_len)) # this gets reset to 0 counts_buff = deque(maxlen=history_len_counts) # this doesn't get reset to 0 counts_buff_regint = deque(maxlen=history_len_counts ) # this doesn't get reset to 0. regint = region of # interest, specified above hist_L3PhotEnergy = Histogram( (numbins_L3PhotEnergy, minhistlim_L3PhotEnergy, maxhistlim_L3PhotEnergy)) hist_FeeGasEnergy = Histogram( (numbins_FeeGasEnergy, minhistlim_FeeGasEnergy, maxhistlim_FeeGasEnergy)) # hist_FeeGasEnergy_Counts = Histogram((numbins_FeeGasEnergy,minhistlim_FeeGasEnergy,maxhistlim_FeeGasEnergy, more, more, more)) image_sum = np.zeros((i_len, j_len)) x_proj_sum = np.zeros(j_len) arcing_freeze = False rolling_count = 0 arc_time_ref = 0.0 # will be set at certain number of seconds if required # Now being looping over events for nevt, evt in enumerate(ds.events()): fee_gas_energy = feeGas.ShotEnergy(evt) cent_pe = l3Proc.CentPE(evt)
vals_std = np.asarray(vals).std() vals_mean = np.asarray(vals).mean() threshold = vals_mean - 2 * vals_std l3_mean = int(np.asarray(vals_l3).mean()) print "threshold: %f, mean: %f, std: %f, l3: %d" % ( threshold, vals_mean, vals_std, EBEAM_beam_energy) numbins_L3Energy = 100 minhistlim_L3Energy = l3_mean - 50 maxhistlim_L3Energy = l3_mean + 50 numbins_L3EnergyWeighted = 100 minhistlim_L3EnergyWeighted = minhistlim_L3Energy maxhistlim_L3EnergyWeighted = maxhistlim_L3Energy hist_L3Energy = Histogram( (numbins_L3Energy, minhistlim_L3Energy, maxhistlim_L3Energy)) hist_L3EnergyWeighted = Histogram( (numbins_L3EnergyWeighted, minhistlim_L3EnergyWeighted, maxhistlim_L3EnergyWeighted)) hist_L3EnergyWeighted_unnorm = Histogram( (numbins_L3EnergyWeighted, minhistlim_L3EnergyWeighted, maxhistlim_L3EnergyWeighted)) ds = DataSource( "exp=AMO/amolr2516:run=%d:smd:dir=/reg/d/psdm/amo/amolr2516/xtc:live" % run) for nevt, evt in enumerate(ds.events()): wf = MINITOF_det.waveform(evt) #wf_time = MINITOF_det.wftime(evt) EBEAM = EBEAM_det.get(evt)
if __name__ == "__main__": import timeit import numpy as np from skbeam.core.accumulators.histogram import Histogram h = Histogram((10, 0, 10.1), (7, 0, 7.1)) x = np.random.random(1000000) * 40 y = np.random.random(1000000) * 10 w = np.ones_like(x) xi = x.astype(int) xi = xi.astype(float) wi = np.ones_like(xi) gg = globals() def timethis(stmt): return np.mean(timeit.repeat(stmt, number=10, repeat=5, globals=gg)) def histfromzero(h, fncname, x, w): h.data[:] = 0 getattr(h, fncname)(x, w) return h.data.copy() print("Timing h.fill", timethis('h.fill(x, y, weights=w)')) h._always_use_fillnd = True print("Timing h.fill with _always_use_fillnd", timethis('h.fill(x, y, weights=w)'))
import numpy as np import os from skbeam.core.accumulators.histogram import Histogram run = 66 folder = "/reg/d/psdm/AMO/amolr2516/results/npzfiles/run00%d" % run files = os.listdir(folder) for nfi, fi in enumerate(files): f = np.load(os.path.join(folder, fi)) if nfi == 0: print "start" gas_mean = np.nanmean(f['GasDetector'][:, 0]) gas_std = np.nanstd(f['GasDetector'][:, 0]) gas_max = np.nanmax(f['GasDetector'][:, 0]) His2d = Histogram((100 / 10, 0, gas_max), (714 / 10, 0, 713)) His2d_weights = Histogram((100 / 10, 0, gas_max), (714 / 10, 0, 713)) His2d_weights_not_norm = Histogram((100 / 10, 0, gas_max), (714 / 10, 0, 713)) for evt in range(len(f['GasDetector'][:, 0])): His2d.fill(list(f['GasDetector'][evt, 0] * np.ones(714)), range(714)) values = list(np.nan_to_num(np.asarray(f['SHESwf'][evt, :]))) His2d_weights.fill(list(f['GasDetector'][evt, 0] * np.ones(714)), range(714), weights=values / f['GasDetector'][evt, 0]) His2d_weights_not_norm.fill(list(f['GasDetector'][evt, 0] * np.ones(714)), range(714), weights=values) print "%.2f done" % (float(nfi) / len(files))
from psana import * from skbeam.core.accumulators.histogram import Histogram import numpy as np # 1D histogram: 11 bins going from -5.5 to 5.5 hist = Histogram((11, -5.5, 5.5)) hist.fill(4) hist.fill(0, weights=2) x = np.array((-4, -4, -4)) weights = np.array((1, 1, 0.5)) hist.fill(x, weights=weights) import matplotlib.pyplot as plt plt.plot(hist.centers[0], hist.values) plt.title('1D Histogram Results') plt.show() # 2D histogram: 3 bins in x going from -0.5 to 2.5 # and 2 bins going from -0.5 to 1.5 hist2d = Histogram((3, -0.5, 2.5), (2, -0.5, 1.5)) hist2d.fill(1, 1) x = np.array((0, 1, 2)) y = np.array((0, 1, 2)) weights = np.array((5.5, 2.5, 9999.)) hist2d.fill(x, y, weights=weights) print '2D histogram values:\n', hist2d.values
# Extract shape of arrays which SHES processor will return _, j_len, i_len = processor.pers_trans_params # for specified x_len_param/y_len_param in SHESPreProcessor, # PerspectiveTransform() returns array of shape (y_len_param,x_len_param) # Other initialisation image_buff = np.zeros( (refresh_rate, i_len, j_len)) # this gets reset to 0 after each reduction x_proj_buff = np.zeros( (refresh_rate, j_len)) # this gets reset to 0 after each reduction counts_buff = np.zeros( (refresh_rate, 2)) # this gets reset to 0 after each reduction counts_buff_roi = np.zeros( (refresh_rate, 2)) # this gets reset to 0 after each reduction hist_L3PhotEnergy = Histogram( (numbins_L3PhotEnergy, minhistlim_L3PhotEnergy, maxhistlim_L3PhotEnergy)) hist_FeeGasEnergy = Histogram( (numbins_FeeGasEnergy, minhistlim_FeeGasEnergy, maxhistlim_FeeGasEnergy)) hist_FeeGasEnergy_CountsROI = Histogram((numbins_FEE_CountsROI_FEE,minhistlim_FEE_CountsROI_FEE,\ maxhistlim_FEE_CountsROI_FEE),(numbins_FEE_CountsROI_CountsROI,\ minhistlim_FEE_CountsROI_CountsROI,maxhistlim_FEE_CountsROI_CountsROI)) # Initialise variables which are only used by root core image_sum = np.zeros( (i_len, j_len)) # accumulated sum of OPAL image over history_len shots x_proj_sum = np.zeros( j_len ) # accumulated sum of projected electron spectrum over history_len shots good_shot_count_all = 0 image_sum_slice = np.zeros(
maxhistlim_L3_elec1mom_L3=3400 numbins_L3_elec1mom_elec1mom=100 minhistlim_L3_elec1mom_elec1mom=0 maxhistlim_L3_elec1mom_elec1mom=714 numbins_L3Energy=100 minhistlim_L3Energy=3300 maxhistlim_L3Energy=3400 numbins_L3EnergyWeighted=100 minhistlim_L3EnergyWeighted=3300 maxhistlim_L3EnergyWeighted=3400 hist_L3_elec1mom = Histogram((numbins_L3_elec1mom_L3,minhistlim_L3_elec1mom_L3,\ mmaxhistlim_L3_elec1mom_L3),(numbins_L3_elec1mom_elec1mom,\ minhistlim_L3_elec1mom_elec1mom,maxhistlim_L3_elec1mom_elec1mom)) hist_L3Energy=Histogram((numbins_L3Energy,minhistlim_L3Energy,maxhistlim_L3Energy)) hist_L3EnergyWeighted=Histogram((numbins_L3EnergyWeighted,minhistlim_L3EnergyWeighted,maxhistlim_L3EnergyWeighted)) for nevt, evt in enumerate(ds.events()): print nevt energyL3=procL3Energy.L3Energy(evt) moments, numhits=procSHES.CalibProcess(evt, inPix=True) mom1=moments[0] hist_L3Energy.fill(energyL3) hist_L3EnergyWeighted.fill([energyL3], weights=[numhits]) if nevt>100:
################################################################## ########################### Histograms & stuff ################### ################################################################## runs = np.array([args.runs], dtype='int').flatten() xLength = 839 # rotated camera 798 yLength = 591 # rotated camera 647 photonE = 508. # 1050. for runs 134 - ~150 |||| 508. for 156, 157 |||| 530 for 158+ cutoff = 0.2 # gas detector cutoff xtcav = True binsOfHist = xLength * len(runs) spectraCombined = Histogram((binsOfHist, 0., float(binsOfHist))) delayhist2d = Histogram((50, 0., 50.), (binsOfHist, 0., float(binsOfHist))) delayCountHist = Histogram((50, 0., 50.)) feegashist2d = Histogram((100, 0., 2.0), (binsOfHist, 0., float(binsOfHist))) feeCountHist = Histogram((100, 0., 2.0)) photonhist2d = Histogram((200, photonE * (1. - 0.02), photonE * (1. + 0.02)), (binsOfHist, 0., float(binsOfHist))) photonCountHist = Histogram( (200, photonE * (1. - 0.02), photonE * (1. + 0.02))) RatiosXTCAV = Histogram((50, -5., 5.), (binsOfHist, 0., float(binsOfHist))) TotalPowXTCAV = Histogram((100, 600., 1200.), (binsOfHist, 0., float(binsOfHist)))
from psana import * import numpy as np from ImgAlgos.PyAlgos import PyAlgos from skbeam.core.accumulators.histogram import Histogram dsource = MPIDataSource('exp=sxr07416:run=28:smd') det = Detector('OPAL1') alg = PyAlgos() alg.set_peak_selection_pars(npix_min=9, npix_max=100, amax_thr=40, atot_thr=300, son_min=0) hist_row = Histogram((1024, 0., 1024.)) hist_col = Histogram((1024, 0., 1024.)) hist_amp = Histogram((1024, 0., 3000.)) smldata = dsource.small_data('run28.h5', gather_interval=100) peakrow = np.zeros((10), dtype=int) peakcol = np.zeros((10), dtype=int) peakamp = np.zeros((10), dtype=float) for nevt, evt in enumerate(dsource.events()): calib = det.calib(evt) if calib is None: continue peaks = alg.peak_finder_v1(calib, thr_low=40, thr_high=40,