def forward(self, x):
"""
Forward pass
Input:
- x: input batch
Helper:
- exp_weights: expanded weights for fast convolution
- exp_x: expanded input for fast convolution
Return:
- out: convolution output batch
"""
out = np.zeros(self.output_shape)
# expand weights
exp_weights = self.weights.reshape([-1, self.out_N]) #KKC*C'
self.exp_x = []
# conv forward
for i in range(self.N):
x_i = x[i][np.newaxis,:] #single image
# unfold input image
exp_x_i = unfold(x_i,self.K,self.S,self.P) #N*KKC
# store expanded images
self.exp_x.append(exp_x_i)
# perform fast convolution
out[i] = np.reshape(np.dot(exp_x_i,exp_weights) + self.bias, self.output_shape[1:]) #H'W'C'
self.exp_x = np.array(self.exp_x)
return out
def backprop(self, grad):
"""
Backward pass
Input:
- grad, d_out: output gradients
Helper:
- flipped_weights: flipped weight for inverse conv
- exp_flipped_weights: expanded weights for fast convolution
- exp_d_out: expanded d_out for fast convolution
Return:
- d_x: input gradients
"""
self.d_out = grad
col_d_out = np.reshape(self.d_out, [self.N, -1, self.out_N])
# weight graidents
for i in range(self.N):
self.d_weights += np.dot(self.exp_x[i].T, col_d_out[i]).reshape(self.weights.shape)
self.d_bias += np.sum(col_d_out, axis=(0, 1))
# inverse conv for d_x
# flip weights
flipped_weights = np.flipud(np.fliplr(self.weights))
# swap input and output dimension
flipped_weights = flipped_weights.swapaxes(2, 3)
# conv backward
d_x = np.zeros(self.input_shape)
# expand flipped weights
exp_flipped_weights = flipped_weights.reshape([-1, self.C])
for i in range(self.N):
d_out_i = self.d_out[i][np.newaxis,:]
# unfold input image
exp_d_out = unfold(d_out_i,self.K,self.S,self.P)
d_x[i] = np.reshape(np.dot(exp_d_out,exp_flipped_weights),self.input_shape[1:]) #HWC
return d_x
def forward(self, x):
"""
Forward pass
Input:
- x: input batch
Helper:
- ufimg: expanded image for fast pooling
- max_col: positions of max values
Return:
- out: output batch
"""
out = np.zeros(self.output_shape)
# perform max pooling by batch and channel
for i in range(self.N):
for c in range(self.C):
frame = x[i, :, :, c].reshape(1, self.H, self.W, 1)
# expand image
self.ufimg = unfold(frame, self.k_size, self.stride, 0)
out[i, :, :,
c] = np.amax(self.ufimg,
axis=-1).reshape(self.output_shape[1],
self.output_shape[2])
# find max position for gradient calculation
max_col = np.argmax(self.ufimg, axis=-1).reshape(
[-1, 1]) # pick first max value
# set 1 for max value
for r in range(max_col.shape[0]):
block_dim = self.output_shape[1]
hh = 2 * int(r / block_dim) + int(max_col[r] / 2)
ww = 2 * int(r % block_dim) + int(max_col[r] % 2)
self.indices[i, hh, ww, c] = 1
return out
def getChi2(s, d, k=20, n=1000, test=False, smooth=True):
# Setup initial information
print 'Calculating unfolding variance...'
#r = np.log10(s['ML_energy'])
r = np.log10(d['ML_energy'])
Emids = getEmids()
cutName = 'llh'
dcut, scut = d['cuts'][cutName], s['cuts'][cutName]
eList = ['p', 'f']
# Load probability tables
p = prob.getProbs(s, scut)
st = len(Emids) - len(p['Rf|Tf'][0])
Emids = Emids[st:]
l2 = len(Emids)
# Create a fake spectrum
#s0 = -0.5
#sDict = {'':powerFit.powerFit(s0, st=st)}
#specCut = fakeSpec.fakeSpec(s, scut, sDict)
#for e in eList:
# N[e] = Nfinder(r, scut*s[e]*specCut)
# Calculate counts for data distribution
N = {}
for e in eList:
N[e] = Nfinder(r, dcut*d[e])
Ntot = sum([N[e] for e in eList])
# Build a table of n "true" distributions after 'wiggling' observed
# distribution in poisson errors
Nnew = {}
nTable = np.zeros((n,k,l2))
for e in eList:
Nnew[e] = np.random.poisson(N[e], (n, len(N[e])))
# Total number of events for each "true" distribution
totN = np.sum([Nnew[e] for e in eList], axis=(0,2)).astype('float')
# Unfold every wiggled distribution k times
for i in range(n):
for e in eList:
p['R'+e] = Nnew[e][i] / totN[i]
p['T'+e] = powerFit.powerFit(-2.7, [[6, -3.0]], st=st)
for j in range(k):
p = unfold(p)
nTable[i][j] = np.sum([p['T'+e] for e in eList], axis=0) * totN[i]
if smooth:
for e in eList:
p['T'+e] = smoother(p['T'+e])
# Option for analyzing the behavior in a single test bin
if test:
f = plt.figure()
ax1 = f.add_subplot(1,1,1)
ax1.set_title('Distribution for log10(E/GeV) =' + str(Emids[test]))
ax1.set_xlabel('Counts')
ax1.set_ylabel('')
counts = np.sum([N[e] for e in eList], axis=0)
newcts = np.sum([Nnew[e] for e in eList], axis=0)
print 'Counts:', (counts)[test]
print 'Variance:', np.sqrt((counts)[test])
print 'Variance (calc):', np.sqrt(np.var((newcts).transpose()[test]))
print 'Mean:', np.mean((newcts).transpose()[test])
ax1.hist((newcts).transpose()[test], bins=100, color='red')
test -= st
counts = (nTable.transpose()[test])
for j in range(k):
print 'Var:', np.sqrt(np.var(counts[j]))
print 'Mean:', np.mean(counts[j])
ax1.hist(counts[j], bins=100, color='blue', label=str(j))
plt.show()
return
# Calculate the variance using nTable
sigma = np.sqrt(np.var(nTable, axis=0))
## ALWAYS GIVING A 0 IN A SPECIFIC BIN. WHY IS THIS HAPPENING
#print nTable
#print nTable.shape
# Unfold the original counts
chi2 = np.zeros(k)
relerr = []
old = {}
for e in eList:
p['R'+e] = N[e] / Ntot
p['T'+e] = powerFit.powerFit(-2.7, st=st)
old[e] = p['T'+e]
seterr(invalid='ignore')
for j in range(1, k+1):
p = unfold(p)
oldsum = np.sum([old[e] for e in eList], axis=0)
new = np.sum([p['T'+e] for e in eList], axis=0)
chi2[j-1] = (1./(l2+1) * np.nansum(((new - oldsum)*Ntot / sigma[j-1])**2))
relerr += [(sigma[j-1] / (new*Ntot))]
for e in eList:
old[e] = p['T'+e]
if smooth:
p['T'+e] = smoother(p['T'+e])
return chi2, relerr
'/home/jinho93/oxides/perobskite/strontium-titanate/2.supc/vasp/1.defect/5.Vo/333/hub/4.4eV/dense/m5.ismear/k444/4.lmaxmix/band'
)
M = [[3.0, 0.0, 0.0], [0.0, 3.0, 0.0], [0.0, 0.0, 3.0]]
# high-symmetry point of a Hexagonal BZ in fractional coordinate
kpts = [
[0.5, 0.0, 0.0], # M
[0.0, 0.0, 0.0], # G
[0.0, 0.0, 0.5]
] # G
# create band path from the high-symmetry points, 30 points inbetween each pair
# of high-symmetry points
kpath = make_kpath(kpts, nseg=30)
K_in_sup = []
for kk in kpath:
kg, g = find_K_from_k(kk, M)
K_in_sup.append(kg)
# remove the duplicate K-points
reducedK = removeDuplicateKpoints(K_in_sup)
# basis vector of the primitive cell
cell = [[3.9082120409162044, 0.0000000000000000, 0.0],
[0, 3.9082120409162044, 0.0],
[0.0000, 0.0000000000000000, 3.9082120409162044]]
WaveSuper = unfold(M=M, wavecar='WAVECAR')
sw = WaveSuper.spectral_weight(kpath)
ef = 4.8211
def flux(d, s, niter, spec=0, smooth=True):
# Starting information
r = np.log10(d['ML_energy'])
t = duration(d)
Emids = getEmids()
w = d['weights']
cutName = 'llh'
dcut, scut = d['cuts'][cutName], s['cuts'][cutName]
eList = ['p', 'f']
colorDict = {'p':'b', 'h':'y', 'o':'c', 'f':'r', 'All':'k'}
# Setup plot
f = plt.figure()
ax1 = f.add_subplot(1,1,1)
ax1.set_title('Energy Spectum using '+cutName+' cut')
ax1.set_xlabel('Log10(Energy/GeV)')
ax1.set_ylabel('Flux (counts/(m^2 s ster GeV)) * E^' + str(spec))
# Create starting arrays
N = {}
N_passed = Nfinder(r, dcut, w=w).sum()
for e in eList:
ecut = d['llh_comp'] == e
N[e] = Nfinder(r, dcut*ecut, w=w)
# Get probabilities for unfolding
p = getProbs(s, scut)
st = len(Emids) - len(p['Rf|Tf'][0])
Emids = Emids[st:]
scale = (10**Emids)**spec
# Relative errors
effarea, sigma, relerr = getEff(s, scut, smooth=smooth)
effarea, sigma, relerr = effarea[st:], sigma[st:], relerr[st:]
# Due to unfolding
#fl = open('unfold_err.pkl', 'rb')
#chi2, unrel = pickle.load(fl)
#fl.close()
# Get starting probabilities
for e in eList:
p['R'+e] = N[e] / N_passed
p['T'+e] = powerFit.powerFit(-2.7, st=st)
# Bayesian unfolding
Nun, Rel, Flux, Err = {},{},{},{}
seterr(invalid='ignore')
for i in range(niter):
p = unfold(p)
Nun['All'] = sum([p['T'+e] for e in eList], axis=0) * N_passed
#Rel['All'] = np.sqrt(1/Nun['All'] + relerr**2 + unrel[i]**2)
Rel['All'] = np.sqrt(1/Nun['All'] + relerr**2)
Flux['All'] = NumToFlux(Nun['All'], effarea, t, st) * scale
Err['All'] = Flux['All'] * Rel['All']
#ax1.errorbar(Emids, Flux['All'], yerr=Err['All'], fmt='k.', label='Unfolded')
if i < niter-1:
for e in eList:
p['T'+e] = smoother(p['T'+e])
seterr(invalid='warn')
# Find bin values and errors
for e in eList:
Nun[e] = p['T'+e] * N_passed
Rel[e] = np.sqrt(1/Nun[e] + relerr**2)
Flux[e] = NumToFlux(Nun[e], effarea, t, st) * scale
Err[e] = Flux[e] * Rel[e]
# Plot
for e in eList + ['All']:
pnt = colorDict[e] + '.'
ax1.errorbar(Emids, Flux[e], yerr=Err[e], fmt=pnt, label=e)
# plot original (not unfolded) spectrum
O_N = (sum([N[e] for e in eList], axis=0))[st:]
O_relerr = np.sqrt(1/O_N + relerr**2)
O_flux = NumToFlux(O_N, effarea, t, st) * scale
O_err = O_flux * O_relerr
ax1.errorbar(Emids, O_flux, yerr=O_err, fmt='kx', label='Orig')
# plot Bakhtiyar's spectrum
#B_mids, B_flux, B_relup, B_reldn = bakhPlot()
#B_flux *= ((10**B_mids)**spec)
#B_errup = (B_flux * B_relup)
#B_errdn = (B_flux * B_reldn)
#ax1.errorbar(B_mids, B_flux, yerr=(B_errup, B_errdn), fmt='gx', label='Bakhtiyar')
# plot IT26 spectrum
#IT26_data = bakh.points['unfolded_twocomponent_th0-110412-shifted']
#IT26_relerr = bakh.geterrorbars('unfolded_twocomponent_th0-110412-shifted')
#IT26_mids = np.log10(IT26_data['E'])
#IT26_flux = np.asarray(IT26_data['dN/dE'])
#IT26_flux *= ((10**IT26_mids)**spec)
#IT26_err = IT26_flux * IT26_relerr
#ax1.errorbar(IT26_mids, IT26_flux, yerr=IT26_err, fmt='rx', label='IT-26 Two-Component')
ax1.set_yscale('log')
ax1.legend(loc='lower left')
#ax1.set_xlim((6, 9.5))
#ax1.set_ylim((10**(3), 10**(5)))
#ax1.set_ylim((10**(-22), 10**(-10)))
#plt.savefig('collab/pics/test.png')
plt.show()
errorEstimate=True
print splitE
model=Flat(ERange,splitE=splitE)
resp=det(*spectrum)
print('Expected detector responses:')
for i in range(len(resp[0])):
print((['No sphere']+milanoReference.names)[i]+' & %.1f'%resp[0][i]+' \\\\')
if errorEstimate:
guess=[]
x=[]
for i in range(50):
r=(np.array(resp[0])+[np.random.normal()*ri for ri in resp[1]],resp[1])
x.append(unfold.unfold(det, r, model))
errors=np.std(x,axis=0)
x=np.mean(x,axis=0)
else:
x=unfold.unfold(det, resp, model)
errors=[0]*len(x)
#print('Base weights:')
#for xp in x:
# print(str(xp))
#resp2=det(modelForGuess.getERange(), model.getFluence(x))#TODO guess-> x
#print('Expected detector responses with calculated spectrum:')
#for i in range(len(resp2[0])):
# print((['No sphere']+milanoReference.names)[i]+' & %.1f'%resp[0][i]+' & %.1f'%resp2[0][i]+' \\\\')
def get_supercell_bandstructure_ppc(self,
kpts_path=None,
kpts_nintersections=None,
supercell_size=[1, 1, 1],
unit_cell=[[1.0, 0, 0], [0, 1.0, 0],
[0.0, 0.0, 0.0]],
show=False):
"""Calculate band structure along :param kpts_path:
:param list kpts_path: list of tuples of (label, k-point) to
calculate path on.
:param int kpts_nintersections: is the number of points between
points in band structures. More makes the bands smoother.
:param list supercell_size: this lists the size of the supercell
[nx ny nz] as multiples of the unit cell in the ordinal directions
:param list unit_cell: list of unit cell vectors
This version uses PyProcar for k-path preparation and unfolding.
See https://romerogroup.github.io/pyprocar/index.html
returns (npoints, band_energies, fighandle)
"""
self.update()
self.stop_if(self.potential_energy is None)
M = [[1.0 * supercell_size[0], 0.0, 0.0],
[0.0, 1.0 * supercell_size[1], 0.0],
[0.0, 0.0, 1.0 * supercell_size[2]]]
dos = DOS(self, width=0.2)
d = dos.get_dos()
e = dos.get_energies()
ef = self.get_fermi_level()
kpts = [k[1] for k in kpts_path]
labels = [k[0] for k in kpts_path]
# by now, the self-consistent calculation is complete
# run in non-selfconsistent directory
wd = os.path.join(self.directory, 'bandstructure')
if not os.path.exists(wd):
self.clone(wd)
calc = Vasp(wd)
# this next line actually writes a K-points file, but we're
# going to use pyprocar to overwrite it
calc.set(
kpts=kpts,
kpts_nintersections=10,
reciprocal=True,
nsw=0, # no ionic updates required
isif=None,
ibrion=None,
icharg=11,
lorbit=12)
os.remove(os.path.join(wd, 'KPOINTS'))
# Let's try generating the default k-points path
# Now I need to learn how to set the k-path using pyprocar
# see:
import pyprocar as ppc
ppc.kpath(os.path.join(wd, 'POSCAR'),
os.path.join(wd, 'KPOINTS'),
supercell_matrix=np.diag(supercell_size))
# calc.update()
# we'll just launch VASP - skip the calc.update()
# Create and run a subprocess that invokes
if self.parameters.get('lsorbit'):
runfile = 'runvasp_ncl.py'
else:
runfile = 'runvasp.py'
CWD = os.getcwd()
VASPDIR = calc.directory
from .vasprc import VASPRC
module = VASPRC['module']
script = """#!/bin/bash
module load {module}
source ~/.bashrc # added by EPB - slight issue with "module load intel"
cd {CWD}
cd {VASPDIR} # this is the vasp directory
{runfile} # this is the vasp command
#end""".format(**locals())
jobname = 'SCBands'
cmdlist = ['{0}'.format(VASPRC['queue.command'])]
cmdlist += ['-o', VASPDIR]
cmdlist += [option for option in VASPRC['queue.options'].split()]
cmdlist += [
'-N', '{0}'.format(jobname), '-l',
'walltime={0}'.format(VASPRC['queue.walltime']), '-l',
'nodes={0}:ppn={1}'.format(VASPRC['queue.nodes'],
VASPRC['queue.ppn']), '-l',
'mem={0}'.format(VASPRC['queue.mem']), '-M', VASPRC['user.email']
]
p = sp.Popen(cmdlist,
stdin=sp.PIPE,
stdout=sp.PIPE,
stderr=sp.PIPE,
universal_newlines=True)
out, err = p.communicate(script)
'''
return None, None, None
# if calc.potential_energy is None:
# return None, None, None
else: # I don't think this will work unless the calculation is complete!
os.chdir(wd)
import pyprocar
pyprocar.unfold(
fname='PROCAR',
poscar='POSCAR',
outcar='OUTCAR',
supercell_matrix=np.diag(supercell_size),
ispin=None, # None for non-spin polarized calculation. For spin polarized case, ispin=1: up, ispin=2: down
efermi=None,
shift_efermi=True,
elimit=(-2, 2), # kticks=[0, 36, 54, 86, 110, 147, 165, 199], knames=['$\Gamma$', 'K', 'M', '$\Gamma$', 'A', 'H', 'L', 'A'],
print_kpts=False,
show_band=True,
width=4,
color='blue',
savetab='unfolding.csv',
savefig='unfolded_band.png',
exportplt=False)
return None, None, None
'''
from unfold import unfold
WaveSuper = unfold(M=M, wavecar='WAVECAR')
sw = WaveSuper.spectral_weight(kpath_uc)
from unfold import EBS_cmaps
e0, sf = WaveSuper.spectral_function(nedos=4000)
# or show the effective band structure with colormap
EBS_cmaps(
kpath_sc,
unit_cell,
e0,
sf,
nseg=nseg, # eref=-4.01,
show=False) #,
# ylim=(-3, 4))
'''
plt.savefig('unfolded_bandstructure.png')
'''
# In the fol
archiveKpts = os.path.join(wd, 'KPOINTS_old')
originalKpts = os.path.join(wd, 'KPOINTS')
if not os.path.exists(
archiveKpts): # archive the original KPOINTS file
shutil.copy(originalKpts, archiveKpts)
with open(archiveKpts, 'r') as kpts_infile: # get lines of old KPOINTS
KPtsLines = kpts_infile.readlines()
# Append labels for high-symmetry points to lines of the KPOINTS file
reject_chars = '$'
for idx, label in enumerate(labels):
newlabel = label
for ch in reject_chars:
newlabel = newlabel.replace(ch, '')
KPtsLines[4 + idx] = KPtsLines[4 + idx][:-1] + f' ! {newlabel}\n'
# Write a new version of the k-points file
with open(originalKpts, 'w') as kpts_outfile:
for line in KPtsLines:
kpts_outfile.write(line)
# This is John Kitchin's original band structure visualization
fig = plt.figure()
with open(os.path.join(wd, 'EIGENVAL')) as f:
# skip 5 lines
f.readline()
f.readline()
f.readline()
f.readline()
f.readline()
unknown, npoints, nbands = [int(x) for x in f.readline().split()]
f.readline() # skip line
band_energies = [[] for i in range(nbands)]
for i in range(npoints):
x, y, z, weight = [float(x) for x in f.readline().split()]
for j in range(nbands):
fields = f.readline().split()
id, energy = int(fields[0]), float(fields[1])
band_energies[id - 1].append(energy)
f.readline() # skip line
ax1 = plt.subplot(121)
for i in range(nbands):
plt.plot(list(range(npoints)), np.array(band_energies[i]) - ef)
ax = plt.gca()
ax.set_xticks([]) # no tick marks
plt.xlabel('k-vector')
plt.ylabel('Energy (eV)')
nticks = len(labels) / 2 + 1
ax.set_xticks(np.linspace(0, npoints, nticks))
L = []
L.append(labels[0])
for i in range(2, len(labels)):
if i % 2 == 0:
L.append(labels[i])
else:
pass
L.append(labels[-1])
ax.set_xticklabels(L)
plt.axhline(0, c='r')
plt.subplot(122, sharey=ax1)
plt.plot(d, e)
plt.axhline(0, c='r')
plt.ylabel('energy (eV)')
plt.xlabel('DOS')
plt.subplots_adjust(wspace=0.26)
if show:
plt.show()
return (npoints, band_energies, fig)
def get_supercell_bandstructure(self,
kpts_path=None,
kpts_nintersections=None,
supercell_size=[1, 1, 1],
unit_cell=[[1.0, 0, 0], [0, 1.0, 0],
[0.0, 0.0, 1.0]],
show=False,
imagedir=None,
imageprefix=None,
eref=None,
ymin=None,
ymax=None,
lsorb=False):
"""Calculate band structure along :param kpts_path:
:param list kpts_path: list of tuples of (label, k-point) to
calculate path on.
:param int kpts_nintersections: is the number of points between
points in band structures. More makes the bands smoother.
:param list supercell_size: this lists the size of the supercell
[nx ny nz] as multiples of the unit cell in the ordinal directions
:param list unit_cell: list of unit cell vectors
returns (npoints, band_energies, fighandle)
"""
self.update()
self.stop_if(self.potential_energy is None)
# Determine whether the colinear (vasp_std) or non-colinear (vasp_ncl) is
# to be used
if self.parameters.get('lsorbit'):
runfile = 'runvasp_ncl.py'
else:
runfile = 'runvasp.py'
'''
I'm following the procedure for creating a k-path in the supercell
https://github.com/QijingZheng/VaspBandUnfolding#band-unfolding-1
'''
# The tranformation matrix between supercell and primitive cell.
M = [[1.0 * supercell_size[0], 0.0, 0.0],
[0.0, 1.0 * supercell_size[1], 0.0],
[0.0, 0.0, 1.0 * supercell_size[2]]]
print(M)
from unfold import find_K_from_k, make_kpath, removeDuplicateKpoints
# Extract unit-cell k-points from kpts_path
uc_k_pts_bare = [k[1] for k in kpts_path]
nseg = 30 # points per segment
kpath_uc = make_kpath(uc_k_pts_bare, nseg=nseg) # interpolated uc k-path
# Map UC (=PC) k-points to SC k-points
Kpath_sc = []
for k_uc in kpath_uc:
K, g = find_K_from_k(k_uc, M)
Kpath_sc.append(K) # add a weight to K
Kpath = removeDuplicateKpoints(Kpath_sc) # this is a numpy.ndarray
# I convert this to a list because the Vasp wrapper prefers a kpts list
# Also, the Vasp wrapper wants a weight for each K point
Kpath_list = [list(np.append(K, 1.0)) for K in Kpath]
# Extract (unit cel) labels from kpts_path
labels = [k[0] for k in kpts_path]
dos = DOS(self, width=0.2)
d = dos.get_dos()
e = dos.get_energies()
ef = self.get_fermi_level()
print(f'Fermi energy: {ef:8.3f} eV')
# run in non-selfconsistent directory
wd = os.path.join(self.directory, 'bandstructure')
if not os.path.exists(wd):
self.clone(wd)
calc = Vasp(wd)
# don't set kpts_nintersections - avoid line mode
calc.set(
kpts=Kpath_list,
reciprocal=True,
nsw=0, # no ionic updates required
ismear=0,
sigma=0.1,
isif=None,
ibrion=None,
icharg=11,
lorbit=12)
calc.update() # updates the calculation files
# The manual calculation might be unnecessary because
# of the calc.update()
'''
CWD = os.getcwd()
VASPDIR = calc.directory
from .vasprc import VASPRC
module = VASPRC['module']
script = """#!/bin/bash
module load {module}
source ~/.bashrc # added by EPB - slight issue with "module load intel"
cd {CWD}
cd {VASPDIR} # this is the vasp directory
{runfile} # this is the vasp command
#end""".format(**locals())
jobname = 'SCBands'
cmdlist = ['{0}'.format(VASPRC['queue.command'])]
cmdlist += ['-o', VASPDIR]
cmdlist += [option for option in VASPRC['queue.options'].split()]
cmdlist += ['-N', '{0}'.format(jobname),
'-l', 'walltime={0}'.format(VASPRC['queue.walltime']),
'-l', 'nodes={0}:ppn={1}'.format(VASPRC['queue.nodes'],
VASPRC['queue.ppn']),
'-l', 'mem={0}'.format(VASPRC['queue.mem']),
'-M', VASPRC['user.email']]
p = sp.Popen(cmdlist,
stdin=sp.PIPE,
stdout=sp.PIPE,
stderr=sp.PIPE,
universal_newlines=True)
out, err = p.communicate(script)
'''
return None, None
else: # I don't think this will work unless the calculation is complete!
os.chdir(wd)
if imagedir is None:
imagedir = os.getcwd()
if imageprefix is None:
imageprefix = 'unfolded_bandstructure'
from unfold import unfold
# nseg = len(kpts_path) -1
WaveSuper = unfold(M=M, wavecar='WAVECAR', lsorbit=lsorb)
sw = WaveSuper.spectral_weight(kpath_uc)
from unfold import EBS_cmaps, EBS_scatter
e0, sf = WaveSuper.spectral_function(nedos=4000)
if eref is None:
eref = ef
if ymin is None:
ymin = ef - 5
if ymax is None:
ymax = ef + 5
print('The unit cell vectors are:')
print(unit_cell)
# Show the effective band structure with a scatter plot
EBS_scatter(kpath_uc,
unit_cell,
sw,
nseg=nseg,
eref=eref,
ylim=(ymin, ymax),
factor=5,
kpath_label=labels)
scatterplot = os.path.join(imagedir, imageprefix) + '_scatter_plot.png'
plt.savefig(scatterplot)
plt.close('all')
# or show the effective band structure with colormap
EBS_cmaps(kpath_uc,
unit_cell,
e0,
sf,
nseg=nseg,
eref=eref,
show=False,
ylim=(ymin, ymax),
kpath_label=labels)
colormap = os.path.join(imagedir, imageprefix) + '_colormap.png'
plt.savefig(colormap)
return scatterplot, colormap
"c" : "#6DE4DF",
"y" : "#FFDC55",
}
N = 100000
xl = 0
xh = 2
# mcd = mc_data_gen.LorentzianUnfoldData(N=N, range=[xl, xh])
mcd = mc_data_gen.FlatUnfoldData(N=N, range=[xl, xh])
true, meas = mcd.get_mc_sample()
unf = unfold.unfold(
range_true=[xl, xh],
range_meas=[xl, xh],
nbins_true=20,
nbins_meas=20,
)
A = unf.fit(true["data"], meas["data"], meas["weight"])
fig, ax = plt.subplots(1, 1)
cax = ax.matshow(A, cmap="bone_r")
cbar = fig.colorbar(cax)
cbar.set_label(r"$\log_{10}(a_{ij})$")
ax.set_xlabel(r"bin $i$ of true MC distribution")
ax.set_ylabel(r"bin $j$ of measured MC distribution")
ax.set_title(r"Response matrix from MC distribution, $N={{{}}}$ events"
.format(N))
def execute_query(query, filters, dates):
cursorMark = '*'
nextCursor = ''
items = []
while (1):
# Execute command and save data in tmp file
cmd = 'curl ' + '"' + query + '&cursor=' + cursorMark + '"' + \
' > tmp/tmp.json'
print('\n\n', cmd, '\n')
os.system(cmd)
print('\n\n======================================================')
try:
# Parse current API response
with open('tmp/tmp.json') as f:
data = json.load(f)
try:
# Save current items
for i in data['items']:
items.append(unfold('items', i))
# Retrive next markup
nextCursor = data['nextCursor']
cursorMark = nextCursor
except KeyError:
os.remove('./tmp/tmp.json')
break
except json.decoder.JSONDecodeError:
return 0
# Write true csv
with open('tmp/true.csv', 'w') as f:
header = []
for i in items:
for k in i:
header.append(k)
header = list(set(header))
writer = csv.DictWriter(f, fieldnames=list(header))
writer.writeheader()
writer.writerows(items)
# Filter items
items = [{k: clean_list(i[k]) for k in i if k in labels} for i in items]
items = parse_date(items)
if dates is not None:
items = filter_dates(items, dates)
items = filter(items, filters)
print('[ + ] Items found:', len(items))
for i in items[-1]:
print(i, ' : ', items[-1][i])
# Save data to csv
with open('./tmp/output.csv', 'w') as f:
header = []
for i in items:
for k in i:
header.append(k)
header = list(set(header))
writer = csv.DictWriter(f, fieldnames=list(header))
writer.writeheader()
writer.writerows(items)
return items
def histWriter(d, s, mp='All'):
##=========================================================================
## Basic setup
# Starting information
r = log10(d['ML_energy'])
wList = [d['w1'], d['w24'], d['w8']]
cutName = 'llh'
dcut = d['cuts'][cutName]
scut = s['cuts'][cutName]
monthList = [mp]
if mp == 'All':
monthList = ['201006', '201007', '201008', '201012', '201101']
# Get probabilities for unfolding
fl = open('probabilities.pkl', 'rb')
p = pickle.load(fl)
fl.close()
st = len(sim.Emids) - len(p['Rf|Tf'][0])
Emids = sim.Emids[st:]
# Relative errors
fl = open('unfold_err.pkl', 'rb')
chi2, unrel = pickle.load(fl)
fl.close()
effarea, effrel = sim.getEff(s, scut, smooth=True)
effarea, effrel = effarea[st:], effrel[st:]
##=========================================================================
## Make histograms
q = {}
q['Emids'] = Emids
q['flux'], q['err'], q['chi2'] = {},{},{}
d['cuts']['none'] = array([True for i in range(len(dcut))])
nameList = ['none', 'left', 'right', 'odds', 'evens']
cutList = [d['cuts'][name] for name in nameList]
niter = 30
for k in range(len(cutList)):
# Get counts
c0 = dcut * cutList[k]
N_passed = Nfinder(r, c0, wList).sum()
N_proton = Nfinder(r, c0*d['p'], wList)
N_iron = Nfinder(r, c0*d['f'], wList)
name = nameList[k]
# Get starting probabilities
p['Rp'] = N_proton / N_passed
p['Rf'] = N_iron / N_passed
p['Tp'], p['Tf'] = [powerFit.powerFit(-2.7, st=st) for i in range(2)]
q['chi2'][name] = []
# Bayesian unfolding
Tf, Tp = p['Tf'], p['Tp']
for i in range(niter):
# Calculate unfolded fluxes, errors, and chi sqaured values
p['Tf'], p['Tp'] = unfold.unfold(p)
old = Tf + Tp
new = p['Tf'] + p['Tp']
q['chi2'][name].append( 1./(len(Emids)+1) *
sum(((new-old) / unrel[i])**2))
counts, flux, relerr, err = {},{},{},{}
counts['A'] = (p['Tf']+p['Tp']) * N_passed
counts['F'] = p['Tf'] * N_passed
counts['P'] = p['Tp'] * N_passed
for key in counts.keys():
relerr[key] = sqrt(1/counts[key] + effrel**2 + unrel[i]**2)
flux[key] = n2f(counts[key], effarea, monthList, st)
err[key] = flux[key] * relerr[key]
# Write to dictionary
q['flux'][name+'_'+key+'_'+str(i+1)] = flux[key]
q['err'][name+'_'+key+'_'+str(i+1)] = err[key]
if i < niter-1:
p['Tf'] = smoother(p['Tf'])
p['Tp'] = smoother(p['Tp'])
# Original (not unfolded) spectrum
O_counts = (N_proton + N_iron)[st:]
O_relerr = sqrt(1/O_counts + effrel**2)
q['flux'][name+'_O'] = n2f(O_counts, effarea, monthList, st)
q['err'][name+'_O'] = q['flux'][name+'_O'] * O_relerr
##=========================================================================
## Write to file
print 'Writing to file...'
fl = open('collab/'+mp+'_hists.pkl', 'wb')
pickle.dump(q, fl)
fl.close()
import unfold
import mnk
import numpy as np
#GET INPUT FILES
r_values, a_values = unfold.unfold('C:\\python\\input_files\\input_ldf_01.txt')
#DATA
x_samp = np.array(r_values)
y_samp = np.array(a_values)
my_module = mnk.least_square(x_samp,y_samp)
my_module.show_plot()
my_module.get_fit()
def flux(niter=5, configs=None, spec=0, smooth=True,
zcorrect=False, weight=False,
orig=True, bakh=True,
emin=None, linear=False,
tax=None, tlabel=None,
comps=True, months=None,
decmin=None, decmax=None,
ramin=None, ramax=None):
# Starting information
Ebins = getEbins(reco=True)
Emids = getMids(Ebins)
scale = (10**Emids)**spec
h = loadHists()
hParams = {'configs':configs, 'months':months,
'decmin':decmin, 'decmax':decmax,
'ramin':ramin, 'ramax':ramax,
'w':weight, 'z':zcorrect}
eList = ['p','h','o','f']
if configs == None:
configs = sorted(list(set([k.split('_')[0] for k in h.keys()])))
# Load simulation information
effarea, sigma, relerr = getEff_fast(linear) # Effective area
p = getProbs_fast(zcorrect=zcorrect) # Probs for unfolding
# Relative error due to unfolding...
#fl = open('unfold_err.pkl', 'rb')
#chi2, unrel = pickle.load(fl)
#fl.close()
# Get detector runtime
t = 0.
for cfg in configs:
t += getRunTime(cfg, months=months)
# Total counts
N, Err = {},{}
N['All'], bins = histReader(h, x='energy', **hParams)
Err['All'], bins = histReader(h, x='energy', err=True, **hParams)
# Counts by composition
for e in eList:
N[e], bins = histReader(h, x='energy', e=e, **hParams)
Err[e], bins = histReader(h, x='energy', e=e, err=True, **hParams)
# Get starting probabilities
Nun, Rel, Flux, Err = {},{},{},{}
N_passed = float(np.sum([N[e] for e in eList]))
for e in eList:
p['R'+e] = N[e] / N_passed
p['T'+e] = powerFit.powerFit(-2.7)
# Setup plot
if tax == None:
fig, ax = plt.subplots()
ax.set_title('Energy Spectum')
ax.set_xlabel('Log10(Energy/GeV)')
ax.set_ylabel('Flux (counts/(m^2 s ster GeV)) * E^' + str(spec))
else:
ax = tax
# Bayesian unfolding
for i in range(niter):
p = unfold(p)
Nun['All'] = np.sum([p['T'+e] for e in eList], axis=0) * N_passed
#Rel['All'] = np.sqrt(1/Nun['All'] + relerr**2 + unrel[i]**2)
with np.errstate(divide='ignore'):
Rel['All'] = np.sqrt(1/Nun['All'] + relerr**2)
Flux['All'] = NumToFlux(Nun['All'], effarea, t) * scale
Err['All'] = Flux['All'] * Rel['All']
#ax.errorbar(Emids, Flux['All'], yerr=Err['All'], fmt='k.', label='Unfolded')
if i < niter-1:
for e in eList:
p['T'+e] = smoother(p['T'+e])
# Find bin values and errors
for e in eList:
Nun[e] = p['T'+e] * N_passed
Rel[e] = np.sqrt(1/Nun[e] + relerr**2)
Flux[e] = NumToFlux(Nun[e], effarea, t) * scale
Err[e] = Flux[e] * Rel[e]
# Plot
pltList = ['All']
if comps:
pltList += eList
for e in pltList:
pnt = getColor(e)+'.' if tax==None else '-'
label = e if tlabel==None else tlabel
ax.errorbar(Emids, Flux[e], yerr=Err[e], fmt=pnt, label=label)
# plot original (not unfolded) spectrum
if orig:
O_N = np.sum([N[e] for e in eList], axis=0)
O_relerr = np.sqrt(1/O_N + relerr**2)
O_flux = NumToFlux(O_N, effarea, t) * scale
O_err = O_flux * O_relerr
ax.errorbar(Emids, O_flux, yerr=O_err, fmt='kx', label='Orig')
# plot Bakhtiyar's spectrum
if bakh:
B_mids, B_flux, B_relup, B_reldn = bakhPlot()
B_flux *= ((10**B_mids)**spec)
B_errup = (B_flux * B_relup)
B_errdn = (B_flux * B_reldn)
ax.errorbar(B_mids, B_flux, yerr=(B_errup, B_errdn),
fmt='gx', label='Bakhtiyar')
# plot IT26 spectrum
#IT26_data = bakh.points['unfolded_twocomponent_th0-110412-shifted']
#IT26_relerr = bakh.geterrorbars('unfolded_twocomponent_th0-110412-shifted')
#IT26_mids = np.log10(IT26_data['E'])
#IT26_flux = np.asarray(IT26_data['dN/dE'])
#IT26_flux *= ((10**IT26_mids)**spec)
#IT26_err = IT26_flux * IT26_relerr
#ax.errorbar(IT26_mids, IT26_flux, yerr=IT26_err, fmt='rx', label='IT-26 Two-Component')
if tax == None:
ax.set_yscale('log')
ax.legend(loc='lower left')
if emin:
ax.set_xlim((emin, 9.5))
#ax.set_ylim((10**(3), 10**(5)))
#ax.set_ylim((10**(-22), 10**(-10)))
#plt.savefig('collab/pics/test.png')
plt.show()
def counts(s, niter, sDict={'':[-0.25, [[7.5, -0.75]]]},
smooth=True, spl=False, diff=False):
t = log10(s['MC_energy'])
r = crapE(t)
cutName = 'llh'
nancut = (r==r)
cut = s['cuts'][cutName] * nancut
f = plt.figure()
ax1 = f.add_subplot(1,1,1)
ax1.set_title('Energy Spectum using '+cutName+' cut ('+str(niter)+' iterations)')
ax1.set_xlabel('Log10(Energy/GeV)')
ax1.set_ylabel('Counts')
# Load probability tables
print 'Getting probability tables...'
p = crapProb(s, cut)
st = len(sim.Emids) - len(p['Rf|Tf'][0])
Emids = sim.Emids[st:]
# Option for splining
if spl:
for key in p.keys():
p[key] = 10**(spline.spline(s, p[key], nk=2, npoly=3))
print sum(p['Rf|Tp']+p['Rp|Tp'], axis=0)
print sum(p['Rf|Tf']+p['Rp|Tf'], axis=0)
# Create our toy MC spectrum
temp = {}
for key in sDict.keys():
s0 = sDict[key][0]
try:
sTable = sDict[key][1]
except IndexError:
sTable=False
temp[key] = powerFit.powerFit(s0, sTable=sTable, st=st)
specCut = sim.fakeSpec(s, cut, temp)
#specCut = array([True for i in range(len(specCut))])
# Create starting arrays
N_passed = Nfinder(r, cut*specCut).sum()
comp = {}
comp['p'], comp['f'] = crapComp(t)
N_proton = Nfinder(r, cut*comp['p']*specCut)
N_iron = Nfinder(r, cut*comp['f']*specCut)
# Get starting probabilities
p['Rp'] = N_proton / N_passed
p['Rf'] = N_iron / N_passed
p['Tf'], p['Tp'] = [powerFit.powerFit(-2.7, st=st) for i in range(2)]
# Get relative errors due to unfolding
# Due to efficiency
effarea, relerr = sim.getEff(s, cut, smooth=smooth)
relerr = relerr[st:]
# Bayesian unfolding
for i in range(niter):
p['Tf'], p['Tp'] = unfold.unfold(p)
# Smooth prior before next iteration (except last time)
if i < niter-1:
p['Tf'] = smoother(p['Tf'])
p['Tp'] = smoother(p['Tp'])
# Find bin values and errors
F_Nunfold = p['Tf'] * N_passed
P_Nunfold = p['Tp'] * N_passed
All_Nunfold = F_Nunfold + P_Nunfold
## NOTE: I don't think you can use the relative errors like this ##
F_relerr = sqrt(1/F_Nunfold + relerr**2)
P_relerr = sqrt(1/P_Nunfold + relerr**2)
F_err = F_Nunfold * F_relerr
P_err = P_Nunfold * P_relerr
All_err = sqrt(F_err**2 + P_err**2)
#ax1.errorbar(Emids, F_Nunfold, yerr=F_err, fmt='r.', label='Iron')
#ax1.errorbar(Emids, P_Nunfold, yerr=P_err, fmt='b.', label='Proton')
# Plot true spectrum
MC_N = Nfinder(log10(s['MC_energy']), cut*specCut)[st:]
MC_F = Nfinder(log10(s['MC_energy']), cut*specCut*s['F'])[st:]
MC_P = Nfinder(log10(s['MC_energy']), cut*specCut*s['P'])[st:]
#ax1.plot(Emids, MC_F, 'rx', label='MC_F')
#ax1.plot(Emids, MC_P, 'bx', label='MC_P')
# plot original (not unfolded) spectrum
O_N = (N_proton + N_iron)[st:]
O_relerr = sqrt(1/O_N + relerr**2)
O_err = O_N * O_relerr
if not diff:
ax1.errorbar(Emids, O_N, yerr=O_err, fmt='k.', label='Original')
ax1.plot(Emids, MC_N, 'rx', label='MC')
ax1.errorbar(Emids, All_Nunfold, yerr=All_err, fmt='g.', label='Unfold')
ax1.set_yscale('log')
if diff:
ax1.plot(Emids, (O_N - MC_N), 'k', label='Original - MC')
ax1.plot(Emids, (All_Nunfold - MC_N), 'r', label='Unfold - MC')
ax1.legend(loc='lower left')
#ax1.set_ylim((10**(-1), 10**(4)))
plt.show()
# Execute command and save data in tmp file
new_query = '"' + args.query + '&cursor=' + cursorMark + '"'
cmd = 'curl ' + new_query + ' > tmp.json'
print('\n\n', cmd, '\n')
os.system(cmd)
print('\n\n======================================================')
# Parse current API response
with open('tmp.json') as f:
data = json.load(f)
try:
# Save current items
for i in data['items']:
items.append(unfold('items', i))
# Retrive next markup
nextCursor = data['nextCursor']
cursorMark = nextCursor
except KeyError:
os.remove('tmp.json')
break
if args.col.endswith('.txt'):
try:
with open(args.col) as f:
to_keep = [s.rstrip() for s in f]
items = [{k: i[k] for k in i if k in to_keep}
for i in items]
def flux(config, niter, spec=0, smooth=True, zcorrect=False, weight=False):
# Starting information
bintype = 'logdist'
dataList = getDataList(config, bintype)
dataList = dataList[:2]
Ebins = getEbins(reco=True)
Emids = getMids(Ebins)
cutName = 'llh'
scale = (10**Emids)**spec
# Load simulation information
s = load_sim(bintype=bintype)
# Relative errors
effarea, sigma, relerr = getEff(s, s['cuts']['llh'])
#effarea, sigma, relerr = getEff(s, scut, smooth=smooth)
# Due to unfolding
#fl = open('unfold_err.pkl', 'rb')
#chi2, unrel = pickle.load(fl)
#fl.close()
# Get detector runtime
configs = sorted(list(set([data[0] for data in dataList])))
t = 0.
for cfg in configs:
dates = sorted([data[1] for data in dataList if data[0]==cfg])
mindate = int(dates[0] + '01')
next = str(int(dates[-1]) + 1)
if dates[-1][-2:] == '12':
next = str(int(date) + 100)
maxdate = int(next + '01')
t += getRunTime(cfg, minDate=mindate, maxDate=maxdate)
# Build histograms of desired information
N, Err = {},{}
for cfg, date in dataList:
d = load_data(cfg, date, bintype)
eList = getComps(d)
dcut = d['cuts'][cutName]
r = np.log10(d['ML_energy'])
if zcorrect:
r -= zfix(d['zenith'], bintype='logdist')
# Create starting arrays
w = d['weights'][dcut] if weight else None
#N_passed = float(np.histogram(r[dcut], bins=Ebins, weights=w)[0].sum())
for e in eList:
ecut = d['llh_comp'] == e
w = d['weights'][dcut*ecut] if weight else None
try: N[e] += np.histogram(r[dcut*ecut], bins=Ebins, weights=w)[0]
except KeyError:
N[e] = np.histogram(r[dcut*ecut], bins=Ebins, weights=w)[0]
# Get probabilities for unfolding
p = getProbs(s, s['cuts'][cutName], zcorrect=zcorrect)
Nun, Rel, Flux, Err = {},{},{},{}
# Get starting probabilities
N_passed = float(np.sum([N[e] for e in eList]))
for e in eList:
p['R'+e] = N[e] / N_passed
p['T'+e] = powerFit.powerFit(-2.7)
# Setup plot
fig, ax = plt.subplots()
ax.set_title('Energy Spectum using '+cutName+' cut')
ax.set_xlabel('Log10(Energy/GeV)')
ax.set_ylabel('Flux (counts/(m^2 s ster GeV)) * E^' + str(spec))
# Bayesian unfolding
for i in range(niter):
p = unfold(p)
Nun['All'] = np.sum([p['T'+e] for e in eList], axis=0) * N_passed
#Rel['All'] = np.sqrt(1/Nun['All'] + relerr**2 + unrel[i]**2)
with np.errstate(divide='ignore'):
Rel['All'] = np.sqrt(1/Nun['All'] + relerr**2)
Flux['All'] = NumToFlux(Nun['All'], effarea, t) * scale
Err['All'] = Flux['All'] * Rel['All']
#ax.errorbar(Emids, Flux['All'], yerr=Err['All'], fmt='k.', label='Unfolded')
if i < niter-1:
for e in eList:
p['T'+e] = smoother(p['T'+e])
# Find bin values and errors
for e in eList:
Nun[e] = p['T'+e] * N_passed
Rel[e] = np.sqrt(1/Nun[e] + relerr**2)
Flux[e] = NumToFlux(Nun[e], effarea, t) * scale
Err[e] = Flux[e] * Rel[e]
# Plot
for e in eList + ['All']:
pnt = getColor(e) + '.'
ax.errorbar(Emids, Flux[e], yerr=Err[e], fmt=pnt, label=e)
# plot original (not unfolded) spectrum
O_N = np.sum([N[e] for e in eList], axis=0)
O_relerr = np.sqrt(1/O_N + relerr**2)
O_flux = NumToFlux(O_N, effarea, t) * scale
O_err = O_flux * O_relerr
ax.errorbar(Emids, O_flux, yerr=O_err, fmt='kx', label='Orig')
# plot Bakhtiyar's spectrum
B_mids, B_flux, B_relup, B_reldn = bakhPlot()
B_flux *= ((10**B_mids)**spec)
B_errup = (B_flux * B_relup)
B_errdn = (B_flux * B_reldn)
ax.errorbar(B_mids, B_flux, yerr=(B_errup, B_errdn), fmt='gx', label='Bakhtiyar')
# plot IT26 spectrum
#IT26_data = bakh.points['unfolded_twocomponent_th0-110412-shifted']
#IT26_relerr = bakh.geterrorbars('unfolded_twocomponent_th0-110412-shifted')
#IT26_mids = np.log10(IT26_data['E'])
#IT26_flux = np.asarray(IT26_data['dN/dE'])
#IT26_flux *= ((10**IT26_mids)**spec)
#IT26_err = IT26_flux * IT26_relerr
#ax.errorbar(IT26_mids, IT26_flux, yerr=IT26_err, fmt='rx', label='IT-26 Two-Component')
ax.set_yscale('log')
ax.legend(loc='lower left')
#ax.set_xlim((6, 9.5))
#ax.set_ylim((10**(3), 10**(5)))
#ax.set_ylim((10**(-22), 10**(-10)))
#plt.savefig('collab/pics/test.png')
plt.show()
model = Linear((1e-12,1e0),4) #This is were the assumption on spectrum shape is made
resp=det(*spectrum)
print('Expected detector responses:')
for r in resp[0]:
print(str(r))
if errorEstimate:
guess=[]
x=[]
for i in range(7):
r=(np.array(resp[0])+[np.random.normal()*ri for ri in resp[1]],resp[1])
guess.append(unfold.unfold(det, r, modelForGuess))
x.append(unfold.unfold(det, r, model, guess=np.array(guess[-1])))
guessErrors=np.std(guess,axis=0)
errors=np.std(x,axis=0)
guess=np.mean(guess,axis=0)
x=np.mean(x,axis=0)
else:
guess=unfold.unfold(det, resp, modelForGuess)
x=unfold.unfold(det, resp, model, guess=guess*2)
guessErrors=[0]*len(guess)
errors=[0]*len(x)
print('Base weights:')
for xp in x:
print(str(xp))
#energy range for model
ERange=(5e-13,3e0)
#first linear model, used for guess to nonlinear
#modelForGuess = Flat(ERange,splitE=optimizeFlat.optimize(det,ERange,n)) #use this instead for pptimized base function shapes
#modelForGuess = Flat(ERange,n)
#second non-linear model, set to Flat also to get faster results, (or comment everywhere)
model = Flat(ERange,n)
guess=[]
x=[]
simx=[]
for i in range(50):
r=(np.array(resp[0])+[np.random.normal()*ri for ri in resp[1]],resp[1])
#guess.append([max(0,p) for p in unfold.unfold(det, r, modelForGuess)])
x.append(unfold.unfold(det, r, model))
r=(np.array(simResp[0])+[np.random.normal()*ri for ri in simResp[1]],simResp[1])
simx.append(unfold.unfold(det, r, model))
#guessErrors=np.std(guess,axis=0)
errors=np.std(x,axis=0)
simerrors=np.std(simx,axis=0)
#guess=np.mean(guess,axis=0)
x=np.mean(x,axis=0)
simx=np.mean(simx,axis=0)
#print('Optimal parameters:')
#for xp in x:
# print(str(xp))
#print responses in latex table friendly way
#respx=det(model.getERange(), model.getFluence(x))
"""
Predict with multplie techniques and plot the resulting distributions.
"""
N = 10000
xl = 0
xh = 2
nbins_true = 30
nbins_meas = 15
mcd = mc_data_gen.FlatUnfoldData(N=N, range=[xl, xh])
true, meas = mcd.get_mc_sample()
#### Create response matrix once
unf = unf.unfold(range_true=[xl, xh], range_meas=[xl, xh], nbins_true=nbins_true, nbins_meas=nbins_meas)
A = unf.fit(true["data"], meas["data"], meas["weight"])
square = nbins_true == nbins_meas
#### Fit true distribution using various models
# LLH fit unfolding and different regularizations
predicted_res = unf.predict(true["data"], ndof=0)
predicted_res_t1 = unf.predict(true["data"], ndof=1)
predicted_res_t10 = unf.predict(true["data"], ndof=10)
predicted_res_tinf = unf.predict(true["data"], ndof=1.0e10)
# Simple inversion if square (nbins_true = nbins_meas)
if square:
predicted_inverse = unf.predict_by_inverse(true["data"])
# Pseudoinverse with least squares. If square, should be equal to simple inversion
def histWriter(d, s, out='test'):
##=========================================================================
## Basic setup
# Starting information
r = log10(d['ML_energy'])
t = duration(d)
Emids = getEmids()
w = d['weights']
cutName = 'llh'
dcut, scut = d['cuts'][cutName], s['cuts'][cutName]
eList = ['p', 'f']
# Get probabilities for unfolding
p = getProbs(s, scut)
st = len(Emids) - len(p['Rf|Tf'][0])
Emids = Emids[st:]
# Relative errors
fl = open('unfold_err2.pkl', 'rb')
chi2, unrel = pickle.load(fl)
fl.close()
effarea, sigma, effrel = getEff(s, scut, smooth=True)
effarea, sigma, effrel = effarea[st:], sigma[st:], effrel[st:]
##=========================================================================
## Make histograms
q = {}
q['Emids'] = Emids
q['flux'], q['err'], q['chi2'] = {},{},{}
d['cuts']['none'] = array([True for i in range(len(dcut))])
nameList = ['none', 'left', 'right', 'odds', 'evens']
cutList = [d['cuts'][name] for name in nameList]
niter = 30
for k in range(len(cutList)):
name = nameList[k]
c0 = dcut * cutList[k]
q['chi2'][name] = []
# Get counts and starting probabilities
N_passed = Nfinder(r, c0, w=w).sum()
N = {}
for e in eList:
N[e] = Nfinder(r, c0*d[e], w=w)
p['R'+e] = N[e] / N_passed
p['T'+e] = powerFit.powerFit(-2.7, st=st)
# Bayesian unfolding
counts, flux, relerr, err = {},{},{},{}
for i in range(niter):
# Calculate unfolded fluxes, errors, and chi sqaured values
old = sum([p['T'+e] for e in eList], axis=0)
p = unfold.unfold(p)
new = sum([p['T'+e] for e in eList], axis=0)
q['chi2'][name].append( 1./(len(Emids)+1) *
sum(((new-old) / unrel[i])**2))
counts['All'] = new * N_passed
for e in eList:
counts[e] = p['T'+e] * N_passed
for e in eList + ['All']:
relerr[e] = sqrt(1/counts[e] + effrel**2 + unrel[i]**2)
flux[e] = n2f(counts[e], effarea, t, st)
err[e] = flux[e] * relerr[e]
# Write to dictionary
q['flux'][name+'_'+e+'_'+str(i+1)] = flux[e]
q['err'][name+'_'+e+'_'+str(i+1)] = err[e]
if i < niter-1:
p['T'+e] = smoother(p['T'+e])
# Original (not unfolded) spectrum
O_counts = (sum([N[e] for e in eList], axis=0))[st:]
O_relerr = sqrt(1/O_counts + effrel**2)
q['flux'][name+'_O'] = n2f(O_counts, effarea, t, st)
q['err'][name+'_O'] = q['flux'][name+'_O'] * O_relerr
##=========================================================================
## Write to file
print 'Writing to file...'
outFile = 'hists/'+out+'_hists.npy'
save(outFile, q)