def reject_outliers(self):
"""
Reject outliers peaks with a distance to the calculated peak position of
more than self.inp.fit['limit'][1] times the mean distance for the given grain
Jette Oddershede, Risoe DTU, May 15 2008
"""
g = self.grain_values()
self.inp.newreject = 0
self.inp.fit['newreject_grain'] = []
# self.inp.rerefine = []
#value = []
new = 1
while new == 1:
new = 0
for i in range(self.inp.no_grains):
#value.append([])
if i+1 in self.inp.fit['skip']:
pass
else:
for j in range(self.inp.nrefl[i]-1,-1,-1): # loop backwards to make pop work
value = fcn.peak(lsqr.mg.values['a'],lsqr.mg.values['b'],lsqr.mg.values['c'],lsqr.mg.values['alpha'],lsqr.mg.values['beta'],lsqr.mg.values['gamma'],
self.inp.h[i][j],self.inp.k[i][j],self.inp.l[i][j],
self.inp.w[self.inp.id[i][j]],self.inp.dety[self.inp.id[i][j]],self.inp.detz[self.inp.id[i][j]],
#n.array([self.inp.Syy[self.inp.id[i][j]],self.inp.Szz[self.inp.id[i][j]],self.inp.Sww[self.inp.id[i][j]]]),
self.inp.vars[i][j],
self.mg.values['wx'],self.mg.values['wy'],
self.mg.values['tx'],self.mg.values['ty'],self.mg.values['tz'],
self.mg.values['py'],self.mg.values['pz'],
self.mg.values['cy'],self.mg.values['cz'],
self.mg.values['L'],
self.mg.values['x%s' %i],self.mg.values['y%s' %i],self.mg.values['z%s' %i],
self.inp.rod[i][0]+self.mg.values['rodx%s' %i],
self.inp.rod[i][1]+self.mg.values['rody%s' %i],
self.inp.rod[i][2]+self.mg.values['rodz%s' %i],
self.mg.values['epsaa%s' %i],self.mg.values['epsab%s' %i],self.mg.values['epsac%s' %i],
self.mg.values['epsbb%s' %i],self.mg.values['epsbc%s' %i],self.mg.values['epscc%s' %i])
if value > self.inp.fit['limit'][1]*g[i]/self.inp.nrefl[i]:
new = 1
print('Rejected peak id %i from grain %i (hkl: %i %i %i, limit: %f): %f' %(self.inp.id[i][j],i+1,self.inp.h[i][j],self.inp.k[i][j],self.inp.l[i][j],self.inp.fit['limit'][1],value*self.inp.nrefl[i]/g[i]))
reject.reject(self.inp,i,j,value*self.inp.nrefl[i]/g[i])
for i in range(self.inp.no_grains):
if self.inp.nrefl[i] < self.inp.fit['min_refl'] and i+1 not in self.inp.fit['skip']:
self.inp.fit['skip'].append(i+1)
self.inp.fit['skip'].sort()
def residual(inp,limit,only=None):
"""
Reject outliers peaks based on residuals until mean<limit*median
Jette Oddershede, Risoe DTU, May 15 2008
"""
# must update inp.vars because the order here is [i][j] in stead of [id[i][j]], the latter doesn't change when peaks are rejected, the former does.
# calculate experimental errors using the present values
import error
error.vars(inp)
# build functions to minimise
import build_fcn
build_fcn.FCN(inp)
#refinement update
import fcn
reload(fcn)
for i in range(inp.no_grains):
if i+1 in inp.fit['skip']:
pass
else:
for j in range(inp.nrefl[i]):
inp.residual[i][j] = fcn.peak(inp.values['a'],inp.values['b'],inp.values['c'],inp.values['alpha'],inp.values['beta'],inp.values['gamma'],
inp.h[i][j],inp.k[i][j],inp.l[i][j],
inp.w[inp.id[i][j]],inp.dety[inp.id[i][j]],inp.detz[inp.id[i][j]],
#n.array([inp.Syy[inp.id[i][j]],inp.Szz[inp.id[i][j]],inp.Sww[inp.id[i][j]]]),
inp.vars[i][j],
inp.values['wx'],inp.values['wy'],
inp.values['tx'],inp.values['ty'],inp.values['tz'],
inp.values['py'],inp.values['pz'],
inp.values['cy'],inp.values['cz'],
inp.values['L'],
inp.values['x%s' %i],inp.values['y%s' %i],inp.values['z%s' %i],
inp.rod[i][0]+inp.values['rodx%s' %i],
inp.rod[i][1]+inp.values['rody%s' %i],
inp.rod[i][2]+inp.values['rodz%s' %i],
inp.values['epsaa%s' %i],inp.values['epsab%s' %i],inp.values['epsac%s' %i],
inp.values['epsbb%s' %i],inp.values['epsbc%s' %i],inp.values['epscc%s' %i])
data = deepcopy(inp.residual)
maxres = [0]*inp.no_grains
for i in range(inp.no_grains):
data[i].sort()
if i+1 in inp.fit['skip']:
pass
else:
mean = int(n.sum(data[i])/len(data[i]))
medi = median(data[i])
# print i, len(data[i]), medi, mean,'\n',data[i]
while mean > limit*medi:
data[i].pop()
mean = int(n.sum(data[i])/inp.nrefl[i])
medi = median(data[i])
maxres[i] = max(data[i])
# print i, len(data[i]),medi,mean,'\n',data[i],'\n'
delete = 0
if only==None:
only = range(1,1+inp.no_grains)
for i in range(inp.no_grains):
if i+1 in inp.fit['skip'] or i+1 not in only:
pass
else:
for j in range(inp.nrefl[i]-1,-1,-1): # loop backwards to make pop work
if inp.residual[i][j] > maxres[i]:
delete = delete + 1
reject(inp,i,j,'residual')
if only != []:
print 'Rejected', delete, 'reflection based on residuals'
insignificant(inp)
def reject_outliers(lsqr):
# """
# Reject outliers peaks with a distance to the calculated peak position of
# more than lsqr.inp.fit['rej_resmean'] times the mean distance for the given grain
#
# Jette Oddershede, Risoe DTU, May 15 2008
# """
g = grain_values(lsqr)
lsqr.inp.newreject = 0
lsqr.inp.fit['newreject_grain'] = []
for i in range(lsqr.inp.no_grains):
if i+1 in lsqr.inp.fit['skip']:
pass
else:
if lsqr.g_old[i]/g[i] > 1.1: # ensure fitting twice, grain and first final to get sufficient error estimation in iminuit where scale_errors not possible
lsqr.inp.fit['newreject_grain'].append(i+1)
#value = []
new = 1
while new == 1:
new = 0
for i in range(lsqr.inp.no_grains):
#value.append([])
if i+1 in lsqr.inp.fit['skip']:
pass
else:
for j in range(lsqr.inp.nrefl[i]-1,-1,-1): # loop backwards to make pop work
value = fcn.peak(lsqr.m.values['a'],lsqr.m.values['b'],lsqr.m.values['c'],lsqr.m.values['alpha'],lsqr.m.values['beta'],lsqr.m.values['gamma'],
lsqr.inp.h[i][j],lsqr.inp.k[i][j],lsqr.inp.l[i][j],
lsqr.inp.w[lsqr.inp.id[i][j]],lsqr.inp.dety[lsqr.inp.id[i][j]],lsqr.inp.detz[lsqr.inp.id[i][j]],
lsqr.inp.vars[i][j],
lsqr.m.values['wx'],lsqr.m.values['wy'],
lsqr.m.values['tx'],lsqr.m.values['ty'],lsqr.m.values['tz'],
lsqr.m.values['py'],lsqr.m.values['pz'],
lsqr.m.values['cy'],lsqr.m.values['cz'],
lsqr.m.values['L'],
lsqr.m.values['x%s' %i],lsqr.m.values['y%s' %i],lsqr.m.values['z%s' %i],
lsqr.inp.rod[i][0]+lsqr.m.values['rodx%s' %i],
lsqr.inp.rod[i][1]+lsqr.m.values['rody%s' %i],
lsqr.inp.rod[i][2]+lsqr.m.values['rodz%s' %i],
lsqr.m.values['epsaa%s' %i],lsqr.m.values['epsab%s' %i],lsqr.m.values['epsac%s' %i],
lsqr.m.values['epsbb%s' %i],lsqr.m.values['epsbc%s' %i],lsqr.m.values['epscc%s' %i])
if value > lsqr.inp.fit['rej_resmean']*g[i]/lsqr.inp.nrefl[i]:
new = 1
print 'Rejected peak id %i from grain %i (hkl: %i %i %i, limit: %f): %f' %(lsqr.inp.id[i][j],i+1,lsqr.inp.h[i][j],lsqr.inp.k[i][j],lsqr.inp.l[i][j],lsqr.inp.fit['rej_resmean'],value*lsqr.inp.nrefl[i]/g[i])
reject.reject(lsqr.inp,i,j,value*lsqr.inp.nrefl[i]/g[i])
if 'globals' not in lsqr.inp.fit['goon']:
lsqr.inp.mean_ia = []
for i in range(lsqr.inp.no_grains):
lsqr.inp.mean_ia.append([])
for j in range(lsqr.inp.nrefl[i]):
lsqr.inp.mean_ia[i].append(1)
reject.mean_ia(lsqr.inp,lsqr.inp.fit['rej_ia'])
lsqr.inp.residual = []
for i in range(lsqr.inp.no_grains):
lsqr.inp.residual.append([])
for j in range(lsqr.inp.nrefl[i]):
lsqr.inp.residual[i].append(1)
reject.residual(lsqr.inp,lsqr.inp.fit['rej_resmedian'])
lsqr.inp.volume = []
for i in range(lsqr.inp.no_grains):
lsqr.inp.volume.append([])
for j in range(lsqr.inp.nrefl[i]):
lsqr.inp.volume[i].append(1)
reject.intensity(lsqr.inp)
reject.merge(lsqr.inp)
reject.multi(lsqr.inp)
for i in range(lsqr.inp.no_grains):
if lsqr.inp.nrefl[i] < lsqr.inp.fit['min_refl'] and i+1 not in lsqr.inp.fit['skip']:
lsqr.inp.fit['skip'].append(i+1)
lsqr.inp.fit['skip'].sort()
def grain_values(lsqr):
# """
# Calculate the contributions from each grain
# For extreme contributions print a warning (*****)
#
# Jette Oddershede, Risoe DTU, May 15 2008
# """
# rebuild function and load
import build_fcn
build_fcn.FCN(lsqr.inp)
import fcn
reload(fcn)
# save values before making a new lsqr of minuit
# temp1 = deepcopy(lsqr.m.values)
# temp2 = deepcopy(lsqr.m.errors)
# temp3 = deepcopy(lsqr.m.fixed)
# temp4 = deepcopy(lsqr.mg.tol)
g = n.zeros((lsqr.inp.no_grains))
lsqr.inp.fit['poor'] = []
lsqr.poor_value = []
lsqr.poor_nrefl = []
for i in range(lsqr.inp.no_grains):
if i+1 in lsqr.inp.fit['skip']:
pass
else:
for j in range(lsqr.inp.nrefl[i]):
g[i] = g[i] + fcn.peak(lsqr.m.values['a'],lsqr.m.values['b'],lsqr.m.values['c'],lsqr.m.values['alpha'],lsqr.m.values['beta'],lsqr.m.values['gamma'],
lsqr.inp.h[i][j],lsqr.inp.k[i][j],lsqr.inp.l[i][j],
lsqr.inp.w[lsqr.inp.id[i][j]],lsqr.inp.dety[lsqr.inp.id[i][j]],lsqr.inp.detz[lsqr.inp.id[i][j]],
lsqr.inp.vars[i][j],
lsqr.m.values['wx'],lsqr.m.values['wy'],
lsqr.m.values['tx'],lsqr.m.values['ty'],lsqr.m.values['tz'],
lsqr.m.values['py'],lsqr.m.values['pz'],
lsqr.m.values['cy'],lsqr.m.values['cz'],
lsqr.m.values['L'],
lsqr.m.values['x%s' %i],lsqr.m.values['y%s' %i],lsqr.m.values['z%s' %i],
lsqr.inp.rod[i][0]+lsqr.m.values['rodx%s' %i],
lsqr.inp.rod[i][1]+lsqr.m.values['rody%s' %i],
lsqr.inp.rod[i][2]+lsqr.m.values['rodz%s' %i],
lsqr.m.values['epsaa%s' %i],lsqr.m.values['epsab%s' %i],lsqr.m.values['epsac%s' %i],
lsqr.m.values['epsbb%s' %i],lsqr.m.values['epsbc%s' %i],lsqr.m.values['epscc%s' %i])
# for i in range(lsqr.inp.no_grains):
# if i+1 not in lsqr.inp.fit['skip']:
# # make new lsqr of minuit
# lsqr.mg = Minuit(fcn.FCNgrain)
# lsqr.mg.values = temp1
# lsqr.mg.values['i'] = i
# lsqr.mg.scan(("L",1,lsqr.mg.values['L']-1,lsqr.mg.values['L']+1)) # scan to set lsqr.m.fval, function starting value
# g[i] = lsqr.mg.fval
data = []
poor = []
for i in range(lsqr.inp.no_grains):
if i+1 not in lsqr.inp.fit['skip']:
data.append(g[i]/lsqr.inp.nrefl[i])
reject.mad(data,poor,lsqr.inp.fit['rej_vol']**2)
for i in range(lsqr.inp.no_grains):
if i+1 not in lsqr.inp.fit['skip']:
print 'Grain %i %i: %e %f' %(i+1,lsqr.inp.nrefl[i],g[i],g[i]/lsqr.inp.nrefl[i])
# give back old values
# lsqr.m.errors = temp2
# lsqr.m.fixed = temp3
# lsqr.mg.tol = temp4
return g
def reject_outliers(lsqr):
# """
# Reject outliers peaks with a distance to the calculated peak position of
# more than lsqr.inp.fit['rej_resmean'] times the mean distance for the given grain
#
# Jette Oddershede, Risoe DTU, May 15 2008
# """
g = grain_values(lsqr)
lsqr.inp.newreject = 0
lsqr.inp.fit['newreject_grain'] = []
#value = []
new = 1
no_ref = []
while new == 1:
new = 0
for i in range(lsqr.inp.no_grains):
#value.append([])
no_ref.append(0)
for k in range(lsqr.inp.fit['no_det']):
no_ref[i] = no_ref[i] + lsqr.inp.nrefl[k][i]
if i+1 in lsqr.inp.fit['skip']:
pass
else:
for k in range(lsqr.inp.fit['no_det']):
for j in range(lsqr.inp.nrefl[k][i]-1,-1,-1): # loop backwards to make pop work
value = fcn.peak(lsqr.m.values['a'],lsqr.m.values['b'],lsqr.m.values['c'],lsqr.m.values['alpha'],lsqr.m.values['beta'],lsqr.m.values['gamma'],
lsqr.inp.h[k][i][j],lsqr.inp.k[k][i][j],lsqr.inp.l[k][i][j],
lsqr.inp.w[k][lsqr.inp.id[k][i][j]],lsqr.inp.dety[k][lsqr.inp.id[k][i][j]],lsqr.inp.detz[k][lsqr.inp.id[k][i][j]],
lsqr.inp.vars[k][i][j],
lsqr.m.values['wx'],lsqr.m.values['wy'],
lsqr.m.values['tx%s' %k],lsqr.m.values['ty%s' %k],lsqr.m.values['tz%s' %k],
lsqr.m.values['py%s' %k],lsqr.m.values['pz%s' %k],
lsqr.m.values['cy%s' %k],lsqr.m.values['cz%s' %k],
lsqr.m.values['L%s' %k],
lsqr.m.values['x%s' %i],lsqr.m.values['y%s' %i],lsqr.m.values['z%s' %i],
lsqr.inp.rod[i][0]+lsqr.m.values['rodx%s' %i],
lsqr.inp.rod[i][1]+lsqr.m.values['rody%s' %i],
lsqr.inp.rod[i][2]+lsqr.m.values['rodz%s' %i],
lsqr.m.values['epsaa%s' %i],lsqr.m.values['epsab%s' %i],lsqr.m.values['epsac%s' %i],
lsqr.m.values['epsbb%s' %i],lsqr.m.values['epsbc%s' %i],lsqr.m.values['epscc%s' %i])
if value > lsqr.inp.fit['rej_resmean']*g[i]/no_ref[i]:
new = 0 # only one cycle of rejection, not until no more as for one detector fitting!!!
print 'Rej peak id %i of det %i grain %i (hkl: %i %i %i, limit: %0.1f): %0.1f' %(lsqr.inp.id[k][i][j],k,i+1,lsqr.inp.h[k][i][j],lsqr.inp.k[k][i][j],lsqr.inp.l[k][i][j],lsqr.inp.fit['rej_resmean'],value*no_ref[i]/g[i])
reject_multidet.reject(lsqr.inp,i,j,k,value*no_ref[i]/g[i])
if 'globals' not in lsqr.inp.fit['goon']:
lsqr.inp.mean_ia = []
for i in range(lsqr.inp.no_grains):
lsqr.inp.mean_ia.append([])
for m in range(lsqr.inp.fit['no_det']):
for j in range(lsqr.inp.nrefl[m][i]):
lsqr.inp.mean_ia[i].append(1)
reject_multidet.mean_ia(lsqr.inp,lsqr.inp.fit['rej_ia'])
lsqr.inp.residual = []
for i in range(lsqr.inp.no_grains):
lsqr.inp.residual.append([])
for m in range(lsqr.inp.fit['no_det']):
for j in range(lsqr.inp.nrefl[m][i]):
lsqr.inp.residual[i].append(1)
reject_multidet.residual(lsqr.inp,lsqr.inp.fit['rej_resmedian'])
# lsqr.inp.volume = []
# for i in range(lsqr.inp.no_grains):
# lsqr.inp.volume.append([])
# for j in range(lsqr.inp.nrefl[i]):
# lsqr.inp.volume[i].append(1)
# reject.intensity(lsqr.inp)
# reject.merge(lsqr.inp)
# reject.multi(lsqr.inp)
for i in range(lsqr.inp.no_grains):
if no_ref[i] < lsqr.inp.fit['min_refl'] and i+1 not in lsqr.inp.fit['skip']:
lsqr.inp.fit['skip'].append(i+1)
lsqr.inp.fit['skip'].sort()