def TestRead(self):
file = open(self.testFile, 'rb')
self.values = cPickle.load(file, encoding='Latin-1')
self.HistoPhases = cPickle.load(file, encoding='Latin-1')
(self.constrDict, self.fixedList,
self.depVarList) = cPickle.load(file, encoding='Latin-1')
self.parmDict = cPickle.load(file, encoding='Latin-1')
self.varylist = cPickle.load(file, encoding='Latin-1')
self.calcControls = cPickle.load(file, encoding='Latin-1')
self.pawleyLookup = cPickle.load(file, encoding='Latin-1')
self.names = self.varylist + self.depVarList
self.use = [False for i in range(len(self.names))]
self.delt = [
max(abs(self.parmDict[name]) * 0.0001, 1e-6) for name in self.names
]
for iname, name in enumerate(self.names):
if name.split(':')[-1] in [
'Shift',
'DisplaceX',
'DisplaceY',
]:
self.delt[iname] = 0.1
file.close()
msg = G2mv.EvaluateMultipliers(self.constrDict, self.parmDict)
if msg:
print('Unable to interpret multiplier(s): ' + msg)
raise Exception
G2mv.GenerateConstraints(self.varylist, self.constrDict,
self.fixedList, self.parmDict)
print(G2mv.VarRemapShow(self.varylist))
print('Dependent Vary List:', self.depVarList)
def OnTestRead(self, event):
dlg = wx.FileDialog(self, 'Open testDeriv.dat file', '.',
'testDeriv.dat')
if self.dirname:
dlg.SetDirectory(self.dirname)
try:
if dlg.ShowModal() == wx.ID_OK:
self.dirname = dlg.GetDirectory()
testFile = dlg.GetPath()
file = open(testFile, 'rb')
self.values = cPickle.load(file)
self.HistoPhases = cPickle.load(file)
(self.constrDict, self.fixedList) = cPickle.load(file)
self.parmDict = cPickle.load(file)
self.varylist = cPickle.load(file)
self.calcControls = cPickle.load(file)
self.pawleyLookup = cPickle.load(file)
self.use = [False for i in range(len(self.varylist))]
self.delt = [
max(abs(self.parmDict[name]) * 0.001, 1e-6)
for name in self.varylist
]
file.close()
groups, parmlist = G2mv.GroupConstraints(self.constrDict)
G2mv.GenerateConstraints(groups, parmlist, self.varylist,
self.constrDict, self.fixedList,
self.parmDict)
self.UpdateControls(event)
print G2mv.VarRemapShow(self.varylist)
finally:
dlg.Destroy()
def RefineCore(Controls,Histograms,Phases,restraintDict,rigidbodyDict,parmDict,varyList,
calcControls,pawleyLookup,ifSeq,printFile,dlg,refPlotUpdate=None):
'''Core optimization routines, shared between SeqRefine and Refine
:returns: 5-tuple of ifOk (bool), Rvals (dict), result, covMatrix, sig
'''
# print 'current',varyList
# for item in parmDict: print item,parmDict[item] ######### show dict just before refinement
G2mv.Map2Dict(parmDict,varyList)
ifPrint = True
if ifSeq:
ifPrint = False
Rvals = {}
while True:
begin = time.time()
values = np.array(G2stMth.Dict2Values(parmDict, varyList))
if np.any(np.isnan(values)):
raise G2obj.G2Exception('ERROR - nan found in LS parameters - use Calculate/View LS parms to locate')
# test code to compute GOF and save for external repeat
#args = ([Histograms,Phases,restraintDict,rigidbodyDict],parmDict,varyList,calcControls,pawleyLookup,dlg)
#print '*** before fit chi**2',np.sum(G2stMth.errRefine(values,*args)**2)
#fl = open('beforeFit.cpickle','wb')
#cPickle.dump(values,fl,1)
#cPickle.dump(args[:-1],fl,1)
#fl.close()
Ftol = Controls['min dM/M']
Xtol = Controls['SVDtol']
Factor = Controls['shift factor']
if 'Jacobian' in Controls['deriv type']:
result = so.leastsq(G2stMth.errRefine,values,Dfun=G2stMth.dervRefine,full_output=True,
ftol=Ftol,col_deriv=True,factor=Factor,
args=([Histograms,Phases,restraintDict,rigidbodyDict],parmDict,varyList,calcControls,pawleyLookup,dlg))
ncyc = int(result[2]['nfev']/2)
if refPlotUpdate is not None: refPlotUpdate(Histograms) # update plot after completion
elif 'analytic Hessian' in Controls['deriv type']:
Lamda = Controls.get('Marquardt',-3)
maxCyc = Controls['max cyc']
result = G2mth.HessianLSQ(G2stMth.errRefine,values,Hess=G2stMth.HessRefine,ftol=Ftol,xtol=Xtol,maxcyc=maxCyc,Print=ifPrint,lamda=Lamda,
args=([Histograms,Phases,restraintDict,rigidbodyDict],parmDict,varyList,calcControls,pawleyLookup,dlg),
refPlotUpdate=refPlotUpdate)
ncyc = result[2]['num cyc']+1
Rvals['lamMax'] = result[2]['lamMax']
Controls['Marquardt'] = -3 #reset to default
elif 'Hessian SVD' in Controls['deriv type']:
maxCyc = Controls['max cyc']
result = G2mth.HessianSVD(G2stMth.errRefine,values,Hess=G2stMth.HessRefine,ftol=Ftol,xtol=Xtol,maxcyc=maxCyc,Print=ifPrint,
args=([Histograms,Phases,restraintDict,rigidbodyDict],parmDict,varyList,calcControls,pawleyLookup,dlg),
refPlotUpdate=refPlotUpdate)
if result[1] is None:
IfOK = False
covMatrix = []
sig = len(varyList)*[None,]
break
ncyc = result[2]['num cyc']+1
else: #'numeric'
result = so.leastsq(G2stMth.errRefine,values,full_output=True,ftol=Ftol,epsfcn=1.e-8,factor=Factor,
args=([Histograms,Phases,restraintDict,rigidbodyDict],parmDict,varyList,calcControls,pawleyLookup,dlg))
ncyc = 1
if len(varyList):
ncyc = int(result[2]['nfev']/len(varyList))
if refPlotUpdate is not None: refPlotUpdate(Histograms) # update plot
# table = dict(zip(varyList,zip(values,result[0],(result[0]-values))))
# for item in table: print item,table[item] #useful debug - are things shifting?
runtime = time.time()-begin
Rvals['SVD0'] = result[2].get('SVD0',0)
Rvals['converged'] = result[2].get('Converged')
Rvals['DelChi2'] = result[2].get('DelChi2',-1.)
Rvals['chisq'] = np.sum(result[2]['fvec']**2)
G2stMth.Values2Dict(parmDict, varyList, result[0])
G2mv.Dict2Map(parmDict,varyList)
Rvals['Nobs'] = Histograms['Nobs']
Rvals['Rwp'] = np.sqrt(Rvals['chisq']/Histograms['sumwYo'])*100. #to %
Rvals['GOF'] = np.sqrt(Rvals['chisq']/(Histograms['Nobs']-len(varyList)))
printFile.write(' Number of function calls: %d No. of observations: %d No. of parameters: %d User rejected: %d Sp. gp. extinct: %d\n'% \
(result[2]['nfev'],Histograms['Nobs'],len(varyList),Histograms['Nrej'],Histograms['Next']))
if ncyc:
printFile.write(' Refinement time = %8.3fs, %8.3fs/cycle, for %d cycles\n'%(runtime,runtime/ncyc,ncyc))
printFile.write(' wR = %7.2f%%, chi**2 = %12.6g, GOF = %6.2f\n'%(Rvals['Rwp'],Rvals['chisq'],Rvals['GOF']))
sig = len(varyList)*[None,]
if 'None' in str(type(result[1])) and ifSeq: #this bails out of a sequential refinement on singular matrix
IfOK = False
covMatrix = []
G2fil.G2Print ('Warning: **** Refinement failed - singular matrix ****')
if 'Hessian' in Controls['deriv type']:
num = len(varyList)-1
for i,val in enumerate(np.flipud(result[2]['psing'])):
if val:
G2fil.G2Print('Bad parameter: '+varyList[num-i],mode='warn')
else:
Ipvt = result[2]['ipvt']
for i,ipvt in enumerate(Ipvt):
if not np.sum(result[2]['fjac'],axis=1)[i]:
G2fil.G2Print('Bad parameter: '+varyList[ipvt-1],mode='warn')
break
IfOK = True
try:
covMatrix = result[1]*Rvals['GOF']**2
sig = np.sqrt(np.diag(covMatrix))
if np.any(np.isnan(sig)) or not sig.shape:
G2fil.G2Print ('*** Least squares aborted - some invalid esds possible ***',mode='error')
# table = dict(zip(varyList,zip(values,result[0],(result[0]-values)/sig)))
# for item in table: print item,table[item] #useful debug - are things shifting?
break #refinement succeeded - finish up!
except TypeError: #result[1] is None on singular matrix or LinAlgError
IfOK = False
if not len(varyList):
covMatrix = []
break
G2fil.G2Print ('**** Refinement failed - singular matrix ****',mode='error')
if 'Hessian' in Controls['deriv type']:
if result[1] is None:
IfOK = False
covMatrix = []
sig = len(varyList)*[None,]
break
num = len(varyList)-1
for i,val in enumerate(np.flipud(result[2]['psing'])):
if val:
G2fil.G2Print ('Removing parameter: '+varyList[num-i])
del(varyList[num-i])
else:
Ipvt = result[2]['ipvt']
for i,ipvt in enumerate(Ipvt):
if not np.sum(result[2]['fjac'],axis=1)[i]:
G2fil.G2Print ('Removing parameter: '+varyList[ipvt-1])
del(varyList[ipvt-1])
break
if IfOK:
G2stMth.GetFobsSq(Histograms,Phases,parmDict,calcControls)
return IfOK,Rvals,result,covMatrix,sig
def SeqRefine(GPXfile,dlg,refPlotUpdate=None):
'''Perform a sequential refinement -- cycles through all selected histgrams,
one at a time
'''
import GSASIImpsubs as G2mp
G2mp.InitMP()
import pytexture as ptx
ptx.pyqlmninit() #initialize fortran arrays for spherical harmonics
printFile = open(ospath.splitext(GPXfile)[0]+'.lst','w')
G2fil.G2Print ('Starting Sequential Refinement')
G2stIO.ShowBanner(printFile)
Controls = G2stIO.GetControls(GPXfile)
G2stIO.ShowControls(Controls,printFile,SeqRef=True)
restraintDict = G2stIO.GetRestraints(GPXfile)
Histograms,Phases = G2stIO.GetUsedHistogramsAndPhases(GPXfile)
if not Phases:
G2fil.G2Print (' *** ERROR - you have no phases to refine! ***')
G2fil.G2Print (' *** Refine aborted ***')
return False,'No phases'
if not Histograms:
G2fil.G2Print (' *** ERROR - you have no data to refine with! ***')
G2fil.G2Print (' *** Refine aborted ***')
return False,'No data'
rigidbodyDict = G2stIO.GetRigidBodies(GPXfile)
rbIds = rigidbodyDict.get('RBIds',{'Vector':[],'Residue':[]})
rbVary,rbDict = G2stIO.GetRigidBodyModels(rigidbodyDict,pFile=printFile)
G2mv.InitVars()
(Natoms,atomIndx,phaseVary,phaseDict,pawleyLookup,FFtables,BLtables,MFtables,
maxSSwave) = G2stIO.GetPhaseData(Phases,restraintDict,rbIds,
Print=False,pFile=printFile,seqRef=True)
for item in phaseVary:
if '::A0' in item:
G2fil.G2Print ('**** WARNING - lattice parameters should not be refined in a sequential refinement ****')
G2fil.G2Print ('**** instead use the Dij parameters for each powder histogram ****')
return False,'Lattice parameter refinement error - see console message'
if '::C(' in item:
G2fil.G2Print ('**** WARNING - phase texture parameters should not be refined in a sequential refinement ****')
G2fil.G2Print ('**** instead use the C(L,N) parameters for each powder histogram ****')
return False,'Phase texture refinement error - see console message'
if 'Seq Data' in Controls:
histNames = Controls['Seq Data']
else: # patch from before Controls['Seq Data'] was implemented?
histNames = G2stIO.GetHistogramNames(GPXfile,['PWDR',])
if Controls.get('Reverse Seq'):
histNames.reverse()
SeqResult = G2stIO.GetSeqResult(GPXfile)
# SeqResult = {'SeqPseudoVars':{},'SeqParFitEqList':[]}
Histo = {}
NewparmDict = {}
G2stIO.SetupSeqSavePhases(GPXfile)
for ihst,histogram in enumerate(histNames):
if GSASIIpath.GetConfigValue('Show_timing'): t1 = time.time()
G2fil.G2Print('\nRefining with '+str(histogram))
G2mv.InitVars()
(Natoms,atomIndx,phaseVary,phaseDict,pawleyLookup,
FFtables,BLtables,MFtables,maxSSwave) = G2stIO.GetPhaseData(
Phases,restraintDict,rbIds,
Print=False,pFile=printFile,seqRef=True)
ifPrint = False
if dlg:
dlg.SetTitle('Residual for histogram '+str(ihst))
calcControls = {}
calcControls['atomIndx'] = atomIndx
calcControls['Natoms'] = Natoms
calcControls['FFtables'] = FFtables
calcControls['BLtables'] = BLtables
calcControls['MFtables'] = MFtables
calcControls['maxSSwave'] = maxSSwave
if histogram not in Histograms:
G2fil.G2Print("Error: not found!")
continue
#TODO - implement "Fix FXU" for seq refinement here - done?
hId = Histograms[histogram]['hId']
redphaseVary = phaseCheck(phaseVary,Phases,histogram)
Histo = {histogram:Histograms[histogram],}
hapVary,hapDict,controlDict = G2stIO.GetHistogramPhaseData(Phases,Histo,Print=False)
calcControls.update(controlDict)
histVary,histDict,controlDict = G2stIO.GetHistogramData(Histo,False)
calcControls.update(controlDict)
varyList = rbVary+redphaseVary+hapVary+histVary
# if not ihst:
# save the initial vary list, but without histogram numbers on parameters
saveVaryList = varyList[:]
for i,item in enumerate(saveVaryList):
items = item.split(':')
if items[1]:
items[1] = ''
item = ':'.join(items)
saveVaryList[i] = item
if not ihst:
SeqResult['varyList'] = saveVaryList
else:
SeqResult['varyList'] = list(set(SeqResult['varyList']+saveVaryList))
parmDict = {}
parmDict.update(rbDict)
parmDict.update(phaseDict)
parmDict.update(hapDict)
parmDict.update(histDict)
if Controls['Copy2Next']: # update with parms from last histogram
#parmDict.update(NewparmDict) # don't use in case extra entries would cause a problem
for parm in NewparmDict:
if parm in parmDict:
parmDict[parm] = NewparmDict[parm]
elif histogram in SeqResult: # update phase from last seq ref
NewparmDict = SeqResult[histogram].get('parmDict',{})
for parm in NewparmDict:
if '::' in parm and parm in parmDict:
parmDict[parm] = NewparmDict[parm]
G2stIO.GetFprime(calcControls,Histo)
# do constraint processing
#reload(G2mv) # debug
constrDict,fixedList = G2stIO.GetConstraints(GPXfile)
varyListStart = tuple(varyList) # save the original varyList before dependent vars are removed
msg = G2mv.EvaluateMultipliers(constrDict,parmDict)
if msg:
return False,'Unable to interpret multiplier(s): '+msg
try:
groups,parmlist = G2mv.GenerateConstraints(varyList,constrDict,fixedList,parmDict,SeqHist=hId)
# if GSASIIpath.GetConfigValue('debug'): print("DBG_"+
# G2mv.VarRemapShow(varyList,True))
# print('DependentVars',G2mv.GetDependentVars())
# print('IndependentVars',G2mv.GetIndependentVars())
constraintInfo = (groups,parmlist,constrDict,fixedList,ihst)
except G2mv.ConstraintException:
G2fil.G2Print (' *** ERROR - your constraints are internally inconsistent ***')
#errmsg, warnmsg = G2mv.CheckConstraints(varyList,constrDict,fixedList)
#print 'Errors',errmsg
#if warnmsg: print 'Warnings',warnmsg
return False,' Constraint error'
#print G2mv.VarRemapShow(varyList)
if not ihst:
# first histogram to refine against
firstVaryList = []
for item in varyList:
items = item.split(':')
if items[1]:
items[1] = ''
item = ':'.join(items)
firstVaryList.append(item)
newVaryList = firstVaryList
else:
newVaryList = []
for item in varyList:
items = item.split(':')
if items[1]:
items[1] = ''
item = ':'.join(items)
newVaryList.append(item)
if newVaryList != firstVaryList and Controls['Copy2Next']:
# variable lists are expected to match between sequential refinements when Copy2Next is on
#print '**** ERROR - variable list for this histogram does not match previous'
#print ' Copy of variables is not possible'
#print '\ncurrent histogram',histogram,'has',len(newVaryList),'variables'
combined = list(set(firstVaryList+newVaryList))
c = [var for var in combined if var not in newVaryList]
p = [var for var in combined if var not in firstVaryList]
G2fil.G2Print('*** Variables change ***')
for typ,vars in [('Removed',c),('Added',p)]:
line = ' '+typ+': '
if vars:
for var in vars:
if len(line) > 70:
G2fil.G2Print(line)
line = ' '
line += var + ', '
else:
line += 'none, '
G2fil.G2Print(line[:-2])
firstVaryList = newVaryList
ifSeq = True
printFile.write('\n Refinement results for histogram: %s\n'%histogram)
printFile.write(135*'-'+'\n')
try:
IfOK,Rvals,result,covMatrix,sig = RefineCore(Controls,Histo,Phases,restraintDict,
rigidbodyDict,parmDict,varyList,calcControls,pawleyLookup,ifSeq,printFile,dlg,
refPlotUpdate=refPlotUpdate)
G2fil.G2Print (' wR = %7.2f%%, chi**2 = %12.6g, reduced chi**2 = %6.2f, last delta chi = %.4f'%(
Rvals['Rwp'],Rvals['chisq'],Rvals['GOF']**2,Rvals['DelChi2']))
# add the uncertainties into the esd dictionary (sigDict)
if not IfOK:
G2fil.G2Print('***** Sequential refinement failed at histogram '+histogram,mode='warn')
break
sigDict = dict(zip(varyList,sig))
# the uncertainties for dependent constrained parms into the esd dict
sigDict.update(G2mv.ComputeDepESD(covMatrix,varyList,parmDict))
# a dict with values & esds for dependent (constrained) parameters - avoid extraneous holds
depParmDict = {i:(parmDict[i],sigDict[i]) for i in varyListStart if i in sigDict and i not in varyList}
newCellDict = copy.deepcopy(G2stMth.GetNewCellParms(parmDict,varyList))
newAtomDict = copy.deepcopy(G2stMth.ApplyXYZshifts(parmDict,varyList))
histRefData = {
'variables':result[0],'varyList':varyList,'sig':sig,'Rvals':Rvals,
'varyListStart':varyListStart,
'covMatrix':covMatrix,'title':histogram,'newAtomDict':newAtomDict,
'newCellDict':newCellDict,'depParmDict':depParmDict,
'constraintInfo':constraintInfo,
'parmDict':parmDict}
SeqResult[histogram] = histRefData
G2stMth.ApplyRBModels(parmDict,Phases,rigidbodyDict,True)
# G2stIO.SetRigidBodyModels(parmDict,sigDict,rigidbodyDict,printFile)
G2stIO.SetHistogramPhaseData(parmDict,sigDict,Phases,Histo,None,ifPrint,printFile)
G2stIO.SetHistogramData(parmDict,sigDict,Histo,None,ifPrint,printFile)
G2stIO.SaveUpdatedHistogramsAndPhases(GPXfile,Histo,Phases,rigidbodyDict,histRefData)
NewparmDict = {}
# make dict of varied parameters in current histogram, renamed to
# next histogram, for use in next refinement.
if Controls['Copy2Next'] and ihst < len(histNames)-1:
hId = Histo[histogram]['hId'] # current histogram
nexthId = Histograms[histNames[ihst+1]]['hId']
for parm in set(list(varyList)+list(varyListStart)):
items = parm.split(':')
if len(items) < 3:
continue
if str(hId) in items[1]:
items[1] = str(nexthId)
newparm = ':'.join(items)
NewparmDict[newparm] = parmDict[parm]
else:
if items[2].startswith('dA'): parm = parm.replace(':dA',':A')
NewparmDict[parm] = parmDict[parm]
except G2obj.G2RefineCancel as Msg:
printFile.close()
G2fil.G2Print (' ***** Refinement stopped *****')
return False,Msg.msg
except G2obj.G2Exception as Msg: # cell metric error, others?
printFile.close()
G2fil.G2Print (' ***** Refinement error *****')
return False,Msg.msg
if GSASIIpath.GetConfigValue('Show_timing'):
t2 = time.time()
G2fil.G2Print("Fit step time {:.2f} sec.".format(t2-t1))
t1 = t2
SeqResult['histNames'] = [itm for itm in G2stIO.GetHistogramNames(GPXfile,['PWDR',]) if itm in SeqResult.keys()]
G2stIO.SetSeqResult(GPXfile,Histograms,SeqResult)
printFile.close()
G2fil.G2Print (' Sequential refinement results are in file: '+ospath.splitext(GPXfile)[0]+'.lst')
G2fil.G2Print (' ***** Sequential refinement successful *****')
return True,'Success'
def Refine(GPXfile,dlg=None,makeBack=True,refPlotUpdate=None):
'Global refinement -- refines to minimize against all histograms'
import GSASIImpsubs as G2mp
G2mp.InitMP()
import pytexture as ptx
ptx.pyqlmninit() #initialize fortran arrays for spherical harmonics
printFile = open(ospath.splitext(GPXfile)[0]+'.lst','w')
G2stIO.ShowBanner(printFile)
varyList = []
parmDict = {}
G2mv.InitVars()
Controls = G2stIO.GetControls(GPXfile)
G2stIO.ShowControls(Controls,printFile)
calcControls = {}
calcControls.update(Controls)
constrDict,fixedList = G2stIO.GetConstraints(GPXfile)
restraintDict = G2stIO.GetRestraints(GPXfile)
Histograms,Phases = G2stIO.GetUsedHistogramsAndPhases(GPXfile)
if not Phases:
G2fil.G2Print (' *** ERROR - you have no phases to refine! ***')
G2fil.G2Print (' *** Refine aborted ***')
return False,'No phases'
if not Histograms:
G2fil.G2Print (' *** ERROR - you have no data to refine with! ***')
G2fil.G2Print (' *** Refine aborted ***')
return False,'No data'
rigidbodyDict = G2stIO.GetRigidBodies(GPXfile)
rbIds = rigidbodyDict.get('RBIds',{'Vector':[],'Residue':[]})
rbVary,rbDict = G2stIO.GetRigidBodyModels(rigidbodyDict,pFile=printFile)
(Natoms,atomIndx,phaseVary,phaseDict,pawleyLookup,FFtables,BLtables,MFtables,
maxSSwave) = G2stIO.GetPhaseData(Phases,restraintDict,rbIds,pFile=printFile)
calcControls['atomIndx'] = atomIndx
calcControls['Natoms'] = Natoms
calcControls['FFtables'] = FFtables
calcControls['BLtables'] = BLtables
calcControls['MFtables'] = MFtables
calcControls['maxSSwave'] = maxSSwave
hapVary,hapDict,controlDict = G2stIO.GetHistogramPhaseData(Phases,Histograms,pFile=printFile)
TwConstr,TwFixed = G2stIO.makeTwinFrConstr(Phases,Histograms,hapVary)
constrDict += TwConstr
fixedList += TwFixed
calcControls.update(controlDict)
histVary,histDict,controlDict = G2stIO.GetHistogramData(Histograms,pFile=printFile)
calcControls.update(controlDict)
varyList = rbVary+phaseVary+hapVary+histVary
parmDict.update(rbDict)
parmDict.update(phaseDict)
parmDict.update(hapDict)
parmDict.update(histDict)
G2stIO.GetFprime(calcControls,Histograms)
# do constraint processing
varyListStart = tuple(varyList) # save the original varyList before dependent vars are removed
msg = G2mv.EvaluateMultipliers(constrDict,parmDict)
if msg:
return False,'Unable to interpret multiplier(s): '+msg
try:
G2mv.GenerateConstraints(varyList,constrDict,fixedList,parmDict)
#print(G2mv.VarRemapShow(varyList))
#print('DependentVars',G2mv.GetDependentVars())
#print('IndependentVars',G2mv.GetIndependentVars())
except G2mv.ConstraintException:
G2fil.G2Print (' *** ERROR - your constraints are internally inconsistent ***')
#errmsg, warnmsg = G2mv.CheckConstraints(varyList,constrDict,fixedList)
#print 'Errors',errmsg
#if warnmsg: print 'Warnings',warnmsg
return False,' Constraint error'
# print G2mv.VarRemapShow(varyList)
ifSeq = False
printFile.write('\n Refinement results:\n')
printFile.write(135*'-'+'\n')
try:
covData = {}
IfOK,Rvals,result,covMatrix,sig = RefineCore(Controls,Histograms,Phases,restraintDict,
rigidbodyDict,parmDict,varyList,calcControls,pawleyLookup,ifSeq,printFile,dlg,
refPlotUpdate=refPlotUpdate)
if IfOK:
sigDict = dict(zip(varyList,sig))
newCellDict = G2stMth.GetNewCellParms(parmDict,varyList)
newAtomDict = G2stMth.ApplyXYZshifts(parmDict,varyList)
covData = {'variables':result[0],'varyList':varyList,'sig':sig,'Rvals':Rvals,
'varyListStart':varyListStart,
'covMatrix':covMatrix,'title':GPXfile,'newAtomDict':newAtomDict,
'newCellDict':newCellDict,'freshCOV':True}
# add the uncertainties into the esd dictionary (sigDict)
sigDict.update(G2mv.ComputeDepESD(covMatrix,varyList,parmDict))
G2mv.PrintIndependentVars(parmDict,varyList,sigDict,pFile=printFile)
G2stMth.ApplyRBModels(parmDict,Phases,rigidbodyDict,True)
G2stIO.SetRigidBodyModels(parmDict,sigDict,rigidbodyDict,printFile)
G2stIO.SetPhaseData(parmDict,sigDict,Phases,rbIds,covData,restraintDict,printFile)
G2stIO.SetHistogramPhaseData(parmDict,sigDict,Phases,Histograms,calcControls['FFtables'],pFile=printFile)
G2stIO.SetHistogramData(parmDict,sigDict,Histograms,calcControls['FFtables'],pFile=printFile)
G2stIO.SetUsedHistogramsAndPhases(GPXfile,Histograms,Phases,rigidbodyDict,covData,makeBack)
printFile.close()
G2fil.G2Print (' Refinement results are in file: '+ospath.splitext(GPXfile)[0]+'.lst')
G2fil.G2Print (' ***** Refinement successful *****')
else:
G2fil.G2Print ('****ERROR - Refinement failed')
raise G2obj.G2Exception('****ERROR - Refinement failed')
except G2obj.G2RefineCancel as Msg:
printFile.close()
G2fil.G2Print (' ***** Refinement stopped *****')
return False,Msg.msg
except G2obj.G2Exception as Msg: # cell metric error, others?
printFile.close()
G2fil.G2Print (' ***** Refinement error *****')
return False,Msg.msg
#for testing purposes, create a file for testderiv
if GSASIIpath.GetConfigValue('debug'): # and IfOK:
#needs: values,HistoPhases,parmDict,varylist,calcControls,pawleyLookup
fl = open(ospath.splitext(GPXfile)[0]+'.testDeriv','wb')
cPickle.dump(result[0],fl,1)
cPickle.dump([Histograms,Phases,restraintDict,rigidbodyDict],fl,1)
cPickle.dump([constrDict,fixedList,G2mv.GetDependentVars()],fl,1)
cPickle.dump(parmDict,fl,1)
cPickle.dump(varyList,fl,1)
cPickle.dump(calcControls,fl,1)
cPickle.dump(pawleyLookup,fl,1)
fl.close()
if dlg:
return True,Rvals
def RefineCore(Controls,Histograms,Phases,restraintDict,rigidbodyDict,parmDict,varyList,
calcControls,pawleyLookup,ifPrint,printFile,dlg):
'Core optimization routines, shared between SeqRefine and Refine'
# print 'current',varyList
# for item in parmDict: print item,parmDict[item] ######### show dict just before refinement
G2mv.Map2Dict(parmDict,varyList)
Rvals = {}
while True:
begin = time.time()
values = np.array(G2stMth.Dict2Values(parmDict, varyList))
# test code to compute GOF and save for external repeat
#args = ([Histograms,Phases,restraintDict,rigidbodyDict],parmDict,varyList,calcControls,pawleyLookup,dlg)
#print '*** before fit chi**2',np.sum(G2stMth.errRefine(values,*args)**2)
#fl = open('beforeFit.cpickle','wb')
#import cPickle
#cPickle.dump(values,fl,1)
#cPickle.dump(args[:-1],fl,1)
#fl.close()
Ftol = Controls['min dM/M']
Factor = Controls['shift factor']
if 'Jacobian' in Controls['deriv type']:
result = so.leastsq(G2stMth.errRefine,values,Dfun=G2stMth.dervRefine,full_output=True,
ftol=Ftol,col_deriv=True,factor=Factor,
args=([Histograms,Phases,restraintDict,rigidbodyDict],parmDict,varyList,calcControls,pawleyLookup,dlg))
ncyc = int(result[2]['nfev']/2)
elif 'Hessian' in Controls['deriv type']:
maxCyc = Controls['max cyc']
result = G2mth.HessianLSQ(G2stMth.errRefine,values,Hess=G2stMth.HessRefine,ftol=Ftol,maxcyc=maxCyc,Print=ifPrint,
args=([Histograms,Phases,restraintDict,rigidbodyDict],parmDict,varyList,calcControls,pawleyLookup,dlg))
ncyc = result[2]['num cyc']+1
Rvals['lamMax'] = result[2]['lamMax']
else: #'numeric'
result = so.leastsq(G2stMth.errRefine,values,full_output=True,ftol=Ftol,epsfcn=1.e-8,factor=Factor,
args=([Histograms,Phases,restraintDict,rigidbodyDict],parmDict,varyList,calcControls,pawleyLookup,dlg))
ncyc = 1
if len(varyList):
ncyc = int(result[2]['nfev']/len(varyList))
# table = dict(zip(varyList,zip(values,result[0],(result[0]-values))))
# for item in table: print item,table[item] #useful debug - are things shifting?
runtime = time.time()-begin
Rvals['converged'] = result[2].get('Converged')
Rvals['DelChi2'] = result[2].get('DelChi2',-1.)
Rvals['chisq'] = np.sum(result[2]['fvec']**2)
G2stMth.Values2Dict(parmDict, varyList, result[0])
G2mv.Dict2Map(parmDict,varyList)
Rvals['Nobs'] = Histograms['Nobs']
Rvals['Rwp'] = np.sqrt(Rvals['chisq']/Histograms['sumwYo'])*100. #to %
Rvals['GOF'] = np.sqrt(Rvals['chisq']/(Histograms['Nobs']-len(varyList)))
print >>printFile,' Number of function calls:',result[2]['nfev'], \
' No. of observations: ',Histograms['Nobs'],' No. of parameters: ',len(varyList), \
' User rejected: ',Histograms['Nrej'],' Sp. gp. extinct: ',Histograms['Next']
print >>printFile,' Refinement time = %8.3fs, %8.3fs/cycle, for %d cycles'%(runtime,runtime/ncyc,ncyc)
print >>printFile,' wR = %7.2f%%, chi**2 = %12.6g, GOF = %6.2f'%(Rvals['Rwp'],Rvals['chisq'],Rvals['GOF'])
IfOK = True
try:
covMatrix = result[1]*Rvals['GOF']**2
sig = np.sqrt(np.diag(covMatrix))
if np.any(np.isnan(sig)):
print '*** Least squares aborted - some invalid esds possible ***'
# table = dict(zip(varyList,zip(values,result[0],(result[0]-values)/sig)))
# for item in table: print item,table[item] #useful debug - are things shifting?
break #refinement succeeded - finish up!
except TypeError,FloatingPointError: #result[1] is None on singular matrix
IfOK = False
if not len(varyList):
covMatrix = []
sig = []
break
print '**** Refinement failed - singular matrix ****'
if 'Hessian' in Controls['deriv type']:
num = len(varyList)-1
for i,val in enumerate(np.flipud(result[2]['psing'])):
if val:
print 'Removing parameter: ',varyList[num-i]
del(varyList[num-i])
else:
Ipvt = result[2]['ipvt']
for i,ipvt in enumerate(Ipvt):
if not np.sum(result[2]['fjac'],axis=1)[i]:
print 'Removing parameter: ',varyList[ipvt-1]
del(varyList[ipvt-1])
break
def SeqRefine(GPXfile,dlg):
'''Perform a sequential refinement -- cycles through all selected histgrams,
one at a time
'''
import pytexture as ptx
ptx.pyqlmninit() #initialize fortran arrays for spherical harmonics
printFile = open(ospath.splitext(GPXfile)[0]+'.lst','w')
print 'Starting Sequential Refinement'
G2stIO.ShowBanner(printFile)
Controls = G2stIO.GetControls(GPXfile)
G2stIO.ShowControls(Controls,printFile,SeqRef=True)
restraintDict = G2stIO.GetRestraints(GPXfile)
Histograms,Phases = G2stIO.GetUsedHistogramsAndPhases(GPXfile)
if not Phases:
print ' *** ERROR - you have no phases to refine! ***'
print ' *** Refine aborted ***'
return False,'No phases'
if not Histograms:
print ' *** ERROR - you have no data to refine with! ***'
print ' *** Refine aborted ***'
return False,'No data'
rigidbodyDict = G2stIO.GetRigidBodies(GPXfile)
rbIds = rigidbodyDict.get('RBIds',{'Vector':[],'Residue':[]})
rbVary,rbDict = G2stIO.GetRigidBodyModels(rigidbodyDict,pFile=printFile)
Natoms,atomIndx,phaseVary,phaseDict,pawleyLookup,FFtables,BLtables,maxSSwave = G2stIO.GetPhaseData(Phases,restraintDict,rbIds,False,printFile,seqRef=True)
for item in phaseVary:
if '::A0' in item:
print '**** WARNING - lattice parameters should not be refined in a sequential refinement ****'
print '**** instead use the Dij parameters for each powder histogram ****'
return False,'Lattice parameter refinement error - see console message'
if '::C(' in item:
print '**** WARNING - phase texture parameters should not be refined in a sequential refinement ****'
print '**** instead use the C(L,N) parameters for each powder histogram ****'
return False,'Phase texture refinement error - see console message'
if 'Seq Data' in Controls:
histNames = Controls['Seq Data']
else:
histNames = G2stIO.GetHistogramNames(GPXfile,['PWDR',])
if 'Reverse Seq' in Controls:
if Controls['Reverse Seq']:
histNames.reverse()
SeqResult = {'histNames':histNames}
makeBack = True
Histo = {}
NewparmDict = {}
for ihst,histogram in enumerate(histNames):
print('Refining with '+str(histogram))
ifPrint = False
if dlg:
dlg.SetTitle('Residual for histogram '+str(ihst))
calcControls = {}
calcControls['atomIndx'] = atomIndx
calcControls['Natoms'] = Natoms
calcControls['FFtables'] = FFtables
calcControls['BLtables'] = BLtables
calcControls['maxSSwave'] = maxSSwave
Histo = {histogram:Histograms[histogram],}
hapVary,hapDict,controlDict = G2stIO.GetHistogramPhaseData(Phases,Histo,Print=False)
calcControls.update(controlDict)
histVary,histDict,controlDict = G2stIO.GetHistogramData(Histo,False)
calcControls.update(controlDict)
varyList = rbVary+phaseVary+hapVary+histVary
if not ihst:
# save the initial vary list, but without histogram numbers on parameters
saveVaryList = varyList[:]
for i,item in enumerate(saveVaryList):
items = item.split(':')
if items[1]:
items[1] = ''
item = ':'.join(items)
saveVaryList[i] = item
SeqResult['varyList'] = saveVaryList
origvaryList = varyList[:]
parmDict = {}
parmDict.update(phaseDict)
parmDict.update(hapDict)
parmDict.update(histDict)
if Controls['Copy2Next']:
parmDict.update(NewparmDict)
G2stIO.GetFprime(calcControls,Histo)
# do constraint processing
#reload(G2mv) # debug
G2mv.InitVars()
constrDict,fixedList = G2stIO.GetConstraints(GPXfile)
varyListStart = tuple(varyList) # save the original varyList before dependent vars are removed
try:
groups,parmlist = G2mv.GroupConstraints(constrDict)
G2mv.GenerateConstraints(groups,parmlist,varyList,constrDict,fixedList,parmDict,SeqHist=ihst)
constraintInfo = (groups,parmlist,constrDict,fixedList,ihst)
except:
print ' *** ERROR - your constraints are internally inconsistent ***'
#errmsg, warnmsg = G2mv.CheckConstraints(varyList,constrDict,fixedList)
#print 'Errors',errmsg
#if warnmsg: print 'Warnings',warnmsg
return False,' Constraint error'
#print G2mv.VarRemapShow(varyList)
if not ihst:
# first histogram to refine against
firstVaryList = []
for item in varyList:
items = item.split(':')
if items[1]:
items[1] = ''
item = ':'.join(items)
firstVaryList.append(item)
newVaryList = firstVaryList
else:
newVaryList = []
for item in varyList:
items = item.split(':')
if items[1]:
items[1] = ''
item = ':'.join(items)
newVaryList.append(item)
if newVaryList != firstVaryList and Controls['Copy2Next']:
# variable lists are expected to match between sequential refinements when Copy2Next is on
print '**** ERROR - variable list for this histogram does not match previous'
print ' Copy of variables is not possible'
print '\ncurrent histogram',histogram,'has',len(newVaryList),'variables'
combined = list(set(firstVaryList+newVaryList))
c = [var for var in combined if var not in newVaryList]
p = [var for var in combined if var not in firstVaryList]
line = 'Variables in previous but not in current: '
if c:
for var in c:
if len(line) > 100:
print line
line = ' '
line += var + ', '
else:
line += 'none'
print line
print '\nPrevious refinement has',len(firstVaryList),'variables'
line = 'Variables in current but not in previous: '
if p:
for var in p:
if len(line) > 100:
print line
line = ' '
line += var + ', '
else:
line += 'none'
print line
return False,line
ifPrint = False
print >>printFile,'\n Refinement results for histogram: v'+histogram
print >>printFile,135*'-'
try:
IfOK,Rvals,result,covMatrix,sig = RefineCore(Controls,Histo,Phases,restraintDict,
rigidbodyDict,parmDict,varyList,calcControls,pawleyLookup,ifPrint,printFile,dlg)
print ' wR = %7.2f%%, chi**2 = %12.6g, reduced chi**2 = %6.2f, last delta chi = %.4f'%(
Rvals['Rwp'],Rvals['chisq'],Rvals['GOF']**2,Rvals['DelChi2'])
# add the uncertainties into the esd dictionary (sigDict)
sigDict = dict(zip(varyList,sig))
# the uncertainties for dependent constrained parms into the esd dict
sigDict.update(G2mv.ComputeDepESD(covMatrix,varyList,parmDict))
# a dict with values & esds for dependent (constrained) parameters
depParmDict = {i:(parmDict[i],sigDict[i]) for i in varyListStart
if i not in varyList}
newCellDict = copy.deepcopy(G2stMth.GetNewCellParms(parmDict,varyList))
newAtomDict = copy.deepcopy(G2stMth.ApplyXYZshifts(parmDict,varyList))
histRefData = {
'variables':result[0],'varyList':varyList,'sig':sig,'Rvals':Rvals,
'varyListStart':varyListStart,
'covMatrix':covMatrix,'title':histogram,'newAtomDict':newAtomDict,
'newCellDict':newCellDict,'depParmDict':depParmDict,
'constraintInfo':constraintInfo,
'parmDict':parmDict}
SeqResult[histogram] = histRefData
G2stMth.ApplyRBModels(parmDict,Phases,rigidbodyDict,True)
# G2stIO.SetRigidBodyModels(parmDict,sigDict,rigidbodyDict,printFile)
G2stIO.SetHistogramPhaseData(parmDict,sigDict,Phases,Histo,None,ifPrint,printFile)
G2stIO.SetHistogramData(parmDict,sigDict,Histo,None,ifPrint,printFile)
G2stIO.SetUsedHistogramsAndPhases(GPXfile,Histo,Phases,rigidbodyDict,histRefData,makeBack)
makeBack = False
NewparmDict = {}
# make dict of varied parameters in current histogram, renamed to
# next histogram, for use in next refinement.
if Controls['Copy2Next'] and ihst < len(histNames)-1:
hId = Histo[histogram]['hId'] # current histogram
nexthId = Histograms[histNames[ihst+1]]['hId']
for parm in set(list(varyList)+list(varyListStart)):
items = parm.split(':')
if len(items) < 3: continue
if str(hId) in items[1]:
items[1] = str(nexthId)
newparm = ':'.join(items)
NewparmDict[newparm] = parmDict[parm]
except G2obj.G2Exception,Msg:
printFile.close()
print ' ***** Refinement aborted *****'
return False,Msg.msg
def Refine(GPXfile,dlg):
'Global refinement -- refines to minimize against all histograms'
import pytexture as ptx
ptx.pyqlmninit() #initialize fortran arrays for spherical harmonics
printFile = open(ospath.splitext(GPXfile)[0]+'.lst','w')
G2stIO.ShowBanner(printFile)
varyList = []
parmDict = {}
G2mv.InitVars()
Controls = G2stIO.GetControls(GPXfile)
G2stIO.ShowControls(Controls,printFile)
calcControls = {}
calcControls.update(Controls)
constrDict,fixedList = G2stIO.GetConstraints(GPXfile)
restraintDict = G2stIO.GetRestraints(GPXfile)
Histograms,Phases = G2stIO.GetUsedHistogramsAndPhases(GPXfile)
if not Phases:
print ' *** ERROR - you have no phases to refine! ***'
print ' *** Refine aborted ***'
return False,'No phases'
if not Histograms:
print ' *** ERROR - you have no data to refine with! ***'
print ' *** Refine aborted ***'
return False,'No data'
rigidbodyDict = G2stIO.GetRigidBodies(GPXfile)
rbIds = rigidbodyDict.get('RBIds',{'Vector':[],'Residue':[]})
rbVary,rbDict = G2stIO.GetRigidBodyModels(rigidbodyDict,pFile=printFile)
Natoms,atomIndx,phaseVary,phaseDict,pawleyLookup,FFtables,BLtables,maxSSwave = G2stIO.GetPhaseData(Phases,restraintDict,rbIds,pFile=printFile)
calcControls['atomIndx'] = atomIndx
calcControls['Natoms'] = Natoms
calcControls['FFtables'] = FFtables
calcControls['BLtables'] = BLtables
calcControls['maxSSwave'] = maxSSwave
hapVary,hapDict,controlDict = G2stIO.GetHistogramPhaseData(Phases,Histograms,pFile=printFile)
calcControls.update(controlDict)
histVary,histDict,controlDict = G2stIO.GetHistogramData(Histograms,pFile=printFile)
calcControls.update(controlDict)
varyList = rbVary+phaseVary+hapVary+histVary
parmDict.update(rbDict)
parmDict.update(phaseDict)
parmDict.update(hapDict)
parmDict.update(histDict)
G2stIO.GetFprime(calcControls,Histograms)
# do constraint processing
varyListStart = tuple(varyList) # save the original varyList before dependent vars are removed
try:
groups,parmlist = G2mv.GroupConstraints(constrDict)
G2mv.GenerateConstraints(groups,parmlist,varyList,constrDict,fixedList,parmDict)
#print G2mv.VarRemapShow(varyList)
#print 'DependentVars',G2mv.GetDependentVars()
#print 'IndependentVars',G2mv.GetIndependentVars()
except:
print ' *** ERROR - your constraints are internally inconsistent ***'
#errmsg, warnmsg = G2mv.CheckConstraints(varyList,constrDict,fixedList)
#print 'Errors',errmsg
#if warnmsg: print 'Warnings',warnmsg
return False,' Constraint error'
# print G2mv.VarRemapShow(varyList)
ifPrint = True
print >>printFile,'\n Refinement results:'
print >>printFile,135*'-'
try:
IfOK,Rvals,result,covMatrix,sig = RefineCore(Controls,Histograms,Phases,restraintDict,
rigidbodyDict,parmDict,varyList,calcControls,pawleyLookup,ifPrint,printFile,dlg)
sigDict = dict(zip(varyList,sig))
newCellDict = G2stMth.GetNewCellParms(parmDict,varyList)
newAtomDict = G2stMth.ApplyXYZshifts(parmDict,varyList)
covData = {'variables':result[0],'varyList':varyList,'sig':sig,'Rvals':Rvals,
'varyListStart':varyListStart,
'covMatrix':covMatrix,'title':GPXfile,'newAtomDict':newAtomDict,
'newCellDict':newCellDict,'freshCOV':True}
# add the uncertainties into the esd dictionary (sigDict)
sigDict.update(G2mv.ComputeDepESD(covMatrix,varyList,parmDict))
G2mv.PrintIndependentVars(parmDict,varyList,sigDict,pFile=printFile)
G2stMth.ApplyRBModels(parmDict,Phases,rigidbodyDict,True)
G2stIO.SetRigidBodyModels(parmDict,sigDict,rigidbodyDict,printFile)
G2stIO.SetPhaseData(parmDict,sigDict,Phases,rbIds,covData,restraintDict,printFile)
G2stIO.SetHistogramPhaseData(parmDict,sigDict,Phases,Histograms,calcControls['FFtables'],pFile=printFile)
G2stIO.SetHistogramData(parmDict,sigDict,Histograms,calcControls['FFtables'],pFile=printFile)
G2stIO.SetUsedHistogramsAndPhases(GPXfile,Histograms,Phases,rigidbodyDict,covData)
printFile.close()
print ' Refinement results are in file: '+ospath.splitext(GPXfile)[0]+'.lst'
print ' ***** Refinement successful *****'
except G2obj.G2Exception,Msg:
printFile.close()
return False,Msg.msg