def funcFitSmoothList(argsOfList, expList, xanesList, smoothType, targetFunc,
normType):
l2 = []
diffs = []
es = []
for j in range(len(expList)):
exp = expList[j]
arg = getOneArg(argsOfList, exp, smoothType)
norm = getNorm(normType, argsOfList, arg)
smoothed_xanes, normOut = funcFitSmoothHelper(arg, xanesList[j],
smoothType, exp, norm)
# print(normOut)
i = (exp.fit_intervals['smooth'][0] <= exp.xanes.energy) & (
exp.xanes.energy <= exp.fit_intervals['smooth'][1])
diff = abs(smoothed_xanes.absorb[i] - exp.xanes.absorb[i])**2
diffs.append(diff)
es.append(exp.xanes.energy[i])
partial_func_val = np.sqrt(utils.integral(exp.xanes.energy[i], diff))
l2.append(partial_func_val)
if len(expList) == 1: return l2[0]
l2 = np.array(l2)
if targetFunc == 'mean': return np.mean(l2)
elif targetFunc == 'max(l2)': return np.max(l2)
elif targetFunc == 'l2(max)':
e = es[0]
newDiffs = np.zeros([len(expList), e.size])
newDiffs[0] = diffs[0]
for j in range(1, len(expList)):
newDiffs[j] = np.interp(e, es[j], diffs[j])
maxDiff = np.max(newDiffs, axis=0)
return np.sqrt(utils.integral(e, maxDiff))
else:
assert False, 'Unknown target func'
def simpleSmooth(e, xanes, sigma):
new_xanes = np.zeros(e.shape)
for i in range(e.size):
kern = kernelCauchy(e, e[i], sigma)
norm = utils.integral(e, kern)
if norm == 0: norm = 1
new_xanes[i] = utils.integral(e, xanes * kern) / norm
return new_xanes
def smooth_my_fdmnes(e, xanes, Gamma_hole, Ecent, Elarg, Gamma_max, Efermi):
xanes = np.copy(xanes)
xanes[e < Efermi] = 0
new_xanes = np.zeros(e.shape)
sigma = YvesWidth(e, Gamma_hole, Ecent, Elarg, Gamma_max, Efermi)
for i in range(e.size):
kern = kernelCauchy(e, e[i], sigma[i])
norm = utils.integral(e, kern)
new_xanes[i] = utils.integral(e, xanes * kern) / norm
return e, new_xanes
def smooth_linear_conv(e, xanes, Gamma_hole, Gamma_max, Efermi):
xanes = np.copy(xanes)
xanes[e < Efermi] = 0
new_xanes = np.zeros(e.shape)
sigma = Gamma_hole + (Gamma_max - Gamma_hole) * (e - e[0]) / (e[-1] - e[0])
sigma[sigma <= 0] = 1e-3
for i in range(e.size):
kern = kernelCauchy(e, e[i], sigma[i])
norm = utils.integral(e, kern)
new_xanes[i] = utils.integral(e, xanes * kern) / norm
return e, new_xanes
def generalSmooth(e, xanes, sigma):
eleft = np.linspace(e[0] - 10, e[0] - (e[1] - e[0]), 10)
xleft = np.zeros(eleft.shape)
eright = np.linspace(e[-1] + (e[-1] - e[-2]), e[-1] + 50, 10)
xright = np.zeros(eleft.shape) + xanes[-1]
e_new = np.hstack((eleft, e, eright))
xanes = np.hstack((xleft, xanes, xright))
new_xanes = np.zeros(e.shape)
for i in range(e.size):
kern = kernelCauchy(e_new, e[i], sigma[i])
norm = utils.integral(e_new, kern)
new_xanes[i] = utils.integral(e_new, xanes * kern) / norm
return e, new_xanes
def smooth_Muller(e, xanes, group, Gamma_hole, Efermi, alpha1, alpha2, alpha3):
xanes = np.copy(xanes)
# xanes = simpleSmooth(e, xanes, Gamma_hole)
xanes[e < Efermi] = 0
new_xanes = np.zeros(e.shape)
sigma = MullerWidth(e, group, Gamma_hole, Efermi, alpha1, alpha2, alpha3)
for i in range(e.size):
kern = kernelCauchy(e, e[i], sigma[i])
norm = utils.integral(e, kern)
new_xanes[i] = utils.integral(e, xanes * kern) / norm
# print(e[i], sigma[i])
# exit(0)
return e, new_xanes
def funcFitSmooth(args, xanes, smoothType, exp, norm=None):
smoothed_xanes, _ = funcFitSmoothHelper(args, xanes, smoothType, exp, norm)
i = (exp.fit_intervals['smooth'][0] <= exp.xanes.energy) & (
exp.xanes.energy <= exp.fit_intervals['smooth'][1])
return np.sqrt(
utils.integral(exp.xanes.energy[i],
abs(smoothed_xanes.absorb[i] - exp.xanes.absorb[i])**2))
def smooth_piecewise(e0, xanes, Gamma_hole, Gamma_max, Ecent):
eleft = np.linspace(e0[0] - 10, e0[0] - (e0[1] - e0[0]), 10)
xleft = np.zeros(eleft.shape)
eright = np.linspace(e0[-1] + (e0[-1] - e0[-2]), e0[-1] + 50, 10)
xright = np.zeros(eleft.shape) + xanes[-1]
e = np.hstack((eleft, e0, eright))
xanes = np.hstack((xleft, xanes, xright))
new_xanes = np.zeros(e0.shape)
sigma = np.zeros(e0.shape) + Gamma_hole
sigma[e0 > Ecent] = Gamma_max
sigma[sigma <= 0] = 1e-3
for i in range(e0.size):
kern = kernelCauchy(e, e0[i], sigma[i])
norm = utils.integral(e, kern)
new_xanes[i] = utils.integral(e, xanes * kern) / norm
return e0, new_xanes
def deconvolve(e, xanes, smooth_params):
xanes = utils.simpleSmooth(e, xanes, 1)
Gamma_hole = value(smooth_params, 'Gamma_hole')
Ecent = value(smooth_params, 'Ecent')
Elarg = value(smooth_params, 'Elarg')
Gamma_max = value(smooth_params, 'Gamma_max')
Efermi = value(smooth_params, 'Efermi')
n = e.size
A = np.zeros([n, n])
for i in range(n):
#sigma = superWidth(e[i], Gamma_hole, Ecent, Elarg, Gamma_max, Efermi)
sigma = 5
kern = kernel(e, e[i], sigma)
A[i] = kern / utils.integral(e, kern)
res, _, _, s = np.linalg.lstsq(A, xanes, rcond=0.001)
return res
def fitSmooth(expList,
xanesList0,
smoothType='fdmnes',
fixParamNames=[],
commonParams0={},
targetFunc='max(l2)',
plotTrace=False,
crossValidationExp=None,
crossValidationXanes=None,
minimizeMethodType='seq',
useGridSearch=True,
useRefinement=True):
xanesList = copy.deepcopy(xanesList0)
if 'norm' in fixParamNames: normType = 'fixed'
else:
if 'norm' in commonParams0: normType = 'variableCommon'
else: normType = 'variablePrivate'
folder = tempfile.mkdtemp(dir='./tmp', prefix='smooth_')
for i in range(len(expList)):
os.makedirs(folder + '/' + expList[i].name)
if xanesList[i].folder is not None:
copyfile(xanesList[i].folder + '/out.txt',
folder + '/' + expList[i].name + '/out.txt')
xanesList[i].folder = folder + '/' + expList[i].name
if xanesList[i].molecula is not None:
xanesList[i].molecula.export_xyz(xanesList[i].folder +
'/molecula.xyz')
arg0 = createArg(expList, smoothType, fixParamNames, commonParams0)
# 0.0001
fmin, smooth_params, trace = optimize.minimizePokoord(
funcFitSmoothList,
arg0,
minDeltaFunc=1e-4,
enableOutput=False,
methodType=minimizeMethodType,
parallel=False,
useRefinement=useRefinement,
useGridSearch=useGridSearch,
returnTrace=True,
f_kwargs={
'expList': expList,
'xanesList': xanesList,
'smoothType': smoothType,
'targetFunc': targetFunc,
'normType': normType
})
with open(folder + '/func_smooth_value.txt', 'w') as f:
json.dump(fmin, f)
with open(folder + '/args_smooth.txt', 'w') as f:
json.dump(smooth_params, f)
with open(folder + '/args_smooth_human.txt', 'w') as f:
f.write(arg2string(smooth_params))
# выдаем предостережение, если достигли границы одного из параметров
for p in smooth_params:
d = p['rightBorder'] - p['leftBorder']
if (abs(p['value'] - p['leftBorder']) <= p['step']) or (
abs(p['value'] - p['rightBorder']) <= p['step']):
print('Warning: parameter ' + p['paramName'] + '=' +
str(p['value']) + ' is near border of domain [' +
str(p['leftBorder']) + '; ' + str(p['rightBorder']) + ']')
# считаем по отдельности размазку каждого xanes
smoothed_xanes = []
argsOfList = copy.deepcopy(smooth_params)
for j in range(len(expList)):
exp = expList[j]
arg = getOneArg(argsOfList, exp, smoothType)
norm = getNorm(normType, argsOfList, arg)
fdmnes_xan, _ = funcFitSmoothHelper(arg, xanesList[j], smoothType, exp,
norm)
smoothed_xanes.append(fdmnes_xan)
with open(folder + '/' + expList[j].name + '/args_smooth.txt',
'w') as f:
json.dump(arg, f)
with open(folder + '/' + expList[j].name + '/args_smooth_human.txt',
'w') as f:
f.write(arg2string(arg))
shift = value(arg, 'shift')
fdmnes.plotToFolder(folder + '/' + expList[j].name,
exp,
xanesList[j],
fdmnes_xan,
append=getSmoothWidth(smoothType,
exp.xanes.energy - shift,
arg))
ind = (exp.fit_intervals['smooth'][0] <= exp.xanes.energy) & (
exp.xanes.energy <= exp.fit_intervals['smooth'][1])
partial_fmin = np.sqrt(
utils.integral(
exp.xanes.energy[ind],
abs(fdmnes_xan.absorb[ind] - exp.xanes.absorb[ind])**2))
with open(
folder + '/' + expList[j].name +
'/func_smooth_partial_value.txt', 'w') as f:
json.dump(partial_fmin, f)
plotSmoothWidthToFolder(smoothType, exp.xanes.energy[ind] - shift, arg,
folder + '/' + expList[j].name)
if plotTrace:
privateFuncData = np.zeros([len(expList), len(trace)])
norms = np.zeros([len(expList), len(trace)])
for j in range(len(trace)):
step = trace[j]
argsOfList = step[0]
for i in range(len(expList)):
arg = getOneArg(argsOfList, expList[i], smoothType)
norm = getNorm(normType, argsOfList, arg)
privateFuncData[i, j] = funcFitSmooth(arg, xanesList[i],
smoothType, expList[i],
norm)
_, norms[i,
j] = funcFitSmoothHelper(arg, xanesList[i],
smoothType, expList[i], norm)
fig, ax = plt.subplots()
steps = np.arange(1, len(trace) + 1)
targetFuncData = [step[1] for step in trace]
for i in range(len(expList)):
ax.plot(steps, privateFuncData[i], label=expList[i].name)
ax.plot(steps, targetFuncData, label='target')
if crossValidationExp is not None:
crossValidationXanes = copy.deepcopy(crossValidationXanes)
if smoothType == 'fdmnes':
folder2 = folder + '/CV_' + crossValidationExp.name
os.makedirs(folder2)
if crossValidationXanes.folder is not None:
copyfile(crossValidationXanes.folder + '/out.txt',
folder2 + '/out.txt')
crossValidationXanes.folder = folder2
crossValidationData = []
for step_i in range(len(trace)):
step = trace[step_i]
argsOfList = step[0]
arg = getOneArg(argsOfList, crossValidationExp, smoothType)
for a in arg:
if value(argsOfList, a['paramName']) is None:
mean = 0
for exp in expList:
mean += value(argsOfList,
exp.name + '_' + a['paramName'])
mean /= len(expList)
print(
'Warning: parameter ' + a['paramName'] +
' is not common. Take mean for cross validation: '
+ str(mean))
a['value'] = mean
fmin1 = funcFitSmooth(arg, crossValidationXanes, smoothType,
crossValidationExp,
np.mean(norms[:, step_i]))
crossValidationData.append(fmin1)
if step_i == len(trace) - 1:
with open(folder2 + '/func_smooth_check_value.txt',
'w') as f:
json.dump(fmin1, f)
with open(folder2 + '/args_smooth.txt', 'w') as f:
json.dump(arg, f)
fdmnes_xan, _ = funcFitSmoothHelper(
arg, crossValidationXanes, smoothType,
crossValidationExp, np.mean(norms[:, step_i]))
fdmnes.plotToFolder(folder2, crossValidationExp,
crossValidationXanes, fdmnes_xan)
#print(np.mean(norms[:,step_i]))
ax.plot(steps, crossValidationData, label='check')
ax.set_xscale('log')
ax.legend()
fig.set_size_inches((16 / 3 * 2, 9 / 3 * 2))
fig.savefig(folder + '/trace.png')
plt.close(fig)
return smoothed_xanes, smooth_params, fmin