def getNorm(normType, argsOfList, arg):
if normType == 'variablePrivate': norm = None
else:
if normType == 'variableCommon': norm = value(argsOfList, 'norm')
else:
norm = value(
arg, 'norm'
) # normType == 'fixed' => норма должна сидеть в параметрах по умолчанию
assert norm is not None, 'When norm is fixed it must be included in deafaul smooth parameters'
return norm
def deconvolve(e, xanes, smooth_params):
xanes = utils.simpleSmooth(e, xanes, 1)
Gamma_hole = value(smooth_params, 'Gamma_hole')
Ecent = value(smooth_params, 'Ecent')
Elarg = value(smooth_params, 'Elarg')
Gamma_max = value(smooth_params, 'Gamma_max')
Efermi = value(smooth_params, 'Efermi')
n = e.size
A = np.zeros([n, n])
for i in range(n):
#sigma = superWidth(e[i], Gamma_hole, Ecent, Elarg, Gamma_max, Efermi)
sigma = 5
kern = kernel(e, e[i], sigma)
A[i] = kern / utils.integral(e, kern)
res, _, _, s = np.linalg.lstsq(A, xanes, rcond=0.001)
return res
def funcFitSmoothHelper(args, xanes, smoothType, exp, norm=None):
shift = value(args, 'shift')
# t1 = time.time()
if smoothType == 'fdmnes':
if xanes.folder is None:
xanes.folder = tempfile.mkdtemp(dir='./tmp', prefix='smooth_')
if not os.path.exists(xanes.folder + '/out.txt'):
xanes.save(xanes.folder + '/out.txt',
exp.defaultSmoothParams.fdmnesSmoothHeader)
Gamma_hole, Ecent, Elarg, Gamma_max, Efermi = getSmoothParams(
args, ['Gamma_hole', 'Ecent', 'Elarg', 'Gamma_max', 'Efermi'])
fdmnes_en1, res = smooth_fdmnes(xanes.energy,
xanes.absorb,
Gamma_hole,
Ecent,
Elarg,
Gamma_max,
Efermi,
folder=xanes.folder)
elif smoothType == 'my_fdmnes':
Gamma_hole, Ecent, Elarg, Gamma_max, Efermi = getSmoothParams(
args, ['Gamma_hole', 'Ecent', 'Elarg', 'Gamma_max', 'Efermi'])
fdmnes_en1, res = smooth_my_fdmnes(xanes.energy, xanes.absorb,
Gamma_hole, Ecent, Elarg, Gamma_max,
Efermi)
elif smoothType == 'linear':
Gamma_hole, Gamma_max, Efermi = getSmoothParams(
args, ['Gamma_hole', 'Gamma_max', 'Efermi'])
fdmnes_en1, res = smooth_linear_conv(xanes.energy, xanes.absorb,
Gamma_hole, Gamma_max, Efermi)
elif smoothType == 'Muller':
group, Efermi, Gamma_hole, alpha1, alpha2, alpha3 = getSmoothParams(
args,
['group', 'Efermi', 'Gamma_hole', 'alpha1', 'alpha2', 'alpha3'])
fdmnes_en1, res = smooth_Muller(xanes.energy, xanes.absorb, group,
Gamma_hole, Efermi, alpha1, alpha2,
alpha3)
elif smoothType == 'piecewise':
Gamma_hole, Gamma_max, Ecent = getSmoothParams(
args, ['Gamma_hole', 'Gamma_max', 'Ecent'])
fdmnes_en1, res = smooth_piecewise(xanes.energy, xanes.absorb,
Gamma_hole, Gamma_max, Ecent)
elif smoothType == 'multi_piecewise':
sigma = getSmoothWidth(smoothType, xanes.energy, args)
fdmnes_en1, res = generalSmooth(xanes.energy, xanes.absorb, sigma)
elif smoothType == 'spline':
sigma = getSmoothWidth(smoothType, xanes.energy, args)
fdmnes_en1, res = generalSmooth(xanes.energy, xanes.absorb, sigma)
else:
error('Unknown smooth type ' + smoothType)
# t2 = time.time()
# print("Smooth time=", t2 - t1)
res, norm = utils.fit_to_experiment_by_norm_or_regression_mult_only(
exp.xanes.energy, exp.xanes.absorb, exp.fit_intervals['norm'],
fdmnes_en1, res, shift, norm)
return utils.Xanes(exp.xanes.energy, res, xanes.folder), norm
def plotToFolder(folder, exp, xanes0, smoothed_xanes, append=None):
exp_e = exp.xanes.energy
exp_xanes = exp.xanes.absorb
shiftIsAbsolute = exp.defaultSmoothParams.shiftIsAbsolute
search_shift_level = exp.defaultSmoothParams.search_shift_level
fit_interval = exp.fit_intervals['norm']
fig, ax = plt.subplots()
fdmnes_en = smoothed_xanes.energy
fdmnes_xan = smoothed_xanes.absorb
with open(folder + '/args_smooth.txt', 'r') as f:
smooth_params = json.load(f)
shift = optimize.value(smooth_params, 'shift')
if not shiftIsAbsolute:
shift += utils.getInitialShift(exp_e, exp_xanes, fdmnes_en, fdmnes_xan,
search_shift_level)
fdmnes_xan0 = utils.fit_arg_to_experiment(xanes0.energy,
exp_e,
xanes0.absorb,
shift,
lastValueNorm=True)
ax.plot(exp_e - shift, fdmnes_xan0, label='initial')
ax.plot(exp_e - shift, fdmnes_xan, label='convolution')
ax.plot(exp_e - shift, exp_xanes, c='k', label="Experiment")
if append is not None:
e_fdmnes = exp_e - shift
ax2 = ax.twinx()
ax2.plot(e_fdmnes, append, c='r', label='Smooth width')
ax2.legend()
# ax3 = ax.twinx()
# ax3.plot(e_fdmnes[1:], (append[1:]-append[:-1])/(e_fdmnes[1:]-e_fdmnes[:-1]), c='g', label='Diff smooth width')
# ax3.legend()
ax.set_xlim([fit_interval[0] - shift, fit_interval[1] - shift])
ax.set_ylim([0, np.max(exp_xanes) * 1.2])
ax.set_xlabel("Energy")
ax.set_ylabel("XANES")
ax.legend()
fig.set_size_inches((16 / 3 * 2, 9 / 3 * 2))
fig.savefig(folder + '/xanes.png')
plt.close(fig)
def fitBySliders(geometryParams, xanes, exp):
# normalize params
geometryParamsMin = np.min(geometryParams.values, axis=0)
geometryParamsMax = np.max(geometryParams.values, axis=0)
geometryParams = 2 * (geometryParams - geometryParamsMin) / (
geometryParamsMax - geometryParamsMin) - 1
# machine learning estimator training
estimator = makeQuadric(RidgeCV(alphas=[0.01, 0.1, 1, 10, 100]))
estimator.fit(geometryParams.values, xanes.values)
# need for smoothing by fdmnes
xanesFolder = tempfile.mkdtemp(prefix='smooth_')
e_names = xanes.columns
xanes_energy = np.array(
[float(e_names[i][2:]) for i in range(e_names.size)])
fig, ax = plt.subplots()
ax2 = ax.twinx()
fig.set_size_inches((16 * 0.6, 9 * 0.8))
def plotXanes(**params):
geomArg = np.array([params[pName] for pName in geometryParams.columns
]).reshape([1, geometryParams.shape[1]])
geomArg = 2 * (geomArg - geometryParamsMin) / (geometryParamsMax -
geometryParamsMin) - 1
# prediction
absorbPrediction = estimator.predict(geomArg)[0]
# smoothing
xanesPrediction = utils.Xanes(xanes_energy, absorbPrediction, None,
None)
xanesPrediction.save(xanesFolder + '/out.txt',
exp.defaultSmoothParams.fdmnesSmoothHeader)
_, smoothedXanes = smoothLib.smooth_fdmnes(
None, None, params['Gamma_hole'], params['Ecent'], params['Elarg'],
params['Gamma_max'], params['Efermi'], xanesFolder)
#plotting
shift = params['shift']
exp_e = exp.xanes.energy
exp_xanes = exp.xanes.absorb
e_fdmnes = exp_e - shift
absorbPredictionNormalized = utils.fit_arg_to_experiment(
xanesPrediction.energy,
exp_e,
xanesPrediction.absorb,
shift,
lastValueNorm=True)
smoothedPredictionNormalized = utils.fit_arg_to_experiment(
xanesPrediction.energy,
exp_e,
smoothedXanes,
shift,
lastValueNorm=True)
if params['notConvoluted']:
ax.plot(e_fdmnes, absorbPredictionNormalized, label='initial')
ax.clear()
ax.plot(e_fdmnes, smoothedPredictionNormalized, label='convolution')
ax.plot(e_fdmnes, exp_xanes, c='k', label="Experiment")
if params['smoothWidth']:
smoothWidth = smoothLib.YvesWidth(e_fdmnes, params['Gamma_hole'],
params['Ecent'], params['Elarg'],
params['Gamma_max'],
params['Efermi'])
ax2.clear()
ax2.plot(e_fdmnes, smoothWidth, c='r', label='Smooth width')
ax2.legend()
ax.set_xlim([params['energyRange'][0], params['energyRange'][1]])
ax.set_ylim([0, np.max(exp_xanes) * 1.2])
ax.set_xlabel("Energy")
ax.set_ylabel("Absorb")
ax.legend()
# plt.show()
controls = []
o = 'vertical'
for pName in exp.geometryParamRanges:
p0 = exp.geometryParamRanges[pName][0]
p1 = exp.geometryParamRanges[pName][1]
controls.append(
widgets.FloatSlider(description=pName,
min=p0,
max=p1,
step=(p1 - p0) / 30,
value=(p0 + p1) / 2,
orientation=o))
shift = optimize.value(exp.defaultSmoothParams['fdmnes'], 'shift')
controls.append(
widgets.FloatSlider(description='shift',
min=shift - 10.0,
max=shift + 10.0,
step=0.3,
value=shift,
orientation=o))
controls.append(
widgets.FloatSlider(description='Gamma_hole',
min=0.1,
max=10,
step=0.2,
value=2,
orientation=o))
controls.append(
widgets.FloatSlider(description='Ecent',
min=0,
max=100,
step=1,
value=50,
orientation=o))
controls.append(
widgets.FloatSlider(description='Elarg',
min=0,
max=100,
step=1,
value=50,
orientation=o))
controls.append(
widgets.FloatSlider(description='Gamma_max',
min=1,
max=100,
step=1,
value=15,
orientation=o))
controls.append(
widgets.FloatSlider(description='Efermi',
min=-30,
max=30,
step=1,
value=0,
orientation=o))
controls.append(widgets.Checkbox(description='smoothWidth', value=True))
controls.append(widgets.Checkbox(description='notConvoluted', value=True))
e0 = xanes_energy[0]
e1 = xanes_energy[-1]
controls.append(
widgets.FloatRangeSlider(description='energyRange',
min=e0,
max=e1,
step=(e1 - e0) / 30,
value=[e0, e1],
orientation='horizontal'))
ui = widgets.HBox(tuple(controls))
ui.layout.flex_flow = 'row wrap'
ui.layout.justify_content = 'space-between'
ui.layout.align_items = 'flex-start'
ui.layout.align_content = 'flex-start'
controlsDict = {}
for c in controls:
controlsDict[c.description] = c
out = widgets.interactive_output(plotXanes, controlsDict)
# out.layout.min_height = '400px'
display(ui, out)
def fitSmooth(expList,
xanesList0,
smoothType='fdmnes',
fixParamNames=[],
commonParams0={},
targetFunc='max(l2)',
plotTrace=False,
crossValidationExp=None,
crossValidationXanes=None,
minimizeMethodType='seq',
useGridSearch=True,
useRefinement=True):
xanesList = copy.deepcopy(xanesList0)
if 'norm' in fixParamNames: normType = 'fixed'
else:
if 'norm' in commonParams0: normType = 'variableCommon'
else: normType = 'variablePrivate'
folder = tempfile.mkdtemp(dir='./tmp', prefix='smooth_')
for i in range(len(expList)):
os.makedirs(folder + '/' + expList[i].name)
if xanesList[i].folder is not None:
copyfile(xanesList[i].folder + '/out.txt',
folder + '/' + expList[i].name + '/out.txt')
xanesList[i].folder = folder + '/' + expList[i].name
if xanesList[i].molecula is not None:
xanesList[i].molecula.export_xyz(xanesList[i].folder +
'/molecula.xyz')
arg0 = createArg(expList, smoothType, fixParamNames, commonParams0)
# 0.0001
fmin, smooth_params, trace = optimize.minimizePokoord(
funcFitSmoothList,
arg0,
minDeltaFunc=1e-4,
enableOutput=False,
methodType=minimizeMethodType,
parallel=False,
useRefinement=useRefinement,
useGridSearch=useGridSearch,
returnTrace=True,
f_kwargs={
'expList': expList,
'xanesList': xanesList,
'smoothType': smoothType,
'targetFunc': targetFunc,
'normType': normType
})
with open(folder + '/func_smooth_value.txt', 'w') as f:
json.dump(fmin, f)
with open(folder + '/args_smooth.txt', 'w') as f:
json.dump(smooth_params, f)
with open(folder + '/args_smooth_human.txt', 'w') as f:
f.write(arg2string(smooth_params))
# выдаем предостережение, если достигли границы одного из параметров
for p in smooth_params:
d = p['rightBorder'] - p['leftBorder']
if (abs(p['value'] - p['leftBorder']) <= p['step']) or (
abs(p['value'] - p['rightBorder']) <= p['step']):
print('Warning: parameter ' + p['paramName'] + '=' +
str(p['value']) + ' is near border of domain [' +
str(p['leftBorder']) + '; ' + str(p['rightBorder']) + ']')
# считаем по отдельности размазку каждого xanes
smoothed_xanes = []
argsOfList = copy.deepcopy(smooth_params)
for j in range(len(expList)):
exp = expList[j]
arg = getOneArg(argsOfList, exp, smoothType)
norm = getNorm(normType, argsOfList, arg)
fdmnes_xan, _ = funcFitSmoothHelper(arg, xanesList[j], smoothType, exp,
norm)
smoothed_xanes.append(fdmnes_xan)
with open(folder + '/' + expList[j].name + '/args_smooth.txt',
'w') as f:
json.dump(arg, f)
with open(folder + '/' + expList[j].name + '/args_smooth_human.txt',
'w') as f:
f.write(arg2string(arg))
shift = value(arg, 'shift')
fdmnes.plotToFolder(folder + '/' + expList[j].name,
exp,
xanesList[j],
fdmnes_xan,
append=getSmoothWidth(smoothType,
exp.xanes.energy - shift,
arg))
ind = (exp.fit_intervals['smooth'][0] <= exp.xanes.energy) & (
exp.xanes.energy <= exp.fit_intervals['smooth'][1])
partial_fmin = np.sqrt(
utils.integral(
exp.xanes.energy[ind],
abs(fdmnes_xan.absorb[ind] - exp.xanes.absorb[ind])**2))
with open(
folder + '/' + expList[j].name +
'/func_smooth_partial_value.txt', 'w') as f:
json.dump(partial_fmin, f)
plotSmoothWidthToFolder(smoothType, exp.xanes.energy[ind] - shift, arg,
folder + '/' + expList[j].name)
if plotTrace:
privateFuncData = np.zeros([len(expList), len(trace)])
norms = np.zeros([len(expList), len(trace)])
for j in range(len(trace)):
step = trace[j]
argsOfList = step[0]
for i in range(len(expList)):
arg = getOneArg(argsOfList, expList[i], smoothType)
norm = getNorm(normType, argsOfList, arg)
privateFuncData[i, j] = funcFitSmooth(arg, xanesList[i],
smoothType, expList[i],
norm)
_, norms[i,
j] = funcFitSmoothHelper(arg, xanesList[i],
smoothType, expList[i], norm)
fig, ax = plt.subplots()
steps = np.arange(1, len(trace) + 1)
targetFuncData = [step[1] for step in trace]
for i in range(len(expList)):
ax.plot(steps, privateFuncData[i], label=expList[i].name)
ax.plot(steps, targetFuncData, label='target')
if crossValidationExp is not None:
crossValidationXanes = copy.deepcopy(crossValidationXanes)
if smoothType == 'fdmnes':
folder2 = folder + '/CV_' + crossValidationExp.name
os.makedirs(folder2)
if crossValidationXanes.folder is not None:
copyfile(crossValidationXanes.folder + '/out.txt',
folder2 + '/out.txt')
crossValidationXanes.folder = folder2
crossValidationData = []
for step_i in range(len(trace)):
step = trace[step_i]
argsOfList = step[0]
arg = getOneArg(argsOfList, crossValidationExp, smoothType)
for a in arg:
if value(argsOfList, a['paramName']) is None:
mean = 0
for exp in expList:
mean += value(argsOfList,
exp.name + '_' + a['paramName'])
mean /= len(expList)
print(
'Warning: parameter ' + a['paramName'] +
' is not common. Take mean for cross validation: '
+ str(mean))
a['value'] = mean
fmin1 = funcFitSmooth(arg, crossValidationXanes, smoothType,
crossValidationExp,
np.mean(norms[:, step_i]))
crossValidationData.append(fmin1)
if step_i == len(trace) - 1:
with open(folder2 + '/func_smooth_check_value.txt',
'w') as f:
json.dump(fmin1, f)
with open(folder2 + '/args_smooth.txt', 'w') as f:
json.dump(arg, f)
fdmnes_xan, _ = funcFitSmoothHelper(
arg, crossValidationXanes, smoothType,
crossValidationExp, np.mean(norms[:, step_i]))
fdmnes.plotToFolder(folder2, crossValidationExp,
crossValidationXanes, fdmnes_xan)
#print(np.mean(norms[:,step_i]))
ax.plot(steps, crossValidationData, label='check')
ax.set_xscale('log')
ax.legend()
fig.set_size_inches((16 / 3 * 2, 9 / 3 * 2))
fig.savefig(folder + '/trace.png')
plt.close(fig)
return smoothed_xanes, smooth_params, fmin
def getSmoothParams(arg, names):
res = ()
for name in names:
res = res + (value(arg, name), )
return res
def plotDiffToFolder(folder0,
exp_e0,
exp_xanes0,
folder,
exp_e,
exp_xanes,
fit_interval,
shiftIsAbsolute=True,
search_shift_level=None,
fitBy='RegressionMultOnly'):
fig, ax = plt.subplots()
figDiff, axDiff = plt.subplots()
fdmnes_en_init, fdmnes_xan_init, _ = parse_one_folder(folder)
fdmnes_en_init0, fdmnes_xan_init0, _ = parse_one_folder(folder0)
fdmnes_en_conv, fdmnes_xan_conv = parse_convolution(folder)
fdmnes_en_conv0, fdmnes_xan_conv0 = parse_convolution(folder0)
with open(folder + '/args_smooth.txt', 'r') as f:
smooth_params = json.load(f)
with open(folder0 + '/args_smooth.txt', 'r') as f:
smooth_params0 = json.load(f)
shift = optimize.value(smooth_params, 'shift')
shift0 = optimize.value(smooth_params0, 'shift')
assert shift == shift0
if not shiftIsAbsolute:
shift += utils.getInitialShift(exp_e, exp_xanes, fdmnes_en_conv,
fdmnes_xan_conv, search_shift_level)
shift0 += utils.getInitialShift(exp_e0, exp_xanes0, fdmnes_en_conv0,
fdmnes_xan_conv0, search_shift_level)
fdmnes_xan_init = utils.fit_arg_to_experiment(fdmnes_en_init,
exp_e,
fdmnes_xan_init,
shift,
lastValueNorm=True)
fdmnes_xan_init0 = utils.fit_arg_to_experiment(fdmnes_en_init0,
exp_e0,
fdmnes_xan_init0,
shift0,
lastValueNorm=True)
axDiff.plot(exp_e - shift,
fdmnes_xan_init - fdmnes_xan_init0,
label='initDiff') # здесь важно чтобы shift==shift0
ax.plot(exp_e - shift, fdmnes_xan_init, label='init')
ax.plot(exp_e - shift, fdmnes_xan_init0, label='initBare')
if fitBy == 'RegressionMultOnly':
fdmnes_xan_conv = utils.fit_arg_to_experiment(fdmnes_en_conv, exp_e,
fdmnes_xan_conv, shift)
fdmnes_xan_conv0 = utils.fit_arg_to_experiment(fdmnes_en_conv0, exp_e0,
fdmnes_xan_conv0,
shift0)
fdmnes_xan_conv = utils.fit_by_regression_mult_only(
exp_e, exp_xanes, fdmnes_xan_conv, fit_interval)
fdmnes_xan_conv0 = utils.fit_by_regression_mult_only(
exp_e0, exp_xanes0, fdmnes_xan_conv0, fit_interval)
elif fitBy == 'Regression':
fdmnes_xan_conv = utils.fit_arg_to_experiment(fdmnes_en_conv, exp_e,
fdmnes_xan_conv, shift)
fdmnes_xan_conv0 = utils.fit_arg_to_experiment(fdmnes_en_conv0, exp_e0,
fdmnes_xan_conv0,
shift0)
fdmnes_xan_conv = utils.fit_by_regression(exp_e, exp_xanes,
fdmnes_xan_conv,
fit_interval)
fdmnes_xan_conv0 = utils.fit_by_regression(exp_e0, exp_xanes0,
fdmnes_xan_conv0,
fit_interval)
elif fitBy == 'FixedNorm':
norm = optimize.value(smooth_params, 'norm')
norm0 = optimize.value(smooth_params0, 'norm')
fdmnes_xan_conv = utils.fit_to_experiment_by_norm_or_regression_mult_only(
exp_e, exp_xanes, fit_interval, fdmnes_en_conv, fdmnes_xan_conv,
shift, norm)
fdmnes_xan_conv0 = utils.fit_to_experiment_by_norm_or_regression_mult_only(
exp_e0, exp_xanes0, fit_interval, fdmnes_en_conv0,
fdmnes_xan_conv0, shift0, norm0)
else:
assert False, 'Unknown fitBy = ' + fitBy
axDiff.plot(exp_e - shift,
fdmnes_xan_conv - fdmnes_xan_conv0,
label='convDiff') # здесь важно чтобы shift==shift0
ax.plot(exp_e - shift, fdmnes_xan_conv, label='conv')
ax.plot(exp_e - shift, fdmnes_xan_conv0, label='convBare')
axDiff.plot(exp_e - shift, exp_xanes - exp_xanes0, c='k',
label="ExpDiff") # здесь важно чтобы shift==shift0
ax.plot(exp_e - shift, exp_xanes, label="Exp")
ax.plot(exp_e - shift, exp_xanes0, label="ExpBare")
axDiff.set_xlim([fit_interval[0] - shift, fit_interval[1] - shift
]) # здесь важно чтобы shift==shift0
ax.set_xlim([fit_interval[0] - shift, fit_interval[1] - shift])
m = np.min(exp_xanes - exp_xanes0)
M = np.max(exp_xanes - exp_xanes0)
mM = M - m
axDiff.set_ylim([m - mM / 5, M + mM / 5])
ax.set_ylim([0, np.max(exp_xanes) * 1.5])
axDiff.set_xlabel("Energy")
ax.set_xlabel("Energy")
axDiff.set_ylabel("XANES Diff")
ax.set_ylabel("XANES")
axDiff.legend()
ax.legend()
figDiff.set_size_inches((16 / 3 * 2, 9 / 3 * 2))
fig.set_size_inches((16 / 3 * 2, 9 / 3 * 2))
figDiff.savefig(folder + '/xanesDiff.png')
fig.savefig(folder + '/xanes.png')
plt.close(figDiff)
plt.close(fig)