def cost(x, axis=None, samples=Ns):
"""upper bound on expected model error, for truth and Learn(data)
Inputs:
x: list of LearnedModel hyperparameters
axis: int, the index of y on which to find bound (all, by default)
samples: int, number of samples, for a non-deterministic OUQ model
Returns:
upper bound on expected value of model error
"""
# CASE 1: F(x|a) = F'(x|a'). Tune A for optimal G.
args = dict(hidden_layer_sizes=hidden_layers(*x),
max_iter=100,
n_iter_no_change=5,
solver='lbfgs',
learning_rate_init=0.001) #FIXME: max_iter=1000
from sklearn.neural_network import MLPRegressor
from sklearn.preprocessing import StandardScaler
from ml import Estimator, MLData, improve_score
kwds = dict(estimator=MLPRegressor(**args), transform=StandardScaler())
# iteratively improve estimator
mlp = Estimator(**kwds) #FIXME: traintest so train != test ?
best = improve_score(mlp,
MLData(data.coords, data.coords, data.values,
data.values),
tries=1,
verbose=True) #FIXME: tries = 2
mlkw = dict(estimator=best.estimator, transform=best.transform)
#print('building estimator G(x) from truth data...')
surrogate = LearnedModel('surrogate', nx=nx, ny=ny, data=truth, **mlkw)
#print('building UQ model of model error...')
error = ErrorModel('error', model=truth, surrogate=surrogate)
rnd = Ns if error.rnd else None
#print('building UQ objective of expected model error...')
b = ExpectedValue(error,
bnd,
constraint=scons,
cvalid=is_cons,
samples=rnd)
i = counter.count()
#print('solving for upper bound on expected model error...')
solved = b.upper_bound(axis=axis, id=i, **param)
if type(solved) is not tuple:
solved = (solved, )
if axis is None:
results = tuple(-s for s in solved) #NOTE: -1 for LUB
else:
results = -solved[axis] #NOTE: -1 for LUB
return results
model = NoisyModel('model', model=toy, nx=nx, ny=ny, **approx)
# build a surrogate model by training on the data
args = dict(hidden_layer_sizes=(100, 75, 50, 25),
max_iter=1000,
n_iter_no_change=5,
solver='lbfgs',
learning_rate_init=0.001)
from sklearn.neural_network import MLPRegressor
from sklearn.preprocessing import StandardScaler
from ml import Estimator, MLData, improve_score
kwds = dict(estimator=MLPRegressor(**args), transform=StandardScaler())
# iteratively improve estimator
mlp = Estimator(**kwds) #FIXME: traintest so train != test ?
best = improve_score(mlp,
MLData(data.coords, data.coords, data.values,
data.values),
tries=10,
verbose=True)
mlkw = dict(estimator=best.estimator, transform=best.transform)
#print('building estimator G(x) from truth data...')
surrogate = LearnedModel('surrogate', nx=nx, ny=ny, data=truth, **mlkw)
#print('building UQ model of model error...')
error = ErrorModel('error', model=model, surrogate=surrogate)
rnd = Ns if error.rnd else None
#print('building UQ objective of expected model error...')
b = ExpectedValue(error,
bnd,
constraint=scons,
cvalid=is_cons,