def test_rternalize():
def f(x, y):
return x[0] + y[0]
rfun = rinterface.rternalize(f)
res = rfun(1, 2)
assert res[0] == 3
def test_rternalize_return_sexp():
def f(x, y):
return rinterface.IntSexpVector([x[0], y[0]])
rfun = rinterface.rternalize(f)
res = rfun(1, 2)
assert tuple(res) == (1, 2)
def _function_to_ri(func):
def wrap(*args):
res = func(*args)
res = conversion.py2ro(res)
return res
rfunc = rinterface.rternalize(wrap)
return rfunc
def testRternalize(self):
def f(x, y):
return x[0] + y[0]
rfun = rinterface.rternalize(f)
res = rfun(1, 2)
self.assertEqual(3, res[0])
def testRternalize(self):
def f(x, y):
return x[0] + y[0]
rfun = rinterface.rternalize(f)
res = rfun(1, 2)
self.assertEqual(3, res[0])
def fit(self, model, testIndices):
"""
"""
# ------------------------------ Function --------------------------- #
def errorFit(parameters):
def fitData(n, testIndices):
for i in xrange(n):
if i not in testIndices:
yield i
# Instantiate the surrogate model
cModel = modena.libmodena.modena_model_t(
model=model, parameters=list(parameters))
return FloatVector(
list(
model.error(cModel,
idxGenerator=fitData(model.nSamples,
testIndices),
checkBounds=False)))
# ------------------------------------------------------------------- #
new_parameters = model.parameters
if not len(new_parameters):
new_parameters = [None] * len(model.surrogateFunction.parameters)
for k, v in model.surrogateFunction.parameters.iteritems():
new_parameters[v.argPos] = (v.min + v.max) / 2
# make objects usable in R
R_par = FloatVector(new_parameters)
R_res = rinterface.rternalize(errorFit)
max_parameters = [None] * len(new_parameters)
min_parameters = [None] * len(new_parameters)
for k, v in model.surrogateFunction.parameters.iteritems():
min_parameters[v.argPos] = v.min
max_parameters[v.argPos] = v.max
# perform fitting (nonlinear MSSQ)
nlfb = nlmrt.nlfb(start=R_par,
resfn=R_res,
jacfn=rinterface.NULL,
trace=rinterface.FALSE,
lower=FloatVector(min_parameters),
upper=FloatVector(max_parameters),
maskidx=rinterface.NULL)
# optimised coefficients and sum of squares
nlfb_coeffs = nlfb[nlfb.names.index('coefficients')]
nlfb_ssqres = nlfb[nlfb.names.index('ssquares')]
new_parameters = list(nlfb_coeffs)
return new_parameters
def testRternalizeNamedArgs(self):
def f(x, y, z=None):
if z is None:
return x[0]+y[0]
else:
return z
rfun = rinterface.rternalize(f)
res = rfun(1, 2)
self.assertEqual(3, res[0])
res = rfun(1, 2, z=8)
self.assertEqual(8, res[0])
def test_rternalize_namedargs():
def f(x, y, z=None):
if z is None:
return x[0]+y[0]
else:
return z[0]
rfun = rinterface.rternalize(f)
res = rfun(1, 2)
assert res[0] == 3
res = rfun(1, 2, z=8)
assert res[0] == 8
def testRternalizeNamedArgs(self):
def f(x, y, z=None):
if z is None:
return x[0] + y[0]
else:
return z
rfun = rinterface.rternalize(f)
res = rfun(1, 2)
self.assertEqual(3, res[0])
res = rfun(1, 2, z=8)
self.assertEqual(8, res[0])
def fit(self, model, testIndices):
"""
"""
# ------------------------------ Function --------------------------- #
def errorFit(parameters):
def fitData(n, testIndices):
for i in xrange(n):
if i not in testIndices:
yield i
# Instantiate the surrogate model
cModel = modena.libmodena.modena_model_t(model=model,parameters=list(parameters))
return FloatVector(list(model.error(cModel,idxGenerator=fitData(model.nSamples, testIndices),checkBounds=False)))
# ------------------------------------------------------------------- #
new_parameters = model.parameters
if not len(new_parameters):
new_parameters = [None] * len(model.surrogateFunction.parameters)
for k, v in model.surrogateFunction.parameters.iteritems():
new_parameters[v.argPos] = (v.min + v.max)/2
# make objects usable in R
R_par = FloatVector(new_parameters)
R_res = rinterface.rternalize(errorFit)
max_parameters = [None]*len(new_parameters)
min_parameters = [None]*len(new_parameters)
for k, v in model.surrogateFunction.parameters.iteritems():
min_parameters[v.argPos] = v.min
max_parameters[v.argPos] = v.max
# perform fitting (nonlinear MSSQ)
nlfb = nlmrt.nlfb(start=R_par,resfn=R_res,jacfn=rinterface.NULL,trace=rinterface.FALSE,lower=FloatVector(min_parameters),upper=FloatVector(max_parameters),maskidx=rinterface.NULL)
# optimised coefficients and sum of squares
nlfb_coeffs = nlfb[nlfb.names.index('coefficients')]
nlfb_ssqres = nlfb[nlfb.names.index('ssquares')]
new_parameters = list(nlfb_coeffs)
return new_parameters
actual = actual.strip()
actual = actual.split()
actual = [int(n) for n in actual]
omitted_list.append(actual)
print "Done"
TEST = False
nloptr = importr('nloptr')
if TEST:
# wrap the function f so it can be exposed to R
cost_Fr = ri.rternalize(F_test)
# starting parameters
#start_params = FloatVector((.1, .1, .1, .1, .1))
start_params = FloatVector((.1, .1))
lower_bound = FloatVector((0, 0))
upper_bound = FloatVector((10, 10))
test = {
'algorithm': 'NLOPT_GN_DIRECT',
"ftol_abs": 1.0e-7,
'maxeval': 1000000
}
rlist = robjects.ListVector(test)
def newPointsFWAction(self, model, **kwargs):
new_parameters = model.parameters
if not len(new_parameters):
new_parameters = [None] * len(model.surrogateFunction.parameters)
for k, v in model.surrogateFunction.parameters.iteritems():
new_parameters[v.argPos] = (v.min + v.max)/2
max_parameters = [None]*len(new_parameters)
min_parameters = [None]*len(new_parameters)
for k, v in model.surrogateFunction.parameters.iteritems():
min_parameters[v.argPos] = v.min
max_parameters[v.argPos] = v.max
maxError = 1000
coeffs = None
for i in xrange(model.nSamples):
testPoint = [i]
# -------------------------- Function --------------------------- #
def errorTest(parameters):
def fitData(testIndices):
for i in testPoint:
yield i
# Instantiate the surrogate model
cModel = modena.libmodena.modena_model_t(
model,
parameters=list(parameters)
)
return max(
abs(i) for i in model.error(
cModel,
idxGenerator=fitData(testPoint),
checkBounds=False
)
)
# -------------------------- Function --------------------------- #
def errorFit(parameters):
def fitData(n, testPoint):
for i in xrange(n):
if i not in testPoint:
yield i
# Instantiate the surrogate model
cModel = modena.libmodena.modena_model_t(
model=model,
parameters=list(parameters)
)
return FloatVector(
list(
model.error(
cModel,
idxGenerator=fitData(model.nSamples, testPoint),
checkBounds=False
)
)
)
# --------------------------------------------------------------- #
# make objects usable in R
R_res = rinterface.rternalize(errorFit)
R_par = FloatVector(new_parameters)
# perform fitting (nonlinear MSSQ)
nlfb = nlmrt.nlfb(
start=R_par,
resfn=R_res,
jacfn=rinterface.NULL,
trace=rinterface.FALSE,
lower=FloatVector(min_parameters),
upper=FloatVector(max_parameters),
maskidx=rinterface.NULL
)
parameterError = errorTest(nlfb[nlfb.names.index('coefficients')])
if parameterError < maxError:
maxError = parameterError
new_parameters = list(nlfb[nlfb.names.index('coefficients')])
coeffs = nlfb
# optimised coefficients and sum of squares
nlfb_coeffs = coeffs[nlfb.names.index('coefficients')]
nlfb_ssqres = coeffs[nlfb.names.index('ssquares')]
print 'Maximum Error = %s' % maxError
print(
'old parameters = [%s]' % ', '.join(
'%g' % k for k in model.parameters
)
)
print(
'new parameters = [%s]' % ', '.join(
'%g' % k for k in new_parameters
)
)
# Update database
# TODO: This is not save if the model is updated by another fitting
# task running concurrently
model.parameters = new_parameters
model.updateMinMax()
model.save()
del parameterError
del max_parameters
del min_parameters
del coeffs
# return nothing to restart normal operation
return FWAction()
actual = actual.strip()
actual = actual.split()
actual = [int(n) for n in actual]
omitted_list.append(actual)
print "Done"
TEST = False
nloptr = importr('nloptr')
if TEST:
# wrap the function f so it can be exposed to R
cost_Fr = ri.rternalize(F_test)
# starting parameters
#start_params = FloatVector((.1, .1, .1, .1, .1))
start_params = FloatVector((.1, .1))
lower_bound = FloatVector((0, 0))
upper_bound = FloatVector((10, 10))
test ={'algorithm':'NLOPT_GN_DIRECT',"ftol_abs":1.0e-7,'maxeval':1000000}
rlist = robjects.ListVector(test)
print 'Starting opt'
res = nloptr.nloptr(x0=start_params, eval_f=cost_Fr, opts = rlist,lb=lower_bound,ub=upper_bound)
print "Opt finished"
def newPointsFWAction(self, model, **kwargs):
new_parameters = model.parameters
if not len(new_parameters):
new_parameters = [None] * len(model.surrogateFunction.parameters)
for k, v in model.surrogateFunction.parameters.iteritems():
new_parameters[v.argPos] = (v.min + v.max) / 2
max_parameters = [None] * len(new_parameters)
min_parameters = [None] * len(new_parameters)
for k, v in model.surrogateFunction.parameters.iteritems():
min_parameters[v.argPos] = v.min
max_parameters[v.argPos] = v.max
maxError = 1000
coeffs = None
for i in xrange(model.nSamples):
testPoint = [i]
# -------------------------- Function --------------------------- #
def errorTest(parameters):
def fitData(testIndices):
for i in testPoint:
yield i
# Instantiate the surrogate model
cModel = modena.libmodena.modena_model_t(
model, parameters=list(parameters))
return max(
abs(i)
for i in model.error(cModel,
idxGenerator=fitData(testPoint),
checkBounds=False))
# -------------------------- Function --------------------------- #
def errorFit(parameters):
def fitData(n, testPoint):
for i in xrange(n):
if i not in testPoint:
yield i
# Instantiate the surrogate model
cModel = modena.libmodena.modena_model_t(
model=model, parameters=list(parameters))
return FloatVector(
list(
model.error(cModel,
idxGenerator=fitData(
model.nSamples, testPoint),
checkBounds=False)))
# --------------------------------------------------------------- #
# make objects usable in R
R_res = rinterface.rternalize(errorFit)
R_par = FloatVector(new_parameters)
# perform fitting (nonlinear MSSQ)
nlfb = nlmrt.nlfb(start=R_par,
resfn=R_res,
jacfn=rinterface.NULL,
trace=rinterface.FALSE,
lower=FloatVector(min_parameters),
upper=FloatVector(max_parameters),
maskidx=rinterface.NULL)
parameterError = errorTest(nlfb[nlfb.names.index('coefficients')])
if parameterError < maxError:
maxError = parameterError
new_parameters = list(nlfb[nlfb.names.index('coefficients')])
coeffs = nlfb
# optimised coefficients and sum of squares
nlfb_coeffs = coeffs[nlfb.names.index('coefficients')]
nlfb_ssqres = coeffs[nlfb.names.index('ssquares')]
print 'Maximum Error = %s' % maxError
print('old parameters = [%s]' % ', '.join('%g' % k
for k in model.parameters))
print('new parameters = [%s]' % ', '.join('%g' % k
for k in new_parameters))
# Update database
# TODO: This is not save if the model is updated by another fitting
# task running concurrently
model.parameters = new_parameters
model.updateMinMax()
model.save()
del parameterError
del max_parameters
del min_parameters
del coeffs
# return nothing to restart normal operation
return FWAction()
def newPointsFWAction(self, model, **kwargs):
# Make sure we get new samples in deterministic manner
seed(model.nSamples)
# TODO: The rest of this function should become a method in model (or /
# strategy class)
# Create indices a subset (~20% of samples) for testing
testIndices = set(
choice(model.nSamples,
size=max(1, self['testDataPercentage'] * model.nSamples),
replace=False))
# ------------------------------ Function --------------------------- #
def errorFit(parameters):
def fitData(n, testIndices):
for i in xrange(n):
if i not in testIndices:
yield i
# Instantiate the surrogate model
cModel = modena.libmodena.modena_model_t(
model=model, parameters=list(parameters))
return FloatVector(
list(
model.error(cModel,
idxGenerator=fitData(model.nSamples,
testIndices),
checkBounds=False)))
# ------------------------------------------------------------------- #
# ------------------------------ Function --------------------------- #
def errorTest(parameters):
def fitData(testIndices):
for i in testIndices:
yield i
# Instantiate the surrogate model
cModel = modena.libmodena.modena_model_t(
model, parameters=list(parameters))
return max(
abs(i) for i in model.error(cModel,
idxGenerator=fitData(testIndices),
checkBounds=False))
# ------------------------------------------------------------------- #
new_parameters = model.parameters
if not len(new_parameters):
new_parameters = [None] * len(model.surrogateFunction.parameters)
for k, v in model.surrogateFunction.parameters.iteritems():
new_parameters[v.argPos] = (v.min + v.max) / 2
# make objects usable in R
R_par = FloatVector(new_parameters)
R_res = rinterface.rternalize(errorFit)
max_parameters = [None] * len(new_parameters)
min_parameters = [None] * len(new_parameters)
for k, v in model.surrogateFunction.parameters.iteritems():
min_parameters[v.argPos] = v.min
max_parameters[v.argPos] = v.max
# perform fitting (nonlinear MSSQ)
nlfb = nlmrt.nlfb(start=R_par,
resfn=R_res,
jacfn=rinterface.NULL,
trace=rinterface.FALSE,
lower=FloatVector(min_parameters),
upper=FloatVector(max_parameters),
maskidx=rinterface.NULL)
del max_parameters
del min_parameters
# optimised coefficients and sum of squares
nlfb_coeffs = nlfb[nlfb.names.index('coefficients')]
nlfb_ssqres = nlfb[nlfb.names.index('ssquares')]
new_parameters = list(nlfb_coeffs)
# The following code block will check the error and call the ImproveEr-
# rorStrategy to add more points to the design of experiments if the m-
# odel is not validated
maxError = errorTest(new_parameters)
print 'Maximum Error = %s' % maxError
if maxError > self['maxError']:
print('Parameters ' + term.red + 'not' + term.normal +
' valid, adding samples.')
print('current parameters = [%s]' %
', '.join('%g' % k for k in new_parameters))
# Update database
model.save()
return FWAction(
detours=self['improveErrorStrategy'].workflow(model))
else:
print('old parameters = [%s]' %
', '.join('%g' % k for k in model.parameters))
print('new parameters = [%s]' % ', '.join('%g' % k
for k in new_parameters))
# Update database
# TODO: This is not save if the model is updated by another fitting
# task running concurrently
model.parameters = new_parameters
model.updateMinMax()
model.save()
# return nothing to restart normal operation
return FWAction()
robj.r.rpois(10, **{'lambda' : 3})
# <codecell>
# Make a python function and call it within R
ri.initr()
stats = importr('stats')
def quad_f(x):
x = x[0]
return x ** -2 + 0.5 * x
# wrap the function f so it can be exposed to R
quad_fr = ri.rternalize(quad_f)
# define the interval to find the minimum over
interval = rv.IntVector((0, 10))
# call R's optimize()
res = stats.optimize(quad_fr, interval)
print res
# <headingcell level=3>
# Part 5: Creating and exporting R graphics
# <codecell>
# plotting
def newPointsFWAction(self, model, **kwargs):
# Make sure we get new samples in deterministic manner
seed(model.nSamples)
# TODO: The rest of this function should become a method in model (or /
# strategy class)
# Create indices a subset (~20% of samples) for testing
testIndices = set(
choice(
model.nSamples,
size=max(1, self['testDataPercentage']*model.nSamples),
replace=False
)
)
# ------------------------------ Function --------------------------- #
def errorFit(parameters):
def fitData(n, testIndices):
for i in xrange(n):
if i not in testIndices:
yield i
# Instantiate the surrogate model
cModel = modena.libmodena.modena_model_t(
model=model,
parameters=list(parameters)
)
return FloatVector(
list(
model.error(
cModel,
idxGenerator=fitData(model.nSamples, testIndices),
checkBounds=False
)
)
)
# ------------------------------------------------------------------- #
# ------------------------------ Function --------------------------- #
def errorTest(parameters):
def fitData(testIndices):
for i in testIndices:
yield i
# Instantiate the surrogate model
cModel = modena.libmodena.modena_model_t(
model,
parameters=list(parameters)
)
return max(
abs(i) for i in model.error(
cModel,
idxGenerator=fitData(testIndices),
checkBounds=False
)
)
# ------------------------------------------------------------------- #
new_parameters = model.parameters
if not len(new_parameters):
new_parameters = [None] * len(model.surrogateFunction.parameters)
for k, v in model.surrogateFunction.parameters.iteritems():
new_parameters[v.argPos] = (v.min + v.max)/2
# make objects usable in R
R_par = FloatVector(new_parameters)
R_res = rinterface.rternalize(errorFit)
max_parameters = [None]*len(new_parameters)
min_parameters = [None]*len(new_parameters)
for k, v in model.surrogateFunction.parameters.iteritems():
min_parameters[v.argPos] = v.min
max_parameters[v.argPos] = v.max
# perform fitting (nonlinear MSSQ)
nlfb = nlmrt.nlfb(
start=R_par,
resfn=R_res,
jacfn=rinterface.NULL,
trace=rinterface.FALSE,
lower=FloatVector(min_parameters),
upper=FloatVector(max_parameters),
maskidx=rinterface.NULL
)
del max_parameters
del min_parameters
# optimised coefficients and sum of squares
nlfb_coeffs = nlfb[nlfb.names.index('coefficients')]
nlfb_ssqres = nlfb[nlfb.names.index('ssquares')]
new_parameters = list(nlfb_coeffs)
# The following code block will check the error and call the ImproveEr-
# rorStrategy to add more points to the design of experiments if the m-
# odel is not validated
maxError = errorTest(new_parameters)
print 'Maximum Error = %s' % maxError
if maxError > self['maxError']:
print(
'Parameters ' + term.red + 'not' + term.normal
+ ' valid, adding samples.'
)
print(
'current parameters = [%s]' % ', '.join(
'%g' % k for k in new_parameters
)
)
# Update database
model.save()
return FWAction(
detours=self['improveErrorStrategy'].workflow(model)
)
else:
print(
'old parameters = [%s]' % ', '.join(
'%g' % k for k in model.parameters
)
)
print(
'new parameters = [%s]' % ', '.join(
'%g' % k for k in new_parameters
)
)
# Update database
# TODO: This is not save if the model is updated by another fitting
# task running concurrently
model.parameters = new_parameters
model.updateMinMax()
model.save()
# return nothing to restart normal operation
return FWAction()
def representfunc(funcpath, forced=False):
if (funcpath.find('@') == 0):
funcpath = path.dirname(__file__) + '/TestFunctions/' + funcpath[1:]
#defining the function name
funcname = path.splitext(path.basename(funcpath))[0]
# loading the function to be represented
spec = importlib.util.spec_from_file_location(funcname, funcpath)
funcmodule = importlib.util.module_from_spec(spec)
spec.loader.exec_module(funcmodule)
# Finding the function characteristics inside the docstring
if funcmodule.main.__doc__:
regex = re.compile(
r"#_#\s?(\w+):(.+)?\n"
) # this regular expression matches the characteristics already specified in the docstring section of the function -- old exp: "#_#\s?(\w+):\s?([-+]?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?)"
characs = re.findall(regex, funcmodule.main.__doc__)
results = {}
for charac in characs:
results[charac[0]] = eval(charac[1].replace('nan', 'NaN'))
# Automatically generate the representation if the docstrings did not return anything
if not ('Represented' in results):
print(
"Warning, the Representation of the Test Function has not been specified\n===\n******Calculating the Characteristics******"
)
n = int(results['dimmensions'])
blocks = int(1 + 10 / n)
if blocks < 3: blocks = 3
# Importing FLACCO using rpy2
flacco = importr('flacco')
# creating the r functions
rlist = robjs.r['list']
rapply = robjs.r['apply']
rvector = robjs.r['c']
r_unlist = robjs.r['unlist']
rtestfunc = rinterface.rternalize(funcmodule.main)
# Verify if a list of limits has been specified for all dimensions or if all dimensions will use the same boundaries
if (type(results['lower']) is list):
lowerval = r_unlist(rvector(results['lower']))
upperval = r_unlist(rvector(results['upper']))
else:
lowerval = results['lower']
upperval = results['upper']
X = flacco.createInitialSample(
n_obs=500,
dim=n,
control=rlist(
**{
'init_sample.type': 'lhs',
'init_sample.lower': lowerval,
'init_sample.upper': upperval
}))
y = rapply(X, 1, rtestfunc)
testfuncobj = flacco.createFeatureObject(
**{
'X': X,
'y': y,
'fun': rtestfunc,
'lower': lowerval,
'upper': upperval,
'blocks': blocks,
'force': forced
})
# these are the retained features. Note that some features are being excluded for being problematic and to avoid overcomplicating the neural network.... the feature sets are redundant and the most relevant ones have been retained
# the excluded feature sets are: 'bt', 'ela_level'
# feature sets that require special attention: 'cm_angle', 'cm_grad', 'limo', 'gcm' (large set with some nans),
featureset = [
'cm_angle', 'cm_conv', 'cm_grad', 'ela_conv', 'ela_curv',
'ela_distr', 'ela_local', 'ela_meta', 'basic', 'disp', 'limo',
'nbc', 'pca', 'gcm', 'ic'
]
pyfeats = dict()
for feature in featureset:
rawfeats = flacco.calculateFeatureSet(testfuncobj, set=feature)
pyfeats[feature] = asarray(rawfeats)
writerepresentation(funcpath, pyfeats)
for feat in results.keys():
if isinstance(results[feat], ndarray):
results[feat] = results[feat].reshape(results[feat].shape[:-1])
return results