def ppf(self, x):
# convert the parameter to the right format
if isList(x):
x = DataVector(x)
elif isNumerical(x):
x = DataVector([x])
elif isMatrix(x):
x = DataMatrix(x)
if isinstance(x, DataMatrix):
A = x
B = DataMatrix(A.getNrows(), A.getNcols())
B.setAll(0.0)
elif isinstance(x, DataVector):
A = DataMatrix(1, len(x))
A.setRow(0, x)
B = DataMatrix(1, len(x))
B.setAll(0)
# do the transformation
opInvRosen = createOperationInverseRosenblattTransformationKDE(self.dist)
opInvRosen.doTransformation(A, B)
# transform the outcome
if isNumerical(x) or isinstance(x, DataVector):
return B.get(0, 0)
elif isinstance(x, DataMatrix):
return B.array()
def computeMoments(self, ts=None):
names = ['time',
'iteration',
'grid_size',
'mean',
'meanDiscretizationError',
'var',
'varDiscretizationError']
# parameters
ts = self.__samples.keys()
nrows = len(ts)
ncols = len(names)
data = DataMatrix(nrows, ncols)
v = DataVector(ncols)
row = 0
for t in ts:
v.setAll(0.0)
v[0] = t
v[1] = 0
v[2] = len(self.__samples[t].values())
v[3], v[4] = self.mean(ts=[t])
v[5], v[6] = self.var(ts=[t])
# write results to matrix
data.setRow(row, v)
row += 1
return {'data': data,
'names': names}
def ppf(self, x):
# convert the parameter to the right format
if isList(x):
x = DataVector(x)
elif isNumerical(x):
x = DataVector([x])
elif isMatrix(x):
x = DataMatrix(x)
if isinstance(x, DataMatrix):
A = x
B = DataMatrix(A.getNrows(), A.getNcols())
B.setAll(0.0)
elif isinstance(x, DataVector):
A = DataMatrix(1, len(x))
A.setRow(0, x)
B = DataMatrix(1, len(x))
B.setAll(0)
# do the transformation
self.dist.ppf(A, B)
# transform the outcome
if isNumerical(x) or isinstance(x, DataVector):
return B.get(0, 0)
elif isinstance(x, DataMatrix):
return B.array()
def checkPositivity(grid, alpha):
# define a full grid of maxlevel of the grid
gs = grid.getStorage()
fullGrid = Grid.createLinearGrid(gs.dim())
fullGrid.createGridGenerator().full(gs.getMaxLevel())
fullHashGridStorage = fullGrid.getStorage()
A = DataMatrix(fullHashGridStorage.size(), fullHashGridStorage.dim())
p = DataVector(gs.dim())
for i in xrange(fullHashGridStorage.size()):
fullHashGridStorage.get(i).getCoords(p)
A.setRow(i, p)
res = evalSGFunctionMulti(grid, alpha, A)
ymin, ymax, cnt = 0, -1e10, 0
for i, yi in enumerate(res.array()):
if yi < 0. and abs(yi) > 1e-13:
cnt += 1
ymin = min(ymin, yi)
ymax = max(ymax, yi)
A.getRow(i, p)
print " %s = %g" % (p, yi)
if cnt > 0:
print "warning: function is not positive"
print "%i/%i: [%g, %g]" % (cnt, fullHashGridStorage.size(), ymin, ymax)
return cnt == 0
def checkPositivity(grid, alpha):
# define a full grid of maxlevel of the grid
gs = grid.getStorage()
fullGrid = Grid.createLinearGrid(gs.dim())
fullGrid.createGridGenerator().full(gs.getMaxLevel())
fullHashGridStorage = fullGrid.getStorage()
A = DataMatrix(fullHashGridStorage.size(), fullHashGridStorage.dim())
p = DataVector(gs.dim())
for i in xrange(fullHashGridStorage.size()):
fullHashGridStorage.get(i).getCoords(p)
A.setRow(i, p)
res = evalSGFunctionMulti(grid, alpha, A)
ymin, ymax, cnt = 0, -1e10, 0
for i, yi in enumerate(res.array()):
if yi < 0. and abs(yi) > 1e-13:
cnt += 1
ymin = min(ymin, yi)
ymax = max(ymax, yi)
A.getRow(i, p)
print " %s = %g" % (p, yi)
if cnt > 0:
print "warning: function is not positive"
print "%i/%i: [%g, %g]" % (cnt, fullHashGridStorage.size(), ymin, ymax)
return cnt == 0
def computeStats(self, dtype):
names = [
'time', 'iteration', 'level', 'grid_size', 'trainMSE',
'trainL2Error', 'testMSE', 'testL2Error', 'L2ErrorSurpluses'
]
ts = self.__learner.getTimeStepsOfInterest()
iterations = self.__learner.iteration + 1
nrows = len(ts) * iterations
ncols = len(names)
data = DataMatrix(nrows, ncols)
v = DataVector(ncols)
v.setAll(0.)
row = 0
for t in ts:
for iteration in xrange(iterations):
v[0] = t
v[1] = iteration
v[2] = self.__learner.level[dtype][iteration]
v[3] = self.__learner.numberPoints[dtype][iteration]
v[4] = self.__learner.trainAccuracy[dtype][t][iteration]
n = self.__learner.trainCount[dtype][t][iteration]
v[5] = float(np.sqrt(v[4] * n)) # == L2 error
if len(self.__learner.testAccuracy[dtype][t]) == \
len(self.__learner.trainAccuracy[dtype][t]):
v[6] = self.__learner.testAccuracy[dtype][t][iteration]
n = self.__learner.testCount[dtype][t][iteration]
v[7] = float(np.sqrt(v[6] * n)) # == L2 error
v[8] = self.computeL2ErrorSurpluses(self._qoi, t, dtype,
iteration)
# write results to matrix
data.setRow(row, v)
row += 1
return {'data': data, 'names': names}
def computeMoments(self, iterations=None, ts=None):
names = [
'time', 'iteration', 'grid_size', 'mean',
'meanDiscretizationError', 'var', 'varDiscretizationError'
]
# parameters
if ts is None:
ts = self.__knowledge.getAvailableTimeSteps()
if iterations is None:
iterations = self.__knowledge.getAvailableIterations()
nrows = len(ts) * len(iterations)
ncols = len(names)
data = DataMatrix(nrows, ncols)
v = DataVector(ncols)
row = 0
for t in ts:
for iteration in iterations:
size = self.__knowledge.getGrid(qoi=self._qoi,
iteration=iteration).getSize()
v.setAll(0.0)
v[0] = t
v[1] = iteration
v[2] = size
v[3], v[4] = self.mean(ts=[t], iterations=[iteration])
v[5], v[6] = self.var(ts=[t], iterations=[iteration])
# write results to matrix
data.setRow(row, v)
row += 1
return {'data': data, 'names': names}
def merge(cls, containerList):
if len(containerList) == 0:
return None
# determine the total number of entries
size = 0
for container in containerList:
size += len(container.getValues())
dim = container.getPoints().getNcols()
# Copy data to the new DataVector's entry by entry
allPoints = DataMatrix(size, dim)
allValues = DataVector(size)
tmpVector = DataVector(dim)
i = 0
for container in containerList:
points = container.getPoints()
values = container.getValues()
for j in xrange(len(values)):
points.getRow(j, tmpVector)
allPoints.setRow(i, tmpVector)
allValues[i] = values[j]
i += 1
# return new DataContainer
return DataContainer(points=allPoints, values=allValues)
def computeMoments(self, ts=None):
names = [
'time', 'iteration', 'grid_size', 'mean', 'mean_err',
'meanConfidenceIntervalBootstrapping_lower',
'meanConfidenceIntervalBootstrapping_upper', 'var', 'var_err',
'varConfidenceIntervalBootstrapping_lower',
'varConfidenceIntervalBootstrapping_upper'
]
# parameters
ts = list(self.__samples.keys())
nrows = len(ts)
ncols = len(names)
data = DataMatrix(nrows, ncols)
v = DataVector(ncols)
row = 0
for t in np.sort(ts):
v.setAll(0.0)
mean = self.mean(ts=[t], iterations=[0])
var = self.var(ts=[t], iterations=[0])
numSamples = len(list(self.__samples[t].values()))
v[0] = t
v[1] = 0
v[2] = numSamples
v[3], v[4] = mean["value"], mean["err"]
v[5], v[6] = mean["confidence_interval"]
v[7], v[8] = var["value"], var["err"]
v[8], v[9] = var["confidence_interval"]
# write results to matrix
data.setRow(row, v)
row += 1
return {'data': data, 'names': names}
def __prepareDataContainer(self, data, name):
"""
Prepare data for learning
@param data: dictionary loaded from UQSetting
@return dictionary {dtype: {t: <DataContainer>}}
"""
ans = {}
U = self.getParameters()\
.activeParams()\
.getIndependentJointDistribution()
for dtype in self.getKnowledgeTypes():
ans[dtype] = {}
dim = self.grid.getStorage().dim()
# prepare data container depending on the given knowledge type
tmp = KnowledgeTypes.transformData(data, U, dtype)
# load data for all time steps
for t, values in tmp.items():
size = len(values)
mydata = DataMatrix(size, dim)
sol = DataVector(size)
for i, (sample, res) in enumerate(values.items()):
p = DataVector(sample.getActiveUnit())
mydata.setRow(i, p)
sol[i] = float(res)
ans[dtype][t] = DataContainer(points=mydata, values=sol, name=name)
return ans
def merge(cls, containerList):
if len(containerList) == 0:
return None
# determine the total number of entries
size = 0
for container in containerList:
size += len(container.getValues())
dim = container.getPoints().getNcols()
# Copy data to the new DataVector's entry by entry
allPoints = DataMatrix(size, dim)
allValues = DataVector(size)
tmpVector = DataVector(dim)
i = 0
for container in containerList:
points = container.getPoints()
values = container.getValues()
for j in range(len(values)):
points.getRow(j, tmpVector)
allPoints.setRow(i, tmpVector)
allValues[i] = values[j]
i += 1
# return new DataContainer
return DataContainer(points=allPoints, values=allValues)
def __prepareDataContainer(self, data, name):
"""
Prepare data for learning
@param data: dictionary loaded from UQSetting
@return dictionary {dtype: {t: <DataContainer>}}
"""
ans = {}
U = self.getParameters()\
.activeParams()\
.getIndependentJointDistribution()
for dtype in self.getKnowledgeTypes():
ans[dtype] = {}
dim = self.grid.getStorage().getDimension()
# prepare data container depending on the given knowledge type
tmp = KnowledgeTypes.transformData(data, U, dtype)
# load data for all time steps
for t, values in list(tmp.items()):
size = len(values)
mydata = DataMatrix(size, dim)
sol = DataVector(size)
for i, (sample, res) in enumerate(values.items()):
p = DataVector(sample.getActiveUnit())
mydata.setRow(i, p)
sol[i] = float(res)
ans[dtype][t] = DataContainer(points=mydata,
values=sol,
name=name)
return ans
def cdf(self, x):
# convert the parameter to the right format
if isList(x):
x = DataVector(x)
elif isNumerical(x):
x = DataVector([x])
elif isMatrix(x):
x = DataMatrix(x)
if isinstance(x, DataMatrix):
A = x
B = DataMatrix(A.getNrows(), A.getNcols())
B.setAll(0.0)
elif isinstance(x, DataVector):
A = DataMatrix(1, len(x))
A.setRow(0, x)
B = DataMatrix(1, len(x))
B.setAll(0)
# do the transformation
self.dist.cdf(A, B)
# transform the outcome
if isNumerical(x) or isinstance(x, DataVector):
return B.get(0, 0)
elif isinstance(x, DataMatrix):
return B.array()
def ppf(self, x):
# convert the parameter to the right format
if isList(x):
x = DataVector(x)
elif isNumerical(x):
x = DataVector([x])
if isinstance(x, DataMatrix):
A = x
B = DataMatrix(A.getNrows(), A.getNcols())
B.setAll(0.0)
elif isinstance(x, DataVector):
A = DataMatrix(1, len(x))
A.setRow(0, x)
B = DataMatrix(1, len(x))
B.setAll(0)
# do the transformation
assert A.getNcols() == B.getNcols() == self.trainData.getNcols()
op = createOperationInverseRosenblattTransformationKDE(self.trainData)
op.doTransformation(A, B)
# transform the outcome
if isNumerical(x) or isinstance(x, DataVector):
return B.get(0, 0)
elif isinstance(x, DataMatrix):
return B.array()
def __getSamples(self, W, T, n):
if self.samples is None:
# draw n ans
ans = W.rvs(n)
# transform them to the unit hypercube
ans = DataMatrix(n, W.getDim())
for i, sample in enumerate(ans):
p = T.probabilisticToUnit(sample)
ans.setRow(i, DataVector(p))
return ans
else:
if self.samples.shape[0] == n:
dataSamples = self.samples
else:
ixs = np.random.randint(0, len(self.samples), n)
dataSamples = self.samples[ixs, :]
# check if there are just samples for a subset of the random
# variables. If so, add the missing ones
if self.__ixs is not None:
ans = W.rvs(n)
# transform them to the unit hypercube
for i, sample in enumerate(dataSamples):
ans[i, :] = T.probabilisticToUnit(ans[i, :])
ans[i, self.__ixs] = sample
else:
ans = dataSamples
return DataMatrix(ans)
def __getSamples(self, W, T, n):
if self.samples is None:
# draw n ans
ans = W.rvs(n)
# transform them to the unit hypercube
ans = DataMatrix(n, W.getDim())
for i, sample in enumerate(ans):
p = T.probabilisticToUnit(sample)
ans.setRow(i, DataVector(p))
return ans
else:
if self.samples.shape[0] == n:
dataSamples = self.samples
else:
ixs = np.random.randint(0, len(self.samples), n)
dataSamples = self.samples[ixs, :]
# check if there are just samples for a subset of the random
# variables. If so, add the missing ones
if self.__ixs is not None:
ans = W.rvs(n)
# transform them to the unit hypercube
for i, sample in enumerate(dataSamples):
ans[i, :] = T.probabilisticToUnit(ans[i, :])
ans[i, self.__ixs] = sample
else:
ans = dataSamples
return DataMatrix(ans)
def loadData(self, qoi='_', name="train", dtype=KnowledgeTypes.SIMPLE):
"""
Reads data set from file
@param qoi: string quatity of interest
@param name: String for category of data set (train or test),
default "train"
@param dtype: knowledge type
@return dictionary {dtype: {t: <DataContainer>}}
WARNING: dtype parameter not supported
"""
data = self.uqSetting.getTimeDependentResults(qoi)
# load result in a dictionary
ans = {}
ans[dtype] = {}
numDims = self.uqSetting.getDim()
numberOfTimesteps = len(next(iter(data.values())))
# load data for all time steps
for t, values in list(data.items()):
mydata = DataMatrix(numberOfTimesteps, numDims)
sol = DataVector(size)
for i, (sample, res) in enumerate(values.items()):
p = DataVector(sample.getActiveUnit())
mydata.setRow(i, p)
sol[i] = res
ans[dtype][t] = DataContainer(mydata, sol, name)
return ans
def loadData(self, qoi='_', name="train", dtype=KnowledgeTypes.SIMPLE):
"""
Reads data set from file
@param qoi: string quatity of interest
@param name: String for category of data set (train or test),
default "train"
@param dtype: knowledge type
@return dictionary {dtype: {t: <DataContainer>}}
WARNING: dtype parameter not supported
"""
# read from file
s = UQSettingFormatter().deserializeFromFile(self.__filename)
testSetting = UQSetting.fromJson(s)
testData = testSetting.getTimeDependentResults(qoi)
# load result in a dictionary
ans = {}
ans[dtype] = {}
size = len(testData.itervalues().next())
# load data for all time steps
for t, values in testData.items():
mydata = DataMatrix(size, self._dim)
sol = DataVector(size)
for i, (sample, res) in enumerate(values.items()):
p = DataVector(sample.getActiveUnit())
mydata.setRow(i, p)
sol[i] = res
ans[dtype][t] = DataContainer(mydata, sol, name)
return ans
def computeMoments(self, ts=None):
names = [
'time', 'iteration', 'grid_size', 'mean',
'meanDiscretizationError', 'var', 'varDiscretizationError'
]
# parameters
ts = self.__samples.keys()
nrows = len(ts)
ncols = len(names)
data = DataMatrix(nrows, ncols)
v = DataVector(ncols)
row = 0
for t in ts:
v.setAll(0.0)
v[0] = t
v[1] = 0
v[2] = len(self.__samples[t].values())
v[3], v[4] = self.mean(ts=[t])
v[5], v[6] = self.var(ts=[t])
# write results to matrix
data.setRow(row, v)
row += 1
return {'data': data, 'names': names}
def to_data_matrix(arr):
(size_x, size_y) = arr.shape
matrix = DataMatrix(size_x, size_y)
cur_row = 0
for x in arr:
x_vec = DataVector(x.tolist())
matrix.setRow(cur_row, x_vec)
cur_row += 1
return matrix
def general_test(self, d, l, bb, xs):
test_desc = "dim=%d, level=%d, len(x)=%s" % (d, l, len(xs))
print(test_desc)
self.grid = Grid.createLinearGrid(d)
self.grid_gen = self.grid.getGenerator()
self.grid_gen.regular(l)
alpha = DataVector(
[self.get_random_alpha() for i in range(self.grid.getSize())])
bb_ = BoundingBox(d)
for d_k in range(d):
dimbb = BoundingBox1D()
dimbb.leftBoundary = bb[d_k][0]
dimbb.rightBoundary = bb[d_k][1]
bb_.setBoundary(d_k, dimbb)
# Calculate the expected value without the bounding box
expected_normal = [
self.calc_exp_value_normal(x, d, bb, alpha) for x in xs
]
#expected_transposed = [self.calc_exp_value_transposed(x, d, bb, alpha) for x in xs]
# Now set the bounding box
self.grid.getStorage().setBoundingBox(bb_)
dm = DataMatrix(len(xs), d)
for k, x in enumerate(xs):
dv = DataVector(x)
dm.setRow(k, dv)
multEval = createOperationMultipleEval(self.grid, dm)
actual_normal = DataVector(len(xs))
#actual_transposed = DataVector(len(xs))
multEval.mult(alpha, actual_normal)
#multEval.mult(alpha, actual_transposed)
actual_normal_list = []
for k in range(len(xs)):
actual_normal_list.append(actual_normal.__getitem__(k))
#actual_transposed_list = []
#for k in xrange(len(xs)):
# actual_transposed_list.append(actual_transposed.__getitem__(k))
self.assertAlmostEqual(actual_normal_list, expected_normal)
#self.assertAlmostEqual(actual_tranposed_list, expected_tranposed)
del self.grid
def dehierarchizeOnNewGrid(gridResult, grid, alpha):
# dehierarchization
gs = gridResult.getStorage()
ps = DataMatrix(gs.size(), gs.dim())
p = DataVector(gs.dim())
for i in xrange(gs.size()):
gs.get(i).getCoords(p)
ps.setRow(i, p)
nodalValues = evalSGFunctionMulti(grid, alpha, ps)
return nodalValues
def dehierarchizeOnNewGrid(gridResult, grid, alpha):
# dehierarchization
gsResult = gridResult.getStorage()
ps = DataMatrix(gsResult.size(), gsResult.dim())
p = DataVector(gsResult.dim())
for i in xrange(gsResult.size()):
gsResult.get(i).getCoords(p)
ps.setRow(i, p)
nodalValues = evalSGFunctionMulti(grid, alpha, ps)
return nodalValues
def general_test(self, d, l, bb, xs):
test_desc = "dim=%d, level=%d, len(x)=%s" % (d, l, len(xs))
print test_desc
self.grid = Grid.createLinearGrid(d)
self.grid_gen = self.grid.createGridGenerator()
self.grid_gen.regular(l)
alpha = DataVector([self.get_random_alpha() for i in xrange(self.grid.getSize())])
bb_ = BoundingBox(d)
for d_k in xrange(d):
dimbb = DimensionBoundary()
dimbb.leftBoundary = bb[d_k][0]
dimbb.rightBoundary = bb[d_k][1]
bb_.setBoundary(d_k, dimbb)
# Calculate the expected value without the bounding box
expected_normal = [self.calc_exp_value_normal(x, d, bb, alpha) for x in xs]
#expected_transposed = [self.calc_exp_value_transposed(x, d, bb, alpha) for x in xs]
# Now set the bounding box
self.grid.getStorage().setBoundingBox(bb_)
dm = DataMatrix(len(xs), d)
for k, x in enumerate(xs):
dv = DataVector(x)
dm.setRow(k, dv)
multEval = createOperationMultipleEval(self.grid, dm)
actual_normal = DataVector(len(xs))
#actual_transposed = DataVector(len(xs))
multEval.mult(alpha, actual_normal)
#multEval.mult(alpha, actual_transposed)
actual_normal_list = []
for k in xrange(len(xs)):
actual_normal_list.append(actual_normal.__getitem__(k))
#actual_transposed_list = []
#for k in xrange(len(xs)):
# actual_transposed_list.append(actual_transposed.__getitem__(k))
self.assertAlmostEqual(actual_normal_list, expected_normal)
#self.assertAlmostEqual(actual_tranposed_list, expected_tranposed)
del self.grid
def sampleGrids(self, filename):
ts = self.__learner.getTimeStepsOfInterest()
names = self.__params.getNames()
names.append('f_\\mathcal{I}(x)')
for t in ts:
grid, surplus = self.__knowledge.getSparseGridFunction(
self._qoi, t)
# init
gs = grid.getStorage()
dim = gs.dim()
# -----------------------------------------
# do full grid sampling of sparse grid function
# -----------------------------------------
data = eval_fullGrid(4, dim)
res = evalSGFunctionMulti(grid, surplus, data)
data.transpose()
data.appendRow()
data.setRow(data.getNrows() - 1, res)
data.transpose()
# write results
writeDataARFF({
'filename': "%s.t%f.samples.arff" % (filename, t),
'data': data,
'names': names
})
# -----------------------------------------
# write sparse grid points to file
# -----------------------------------------
data = DataMatrix(gs.size(), dim)
data.setAll(0.0)
for i in xrange(gs.size()):
gp = gs.get(i)
v = np.array([gp.getCoord(j) for j in xrange(dim)])
data.setRow(i, DataVector(v))
# write results
writeDataARFF({
'filename': "%s.t%f.gridpoints.arff" % (filename, t),
'data': data,
'names': names
})
# -----------------------------------------
# write alpha
# -----------------------------------------
writeAlphaARFF("%s.t%f.alpha.arff" % (filename, t), surplus)
def dehierarchize(grid, alpha):
# dehierarchization
gs = grid.getStorage()
p = DataVector(gs.dim())
nodalValues = DataVector(gs.size())
A = DataMatrix(gs.size(), gs.dim())
for i in xrange(gs.size()):
gs.get(i).getCoords(p)
A.setRow(i, p)
opEval = createOperationMultipleEval(grid, A)
opEval.mult(alpha, nodalValues)
return nodalValues
def loadData(self, name="train"):
fin = self.__gzOpen(self.filename, "r")
data = []
classes = []
hasclass = False
# get the different section of ARFF-File
for line in fin:
sline = line.strip().lower()
if sline.startswith(b"%") or len(sline) == 0:
continue
if sline.startswith(b"@data"):
break
if sline.startswith(b"@attribute"):
value = sline.split()
if value[1].startswith(b"class"):
hasclass = True
else:
data.append([])
#read in the data stored in the ARFF file
for line in fin:
sline = line.strip()
if sline.startswith(b"%") or len(sline) == 0:
continue
values = sline.split(b",")
if hasclass:
classes.append(float(values[-1]))
values = values[:-1]
for i in range(len(values)):
data[i].append(float(values[i]))
# cleaning up and return
fin.close()
dim = len(data)
size = len(data[0])
dataMatrix = DataMatrix(size, dim)
tempVector = DataVector(dim)
valuesVector = DataVector(size)
for rowIndex in range(size):
for colIndex in range(dim):
tempVector[colIndex] = data[colIndex][rowIndex]
dataMatrix.setRow(rowIndex, tempVector)
valuesVector[rowIndex] = classes[rowIndex]
return DataContainer(points=dataMatrix,
values=valuesVector,
name=name,
filename=self.filename)
def dehierarchize(grid, alpha):
# dehierarchization
gs = grid.getStorage()
p = DataVector(gs.dim())
nodalValues = DataVector(gs.size())
A = DataMatrix(gs.size(), gs.dim())
for i in xrange(gs.size()):
gs.get(i).getCoords(p)
A.setRow(i, p)
opEval = createOperationMultipleEval(grid, A)
opEval.mult(alpha, nodalValues)
return nodalValues
def writeSensitivityValues(self, filename):
def keymap(key):
names = self.__uqManager.getParameters().activeParams().getNames()
ans = [names[i] for i in key]
return ",".join(ans)
# parameters
ts = self.__knowledge.getAvailableTimeSteps()
gs = self.__knowledge.getGrid(self._qoi).getStorage()
n = len(ts)
n1 = gs.getDimension()
n2 = 2**n1 - 1
data = DataMatrix(n, n1 + n2 + 1)
names = ['time'] + [None] * (n1 + n2)
for k, t in enumerate(ts):
# estimated anova decomposition
anova = self.getAnovaDecomposition(t=t)
me = anova.getSobolIndices()
if len(me) != n2:
import ipdb
ipdb.set_trace()
n2 = len(me)
te = anova.getTotalEffects()
n1 = len(te)
v = DataVector(n1 + n2 + 1)
v.setAll(0.0)
v[0] = t
for i, key in enumerate(
anova.getSortedPermutations(list(te.keys()))):
v[i + 1] = te[key]
if k == 0:
names[i + 1] = '"$T_{' + keymap(key) + '}$"'
for i, key in enumerate(
anova.getSortedPermutations(list(me.keys()))):
v[n1 + i + 1] = me[key]
if k == 0:
names[n1 + 1 + i] = '"$S_{' + keymap(key) + '}$"'
data.setRow(k, v)
writeDataARFF({
'filename': filename + ".sa.stats.arff",
'data': data,
'names': names
})
def testSave(self):
filename = pathlocal + '/datasets/saving.arff.gz'
testPoints = [[0.307143, 0.130137, 0.050000],
[0.365584, 0.105479, 0.050000],
[0.178571, 0.201027, 0.050000],
[0.272078, 0.145548, 0.050000],
[0.318831, 0.065411, 0.050000],
[0.190260, 0.086986, 0.050000],
[0.190260, 0.062329, 0.072500],
[0.120130, 0.068493, 0.072500],
[0.225325, 0.056164, 0.072500],
[0.213636, 0.050000, 0.072500]]
testValues = [
-1.000000, 1.000000, 1.000000, 1.000000, 1.000000, 1.000000,
-1.000000, -1.000000, -1.000000, -1.000000
]
attributes = {
"x0": "NUMERIC",
"x1": "NUMERIC",
"x2": "NUMERIC",
"class": "NUMERIC",
}
size = len(testPoints)
dim = len(testPoints[0])
point = DataVector(dim)
points = DataMatrix(size, dim)
for row in xrange(size):
for col in xrange(dim):
point[col] = testPoints[row][col]
points.setRow(row, point)
adapter = ARFFAdapter(filename)
adapter.save(points, testValues, attributes)
(points, values) = adapter.loadData().getPointsValues()
size = len(testPoints)
dim = len(testPoints[0])
testVector = DataVector(dim)
for rowIdx in xrange(size):
points.getRow(rowIdx, testVector)
for colIdx in xrange(dim):
if cvar.USING_DOUBLE_PRECISION:
self.assertEqual(testVector[colIdx],
testPoints[rowIdx][colIdx])
else:
self.assertAlmostEqual(testVector[colIdx],
testPoints[rowIdx][colIdx])
self.assertEqual(values[rowIdx], testValues[rowIdx])
os.remove(filename)
def sampleGrids(self, filename):
ts = self.__learner.getTimeStepsOfInterest()
names = self.__params.getNames()
names.append('f_\\mathcal{I}(x)')
for t in ts:
grid, surplus = self.__knowledge.getSparseGridFunction(self._qoi, t)
# init
gs = grid.getStorage()
dim = gs.dim()
# -----------------------------------------
# do full grid sampling of sparse grid function
# -----------------------------------------
data = eval_fullGrid(4, dim)
res = evalSGFunctionMulti(grid, surplus, data)
data.transpose()
data.appendRow()
data.setRow(data.getNrows() - 1, res)
data.transpose()
# write results
writeDataARFF({'filename': "%s.t%f.samples.arff" % (filename, t),
'data': data,
'names': names})
# -----------------------------------------
# write sparse grid points to file
# -----------------------------------------
data = DataMatrix(gs.size(), dim)
data.setAll(0.0)
for i in xrange(gs.size()):
gp = gs.get(i)
v = np.array([gp.getCoord(j) for j in xrange(dim)])
data.setRow(i, DataVector(v))
# write results
writeDataARFF({'filename': "%s.t%f.gridpoints.arff" % (filename, t),
'data': data,
'names': names})
# -----------------------------------------
# write alpha
# -----------------------------------------
writeAlphaARFF("%s.t%f.alpha.arff" % (filename, t),
surplus)
def loadData(self, name = "train"):
fin = self.__gzOpen(self.filename, "r")
data = []
classes = []
hasclass = False
# get the different section of ARFF-File
for line in fin:
sline = line.strip().lower()
if sline.startswith("%") or len(sline) == 0:
continue
if sline.startswith("@data"):
break
if sline.startswith("@attribute"):
value = sline.split()
if value[1].startswith("class"):
hasclass = True
else:
data.append([])
#read in the data stored in the ARFF file
for line in fin:
sline = line.strip()
if sline.startswith("%") or len(sline) == 0:
continue
values = sline.split(",")
if hasclass:
classes.append(float(values[-1]))
values = values[:-1]
for i in xrange(len(values)):
data[i].append(float(values[i]))
# cleaning up and return
fin.close()
dim = len(data)
size = len(data[0])
dataMatrix = DataMatrix(size, dim)
tempVector = DataVector(dim)
valuesVector = DataVector(size)
for rowIndex in xrange(size):
for colIndex in xrange(dim):
tempVector[colIndex] = data[colIndex][rowIndex]
dataMatrix.setRow(rowIndex, tempVector)
valuesVector[rowIndex] = classes[rowIndex]
return DataContainer(points=dataMatrix, values=valuesVector, name=name, filename=self.filename)
def getDataSubsetByIndexList(self, indices, name="train"):
size = len(indices)
subset_points = DataMatrix(size, self.dim)
subset_values = DataVector(size)
row = DataVector(self.dim)
points = self.getPoints()
values = self.getValues()
i = 0
for index in indices:
points.getRow(index, row)
subset_points.setRow(i, row)
subset_values[i] = values[index]
i = i + 1
return DataContainer(points=subset_points, values=subset_values, name=name)
def getDataSubsetByIndexList(self, indices, name="train"):
size = len(indices)
subset_points = DataMatrix(size, self.dim)
subset_values = DataVector(size)
row = DataVector(self.dim)
points = self.getPoints()
values = self.getValues()
i = 0
for index in indices:
points.getRow(index, row)
subset_points.setRow(i, row)
subset_values[i] = values[index]
i = i + 1
return DataContainer(points=subset_points, values=subset_values, name=name)
def writeSensitivityValues(self, filename):
def keymap(key):
names = self.getLearner().getParameters().activeParams().getNames()
ans = [names[i] for i in key]
return ",".join(ans)
# parameters
ts = self.__knowledge.getAvailableTimeSteps()
gs = self.__knowledge.getGrid(self._qoi).getStorage()
n = len(ts)
n1 = gs.dim()
n2 = 2 ** n1 - 1
data = DataMatrix(n, n1 + n2 + 1)
names = ['time'] + [None] * (n1 + n2)
for k, t in enumerate(ts):
# estimated anova decomposition
anova = self.getAnovaDecomposition(t=t)
me = anova.getSobolIndices()
if len(me) != n2:
import ipdb; ipdb.set_trace()
n2 = len(me)
te = anova.getTotalEffects()
n1 = len(te)
v = DataVector(n1 + n2 + 1)
v.setAll(0.0)
v[0] = t
for i, key in enumerate(anova.getSortedPermutations(te.keys())):
v[i + 1] = te[key]
if k == 0:
names[i + 1] = '"$T_{' + keymap(key) + '}$"'
for i, key in enumerate(anova.getSortedPermutations(me.keys())):
v[n1 + i + 1] = me[key]
if k == 0:
names[n1 + 1 + i] = '"$S_{' + keymap(key) + '}$"'
data.setRow(k, v)
writeDataARFF({'filename': filename + ".sa.stats.arff",
'data': data,
'names': names})
def computeStats(self, dtype):
names = [
'time', # 0
'iteration', # 1
'level', # 2
'grid_size', # 3
'trainMSE', # 4
'trainL2Error', # 5
'testMSE', # 6
'testL2Error', # 7
'testL1Error', # 8
'testMaxError', # 9
'L2ErrorSurpluses'
] # 10
knowledge = self.__uqManager.getKnowledge()
ts = knowledge.getAvailableTimeSteps()
iterations = knowledge.getAvailableIterations()
nrows = len(ts) * len(iterations)
ncols = len(names)
data = DataMatrix(nrows, ncols)
v = DataVector(ncols)
v.setAll(0.)
row = 0
for t in ts:
for iteration in iterations:
v[0] = t
v[1] = iteration
v[2] = self.__uqManager.stats.level[dtype][iteration]
v[3] = self.__uqManager.stats.numberPoints[dtype][iteration]
v[4] = self.__uqManager.stats.trainMSE[dtype][t][iteration]
v[5] = self.__uqManager.stats.trainL2Norm[dtype][t][iteration]
if len(self.__uqManager.stats.testMSE[dtype][t]) == \
len(self.__uqManager.stats.trainMSE[dtype][t]):
v[6] = self.__uqManager.stats.testMSE[dtype][t][iteration]
v[7] = self.__uqManager.stats.testL2Norm[dtype][t][
iteration]
v[8] = self.__uqManager.stats.testL1Norm[dtype][t][
iteration]
v[9] = self.__uqManager.stats.testMaxError[dtype][t][
iteration]
v[10] = self.computeL2ErrorSurpluses(self._qoi, t, dtype,
iteration)
# write results to matrix
data.setRow(row, v)
row += 1
return {'data': data, 'names': names}
def dehierarchizeList(grid, alpha, gps):
"""
evaluate sparse grid function at grid points in gps
@param grid: Grid
@param alpha: DataVector
@param gps: list of HashGridIndex
"""
dim = grid.getStorage().dim()
p = DataVector(dim)
nodalValues = DataVector(len(gps))
A = DataMatrix(len(gps), dim)
for i, gp in enumerate(gps):
gp.getCoords(p)
A.setRow(i, p)
createOperationMultipleEval(grid, A).mult(alpha, nodalValues)
return nodalValues
def dehierarchizeList(grid, alpha, gps):
"""
evaluate sparse grid function at grid points in gps
@param grid: Grid
@param alpha: DataVector
@param gps: list of HashGridIndex
"""
dim = grid.getStorage().dim()
p = DataVector(dim)
nodalValues = DataVector(len(gps))
A = DataMatrix(len(gps), dim)
for i, gp in enumerate(gps):
gp.getCoords(p)
A.setRow(i, p)
createOperationMultipleEval(grid, A).mult(alpha, nodalValues)
return nodalValues
def eval_fullGrid(level, dim, border=True):
if border:
grid = Grid.createLinearBoundaryGrid(dim)
else:
grid = Grid.createLinearGrid(dim)
grid.createGridGenerator().full(level)
gs = grid.getStorage()
ans = DataMatrix(gs.size(), dim)
p = DataVector(dim)
for i in xrange(gs.size()):
gs.get(i).getCoords(p)
ans.setRow(i, p)
return ans
def eval_fullGrid(level, dim, border=True):
if border:
grid = Grid.createLinearBoundaryGrid(dim)
else:
grid = Grid.createLinearGrid(dim)
grid.createGridGenerator().full(level)
gs = grid.getStorage()
ans = DataMatrix(gs.size(), dim)
p = DataVector(dim)
for i in xrange(gs.size()):
gs.get(i).getCoords(p)
ans.setRow(i, p)
return ans
def testSave(self):
filename = pathlocal + '/datasets/saving.arff.gz'
testPoints = [[0.307143,0.130137,0.050000],
[0.365584,0.105479,0.050000],
[0.178571,0.201027,0.050000],
[0.272078,0.145548,0.050000],
[0.318831,0.065411,0.050000],
[0.190260,0.086986,0.050000],
[0.190260,0.062329,0.072500],
[0.120130,0.068493,0.072500],
[0.225325,0.056164,0.072500],
[0.213636,0.050000,0.072500]
]
testValues = [-1.000000, 1.000000, 1.000000, 1.000000, 1.000000, 1.000000, -1.000000, -1.000000, -1.000000, -1.000000]
attributes = {
"x0":"NUMERIC",
"x1":"NUMERIC",
"x2":"NUMERIC",
"class":"NUMERIC",
}
size = len(testPoints)
dim = len(testPoints[0])
point = DataVector(dim)
points = DataMatrix(size, dim)
for row in xrange(size):
for col in xrange(dim):
point[col] = testPoints[row][col]
points.setRow(row, point)
adapter = ARFFAdapter(filename)
adapter.save(points, testValues, attributes)
(points, values) = adapter.loadData().getPointsValues()
size = len(testPoints)
dim = len(testPoints[0])
testVector = DataVector(dim)
for rowIdx in xrange(size):
points.getRow(rowIdx, testVector)
for colIdx in xrange(dim):
if cvar.USING_DOUBLE_PRECISION:
self.assertEqual(testVector[colIdx], testPoints[rowIdx][colIdx])
else:
self.assertAlmostEqual(testVector[colIdx], testPoints[rowIdx][colIdx])
self.assertEqual(values[rowIdx], testValues[rowIdx])
os.remove(filename)
def computeBilinearFormByList(self, gpsi, basisi, gpsj, basisj):
"""
Compute bilinear form for two lists of grid points
@param gpsi: list of HashGridIndex
@param basisi: SG++ basis for grid indices gpsi
@param gpsj: list of HashGridIndex
@param basisj: SG++ basis for grid indices gpsj
@return: DataMatrix
"""
A = DataMatrix(len(gpsi), len(gpsj))
err = 0.
# run over all rows
for i, gpi in enumerate(gpsi):
b, erri = self.computeBilinearFormByRow(gpi, basisi, gpsj, basisj)
A.setRow(i, b)
err += erri
return A, err
def computeBilinearFormByList(self, gpsi, basisi, gpsj, basisj):
"""
Compute bilinear form for two lists of grid points
@param gpsi: list of HashGridIndex
@param basisi: SG++ basis for grid indices gpsi
@param gpsj: list of HashGridIndex
@param basisj: SG++ basis for grid indices gpsj
@return: DataMatrix
"""
A = DataMatrix(len(gpsi), len(gpsj))
err = 0.
# run over all rows
for i, gpi in enumerate(gpsi):
b, erri = self.computeBilinearFormByRow(gpi, basisi, gpsj, basisj)
A.setRow(i, b)
err += erri
return A, err
def computeMoments(self, iterations=None, ts=None):
names = [
'time', 'iteration', 'grid_size', 'mean',
'meanDiscretizationError',
'meanConfidenceIntervalBootstrapping_lower',
'meanConfidenceIntervalBootstrapping_upper', 'var',
'varDiscretizationError',
'varConfidenceIntervalBootstrapping_lower',
'varConfidenceIntervalBootstrapping_upper'
]
# parameters
if ts is None:
ts = self.__knowledge.getAvailableTimeSteps()
if iterations is None:
iterations = self.__knowledge.getAvailableIterations()
nrows = len(ts) * len(iterations)
ncols = len(names)
data = DataMatrix(nrows, ncols)
v = DataVector(ncols)
row = 0
for t in ts:
for iteration in iterations:
size = self.__knowledge.getGrid(qoi=self._qoi,
iteration=iteration).getSize()
mean = self.mean(ts=[t],
iterations=[iteration],
totalNumIterations=len(iterations))
var = self.var(ts=[t],
iterations=[iteration],
totalNumIterations=len(iterations))
v.setAll(0.0)
v[0] = t
v[1] = iteration
v[2] = size
v[3], v[4] = mean["value"], mean["err"]
v[5], v[6] = mean["confidence_interval"]
v[7], v[8] = var["value"], var["err"]
v[9], v[10] = var["confidence_interval"]
# write results to matrix
data.setRow(row, v)
row += 1
return {'data': data, 'names': names}
def computeNodalValues(jgrid, grid, alpha):
"""
Dehierarchization of sparse grid function (grid, alpha) on new grid (jgrid)
@param jgrid: Grid, new discretization
@param grid: Grid, old discretization
@param alpha: DataVector, surpluses for grid
@return: DataVector, nodalValues of jgrid
"""
jgs = jgrid.getStorage()
# dehierarchization
p = DataVector(jgs.getDimension())
A = DataMatrix(jgs.size(), jgs.getDimension())
for i in range(jgs.size()):
jgs.getCoordinates(jgs.getPoint(i), p)
A.setRow(i, p)
return evalSGFunctionMulti(grid, alpha, A)
def computeNodalValues(jgrid, grid, alpha):
"""
Dehierarchization of sparse grid function (grid, alpha) on new grid (jgrid)
@param jgrid: Grid, new discretization
@param grid: Grid, old discretization
@param alpha: DataVector, surpluses for grid
@return: DataVector, nodalValues of jgrid
"""
jgs = jgrid.getStorage()
# dehierarchization
p = DataVector(jgs.dim())
A = DataMatrix(jgs.size(), jgs.dim())
for i in xrange(jgs.size()):
jgs.get(i).getCoords(p)
A.setRow(i, p)
return evalSGFunctionMulti(grid, alpha, A)
def dehierarchizeList(grid, alpha, gps):
"""
evaluate sparse grid function at grid points in gps
@param grid: Grid
@param alpha: DataVector
@param gps: list of HashGridPoint
"""
dim = grid.getDimension()
p = DataVector(dim)
nodalValues = DataVector(len(gps))
A = DataMatrix(len(gps), dim)
for i, gp in enumerate(gps):
gs.getCoordinates(gp, p)
A.setRow(i, p)
opEval = createOperationMultipleEval(grid, A)
opEval.mult(alpha, nodalValues)
del opEval
del A
return nodalValues
def computeStats(self, dtype):
names = ['time',
'iteration',
'level',
'grid_size',
'trainMSE',
'trainL2Error',
'testMSE',
'testL2Error',
'L2ErrorSurpluses']
ts = self.__learner.getTimeStepsOfInterest()
iterations = self.__learner.iteration + 1
nrows = len(ts) * iterations
ncols = len(names)
data = DataMatrix(nrows, ncols)
v = DataVector(ncols)
v.setAll(0.)
row = 0
for t in ts:
for iteration in xrange(iterations):
v[0] = t
v[1] = iteration
v[2] = self.__learner.level[dtype][iteration]
v[3] = self.__learner.numberPoints[dtype][iteration]
v[4] = self.__learner.trainAccuracy[dtype][t][iteration]
n = self.__learner.trainCount[dtype][t][iteration]
v[5] = float(np.sqrt(v[4] * n)) # == L2 error
if len(self.__learner.testAccuracy[dtype][t]) == \
len(self.__learner.trainAccuracy[dtype][t]):
v[6] = self.__learner.testAccuracy[dtype][t][iteration]
n = self.__learner.testCount[dtype][t][iteration]
v[7] = float(np.sqrt(v[6] * n)) # == L2 error
v[8] = self.computeL2ErrorSurpluses(self._qoi, t,
dtype, iteration)
# write results to matrix
data.setRow(row, v)
row += 1
return {'data': data, 'names': names}
def pdf(self, x):
# convert the parameter to the right format
if isList(x):
x = DataVector(x)
elif isNumerical(x):
x = DataVector([x])
if isinstance(x, DataMatrix):
A = x
res = DataVector(A.getNrows())
res.setAll(0.0)
elif isinstance(x, DataVector):
A = DataMatrix(1, len(x))
A.setRow(0, x)
res = DataVector(1)
res.setAll(0)
self.dist.pdf(A, res)
if len(res) == 1:
return res[0]
else:
return res.array()
def pdf(self, x):
# convert the parameter to the right format
if isList(x):
x = DataVector(x)
elif isNumerical(x):
x = DataVector([x])
if isinstance(x, DataMatrix):
A = x
res = DataVector(A.getNrows())
res.setAll(0.0)
elif isinstance(x, DataVector):
A = DataMatrix(1, len(x))
A.setRow(0, x)
res = DataVector(1)
res.setAll(0)
self.dist.pdf(A, res)
if len(res) == 1:
return res[0]
else:
return res.array()
def pdf(self, x):
# convert the parameter to the right format
if isList(x):
x = DataVector(x)
elif isNumerical(x):
return evalSGFunction(self.grid, self.alpha, DataVector([x]))
if isinstance(x, DataMatrix):
A = x
elif isinstance(x, DataVector):
A = DataMatrix(1, len(x))
A.setRow(0, x)
else:
raise AttributeError('data type "%s" is not supported in SGDEdist' % type(x))
# evaluate the sparse grid density
fx = evalSGFunctionMulti(self.grid, self.alpha, A)
# if there is just one value given, extract it from the list
if len(fx) == 1:
fx = fx[0]
return fx
def computeCoefficients(jgrid, grid, alpha, f):
"""
Interpolate function f, which depends on some sparse grid function
(grid, alpha) on jgrid
@param jgrid: Grid, new discretization
@param grid: Grid, old discretization
@param alpha: DataVector, surpluses for grid
@param f: function, to be interpolated
@return: DataVector, surpluses for jgrid
"""
jgs = jgrid.getStorage()
# dehierarchization
p = DataVector(jgs.dim())
A = DataMatrix(jgs.size(), jgs.dim())
for i in xrange(jgs.size()):
jgs.get(i).getCoords(p)
A.setRow(i, p)
nodalValues = evalSGFunctionMulti(grid, alpha, A)
# apply f to all grid points
jnodalValues = DataVector(jgs.size())
for i in xrange(len(nodalValues)):
A.getRow(i, p)
# print i, p.array(), nodalValues[i], alpha.min(), alpha.max()
# if nodalValues[i] < -1e20 or nodalValues[i] > 1e20:
# from pysgpp.extensions.datadriven.uq.operations import evalSGFunction, evalSGFunctionMultiVectorized
# print alpha.min(), alpha.max()
# print evalSGFunction(grid, alpha, p)
# print evalSGFunctionMulti(grid, alpha, DataMatrix([p.array()]))
# print evalSGFunctionMultiVectorized(grid, alpha, DataMatrix([p.array()]))
# import ipdb; ipdb.set_trace()
jnodalValues[i] = f(p.array(), nodalValues[i])
jalpha = hierarchize(jgrid, jnodalValues)
return jalpha
def computeCoefficients(jgrid, grid, alpha, f):
"""
Interpolate function f, which depends on some sparse grid function
(grid, alpha) on jgrid
@param jgrid: Grid, new discretization
@param grid: Grid, old discretization
@param alpha: DataVector, surpluses for grid
@param f: function, to be interpolated
@return: DataVector, surpluses for jgrid
"""
jgs = jgrid.getStorage()
# dehierarchization
p = DataVector(jgs.dim())
A = DataMatrix(jgs.size(), jgs.dim())
for i in xrange(jgs.size()):
jgs.get(i).getCoords(p)
A.setRow(i, p)
nodalValues = evalSGFunctionMulti(grid, alpha, A)
# apply f to all grid points
jnodalValues = DataVector(jgs.size())
for i in xrange(len(nodalValues)):
A.getRow(i, p)
# print i, p.array(), nodalValues[i], alpha.min(), alpha.max()
# if nodalValues[i] < -1e20 or nodalValues[i] > 1e20:
# from pysgpp.extensions.datadriven.uq.operations import evalSGFunction, evalSGFunctionMultiVectorized
# print alpha.min(), alpha.max()
# print evalSGFunction(grid, alpha, p)
# print evalSGFunctionMulti(grid, alpha, DataMatrix([p.array()]))
# print evalSGFunctionMultiVectorized(grid, alpha, DataMatrix([p.array()]))
# import ipdb; ipdb.set_trace()
jnodalValues[i] = f(p.array(), nodalValues[i])
jalpha = hierarchize(jgrid, jnodalValues)
return jalpha
def ppf(self, x):
# convert the parameter to the right format
if isList(x):
x = DataVector(x)
elif isNumerical(x):
x = DataVector([x])
# do the transformation
if self.grid.getStorage().dim() == 1:
op = createOperationInverseRosenblattTransformation1D(self.grid)
ans = np.ndarray(len(x))
for i, xi in enumerate(x.array()):
ans[i] = op.doTransformation1D(self.alpha, xi)
if len(ans) == 1:
return ans[0]
else:
return ans
else:
if isinstance(x, DataMatrix):
A = x
B = DataMatrix(A.getNrows(), A.getNcols())
B.setAll(0.0)
elif isinstance(x, DataVector):
A = DataMatrix(1, len(x))
A.setRow(0, x)
B = DataMatrix(1, len(x))
B.setAll(0)
# do the transformation
op = createOperationInverseRosenblattTransformation(self.grid)
op.doTransformation(self.alpha, A, B)
# extract the outcome
if isNumerical(x) or isinstance(x, DataVector):
return B.get(0, 0)
elif isinstance(x, DataMatrix):
return B.array()
def computeMoments(self, iterations=None, ts=None):
names = ['time',
'iteration',
'grid_size',
'mean',
'meanDiscretizationError',
'var',
'varDiscretizationError']
# parameters
if ts is None:
ts = self.__knowledge.getAvailableTimeSteps()
if iterations is None:
iterations = self.__knowledge.getAvailableIterations()
nrows = len(ts) * len(iterations)
ncols = len(names)
data = DataMatrix(nrows, ncols)
v = DataVector(ncols)
row = 0
for t in ts:
for iteration in iterations:
size = self.__knowledge.getGrid(qoi=self._qoi,
iteration=iteration).getSize()
v.setAll(0.0)
v[0] = t
v[1] = iteration
v[2] = size
v[3], v[4] = self.mean(ts=[t], iterations=[iteration])
v[5], v[6] = self.var(ts=[t], iterations=[iteration])
# write results to matrix
data.setRow(row, v)
row += 1
return {'data': data,
'names': names}
def loadData(self, name = "train", delimiter=',', target_col=-1):
fin = self.__gzOpen(self.filename, "r")
reader = csv.reader(fin, delimiter = delimiter)
data = []
classes = []
hasclass = False
target_col = -1
first_line = reader.next()
# training set has to contain targets
if name == 'train':
if len(first_line) <= target_col:
raise Exception('Target column does not match total column number.')
for i in xrange(len(first_line)-1):
data.append([])
hasclass = True
# test set may contain target values
if name == 'test':
if len(first_line) > target_col:
for i in xrange(len(first_line)-1): data.append([])
hasclass = True
else:
for i in xrange(len(first_line)): data.append([])
hasclass = False
# skip header if available
if first_line[0].isalpha():
pass
else:
line = first_line
if hasclass:
classes.append(float(line[target_col]))
line.remove(line[target_col])
for i in xrange(len(line)):
data[i].append(float(line[i]))
for line in reader:
if hasclass:
classes.append(float(line[target_col]))
line.remove(line[target_col])
for i in xrange(len(line)):
data[i].append(float(line[i]))
# cleaning up and return
fin.close()
dim = len(data)
size = len(data[0])
dataMatrix = DataMatrix(size, dim)
tempVector = DataVector(dim)
valuesVector = DataVector(size)
for rowIndex in xrange(size):
for colIndex in xrange(dim):
tempVector[colIndex] = data[colIndex][rowIndex]
dataMatrix.setRow(rowIndex, tempVector)
valuesVector[rowIndex] = classes[rowIndex]
return DataContainer(dataMatrix, valuesVector, name, self.filename)
def plotSG2d(grid, alpha, addContour=True, n=100):
gs = grid.getStorage()
gpxp = []
gpyp = []
gpxn = []
gpyn = []
for i in xrange(gs.size()):
if alpha[i] > 0:
gpxp.append(gs.get(i).getCoord(0))
gpyp.append(gs.get(i).getCoord(1))
else:
gpxn.append(gs.get(i).getCoord(0))
gpyn.append(gs.get(i).getCoord(1))
x = np.linspace(0, 1, n)
y = np.linspace(0, 1, n)
X, Y = np.meshgrid(x, y)
Z = np.ones(n * n).reshape(n, n)
neg_x = []
neg_y = []
neg_z = []
A = DataMatrix(n * n, 2)
p = DataVector(2)
# do vectorized evaluation
k = 0
for i in xrange(len(X)):
for j, (xi, yi) in enumerate(zip(X[i], Y[i])):
p[0] = xi
p[1] = 1 - yi
A.setRow(k, p)
k += 1
res = evalSGFunctionMulti(grid, alpha, A)
k = 0
for i in xrange(len(X)):
for j, (xi, yi) in enumerate(zip(X[i], Y[i])):
Z[i, j] = res[k]
if Z[i, j] < 0 and abs(Z[i, j]) > 1e-13:
neg_x.append(xi)
neg_y.append(1 - yi)
neg_z.append(res[k])
k += 1
plt.imshow(Z, interpolation='bilinear', extent=(0, 1, 0, 1))
if len(neg_z) > 0:
plt.plot(neg_x, neg_y, linestyle=' ', marker='o', color='red')
plt.title("[%g, %g]" % (min(neg_z), max(neg_z)))
# plot surpluses
plt.plot(gpxp, gpyp, "^ ", color="white")
plt.plot(gpxn, gpyn, "v ", color="red")
plt.jet()
plt.colorbar()
if addContour:
cs = plt.contour(X, 1 - Y, Z, colors='black')
plt.clabel(cs, inline=1, fontsize=18)
return res
