def findNextBestCovariateIdDEBUG(fullCovMatrix, trainData, trainLabels,
misclassificationCosts, definedFeatureCosts,
idsGivenCovariates, observedCovariates,
allFeatureListsInOrder):
assert (trainData.shape[0] == trainLabels.shape[0])
p = trainData.shape[1]
remainingCovariates = set(numpy.arange(p)) - set(idsGivenCovariates)
fixedQueryCovariates = numpy.asarray([], dtype=numpy.int)
if len(remainingCovariates) == 0:
# no new covariates can be acquired
return None, float("inf")
numberOfFeatureSets = len(allFeatureListsInOrder)
currentIdForGivenFeatures = None
for i in range(numberOfFeatureSets):
if len(idsGivenCovariates) == len(allFeatureListsInOrder[i]):
assert (numpy.all(
numpy.equal(idsGivenCovariates, allFeatureListsInOrder[i])))
currentIdForGivenFeatures = i
break
assert (currentIdForGivenFeatures is not None)
bestLocalTotalExpectedFutureCosts = float("inf")
for j in range(currentIdForGivenFeatures + 1, numberOfFeatureSets):
idsNextCovariates = allFeatureListsInOrder[j]
idsQueryCovariates = idsNextCovariates[
idsGivenCovariates.shape[0]:idsNextCovariates.shape[0]]
covariatesForPrediction = numpy.append(idsGivenCovariates,
idsQueryCovariates)
idsMarginalizeOutCovariates = numpy.arange(p)
idsMarginalizeOutCovariates = numpy.delete(idsMarginalizeOutCovariates,
covariatesForPrediction)
assert (len(
set(idsMarginalizeOutCovariates) | set(idsQueryCovariates)
| set(idsGivenCovariates)) == p)
predictionModel, _ = prepareFeatureSets.getOptimalTrainedModel(
trainData, trainLabels, covariatesForPrediction)
totalExpectedFutureCosts = getTotalExpectedFutureCosts(
fullCovMatrix, predictionModel, misclassificationCosts,
definedFeatureCosts, idsQueryCovariates,
idsMarginalizeOutCovariates, idsGivenCovariates,
observedCovariates)
if totalExpectedFutureCosts < bestLocalTotalExpectedFutureCosts:
bestLocalTotalExpectedFutureCosts = totalExpectedFutureCosts
fixedQueryCovariates = numpy.append(fixedQueryCovariates,
idsQueryCovariates)
return fixedQueryCovariates[0], bestLocalTotalExpectedFutureCosts
def findNextBestCovariateIdForDebugging(fullCovMatrix, trainData, trainLabels,
misclassificationCosts,
definedFeatureCosts,
idsGivenCovariates,
observedCovariates):
assert (trainData.shape[0] == trainLabels.shape[0])
p = trainData.shape[1]
remainingCovariates = set(numpy.arange(p)) - set(idsGivenCovariates)
fixedQueryCovariates = numpy.asarray([], dtype=numpy.int)
globalBestTotalExpectedFutureCosts = float("inf")
if len(remainingCovariates) == 0:
# no new covariates can be acquired
return None, float("inf")
while len(remainingCovariates) > 0:
bestLocalNextQueryId = None
bestLocalTotalExpectedFutureCosts = float("inf")
for j in remainingCovariates:
idsQueryCovariates = numpy.append(fixedQueryCovariates, [j])
covariatesForPrediction = numpy.append(idsGivenCovariates,
idsQueryCovariates)
idsMarginalizeOutCovariates = numpy.arange(p)
idsMarginalizeOutCovariates = numpy.delete(
idsMarginalizeOutCovariates, covariatesForPrediction)
assert (len(
set(idsMarginalizeOutCovariates) | set(idsQueryCovariates)
| set(idsGivenCovariates)) == p)
predictionModel, _ = prepareFeatureSets.getOptimalTrainedModel(
trainData, trainLabels, covariatesForPrediction)
totalExpectedFutureCosts = getTotalExpectedFutureCosts(
fullCovMatrix, predictionModel, misclassificationCosts,
definedFeatureCosts, idsQueryCovariates,
idsMarginalizeOutCovariates, idsGivenCovariates,
observedCovariates)
if totalExpectedFutureCosts < bestLocalTotalExpectedFutureCosts:
bestLocalNextQueryId = j
bestLocalTotalExpectedFutureCosts = totalExpectedFutureCosts
fixedQueryCovariates = numpy.append(fixedQueryCovariates,
[bestLocalNextQueryId])
remainingCovariates.remove(bestLocalNextQueryId)
if bestLocalTotalExpectedFutureCosts < globalBestTotalExpectedFutureCosts:
globalBestTotalExpectedFutureCosts = bestLocalTotalExpectedFutureCosts
return fixedQueryCovariates[0], globalBestTotalExpectedFutureCosts
def getBayesRiskFromCurrentTestSample(trainData, trainLabels, testSample,
misclassificationCosts,
selectedFeatureIds):
predictionModel, holdOutDataAccuracyEstimate = prepareFeatureSets.getOptimalTrainedModel(
trainData, trainLabels, selectedFeatureIds)
predictedProbs = predictionModel.predict_proba(testSample)
print(predictedProbs)
br = prepareFeatureSets.bayesRisk(predictedProbs, misclassificationCosts)
print(br)
assert (False)
return prepareFeatureSets.bayesRisk(predictedProbs, misclassificationCosts)
def prepareForLinearSequence(trainDataFullCovariates, trainLabels, allData,
allFeatureListsInOrder):
p = trainDataFullCovariates.shape[1]
allPredictionModels = []
allSamplerInfos = []
numberOfFeatureSets = len(allFeatureListsInOrder)
for i in range(numberOfFeatureSets):
idsGivenCovariates = allFeatureListsInOrder[i]
# print("******************** i = " + str(i) + " ***********************")
# train classifier
predictionModel, _ = prepareFeatureSets.getOptimalTrainedModel(
trainDataFullCovariates, trainLabels, idsGivenCovariates)
samplerInfo = []
for j in range(i + 1, numberOfFeatureSets):
idsNextCovariates = allFeatureListsInOrder[j]
assert ((trainDataFullCovariates[:, idsNextCovariates]).shape[1] >
0)
idsQueryCovariates = idsNextCovariates[
idsGivenCovariates.shape[0]:idsNextCovariates.shape[0]]
idsMarginalizeOutCovariates = numpy.arange(p)
idsMarginalizeOutCovariates = numpy.delete(
idsMarginalizeOutCovariates,
numpy.hstack((idsQueryCovariates, idsGivenCovariates)))
assert (len(
set(idsMarginalizeOutCovariates) | set(idsQueryCovariates)
| set(idsGivenCovariates)) == p)
samplerInfo.append(
(idsQueryCovariates, idsMarginalizeOutCovariates,
idsGivenCovariates))
allSamplerInfos.append(samplerInfo)
allPredictionModels.append(predictionModel)
fullCovMatrix = numpy.cov(allData.transpose(), bias=True)
return fullCovMatrix, allSamplerInfos, allPredictionModels
def fullDynamicGreedyMethodForDebugging(fullCovMatrix, trainData, trainLabels,
testSample, misclassificationCosts,
definedFeatureCosts, initialFeatureId):
if initialFeatureId is None:
idsGivenCovariates = numpy.asarray([], dtype=numpy.int)
else:
idsGivenCovariates = numpy.asarray([initialFeatureId], dtype=numpy.int)
while (True):
observedCovariates = testSample[idsGivenCovariates]
currentPredictionModel, _ = prepareFeatureSets.getOptimalTrainedModel(
trainData, trainLabels, idsGivenCovariates)
currentBayesRisk = prepareFeatureSets.getBayesRiskSingleObservation(
currentPredictionModel, misclassificationCosts, observedCovariates)
bestNextQueryId, expectedTotalFutureCosts = findNextBestCovariateIdForDebugging(
fullCovMatrix, trainData, trainLabels, misclassificationCosts,
definedFeatureCosts, idsGivenCovariates, observedCovariates)
if expectedTotalFutureCosts < currentBayesRisk:
# acquire covariate "bestNextQueryId"
idsGivenCovariates = numpy.append(idsGivenCovariates,
[bestNextQueryId])
else:
# decide on class label
predictedLabel = experimentHelper.getPredictionLabelSingleObservation(
currentPredictionModel, observedCovariates)
featureCosts = numpy.sum(definedFeatureCosts[idsGivenCovariates])
print("covariateIds = " + str(idsGivenCovariates) +
"| costs of acquired covariates = " + str(featureCosts) +
" | bayes risk = " + str(currentBayesRisk))
return idsGivenCovariates, featureCosts, predictedLabel
return
def evaluateConditionalDensity(trainDataFullCovariates, trainLabels, allData,
allFeatureListsInOrder, testData):
allPredictionModels = []
numberOfFeatureSets = len(allFeatureListsInOrder)
for i in range(numberOfFeatureSets):
idsGivenCovariates = allFeatureListsInOrder[i]
predictionModel, _ = prepareFeatureSets.getOptimalTrainedModel(
trainDataFullCovariates, trainLabels, idsGivenCovariates)
allPredictionModels.append(predictionModel)
for i in range(numberOfFeatureSets):
idsGivenCovariates = allFeatureListsInOrder[i]
if idsGivenCovariates.shape[0] != 6:
continue
print("---------------------------")
print("idsGivenCovariates = ", idsGivenCovariates)
for j in range(i + 1, numberOfFeatureSets):
nextPredictionModel = allPredictionModels[j]
idsNextCovariates = allFeatureListsInOrder[j]
assert ((trainDataFullCovariates[:, idsNextCovariates]).shape[1] >
0)
idsQueryCovariates = idsNextCovariates[
idsGivenCovariates.shape[0]:idsNextCovariates.shape[0]]
beta = nextPredictionModel.coef_[0]
assert (beta.shape[0] == idsNextCovariates.shape[0])
queryBetaPart = beta[idsGivenCovariates.shape[0]:idsNextCovariates.
shape[0]]
print("idsQueryCovariates = ", idsQueryCovariates)
# train simple linear regression model
sigma12TimesSigma22InvQueryPart, newCovMatrixQueryPart = dynamicAcquisition.getConditionalMeanAndVariance(
allData, idsGivenCovariates, idsQueryCovariates)
densityTrainDataCovariates = allData[:, idsGivenCovariates]
densityTrainDataResponse = allData[:,
idsQueryCovariates] @ queryBetaPart
# print("densityTrainDataResponse average = ", numpy.average(densityTrainDataResponse))
densityTestDataCovariates = testData[:, idsGivenCovariates]
densityTestDataResponse = testData[:,
idsQueryCovariates] @ queryBetaPart
# train Gaussian process regression model
# for lengthScale in [2.0, 1.0, 0.5]:
for lengthScale in [100.0, 10.0, 1.0, 0.1, 0.001]:
for alphaValue in [20.0, 10.0, 5.0]:
covFunc = sklearn.gaussian_process.kernels.DotProduct(
sigma_0=lengthScale)
# covFunc = sklearn.gaussian_process.kernels.RBF(length_scale = lengthScale)
GP = sklearn.gaussian_process.GaussianProcessRegressor(
kernel=covFunc,
alpha=alphaValue,
optimizer=None,
normalize_y=False)
# GP = sklearn.gaussian_process.GaussianProcessRegressor(kernel=sklearn.gaussian_process.kernels.RBF(), alpha=0.01, n_restarts_optimizer=5, normalize_y=False)
GP.fit(densityTrainDataCovariates,
densityTrainDataResponse)
# print("GP.kernel_.length_scale = ", GP.kernel_.length_scale)
print("lengthScale = " + str(lengthScale) +
", alphaValue = " + str(alphaValue))
print("GP.log_marginal_likelihood_value_ = ",
GP.log_marginal_likelihood_value_)
heldOutLogLikelihoodGP, mseGP = evalHeldOutGP(
GP, densityTestDataCovariates, densityTestDataResponse)
print("heldOutLogLikelihoodGP = ", heldOutLogLikelihoodGP)
heldOutLogLikelihoodNormal = 0.0
mseNormal = 0.0
for t in range(densityTestDataCovariates.shape[0]):
observedCovariates = densityTestDataCovariates[t]
meanQueryPart = numpy.matmul(sigma12TimesSigma22InvQueryPart,
observedCovariates)
mean = numpy.dot(queryBetaPart, meanQueryPart)
variance = queryBetaPart @ newCovMatrixQueryPart @ queryBetaPart.transpose(
)
heldOutLogLikelihoodNormal += scipy.stats.norm.logpdf(
x=densityTestDataResponse[t],
loc=mean,
scale=numpy.sqrt(variance))
mseNormal += numpy.square(mean - densityTestDataResponse[t])
mseNormal = mseNormal / float(densityTestDataCovariates.shape[0])
print("heldOutLogLikelihoodNormal = ", heldOutLogLikelihoodNormal)
print("mseNormal = ", mseNormal)
return
def evaluateConditionalDensitySimple(trainDataFullCovariates, trainLabels,
allData, testData):
p = trainDataFullCovariates.shape[1]
# idsGivenCovariates = numpy.arange(p-5)
# idsNextCovariates = numpy.arange(p-4)
idsGivenCovariates = numpy.arange(p - 3)
idsNextCovariates = numpy.arange(p - 1)
print("idsGivenCovariates = ", idsGivenCovariates)
print("idsNextCovariates = ", idsNextCovariates)
nextPredictionModel, _ = prepareFeatureSets.getOptimalTrainedModel(
trainDataFullCovariates, trainLabels, idsNextCovariates)
idsQueryCovariates = idsNextCovariates[idsGivenCovariates.
shape[0]:idsNextCovariates.shape[0]]
beta = nextPredictionModel.coef_[0]
assert (beta.shape[0] == idsNextCovariates.shape[0])
queryBetaPart = beta[idsGivenCovariates.shape[0]:idsNextCovariates.
shape[0]]
densityTrainDataCovariates = allData[:, idsGivenCovariates]
densityTrainDataResponse = allData[:, idsQueryCovariates] @ queryBetaPart
print("idsQueryCovariates = ", idsQueryCovariates)
# train simple linear regression model
sigma12TimesSigma22InvQueryPart, newCovMatrixQueryPart = dynamicAcquisition.getConditionalMeanAndVariance(
allData, idsGivenCovariates, idsQueryCovariates)
# print("densityTrainDataResponse average = ", numpy.average(densityTrainDataResponse))
densityTestDataCovariates = testData[:, idsGivenCovariates]
densityTestDataResponse = testData[:, idsQueryCovariates] @ queryBetaPart
# train Gaussian process regression model
# for lengthScale in [2.0, 1.0, 0.5]:
# for lengthScale in [1.0]: # [10000.0, 100.0, 10.0, 1.0, 0.1, 0.001, 0.00001]:
# for alphaValue in [1.0]: # [100.0, 10.0, 1.0, 0.1, 0.001]:
# # covFunc = sklearn.gaussian_process.kernels.DotProduct(sigma_0 = lengthScale)
# covFunc = sklearn.gaussian_process.kernels.RBF(length_scale = lengthScale)
# GP = sklearn.gaussian_process.GaussianProcessRegressor(kernel=covFunc, alpha=alphaValue, optimizer=None, normalize_y=False)
# # GP = sklearn.gaussian_process.GaussianProcessRegressor(kernel=sklearn.gaussian_process.kernels.RBF(), alpha=0.01, n_restarts_optimizer=5, normalize_y=False)
# GP.fit(densityTrainDataCovariates, densityTrainDataResponse)
# # print("GP.kernel_.length_scale = ", GP.kernel_.length_scale)
# print("lengthScale = " + str(lengthScale) + ", alphaValue = " + str(alphaValue))
# # print("GP.log_marginal_likelihood_value_ = ", GP.log_marginal_likelihood_value_)
# heldOutLogLikelihoodGP, mseGP = evalHeldOutGP(GP, densityTestDataCovariates, densityTestDataResponse)
# # print("heldOutLogLikelihood (GP) = ", heldOutLogLikelihoodGP)
# print("MSE (GP) = ", mseGP)
# logLikelihoodTrainGP, mseTrainGP = evalHeldOutGP(GP, densityTrainDataCovariates, densityTrainDataResponse)
# # print("logLikelihoodTrain (GP) = ", logLikelihoodTrainGP)
# # print("mseTrain (GP)= ", mseTrainGP)
#
# train linear regression model
# lrModel = sklearn.linear_model.LinearRegression()
# for alphaValue in [100.0, 10.0, 1.0, 0.1, 0.001, 0.0001, 0.00001]:
# print("alphaValue = ", alphaValue)
# # lrModel = sklearn.linear_model.Ridge(alpha = alphaValue)
# # lrModel.fit(densityTrainDataCovariates, densityTrainDataResponse)
#
# lrModel = sklearn.neural_network.MLPRegressor(hidden_layer_sizes=100, alpha = alphaValue)
# lrModel.fit(densityTrainDataCovariates, densityTrainDataResponse)
# allMeansLR = lrModel.predict(densityTestDataCovariates)
# mseLR = numpy.mean(numpy.square(allMeansLR - densityTestDataResponse))
# print("MSE (linear regression) = ", mseLR)
normalVariance = queryBetaPart @ newCovMatrixQueryPart @ queryBetaPart.transpose(
)
lengthScale = 2.0
alphaValue = 100000.0 # 0.00001 corresponds to 1/sigma_noise
# alphaValue = 1.0 # 1.0 / normalVariance
BRModel(densityTrainDataCovariates, densityTrainDataResponse,
densityTestDataCovariates, densityTestDataResponse, normalVariance)
GPwithLinearMeanModel(densityTrainDataCovariates, densityTrainDataResponse,
densityTestDataCovariates, densityTestDataResponse,
lengthScale, alphaValue)
heldOutLogLikelihoodNormal, mseNormal = evalHeldOutNormal(
queryBetaPart, sigma12TimesSigma22InvQueryPart, newCovMatrixQueryPart,
densityTestDataCovariates, densityTestDataResponse)
print("heldOutLogLikelihoodNormal = ", heldOutLogLikelihoodNormal)
print("mseNormal = ", mseNormal)
logLikelihoodTrainNormal, mseTrainNormal = evalHeldOutNormal(
queryBetaPart, sigma12TimesSigma22InvQueryPart, newCovMatrixQueryPart,
densityTrainDataCovariates, densityTrainDataResponse)
print("logLikelihoodTrainNormal = ", logLikelihoodTrainNormal)
print("mseTrainNormal = ", mseTrainNormal)
print("numpy.mean(densityTestDataResponse) = ",
numpy.mean(densityTestDataResponse))
print("mse of zero estimate = ",
numpy.mean(numpy.square(densityTestDataResponse)))
return
def fullDynamicGreedyMethodWithGoBack(fullCovMatrix, trainData, trainLabels,
testSample, misclassificationCosts,
definedFeatureCosts, initialFeatureId):
if initialFeatureId is None:
idsAllGivenCovariates = numpy.asarray([], dtype=numpy.int)
else:
idsAllGivenCovariates = numpy.asarray([initialFeatureId],
dtype=numpy.int)
lowestBayesRiskCovariateIds = numpy.asarray([], dtype=numpy.int)
lowestBayesRiskClassifier, _ = prepareFeatureSets.getOptimalTrainedModel(
trainData, trainLabels, lowestBayesRiskCovariateIds)
lowestBayesRisk = prepareFeatureSets.getBayesRiskSingleObservation(
lowestBayesRiskClassifier, misclassificationCosts,
testSample[lowestBayesRiskCovariateIds])
print("lowestBayesRisk = ", lowestBayesRisk)
while (True):
observedCovariates = testSample[idsAllGivenCovariates]
currentPredictionModel, _ = prepareFeatureSets.getOptimalTrainedModel(
trainData, trainLabels, idsAllGivenCovariates)
currentBayesRisk = prepareFeatureSets.getBayesRiskSingleObservation(
currentPredictionModel, misclassificationCosts, observedCovariates)
print("idsGivenCovariates = " + str(idsAllGivenCovariates) +
" | bayesRisk = " + str(currentBayesRisk))
if currentBayesRisk < lowestBayesRisk:
lowestBayesRisk = currentBayesRisk
lowestBayesRiskClassifier = currentPredictionModel
lowestBayesRiskCovariateIds = numpy.copy(idsAllGivenCovariates)
bestNextQueryId, expectedTotalFutureCosts = findNextBestCovariateIdForDebugging(
fullCovMatrix, trainData, trainLabels, misclassificationCosts,
definedFeatureCosts, idsAllGivenCovariates, observedCovariates)
if expectedTotalFutureCosts < lowestBayesRisk:
# acquire covariate "bestNextQueryId"
idsAllGivenCovariates = numpy.append(idsAllGivenCovariates,
[bestNextQueryId])
else:
# lowestBayesRisk = float("inf")
# lowestBayesRiskClassifier = None
# lowestBayesRiskCovariateIds = None
#
# # decide on class label
# print("NOW DECIDING ON CLASS LABEL")
# print("idsAllGivenCovariates = ", idsAllGivenCovariates)
# for j in range(len(idsAllGivenCovariates) + 1):
# idsGivenCovariates = idsAllGivenCovariates[0:j]
# observedCovariates = testSample[idsGivenCovariates]
# predictionModel, _ = prepareFeatureSets.getOptimalTrainedModel(trainData, trainLabels, idsGivenCovariates)
# bayesRisk = prepareFeatureSets.getBayesRiskSingleObservation(predictionModel, misclassificationCosts, observedCovariates)
# print("idsGivenCovariates = " + str(idsGivenCovariates) + " | bayesRisk = " + str(bayesRisk))
# if bayesRisk < lowestBayesRisk:
# lowestBayesRisk = bayesRisk
# lowestBayesRiskClassifier = predictionModel
# lowestBayesRiskCovariateIds = idsGivenCovariates
#
# if len(lowestBayesRiskCovariateIds) < len(idsAllGivenCovariates):
# print("!!!! BEST CLASSIFIER REQUIRES LESS COVARIATES THAN REQUIRED !!!! ")
predictedLabel = experimentHelper.getPredictionLabelSingleObservation(
lowestBayesRiskClassifier,
testSample[lowestBayesRiskCovariateIds])
featureCosts = numpy.sum(
definedFeatureCosts[idsAllGivenCovariates])
print("used covariate ids = " + str(lowestBayesRiskCovariateIds) +
", acquired covariateIds = " +
str(lowestBayesRiskCovariateIds) +
"| costs of acquired covariates = " + str(featureCosts) +
" | bayes risk = " + str(lowestBayesRisk))
# return the ids of the acquired(!) covariates, its costs, and the predicted label (which is based on lowestBayesRiskCovariateIds)
return idsAllGivenCovariates, featureCosts, predictedLabel
return