def getAIC(S, n, clusterAssignments):
allBlockCovs = idcHelper.getBlockCovariance(S, clusterAssignments)
nrEdges = getNumberOfEdges(allBlockCovs)
ln = getUnpenalizedLogLikelihood(allBlockCovs, n)
EBICcriteria = -2.0 * ln + 2.0 * nrEdges
return EBICcriteria
def getNegLogMarginalLikelihoodNew(S, n, clusterAssignments, df):
p = S.shape[0]
allBlockCovs = idcHelper.getBlockCovariance(S, clusterAssignments)
totalLogMarginal = -0.5 * n * p * numpy.log(
2.0 * numpy.pi) # needs to be updated
for blockCov in allBlockCovs:
totalLogMarginal += getLogMarginalOfWishartOneBlock(blockCov, n, df)
return -1.0 * totalLogMarginal
def getEBIC(S, n, clusterAssignments, gamma):
p = S.shape[0]
# if p >= n:
# print "********* WARNING: p >= n ***********"
allBlockCovs = idcHelper.getBlockCovariance(S, clusterAssignments)
nrEdges = getNumberOfEdges(allBlockCovs)
ln = getUnpenalizedLogLikelihood(allBlockCovs, n)
EBICcriteria = -2.0 * ln + nrEdges * numpy.log(
n) + 4.0 * nrEdges * gamma * numpy.log(p)
return EBICcriteria
def optimizePosteriorNus(PRIOR_SCALE_FOR_CLUSTER, PRIOR_SCALE_FOR_NOISE, n,
sortedSampleCovariance, allClusterSizes,
sortedClustering, noisePrecisionMode,
allClusterPrecisionModes):
p = sortedSampleCovariance.shape[0]
allBlockSampleCovs = idcHelper.getBlockCovariance(sortedSampleCovariance,
sortedClustering)
allJointTraces, allClusterTraces, noiseTrace = KLapproximation.precalcuateRelevantTraces(
sortedClustering, PRIOR_SCALE_FOR_CLUSTER, PRIOR_SCALE_FOR_NOISE, n,
sortedSampleCovariance, allBlockSampleCovs, noisePrecisionMode,
allClusterPrecisionModes, 1.0)
def funcPosteriorNuNoiseScalar(posteriorNu_local):
priorNuNoise = float(p + 1)
return KLapproximation.oneSigmaPart(priorNuNoise, n, p, noiseTrace,
posteriorNu_local, 0)
SEARCH_INTERVAL = (float(p + 1), float(100 * (p + n + 1)))
result = scipy.optimize.minimize_scalar(funcPosteriorNuNoiseScalar,
bounds=SEARCH_INTERVAL,
method='bounded')
posteriorNuNoise = result.x
totalKL = funcPosteriorNuNoiseScalar(posteriorNuNoise)
allPosteriorNuClusters = numpy.zeros(len(allClusterSizes))
for j in xrange(len(allClusterSizes)):
clusterSize = allClusterSizes[j]
assert (clusterSize == allClusterPrecisionModes[j].shape[0])
priorNuCluster = float(clusterSize + 1)
def funcPosteriorNuClusterScalar(posteriorNu_local):
returnValue = KLapproximation.oneSigmaPart(priorNuCluster, n,
clusterSize,
allClusterTraces[j],
posteriorNu_local, n)
return returnValue
SEARCH_INTERVAL = (float(clusterSize + 1),
float(100.0 * (clusterSize + n + 1)))
result = scipy.optimize.minimize_scalar(funcPosteriorNuClusterScalar,
bounds=SEARCH_INTERVAL,
method='bounded')
allPosteriorNuClusters[j] = result.x
totalKL += funcPosteriorNuClusterScalar(allPosteriorNuClusters[j])
infoStr = str(posteriorNuNoise) + "," + str(allPosteriorNuClusters)
print infoStr + " klDivApprox = " + str(totalKL)
return posteriorNuNoise, allPosteriorNuClusters
def testMatrix(fullCovarianceMatrix, clusterAssignments):
p = fullCovarianceMatrix.shape[0]
fullPrecisionMatrix = numpy.linalg.inv(fullCovarianceMatrix)
fullPrecisionMatrixOnlyBlocks = idcHelper.createFullX(
p, idcHelper.getBlockCovariance(fullPrecisionMatrix,
clusterAssignments))
reducedPrecision = fullPrecisionMatrix - fullPrecisionMatrixOnlyBlocks
alphaMin = 0.0
alphaMax = 1.0
alpha = None
for i in xrange(50):
assert (alphaMax > alphaMin)
if (alphaMax - alphaMin) < 0.00001:
break
alpha = (alphaMax + alphaMin) / 2.0
lambdaMin = getLambdaMin(alpha, fullPrecisionMatrixOnlyBlocks,
reducedPrecision)
# print "alphaMin = ", alphaMin
# print "alphaMax = ", alphaMax
# print "lambdaMin = ", lambdaMin
if lambdaMin <= 0.0:
# need to increase alpha
alphaMin = alpha
else:
alphaMax = alpha
alpha += 0.0001
print "alpha = ", alpha
assert (getLambdaMin(alpha, fullPrecisionMatrixOnlyBlocks,
reducedPrecision) > 0.0)
return
def precalcuateRelevantTraces(sortedClustering, PRIOR_SCALE_FOR_CLUSTER,
PRIOR_SCALE_FOR_NOISE, n, sampleCov,
allBlockSampleCovs, noisePrecisionMode,
allClusterPrecisionModes, beta):
allNoisePrecisionModeBlocks = idcHelper.getBlockCovariance(
noisePrecisionMode, sortedClustering)
assert (len(allNoisePrecisionModeBlocks) == len(allClusterPrecisionModes))
allJointTraces = []
for j in xrange(len(allClusterPrecisionModes)):
covModeBlock = numpy.linalg.inv(allClusterPrecisionModes[j])
jointTrace = idcHelper.matrixInnerProdSymmetric(
covModeBlock, allNoisePrecisionModeBlocks[j])
allJointTraces.append(jointTrace)
allClusterTraces = []
for j in xrange(len(allClusterPrecisionModes)):
clusterPrecisionMode = allClusterPrecisionModes[j]
clusterSize = clusterPrecisionMode.shape[0]
priorSigma = PRIOR_SCALE_FOR_CLUSTER * numpy.eye(clusterSize)
assert (clusterSize == clusterPrecisionMode.shape[0])
traceMat = n * allBlockSampleCovs[j] + priorSigma
clusterTrace = idcHelper.matrixInnerProdSymmetric(
traceMat, clusterPrecisionMode)
allClusterTraces.append(clusterTrace)
p = noisePrecisionMode.shape[0]
priorSigma = PRIOR_SCALE_FOR_NOISE * numpy.eye(p)
traceMatNoise = beta * n * sampleCov + priorSigma
noiseTrace = idcHelper.matrixInnerProdSymmetric(traceMatNoise,
noisePrecisionMode)
return allJointTraces, allClusterTraces, noiseTrace
def findPosteriorMode3Block(PRIOR_SCALE_FOR_CLUSTER, PRIOR_SCALE_FOR_NOISE, n,
dataSampleCov, allClusterSizes, beta, df):
assert (type(dataSampleCov) == numpy.matrixlib.defmatrix.matrix)
clusterAssignments = marginalHelper.getSimpleClusterAssignments(
allClusterSizes)
p = dataSampleCov.shape[0]
priorNuNoise = df + p + 1
aFacNoise = priorNuNoise + p + 1
priorSigmaNoise = PRIOR_SCALE_FOR_NOISE * numpy.eye(p)
# nS = n * dataSampleCov
assert (PRIOR_SCALE_FOR_NOISE >= 0.01)
assert (PRIOR_SCALE_FOR_CLUSTER >= 0.01)
k = len(allClusterSizes)
assert (k >= 1)
# initialization
allSblocks = idcHelper.getBlockCovariance(dataSampleCov,
clusterAssignments)
U = numpy.zeros((p, p))
Z = numpy.zeros((p, p))
allX = []
allAFactors = []
allPriorSigmas = []
for j in xrange(k):
clusterSize = allClusterSizes[j]
priorNuCluster = df + clusterSize + 1
priorSigmaCluster = PRIOR_SCALE_FOR_CLUSTER * numpy.eye(clusterSize)
aFactor = priorNuCluster + clusterSize + 1
allAFactors.append(aFactor)
allPriorSigmas.append(priorSigmaCluster)
allX.append(
getInitialPrecisionMatrix(n, aFactor, allSblocks[j],
priorSigmaCluster)) # initial X
Xnoise = getInitialPrecisionMatrix(n, aFacNoise, dataSampleCov,
priorSigmaNoise)
MAX_ADMM_ITERATIONS = 100000
rho = 1.0
previousObjValue = float("inf")
for admmIt in xrange(MAX_ADMM_ITERATIONS):
allUs = idcHelper.getBlockCovariance(U, clusterAssignments)
allZs = idcHelper.getBlockCovariance(Z, clusterAssignments)
# perform X update
allXnoise = idcHelper.getBlockCovariance(Xnoise, clusterAssignments)
for j in xrange(k):
rightHandSideM = (-1.0 / allAFactors[j]) * (
allPriorSigmas[j] + rho *
(beta * allXnoise[j] - allZs[j]) + allUs[j])
allX[j] = idcHelper.getFastDiagonalSolutionMoreStable(
rightHandSideM, rho / allAFactors[j])
# perform X_epsilon update
X = idcHelper.createFullX(p, allX)
rightHandSideM = (-1.0 / aFacNoise) * (priorSigmaNoise + beta * rho *
(X - Z) + beta * U)
Xnoise = idcHelper.getFastDiagonalSolutionMoreStable(
rightHandSideM, (rho * beta * beta) / aFacNoise)
# perform Z update
rightHandSideM = (1.0 /
float(n)) * (U + rho *
(X + beta * Xnoise)) - dataSampleCov
Z = idcHelper.getFastDiagonalSolutionMoreStable(
rightHandSideM, rho / float(n))
# perform U update
U = U + rho * (X + beta * Xnoise - Z)
objValue = getObjValue(PRIOR_SCALE_FOR_CLUSTER, PRIOR_SCALE_FOR_NOISE,
n, dataSampleCov, allClusterSizes, Xnoise, allX,
beta, df)
if admmIt % 100 == 0:
print(str(admmIt) + ", objValue = " + str(objValue))
rho = rho * 1.1
# print "rho = ", rho
# idcHelper.assertValidAndNotInfinite(objValue)
VERY_ACCURATE = 0.000001
if numpy.isfinite(objValue) and numpy.abs(
objValue - previousObjValue) < VERY_ACCURATE:
print "REACHED CONVERGENCE"
break
previousObjValue = objValue
idcHelper.assertValidAndNotInfinite(objValue)
return Xnoise, allX, objValue