def testGetOmegaListPtr(self):
import sppy
m = 10
n = 5
X = scipy.sparse.rand(m, n, 0.1)
X = X.tocsr()
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
for i in range(m):
omegai = colInds[indPtr[i]:indPtr[i + 1]]
nptst.assert_array_almost_equal(omegai,
X.toarray()[i, :].nonzero()[0])
Xsppy = sppy.csarray(X)
indPtr, colInds = SparseUtils.getOmegaListPtr(Xsppy)
for i in range(m):
omegai = colInds[indPtr[i]:indPtr[i + 1]]
nptst.assert_array_almost_equal(omegai,
X.toarray()[i, :].nonzero()[0])
#Test a zero array (scipy doesn't work in this case)
X = sppy.csarray((m, n))
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
for i in range(m):
omegai = colInds[indPtr[i]:indPtr[i + 1]]
def testLocalAucApprox(self):
m = 100
n = 200
k = 2
X, U, s, V, wv = SparseUtils.generateSparseBinaryMatrix((m, n),
k,
csarray=True,
verbose=True)
w = 1.0
localAuc = MCEvaluator.localAUC(X, U, V, w)
samples = numpy.arange(150, 200, 10)
for i, sampleSize in enumerate(samples):
numAucSamples = sampleSize
localAuc2 = MCEvaluator.localAUCApprox(
SparseUtils.getOmegaListPtr(X), U, V, w, numAucSamples)
self.assertAlmostEqual(localAuc2, localAuc, 1)
#Try smaller w
w = 0.5
localAuc = MCEvaluator.localAUC(X, U, V, w)
samples = numpy.arange(50, 200, 10)
for i, sampleSize in enumerate(samples):
numAucSamples = sampleSize
localAuc2 = MCEvaluator.localAUCApprox(
SparseUtils.getOmegaListPtr(X), U, V, w, numAucSamples)
self.assertAlmostEqual(localAuc2, localAuc, 1)
def testLocalAucApprox(self):
m = 100
n = 200
k = 2
X, U, s, V, wv = SparseUtils.generateSparseBinaryMatrix((m, n), k, csarray=True, verbose=True)
w = 1.0
localAuc = MCEvaluator.localAUC(X, U, V, w)
samples = numpy.arange(150, 200, 10)
for i, sampleSize in enumerate(samples):
numAucSamples = sampleSize
localAuc2 = MCEvaluator.localAUCApprox(SparseUtils.getOmegaListPtr(X), U, V, w, numAucSamples)
self.assertAlmostEqual(localAuc2, localAuc, 1)
# Try smaller w
w = 0.5
localAuc = MCEvaluator.localAUC(X, U, V, w)
samples = numpy.arange(50, 200, 10)
for i, sampleSize in enumerate(samples):
numAucSamples = sampleSize
localAuc2 = MCEvaluator.localAUCApprox(SparseUtils.getOmegaListPtr(X), U, V, w, numAucSamples)
self.assertAlmostEqual(localAuc2, localAuc, 1)
def testGetOmegaListPtr(self):
import sppy
m = 10
n = 5
X = scipy.sparse.rand(m, n, 0.1)
X = X.tocsr()
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
for i in range(m):
omegai = colInds[indPtr[i]:indPtr[i+1]]
nptst.assert_array_almost_equal(omegai, X.toarray()[i, :].nonzero()[0])
Xsppy = sppy.csarray(X)
indPtr, colInds = SparseUtils.getOmegaListPtr(Xsppy)
for i in range(m):
omegai = colInds[indPtr[i]:indPtr[i+1]]
nptst.assert_array_almost_equal(omegai, X.toarray()[i, :].nonzero()[0])
#Test a zero array (scipy doesn't work in this case)
X = sppy.csarray((m,n))
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
for i in range(m):
omegai = colInds[indPtr[i]:indPtr[i+1]]
def learnPredictRanking(args):
"""
A function to train on a training set and test on a test set, for a number
of values of rho.
"""
learner, trainX, testX, rhos = args
logging.debug("k=" + str(learner.getK()))
logging.debug(learner)
testInds = testX.nonzero()
trainXIter = []
testIndList = []
for rho in rhos:
trainXIter.append(trainX)
testIndList.append(testInds)
trainXIter = iter(trainXIter)
ZIter = learner.learnModel(trainXIter, iter(rhos))
metrics = numpy.zeros(rhos.shape[0])
for j, Z in enumerate(ZIter):
U, s, V = Z
U = U * s
U = numpy.ascontiguousarray(U)
V = numpy.ascontiguousarray(V)
testOrderedItems = MCEvaluatorCython.recommendAtk(
U, V, learner.recommendSize, trainX)
if learner.metric == "mrr":
metrics[j] = MCEvaluator.mrrAtK(SparseUtils.getOmegaListPtr(testX),
testOrderedItems,
learner.recommendSize)
logging.debug("MRR@" + str(learner.recommendSize) + ": " +
str('%.4f' % metrics[j]) + " " + str(learner))
elif learner.metric == "f1":
metrics[j] = MCEvaluator.mrrAtK(SparseUtils.getOmegaListPtr(testX),
testOrderedItems,
learner.recommendSize)
logging.debug("F1@" + str(learner.recommendSize) + ": " +
str('%.4f' % metrics[j]) + " " + str(learner))
else:
raise ValueError("Unknown metric " + learner.metric)
gc.collect()
return metrics
def localAUCApprox(positiveArray,
U,
V,
w,
numAucSamples=50,
r=None,
allArray=None):
"""
Compute the estimated local AUC for the score functions UV^T relative to X with
quantile w. The AUC is computed using positiveArray which is a tuple (indPtr, colInds)
assuming allArray is None. If allArray is not None then positive items are chosen
from positiveArray and negative ones are chosen to complement allArray.
"""
if type(positiveArray) != tuple:
positiveArray = SparseUtils.getOmegaListPtr(positiveArray)
indPtr, colInds = positiveArray
U = numpy.ascontiguousarray(U)
V = numpy.ascontiguousarray(V)
if r is None:
r = SparseUtilsCython.computeR(U, V, w, numAucSamples)
if allArray is None:
return MCEvaluatorCython.localAUCApprox(indPtr, colInds, indPtr,
colInds, U, V,
numAucSamples, r)
else:
allIndPtr, allColInd = allArray
return MCEvaluatorCython.localAUCApprox(indPtr, colInds, allIndPtr,
allColInd, U, V,
numAucSamples, r)
def f1AtK(positiveArray, orderedItems, k, verbose=False):
"""
Return the F1@k measure for each row of the predicted matrix UV.T
using real values in positiveArray. positiveArray is a tuple (indPtr, colInds)
:param orderedItems: The ordered items for each user (users are rows, items are cols)
:param verbose: If true return recall and first k recommendation for each row, otherwise just precisions
"""
if type(positiveArray) != tuple:
positiveArray = SparseUtils.getOmegaListPtr(positiveArray)
orderedItems = orderedItems[:, 0:k]
indPtr, colInds = positiveArray
precisions = MCEvaluatorCython.precisionAtk(indPtr, colInds,
orderedItems)
recalls = MCEvaluatorCython.recallAtk(indPtr, colInds, orderedItems)
denominator = precisions + recalls
denominator += denominator == 0
f1s = 2 * precisions * recalls / denominator
if verbose:
return f1s, orderedItems
else:
return f1s.mean()
def stratifiedRecallAtK(positiveArray,
orderedItems,
k,
itemCounts,
beta=0.5,
verbose=False):
"""
Compute the average recall@k score for each row of the predicted matrix UV.T
using real values in positiveArray. positiveArray is a tuple (indPtr, colInds)
:param orderedItems: The ordered items for each user (users are rows, items are cols)
:param verbose: If true return recall and first k recommendation for each row, otherwise just precisions
"""
if type(positiveArray) != tuple:
positiveArray = SparseUtils.getOmegaListPtr(positiveArray)
orderedItems = orderedItems[:, 0:k]
indPtr, colInds = positiveArray
recalls, denominators = MCEvaluatorCython.stratifiedRecallAtk(
indPtr, colInds, orderedItems, itemCounts, beta)
if verbose:
return recalls, orderedItems
else:
return numpy.average(recalls, weights=denominators)
def profileDerivativeUiApprox(self):
k = 10
U = numpy.random.rand(self.m, k)
V = numpy.random.rand(self.n, k)
indPtr, colInds = SparseUtils.getOmegaListPtr(self.X)
gp = numpy.random.rand(self.n)
gp /= gp.sum()
gq = numpy.random.rand(self.n)
gq /= gq.sum()
j = 3
numRowSamples = 100
numAucSamples = 10
permutedRowInds = numpy.array(numpy.random.permutation(self.m), numpy.uint32)
permutedColInds = numpy.array(numpy.random.permutation(self.n), numpy.uint32)
maxLocalAuc = MaxLocalAUC(k, w=0.9)
normGp, normGq = maxLocalAuc.computeNormGpq(indPtr, colInds, gp, gq, self.m)
lmbda = 0.001
normalise = True
learner = MaxLocalAUCCython()
def run():
numRuns = 10
for j in range(numRuns):
for i in range(self.m):
learner.derivativeUiApprox(indPtr, colInds, U, V, gp, gq, permutedColInds, i)
ProfileUtils.profile("run()", globals(), locals())
def profileObjective(self):
k = 10
U = numpy.random.rand(self.m, k)
V = numpy.random.rand(self.n, k)
indPtr, colInds = SparseUtils.getOmegaListPtr(self.X)
colIndsProbabilities = numpy.ones(colInds.shape[0])
for i in range(self.m):
colIndsProbabilities[indPtr[i] : indPtr[i + 1]] /= colIndsProbabilities[indPtr[i] : indPtr[i + 1]].sum()
colIndsProbabilities[indPtr[i] : indPtr[i + 1]] = numpy.cumsum(
colIndsProbabilities[indPtr[i] : indPtr[i + 1]]
)
r = numpy.zeros(self.m)
lmbda = 0.001
rho = 1.0
numAucSamples = 100
def run():
numRuns = 10
for i in range(numRuns):
objectiveApprox(indPtr, colInds, indPtr, colInds, U, V, r, numAucSamples, lmbda, rho, False)
ProfileUtils.profile("run()", globals(), locals())
def profileObjective(self):
k = 10
U = numpy.random.rand(self.m, k)
V = numpy.random.rand(self.n, k)
indPtr, colInds = SparseUtils.getOmegaListPtr(self.X)
colIndsProbabilities = numpy.ones(colInds.shape[0])
for i in range(self.m):
colIndsProbabilities[indPtr[i]:indPtr[
i + 1]] /= colIndsProbabilities[indPtr[i]:indPtr[i + 1]].sum()
colIndsProbabilities[indPtr[i]:indPtr[i + 1]] = numpy.cumsum(
colIndsProbabilities[indPtr[i]:indPtr[i + 1]])
r = numpy.zeros(self.m)
lmbda = 0.001
rho = 1.0
numAucSamples = 100
def run():
numRuns = 10
for i in range(numRuns):
objectiveApprox(indPtr, colInds, indPtr, colInds, U, V, r,
numAucSamples, lmbda, rho, False)
ProfileUtils.profile('run()', globals(), locals())
def f1AtK(positiveArray, orderedItems, k, verbose=False):
"""
Return the F1@k measure for each row of the predicted matrix UV.T
using real values in positiveArray. positiveArray is a tuple (indPtr, colInds)
:param orderedItems: The ordered items for each user (users are rows, items are cols)
:param verbose: If true return recall and first k recommendation for each row, otherwise just precisions
"""
if type(positiveArray) != tuple:
positiveArray = SparseUtils.getOmegaListPtr(positiveArray)
orderedItems = orderedItems[:, 0:k]
indPtr, colInds = positiveArray
precisions = MCEvaluatorCython.precisionAtk(indPtr, colInds, orderedItems)
recalls = MCEvaluatorCython.recallAtk(indPtr, colInds, orderedItems)
denominator = precisions+recalls
denominator += denominator == 0
f1s = 2*precisions*recalls/denominator
if verbose:
return f1s, orderedItems
else:
return f1s.mean()
def testLocalAucApprox2(self):
m = 100
n = 200
k = 5
numInds = 100
X, U, s, V, wv = SparseUtils.generateSparseBinaryMatrix((m, n),
k,
csarray=True,
verbose=True)
r = numpy.ones(m) * -10
w = 0.5
localAuc = MCEvaluator.localAUC(X, U, V, w)
samples = numpy.arange(50, 200, 10)
for i, sampleSize in enumerate(samples):
localAuc2 = MCEvaluator.localAUCApprox(
SparseUtils.getOmegaListPtr(X), U, V, w, sampleSize)
self.assertAlmostEqual(localAuc2, localAuc, 1)
#Test more accurately
sampleSize = 1000
localAuc2 = MCEvaluator.localAUCApprox(SparseUtils.getOmegaListPtr(X),
U, V, w, sampleSize)
self.assertAlmostEqual(localAuc2, localAuc, 2)
#Now set a high r
Z = U.dot(V.T)
localAuc = MCEvaluator.localAUCApprox(SparseUtils.getOmegaListPtr(X),
U, V, w, sampleSize)
for i, sampleSize in enumerate(samples):
localAuc2 = MCEvaluator.localAUCApprox(
SparseUtils.getOmegaListPtr(X), U, V, w, sampleSize)
self.assertAlmostEqual(localAuc2, localAuc, 1)
#Test more accurately
sampleSize = 1000
localAuc2 = MCEvaluator.localAUCApprox(SparseUtils.getOmegaListPtr(X),
U, V, w, sampleSize)
self.assertAlmostEqual(localAuc2, localAuc, 2)
def testScale(self):
"""
Look at the scales of the unnormalised gradients.
"""
m = 100
n = 400
k = 3
X = SparseUtils.generateSparseBinaryMatrix((m, n), k, csarray=True)
w = 0.1
eps = 0.001
learner = MaxAUCTanh(k, w)
learner.normalise = False
learner.lmbdaU = 1.0
learner.lmbdaV = 1.0
learner.rho = 1.0
learner.numAucSamples = 100
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
r = numpy.random.rand(m)
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
gi = numpy.random.rand(m)
gi /= gi.sum()
gp = numpy.random.rand(n)
gp /= gp.sum()
gq = numpy.random.rand(n)
gq /= gq.sum()
permutedRowInds = numpy.array(numpy.random.permutation(m),
numpy.uint32)
permutedColInds = numpy.array(numpy.random.permutation(n),
numpy.uint32)
maxLocalAuc = MaxLocalAUC(k, w)
normGp, normGq = maxLocalAuc.computeNormGpq(indPtr, colInds, gp, gq, m)
normDui = 0
for i in range(m):
du = learner.derivativeUi(indPtr, colInds, U, V, r, gi, gp, gq, i)
normDui += numpy.linalg.norm(du)
normDui /= float(m)
print(normDui)
normDvi = 0
for i in range(n):
dv = learner.derivativeVi(indPtr, colInds, U, V, r, gi, gp, gq, i)
normDvi += numpy.linalg.norm(dv)
normDvi /= float(n)
print(normDvi)
def learnPredictRanking(args):
"""
A function to train on a training set and test on a test set, for a number
of values of rho.
"""
learner, trainX, testX, rhos = args
logging.debug("k=" + str(learner.getK()))
logging.debug(learner)
testInds = testX.nonzero()
trainXIter = []
testIndList = []
for rho in rhos:
trainXIter.append(trainX)
testIndList.append(testInds)
trainXIter = iter(trainXIter)
ZIter = learner.learnModel(trainXIter, iter(rhos))
metrics = numpy.zeros(rhos.shape[0])
for j, Z in enumerate(ZIter):
U, s, V = Z
U = U*s
U = numpy.ascontiguousarray(U)
V = numpy.ascontiguousarray(V)
testOrderedItems = MCEvaluatorCython.recommendAtk(U, V, learner.recommendSize, trainX)
if learner.metric == "mrr":
metrics[j] = MCEvaluator.mrrAtK(SparseUtils.getOmegaListPtr(testX), testOrderedItems, learner.recommendSize)
logging.debug("MRR@" + str(learner.recommendSize) + ": " + str('%.4f' % metrics[j]) + " " + str(learner))
elif learner.metric == "f1":
metrics[j] = MCEvaluator.mrrAtK(SparseUtils.getOmegaListPtr(testX), testOrderedItems, learner.recommendSize)
logging.debug("F1@" + str(learner.recommendSize) + ": " + str('%.4f' % metrics[j]) + " " + str(learner))
else:
raise ValueError("Unknown metric " + learner.metric)
gc.collect()
return metrics
def testScale(self):
"""
Look at the scales of the unnormalised gradients.
"""
m = 100
n = 400
k = 3
X = SparseUtils.generateSparseBinaryMatrix((m, n), k, csarray=True)
w = 0.1
eps = 0.001
learner = MaxAUCTanh(k, w)
learner.normalise = False
learner.lmbdaU = 1.0
learner.lmbdaV = 1.0
learner.rho = 1.0
learner.numAucSamples = 100
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
r = numpy.random.rand(m)
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
gi = numpy.random.rand(m)
gi /= gi.sum()
gp = numpy.random.rand(n)
gp /= gp.sum()
gq = numpy.random.rand(n)
gq /= gq.sum()
permutedRowInds = numpy.array(numpy.random.permutation(m), numpy.uint32)
permutedColInds = numpy.array(numpy.random.permutation(n), numpy.uint32)
maxLocalAuc = MaxLocalAUC(k, w)
normGp, normGq = maxLocalAuc.computeNormGpq(indPtr, colInds, gp, gq, m)
normDui = 0
for i in range(m):
du = learner.derivativeUi(indPtr, colInds, U, V, r, gi, gp, gq, i)
normDui += numpy.linalg.norm(du)
normDui /= float(m)
print(normDui)
normDvi = 0
for i in range(n):
dv = learner.derivativeVi(indPtr, colInds, U, V, r, gi, gp, gq, i)
normDvi += numpy.linalg.norm(dv)
normDvi /= float(n)
print(normDvi)
def profileRestrictOmega(self):
X, U, V = DatasetUtils.syntheticDataset1(u=0.01, m=1000, n=2000)
m, n = X.shape
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
colIndsSubset = numpy.random.choice(n, 500, replace=False)
def run():
for i in range(100):
newIndPtr, newColInds = restrictOmega(indPtr, colInds,
colIndsSubset)
ProfileUtils.profile('run()', globals(), locals())
def testLocalAucApprox2(self):
m = 100
n = 200
k = 5
numInds = 100
X, U, s, V, wv = SparseUtils.generateSparseBinaryMatrix((m, n), k, csarray=True, verbose=True)
r = numpy.ones(m) * -10
w = 0.5
localAuc = MCEvaluator.localAUC(X, U, V, w)
samples = numpy.arange(50, 200, 10)
for i, sampleSize in enumerate(samples):
localAuc2 = MCEvaluator.localAUCApprox(SparseUtils.getOmegaListPtr(X), U, V, w, sampleSize)
self.assertAlmostEqual(localAuc2, localAuc, 1)
# Test more accurately
sampleSize = 1000
localAuc2 = MCEvaluator.localAUCApprox(SparseUtils.getOmegaListPtr(X), U, V, w, sampleSize)
self.assertAlmostEqual(localAuc2, localAuc, 2)
# Now set a high r
Z = U.dot(V.T)
localAuc = MCEvaluator.localAUCApprox(SparseUtils.getOmegaListPtr(X), U, V, w, sampleSize)
for i, sampleSize in enumerate(samples):
localAuc2 = MCEvaluator.localAUCApprox(SparseUtils.getOmegaListPtr(X), U, V, w, sampleSize)
self.assertAlmostEqual(localAuc2, localAuc, 1)
# Test more accurately
sampleSize = 1000
localAuc2 = MCEvaluator.localAUCApprox(SparseUtils.getOmegaListPtr(X), U, V, w, sampleSize)
self.assertAlmostEqual(localAuc2, localAuc, 2)
def computeTestMRR(args):
"""
A simple function for outputing F1 for a learner in conjunction e.g. with
parallel model selection.
"""
trainX, testX, learner = args
learner.learnModel(trainX)
testOrderedItems = MCEvaluatorCython.recommendAtk(learner.U, learner.V, learner.recommendSize, trainX)
mrr = MCEvaluator.mrrAtK(SparseUtils.getOmegaListPtr(testX), testOrderedItems, learner.recommendSize)
try:
learnerStr = learner.modelParamsStr()
except:
learnerStr = str(learner)
logging.debug("MRR@" + str(learner.recommendSize) + ": " + str("%.4f" % mrr) + " " + learnerStr)
return mrr
def stratifiedRecallAtK(positiveArray, orderedItems, k, itemCounts, beta=0.5, verbose=False):
"""
Compute the average recall@k score for each row of the predicted matrix UV.T
using real values in positiveArray. positiveArray is a tuple (indPtr, colInds)
:param orderedItems: The ordered items for each user (users are rows, items are cols)
:param verbose: If true return recall and first k recommendation for each row, otherwise just precisions
"""
if type(positiveArray) != tuple:
positiveArray = SparseUtils.getOmegaListPtr(positiveArray)
orderedItems = orderedItems[:, 0:k]
indPtr, colInds = positiveArray
recalls, denominators = MCEvaluatorCython.stratifiedRecallAtk(indPtr, colInds, orderedItems, itemCounts, beta)
if verbose:
return recalls, orderedItems
else:
return numpy.average(recalls, weights=denominators)
def precisionAtK(positiveArray, orderedItems, k, verbose=False):
"""
Compute the average precision@k score for each row of the predicted matrix UV.T
using real values in positiveArray. positiveArray is a tuple (indPtr, colInds)
:param orderedItems: The ordered items for each user (users are rows, items are cols)
:param verbose: If true return precision and first k recommendation for each row, otherwise just precisions
"""
if type(positiveArray) != tuple:
positiveArray = SparseUtils.getOmegaListPtr(positiveArray)
orderedItems = orderedItems[:, 0:k]
indPtr, colInds = positiveArray
precisions = MCEvaluatorCython.precisionAtk(indPtr, colInds, orderedItems)
if verbose:
return precisions, orderedItems
else:
return precisions.mean()
def precisionAtK(positiveArray, orderedItems, k, verbose=False):
"""
Compute the average precision@k score for each row of the predicted matrix UV.T
using real values in positiveArray. positiveArray is a tuple (indPtr, colInds)
:param orderedItems: The ordered items for each user (users are rows, items are cols)
:param verbose: If true return precision and first k recommendation for each row, otherwise just precisions
"""
if type(positiveArray) != tuple:
positiveArray = SparseUtils.getOmegaListPtr(positiveArray)
orderedItems = orderedItems[:, 0:k]
indPtr, colInds = positiveArray
precisions = MCEvaluatorCython.precisionAtk(indPtr, colInds,
orderedItems)
if verbose:
return precisions, orderedItems
else:
return precisions.mean()
def profileDerivativeVjApprox(self):
k = 10
U = numpy.random.rand(self.m, k)
V = numpy.random.rand(self.n, k)
indPtr, colInds = SparseUtils.getOmegaListPtr(self.X)
gp = numpy.random.rand(self.n)
gp /= gp.sum()
gq = numpy.random.rand(self.n)
gq /= gq.sum()
j = 3
numRowSamples = 100
numAucSamples = 10
permutedRowInds = numpy.array(numpy.random.permutation(self.m),
numpy.uint32)
permutedColInds = numpy.array(numpy.random.permutation(self.n),
numpy.uint32)
maxLocalAuc = MaxLocalAUC(k, w=0.9)
normGp, normGq = maxLocalAuc.computeNormGpq(indPtr, colInds, gp, gq,
self.m)
lmbda = 0.001
normalise = True
learner = MaxLocalAUCCython()
def run():
numRuns = 1
for i in range(numRuns):
for j in range(self.n):
learner.derivativeViApprox(indPtr, colInds, U, V, gp, gq,
normGp, normGq, permutedRowInds,
permutedColInds, i)
ProfileUtils.profile('run()', globals(), locals())
def localAUC(positiveArray, U, V, w, numRowInds=None):
"""
Compute the local AUC for the score functions UV^T relative to X with
quantile w.
"""
if numRowInds == None:
numRowInds = V.shape[0]
if type(positiveArray) != tuple:
positiveArray = SparseUtils.getOmegaListPtr(positiveArray)
#For now let's compute the full matrix
Z = U.dot(V.T)
r = SparseUtilsCython.computeR(U, V, w, numRowInds)
localAuc = numpy.zeros(U.shape[0])
allInds = numpy.arange(V.shape[0])
indPtr, colInds = positiveArray
for i in range(U.shape[0]):
omegai = colInds[indPtr[i]:indPtr[i + 1]]
omegaBari = numpy.setdiff1d(allInds, omegai, assume_unique=True)
if omegai.shape[0] * omegaBari.shape[0] != 0:
partialAuc = 0
for p in omegai:
for q in omegaBari:
if Z[i, p] > Z[i, q] and Z[i, p] > r[i]:
partialAuc += 1
localAuc[i] = partialAuc / float(
omegai.shape[0] * omegaBari.shape[0])
localAuc = localAuc.mean()
return localAuc
def testRestrictOmega(self):
m = 50
n = 100
k = 5
u = 0.5
w = 1 - u
X = SparseUtils.generateSparseBinaryMatrix((m, n), k, w, csarray=True)
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
runs = 100
for i in range(runs):
colSubset = numpy.random.choice(n, 20, replace=False)
newIndPtr, newColInds = restrictOmega(indPtr, colInds, colSubset)
for i in range(m):
omegai = colInds[indPtr[i]:indPtr[i + 1]]
omegai2 = newColInds[newIndPtr[i]:newIndPtr[i + 1]]
a = numpy.setdiff1d(omegai, omegai2)
self.assertEquals(numpy.intersect1d(a, colSubset).shape[0], 0)
def localAUCApprox(positiveArray, U, V, w, numAucSamples=50, r=None, allArray=None):
"""
Compute the estimated local AUC for the score functions UV^T relative to X with
quantile w. The AUC is computed using positiveArray which is a tuple (indPtr, colInds)
assuming allArray is None. If allArray is not None then positive items are chosen
from positiveArray and negative ones are chosen to complement allArray.
"""
if type(positiveArray) != tuple:
positiveArray = SparseUtils.getOmegaListPtr(positiveArray)
indPtr, colInds = positiveArray
U = numpy.ascontiguousarray(U)
V = numpy.ascontiguousarray(V)
if r is None:
r = SparseUtilsCython.computeR(U, V, w, numAucSamples)
if allArray is None:
return MCEvaluatorCython.localAUCApprox(indPtr, colInds, indPtr, colInds, U, V, numAucSamples, r)
else:
allIndPtr, allColInd = allArray
return MCEvaluatorCython.localAUCApprox(indPtr, colInds, allIndPtr, allColInd, U, V, numAucSamples, r)
def testRestrictOmega(self):
m = 50
n = 100
k = 5
u = 0.5
w = 1-u
X = SparseUtils.generateSparseBinaryMatrix((m, n), k, w, csarray=True)
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
runs = 100
for i in range(runs):
colSubset = numpy.random.choice(n, 20, replace=False)
newIndPtr, newColInds = restrictOmega(indPtr, colInds, colSubset)
for i in range(m):
omegai = colInds[indPtr[i]:indPtr[i+1]]
omegai2 = newColInds[newIndPtr[i]:newIndPtr[i+1]]
a = numpy.setdiff1d(omegai, omegai2)
self.assertEquals(numpy.intersect1d(a, colSubset).shape[0], 0)
def localAUC(positiveArray, U, V, w, numRowInds=None):
"""
Compute the local AUC for the score functions UV^T relative to X with
quantile w.
"""
if numRowInds == None:
numRowInds = V.shape[0]
if type(positiveArray) != tuple:
positiveArray = SparseUtils.getOmegaListPtr(positiveArray)
#For now let's compute the full matrix
Z = U.dot(V.T)
r = SparseUtilsCython.computeR(U, V, w, numRowInds)
localAuc = numpy.zeros(U.shape[0])
allInds = numpy.arange(V.shape[0])
indPtr, colInds = positiveArray
for i in range(U.shape[0]):
omegai = colInds[indPtr[i]:indPtr[i+1]]
omegaBari = numpy.setdiff1d(allInds, omegai, assume_unique=True)
if omegai.shape[0] * omegaBari.shape[0] != 0:
partialAuc = 0
for p in omegai:
for q in omegaBari:
if Z[i, p] > Z[i, q] and Z[i, p] > r[i]:
partialAuc += 1
localAuc[i] = partialAuc/float(omegai.shape[0] * omegaBari.shape[0])
localAuc = localAuc.mean()
return localAuc
def computeTestF1(args):
"""
A simple function for outputing F1 for a learner in conjunction e.g. with
parallel model selection.
"""
trainX, testX, learner = args
learner.learnModel(trainX)
testOrderedItems = MCEvaluatorCython.recommendAtk(learner.U, learner.V,
learner.recommendSize,
trainX)
f1 = MCEvaluator.f1AtK(SparseUtils.getOmegaListPtr(testX),
testOrderedItems, learner.recommendSize)
try:
learnerStr = learner.modelParamsStr()
except:
learnerStr = str(learner)
logging.debug("F1@" + str(learner.recommendSize) + ": " +
str('%.4f' % f1) + " " + learnerStr)
return f1
def testDerivativeUiApprox(self):
"""
We'll test the case in which we apprormate using a large number of samples
for the AUC and see if we get close to the exact derivative
"""
m = 20
n = 30
k = 3
X = SparseUtils.generateSparseBinaryMatrix((m, n), k, csarray=True)
w = 0.1
learner = MaxAUCSigmoid(k, w)
learner.normalise = False
learner.lmbdaU = 0
learner.lmbdaV = 0
learner.rho = 1.0
learner.numAucSamples = 100
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
gp = numpy.random.rand(n)
gp /= gp.sum()
gq = numpy.random.rand(n)
gq /= gq.sum()
numRuns = 200
numTests = 5
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
permutedColInds = numpy.arange(n, dtype=numpy.uint32)
#Test with small number of AUC samples, but normalise
learner.numAucSamples = n
numRuns = 1000
for i in numpy.random.permutation(m)[0:numTests]:
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
du1 = numpy.zeros(k)
for j in range(numRuns):
du1 += learner.derivativeUiApprox(indPtr, colInds, U, V, gp, gq, permutedColInds, i)
du1 /= numRuns
du2 = learner.derivativeUi(indPtr, colInds, U, V, gp, gq, i)
#print(du1, du2)
print(du1/numpy.linalg.norm(du1), du2/numpy.linalg.norm(du2))
#print(numpy.linalg.norm(du1 - du2)/numpy.linalg.norm(du1))
self.assertTrue(numpy.linalg.norm(du1 - du2)/numpy.linalg.norm(du1) < 0.5)
#Let's compare against using the exact derivative
for i in numpy.random.permutation(m)[0:numTests]:
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
du1 = numpy.zeros(k)
for j in range(numRuns):
du1 += learner.derivativeUiApprox(indPtr, colInds, U, V, gp, gq, permutedColInds, i)
du1 /= numRuns
du2 = learner.derivativeUi(indPtr, colInds, U, V, gp, gq, i)
print(du1/numpy.linalg.norm(du1), du2/numpy.linalg.norm(du2))
nptst.assert_array_almost_equal(du1, du2, 2)
learner.lmbdaV = 0.5
learner.rho = 0.5
for i in numpy.random.permutation(m)[0:numTests]:
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
du1 = numpy.zeros(k)
for j in range(numRuns):
du1 += learner.derivativeUiApprox(indPtr, colInds, U, V, gp, gq, permutedColInds, i)
du1 /= numRuns
du2 = learner.derivativeUi(indPtr, colInds, U, V, gp, gq, i)
nptst.assert_array_almost_equal(du1, du2, 2)
print(du1/numpy.linalg.norm(du1), du2/numpy.linalg.norm(du2))
def testComputeV1V2(self):
m = 10
n = 20
nnzPerRow = 5
X = SparseUtils.generateSparseBinaryMatrix((m, n),
nnzPerRow,
csarray=True)
k = 5
learner = MaxAUCSquare(k)
learner.normalise = False
learner.lmbdaU = 0
learner.lmbdaV = 0
learner.rho = 1.0
learner.numAucSamples = 20
numRuns = 500
gp = numpy.random.rand(n)
gp /= gp.sum()
gq = numpy.random.rand(n)
gq /= gq.sum()
permutedRowInds = numpy.arange(m, dtype=numpy.uint32)
permutedColInds = numpy.arange(n, dtype=numpy.uint32)
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
U = numpy.random.randn(m, k)
V = numpy.random.randn(n, k)
V11 = numpy.zeros((m, k))
V21 = numpy.zeros((m, k))
W11 = numpy.zeros((m, k))
W21 = numpy.zeros((m, k))
for i in range(numRuns):
tempV1, tempV2, tempW1, tempW2 = learner.computeMeansVW(
indPtr, colInds, U, V, permutedRowInds, permutedColInds, gp,
gq)
V11 += tempV1
V21 += tempV2
W11 += tempW1
W21 += tempW2
V11 /= numRuns
V21 /= numRuns
W11 /= numRuns
W21 /= numRuns
#print(V11)
#print(V21)
#Now compute real solution
V12 = numpy.zeros((m, k))
V22 = numpy.zeros((m, k))
W12 = numpy.zeros((m, k))
W22 = numpy.zeros((m, k))
#The bootstrap sampling causes slight errors
for i in range(m):
normGp = 0
omegai = colInds[indPtr[i]:indPtr[i + 1]]
for j in omegai:
V12[i, :] += V[j, :] * gp[j]
W12[i, :] += V[j, :] * gp[j] * (U[i, :].dot(V[j, :]))
normGp += gp[j]
V12[i, :] /= normGp
W12[i, :] /= normGp
normGq = 0
omegaBari = numpy.setdiff1d(numpy.arange(n, dtype=numpy.uint32),
omegai,
assume_unique=True)
for j in omegaBari:
V22[i, :] += V[j, :] * gq[j]
W22[i, :] += V[j, :] * gq[j] * (U[i, :].dot(V[j, :]))
normGq += gq[j]
V22[i, :] /= normGq
W22[i, :] /= normGq
#print(W21)
#print(W22)
nptst.assert_array_almost_equal(V11, V12, 1)
nptst.assert_array_almost_equal(V21, V22, 1)
nptst.assert_array_almost_equal(W11, W12, 1)
nptst.assert_array_almost_equal(W21, W22, 1)
def parallelLearnModel(self, X, verbose=False, U=None, V=None):
"""
Max local AUC with Frobenius norm penalty on V. Solve with parallel (stochastic) gradient descent.
The input is a sparse array.
"""
#Convert to a csarray for faster access
if scipy.sparse.issparse(X):
logging.debug("Converting to csarray")
X2 = sppy.csarray(X, storagetype="row")
X = X2
m, n = X.shape
#We keep a validation set in order to determine when to stop
if self.validationUsers != 0:
numValidationUsers = int(m * self.validationUsers)
trainX, testX, rowSamples = Sampling.shuffleSplitRows(
X, 1, self.validationSize, numRows=numValidationUsers)[0]
testIndPtr, testColInds = SparseUtils.getOmegaListPtr(testX)
else:
trainX = X
testX = None
rowSamples = None
testIndPtr, testColInds = None, None
#Not that to compute the test AUC we pick i \in X and j \notin X \cup testX
indPtr, colInds = SparseUtils.getOmegaListPtr(trainX)
allIndPtr, allColInds = SparseUtils.getOmegaListPtr(X)
if U == None or V == None:
U, V = self.initUV(trainX)
if self.metric == "f1":
metricInd = 2
elif self.metric == "mrr":
metricInd = 3
else:
raise ValueError("Unknown metric: " + self.metric)
bestMetric = 0
bestU = 0
bestV = 0
trainMeasures = []
testMeasures = []
loopInd = 0
lastObj = 0
currentObj = lastObj - 2 * self.eps
numBlocks = self.numProcesses + 1
gi, gp, gq = self.computeGipq(X)
normGp, normGq = self.computeNormGpq(indPtr, colInds, gp, gq, m)
#Some shared variables
rowIsFree = sharedmem.ones(numBlocks, dtype=numpy.bool)
colIsFree = sharedmem.ones(numBlocks, dtype=numpy.bool)
#Create shared factors
U2 = sharedmem.zeros((m, self.k))
V2 = sharedmem.zeros((n, self.k))
muU2 = sharedmem.zeros((m, self.k))
muV2 = sharedmem.zeros((n, self.k))
U2[:] = U[:]
V2[:] = V[:]
muU2[:] = U[:]
muV2[:] = V[:]
del U, V
rowBlockSize = int(numpy.ceil(float(m) / numBlocks))
colBlockSize = int(numpy.ceil(float(n) / numBlocks))
lock = multiprocessing.Lock()
startTime = time.time()
loopInd = 0
iterationsPerBlock = sharedmem.zeros((numBlocks, numBlocks))
self.learnerCython = self.getCythonLearner()
nextRecord = 0
while loopInd < self.maxIterations and abs(lastObj -
currentObj) > self.eps:
if loopInd >= nextRecord:
if loopInd != 0:
print("")
printStr = self.recordResults(muU2, muV2, trainMeasures,
testMeasures, loopInd,
rowSamples, indPtr, colInds,
testIndPtr, testColInds,
allIndPtr, allColInds, gi, gp,
gq, trainX, startTime)
logging.debug(printStr)
if testIndPtr is not None and testMeasures[-1][
metricInd] >= bestMetric:
bestMetric = testMeasures[-1][metricInd]
bestU = muU2.copy()
bestV = muV2.copy()
elif testIndPtr is None:
bestU = muU2.copy()
bestV = muV2.copy()
#Compute objective averaged over last 5 recorded steps
trainMeasuresArr = numpy.array(trainMeasures)
lastObj = currentObj
currentObj = numpy.mean(trainMeasuresArr[-5:, 0])
nextRecord += self.recordStep
iterationsPerBlock = sharedmem.zeros((numBlocks, numBlocks))
self.parallelUpdateUV(X, U2, V2, muU2, muV2, numBlocks,
rowBlockSize, colBlockSize, rowIsFree,
colIsFree, indPtr, colInds, lock, gi, gp, gq,
normGp, normGq, iterationsPerBlock, loopInd)
loopInd += numpy.floor(iterationsPerBlock.mean())
totalTime = time.time() - startTime
#Compute quantities for last U and V
print("")
totalTime = time.time() - startTime
printStr = "Finished, time=" + str('%.1f' % totalTime) + " "
printStr += self.recordResults(muU2, muV2, trainMeasures, testMeasures,
loopInd, rowSamples, indPtr, colInds,
testIndPtr, testColInds, allIndPtr,
allColInds, gi, gp, gq, trainX,
startTime)
printStr += " delta obj" + "%.3e" % abs(lastObj - currentObj)
logging.debug(printStr)
self.U = bestU
self.V = bestV
self.gi = gi
self.gp = gp
self.gq = gq
if verbose:
return self.U, self.V, numpy.array(trainMeasures), numpy.array(
testMeasures), loopInd, totalTime
else:
return self.U, self.V
def testComputeV1V2(self):
m = 10
n = 20
nnzPerRow = 5
X = SparseUtils.generateSparseBinaryMatrix((m, n), nnzPerRow, csarray=True)
k = 5
learner = MaxAUCSquare(k)
learner.normalise = False
learner.lmbdaU = 0
learner.lmbdaV = 0
learner.rho = 1.0
learner.numAucSamples = 20
numRuns = 500
gp = numpy.random.rand(n)
gp /= gp.sum()
gq = numpy.random.rand(n)
gq /= gq.sum()
permutedRowInds = numpy.arange(m, dtype=numpy.uint32)
permutedColInds = numpy.arange(n, dtype=numpy.uint32)
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
U = numpy.random.randn(m, k)
V = numpy.random.randn(n, k)
V11 = numpy.zeros((m, k))
V21 = numpy.zeros((m, k))
W11 = numpy.zeros((m, k))
W21 = numpy.zeros((m, k))
for i in range(numRuns):
tempV1, tempV2, tempW1, tempW2 = learner.computeMeansVW(indPtr, colInds, U, V, permutedRowInds, permutedColInds, gp, gq)
V11 += tempV1
V21 += tempV2
W11 += tempW1
W21 += tempW2
V11 /= numRuns
V21 /= numRuns
W11 /= numRuns
W21 /= numRuns
#print(V11)
#print(V21)
#Now compute real solution
V12 = numpy.zeros((m, k))
V22 = numpy.zeros((m, k))
W12 = numpy.zeros((m, k))
W22 = numpy.zeros((m, k))
#The bootstrap sampling causes slight errors
for i in range(m):
normGp = 0
omegai = colInds[indPtr[i]:indPtr[i+1]]
for j in omegai:
V12[i, :] += V[j, :]*gp[j]
W12[i, :] += V[j, :]*gp[j]*(U[i, :].dot(V[j, :]))
normGp += gp[j]
V12[i, :] /= normGp
W12[i, :] /= normGp
normGq = 0
omegaBari = numpy.setdiff1d(numpy.arange(n, dtype=numpy.uint32), omegai, assume_unique=True)
for j in omegaBari:
V22[i, :] += V[j, :]*gq[j]
W22[i, :] += V[j, :]*gq[j]*(U[i, :].dot(V[j, :]))
normGq += gq[j]
V22[i, :] /= normGq
W22[i, :] /= normGq
#print(W21)
#print(W22)
nptst.assert_array_almost_equal(V11, V12, 1)
nptst.assert_array_almost_equal(V21, V22, 1)
nptst.assert_array_almost_equal(W11, W12, 1)
nptst.assert_array_almost_equal(W21, W22, 1)
def parallelLearnModel(self, X, verbose=False, U=None, V=None):
"""
Max local AUC with Frobenius norm penalty on V. Solve with parallel (stochastic) gradient descent.
The input is a sparse array.
"""
# Convert to a csarray for faster access
if scipy.sparse.issparse(X):
logging.debug("Converting to csarray")
X2 = sppy.csarray(X, storagetype="row")
X = X2
m, n = X.shape
# We keep a validation set in order to determine when to stop
if self.validationUsers != 0:
numValidationUsers = int(m * self.validationUsers)
trainX, testX, rowSamples = Sampling.shuffleSplitRows(
X, 1, self.validationSize, numRows=numValidationUsers
)[0]
testIndPtr, testColInds = SparseUtils.getOmegaListPtr(testX)
else:
trainX = X
testX = None
rowSamples = None
testIndPtr, testColInds = None, None
# Not that to compute the test AUC we pick i \in X and j \notin X \cup testX
indPtr, colInds = SparseUtils.getOmegaListPtr(trainX)
allIndPtr, allColInds = SparseUtils.getOmegaListPtr(X)
if U == None or V == None:
U, V = self.initUV(trainX)
if self.metric == "f1":
metricInd = 2
elif self.metric == "mrr":
metricInd = 3
else:
raise ValueError("Unknown metric: " + self.metric)
bestMetric = 0
bestU = 0
bestV = 0
trainMeasures = []
testMeasures = []
loopInd = 0
lastObj = 0
currentObj = lastObj - 2 * self.eps
numBlocks = self.numProcesses + 1
gi, gp, gq = self.computeGipq(X)
normGp, normGq = self.computeNormGpq(indPtr, colInds, gp, gq, m)
# Some shared variables
rowIsFree = sharedmem.ones(numBlocks, dtype=numpy.bool)
colIsFree = sharedmem.ones(numBlocks, dtype=numpy.bool)
# Create shared factors
U2 = sharedmem.zeros((m, self.k))
V2 = sharedmem.zeros((n, self.k))
muU2 = sharedmem.zeros((m, self.k))
muV2 = sharedmem.zeros((n, self.k))
U2[:] = U[:]
V2[:] = V[:]
muU2[:] = U[:]
muV2[:] = V[:]
del U, V
rowBlockSize = int(numpy.ceil(float(m) / numBlocks))
colBlockSize = int(numpy.ceil(float(n) / numBlocks))
lock = multiprocessing.Lock()
startTime = time.time()
loopInd = 0
iterationsPerBlock = sharedmem.zeros((numBlocks, numBlocks))
self.learnerCython = self.getCythonLearner()
nextRecord = 0
while loopInd < self.maxIterations and abs(lastObj - currentObj) > self.eps:
if loopInd >= nextRecord:
if loopInd != 0:
print("")
printStr = self.recordResults(
muU2,
muV2,
trainMeasures,
testMeasures,
loopInd,
rowSamples,
indPtr,
colInds,
testIndPtr,
testColInds,
allIndPtr,
allColInds,
gi,
gp,
gq,
trainX,
startTime,
)
logging.debug(printStr)
if testIndPtr is not None and testMeasures[-1][metricInd] >= bestMetric:
bestMetric = testMeasures[-1][metricInd]
bestU = muU2.copy()
bestV = muV2.copy()
elif testIndPtr is None:
bestU = muU2.copy()
bestV = muV2.copy()
# Compute objective averaged over last 5 recorded steps
trainMeasuresArr = numpy.array(trainMeasures)
lastObj = currentObj
currentObj = numpy.mean(trainMeasuresArr[-5:, 0])
nextRecord += self.recordStep
iterationsPerBlock = sharedmem.zeros((numBlocks, numBlocks))
self.parallelUpdateUV(
X,
U2,
V2,
muU2,
muV2,
numBlocks,
rowBlockSize,
colBlockSize,
rowIsFree,
colIsFree,
indPtr,
colInds,
lock,
gi,
gp,
gq,
normGp,
normGq,
iterationsPerBlock,
loopInd,
)
loopInd += numpy.floor(iterationsPerBlock.mean())
totalTime = time.time() - startTime
# Compute quantities for last U and V
print("")
totalTime = time.time() - startTime
printStr = "Finished, time=" + str("%.1f" % totalTime) + " "
printStr += self.recordResults(
muU2,
muV2,
trainMeasures,
testMeasures,
loopInd,
rowSamples,
indPtr,
colInds,
testIndPtr,
testColInds,
allIndPtr,
allColInds,
gi,
gp,
gq,
trainX,
startTime,
)
printStr += " delta obj" + "%.3e" % abs(lastObj - currentObj)
logging.debug(printStr)
self.U = bestU
self.V = bestV
self.gi = gi
self.gp = gp
self.gq = gq
if verbose:
return self.U, self.V, numpy.array(trainMeasures), numpy.array(testMeasures), loopInd, totalTime
else:
return self.U, self.V
def testDerivativeViApprox(self):
"""
We'll test the case in which we apprormate using a large number of samples
for the AUC and see if we get close to the exact derivative
"""
m = 20
n = 30
k = 3
X = SparseUtils.generateSparseBinaryMatrix((m, n), k, csarray=True)
for i in range(m):
X[i, 0] = 1
X[i, 1] = 0
w = 0.1
eps = 0.001
learner = MaxAUCSigmoid(k, w)
learner.normalise = False
learner.lmbdaU = 0
learner.lmbdaV = 0
learner.numAucSamples = n
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
gp = numpy.random.rand(n)
gp /= gp.sum()
gq = numpy.random.rand(n)
gq /= gq.sum()
permutedRowInds = numpy.array(numpy.random.permutation(m), numpy.uint32)
permutedColInds = numpy.array(numpy.random.permutation(n), numpy.uint32)
maxLocalAuc = MaxLocalAUC(k, w)
normGp, normGq = maxLocalAuc.computeNormGpq(indPtr, colInds, gp, gq, m)
numRuns = 200
numTests = 5
#Let's compare against using the exact derivative
for i in numpy.random.permutation(m)[0:numTests]:
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
dv1 = numpy.zeros(k)
for j in range(numRuns):
dv1 += learner.derivativeViApprox(indPtr, colInds, U, V, gp, gq, normGp, normGq, permutedRowInds, permutedColInds, i)
dv1 /= numRuns
dv2 = learner.derivativeVi(indPtr, colInds, U, V, gp, gq, i)
dv3 = numpy.zeros(k)
for j in range(k):
eps = 10**-6
tempV = V.copy()
tempV[i,j] += eps
obj1 = learner.objective(indPtr, colInds, indPtr, colInds, U, tempV, gp, gq)
tempV = V.copy()
tempV[i,j] -= eps
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, U, tempV, gp, gq)
dv3[j] = (obj1-obj2)/(2*eps)
print(dv1, dv2, dv3)
nptst.assert_array_almost_equal(dv1, dv2, 3)
learner.lmbdaV = 0.5
learner.rho = 0.5
for i in numpy.random.permutation(m)[0:numTests]:
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
dv1 = numpy.zeros(k)
for j in range(numRuns):
dv1 += learner.derivativeViApprox(indPtr, colInds, U, V, gp, gq, normGp, normGq, permutedRowInds, permutedColInds, i)
dv1 /= numRuns
dv2 = learner.derivativeVi(indPtr, colInds, U, V, gp, gq, i)
print(dv1, dv2)
nptst.assert_array_almost_equal(dv1, dv2, 3)
learner.numRowSamples = 10
numRuns = 1000
for i in numpy.random.permutation(m)[0:numTests]:
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
dv1 = numpy.zeros(k)
for j in range(numRuns):
dv1 += learner.derivativeViApprox(indPtr, colInds, U, V, gp, gq, normGp, normGq, permutedRowInds, permutedColInds, i)
dv1 /= numRuns
dv2 = learner.derivativeVi(indPtr, colInds, U, V, gp, gq, i)
print(dv1, dv2)
nptst.assert_array_almost_equal(dv1, dv2, 3)
maxLocalAuc.numRowSamples = m
maxLocalAuc.numAucSamples = 20
maxLocalAuc.lmbdaV = 0
numRuns = 1000
print("Final test")
#for i in numpy.random.permutation(m)[0:numTests]:
for i in range(m):
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
dv1 = numpy.zeros(k)
for j in range(numRuns):
dv1 += learner.derivativeViApprox(indPtr, colInds, U, V, gp, gq, normGp, normGq, permutedRowInds, permutedColInds, i)
dv1 /= numRuns
#dv1 = learner.derivativeVi(indPtr, colInds, U, V, gp, gq, i)
dv2 = learner.derivativeVi(indPtr, colInds, U, V, gp, gq, i)
print(i, dv1, dv2)
nptst.assert_array_almost_equal(dv1, dv2, 3)
def testDerivativeViApprox(self):
"""
We'll test the case in which we apprormate using a large number of samples
for the AUC and see if we get close to the exact derivative
"""
m = 20
n = 30
k = 3
X = SparseUtils.generateSparseBinaryMatrix((m, n), k, csarray=True)
for i in range(m):
X[i, 0] = 1
X[i, 1] = 0
w = 0.1
eps = 0.001
learner = MaxAUCSigmoid(k, w)
learner.normalise = False
learner.lmbdaU = 0
learner.lmbdaV = 0
learner.numAucSamples = n
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
gp = numpy.random.rand(n)
gp /= gp.sum()
gq = numpy.random.rand(n)
gq /= gq.sum()
permutedRowInds = numpy.array(numpy.random.permutation(m),
numpy.uint32)
permutedColInds = numpy.array(numpy.random.permutation(n),
numpy.uint32)
maxLocalAuc = MaxLocalAUC(k, w)
normGp, normGq = maxLocalAuc.computeNormGpq(indPtr, colInds, gp, gq, m)
numRuns = 200
numTests = 5
#Let's compare against using the exact derivative
for i in numpy.random.permutation(m)[0:numTests]:
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
dv1 = numpy.zeros(k)
for j in range(numRuns):
dv1 += learner.derivativeViApprox(indPtr, colInds, U, V, gp,
gq, normGp, normGq,
permutedRowInds,
permutedColInds, i)
dv1 /= numRuns
dv2 = learner.derivativeVi(indPtr, colInds, U, V, gp, gq, i)
dv3 = numpy.zeros(k)
for j in range(k):
eps = 10**-6
tempV = V.copy()
tempV[i, j] += eps
obj1 = learner.objective(indPtr, colInds, indPtr, colInds, U,
tempV, gp, gq)
tempV = V.copy()
tempV[i, j] -= eps
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, U,
tempV, gp, gq)
dv3[j] = (obj1 - obj2) / (2 * eps)
print(dv1, dv2, dv3)
nptst.assert_array_almost_equal(dv1, dv2, 3)
learner.lmbdaV = 0.5
learner.rho = 0.5
for i in numpy.random.permutation(m)[0:numTests]:
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
dv1 = numpy.zeros(k)
for j in range(numRuns):
dv1 += learner.derivativeViApprox(indPtr, colInds, U, V, gp,
gq, normGp, normGq,
permutedRowInds,
permutedColInds, i)
dv1 /= numRuns
dv2 = learner.derivativeVi(indPtr, colInds, U, V, gp, gq, i)
print(dv1, dv2)
nptst.assert_array_almost_equal(dv1, dv2, 3)
learner.numRowSamples = 10
numRuns = 1000
for i in numpy.random.permutation(m)[0:numTests]:
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
dv1 = numpy.zeros(k)
for j in range(numRuns):
dv1 += learner.derivativeViApprox(indPtr, colInds, U, V, gp,
gq, normGp, normGq,
permutedRowInds,
permutedColInds, i)
dv1 /= numRuns
dv2 = learner.derivativeVi(indPtr, colInds, U, V, gp, gq, i)
print(dv1, dv2)
nptst.assert_array_almost_equal(dv1, dv2, 3)
maxLocalAuc.numRowSamples = m
maxLocalAuc.numAucSamples = 20
maxLocalAuc.lmbdaV = 0
numRuns = 1000
print("Final test")
#for i in numpy.random.permutation(m)[0:numTests]:
for i in range(m):
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
dv1 = numpy.zeros(k)
for j in range(numRuns):
dv1 += learner.derivativeViApprox(indPtr, colInds, U, V, gp,
gq, normGp, normGq,
permutedRowInds,
permutedColInds, i)
dv1 /= numRuns
#dv1 = learner.derivativeVi(indPtr, colInds, U, V, gp, gq, i)
dv2 = learner.derivativeVi(indPtr, colInds, U, V, gp, gq, i)
print(i, dv1, dv2)
nptst.assert_array_almost_equal(dv1, dv2, 3)
def testDerivativeV(self):
m = 10
n = 20
nnzPerRow = 5
X = SparseUtils.generateSparseBinaryMatrix((m, n),
nnzPerRow,
csarray=True)
for i in range(m):
X[i, 0] = 1
X[i, 1] = 0
k = 5
u = 0.1
w = 1 - u
eps = 0.05
learner = MaxAUCSigmoid(k, w)
learner.normalise = False
learner.lmbdaU = 0
learner.lmbdaV = 0
learner.rho = 1.0
learner.numAucSamples = 100
numRuns = 20
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
gp = numpy.random.rand(n)
gp /= gp.sum()
gq = numpy.random.rand(n)
gq /= gq.sum()
for s in range(numRuns):
U = numpy.random.randn(m, k)
V = numpy.random.randn(n, k)
deltaV = numpy.zeros(V.shape)
for j in range(n):
deltaV[j, :] = learner.derivativeVi(indPtr, colInds, U, V, gp,
gq, j)
deltaV2 = numpy.zeros(V.shape)
eps = 0.00001
for i in range(n):
for j in range(k):
tempV = V.copy()
tempV[i, j] += eps
obj1 = learner.objective(indPtr, colInds, indPtr, colInds,
U, tempV, gp, gq)
tempV = V.copy()
tempV[i, j] -= eps
obj2 = learner.objective(indPtr, colInds, indPtr, colInds,
U, tempV, gp, gq)
deltaV2[i, j] = (obj1 - obj2) / (2 * eps)
#deltaV2[i,:] = deltaV2[i,:]/numpy.linalg.norm(deltaV2[i,:])
nptst.assert_almost_equal(deltaV, deltaV2, 3)
#Try r != 0 and rho > 0
for s in range(numRuns):
U = numpy.random.randn(m, k)
V = numpy.random.randn(n, k)
learner.rho = 1.0
deltaV = numpy.zeros(V.shape)
for j in range(n):
deltaV[j, :] = learner.derivativeVi(indPtr, colInds, U, V, gp,
gq, j)
deltaV2 = numpy.zeros(V.shape)
for i in range(n):
for j in range(k):
tempV = V.copy()
tempV[i, j] += eps
obj1 = learner.objective(indPtr, colInds, indPtr, colInds,
U, tempV, gp, gq)
tempV = V.copy()
tempV[i, j] -= eps
obj2 = learner.objective(indPtr, colInds, indPtr, colInds,
U, tempV, gp, gq)
deltaV2[i, j] = (obj1 - obj2) / (2 * eps)
#deltaV2[i,:] = deltaV2[i,:]/numpy.linalg.norm(deltaV2[i,:])
nptst.assert_almost_equal(deltaV, deltaV2, 3)
#Try r != 0 and rho > 0
for s in range(numRuns):
U = numpy.random.randn(m, k)
V = numpy.random.randn(n, k)
learner.lmbdaV = 100
learner.rho = 0.1
deltaV = numpy.zeros(V.shape)
for j in range(n):
deltaV[j, :] = learner.derivativeVi(indPtr, colInds, U, V, gp,
gq, j)
deltaV2 = numpy.zeros(V.shape)
for i in range(n):
for j in range(k):
tempV = V.copy()
tempV[i, j] += eps
obj1 = learner.objective(indPtr, colInds, indPtr, colInds,
U, tempV, gp, gq)
tempV = V.copy()
tempV[i, j] -= eps
obj2 = learner.objective(indPtr, colInds, indPtr, colInds,
U, tempV, gp, gq)
deltaV2[i, j] = (obj1 - obj2) / (2 * eps)
#deltaV2[i,:] = deltaV2[i,:]/numpy.linalg.norm(deltaV2[i,:])
nptst.assert_almost_equal(deltaV, deltaV2, 3)
def testDerivativeU(self):
m = 10
n = 20
nnzPerRow = 5
X = SparseUtils.generateSparseBinaryMatrix((m, n),
nnzPerRow,
csarray=True)
k = 5
eps = 0.05
learner = MaxAUCSigmoid(k)
learner.normalise = False
learner.lmbdaU = 0
learner.lmbdaV = 0
learner.rho = 1.0
learner.numAucSamples = n
numRuns = 20
gi = numpy.random.rand(m)
gi /= gi.sum()
gp = numpy.random.rand(n)
gp /= gp.sum()
gq = numpy.random.rand(n)
gq /= gq.sum()
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
for s in range(numRuns):
U = numpy.random.randn(m, k)
V = numpy.random.randn(n, k)
deltaU = numpy.zeros(U.shape)
for i in range(X.shape[0]):
deltaU[i, :] = learner.derivativeUi(indPtr, colInds, U, V, gp,
gq, i)
deltaU2 = numpy.zeros(U.shape)
eps = 10**-8
for i in range(m):
for j in range(k):
tempU = U.copy()
tempU[i, j] += eps
obj1 = learner.objective(indPtr, colInds, indPtr, colInds,
tempU, V, gp, gq)
tempU = U.copy()
tempU[i, j] -= eps
obj2 = learner.objective(indPtr, colInds, indPtr, colInds,
tempU, V, gp, gq)
deltaU2[i, j] = (obj1 - obj2) / (2 * eps)
#deltaU2[i,:] = deltaU2[i,:]/numpy.linalg.norm(deltaU2[i,:])
#print(deltaU*100)
#print(deltaU2*100)
nptst.assert_almost_equal(deltaU, deltaU2, 3)
#Try r != 0 and rho > 0
for s in range(numRuns):
U = numpy.random.randn(m, k)
V = numpy.random.randn(n, k)
learner.rho = 0.1
deltaU = numpy.zeros(U.shape)
for i in range(X.shape[0]):
deltaU[i, :] = learner.derivativeUi(indPtr, colInds, U, V, gp,
gq, i)
deltaU2 = numpy.zeros(U.shape)
eps = 10**-9
for i in range(m):
for j in range(k):
tempU = U.copy()
tempU[i, j] += eps
obj1 = learner.objective(indPtr, colInds, indPtr, colInds,
tempU, V, gp, gq)
tempU = U.copy()
tempU[i, j] -= eps
obj2 = learner.objective(indPtr, colInds, indPtr, colInds,
tempU, V, gp, gq)
deltaU2[i, j] = (obj1 - obj2) / (2 * eps)
nptst.assert_almost_equal(deltaU, deltaU2, 3)
#Try lmbda > 0
for s in range(numRuns):
U = numpy.random.randn(m, k)
V = numpy.random.randn(n, k)
learner.lmbdaU = 0.5
deltaU = numpy.zeros(U.shape)
for i in range(X.shape[0]):
deltaU[i, :] = learner.derivativeUi(indPtr, colInds, U, V, gp,
gq, i)
deltaU2 = numpy.zeros(U.shape)
eps = 10**-9
for i in range(m):
for j in range(k):
tempU = U.copy()
tempU[i, j] += eps
obj1 = learner.objective(indPtr, colInds, indPtr, colInds,
tempU, V, gp, gq)
tempU = U.copy()
tempU[i, j] -= eps
obj2 = learner.objective(indPtr, colInds, indPtr, colInds,
tempU, V, gp, gq)
deltaU2[i, j] = (obj1 - obj2) / (2 * eps)
nptst.assert_almost_equal(deltaU, deltaU2, 3)
def testDerivativeUiApprox(self):
"""
We'll test the case in which we apprormate using a large number of samples
for the AUC and see if we get close to the exact derivative
"""
m = 20
n = 30
k = 3
X = SparseUtils.generateSparseBinaryMatrix((m, n), k, csarray=True)
w = 0.1
learner = MaxAUCSigmoid(k, w)
learner.normalise = False
learner.lmbdaU = 0
learner.lmbdaV = 0
learner.rho = 1.0
learner.numAucSamples = 100
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
gp = numpy.random.rand(n)
gp /= gp.sum()
gq = numpy.random.rand(n)
gq /= gq.sum()
numRuns = 200
numTests = 5
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
permutedColInds = numpy.arange(n, dtype=numpy.uint32)
#Test with small number of AUC samples, but normalise
learner.numAucSamples = n
numRuns = 1000
for i in numpy.random.permutation(m)[0:numTests]:
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
du1 = numpy.zeros(k)
for j in range(numRuns):
du1 += learner.derivativeUiApprox(indPtr, colInds, U, V, gp,
gq, permutedColInds, i)
du1 /= numRuns
du2 = learner.derivativeUi(indPtr, colInds, U, V, gp, gq, i)
#print(du1, du2)
print(du1 / numpy.linalg.norm(du1), du2 / numpy.linalg.norm(du2))
#print(numpy.linalg.norm(du1 - du2)/numpy.linalg.norm(du1))
self.assertTrue(
numpy.linalg.norm(du1 - du2) / numpy.linalg.norm(du1) < 0.5)
#Let's compare against using the exact derivative
for i in numpy.random.permutation(m)[0:numTests]:
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
du1 = numpy.zeros(k)
for j in range(numRuns):
du1 += learner.derivativeUiApprox(indPtr, colInds, U, V, gp,
gq, permutedColInds, i)
du1 /= numRuns
du2 = learner.derivativeUi(indPtr, colInds, U, V, gp, gq, i)
print(du1 / numpy.linalg.norm(du1), du2 / numpy.linalg.norm(du2))
nptst.assert_array_almost_equal(du1, du2, 2)
learner.lmbdaV = 0.5
learner.rho = 0.5
for i in numpy.random.permutation(m)[0:numTests]:
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
du1 = numpy.zeros(k)
for j in range(numRuns):
du1 += learner.derivativeUiApprox(indPtr, colInds, U, V, gp,
gq, permutedColInds, i)
du1 /= numRuns
du2 = learner.derivativeUi(indPtr, colInds, U, V, gp, gq, i)
nptst.assert_array_almost_equal(du1, du2, 2)
print(du1 / numpy.linalg.norm(du1), du2 / numpy.linalg.norm(du2))
def singleLearnModel(self, X, verbose=False, U=None, V=None):
"""
Max local AUC with Frobenius norm penalty on V. Solve with (stochastic) gradient descent.
The input is a sparse array.
"""
#Convert to a csarray for faster access
if scipy.sparse.issparse(X):
logging.debug("Converting to csarray")
X2 = sppy.csarray(X, storagetype="row")
X = X2
m, n = X.shape
#We keep a validation set in order to determine when to stop
if self.validationUsers != 0:
numValidationUsers = int(m * self.validationUsers)
trainX, testX, rowSamples = Sampling.shuffleSplitRows(
X, 1, self.validationSize, numRows=numValidationUsers)[0]
testIndPtr, testColInds = SparseUtils.getOmegaListPtr(testX)
logging.debug("Train X shape and nnz: " + str(trainX.shape) + " " +
str(trainX.nnz))
logging.debug("Validation X shape and nnz: " + str(testX.shape) +
" " + str(testX.nnz))
else:
trainX = X
testX = None
rowSamples = None
testIndPtr, testColInds = None, None
#Note that to compute the test AUC we pick i \in X and j \notin X \cup testX
indPtr, colInds = SparseUtils.getOmegaListPtr(trainX)
allIndPtr, allColInds = SparseUtils.getOmegaListPtr(X)
if type(U) != numpy.ndarray and type(V) != numpy.ndarray:
U, V = self.initUV(trainX)
if self.metric == "f1":
metricInd = 2
elif self.metric == "mrr":
metricInd = 3
else:
raise ValueError("Unknown metric: " + self.metric)
muU = U.copy()
muV = V.copy()
bestMetric = 0
bestU = 0
bestV = 0
trainMeasures = []
testMeasures = []
loopInd = 0
lastObj = 0
currentObj = lastObj - 2 * self.eps
#Try alternative number of iterations
#numIterations = trainX.nnz/self.numAucSamples
numIterations = max(m, n)
self.learnerCython = self.getCythonLearner()
#Set up order of indices for stochastic methods
permutedRowInds = numpy.array(numpy.random.permutation(m),
numpy.uint32)
permutedColInds = numpy.array(numpy.random.permutation(n),
numpy.uint32)
startTime = time.time()
gi, gp, gq = self.computeGipq(X)
normGp, normGq = self.computeNormGpq(indPtr, colInds, gp, gq, m)
while loopInd < self.maxIterations and abs(lastObj -
currentObj) > self.eps:
sigmaU = self.getSigma(loopInd, self.alpha, m)
sigmaV = self.getSigma(loopInd, self.alpha, m)
if loopInd % self.recordStep == 0:
if loopInd != 0 and self.stochastic:
print("")
printStr = self.recordResults(muU, muV, trainMeasures,
testMeasures, loopInd,
rowSamples, indPtr, colInds,
testIndPtr, testColInds,
allIndPtr, allColInds, gi, gp,
gq, trainX, startTime)
logging.debug(printStr)
if testIndPtr is not None and testMeasures[-1][
metricInd] >= bestMetric:
bestMetric = testMeasures[-1][metricInd]
logging.debug("Current best metric=" + str(bestMetric))
bestU = muU.copy()
bestV = muV.copy()
elif testIndPtr is None:
bestU = muU.copy()
bestV = muV.copy()
#Compute objective averaged over last 5 recorded steps
trainMeasuresArr = numpy.array(trainMeasures)
lastObj = currentObj
currentObj = numpy.mean(trainMeasuresArr[-5:, 0])
U = numpy.ascontiguousarray(U)
self.updateUV(indPtr, colInds, U, V, muU, muV, permutedRowInds,
permutedColInds, gp, gq, normGp, normGq, loopInd,
sigmaU, sigmaV, numIterations)
loopInd += 1
#Compute quantities for last U and V
totalTime = time.time() - startTime
printStr = "\nFinished, time=" + str('%.1f' % totalTime) + " "
printStr += self.recordResults(muU, muV, trainMeasures, testMeasures,
loopInd, rowSamples, indPtr, colInds,
testIndPtr, testColInds, allIndPtr,
allColInds, gi, gp, gq, trainX,
startTime)
printStr += " delta obj=" + "%.3e" % abs(lastObj - currentObj)
logging.debug(printStr)
self.U = bestU
self.V = bestV
self.gi = gi
self.gp = gp
self.gq = gq
trainMeasures = numpy.array(trainMeasures)
testMeasures = numpy.array(testMeasures)
if verbose:
return self.U, self.V, trainMeasures, testMeasures, loopInd, totalTime
else:
return self.U, self.V
def recordResults(self, X, trainX, testX, learner, fileName):
"""
Save results for a particular recommendation
"""
if self.algoArgs.skipRecordResults:
logging.debug("Skipping final evaluation of algorithm")
return
allTrainMeasures = []
allTestMeasures = []
allMetaData = []
for i in range(self.algoArgs.recordFolds):
metaData = []
w = 1-self.algoArgs.u
logging.debug("Computing recommendation errors")
maxItems = self.ps[-1]
start = time.time()
if type(learner) == IterativeSoftImpute:
trainIterator = iter([trainX])
ZList = learner.learnModel(trainIterator)
U, s, V = ZList.next()
U = U*s
#trainX = sppy.csarray(trainX)
#testX = sppy.csarray(testX)
U = numpy.ascontiguousarray(U)
V = numpy.ascontiguousarray(V)
else:
learner.learnModel(trainX)
U = learner.U
V = learner.V
learnTime = time.time()-start
metaData.append(learnTime)
logging.debug("Getting all omega")
allOmegaPtr = SparseUtils.getOmegaListPtr(X)
logging.debug("Getting train omega")
trainOmegaPtr = SparseUtils.getOmegaListPtr(trainX)
logging.debug("Getting test omega")
testOmegaPtr = SparseUtils.getOmegaListPtr(testX)
logging.debug("Getting recommendations")
trainOrderedItems = MCEvaluator.recommendAtk(U, V, maxItems)
testOrderedItems = MCEvaluatorCython.recommendAtk(U, V, maxItems, trainX)
colNames = []
trainMeasures = []
testMeasures = []
for p in self.ps:
trainMeasures.append(MCEvaluator.precisionAtK(trainOmegaPtr, trainOrderedItems, p))
testMeasures.append(MCEvaluator.precisionAtK(testOmegaPtr, testOrderedItems, p))
colNames.append("precision@" + str(p))
for p in self.ps:
trainMeasures.append(MCEvaluator.recallAtK(trainOmegaPtr, trainOrderedItems, p))
testMeasures.append(MCEvaluator.recallAtK(testOmegaPtr, testOrderedItems, p))
colNames.append("recall@" + str(p))
for p in self.ps:
trainMeasures.append(MCEvaluator.f1AtK(trainOmegaPtr, trainOrderedItems, p))
testMeasures.append(MCEvaluator.f1AtK(testOmegaPtr, testOrderedItems, p))
colNames.append("f1@" + str(p))
for p in self.ps:
trainMeasures.append(MCEvaluator.mrrAtK(trainOmegaPtr, trainOrderedItems, p))
testMeasures.append(MCEvaluator.mrrAtK(testOmegaPtr, testOrderedItems, p))
colNames.append("mrr@" + str(p))
try:
r = SparseUtilsCython.computeR(U, V, w, self.algoArgs.numRecordAucSamples)
trainMeasures.append(MCEvaluator.localAUCApprox(trainOmegaPtr, U, V, w, self.algoArgs.numRecordAucSamples, r=r))
testMeasures.append(MCEvaluator.localAUCApprox(testOmegaPtr, U, V, w, self.algoArgs.numRecordAucSamples, allArray=allOmegaPtr, r=r))
w = 0.0
r = SparseUtilsCython.computeR(U, V, w, self.algoArgs.numRecordAucSamples)
trainMeasures.append(MCEvaluator.localAUCApprox(trainOmegaPtr, U, V, w, self.algoArgs.numRecordAucSamples, r=r))
testMeasures.append(MCEvaluator.localAUCApprox(testOmegaPtr, U, V, w, self.algoArgs.numRecordAucSamples, allArray=allOmegaPtr, r=r))
colNames.append("LAUC@" + str(self.algoArgs.u))
colNames.append("AUC")
except:
logging.debug("Could not compute AUCs")
raise
trainMeasures = numpy.array(trainMeasures)
testMeasures = numpy.array(testMeasures)
metaData = numpy.array(metaData)
allTrainMeasures.append(trainMeasures)
allTestMeasures.append(testMeasures)
allMetaData.append(metaData)
allTrainMeasures = numpy.array(allTrainMeasures)
allTestMeasures = numpy.array(allTestMeasures)
allMetaData = numpy.array(allMetaData)
meanTrainMeasures = numpy.mean(allTrainMeasures, 0)
meanTestMeasures = numpy.mean(allTestMeasures, 0)
meanMetaData = numpy.mean(allMetaData, 0)
logging.debug("Mean metrics")
for i, colName in enumerate(colNames):
logging.debug(colName + ":" + str('%.4f' % meanTrainMeasures[i]) + "/" + str('%.4f' % meanTestMeasures[i]))
numpy.savez(fileName, meanTrainMeasures, meanTestMeasures, meanMetaData, trainOrderedItems, testOrderedItems)
logging.debug("Saved file as " + fileName)
def testDerivativeV(self):
m = 10
n = 20
nnzPerRow = 5
X = SparseUtils.generateSparseBinaryMatrix((m, n), nnzPerRow, csarray=True)
for i in range(m):
X[i, 0] = 1
X[i, 1] = 0
k = 5
u = 0.1
w = 1-u
eps = 0.05
learner = MaxAUCSigmoid(k, w)
learner.normalise = False
learner.lmbdaU = 0
learner.lmbdaV = 0
learner.rho = 1.0
learner.numAucSamples = 100
numRuns = 20
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
gp = numpy.random.rand(n)
gp /= gp.sum()
gq = numpy.random.rand(n)
gq /= gq.sum()
for s in range(numRuns):
U = numpy.random.randn(m, k)
V = numpy.random.randn(n, k)
deltaV = numpy.zeros(V.shape)
for j in range(n):
deltaV[j, :] = learner.derivativeVi(indPtr, colInds, U, V, gp, gq, j)
deltaV2 = numpy.zeros(V.shape)
eps = 0.00001
for i in range(n):
for j in range(k):
tempV = V.copy()
tempV[i,j] += eps
obj1 = learner.objective(indPtr, colInds, indPtr, colInds, U, tempV, gp, gq)
tempV = V.copy()
tempV[i,j] -= eps
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, U, tempV, gp, gq)
deltaV2[i,j] = (obj1-obj2)/(2*eps)
#deltaV2[i,:] = deltaV2[i,:]/numpy.linalg.norm(deltaV2[i,:])
nptst.assert_almost_equal(deltaV, deltaV2, 3)
#Try r != 0 and rho > 0
for s in range(numRuns):
U = numpy.random.randn(m, k)
V = numpy.random.randn(n, k)
learner.rho = 1.0
deltaV = numpy.zeros(V.shape)
for j in range(n):
deltaV[j, :] = learner.derivativeVi(indPtr, colInds, U, V, gp, gq, j)
deltaV2 = numpy.zeros(V.shape)
for i in range(n):
for j in range(k):
tempV = V.copy()
tempV[i,j] += eps
obj1 = learner.objective(indPtr, colInds, indPtr, colInds, U, tempV, gp, gq)
tempV = V.copy()
tempV[i,j] -= eps
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, U, tempV, gp, gq)
deltaV2[i,j] = (obj1-obj2)/(2*eps)
#deltaV2[i,:] = deltaV2[i,:]/numpy.linalg.norm(deltaV2[i,:])
nptst.assert_almost_equal(deltaV, deltaV2, 3)
#Try r != 0 and rho > 0
for s in range(numRuns):
U = numpy.random.randn(m, k)
V = numpy.random.randn(n, k)
learner.lmbdaV = 100
learner.rho = 0.1
deltaV = numpy.zeros(V.shape)
for j in range(n):
deltaV[j, :] = learner.derivativeVi(indPtr, colInds, U, V, gp, gq, j)
deltaV2 = numpy.zeros(V.shape)
for i in range(n):
for j in range(k):
tempV = V.copy()
tempV[i,j] += eps
obj1 = learner.objective(indPtr, colInds, indPtr, colInds, U, tempV, gp, gq)
tempV = V.copy()
tempV[i,j] -= eps
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, U, tempV, gp, gq)
deltaV2[i,j] = (obj1-obj2)/(2*eps)
#deltaV2[i,:] = deltaV2[i,:]/numpy.linalg.norm(deltaV2[i,:])
nptst.assert_almost_equal(deltaV, deltaV2, 3)
def testObjectiveApprox(self):
"""
We'll test the case in which we apprormate using a large number of samples
for the AUC and see if we get close to the exact objective
"""
m = 20
n = 30
k = 3
X = SparseUtils.generateSparseBinaryMatrix((m, n), k, csarray=True)
learner = MaxAUCSigmoid(k)
learner.normalise = False
learner.lmbdaU = 0
learner.lmbdaV = 0
learner.rho = 1.0
learner.numAucSamples = n
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
numRuns = 100
numTests = 5
gi = numpy.random.rand(m)
gi /= gi.sum()
gp = numpy.random.rand(n)
gp /= gp.sum()
gq = numpy.random.rand(n)
gq /= gq.sum()
#gi = numpy.ones(m)
#gp = numpy.ones(n)
#gq = numpy.ones(n)
#Let's compare against using the exact derivative
for i in range(numTests):
obj = 0
for j in range(numRuns):
obj += learner.objectiveApprox(indPtr, colInds, indPtr,
colInds, U, V, gp, gq)
obj /= numRuns
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, U, V,
gp, gq)
self.assertAlmostEquals(obj, obj2, 2)
learner.rho = 0.2
for i in range(numTests):
obj = 0
for j in range(numRuns):
obj += learner.objectiveApprox(indPtr, colInds, indPtr,
colInds, U, V, gp, gq)
obj /= numRuns
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, U, V,
gp, gq)
self.assertAlmostEquals(obj, obj2, 2)
learner.lmbdaV = 0.2
for i in range(numTests):
obj = 0
for j in range(numRuns):
obj += learner.objectiveApprox(indPtr, colInds, indPtr,
colInds, U, V, gp, gq)
obj /= numRuns
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, U, V,
gp, gq)
self.assertAlmostEquals(obj, obj2, 2)
#Check full and summary versions are the same
obj = learner.objective(indPtr, colInds, indPtr, colInds, U, V, gp, gq)
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, U, V, gp,
gq)
self.assertAlmostEquals(obj, obj2, 2)
def testDerivativeU(self):
m = 10
n = 20
nnzPerRow = 5
X = SparseUtils.generateSparseBinaryMatrix((m, n), nnzPerRow, csarray=True)
k = 5
eps = 0.05
learner = MaxAUCSigmoid(k)
learner.normalise = False
learner.lmbdaU = 0
learner.lmbdaV = 0
learner.rho = 1.0
learner.numAucSamples = n
numRuns = 20
gi = numpy.random.rand(m)
gi /= gi.sum()
gp = numpy.random.rand(n)
gp /= gp.sum()
gq = numpy.random.rand(n)
gq /= gq.sum()
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
for s in range(numRuns):
U = numpy.random.randn(m, k)
V = numpy.random.randn(n, k)
deltaU = numpy.zeros(U.shape)
for i in range(X.shape[0]):
deltaU[i, :] = learner.derivativeUi(indPtr, colInds, U, V, gp, gq, i)
deltaU2 = numpy.zeros(U.shape)
eps = 10**-8
for i in range(m):
for j in range(k):
tempU = U.copy()
tempU[i,j] += eps
obj1 = learner.objective(indPtr, colInds, indPtr, colInds, tempU, V, gp, gq)
tempU = U.copy()
tempU[i,j] -= eps
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, tempU, V, gp, gq)
deltaU2[i,j] = (obj1-obj2)/(2*eps)
#deltaU2[i,:] = deltaU2[i,:]/numpy.linalg.norm(deltaU2[i,:])
#print(deltaU*100)
#print(deltaU2*100)
nptst.assert_almost_equal(deltaU, deltaU2, 3)
#Try r != 0 and rho > 0
for s in range(numRuns):
U = numpy.random.randn(m, k)
V = numpy.random.randn(n, k)
learner.rho = 0.1
deltaU = numpy.zeros(U.shape)
for i in range(X.shape[0]):
deltaU[i, :] = learner.derivativeUi(indPtr, colInds, U, V, gp, gq, i)
deltaU2 = numpy.zeros(U.shape)
eps = 10**-9
for i in range(m):
for j in range(k):
tempU = U.copy()
tempU[i,j] += eps
obj1 = learner.objective(indPtr, colInds, indPtr, colInds, tempU, V, gp, gq)
tempU = U.copy()
tempU[i,j] -= eps
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, tempU, V, gp, gq)
deltaU2[i,j] = (obj1-obj2)/(2*eps)
nptst.assert_almost_equal(deltaU, deltaU2, 3)
#Try lmbda > 0
for s in range(numRuns):
U = numpy.random.randn(m, k)
V = numpy.random.randn(n, k)
learner.lmbdaU = 0.5
deltaU = numpy.zeros(U.shape)
for i in range(X.shape[0]):
deltaU[i, :] = learner.derivativeUi(indPtr, colInds, U, V, gp, gq, i)
deltaU2 = numpy.zeros(U.shape)
eps = 10**-9
for i in range(m):
for j in range(k):
tempU = U.copy()
tempU[i,j] += eps
obj1 = learner.objective(indPtr, colInds, indPtr, colInds, tempU, V, gp, gq)
tempU = U.copy()
tempU[i,j] -= eps
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, tempU, V, gp, gq)
deltaU2[i,j] = (obj1-obj2)/(2*eps)
nptst.assert_almost_equal(deltaU, deltaU2, 3)
learningRateParams = [(4.0, 1.0), (4.0, 0.5), (4.0, 0.1), (1.0, 1.0), (1.0, 0.5), (1.0, 0.1), (0.25, 1.0), (0.25, 0.5), (0.25, 0.1)]
print(startAverages)
def computeTestObj(args):
trainX, maxLocalAuc = args
numpy.random.seed(21)
U, V, trainMeasures, testMeasures, iterations, totalTime = maxLocalAuc.learnModel(trainX, verbose=True)
return U, V, trainMeasures[-1, 0], testMeasures[-1, 0]
if saveResults:
trainObjectives = numpy.zeros((startAverages.shape[0], len(learningRateParams)))
testObjectives = numpy.zeros((startAverages.shape[0], len(learningRateParams)))
for trainX, testX in trainTestXs:
trainOmegaPtr = SparseUtils.getOmegaListPtr(trainX)
testOmegaPtr = SparseUtils.getOmegaListPtr(testX)
allOmegaPtr = SparseUtils.getOmegaListPtr(X)
logging.debug("Number of non-zero elements: " + str((trainX.nnz, testX.nnz)))
paramList = []
for j, startAverage in enumerate(startAverages):
for i, (alpha, t0) in enumerate(learningRateParams):
maxLocalAuc.startAverage = startAverage
maxLocalAuc.alpha = alpha
maxLocalAuc.t0 = t0
logging.debug(maxLocalAuc)
learner = maxLocalAuc.copy()
paramList.append((trainX, learner))
def testObjectiveApprox(self):
"""
We'll test the case in which we apprormate using a large number of samples
for the AUC and see if we get close to the exact objective
"""
m = 20
n = 30
k = 3
X = SparseUtils.generateSparseBinaryMatrix((m, n), k, csarray=True)
learner = MaxAUCSigmoid(k)
learner.normalise = False
learner.lmbdaU = 0
learner.lmbdaV = 0
learner.rho = 1.0
learner.numAucSamples = n
indPtr, colInds = SparseUtils.getOmegaListPtr(X)
U = numpy.random.rand(X.shape[0], k)
V = numpy.random.rand(X.shape[1], k)
numRuns = 100
numTests = 5
gi = numpy.random.rand(m)
gi /= gi.sum()
gp = numpy.random.rand(n)
gp /= gp.sum()
gq = numpy.random.rand(n)
gq /= gq.sum()
#gi = numpy.ones(m)
#gp = numpy.ones(n)
#gq = numpy.ones(n)
#Let's compare against using the exact derivative
for i in range(numTests):
obj = 0
for j in range(numRuns):
obj += learner.objectiveApprox(indPtr, colInds, indPtr, colInds, U, V, gp, gq)
obj /= numRuns
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, U, V, gp, gq)
self.assertAlmostEquals(obj, obj2, 2)
learner.rho = 0.2
for i in range(numTests):
obj = 0
for j in range(numRuns):
obj += learner.objectiveApprox(indPtr, colInds, indPtr, colInds, U, V, gp, gq)
obj /= numRuns
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, U, V, gp, gq)
self.assertAlmostEquals(obj, obj2, 2)
learner.lmbdaV = 0.2
for i in range(numTests):
obj = 0
for j in range(numRuns):
obj += learner.objectiveApprox(indPtr, colInds, indPtr, colInds, U, V, gp, gq)
obj /= numRuns
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, U, V, gp, gq)
self.assertAlmostEquals(obj, obj2, 2)
#Check full and summary versions are the same
obj = learner.objective(indPtr, colInds, indPtr, colInds, U, V, gp, gq)
obj2 = learner.objective(indPtr, colInds, indPtr, colInds, U, V, gp, gq)
self.assertAlmostEquals(obj, obj2, 2)
def singleLearnModel(self, X, verbose=False, U=None, V=None):
"""
Max local AUC with Frobenius norm penalty on V. Solve with (stochastic) gradient descent.
The input is a sparse array.
"""
# Convert to a csarray for faster access
if scipy.sparse.issparse(X):
logging.debug("Converting to csarray")
X2 = sppy.csarray(X, storagetype="row")
X = X2
m, n = X.shape
# We keep a validation set in order to determine when to stop
if self.validationUsers != 0:
numValidationUsers = int(m * self.validationUsers)
trainX, testX, rowSamples = Sampling.shuffleSplitRows(
X, 1, self.validationSize, numRows=numValidationUsers
)[0]
testIndPtr, testColInds = SparseUtils.getOmegaListPtr(testX)
logging.debug("Train X shape and nnz: " + str(trainX.shape) + " " + str(trainX.nnz))
logging.debug("Validation X shape and nnz: " + str(testX.shape) + " " + str(testX.nnz))
else:
trainX = X
testX = None
rowSamples = None
testIndPtr, testColInds = None, None
# Note that to compute the test AUC we pick i \in X and j \notin X \cup testX
indPtr, colInds = SparseUtils.getOmegaListPtr(trainX)
allIndPtr, allColInds = SparseUtils.getOmegaListPtr(X)
if type(U) != numpy.ndarray and type(V) != numpy.ndarray:
U, V = self.initUV(trainX)
if self.metric == "f1":
metricInd = 2
elif self.metric == "mrr":
metricInd = 3
else:
raise ValueError("Unknown metric: " + self.metric)
muU = U.copy()
muV = V.copy()
bestMetric = 0
bestU = 0
bestV = 0
trainMeasures = []
testMeasures = []
loopInd = 0
lastObj = 0
currentObj = lastObj - 2 * self.eps
# Try alternative number of iterations
# numIterations = trainX.nnz/self.numAucSamples
numIterations = max(m, n)
self.learnerCython = self.getCythonLearner()
# Set up order of indices for stochastic methods
permutedRowInds = numpy.array(numpy.random.permutation(m), numpy.uint32)
permutedColInds = numpy.array(numpy.random.permutation(n), numpy.uint32)
startTime = time.time()
gi, gp, gq = self.computeGipq(X)
normGp, normGq = self.computeNormGpq(indPtr, colInds, gp, gq, m)
while loopInd < self.maxIterations and abs(lastObj - currentObj) > self.eps:
sigmaU = self.getSigma(loopInd, self.alpha, m)
sigmaV = self.getSigma(loopInd, self.alpha, m)
if loopInd % self.recordStep == 0:
if loopInd != 0 and self.stochastic:
print("")
printStr = self.recordResults(
muU,
muV,
trainMeasures,
testMeasures,
loopInd,
rowSamples,
indPtr,
colInds,
testIndPtr,
testColInds,
allIndPtr,
allColInds,
gi,
gp,
gq,
trainX,
startTime,
)
logging.debug(printStr)
if testIndPtr is not None and testMeasures[-1][metricInd] >= bestMetric:
bestMetric = testMeasures[-1][metricInd]
logging.debug("Current best metric=" + str(bestMetric))
bestU = muU.copy()
bestV = muV.copy()
elif testIndPtr is None:
bestU = muU.copy()
bestV = muV.copy()
# Compute objective averaged over last 5 recorded steps
trainMeasuresArr = numpy.array(trainMeasures)
lastObj = currentObj
currentObj = numpy.mean(trainMeasuresArr[-5:, 0])
U = numpy.ascontiguousarray(U)
self.updateUV(
indPtr,
colInds,
U,
V,
muU,
muV,
permutedRowInds,
permutedColInds,
gp,
gq,
normGp,
normGq,
loopInd,
sigmaU,
sigmaV,
numIterations,
)
loopInd += 1
# Compute quantities for last U and V
totalTime = time.time() - startTime
printStr = "\nFinished, time=" + str("%.1f" % totalTime) + " "
printStr += self.recordResults(
muU,
muV,
trainMeasures,
testMeasures,
loopInd,
rowSamples,
indPtr,
colInds,
testIndPtr,
testColInds,
allIndPtr,
allColInds,
gi,
gp,
gq,
trainX,
startTime,
)
printStr += " delta obj=" + "%.3e" % abs(lastObj - currentObj)
logging.debug(printStr)
self.U = bestU
self.V = bestV
self.gi = gi
self.gp = gp
self.gq = gq
trainMeasures = numpy.array(trainMeasures)
testMeasures = numpy.array(testMeasures)
if verbose:
return self.U, self.V, trainMeasures, testMeasures, loopInd, totalTime
else:
return self.U, self.V
