def testRandCrossValidation(self):
numExamples = 10
folds = 3
indices = Sampling.randCrossValidation(folds, numExamples)
for i in range(folds):
self.assertTrue((numpy.union1d(indices[i][0], indices[i][1]) == numpy.arange(numExamples)).all())
def testParallelModelSelect(self):
X = scipy.sparse.rand(10, 10, 0.5)
X = X.tocsr()
numExamples = X.getnnz()
paramDict = {}
paramDict["setK"] = numpy.array([5, 10, 20])
folds = 3
idx = Sampling.randCrossValidation(folds, numExamples)
lmbdas = numpy.array([0.1])
softImpute = SoftImpute(lmbdas, k=10)
learner, meanErrors = softImpute.parallelModelSelect(X, idx, paramDict)
def runExperiment(self):
"""
Run the selected clustering experiments and save results
"""
if self.algoArgs.runSoftImpute:
logging.debug("Running soft impute")
for svdAlg in self.algoArgs.svdAlgs:
if svdAlg == "rsvd" or svdAlg == "rsvdUpdate" or svdAlg == "rsvdUpdate2":
resultsFileName = self.resultsDir + "ResultsSoftImpute_alg=" + svdAlg + "_p=" + str(self.algoArgs.p)+ "_q=" + str(self.algoArgs.q) + "_updateAlg=" + self.algoArgs.updateAlg + ".npz"
else:
resultsFileName = self.resultsDir + "ResultsSoftImpute_alg=" + svdAlg + "_updateAlg=" + self.algoArgs.updateAlg + ".npz"
fileLock = FileLock(resultsFileName)
if not fileLock.isLocked() and not fileLock.fileExists():
fileLock.lock()
try:
learner = IterativeSoftImpute(svdAlg=svdAlg, logStep=self.logStep, kmax=self.algoArgs.kmax, postProcess=self.algoArgs.postProcess, weighted=self.algoArgs.weighted, p=self.algoArgs.p, q=self.algoArgs.q, verbose=self.algoArgs.verbose, updateAlg=self.algoArgs.updateAlg)
if self.algoArgs.modelSelect:
trainIterator = self.getTrainIterator()
#Let's find the optimal lambda using the first matrix
X = trainIterator.next()
logging.debug("Performing model selection, taking subsample of entries of size " + str(self.sampleSize))
X = SparseUtils.submatrix(X, self.sampleSize)
cvInds = Sampling.randCrossValidation(self.algoArgs.folds, X.nnz)
meanErrors, stdErrors = learner.modelSelect(X, self.algoArgs.rhos, self.algoArgs.ks, cvInds)
logging.debug("Mean errors = " + str(meanErrors))
logging.debug("Std errors = " + str(stdErrors))
modelSelectFileName = resultsFileName.replace("Results", "ModelSelect")
numpy.savez(modelSelectFileName, meanErrors, stdErrors)
logging.debug("Saved model selection grid as " + modelSelectFileName)
rho = self.algoArgs.rhos[numpy.unravel_index(numpy.argmin(meanErrors), meanErrors.shape)[0]]
k = self.algoArgs.ks[numpy.unravel_index(numpy.argmin(meanErrors), meanErrors.shape)[1]]
else:
rho = self.algoArgs.rhos[0]
k = self.algoArgs.ks[0]
learner.setK(k)
learner.setRho(rho)
logging.debug(learner)
trainIterator = self.getTrainIterator()
ZIter = learner.learnModel(trainIterator)
self.recordResults(ZIter, learner, resultsFileName)
finally:
fileLock.unlock()
else:
logging.debug("File is locked or already computed: " + resultsFileName)
if self.algoArgs.runSgdMf:
logging.debug("Running SGD MF")
resultsFileName = self.resultsDir + "ResultsSgdMf.npz"
fileLock = FileLock(resultsFileName)
if not fileLock.isLocked() and not fileLock.fileExists():
fileLock.lock()
try:
learner = IterativeSGDNorm2Reg(k=self.algoArgs.ks[0], lmbda=self.algoArgs.lmbdas[0], gamma=self.algoArgs.gammas[0], eps=self.algoArgs.eps)
if self.algoArgs.modelSelect:
# Let's find optimal parameters using the first matrix
learner.modelSelect(self.getTrainIterator().next(), self.algoArgs.ks, self.algoArgs.lmbdas, self.algoArgs.gammas, self.algoArgs.folds)
trainIterator = self.getTrainIterator()
trainIterator = self.getTrainIterator()
ZIter = learner.learnModel(trainIterator)
self.recordResults(ZIter, learner, resultsFileName)
finally:
fileLock.unlock()
else:
logging.debug("File is locked or already computed: " + resultsFileName)
logging.info("All done: see you around!")
