def get_global_suff_stats(self, Data, LP, doPrecompEntropy=0, **kwargs): ''' Compute sufficient stats for provided dataset and local params Returns ------- SS : SuffStatBag with K components and fields * sumSource : nNodes x K * sumReceiver : nNodes x K ''' V = Data.nNodes K = LP['resp'].shape[-1] SS = SuffStatBag(K=K, D=Data.dim, V=V) if 'NodeStateCount' not in LP: assert 'resp' in LP LP = self.initLPFromResp(Data, LP) SS.setField('NodeStateCount', LP['NodeStateCount'], dims=('V', 'K')) if np.allclose(LP['resp'].sum(axis=1).min(), 1.0): # If the LP fully represents all present edges, # then the NodeStateCount should as well. assert np.allclose(SS.NodeStateCount, Data.nEdges * 2) SS.setField('N', LP['N_fg'], dims=('K', )) SS.setField('scaleFactor', Data.nEdges, dims=None) if 'Ldata_bg' in LP: SS.setELBOTerm('Ldata_bg', LP['Ldata_bg'], dims=None) if doPrecompEntropy: Hresp_fg = LP['Lentropy_fg'] # = -1 * calcRlogR(LP['resp']) Hresp_bg = LP['Lentropy_bg'] SS.setELBOTerm('Hresp', Hresp_fg, dims='K') SS.setELBOTerm('Hresp_bg', Hresp_bg, dims=None) return SS
def test_BPlanner_makePlanAtBatch_someDisqualifiedForPrevFailures(K=10): SS = SuffStatBag(K=K) SS.setField('N', np.arange(K), dims='K') SSbatch = SS.copy() # Do the same test, while eliminating some uids MoveRecordsByUID = defaultdict(lambda: defaultdict(int)) for uid in [0, 6, 9]: MoveRecordsByUID[uid]['b_nFail'] = 1 MoveRecordsByUID[uid]['b_nFailRecent'] = 1 MoveRecordsByUID[uid]['b_batchIDsWhoseProposalFailed'] = set([0]) for b_minNumAtomsForTargetComp in [2, 5, K]: BArgs['b_minNumAtomsForTargetComp'] = b_minNumAtomsForTargetComp MovePlans = selectCompsForBirthAtCurrentBatch( SS=SS, SSbatch=SSbatch, MovePlans=dict(), MoveRecordsByUID=MoveRecordsByUID, **BArgs) nChosen = len(MovePlans['b_targetUIDs']) nFailPerUID = list() for uid in SS.uids: bIDs = MoveRecordsByUID[uid]['b_batchIDsWhoseProposalFailed'] if isinstance(bIDs, set): nFailPerUID.append(len(bIDs)) else: nFailPerUID.append(0) nFailPerUID = np.asarray(nFailPerUID) nExpected = np.sum( np.logical_and(SS.N >= b_minNumAtomsForTargetComp, nFailPerUID < 1)) assert nChosen == nExpected
def get_global_suff_stats(self, Data, LP, doPrecompEntropy=None, **kwargs): ''' Calculate the sufficient statistics for global parameter updates Only adds stats relevant for this allocModel. Other stats are added by the obsModel. Args ------- Data : bnpy data object LP : local param dict with fields resp : Data.nObs x K array, where resp[n,k] = posterior resp of comp k doPrecompEntropy : boolean flag indicates whether to precompute ELBO terms in advance used for memoized learning algorithms (moVB) Returns ------- SS : SuffStats for K components, with field N : vector of length-K, effective number of observations assigned to each comp ''' Nvec = np.sum( LP['resp'], axis=0 ) SS = SuffStatBag(K=Nvec.size, D=Data.dim) SS.setField('N', Nvec, dims=('K')) if doPrecompEntropy is not None: ElogqZ_vec = self.E_logqZ(LP) SS.setELBOTerm('ElogqZ', ElogqZ_vec, dims=('K')) return SS
def get_global_suff_stats(self, Data, LP, doPrecompEntropy=False, doPrecompMergeEntropy=False, mPairIDs=None): ''' Count expected number of times each topic is used across all docs ''' wv = LP['word_variational'] _, K = wv.shape # Turn dim checking off, since some stats have dim K+1 instead of K SS = SuffStatBag(K=K, D=Data.vocab_size) SS.setField('nDoc', Data.nDoc, dims=None) sumLogPi = np.sum(LP['E_logPi'], axis=0) SS.setField('sumLogPiActive', sumLogPi[:K], dims='K') SS.setField('sumLogPiUnused', sumLogPi[-1], dims=None) if 'DocTopicFrac' in LP: Nmajor = LP['DocTopicFrac'] Nmajor[Nmajor < 0.05] = 0 SS.setField('Nmajor', np.sum(Nmajor, axis=0), dims='K') if doPrecompEntropy: # ---------------- Z terms SS.setELBOTerm('ElogpZ', self.E_logpZ(Data, LP), dims='K') # ---------------- Pi terms # Note: no terms needed for ElogpPI # SS already has field sumLogPi, which is sufficient for this term ElogqPiC, ElogqPiA, ElogqPiU = self.E_logqPi_Memoized_from_LP(LP) SS.setELBOTerm('ElogqPiConst', ElogqPiC, dims=None) SS.setELBOTerm('ElogqPiActive', ElogqPiA, dims='K') SS.setELBOTerm('ElogqPiUnused', ElogqPiU, dims=None) if doPrecompMergeEntropy: ElogpZMat, sLgPiMat, ElogqPiMat = self.memo_elbo_terms_for_merge(LP) SS.setMergeTerm('ElogpZ', ElogpZMat, dims=('K','K')) SS.setMergeTerm('ElogqPiActive', ElogqPiMat, dims=('K','K')) SS.setMergeTerm('sumLogPiActive', sLgPiMat, dims=('K','K')) return SS
def calcSummaryStats(Data, LP, doPrecompEntropy=0, doPrecompMergeEntropy=0, mPairIDs=None, trackDocUsage=0, **kwargs): ''' Calculate summary statistics for given data slice and local params. Returns ------- SS : SuffStatBag ''' if mPairIDs is None: M = 0 else: M = len(mPairIDs) resp = LP['resp'] K = resp.shape[1] startLocIDs = Data.doc_range[:-1] StartStateCount = np.sum(resp[startLocIDs], axis=0) N = np.sum(resp, axis=0) if 'TransCount' in LP: TransStateCount = np.sum(LP['TransCount'], axis=0) else: respPair = LP['respPair'] TransStateCount = np.sum(respPair, axis=0) SS = SuffStatBag(K=K, D=Data.dim, M=M) SS.setField('StartStateCount', StartStateCount, dims=('K')) SS.setField('TransStateCount', TransStateCount, dims=('K', 'K')) SS.setField('N', N, dims=('K')) SS.setField('nDoc', Data.nDoc, dims=None) if doPrecompEntropy or 'Htable' in LP: # Compute entropy terms! # 'Htable', 'Hstart' will both be in Mdict Mdict = calcELBO_NonlinearTerms(Data=Data, LP=LP, returnMemoizedDict=1) SS.setELBOTerm('Htable', Mdict['Htable'], dims=('K', 'K')) SS.setELBOTerm('Hstart', Mdict['Hstart'], dims=('K')) if doPrecompMergeEntropy: subHstart, subHtable = HMMUtil.PrecompMergeEntropy_SpecificPairs( LP, Data, mPairIDs) SS.setMergeTerm('Hstart', subHstart, dims=('M')) SS.setMergeTerm('Htable', subHtable, dims=('M', 2, 'K')) SS.mPairIDs = np.asarray(mPairIDs) if trackDocUsage: # Track how often topic appears in a seq. with mass > thresh. DocUsage = np.zeros(K) for n in range(Data.nDoc): start = Data.doc_range[n] stop = Data.doc_range[n + 1] DocUsage += np.sum(LP['resp'][start:stop], axis=0) > 0.01 SS.setSelectionTerm('DocUsageCount', DocUsage, dims='K') return SS
def calcHardMergeGap(self, SS, kA, kB): ''' Calculate scalar improvement in ELBO for hard merge of comps kA, kB Does *not* include any entropy. Returns --------- L : scalar ''' m_K = SS.K - 1 m_SS = SuffStatBag(K=SS.K, D=0) m_SS.setField('StartStateCount', SS.StartStateCount.copy(), dims='K') m_SS.setField('TransStateCount', SS.TransStateCount.copy(), dims=('K', 'K')) m_SS.mergeComps(kA, kB) # Create candidate beta vector m_beta = StickBreakUtil.rho2beta(self.rho) m_beta[kA] += m_beta[kB] m_beta = np.delete(m_beta, kB, axis=0) # Create candidate rho and omega vectors m_rho = StickBreakUtil.beta2rho(m_beta, m_K) m_omega = np.delete(self.omega, kB) # Create candidate startTheta m_startTheta = self.startAlpha * m_beta.copy() m_startTheta[:m_K] += m_SS.StartStateCount # Create candidate transTheta m_transTheta = self.alpha * np.tile(m_beta, (m_K, 1)) if self.kappa > 0: m_transTheta[:, :m_K] += self.kappa * np.eye(m_K) m_transTheta[:, :m_K] += m_SS.TransStateCount # Evaluate objective func. for both candidate and current model Lcur = calcELBO_LinearTerms(SS=SS, rho=self.rho, omega=self.omega, startTheta=self.startTheta, transTheta=self.transTheta, alpha=self.alpha, startAlpha=self.startAlpha, gamma=self.gamma, kappa=self.kappa) Lprop = calcELBO_LinearTerms(SS=m_SS, rho=m_rho, omega=m_omega, startTheta=m_startTheta, transTheta=m_transTheta, alpha=self.alpha, startAlpha=self.startAlpha, gamma=self.gamma, kappa=self.kappa) # Note: This gap relies on fact that all nonlinear terms are entropies, return Lprop - Lcur
def setUp(self): self.alpha0 = 2.0 self.allocM = MixModel('EM', dict(alpha0=self.alpha0)) self.N = np.asarray([1., 2., 3, 4, 5.]) self.SS = SuffStatBag(K=5, D=1) self.SS.setField('N', self.N, dims='K') self.resp = np.random.rand(100, 3) self.precompEntropy = np.sum(self.resp * np.log(self.resp), axis=0)
def setUp(self): self.gamma = 2.0 self.allocM = FiniteMixtureModel('EM', dict(gamma=self.gamma)) self.N = np.asarray([1., 2., 3, 4, 5.]) self.SS = SuffStatBag(K=5, D=1) self.SS.setField('N', self.N, dims='K') self.resp = np.random.rand(100, 3) self.precompEntropy = -1 * np.sum(self.resp * np.log(self.resp), axis=0)
def setUp(self): ''' Create a stupid simple case for making sure we're calculating things correctly ''' self.alpha0 = 1.0 self.allocM = MixModel('EM', dict(alpha0=self.alpha0)) self.N = np.asarray([1., 2., 3, 4, 5.]) self.SS = SuffStatBag(K=5, D=1) self.SS.setField('N', self.N, dims='K') self.resp = np.random.rand(100, 3) self.precompEntropy = np.sum(self.resp * np.log(self.resp), axis=0)
def test_BPlanner_makePlanAtBatch_noPrevFailures(K=10): SS = SuffStatBag(K=K) SS.setField('N', np.arange(K), dims='K') SSbatch = SS.copy() for b_minNumAtomsForTargetComp in [2, 5, K]: BArgs['b_minNumAtomsForTargetComp'] = b_minNumAtomsForTargetComp MovePlans = selectCompsForBirthAtCurrentBatch( SS=SS, SSbatch=SSbatch, MovePlans=dict(), **BArgs) nChosen = len(MovePlans['b_targetUIDs']) assert nChosen == np.sum(SS.N >= b_minNumAtomsForTargetComp)
def setUp(self): ''' Create simple case to double-check calculations. ''' self.gamma = 1.0 self.allocM = FiniteMixtureModel('EM', dict(gamma=self.gamma)) self.N = np.asarray([1., 2., 3, 4, 5.]) self.SS = SuffStatBag(K=5, D=1) self.SS.setField('N', self.N, dims='K') self.resp = np.random.rand(100, 3) self.precompEntropy = -1 * np.sum(self.resp * np.log(self.resp), axis=0)
def calcSummaryStatsForContigBlock(self, Data, SS=None, a=0, b=0): ''' Calculate sufficient stats for a single contiguous block of data ''' if SS is None: SS = SuffStatBag(K=1, D=Data.dim) SS.setField('N', (b - a) * np.ones(1), dims='K') SS.setField( 'x', np.sum(Data.X[a:b], axis=0)[np.newaxis, :], dims=('K', 'D')) SS.setField( 'xxT', dotATA(Data.X[a:b])[np.newaxis, :, :], dims=('K', 'D', 'D')) return SS
def calcSummaryStats(Data, SS, LP, **kwargs): ''' Calculate summary statistics for given dataset and local parameters Returns -------- SS : SuffStatBag object, with K components. ''' if not hasattr(Data, 'X_NE'): Data.X_NE = np.hstack([Data.X, np.ones(Data.nObs)[:, np.newaxis]]) Y_N = Data.Y X_NE = Data.X_NE E = X_NE.shape[1] if 'resp' in LP: # Dense responsibility calculations resp = LP['resp'] K = resp.shape[1] S_yy_K = dotATB(resp, np.square(Y_N)).flatten() S_yx_KE = dotATB(resp, Y_N * X_NE) # Expected outer product S_xxT_KEE = np.zeros((K, E, E)) sqrtResp_k_N = np.sqrt(resp[:, 0]) sqrtR_X_k_NE = sqrtResp_k_N[:, np.newaxis] * X_NE S_xxT_KEE[0] = dotATA(sqrtR_X_k_NE) for k in xrange(1, K): np.sqrt(resp[:, k], out=sqrtResp_k_N) np.multiply(sqrtResp_k_N[:, np.newaxis], X_NE, out=sqrtR_X_k_NE) S_xxT_KEE[k] = dotATA(sqrtR_X_k_NE) else: raise ValueError("TODO") spR = LP['spR'] K = spR.shape[1] if SS is None: SS = SuffStatBag(K=K, D=Data.dim, E=E) elif not hasattr(SS, 'E'): SS._Fields.E = E SS.setField('xxT_KEE', S_xxT_KEE, dims=('K', 'E', 'E')) SS.setField('yx_KE', S_yx_KE, dims=('K', 'E')) SS.setField('yy_K', S_yy_K, dims=('K')) # Expected count for each k # Usually computed by allocmodel. But just in case... if not hasattr(SS, 'N'): if 'resp' in LP: SS.setField('N', LP['resp'].sum(axis=0), dims='K') else: SS.setField('N', as1D(toCArray(LP['spR'].sum(axis=0))), dims='K') #SS.setField("N_K", SS.N, dims="K") return SS
def get_global_suff_stats(self, Data, LP, doPrecompEntropy=None, **kwargs): ''' Calculate sufficient statistics. Admixture models have no suff stats for allocation ''' wv = LP['word_variational'] _, K = wv.shape SS = SuffStatBag(K=K, D=Data.vocab_size) SS.setField('nDoc', Data.nDoc, dims=None) if doPrecompEntropy: SS.setELBOTerm('ElogpZ', self.E_log_pZ(Data, LP), dims='K') SS.setELBOTerm('ElogqZ', self.E_log_qZ(Data, LP), dims='K') SS.setELBOTerm('ElogpPi', self.E_log_pPI(Data, LP), dims=None) SS.setELBOTerm('ElogqPi', self.E_log_qPI(Data, LP), dims=None) return SS
def calcSummaryStats(Dslice, LP=None, alpha=None, doPrecompEntropy=False, cslice=(0, None), **kwargs): """ Calculate summary from local parameters for given data slice. Parameters ------- Data : bnpy data object LP : local param dict with fields resp : Data.nObs x K array, where resp[n,k] = posterior resp of comp k doPrecompEntropy : boolean flag indicates whether to precompute ELBO terms in advance used for memoized learning algorithms (moVB) Returns ------- SS : SuffStatBag with K components * nDoc : scalar float Counts total documents available in provided data. Also has optional ELBO field when precompELBO is True * Hvec : 1D array, size K Vector of entropy contributions from each comp. Hvec[k] = \sum_{n=1}^N H[q(z_n)], a function of 'resp' """ K = LP['DocTopicCount'].shape[1] SS = SuffStatBag(K=K, D=Dslice.dim) if cslice[1] is None: SS.setField('nDoc', Dslice.nDoc, dims=None) else: SS.setField('nDoc', cslice[1] - cslice[0], dims=None) SS.setField('nDoc', Dslice.nDoc, dims=None) if doPrecompEntropy: assert 'theta' in LP Lalloc = L_alloc(Dslice, LP, alpha=alpha) SS.setELBOTerm('L_alloc', Lalloc, dims=None) if 'nnzPerRow' in LP and LP['nnzPerRow'] == 1: SS.setELBOTerm('Hvec', 0.0, dims=None) else: Hvec = L_entropy(Dslice, LP, returnVector=1) SS.setELBOTerm('Hvec', Hvec, dims='K') return SS
def init_global_params(self, Data, K=0, **initArgs): ''' Initialize rho, omega, and theta to reasonable values. This is only called by "from scratch" init routines. ''' self.K = K self.rho = OptimizerRhoOmega.create_initrho(K) self.omega = (1.0 + self.gamma) * np.ones(K) # To initialize theta, perform standard update given rho, omega # but with "empty" sufficient statistics. SS = SuffStatBag(K=self.K, D=Data.dim) SS.setField('StartStateCount', np.ones(K), dims=('K')) SS.setField('TransStateCount', np.ones((K, K)), dims=('K', 'K')) self.transTheta, self.startTheta = self._calcTheta(SS)
def get_global_suff_stats(self, Data, LP, doPrecompEntropy=None, **kwargs): ''' Calculate sufficient statistics. ''' resp = LP['resp'] _, K = resp.shape SS = SuffStatBag(K=K, D=Data.get_dim()) SS.setField('nDoc', Data.nDoc, dims=None) SS.setField('sumLogVd', np.sum(LP['ElogV'], axis=0), dims='K') SS.setField('sumLog1mVd', np.sum(LP['Elog1mV'], axis=0), dims='K') if doPrecompEntropy: ElogqZ = self.E_logqZ(Data, LP) VZlocal = self.E_logpVZ_logqV(Data, LP) SS.setELBOTerm('ElogqZ', ElogqZ, dims='K') SS.setELBOTerm('VZlocal', VZlocal, dims=None) return SS
def test_entropy_posterior_gets_smaller(self, N=10): PRNG = np.random.RandomState(seed=8675309) for trial in range(3): X = PRNG.randn(N, self.distr.D) + self.distr.m x = np.sum(X, axis=0) xxT = np.dot(X.T, X) SS = SuffStatBag(K=1, D=self.distr.D) SS.setField('N', [N], dims='K') SS.setField('x', [x], dims=('K', 'D')) SS.setField('xxT', [xxT], dims=('K', 'D', 'D')) postD = self.distr.get_post_distr(SS, 0) assert postD.D == self.distr.D Hpost = postD.entropyWish() Hprior = self.distr.entropyWish() print 'Prior %.3g, Post %.3g' % (Hprior, Hpost) assert Hpost < Hprior
def calcSummaryStatsForContigBlock(self, Data, a=0, b=0, **kwargs): ''' Calculate summary stats for a contiguous block of the data. Returns -------- SS : SuffStatBag object, with 1 component. ''' Xab = Data.X[a:b] # 2D array, Nab x D CountON = np.sum(Xab, axis=0)[np.newaxis, :] CountOFF = (b - a) - CountON SS = SuffStatBag(K=1, D=Data.dim) SS.setField('N', np.asarray([b - a], dtype=np.float64), dims='K') SS.setField('Count1', CountON, dims=('K', 'D')) SS.setField('Count0', CountOFF, dims=('K', 'D')) return SS
def test_BPlanner_makePlanAtBatch_someDQForPrevFailuresWithOtherBatches(K=20): print('') SS = SuffStatBag(K=K) SS.setField('N', np.arange(K), dims='K') SSbatch = SS.copy() # Select some subset of uids to be disqualified PRNG = np.random.RandomState(11) dqUIDs = PRNG.choice(K, size=3, replace=False) otherfailUIDs = PRNG.choice(K, size=3, replace=False) # Do the same test, while eliminating some uids MoveRecordsByUID = defaultdict(lambda: defaultdict(int)) for uid in dqUIDs: MoveRecordsByUID[uid]['b_nFail'] = 1 MoveRecordsByUID[uid]['b_nFailRecent'] = 1 MoveRecordsByUID[uid]['b_batchIDsWhoseProposalFailed'] = set([0]) print('PREV FAIL AT THIS BATCH: uid ', uid) for uid in otherfailUIDs: if uid in dqUIDs: continue MoveRecordsByUID[uid]['b_nFail'] = 1 MoveRecordsByUID[uid]['b_nFailRecent'] = 1 MoveRecordsByUID[uid]['b_batchIDsWhoseProposalFailed'] = set([1]) print('PREV FAIL AT ANOTHER BATCH: uid ', uid) for b_minNumAtomsForTargetComp in [2, 5, 10, K]: BArgs['b_minNumAtomsForTargetComp'] = b_minNumAtomsForTargetComp MovePlans = selectCompsForBirthAtCurrentBatch( SS=SS, SSbatch=SSbatch, MovePlans=dict(), MoveRecordsByUID=MoveRecordsByUID, **BArgs) nChosen = len(MovePlans['b_targetUIDs']) nFailPerUID = list() for uid in SS.uids: bIDs = MoveRecordsByUID[uid]['b_batchIDsWhoseProposalFailed'] if isinstance(bIDs, set) and 0 in bIDs: nFailPerUID.append(len(bIDs)) else: nFailPerUID.append(0) nFailPerUID = np.asarray(nFailPerUID) nExpected = np.sum( np.logical_and(SS.N >= b_minNumAtomsForTargetComp, nFailPerUID < 1)) assert nChosen == nExpected
def get_global_suff_stats(self, Data, LP, doPrecompEntropy=False, doPrecompMergeEntropy=False, mPairIDs=None): ''' Calculate the sufficient statistics for global parameter updates Only adds stats relevant for this allocModel. Other stats are added by the obsModel. Args ------- Data : bnpy data object LP : local param dict with fields resp : Data.nObs x K array, where resp[n,k] = posterior resp of comp k doPrecompEntropy : boolean flag indicates whether to precompute ELBO terms in advance used for memoized learning algorithms (moVB) doPrecompMergeEntropy : boolean flag indicates whether to precompute ELBO terms in advance for all possible merges of pairs of components used for optional merge moves Returns ------- SS : SuffStats for K components, with field N : vector of length-K, effective number of observations assigned to each comp ''' Nvec = np.sum(LP['resp'], axis=0) SS = SuffStatBag(K=Nvec.size, D=Data.dim) SS.setField('N', Nvec, dims=('K')) if doPrecompEntropy: ElogqZ_vec = self.E_logqZ(LP) SS.setELBOTerm('ElogqZ', ElogqZ_vec, dims=('K')) if doPrecompMergeEntropy: # Hmerge : KxK matrix of entropies for all possible pair-wise merges # for example, if we had only 3 components {0,1,2} # Hmerge = [ 0 H(0,1) H(0,2) # 0 0 H(1,2) # 0 0 0 ] # where H(i,j) is entropy if components i and j merged. Hmerge = np.zeros((self.K, self.K)) for jj in range(self.K): compIDs = np.arange(jj+1, self.K) Rcombo = LP['resp'][:,jj][:,np.newaxis] + LP['resp'][:,compIDs] Hmerge[jj,compIDs] = np.sum(Rcombo*np.log(Rcombo+EPS), axis=0) SS.setMergeTerm('ElogqZ', Hmerge, dims=('K','K')) return SS
def calcSummaryStats(Data, SS, LP, **kwargs): ''' Calculate summary statistics for given dataset and local parameters Returns -------- SS : SuffStatBag object, with K components. ''' X = Data.X D = Data.dim if 'resp' in LP: resp = LP['resp'] K = resp.shape[1] # Compute expected outer-product statistic S_xxT = np.zeros((K, Data.dim, Data.dim)) sqrtResp_k = np.sqrt(resp[:, 0]) sqrtRX_k = sqrtResp_k[:, np.newaxis] * Data.X S_xxT[0] = dotATA(sqrtRX_k) for k in xrange(1, K): np.sqrt(resp[:, k], out=sqrtResp_k) np.multiply(sqrtResp_k[:, np.newaxis], Data.X, out=sqrtRX_k) S_xxT[k] = dotATA(sqrtRX_k) sqrtResp = np.sqrt(resp) xxT = np.zeros((K, D, D)) for k in xrange(K): xxT[k] = dotATA(sqrtResp[:, k][:, np.newaxis] * Data.X) assert np.allclose(xxT, S_xxT) else: spR = LP['spR'] K = spR.shape[1] # Compute expected outer-product statistic S_xxT = calcSpRXXT(X=X, spR_csr=spR) if SS is None: SS = SuffStatBag(K=K, D=D) # Expected outer-product for each state k SS.setField('xxT', S_xxT, dims=('K', 'D', 'D')) # Expected count for each k # Usually computed by allocmodel. But sometimes not (eg TopicModel) if not hasattr(SS, 'N'): if 'resp' in LP: SS.setField('N', LP['resp'].sum(axis=0), dims='K') else: SS.setField('N', as1D(toCArray(LP['spR'].sum(axis=0))), dims='K') return SS
def get_global_suff_stats(self, Data, LP, doPrecompEntropy=None, **kwargs): ''' Create sufficient stats needed for global param updates Args ------- Data : bnpy data object LP : Dictionary containing the local parameters. Expected to contain: resp : Data.nObs x K array respPair : Data.nObs x K x K array (from the def. of respPair, note respPair[0,:,:] is undefined) Returns ------- SS : SuffStatBag with fields StartStateCount : A vector of length K with entry i being resp(z_{1k}) = resp[0,:] TransStateCount : A K x K matrix where TransStateCount[i,j] = sum_{n=2}^K respPair(z_{n-1,j}, z_{nk}) N : A vector of length K with entry k being sum_{n=1}^Data.nobs resp(z_{nk}) The first two of these are used by FiniteHMM.update_global_params, and the third is used by ObsModel.update_global_params. (see the documentation for information about resp and respPair) ''' resp = LP['resp'] respPair = LP['respPair'] K = resp.shape[1] startLocIDs = Data.doc_range[:-1] StartStateCount = np.sum(resp[startLocIDs], axis=0) N = np.sum(resp, axis=0) TransStateCount = np.sum(respPair, axis=0) SS = SuffStatBag(K=K, D=Data.dim) SS.setField('StartStateCount', StartStateCount, dims=('K')) SS.setField('TransStateCount', TransStateCount, dims=('K', 'K')) SS.setField('N', N, dims=('K')) if doPrecompEntropy is not None: entropy = self.elbo_entropy(Data, LP) SS.setELBOTerm('Elogqz', entropy, dims=None) return SS
def get_global_suff_stats(self, Data, LP, doPrecompEntropy=0, **kwargs): ''' Compute sufficient stats for provided dataset and local params Returns ------- SS : SuffStatBag Updated fields * NodeStateCount : 2D array, nNodes x K * N : 2D array, size K x K ''' K = LP['resp'].shape[-1] V = Data.nNodes SS = SuffStatBag(K=K, D=Data.dim, V=V) # NodeStateCount_src[i,k] # Num edges assigned to topic k associated with node i as source srcResp = LP['resp'].sum(axis=2) NodeStateCount_src = Data.getSparseSrcNodeMat() * srcResp # Equivalent but slower: for loop # NodeStateCount_src = np.zeros((Data.nNodes, K)) # for i in xrange(Data.nNodes): # mask_i = Data.edges[:,0] == i # NodeStateCount_src[i,:] = srcResp[mask_i].sum(axis=0) # NodeStateCount_rcv[i,k] # Num edges assigned to topic k associated with node i as receiver rcvResp = LP['resp'].sum(axis=1) NodeStateCount_rcv = Data.getSparseRcvNodeMat() * rcvResp # Summing src counts and rcv counts gives the total SS.setField('NodeStateCount', NodeStateCount_src + NodeStateCount_rcv, dims=('V', 'K')) # Compute total atoms assigned to each cluster pair Nresp = np.sum(LP['resp'], axis=0) SS.setField('N', Nresp, dims=('K', 'K')) if doPrecompEntropy: # Remember, resp has shape nEdges x K x K # So, need to sum so we track scalar entropy, not K x K Hresp = calcLentropyAsScalar(LP) SS.setELBOTerm('Hresp', Hresp, dims=None) return SS
def calcSummaryStats(Data, SS, LP, DataAtomType='doc', **kwargs): ''' Calculate summary statistics for given dataset and local parameters Returns -------- SS : SuffStatBag object, with K components. ''' if 'resp' in LP: K = LP['resp'].shape[1] else: K = LP['spR'].shape[1] nnzPerRow = LP['nnzPerRow'] if SS is None: SS = SuffStatBag(K=K, D=Data.vocab_size) if DataAtomType == 'doc': # X : 2D sparse matrix, size nDoc x vocab_size X = Data.getSparseDocTypeCountMatrix() # WordCounts : 2D array, size K x vocab_size # obtained by sparse matrix multiply # here, '*' operator does this because X is sparse matrix type Nvec = None if 'resp' in LP: WordCounts = LP['resp'].T * X if not hasattr(SS, 'N'): Nvec = LP['resp'].sum(axis=0) else: WordCounts = (LP['spR'].T * X).toarray() if not hasattr(SS, 'N'): Nvec = as1D(toCArray(LP['spR'].sum(axis=0))) if Nvec is not None: SS.setField('N', Nvec, dims=('K')) else: # 2D sparse matrix, size V x N X = Data.getSparseTokenTypeCountMatrix() if 'resp' in LP: WordCounts = (X * LP['resp']).T # matrix-matrix product else: WordCounts = (X * LP['spR']).T.toarray() SS.setField('WordCounts', WordCounts, dims=('K', 'D')) SS.setField('SumWordCounts', np.sum(WordCounts, axis=1), dims=('K')) return SS """
def calcSummaryStatsForContigBlock(self, Data, SS=None, a=0, b=0): ''' Calculate sufficient stats for a single contiguous block of data ''' D = Data.X.shape[1] E = Data.Xprev.shape[1] if SS is None: SS = SuffStatBag(K=1, D=D, E=E) elif not hasattr(SS, 'E'): SS._Fields.E = E ppT = dotATA(Data.Xprev[a:b])[np.newaxis, :, :] xxT = dotATA(Data.X[a:b])[np.newaxis, :, :] pxT = dotATB(Data.Xprev[a:b], Data.X[a:b])[np.newaxis, :, :] SS.setField('N', (b - a) * np.ones(1), dims='K') SS.setField('xxT', xxT, dims=('K', 'D', 'D')) SS.setField('ppT', ppT, dims=('K', 'E', 'E')) SS.setField('pxT', pxT, dims=('K', 'E', 'D')) return SS
def calcSummaryStatsForContigBlock(self, Data, SS=None, a=None, b=None, **kwargs): ''' Calculate summary statistics for specific block of dataset Returns -------- SS : SuffStatBag object, with K components. ''' SS = SuffStatBag(K=1, D=Data.dim) # Expected count SS.setField('N', (b - a) * np.ones(1, dtype=np.float64), dims='K') # Expected outer-product xxT = dotATA(Data.X[a:b])[np.newaxis, :, :] SS.setField('xxT', xxT, dims=('K', 'D', 'D')) return SS
def calcSummaryStats(Data, SS, LP, **kwargs): ''' Calculate sufficient statistics for local params at data slice. Returns ------- SS ''' X = Data.X Xprev = Data.Xprev resp = LP['resp'] K = resp.shape[1] D = Data.X.shape[1] E = Data.Xprev.shape[1] if SS is None: SS = SuffStatBag(K=K, D=D, E=E) elif not hasattr(SS, 'E'): SS._Fields.E = E # Expected count for each k # Usually computed by allocmodel. But just in case... if not hasattr(SS, 'N'): SS.setField('N', np.sum(resp, axis=0), dims='K') # Expected outer products sqrtResp = np.sqrt(resp) xxT = np.empty((K, D, D)) ppT = np.empty((K, E, E)) pxT = np.empty((K, E, D)) for k in xrange(K): sqrtResp_k = sqrtResp[:, k][:, np.newaxis] xxT[k] = dotATA(sqrtResp_k * Data.X) ppT[k] = dotATA(sqrtResp_k * Data.Xprev) pxT[k] = np.dot(Data.Xprev.T, resp[:, k][:, np.newaxis] * Data.X) SS.setField('xxT', xxT, dims=('K', 'D', 'D')) SS.setField('ppT', ppT, dims=('K', 'E', 'E')) SS.setField('pxT', pxT, dims=('K', 'E', 'D')) return SS
def get_global_suff_stats(self, Data, LP, doPrecompEntropy=False, doPrecompMergeEntropy=False, mPairIDs=None): ''' Theta is a global parameter here so we need to get its sufficient stats Sufficient statistics for these require precomputing certain terms ''' E, K = LP['E_logsoftev_EdgeLik'].shape # Turn dim checking off, since some stats have dim K+1 instead of K N = Data.nNodeTotal SS = SuffStatBag(K=K, D=N) # Summary statistics node_ss = np.zeros((N, K)) node_z_ss = np.zeros((N, K)) node_offset = np.zeros((E, K)) # used to cache ELBO ev = LP['edge_variational'] edgeEps = LP['E_logsoftev_EdgeEps'] for e in xrange(E): ii = Data.edges[e, 0] jj = Data.edges[e, 1] node_ss[ii, :] += ev[e] node_ss[jj, :] += ev[e] node_z_ss[ii, :] += LP['E_logsoftev_EdgeEps'][ e] # need to check this if there's a better way SS.setField('nNodeTotal', N, dims=None) SS.setField('nEdgeTotal', E, dims=None) SS.setField('node_ss', node_ss, dims=('D', 'K')) SS.setField('node_z_ss', node_z_ss, dims=('D', 'K')) SS.setField('sumLogPiActive', LP['E_logPiSumK'][:self.K], dims='K') SS.setField('sumLogPiUnused', LP['E_logPiSumK'][-1], dims=None) return SS
def get_global_suff_stats(self, Data, LP, doPrecompEntropy=False, doPrecompMergeEntropy=False, mPairIDs=None): ''' Count expected number of times each topic is used across all docs ''' K = LP['DocTopicCount'].shape[1] SS = SuffStatBag(K=K, D=Data.vocab_size) SS.setField('nDoc', Data.nDoc, dims=None) sumLogPi = np.sum(LP['E_logPi'], axis=0) SS.setField('sumLogPiActive', sumLogPi[:K], dims='K') SS.setField('sumLogPiUnused', sumLogPi[-1], dims=None) if doPrecompEntropy: # ---------------- Z terms SS.setELBOTerm('ElogpZ', self.E_logpZ(Data, LP), dims='K') logFactData, logFactZ = self.E_logfactorialZ(Data, LP) SS.setELBOTerm('logFactData', logFactData, dims=None) SS.setELBOTerm('logFactZ', logFactZ, dims='K') # ---------------- Pi terms # Note: no terms needed for ElogpPI # SS already has field sumLogPi, which is sufficient for this term ElogqPiC, ElogqPiA, ElogqPiU = self.E_logqPi_Memoized_from_LP(LP) SS.setELBOTerm('ElogqPiConst', ElogqPiC, dims=None) SS.setELBOTerm('ElogqPiActive', ElogqPiA, dims='K') SS.setELBOTerm('ElogqPiUnused', ElogqPiU, dims=None) if doPrecompMergeEntropy: ElogpZMat, sLgPiMat, ElogqPiMat = self.memo_elbo_terms_for_merge(LP) SS.setMergeTerm('ElogpZ', ElogpZMat, dims=('K','K')) SS.setMergeTerm('ElogqPiActive', ElogqPiMat, dims=('K','K')) SS.setMergeTerm('sumLogPiActive', sLgPiMat, dims=('K','K')) SS.setMergeTerm('logFactZ', self.memo_factorial_term_for_merge(LP, mPairIDs), dims=('K', 'K')) return SS