def testAccountForType(self): result = iP.firstIndexHash(self.entrySet, self.increasingList) self.assertEqual(result, { 100: 1, 200: 3, 300: 6, 400: 7, 1000: 7, 2000: 10, 10000: 11 })
def unSerialize(self, pickleFile): #TODO: refactor to name "deSerialize" #Load in results stored in serialized pickle format #auxiliary function def N_KLDiv_Pts(arrayOfResultsForThisEta,p_eta): N_KLDiv_Pts = [tuple([alist['N'], alist['gamma'], alist['beta']]) for alist in arrayOfResultsForThisEta] N_KLDiv_Pts = np.array( N_KLDiv_Pts, dtype=[('N', '<i8'), ('KL_Div','<f8'), ('beta', '<f8')]) #import ipdb; ipdb.set_trace() for row in N_KLDiv_Pts: row['KL_Div'] = p_eta.KLDivergenceOfP_gammaFromDist(row['KL_Div']) N_KLDiv_Pts.sort(order=['N','KL_Div']) return N_KLDiv_Pts #main body: if not self.eta: raise Exception("Eta must be defined prior to unserializing results of beta computation") probabilityDistFactoryUniformMarginals = pdf.probabilityDistributionFactory(self.k, self.l) p_eta = probabilityDistFactoryUniformMarginals.get_p_eta(self.eta) listOfResults= pickle.load(open(pickleFile,'rb')) print pickleFile listOfResultsAsTuples = [tuple(alist) for alist in listOfResults] arrayOfResults = np.array(listOfResultsAsTuples, dtype=[('time', '<f8'),('eta', '<f8'), ('N', '<i8'), ('gamma', '<f8'), ('beta', '<f8')]) arrayOfResultsForThisEtaGammaLessThanEta = arrayOfResults[ arrayOfResults['eta'] == self.eta] #and arrayOfResults['gamma'] < self.eta] arrayOfResultsForThisEtaGammaLessThanEta = arrayOfResultsForThisEtaGammaLessThanEta[arrayOfResultsForThisEtaGammaLessThanEta['gamma'] < self.eta] arrayOfResultsForThisEtaGammaAtLeastEta = arrayOfResults[ arrayOfResults['eta'] == self.eta] arrayOfResultsForThisEtaGammaAtLeastEta = arrayOfResultsForThisEtaGammaAtLeastEta[ arrayOfResultsForThisEtaGammaAtLeastEta ['gamma'] >= self.eta ] self.betasByAscendingNAscendingGamma = np.array(listOfResultsAsTuples, dtype=[('time', '<f8'),('eta', '<f8'), ('N', '<i8'), ('gamma', '<f8'), ('beta', '<f8')]) self.betasByAscendingNAscendingGamma.sort(order = ['N', 'gamma']) self.N_KLDivPtsForInterpolationGammaLessThanEta = N_KLDiv_Pts(arrayOfResultsForThisEtaGammaLessThanEta,p_eta) self.N_KLDivPtsForInterpolationGammaAtLeastEta = N_KLDiv_Pts(arrayOfResultsForThisEtaGammaAtLeastEta,p_eta) #import ipdb; ipdb.set_trace() N_KLDivPtsForInterpolation_nd_GammaLessThanEta = self.N_KLDivPtsForInterpolationGammaLessThanEta[['N', 'KL_Div']].view(np.ndarray).reshape(len(self.N_KLDivPtsForInterpolationGammaLessThanEta), -1) self.points_GammaLessThanEta = np.array([list(arow[0].view(np.ndarray)) for arow in N_KLDivPtsForInterpolation_nd_GammaLessThanEta]) N_KLDivPtsForInterpolation_nd_GammaAtLeastEta = self.N_KLDivPtsForInterpolationGammaAtLeastEta[['N', 'KL_Div']].view(np.ndarray).reshape(len(self.N_KLDivPtsForInterpolationGammaAtLeastEta), -1) self.points_GammaAtLeastEta = np.array([list(arow[0].view(np.ndarray)) for arow in N_KLDivPtsForInterpolation_nd_GammaAtLeastEta]) self.valuesForInterpolationGammaLessthanEta = [np.log(row['beta']) for row in self.N_KLDivPtsForInterpolationGammaLessThanEta] self.valuesForInterpolationGammaAtLeastEta = [np.log(row['beta']) for row in self.N_KLDivPtsForInterpolationGammaAtLeastEta] self.largestN = self.betasByAscendingNAscendingGamma['N'][-1] # if N is greater than the last (largest) N in the N_KLDivPtsForInterpolation['N'], compute value of function # on this largest N and the given gamma, then second largest N and given gamma, and extrapolate from those two. self.nextToLargestN, self.nextToLargestNLastIndex = self.nextLargestNAndLastIndex(self.largestN) self.largestNFirstIndex = self.nextToLargestNLastIndex thirdLargestN,thirdLargestNLastIndex = self.nextLargestNAndLastIndex(self.nextToLargestN, self.nextToLargestNLastIndex) self.nextToLargestNFirstIndex = thirdLargestNLastIndex self.NList = sorted(set(self.betasByAscendingNAscendingGamma['N'])) self.NFirstIndexHash = iP.firstIndexHash(set(self.NList),list(self.betasByAscendingNAscendingGamma['N']))
def testAccountForType(self): result = iP.firstIndexHash(self.entrySet,self.increasingList) self.assertEqual(result,{100: 1, 200: 3, 300: 6, 400: 7, 1000: 7, 2000: 10, 10000: 11})
def unSerialize(self, pickleFile): #TODO: refactor to name "deSerialize" #Load in results stored in serialized pickle format #auxiliary function def N_KLDiv_Pts(arrayOfResultsForThisEta, p_eta): N_KLDiv_Pts = [ tuple([alist['N'], alist['gamma'], alist['beta']]) for alist in arrayOfResultsForThisEta ] N_KLDiv_Pts = np.array(N_KLDiv_Pts, dtype=[('N', '<i8'), ('KL_Div', '<f8'), ('beta', '<f8')]) #import ipdb; ipdb.set_trace() for row in N_KLDiv_Pts: row['KL_Div'] = p_eta.KLDivergenceOfP_gammaFromDist( row['KL_Div']) N_KLDiv_Pts.sort(order=['N', 'KL_Div']) return N_KLDiv_Pts #main body: if not self.eta: raise Exception( "Eta must be defined prior to unserializing results of beta computation" ) probabilityDistFactoryUniformMarginals = pdf.probabilityDistributionFactory( self.k, self.l) p_eta = probabilityDistFactoryUniformMarginals.get_p_eta(self.eta) listOfResults = pickle.load(open(pickleFile, 'rb')) print pickleFile listOfResultsAsTuples = [tuple(alist) for alist in listOfResults] arrayOfResults = np.array(listOfResultsAsTuples, dtype=[('time', '<f8'), ('eta', '<f8'), ('N', '<i8'), ('gamma', '<f8'), ('beta', '<f8')]) arrayOfResultsForThisEtaGammaLessThanEta = arrayOfResults[ arrayOfResults['eta'] == self.eta] #and arrayOfResults['gamma'] < self.eta] arrayOfResultsForThisEtaGammaLessThanEta = arrayOfResultsForThisEtaGammaLessThanEta[ arrayOfResultsForThisEtaGammaLessThanEta['gamma'] < self.eta] arrayOfResultsForThisEtaGammaAtLeastEta = arrayOfResults[ arrayOfResults['eta'] == self.eta] arrayOfResultsForThisEtaGammaAtLeastEta = arrayOfResultsForThisEtaGammaAtLeastEta[ arrayOfResultsForThisEtaGammaAtLeastEta['gamma'] >= self.eta] self.betasByAscendingNAscendingGamma = np.array(listOfResultsAsTuples, dtype=[ ('time', '<f8'), ('eta', '<f8'), ('N', '<i8'), ('gamma', '<f8'), ('beta', '<f8') ]) self.betasByAscendingNAscendingGamma.sort(order=['N', 'gamma']) self.N_KLDivPtsForInterpolationGammaLessThanEta = N_KLDiv_Pts( arrayOfResultsForThisEtaGammaLessThanEta, p_eta) self.N_KLDivPtsForInterpolationGammaAtLeastEta = N_KLDiv_Pts( arrayOfResultsForThisEtaGammaAtLeastEta, p_eta) #import ipdb; ipdb.set_trace() N_KLDivPtsForInterpolation_nd_GammaLessThanEta = self.N_KLDivPtsForInterpolationGammaLessThanEta[ ['N', 'KL_Div']].view(np.ndarray).reshape( len(self.N_KLDivPtsForInterpolationGammaLessThanEta), -1) self.points_GammaLessThanEta = np.array([ list(arow[0].view(np.ndarray)) for arow in N_KLDivPtsForInterpolation_nd_GammaLessThanEta ]) N_KLDivPtsForInterpolation_nd_GammaAtLeastEta = self.N_KLDivPtsForInterpolationGammaAtLeastEta[ ['N', 'KL_Div']].view(np.ndarray).reshape( len(self.N_KLDivPtsForInterpolationGammaAtLeastEta), -1) self.points_GammaAtLeastEta = np.array([ list(arow[0].view(np.ndarray)) for arow in N_KLDivPtsForInterpolation_nd_GammaAtLeastEta ]) self.valuesForInterpolationGammaLessthanEta = [ np.log(row['beta']) for row in self.N_KLDivPtsForInterpolationGammaLessThanEta ] self.valuesForInterpolationGammaAtLeastEta = [ np.log(row['beta']) for row in self.N_KLDivPtsForInterpolationGammaAtLeastEta ] self.largestN = self.betasByAscendingNAscendingGamma['N'][-1] # if N is greater than the last (largest) N in the N_KLDivPtsForInterpolation['N'], compute value of function # on this largest N and the given gamma, then second largest N and given gamma, and extrapolate from those two. self.nextToLargestN, self.nextToLargestNLastIndex = self.nextLargestNAndLastIndex( self.largestN) self.largestNFirstIndex = self.nextToLargestNLastIndex thirdLargestN, thirdLargestNLastIndex = self.nextLargestNAndLastIndex( self.nextToLargestN, self.nextToLargestNLastIndex) self.nextToLargestNFirstIndex = thirdLargestNLastIndex self.NList = sorted(set(self.betasByAscendingNAscendingGamma['N'])) self.NFirstIndexHash = iP.firstIndexHash( set(self.NList), list(self.betasByAscendingNAscendingGamma['N']))