def mode_importance(weights): """ Detects separate modes in the posterior using clustering of the top 99% of the posterior. weights is an array of entries [u, x, L, logwidth] Returns: A list of modes with their median, standard deviation, and local evidence. """ # strip lowest 1% of posterior weights = sorted(weights, key=getitem(2)) u, x, L, lw = weights[-1] # use highest as normalization to avoid small numbers offset = L + lw total = sum([exp(L + lw - offset) for u, x, L, lw in weights]) logtotal = log(total) + offset bottom_sum = 0 high_parts = [] for i, (u, x, L, lw) in enumerate(weights): bottom_sum += exp(L + lw) if bottom_sum > total * 0.01: high_parts.append(i) # perform clustering on the top # in prior space or in transformed space? # transformed space is probably pretty weird pos = [u for u, x, L, lw in weights[high_parts]] distances = scipy.spatial.distance.cdist(pos, pos) cluster = scipy.cluster.hierarchy.single(distances) # the idea here is that the inter-cluster distances must be much smaller than the cluster-distances # small cluster distances multiplied by 10 will remain small, if there is a well-separated cluster # if no cluster, then this will be ~clusterdists.max()/3, and no splitting will be done threshold = scipy.stats.mstats.mquantiles(clusterdists, 0.1)*10 + clusterdists.max()/3 assigned = clusterdetect.cut_cluster(cluster, distances, threshold) # now we have clusters with some members clusterids = sorted(set(assigned)) results = [] for i in clusterids: inside = assigned == i inweights = weights[high_parts][inside] membersu = [x for u, x, L, lw in inweights] membersx = [u for u, x, L, lw in inweights] membersw = [L + lw for u, x, L, lw in inweights] probs = exp(numpy.array(membersw) - offset) # compute weighted mean # compute weighted standard deviation umean, ustdev = _weighted_avg_and_std(values=membersu, weights=probs) xmean, xstdev = _weighted_avg_and_std(values=membersx, weights=probs) # compute evidence local_evidence = log(sum(probs)) + offset relative_evidence = exp(log(sum(probs)) - log(total)) results.append(dict( members = [membersu, membersx, membersw], mean = xmean, stdev = xstdev, untransformed_mean = umean, untransformed_stdev = ustdev, relative_posterior_probability = relative_evidence, local_evidence = local_evidence, )) return results
def draw_constrained(self, Lmin, priortransform, loglikelihood, previous, ndim, **kwargs): # previous is [[u, x, L], ...] previousL = numpy.array([L for _, _, L in previous]) previousu = numpy.array([u for u, _, _ in previous]) assert previousu.shape[1] == ndim, previousu.shape self.iter += 1 rebuild = self.iter % 50 == 1 if rebuild: high = previousL > Lmin u = previousu[high] L = previousL[high] # detect clusters using hierarchical clustering assert len(u.shape) == 2, u.shape distances = scipy.spatial.distance.cdist(u, u) cluster = scipy.cluster.hierarchy.single(distances) n = len(distances) clusterdists = cluster[:, 2] threshold = scipy.stats.mstats.mquantiles( clusterdists, 0.1) * 20 + clusterdists.max() / 2 assigned = clusterdetect.cut_cluster(cluster, distances, threshold) # now we have clusters with some members # find some rough boundaries # make sure to make them so that they enclose all the points clusterids = sorted(set(assigned)) rects = [] for i in clusterids: inside = assigned == i ulow = u[inside].min(axis=0) uhigh = u[inside].max(axis=0) width = uhigh - ulow # expand, to avoid over-shrinkage ulow -= width * 0.2 uhigh += width * 0.2 j = L[inside].argmax() ustart = u[inside][j] Lstart = L[inside][j] rects.append((i, (ulow, uhigh, (log(uhigh - ulow)).sum()))) #print 'adding new rectangle:', (i, (ulow, uhigh)) rects = dict(rects) # now that we got a little more familiar with out clusters, # we want to sample from them # for this, we want to create boundaries between high and -high # we will do a multi-stage SVM, for every cluster rectid = numpy.zeros(len(previous), dtype=int) - 1 rectid[high] = assigned #try: #if True: #if high.mean() >= 0.9: # print 'not worth it yet to apply svm' clf, svmtransform = None, None if high.mean() < 0.9 and self.iter % 200 == 1: clf, svmtransform = svm_classify(previousu, rectid) #except ValueError as e: # clf, svmtransform = None, None # print 'WARNING: SVM step failed: ', e self.clf = clf self.svmtransform = svmtransform self.rects = rects if len(self.rects) == 1: def get_rect_id(x): return 0 else: x = range(len(self.rects)) y = numpy.array([self.rects[i][2] for i in x]) minlogsize = y.min() y -= y.min() y = exp(y) totalsize = y.sum() y /= y.sum() y = [0] + y.cumsum().tolist() + [1] x = [x[0]] + list(x) + [x[-1]] get_rect_id = scipy.interpolate.interp1d(y, x, kind='zero') ntoaccept = 0 while True: # sample from rectangles, and through against SVM dice = numpy.random.random() i = int(get_rect_id(dice)) ulow, uhigh, logsize = self.rects[i] u = numpy.random.uniform(ulow, uhigh, size=ndim) if len(self.rects) != 1: # count in how many rectangles it is nrect = sum([ exp(logsize - minlogsize) for ulow, uhigh, logsize in self.rects.values() if ((u >= ulow).all() and (u <= uhigh).all()) ]) # reject proportionally ~ 1. / nrect coin = numpy.random.uniform(0, 1) accept = coin < exp(logsize) / nrect if not accept: continue # if survives (classified to be in high region) # then evaluate if self.clf is not None: prob = self.clf.predict_proba(self.svmtransform(u))[0][0] #print 'svm evaluation:', u, prob # we allow 1 false positive classified if prob > 1 - 1. / len(previous) and ntoaccept % 100 != 95: continue x = priortransform(u) L = loglikelihood(x) ntoaccept += 1 if L >= Lmin: # yay, we win if ntoaccept > 5: print '%d samples before accept' % ntoaccept, u, x, L return u, x, L, ntoaccept
def draw_constrained(self, Lmin, priortransform, loglikelihood, previous, ndim, **kwargs): # previous is [[u, x, L], ...] previousL = numpy.array([L for _, _, L in previous]) previousu = numpy.array([u for u, _, _ in previous]) assert previousu.shape[1] == ndim, previousu.shape self.iter += 1 rebuild = self.iter % 50 == 1 if rebuild: high = previousL > Lmin u = previousu[high] L = previousL[high] # detect clusters using hierarchical clustering assert len(u.shape) == 2, u.shape distances = scipy.spatial.distance.cdist(u, u) cluster = scipy.cluster.hierarchy.single(distances) n = len(distances) clusterdists = cluster[:, 2] threshold = scipy.stats.mstats.mquantiles( clusterdists, 0.1) * 20 + clusterdists.max() / 2 assigned = clusterdetect.cut_cluster(cluster, distances, threshold) # now we have clusters with some members # find some rough boundaries # make sure to make them so that they enclose all the points clusterids = sorted(set(assigned)) rects = [] for i in clusterids: inside = assigned == i ulow = u[inside].min(axis=0) uhigh = u[inside].max(axis=0) j = L[inside].argmax() ustart = u[inside][j] Lstart = L[inside][j] assert len(ulow) == ndim assert len(uhigh) == ndim assert len(ustart) == ndim # find maximum in each cluster isinside = lambda ui: (ui >= ulow).all() and (ui <= uhigh).all( ) assert isinside(ustart) clustermaxima = [[mu, mL] for mu, mL in self.maxima if isinside(mu)] if len(clustermaxima) == 0: print 'optimizing in cluster', i, ulow, uhigh def minfunc(ui): if not isinside(ui): return 1e300 return -loglikelihood(priortransform(ui)) ubest = self.optimizer(minfunc, ustart) assert len(ubest) == ndim #ulow = numpy.min([ulow, ubest], axis=0) #uhigh = numpy.max([uhigh, ubest], axis=0) Lbest = loglikelihood(priortransform(ubest)) print 'new best:', ubest, Lbest if self.sampler: self.sampler.Lmax = max(self.sampler.Lmax, Lbest) self.maxima.append([ubest, Lbest]) else: if len(clustermaxima) > 1: print 'WARNING: multiple maxima fitted already', clustermaxima ubest, Lbest = clustermaxima[0] rects.append((i, (ulow, uhigh, ubest, Lbest))) print 'adding new rectangle:', (i, (ulow, uhigh, ubest, Lbest)) rects = dict(rects) # now that we got a little more familiar with out clusters, # we want to sample from them # for this, we want to create boundaries between high and -high # we will do a multi-stage SVM, for every cluster rectid = numpy.zeros(len(previous), dtype=int) - 1 rectid[high] = assigned try: if high.mean() >= 0.9: raise ValueError('not worth it yet') clf, svmtransform = svm_classify(previousu, rectid) except ValueError as e: clf, svmtransform = None, None print 'WARNING: SVM step failed: ', e self.clf = clf self.svmtransform = svmtransform self.rects = rects ntoaccept = 0 while True: # sample from rectangles, and through against SVM i = numpy.random.randint(0, len(self.rects)) ulow, uhigh, ubest, Lbest = self.rects[i] assert len(ulow) == ndim assert len(uhigh) == ndim u = numpy.random.uniform(ulow, uhigh, size=ndim) assert len(u) == ndim # count in how many rectangles it is nrect = sum([((u >= ulow).all() and (u <= uhigh).all()) for ulow, uhigh, ubest, Lbest in self.rects.values()]) # reject proportionally if nrect > 1 and numpy.random.uniform(0, 1) > 1. / nrect: continue # if survives (classified to be in high region) # then evaluate if self.clf is not None: prob = self.clf.predict_proba(self.svmtransform(u))[0][0] #print 'svm evaluation:', u, prob if prob > 1 - 1. / len(previous) and ntoaccept % 100 != 95: continue x = priortransform(u) L = loglikelihood(x) ntoaccept += 1 if L > Lmin: # yay, we win #print '%d samples before accept' % ntoaccept, u, x, L return u, x, L, ntoaccept
def mode_importance(weights): """ Detects separate modes in the posterior using clustering of the top 99% of the posterior. weights is an array of entries [u, x, L, logwidth] Returns: A list of modes with their median, standard deviation, and local evidence. """ # strip lowest 1% of posterior weights = sorted(weights, key=getitem(2)) u, x, L, lw = weights[-1] # use highest as normalization to avoid small numbers offset = L + lw total = sum([exp(L + lw - offset) for u, x, L, lw in weights]) logtotal = log(total) + offset bottom_sum = 0 high_parts = [] for i, (u, x, L, lw) in enumerate(weights): bottom_sum += exp(L + lw) if bottom_sum > total * 0.01: high_parts.append(i) # perform clustering on the top # in prior space or in transformed space? # transformed space is probably pretty weird pos = [u for u, x, L, lw in weights[high_parts]] distances = scipy.spatial.distance.cdist(pos, pos) cluster = scipy.cluster.hierarchy.single(distances) # the idea here is that the inter-cluster distances must be much smaller than the cluster-distances # small cluster distances multiplied by 10 will remain small, if there is a well-separated cluster # if no cluster, then this will be ~clusterdists.max()/3, and no splitting will be done threshold = scipy.stats.mstats.mquantiles( clusterdists, 0.1) * 10 + clusterdists.max() / 3 assigned = clusterdetect.cut_cluster(cluster, distances, threshold) # now we have clusters with some members clusterids = sorted(set(assigned)) results = [] for i in clusterids: inside = assigned == i inweights = weights[high_parts][inside] membersu = [x for u, x, L, lw in inweights] membersx = [u for u, x, L, lw in inweights] membersw = [L + lw for u, x, L, lw in inweights] probs = exp(numpy.array(membersw) - offset) # compute weighted mean # compute weighted standard deviation umean, ustdev = _weighted_avg_and_std(values=membersu, weights=probs) xmean, xstdev = _weighted_avg_and_std(values=membersx, weights=probs) # compute evidence local_evidence = log(sum(probs)) + offset relative_evidence = exp(log(sum(probs)) - log(total)) results.append( dict( members=[membersu, membersx, membersw], mean=xmean, stdev=xstdev, untransformed_mean=umean, untransformed_stdev=ustdev, relative_posterior_probability=relative_evidence, local_evidence=local_evidence, )) return results
def draw_constrained(self, Lmin, priortransform, loglikelihood, previous, ndim, **kwargs): # previous is [[u, x, L], ...] previousL = numpy.array([L for _, _, L in previous]) previousu = numpy.array([u for u, _, _ in previous]) assert previousu.shape[1] == ndim, previousu.shape self.iter += 1 rebuild = self.iter % 50 == 1 if rebuild: high = previousL > Lmin u = previousu[high] L = previousL[high] # detect clusters using hierarchical clustering assert len(u.shape) == 2, u.shape distances = scipy.spatial.distance.cdist(u, u) cluster = scipy.cluster.hierarchy.single(distances) n = len(distances) clusterdists = cluster[:,2] threshold = scipy.stats.mstats.mquantiles(clusterdists, 0.1)*20 + clusterdists.max()/2 assigned = clusterdetect.cut_cluster(cluster, distances, threshold) # now we have clusters with some members # find some rough boundaries # make sure to make them so that they enclose all the points clusterids = sorted(set(assigned)) rects = [] for i in clusterids: inside = assigned == i ulow = u[inside].min(axis=0) uhigh = u[inside].max(axis=0) width = uhigh - ulow # expand, to avoid over-shrinkage ulow -= width*0.2 uhigh += width*0.2 j = L[inside].argmax() ustart = u[inside][j] Lstart = L[inside][j] rects.append((i, (ulow, uhigh, (log(uhigh - ulow)).sum()))) #print 'adding new rectangle:', (i, (ulow, uhigh)) rects = dict(rects) # now that we got a little more familiar with out clusters, # we want to sample from them # for this, we want to create boundaries between high and -high # we will do a multi-stage SVM, for every cluster rectid = numpy.zeros(len(previous), dtype=int) - 1 rectid[high] = assigned #try: #if True: #if high.mean() >= 0.9: # print 'not worth it yet to apply svm' clf, svmtransform = None, None if high.mean() < 0.9 and self.iter % 200 == 1: clf, svmtransform = svm_classify(previousu, rectid) #except ValueError as e: # clf, svmtransform = None, None # print 'WARNING: SVM step failed: ', e self.clf = clf self.svmtransform = svmtransform self.rects = rects if len(self.rects) == 1: def get_rect_id(x): return 0 else: x = range(len(self.rects)) y = numpy.array([self.rects[i][2] for i in x]) minlogsize = y.min() y -= y.min() y = exp(y) totalsize = y.sum() y /= y.sum() y = [0] + y.cumsum().tolist() + [1] x = [x[0]] + list(x) + [x[-1]] get_rect_id = scipy.interpolate.interp1d(y, x, kind='zero') ntoaccept = 0 while True: # sample from rectangles, and through against SVM dice = numpy.random.random() i = int(get_rect_id(dice)) ulow, uhigh, logsize = self.rects[i] u = numpy.random.uniform(ulow, uhigh, size=ndim) if len(self.rects) != 1: # count in how many rectangles it is nrect = sum([exp(logsize - minlogsize) for ulow, uhigh, logsize in self.rects.values() if ((u >= ulow).all() and (u <= uhigh).all())]) # reject proportionally ~ 1. / nrect coin = numpy.random.uniform(0, 1) accept = coin < exp(logsize) / nrect if not accept: continue # if survives (classified to be in high region) # then evaluate if self.clf is not None: prob = self.clf.predict_proba(self.svmtransform(u))[0][0] #print 'svm evaluation:', u, prob # we allow 1 false positive classified if prob > 1 - 1. / len(previous) and ntoaccept % 100 != 95: continue x = priortransform(u) L = loglikelihood(x) ntoaccept += 1 if L >= Lmin: # yay, we win if ntoaccept > 5: print '%d samples before accept' % ntoaccept, u, x, L return u, x, L, ntoaccept
def draw_constrained(self, Lmin, priortransform, loglikelihood, previous, ndim, **kwargs): # previous is [[u, x, L], ...] previousL = numpy.array([L for _, _, L in previous]) previousu = numpy.array([u for u, _, _ in previous]) assert previousu.shape[1] == ndim, previousu.shape self.iter += 1 rebuild = self.iter % 50 == 1 if rebuild: high = previousL > Lmin u = previousu[high] L = previousL[high] # detect clusters using hierarchical clustering assert len(u.shape) == 2, u.shape distances = scipy.spatial.distance.cdist(u, u) cluster = scipy.cluster.hierarchy.single(distances) n = len(distances) clusterdists = cluster[:,2] threshold = scipy.stats.mstats.mquantiles(clusterdists, 0.1)*20 + clusterdists.max()/2 assigned = clusterdetect.cut_cluster(cluster, distances, threshold) # now we have clusters with some members # find some rough boundaries # make sure to make them so that they enclose all the points clusterids = sorted(set(assigned)) rects = [] for i in clusterids: inside = assigned == i ulow = u[inside].min(axis=0) uhigh = u[inside].max(axis=0) j = L[inside].argmax() ustart = u[inside][j] Lstart = L[inside][j] assert len(ulow) == ndim assert len(uhigh) == ndim assert len(ustart) == ndim # find maximum in each cluster isinside = lambda ui: (ui >= ulow).all() and (ui <= uhigh).all() assert isinside(ustart) clustermaxima = [[mu, mL] for mu, mL in self.maxima if isinside(mu)] if len(clustermaxima) == 0: print('optimizing in cluster', i, ulow, uhigh) def minfunc(ui): if not isinside(ui): return 1e300 return -loglikelihood(priortransform(ui)) ubest = self.optimizer(minfunc, ustart) assert len(ubest) == ndim #ulow = numpy.min([ulow, ubest], axis=0) #uhigh = numpy.max([uhigh, ubest], axis=0) Lbest = loglikelihood(priortransform(ubest)) print('new best:', ubest, Lbest) if self.sampler: self.sampler.Lmax = max(self.sampler.Lmax, Lbest) self.maxima.append([ubest, Lbest]) else: if len(clustermaxima) > 1: print('WARNING: multiple maxima fitted already', clustermaxima) ubest, Lbest = clustermaxima[0] rects.append((i, (ulow, uhigh, ubest, Lbest))) print('adding new rectangle:', (i, (ulow, uhigh, ubest, Lbest))) rects = dict(rects) # now that we got a little more familiar with out clusters, # we want to sample from them # for this, we want to create boundaries between high and -high # we will do a multi-stage SVM, for every cluster rectid = numpy.zeros(len(previous), dtype=int) - 1 rectid[high] = assigned try: if high.mean() >= 0.9: raise ValueError('not worth it yet') clf, svmtransform = svm_classify(previousu, rectid) except ValueError as e: clf, svmtransform = None, None print('WARNING: SVM step failed: ', e) self.clf = clf self.svmtransform = svmtransform self.rects = rects ntoaccept = 0 while True: # sample from rectangles, and through against SVM i = numpy.random.randint(0, len(self.rects)) ulow, uhigh, ubest, Lbest = self.rects[i] assert len(ulow) == ndim assert len(uhigh) == ndim u = numpy.random.uniform(ulow, uhigh, size=ndim) assert len(u) == ndim # count in how many rectangles it is nrect = sum([((u >= ulow).all() and (u <= uhigh).all()) for ulow, uhigh, ubest, Lbest in self.rects.values()]) # reject proportionally if nrect > 1 and numpy.random.uniform(0, 1) > 1./nrect: continue # if survives (classified to be in high region) # then evaluate if self.clf is not None: prob = self.clf.predict_proba(self.svmtransform(u))[0][0] #print 'svm evaluation:', u, prob if prob > 1 - 1./len(previous) and ntoaccept % 100 != 95: continue x = priortransform(u) L = loglikelihood(x) ntoaccept += 1 if L > Lmin: # yay, we win #print '%d samples before accept' % ntoaccept, u, x, L return u, x, L, ntoaccept