def evaluate_kernel(self,inds,vals,odomain,params,logspace=False):
sqdist_mat = self.domain.get_sqdist_mat(odomain,curr_inds = None, other_inds = inds)
sigma_scale = params.get('is_gaussian_sigmascale'+self.suffix,params.get('is_gaussian_sigmascale',1.0))
# sigma = sigma_scale*odomain.resolution
sigma = sigma_scale*n.reshape(odomain.get_pt_resolution(inds),(1,-1))
if logspace:
logK = (-0.5/sigma**2)*sqdist_mat
ret = logsumexp(logK + vals.reshape((1,-1)),axis=1)
ret -= logsumexp(ret)
else:
K = n.exp((-0.5/sigma**2)*sqdist_mat)
ret = n.dot(K,vals)
ret /= ret.sum()
return ret
def record_update(self,
dist,
samples,
samplevalues,
samplews,
pinlier,
testImg,
logspace=False):
do_importance_sampling = self.params.get('is_on' + self.suffix, False)
if not do_importance_sampling or dist in self.updates:
return
# IMPORTANT: Must make sure our stored memory here is distinct from the memory passed in with
# samplevalues, otherwise it may get changed out from underneath us!!!
samplevalues = n.require(samplevalues, dtype=n.float64)
if not logspace:
logsamplevalues = n.log(n.abs(samplevalues) / samplews)
else:
logsamplevalues = samplevalues - n.log(samplews)
# Anneal the new values a bit
temp = self.params.get('is_temperature' + self.suffix,
self.params.get('is_temperature', 1.0))
assert n.isfinite(temp) and temp > 0
if temp != 1.0:
logsamplevalues *= (1.0 / temp)
logsamplevalues[n.logical_not(n.isfinite(logsamplevalues))] = -n.inf
lse = logsumexp(logsamplevalues)
assert n.isfinite(lse)
logsamplevalues -= lse
expsamplevalues = n.exp(logsamplevalues)
self.updates[dist] = (samples, logsamplevalues, expsamplevalues,
self.domain, self.params, pinlier, testImg)
# Compute the global update here to allow it to be done in parallel
# NOTE: there is some threading level non-determinism here due to
# numerical precision and the fact that global_upd may get added in
# different orders with different thread timings.
global_prob = self.params.get('is_global_prob' + self.suffix,
self.params.get('is_global_prob', 0))
if not testImg and pinlier > 0 and global_prob > 0:
self.global_upd += pinlier * self.evaluate_kernel(samples,
expsamplevalues,
self.domain,
self.params,
logspace=False)
if self.global_contrib[dist] == 1:
psamples, _, pexpsamplevalues, pdom, pparams, ppinlier, ptestImg = self.prev_vals[
dist]
global_prob = pparams.get('is_global_prob' + self.suffix,
pparams.get('is_global_prob', 0))
if not ptestImg and ppinlier > 0 and global_prob > 0:
self.global_upd -= ppinlier * self.evaluate_kernel(psamples, pexpsamplevalues, \
pdom, pparams, logspace=False)
def score_seq_by_word(possible_word_indices, offset_of_chosen_word,
features_concat, weights):
logprobs = []
for indices, chosen, features in zip(possible_word_indices,
offset_of_chosen_word,
features_concat):
logits = np.dot(features, weights)
chosen_logprob = logits[chosen] - logsumexp(logits)
logprobs.append(chosen_logprob)
return logprobs
def p_ratio(examples_support, examples_test):
examples_support = tuple(sorted(examples_support))
examples_test = tuple(sorted(examples_test))
examples_joint = tuple(sorted(examples_support + examples_test))
if examples_support in prob_cache:
logp_support = prob_cache[examples_support]
else:
proposals_support = getProposals(M['net'],
M['trace'],
examples_support,
modes=("regex", ),
nProposals=args.nProposals)
logp_support = util.logsumexp(
[proposal.final_trace.score for proposal in proposals_support])
prob_cache[examples_support] = logp_support
if examples_test in prob_cache:
logp_test = prob_cache[examples_test]
else:
proposals_test = getProposals(M['net'],
M['trace'],
examples_test,
modes=("regex", ),
nProposals=args.nProposals)
logp_test = util.logsumexp(
[proposal.final_trace.score for proposal in proposals_test])
prob_cache[examples_test] = logp_test
if examples_joint in prob_cache:
logp_joint = prob_cache[examples_joint]
else:
proposals_joint = getProposals(M['net'],
M['trace'],
examples_joint,
modes=("regex", ),
nProposals=args.nProposals)
logp_joint = util.logsumexp(
[proposal.final_trace.score for proposal in proposals_joint])
prob_cache[examples_joint] = logp_joint
score = logp_joint - logp_test - logp_support
return score
def store_results(self, idx, isw,
cphi_R, sampleinfo_R,
cphi_I, sampleinfo_I,
cphi_S, sampleinfo_S, res,
logspace_phis=False):
Idx = self.minibatch['img_idxs'][idx]
testImg = self.minibatch['test_batch']
N_R_sampled = sampleinfo_R[0]
N_I_sampled = sampleinfo_I[0]
N_S_sampled = sampleinfo_S[0]
if not self.factoredRI:
cphi_R = cphi_R.reshape((N_R_sampled, N_I_sampled))
if logspace_phis:
cphi_I = logsumexp(cphi_R, axis=0)
cphi_R = logsumexp(cphi_R, axis=1)
else:
cphi_I = n.sum(cphi_R, axis=0)
cphi_R = n.sum(cphi_R, axis=1)
self.sampler_R.record_update(Idx, sampleinfo_R[1], cphi_R, sampleinfo_R[2], isw, testImg,
logspace=logspace_phis)
self.sampler_I.record_update(Idx, sampleinfo_I[1], cphi_I, sampleinfo_I[2], isw, testImg,
logspace=logspace_phis)
self.sampler_S.record_update(Idx, sampleinfo_S[1], cphi_S, sampleinfo_S[2], isw, testImg,
logspace=logspace_phis)
res['N_R_sampled'][idx] = N_R_sampled
res['N_I_sampled'][idx] = N_I_sampled
res['N_S_sampled'][idx] = N_S_sampled
res['N_Total_sampled'][idx] = N_R_sampled * N_I_sampled * N_S_sampled
res['Evar_prior'][idx] = self.imgpower[idx] / self.N ** 2
if logspace_phis:
res['CV2_R'][idx] = n.exp(-logsumexp(2 * cphi_R, dtype=n.float64))
res['CV2_I'][idx] = n.exp(-logsumexp(2 * cphi_I, dtype=n.float64))
res['CV2_S'][idx] = n.exp(-logsumexp(2 * cphi_S, dtype=n.float64))
else:
res['CV2_R'][idx] = (1.0 / n.sum(cphi_R ** 2, dtype=n.float64))
res['CV2_I'][idx] = (1.0 / n.sum(cphi_I ** 2, dtype=n.float64))
res['CV2_S'][idx] = (1.0 / n.sum(cphi_S ** 2, dtype=n.float64))
def evaluate_kernel(self, inds, vals, odomain, params, logspace=False):
sqdist_mat = self.domain.get_sqdist_mat(odomain,
curr_inds=None,
other_inds=inds)
sigma_scale = params.get('is_gaussian_sigmascale' + self.suffix,
params.get('is_gaussian_sigmascale', 1.0))
# sigma = sigma_scale*odomain.resolution
sigma = sigma_scale * n.reshape(odomain.get_pt_resolution(inds),
(1, -1))
if logspace:
logK = (-0.5 / sigma**2) * sqdist_mat
ret = logsumexp(logK + vals.reshape((1, -1)), axis=1)
ret -= logsumexp(ret)
else:
K = n.exp((-0.5 / sigma**2) * sqdist_mat)
ret = n.dot(K, vals)
ret /= ret.sum()
return ret
def calc_local_params(self, Data, LP, nnzPerRowLP=0, **kwargs):
''' Compute local parameters for each data item and component.
Parameters
-------
Data : bnpy.data.DataObj subclass
LP : dict
Local parameters as key-value string/array pairs
* E_log_soft_ev : 2D array, N x K
E_log_soft_ev[n,k] = log p(data obs n | comp k)
Returns
-------
LP : dict
Local parameters, with updated fields
* resp : 2D array, size N x K array
Posterior responsibility each comp has for each item
resp[n, k] = p(z[n] = k | x[n])
'''
lpr = LP['E_log_soft_ev']
K = lpr.shape[1]
if self.inferType.count('EM') > 0:
# Using point estimates, for EM algorithm
lpr += np.log(self.w)
if nnzPerRowLP and (nnzPerRowLP > 0 and nnzPerRowLP < K):
# SPARSE Assignments
LP['nnzPerRow'] = nnzPerRowLP
LP['spR'] = sparsifyLogResp(lpr, nnzPerRow=nnzPerRowLP)
assert np.all(np.isfinite(LP['spR'].data))
else:
lprPerItem = logsumexp(lpr, axis=1)
lpr -= lprPerItem[:, np.newaxis]
np.exp(lpr, out=lpr)
LP['resp'] = lpr
LP['evidence'] = lprPerItem.sum()
else:
# Full Bayesian approach, for VB or GS algorithms
lpr += self.Elogw
if nnzPerRowLP and (nnzPerRowLP > 0 and nnzPerRowLP < K):
# SPARSE Assignments
LP['nnzPerRow'] = nnzPerRowLP
LP['spR'] = sparsifyLogResp(lpr, nnzPerRow=nnzPerRowLP)
assert np.all(np.isfinite(LP['spR'].data))
else:
# DENSE Assignments
# Calculate exp in numerically safe way,
# in-place so no new allocations occur
NumericUtil.inplaceExpAndNormalizeRows(lpr)
LP['resp'] = lpr
assert np.allclose(lpr.sum(axis=1), 1)
return LP
def store_results(self, idx, isw, \
cphi_R, sampleinfo_R, \
cphi_I, sampleinfo_I, \
cphi_S, sampleinfo_S, res,
logspace_phis = False):
Idx = self.minibatch['img_idxs'][idx]
testImg = self.minibatch['test_batch']
N_R_sampled = sampleinfo_R[0]
N_I_sampled = sampleinfo_I[0]
N_S_sampled = sampleinfo_S[0]
if not self.factoredRI:
cphi_R = cphi_R.reshape((N_R_sampled,N_I_sampled))
if logspace_phis:
cphi_I = logsumexp(cphi_R,axis=0)
cphi_R = logsumexp(cphi_R,axis=1)
else:
cphi_I = np.sum(cphi_R,axis=0)
cphi_R = np.sum(cphi_R,axis=1)
self.sampler_R.record_update(Idx, sampleinfo_R[1], cphi_R, sampleinfo_R[2], isw, testImg, logspace = logspace_phis)
self.sampler_I.record_update(Idx, sampleinfo_I[1], cphi_I, sampleinfo_I[2], isw, testImg, logspace = logspace_phis)
self.sampler_S.record_update(Idx, sampleinfo_S[1], cphi_S, sampleinfo_S[2], isw, testImg, logspace = logspace_phis)
res['N_R_sampled'][idx] = N_R_sampled
res['N_I_sampled'][idx] = N_I_sampled
res['N_S_sampled'][idx] = N_S_sampled
res['N_Total_sampled'][idx] = N_R_sampled*N_I_sampled*N_S_sampled
res['Evar_prior'][idx] = self.imgpower[idx]/self.N**2
if logspace_phis:
res['CV2_R'][idx] = np.exp(-logsumexp(2*cphi_R,dtype=np.float64))
res['CV2_I'][idx] = np.exp(-logsumexp(2*cphi_I,dtype=np.float64))
res['CV2_S'][idx] = np.exp(-logsumexp(2*cphi_S,dtype=np.float64))
else:
res['CV2_R'][idx] = (1.0/np.sum(cphi_R**2,dtype=np.float64))
res['CV2_I'][idx] = (1.0/np.sum(cphi_I**2,dtype=np.float64))
res['CV2_S'][idx] = (1.0/np.sum(cphi_S**2,dtype=np.float64))
def conditionalProbability(
examples_support,
examples_test): #p(examples_test | examples_support)
proposals = getProposals(M['net'], M['trace'], examples_support)
#Normalise scores
total_logprob = util.logsumexp(
[proposal.final_trace.score for proposal in proposals])
for proposal in proposals:
proposal.final_trace.score -= total_logprob
#Observe new examples
new_traces = []
for proposal in proposals:
trace, observations, counterexamples, p_valid = proposal.final_trace.observe_all(
proposal.concept, examples_test)
if trace is not None:
new_traces.append(trace)
#Conditional probability
return util.logsumexp([trace.score for trace in new_traces])
def generate_variable(charset, policy, min_length, max_length, urandom):
"""
min_length, max_length inclusive
"""
assert min_length > 0
assert min_length <= max_length
n = len(charset)
choices = np.arange(min_length, max_length + 1)
scores = choices * np.log(n)
scores -= logsumexp(scores)
probs = np.exp(scores)
assert almost_equals(probs.sum(), 1.0)
fname = '/dev/urandom' if urandom else '/dev/random'
with open(fname) as f:
choice = choices[sample_discrete(probs, total=1.0, r=f)]
return generate0(charset, policy, choice, f)
def record_update(self, dist, samples, samplevalues, samplews, pinlier, testImg, logspace=False):
do_importance_sampling = self.params.get('is_on'+self.suffix, False)
if not do_importance_sampling or dist in self.updates:
return
# IMPORTANT: Must make sure our stored memory here is distinct from the memory passed in with
# samplevalues, otherwise it may get changed out from underneath us!!!
samplevalues = np.require(samplevalues,dtype=np.float64)
if not logspace:
logsamplevalues = np.log(np.abs(samplevalues)/samplews)
else:
logsamplevalues = samplevalues - np.log(samplews)
# Anneal the new values a bit
temp = self.params.get('is_temperature'+self.suffix, self.params.get('is_temperature',1.0))
assert np.isfinite(temp) and temp > 0
if temp != 1.0:
logsamplevalues *= (1.0/temp)
logsamplevalues[np.logical_not(np.isfinite(logsamplevalues))] = -np.inf
lse = logsumexp(logsamplevalues)
assert np.isfinite(lse)
logsamplevalues -= lse
expsamplevalues = np.exp(logsamplevalues)
self.updates[dist] = (samples,logsamplevalues,expsamplevalues,self.domain,self.params,pinlier,testImg)
# Compute the global update here to allow it to be done in parallel
# NOTE: there is some threading level non-determinism here due to
# numerical precision and the fact that global_upd may get added in
# different orders with different thread timings.
global_prob = self.params.get('is_global_prob'+self.suffix,self.params.get('is_global_prob',0))
if not testImg and pinlier > 0 and global_prob > 0:
self.global_upd += pinlier*self.evaluate_kernel(samples,
expsamplevalues,
self.domain,
self.params,
logspace=False)
if self.global_contrib[dist] == 1:
psamples, _, pexpsamplevalues, pdom, pparams, ppinlier, ptestImg = self.prev_vals[dist]
global_prob = pparams.get('is_global_prob'+self.suffix,pparams.get('is_global_prob',0))
if not ptestImg and ppinlier > 0 and global_prob > 0:
self.global_upd -= ppinlier*self.evaluate_kernel(psamples, pexpsamplevalues, \
pdom, pparams, logspace=False)
def _build_entropy(self, weights, means, covars):
# First build half a square matrix of normals. This avoids re-computing symmetric normals.
log_normal_probs = util.init_list(
0.0, [self.num_components, self.num_components])
for i in xrange(self.num_components):
for j in xrange(i, self.num_components):
for k in xrange(self.num_latent):
if self.diag_post:
normal = util.DiagNormal(
means[i, k, :], covars[i, k, :] + covars[j, k, :])
else:
if i == j:
# Compute chol(2S) = sqrt(2)*chol(S).
covars_sum = tf.sqrt(2.0) * covars[i, k, :, :]
else:
# TODO(karl): Can we just stay in cholesky space somehow?
covars_sum = tf.cholesky(
util.mat_square(covars[i, k, :, :]) +
util.mat_square(covars[j, k, :, :]))
normal = util.CholNormal(means[i, k, :], covars_sum)
log_normal_probs[i][j] += normal.log_prob(means[j, k, :])
# Now compute the entropy.
entropy = 0.0
for i in xrange(self.num_components):
weighted_log_probs = util.init_list(0.0, [self.num_components])
for j in xrange(self.num_components):
if i <= j:
weighted_log_probs[j] = tf.log(
weights[j]) + log_normal_probs[i][j]
else:
weighted_log_probs[j] = tf.log(
weights[j]) + log_normal_probs[j][i]
entropy -= weights[i] * util.logsumexp(
tf.stack(weighted_log_probs))
return entropy
def simbased_sample(k, L):
N = L.shape[0]
D = np.real(np.log(1-L))
# Sample a random element
j = np.random.randint(N)
Y = [j]
for i in range(1, k):
# Calculate distances
p = np.sum(D[:, Y], axis=1)
# Normalize
p = np.exp(p - logsumexp(p))
# Sample
p_cdf = np.cumsum(p)
r = np.random.rand()
j = np.argwhere(r <= p_cdf)[0]
Y.append(j)
return np.array(Y)
def _compute_posteriors(self, X):
n_samples, ndim = X.shape
for k in xrange(self.n_components):
Cinv = np.linalg.pinv(self.cov[k])
diff = X - self.mu[k]
# print("Cinv")
# print(Cinv)
# print("diff")
# print(diff)
# print("-" * 20)
# print("cov[{}]".format(self.cov[k]))
# print("Det: {}".format(np.linalg.det(self.cov[k])))
self.posteriors[k] = -0.5 * ndim * np.log(2*pi) - \
0.5 * np.log(np.linalg.det(self.cov[k])) - \
0.5 * np.diag(diff.dot(Cinv).dot(diff.T)) + \
np.log(self.mixprobs[0, k]) # This is log P(x(i), z(i)) i.e., log of joint probability.
# print("posterior[{}]".format(k))
# print(self.posteriors[k])
self.posteriors -= logsumexp(self.posteriors, axis=0) # Compute log P(z(i) | x(i)) = log ( P(x(i), z(i)) / sum_z(i) (P(x(i), z(i))) )
self.posteriors = np.exp(self.posteriors)
return self.posteriors
def _compute_value(self):
X = self.X.value
return np.exp(X - util.logsumexp(X, axis=self.axis))
def _compute_value(self):
X = self.X.value
A = np.reshape(X, (X.shape[0], self.ncolors, X.shape[1]//self.ncolors))
X = A
return np.exp(X - util.logsumexp(X, axis=1)).reshape((self.X.shape))
def softmax(scores):
return np.exp(scores - logsumexp(scores))
def sample(self, dist):
# sample from dist (index) using params (is_*)
# returns sample indices and sample weights (may be smaller than
# computed num_samples if there are overlaps)
# the sample weights are not the inverse probabilities, but rather
# the actual weights you need to multiply the weighted function
# evaluations by.
# So before we computed sum_i w_i f(x_i)
# now we compute sum_j sampleweight_j w_sample_j f(x_sample_j)
# for instance, if the number of samples == domain and the importance
# distribution is uniform then the sampleweights are all 1.
# 1) generate prob dist over domain
# 2) sample from this dist
# 3) compute weights from samples and probs (weight = 1 / prob / numsamples)
# 4) remove dups and multiply weights by number of dups
# 5) return final samples and weights
if dist in self.cached_samples:
# If we've already drawn samples since our last update, use those.
# This ensures that multiple evaluations of the objective uses a
# consistent set of IS samples.
return self.cached_samples[dist][0:2]
params = self.params
global_prob = params.get('is_global_prob'+self.suffix,params.get('is_global_prob',0))
num_samples = params.get('is_num_samples'+self.suffix,params.get('is_num_samples','auto'))
ess_scale = params.get('is_ess_scale'+self.suffix,params.get('is_ess_scale',5.0))
do_importance_sampling = params.get('is_on'+self.suffix, False) and len(self.domain) > 1
if do_importance_sampling:
prior_prob = self.get_image_priorprob(dist)
if prior_prob < 1.0:
# Use some of the prior and some of the previous distribution
logprobs = self.get_image_logdist(dist)
# Mix the current estimate with a prior distribution
# probs *= (1.0 - prior_prob)
logprobs += np.log(1.0-prior_prob)
# probs += (1.0 - global_prob) * prior_prob / float(len(self.domain))
logprobs = np.logaddexp(logprobs,np.log((1.0 - global_prob) * prior_prob / float(len(self.domain))))
else:
# Using only the prior distribution
logprobs = np.empty(len(self.domain))
logprobs[:] = np.log((1.0 - global_prob) / float(len(self.domain)))
if global_prob > 0 and prior_prob > 0:
# Part of the prior prob is the global estimate of directions
# probs += (prior_prob * global_prob) * self.globalvals
logprobs = np.logaddexp(logprobs, np.log(prior_prob * global_prob) + self.get_global_logdist())
# Normalize the distribution
# probs /= probs.sum()
logprobs[np.logical_not(np.isfinite(logprobs))] = -np.inf
lse = logsumexp(logprobs)
if np.isfinite(lse):
logprobs -= lse
else:
print("WARNING: lse for {1} is not finite: {0}".format(lse,self.suffix))
print("logprobs = {0}".format(logprobs))
logprobs[:] = -np.log(len(self.domain))
# Compute effective sample size.
# ess ~=~ the number of significant components in probs.
# Note that large values of prior_prob will cause ess to be large.
# ess = 1.0/n.sum(probs**2)
# logess = -n.log(n.sum(n.exp(2*logprobs))
logess = -logsumexp(2*logprobs)
if not (np.isfinite(logess) and logess > 0):
print("WARNING: logess is not finite and positive: {0}, {1}".format(logess,np.exp(logess)))
ess = 1
else:
ess = np.exp(logess)
# Tune the number of samples based on the effective sample size.
if num_samples == 'auto':
num_samples = min(np.ceil(ess_scale*ess),len(self.domain))
# Only do sampling if it's likely to save us at least 5%
if min(ess,num_samples)/len(self.domain) < (1 - 0.05):
probs = np.exp(logprobs)
samplecount = np.random.multinomial(num_samples,probs)
# samplecount = multinomial_sample_residual(num_samples,probs)
# only evaluate samples with nonzero counts
samples = np.where(samplecount)[0]
weights = samplecount[samples] / (probs[samples] * num_samples)
fullsampled = False
else:
fullsampled = True
if (not do_importance_sampling) or fullsampled:
samples = None
weights = 1.0
fullsampled = True
self.cached_samples[dist] = (samples, weights)
return self.cached_samples[dist][0:2]
def softmax_log(a):
return a - logsumexp(a)
def _compute_value(self):
X = self.X.value
lX = np.log(X + EPSILON)
return np.exp(lX - 0.5*util.logsumexp(2*lX, axis=self.axis))
def online_loglikelihood(self, logl_A_B, logl_B_A):
n = logl_A_B.size(0)
log_alpha, log_1_m_alpha = F.logsigmoid(self.w), F.logsigmoid(-self.w)
return logsumexp(log_alpha + torch.sum(logl_A_B),
log_1_m_alpha + torch.sum(logl_B_A)) # / float(n)
classes = [random.choice(test_data) for _ in range(args.way)]
exampless = [list(np.random.choice(X, size=args.shot)) for X in classes]
if set(classes[0]) == set(exampless[0]):
continue
examples_test = [
random.choice([x for x in classes[0] if x not in exampless[0]])
]
print("Support Sets:")
for examples in exampless:
print(examples)
print("Test:")
print(examples_test[0])
print()
#scores = [conditionalProbability(examples, examples_test) for examples in exampless]
scores = [p_ratio(examples, examples_test) for examples in exampless]
print("Scores:", scores)
confidence = math.exp(max(scores) - util.logsumexp(scores))
if max(scores) == scores[0]:
print("HIT (confidence %2.2f%%)" % (confidence * 100))
hits += 1
else:
print("MISS (confidence %2.2f%%)" % (confidence * 100))
misses += 1
print("Accuracy: %d out of %d (%2.2f%%)" %
(hits, hits + misses, (hits / (hits + misses) * 100)))
def _compute_value(self):
X = self.X.value
return np.exp(X - util.logsumexp(X, axis=self.axis))
def _compute_value(self):
X = self.X.value
A = np.reshape(X, (X.shape[0], self.ncolors, X.shape[1]//self.ncolors))
X = A
return np.exp(X - util.logsumexp(X, axis=1)).reshape((self.X.shape))
networkCache.clear()
for examples in defaultExamples:
proposals = list(
getProposals(M['net'],
M['trace'],
examples,
nProposals=50,
maxNetworkEvals=100,
doPrint=False))
proposals = [
x for x in proposals if not (all(
x.concept.sample(x.trace) == examples[0] for _ in range(100)))
]
totalJoint = util.logsumexp([x.final_trace.score for x in proposals])
probs = [math.exp(x.final_trace.score - totalJoint) for x in proposals]
samples = []
if len(proposals) > 0:
#print(examples)
#for p in sorted(proposals, key=lambda p: p.final_trace.score, reverse=True)[:5]:
# print(p.concept.str(p.trace), p.concept.sample(p.trace))
k = 0
for _ in range(500):
p = proposals[np.random.choice(range(len(proposals)), p=probs)]
s = p.concept.sample(p.trace)
if s not in examples and s not in samples:
samples.append(s)
k += 1
if k == nSamples:
break
def _compute_value(self):
X = self.X.value
lX = np.log(X + EPSILON)
return np.exp(lX - 0.5 * util.logsumexp(2 * lX, axis=self.axis))
def sample(self, dist):
# sample from dist (index) using params (is_*)
# returns sample indices and sample weights (may be smaller than
# computed num_samples if there are overlaps)
# the sample weights are not the inverse probabilities, but rather
# the actual weights you need to multiply the weighted function
# evaluations by.
# So before we computed sum_i w_i f(x_i)
# now we compute sum_j sampleweight_j w_sample_j f(x_sample_j)
# for instance, if the number of samples == domain and the importance
# distribution is uniform then the sampleweights are all 1.
# 1) generate prob dist over domain
# 2) sample from this dist
# 3) compute weights from samples and probs (weight = 1 / prob / numsamples)
# 4) remove dups and multiply weights by number of dups
# 5) return final samples and weights
if dist in self.cached_samples:
# If we've already drawn samples since our last update, use those.
# This ensures that multiple evaluations of the objective uses a
# consistent set of IS samples.
return self.cached_samples[dist][0:2]
params = self.params
global_prob = params.get('is_global_prob' + self.suffix,
params.get('is_global_prob', 0))
num_samples = params.get('is_num_samples' + self.suffix,
params.get('is_num_samples', 'auto'))
ess_scale = params.get('is_ess_scale' + self.suffix,
params.get('is_ess_scale', 5.0))
do_importance_sampling = params.get('is_on' + self.suffix,
False) and len(self.domain) > 1
if do_importance_sampling:
prior_prob = self.get_image_priorprob(dist)
if prior_prob < 1.0:
# Use some of the prior and some of the previous distribution
logprobs = self.get_image_logdist(dist)
# Mix the current estimate with a prior distribution
# probs *= (1.0 - prior_prob)
logprobs += np.log(1.0 - prior_prob)
# probs += (1.0 - global_prob) * prior_prob / float(len(self.domain))
logprobs = np.logaddexp(
logprobs,
np.log((1.0 - global_prob) * prior_prob /
float(len(self.domain))))
else:
# Using only the prior distribution
logprobs = np.empty(len(self.domain))
logprobs[:] = np.log(
(1.0 - global_prob) / float(len(self.domain)))
if global_prob > 0 and prior_prob > 0:
# Part of the prior prob is the global estimate of directions
# probs += (prior_prob * global_prob) * self.globalvals
logprobs = np.logaddexp(
logprobs,
np.log(prior_prob * global_prob) +
self.get_global_logdist())
# Normalize the distribution
# probs /= probs.sum()
logprobs[np.logical_not(np.isfinite(logprobs))] = -np.inf
lse = logsumexp(logprobs)
if np.isfinite(lse):
logprobs -= lse
else:
print("WARNING: lse for {1} is not finite: {0}".format(
lse, self.suffix))
print("logprobs = {0}".format(logprobs))
logprobs[:] = -np.log(len(self.domain))
# Compute effective sample size.
# ess ~=~ the number of significant components in probs.
# Note that large values of prior_prob will cause ess to be large.
# ess = 1.0/n.sum(probs**2)
# logess = -n.log(n.sum(n.exp(2*logprobs))
logess = -logsumexp(2 * logprobs)
if not (np.isfinite(logess) and logess > 0):
print("WARNING: logess is not finite and positive: {0}, {1}".
format(logess, np.exp(logess)))
ess = 1
else:
ess = np.exp(logess)
# Tune the number of samples based on the effective sample size.
if num_samples == 'auto':
num_samples = min(np.ceil(ess_scale * ess), len(self.domain))
# Only do sampling if it's likely to save us at least 5%
if min(ess, num_samples) / len(self.domain) < (1 - 0.05):
probs = np.exp(logprobs)
samplecount = np.random.multinomial(num_samples, probs)
# samplecount = multinomial_sample_residual(num_samples,probs)
# only evaluate samples with nonzero counts
samples = np.where(samplecount)[0]
weights = samplecount[samples] / (probs[samples] * num_samples)
fullsampled = False
else:
fullsampled = True
if (not do_importance_sampling) or fullsampled:
samples = None
weights = 1.0
fullsampled = True
self.cached_samples[dist] = (samples, weights)
return self.cached_samples[dist][0:2]
def online_loglikelihood(self, logl_model_1, logl_model_2):
n = logl_model_1.size(0)
log_alpha, log_1_m_alpha = F.logsigmoid(self.w), F.logsigmoid(-self.w)
return logsumexp(log_alpha + torch.sum(logl_model_1),
log_1_m_alpha + torch.sum(logl_model_2)) # / float(n)
samples = []
for _ in range(1000):
s = proposal.concept.sample(proposal.trace)
if s not in examples and s not in samples:
samples.append(s)
if len(samples) == 3:
break
if len(samples) > 0:
print("\n%5.2f: %s" % (proposal.final_trace.score,
proposal.concept.str(proposal.trace)))
print("; ".join(samples))
j += 1
if j == 4: break
totalJoint = util.logsumexp([x.final_trace.score for x in proposals])
probs = [math.exp(x.final_trace.score - totalJoint) for x in proposals]
posteriorConceptSamples = np.random.choice(range(len(proposals)),
size=3,
p=probs)
print("\nPosterior predictive samples:")
print("; ".join(proposals[i].concept.sample(proposals[i].trace)
for i in posteriorConceptSamples))
elif mode.lower() in ["c", "classification"]:
raise NotImplementedError() #which classification mode? Total correlation?
def getExamples():
examples = []
nextInput = True
while nextInput:
