def sample(self, p, size=None, split=False):
if size is None:
size = p.shape
start = 0
x = []
for o, v in self.outs.iteritems():
dim = self.layers[o]['dim']
f_sample = v['f_sample']
p_ = _slice2(p, start, start + dim)
if self.layers[o]['act'] == 'lambda x: x':
scale = 2
else:
scale = 1
if size is None:
size_ = None
else:
if p.ndim == 1:
size_ = (size[0], p_.shape[0] // scale)
elif p.ndim == 2:
size_ = (size[0], p_.shape[0], p_.shape[1] // scale)
elif p.ndim == 3:
size_ = (size[0], p_.shape[0], p_.shape[1],
p_.shape[2] // scale)
else:
raise ValueError()
x.append(f_sample(self.trng, p_, size=size_))
start += dim
if split:
return x
else:
return concatenate(x, axis=(x[0].ndim - 1))
def e_step(self, y, qs, *params):
prior = concatenate(params[:1], axis=0)
consider_constant = [y, prior]
cost = T.constant(0.).astype(floatX)
for l in xrange(self.n_layers):
q = qs[l]
mu_q = _slice(q, 0, self.dim_h)
log_sigma_q = _slice(q, 1, self.dim_h)
kl_term = self.kl_divergence(q, prior).mean(axis=0)
epsilon = self.trng.normal(avg=0,
std=1.0,
size=(self.n_inference_samples,
mu_q.shape[0], mu_q.shape[1]))
h = mu_q + epsilon * T.exp(log_sigma_q)
p = self.p_y_given_h(h, *params)
if l == 0:
cond_term = self.conditional.neg_log_prob(y[None, :, :],
p).mean(axis=0)
else:
cond_term = self.kl_divergence(q[l - 1][None, :, :], p)
cost += (kl_term + cond_term).sum(axis=0)
grads = theano.grad(cost, wrt=qs, consider_constant=consider_constant)
cost = kl_term.mean()
return cost, grads
def m_step(self, p_hs, y, qs, n_samples=10):
constants = []
prior_energy = T.constant(0.).astype(floatX)
y_energy = T.constant(0.).astype(floatX)
h_energy = T.constant(0.).astype(floatX)
mu = self.mu[None, None, :]
log_sigma = self.log_sigma[None, None, :]
p_y = concatenate([mu, log_sigma], axis=2)
for l in xrange(self.n_layers - 1, -1, -1):
q = qs[l]
p_h = p_hs[l]
h_energy += self.kl_divergence(q, p_h).mean()
prior_energy += self.kl_divergence(q[None, :, :], p_y).mean()
if n_samples == 0:
h = mu[None, :, :]
else:
h = self.posteriors[l].sample(p=q,
size=(n_samples, q.shape[0],
q.shape[1] / 2))
p_y = self.conditional(h)
if l == 0:
y_energy += self.conditionals[l].neg_log_prob(
y[None, :, :], p_y).mean()
else:
y_energy += self.kl_divergence(q[l - 1][None, :, :],
p_y).mean()
return (prior_energy, h_energy, y_energy), constants
def neg_log_prob(self, x, p=None):
if p is None:
p = self.get_prob(*self.get_params())
mu = _slice(p, 0, p.shape[p.ndim - 1] // 2)
log_sigma = _slice(p, 1, p.shape[p.ndim - 1] // 2)
mu = T.clip(mu, self.min, self.max)
p = concatenate([mu, log_sigma], axis=mu.ndim - 1)
return self.f_neg_log_prob(x, p)
def determiner_check_case_by_case(determiner, word, limit_len=100):
_, determiner_word2sentence = determiner_check(word, limit_len=limit_len)
determiner_word_usages = []
key_ = tools.concatenate([determiner, word])
if key_ in determiner_word2sentence:
determiner_word_usages = determiner_word2sentence[key_]
return determiner_word_usages
def preposition_check_case_by_case(word, prep):
_, word_prep2sentence = preposition_check(word)
word_prep_usages = []
key_ = tools.concatenate([word, prep])
if key_ in word_prep2sentence:
word_prep_usages = word_prep2sentence[key_]
return word_prep_usages
def step_call(self, x, *params):
x = self.preact(x, *params)
start = 0
y = []
for o in self.outs.keys():
dim = self.layers[o]['dim']
act = self.layers[o]['act']
z_ = _slice2(x, start, start + dim)
x_ = eval(act)(z_)
y.append(x_)
start += dim
return concatenate(y, axis=(y[0].ndim - 1))
def determiner_check(word, limit_len=100):
sentences = global_().load_sentences()
determiner_word2sentence = {}
determiner_word2count = {}
cnt = 0
for sent in sentences:
if cnt >= limit_len:
break
if not word.lower() in sent.lower():
continue
sent = sent.lower()
word = word.lower()
sent = tools.remove_non_words(sent)
sent = tools.remove_special_tokens(sent)
pos_tagged_sent = tools.pos_tagging(sent)
for idx, tuple in enumerate(pos_tagged_sent):
if idx == 0:
continue
token, pos_tag = tuple
previous_token, previous_pos_tag = pos_tagged_sent[idx - 1]
if token == word and tools.is_pos_determiner(previous_pos_tag):
sent = sent.lower()
sent = sent.replace(" " + token.lower() +
" ", " " + token.upper() + " ")
sent = sent.replace(
" " + previous_token.lower() + " ", " " + previous_token.upper() + " ")
cnt += 1
determiner_word2sentence = tools.add_allowing_duplication(determiner_word2sentence,
tools.concatenate(
[previous_token, token]),
sent)
for determiner_word, list_ in determiner_word2sentence.items():
determiner_word2count[determiner_word] = len(list_)
return determiner_word2count, determiner_word2sentence
def preact(self, x, *params):
# Used within scan with `get_params`
params = list(params)
outs = dict(i=x)
for i, o in self.edges:
x = outs[i]
assert not o in outs.keys()
W = params.pop(0)
b = params.pop(0)
if o in self.outs:
x = T.dot(x, W) + b
else:
act = self.layers[o]['act']
x = eval(act)(T.dot(x, W) + b)
outs[o] = x
x = []
for o in self.outs.keys():
x.append(outs[o])
return concatenate(x, axis=(x[0].ndim - 1))
def step_neg_log_prob(self, x, p):
mu = _slice(p, 0, p.shape[p.ndim - 1] // 2)
log_sigma = _slice(p, 1, p.shape[p.ndim - 1] // 2)
mu = T.clip(mu, self.min, self.max)
p = concatenate([mu, log_sigma], axis=mu.ndim - 1)
return self.f_neg_log_prob(x, p)
def __call__(self, p):
mu = _slice(p, 0, p.shape[p.ndim - 1] // 2)
log_sigma = _slice(p, 1, p.shape[p.ndim - 1] // 2)
mu = T.clip(mu, self.min, self.max)
return concatenate([mu, log_sigma], axis=mu.ndim - 1)
def get_prob(self, mu, log_sigma):
return concatenate([mu, log_sigma], axis=mu.ndim - 1)