def __init__(self, num_args, vocab_size, ont_size, hyp_hidden_size, wc_hidden_sizes, cc_hidden_sizes, word_dim=50, concept_dim=50, word_rep_param=False, hyp_model_type="weighted_prod", wc_pref_model_type="tanhlayer", cc_pref_model_type="tanhlayer", rec_model_type="gaussian", init_hyp_strengths=None, relaxed=False, no_hyp=False, wc_lr_wp_rank=10, cc_lr_wp_rank=10):
print >>sys.stderr, "Initializing SPADE"
print >>sys.stderr, "num_args: %d"%(num_args)
print >>sys.stderr, "vocab_size: %d"%(vocab_size)
print >>sys.stderr, "ont_size: %d"%(ont_size)
print >>sys.stderr, "word_dim: %d"%(word_dim)
print >>sys.stderr, "concept_dim: %d"%(concept_dim)
print >>sys.stderr, "word_rep_param: %s"%(word_rep_param)
if no_hyp:
print >>sys.stderr, "Running without hypernymy links"
else:
print >>sys.stderr, "hyp_model: %s"%(hyp_model_type)
print >>sys.stderr, "wc_pref_model: %s"%(wc_pref_model_type)
if wc_pref_model_type == "lr_weighted_prod":
print >>sys.stderr, "wc_lr_wp_rank: %d"%(wc_lr_wp_rank)
if relaxed:
print >>sys.stderr, "Running without inter-concept preferences"
else:
print >>sys.stderr, "cc_pref_model: %s"%(cc_pref_model_type)
if cc_pref_model_type == "lr_weighted_prod":
print >>sys.stderr, "cc_lr_wp_rank: %d"%(cc_lr_wp_rank)
print >>sys.stderr, "rec_model: %s"%rec_model_type
numpy_rng = numpy.random.RandomState(12345)
self.theano_rng = RandomStreams(12345)
self.ont_size = ont_size
vocab_rep_range = 4 * numpy.sqrt(6. / (vocab_size + word_dim))
init_vocab_rep = numpy.asarray(numpy_rng.uniform(low = -vocab_rep_range, high = vocab_rep_range, size=(vocab_size, word_dim)) )
ont_rep_range = 4 * numpy.sqrt(6. / (ont_size + concept_dim))
init_ont_rep = numpy.asarray(numpy_rng.uniform(low = -ont_rep_range, high = ont_rep_range, size=(ont_size, concept_dim)) )
self.vocab_rep = theano.shared(value=init_vocab_rep, name='vocab_rep')
self.ont_rep = theano.shared(value=init_ont_rep, name='ont_rep')
self.repr_params = [self.vocab_rep] if word_rep_param else []
self.repr_params.append(self.ont_rep)
self.enc_params = []
self.relaxed = relaxed
self.no_hyp = no_hyp
if not self.no_hyp:
self.hyp_model = HypernymModel(hyp_model_type, hyp_hidden_size, self.vocab_rep, self.ont_rep)
self.enc_params.extend(self.hyp_model.get_params())
self.wc_pref_models = []
self.cc_pref_models = []
self.num_slots = num_args + 1 # +1 for the predicate
self.num_args = num_args
self.wc_pref_models = [{} for _ in range(self.num_slots)]
for i in range(self.num_slots):
for j in range(self.num_slots):
if i == j:
continue
wc_pref_model = PreferenceModel('word_concept', wc_pref_model_type, wc_hidden_sizes[i], self.ont_rep, "wc_%d_%d"%(i, j), self.vocab_rep, lr_wp_rank=wc_lr_wp_rank)
self.wc_pref_models[i][j] = wc_pref_model
self.enc_params.extend(wc_pref_model.get_params())
if not self.relaxed:
for i in range(num_args):
cc_pref_model = PreferenceModel('concept_concept', cc_pref_model_type, cc_hidden_sizes[i], self.ont_rep, "cc_%d"%i, lr_wp_rank=cc_lr_wp_rank)
self.cc_pref_models.append(cc_pref_model)
self.enc_params.extend(cc_pref_model.get_params())
self.rec_model = ReconstructionModel(self.ont_rep, self.vocab_rep, init_hyp_strengths=init_hyp_strengths, rec_model_type=rec_model_type)
self.rec_params = self.rec_model.get_params()
# Random y, sampled from uniform(|ont|^num_slots)
self.y_r = T.cast(self.theano_rng.uniform(low=0, high=self.ont_size-1, size=(self.num_slots,)), 'int32')
self.num_enc_ns = 1
self.num_label_ns = 1
class EventAE(object):
def __init__(self, num_args, vocab_size, ont_size, hyp_hidden_size, wc_hidden_sizes, cc_hidden_sizes, word_dim=50, concept_dim=50, word_rep_param=False, hyp_model_type="weighted_prod", wc_pref_model_type="tanhlayer", cc_pref_model_type="tanhlayer", rec_model_type="gaussian", init_hyp_strengths=None, relaxed=False, no_hyp=False, wc_lr_wp_rank=10, cc_lr_wp_rank=10):
print >>sys.stderr, "Initializing SPADE"
print >>sys.stderr, "num_args: %d"%(num_args)
print >>sys.stderr, "vocab_size: %d"%(vocab_size)
print >>sys.stderr, "ont_size: %d"%(ont_size)
print >>sys.stderr, "word_dim: %d"%(word_dim)
print >>sys.stderr, "concept_dim: %d"%(concept_dim)
print >>sys.stderr, "word_rep_param: %s"%(word_rep_param)
if no_hyp:
print >>sys.stderr, "Running without hypernymy links"
else:
print >>sys.stderr, "hyp_model: %s"%(hyp_model_type)
print >>sys.stderr, "wc_pref_model: %s"%(wc_pref_model_type)
if wc_pref_model_type == "lr_weighted_prod":
print >>sys.stderr, "wc_lr_wp_rank: %d"%(wc_lr_wp_rank)
if relaxed:
print >>sys.stderr, "Running without inter-concept preferences"
else:
print >>sys.stderr, "cc_pref_model: %s"%(cc_pref_model_type)
if cc_pref_model_type == "lr_weighted_prod":
print >>sys.stderr, "cc_lr_wp_rank: %d"%(cc_lr_wp_rank)
print >>sys.stderr, "rec_model: %s"%rec_model_type
numpy_rng = numpy.random.RandomState(12345)
self.theano_rng = RandomStreams(12345)
self.ont_size = ont_size
vocab_rep_range = 4 * numpy.sqrt(6. / (vocab_size + word_dim))
init_vocab_rep = numpy.asarray(numpy_rng.uniform(low = -vocab_rep_range, high = vocab_rep_range, size=(vocab_size, word_dim)) )
ont_rep_range = 4 * numpy.sqrt(6. / (ont_size + concept_dim))
init_ont_rep = numpy.asarray(numpy_rng.uniform(low = -ont_rep_range, high = ont_rep_range, size=(ont_size, concept_dim)) )
self.vocab_rep = theano.shared(value=init_vocab_rep, name='vocab_rep')
self.ont_rep = theano.shared(value=init_ont_rep, name='ont_rep')
self.repr_params = [self.vocab_rep] if word_rep_param else []
self.repr_params.append(self.ont_rep)
self.enc_params = []
self.relaxed = relaxed
self.no_hyp = no_hyp
if not self.no_hyp:
self.hyp_model = HypernymModel(hyp_model_type, hyp_hidden_size, self.vocab_rep, self.ont_rep)
self.enc_params.extend(self.hyp_model.get_params())
self.wc_pref_models = []
self.cc_pref_models = []
self.num_slots = num_args + 1 # +1 for the predicate
self.num_args = num_args
self.wc_pref_models = [{} for _ in range(self.num_slots)]
for i in range(self.num_slots):
for j in range(self.num_slots):
if i == j:
continue
wc_pref_model = PreferenceModel('word_concept', wc_pref_model_type, wc_hidden_sizes[i], self.ont_rep, "wc_%d_%d"%(i, j), self.vocab_rep, lr_wp_rank=wc_lr_wp_rank)
self.wc_pref_models[i][j] = wc_pref_model
self.enc_params.extend(wc_pref_model.get_params())
if not self.relaxed:
for i in range(num_args):
cc_pref_model = PreferenceModel('concept_concept', cc_pref_model_type, cc_hidden_sizes[i], self.ont_rep, "cc_%d"%i, lr_wp_rank=cc_lr_wp_rank)
self.cc_pref_models.append(cc_pref_model)
self.enc_params.extend(cc_pref_model.get_params())
self.rec_model = ReconstructionModel(self.ont_rep, self.vocab_rep, init_hyp_strengths=init_hyp_strengths, rec_model_type=rec_model_type)
self.rec_params = self.rec_model.get_params()
# Random y, sampled from uniform(|ont|^num_slots)
self.y_r = T.cast(self.theano_rng.uniform(low=0, high=self.ont_size-1, size=(self.num_slots,)), 'int32')
self.num_enc_ns = 1
self.num_label_ns = 1
### Direct prob functions ###
def get_sym_encoder_energy(self, x, y):
# Works with NCE
hsum = T.constant(0)
if not self.no_hyp:
for i in range(self.num_slots):
hsum += self.hyp_model.get_symb_score(x[i], y[i])
p_w_c_sum = T.constant(0)
for i in range(self.num_slots):
for j in range(self.num_slots):
if i == j:
continue
p_w_c_sum += self.wc_pref_models[i][j].get_symb_score(x[i], y[j])
p_c_c_sum = T.constant(0)
for i in range(self.num_args):
p_c_c_sum += self.cc_pref_models[i].get_symb_score(y[0], y[i + 1])
return hsum + p_w_c_sum + p_c_c_sum
def get_sym_encoder_partition(self, x, y_s):
partial_sums, _ = theano.scan(fn=lambda y, interm_sum, x_0: interm_sum + T.exp(self.get_sym_encoder_energy(x_0, y)), outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=x)
encoder_partition = partial_sums[-1]
return encoder_partition
def get_sym_rec_prob(self, x, y):
# Works with NCE
init_prob = T.constant(1.0, dtype='float64')
partial_prods, _ = theano.scan(fn = lambda x_i, y_i, interm_prod: interm_prod * self.rec_model.get_sym_rec_prob(x_i, y_i), outputs_info=init_prob, sequences=[x, y])
rec_prob = partial_prods[-1]
return rec_prob
def get_sym_posterior_num(self, x, y):
# Needed for NCE
enc_energy = self.get_sym_encoder_energy(x, y)
rec_prob = self.get_sym_rec_prob(x, y)
return T.exp(enc_energy) * rec_prob
def get_sym_posterior_partition(self, x, y_s):
partial_sums, _ = theano.scan(fn=lambda y, interm_sum, x_0: interm_sum + self.get_sym_posterior_num(x_0, y), outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=x)
posterior_partition = partial_sums[-1]
return posterior_partition
def get_sym_direct_prob(self, x, y_s):
def get_post_num_sum(y_0, interm_sum, x_0):
posterior_num = self.get_sym_posterior_num(x_0, y_0)
return interm_sum + posterior_num
res, _ = theano.scan(fn=get_post_num_sum, outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=[x])
direct_prob = res[-1] / self.get_sym_encoder_partition(x, y_s)
return direct_prob
# Following function is useless.
def get_sym_posterior(self, x, y, y_s):
return self.get_sym_posterior_num(x, y) / self.get_sym_posterior_partition(x, y_s)
### Complete expectation functions ###
def get_sym_complete_expectation(self, x, y_s):
encoder_partition = self.get_sym_encoder_partition(x, y_s)
posterior_partition = self.get_sym_posterior_partition(x, y_s)
def prod_fun(y_0, interm_sum, x_0):
post_num = self.get_sym_posterior_num(x_0, y_0)
fixed_post_num = ifelse(T.le(post_num, SMALL_NUM), T.constant(0.0, dtype='float64'), post_num)
return interm_sum + ifelse(T.le(fixed_post_num, SMALL_NUM), T.constant(0.0, dtype='float64'), fixed_post_num * T.log(fixed_post_num))
partial_sums, _ = theano.scan(fn=prod_fun, outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=x)
data_term = ifelse(T.eq(posterior_partition, T.constant(0.0, dtype='float64')), T.constant(0.0, dtype='float64'), partial_sums[-1] / posterior_partition)
#data_term = partial_sums[-1]
complete_expectation = data_term - T.log(encoder_partition)
#complete_expectation = data_term
return complete_expectation
### NCE functions ###
def get_sym_rand_y(self, y_s):
# NCE function
# Sample randomly from y|x
rand_ind = T.cast(self.theano_rng.uniform(low=0, high=y_s.shape[0]-1, size=(1,)), 'int32')
sample = y_s[rand_ind[0]]
return sample
def get_sym_nc_encoder_prob(self, x, y, y_s, num_noise_samples=None):
# NCE function
if num_noise_samples is None:
num_noise_samples = self.num_enc_ns
enc_energy = T.exp(self.get_sym_encoder_energy(x, y))
ns_prob = num_noise_samples * ((1. / self.ont_size) ** self.num_slots)
true_prob = enc_energy / (enc_energy + ns_prob)
noise_prob = T.constant(1.0, dtype='float64')
for _ in range(num_noise_samples):
# Noise distribution is not conditioned on x. So we sample directly from ont, not from y_s
ns_enc_energy = T.exp(self.get_sym_encoder_energy(x, self.y_r))
#ns_enc_energy = T.exp(self.get_sym_encoder_energy(x, self.get_sym_rand_y(y_s)))
noise_prob *= ns_prob / (ns_enc_energy + ns_prob)
return true_prob * noise_prob
def get_sym_nc_posterior(self, x, y, y_s, num_noise_samples=None):
# NCE function
# p(\hat{x}, y | x)
if num_noise_samples is None:
num_noise_samples = self.num_enc_ns
return self.get_sym_nc_encoder_prob(x, y, y_s, num_noise_samples=num_noise_samples) * self.get_sym_rec_prob(x, y)
def get_sym_nc_direct_prob(self, x, y_s):
# NCE function
def get_prob(y_0, interm_sum, x_0, Y):
posterior = self.get_sym_nc_posterior(x_0, y_0, Y)
return interm_sum + posterior
res, _ = theano.scan(fn=get_prob, outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=[x, y_s])
direct_prob = res[-1]
return direct_prob
def get_sym_nc_label_prob(self, x, y, y_s, num_noise_samples=None):
# NCE function
# p(y | x, \hat{x})
if num_noise_samples is None:
num_noise_samples = self.num_label_ns
true_posterior = self.get_sym_nc_posterior(x, y)
#TODO: Can make this more efficient
noise_posterior = self.get_sym_nc_posterior(x, self.get_sym_rand_y(y_s), num_noise_samples=1)
ns_prob = num_noise_samples * T.pow(1. / y_s.shape[0], self.num_slots)
true_prob = true_posterior / (true_posterior + ns_prob)
noise_prob = T.constant(1.0, dtype='float64')
for _ in range(num_noise_samples):
#noise_posterior = self.get_sym_nc_posterior(x, self.get_sym_rand_y(y_s))
noise_prob *= ns_prob / (noise_posterior + ns_prob)
return true_prob * noise_prob
def get_sym_nc_complete_expectation(self, x, y_s):
# NCE function
def get_expectation(y_0, interm_sum, x_0, Y):
label_prob = self.get_sym_nc_label_prob(x_0, y_0, Y)
posterior = self.get_sym_nc_posterior(x_0, y_0)
log_posterior = ifelse(T.le(posterior, SMALL_NUM), T.constant(LOG_SMALL_NUM, dtype='float64'), T.log(posterior))
return interm_sum + (label_prob * log_posterior)
res, _ = theano.scan(fn=get_expectation, outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=[x, y_s])
complete_expectation = res[-1]
return complete_expectation
def get_train_func(self, learning_rate, nce=True, em=False):
print >>sys.stderr, "Trainining type: EM = %s, NCE = %s"%(em, nce)
# TODO: Implement AdaGrad
x, y_s = T.ivector("x"), T.imatrix("y_s")
if em:
cost = -self.get_sym_nc_complete_expectation(x, y_s) if nce else -self.get_sym_complete_expectation(x, y_s)
else:
cost = -T.log(self.get_sym_nc_direct_prob(x, y_s)) if nce else -T.log(self.get_sym_direct_prob(x, y_s))
params = self.repr_params + self.enc_params + self.rec_params
g_params = T.grad(cost, params)
# Updating the parameters only if the norm of the gradient is less than 100.
# Important: This check also takes care of any element in the gradients being nan. The conditional returns False even in that case.
updates=[ (p, ifelse(T.le(T.nlinalg.norm(g, None), T.constant(100.0, dtype='float64')), p - learning_rate * g, p)) for p, g in zip(params, g_params) ]
train_func = theano.function([x, y_s], cost, updates=updates)
return train_func
def get_posterior_func(self):
# Works with NCE
x, y = T.ivectors('x', 'y')
posterior_func = theano.function([x, y], self.get_sym_posterior_num(x, y))
return posterior_func
def get_rec_prob_func(self):
# Works with NCE
x, y = T.ivectors('x', 'y')
rec_prob_func = theano.function([x, y], self.get_sym_rec_prob(x, y))
return rec_prob_func
### Relaxed variant functions ###
def get_sym_relaxed_encoder_energy(self, x, y, s):
h = self.hyp_model.get_symb_score(x[s], y) if not self.no_hyp else T.constant(0.0)
p_sum = T.constant(0.0)
# We need to sum up the preference scores of words in all slots except s with y
for i in range(self.num_slots):
if i == s:
continue
p_sum += self.wc_pref_models[i][s].get_symb_score(x[i], y)
return h + p_sum
def get_sym_relaxed_encoder_partition(self, x, y_s, s):
partial_sums, _ = theano.scan(fn=lambda y, interm_sum, x_0: interm_sum + T.exp(self.get_sym_relaxed_encoder_energy(x_0, y, s)), outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=[x])
encoder_partition = partial_sums[-1]
return encoder_partition
def get_sym_relaxed_posterior_num(self, x, y, s):
# Needed for NCE
enc_energy = self.get_sym_relaxed_encoder_energy(x, y, s)
rec_prob = self.rec_model.get_sym_rec_prob(x[s], y)
return T.exp(enc_energy) * rec_prob
def get_sym_relaxed_posterior_partition(self, x, y_s, s):
partial_sums, _ = theano.scan(fn=lambda y, interm_sum, x_0, s: interm_sum + self.get_sym_relaxed_posterior_num(x_0, y, s), outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=[x,s])
posterior_partition = partial_sums[-1]
return posterior_partition
def get_sym_relaxed_direct_prob(self, x, y_s, s):
def get_post_num_sum(y_0, interm_sum, x_0):
posterior_num = self.get_sym_relaxed_posterior_num(x_0, y_0, s)
return interm_sum + posterior_num
res, _ = theano.scan(fn=get_post_num_sum, outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=[x])
direct_prob = res[-1] / self.get_sym_relaxed_encoder_partition(x, y_s, s)
return direct_prob
def get_relaxed_train_func(self, learning_rate, s):
# TODO: Implement AdaGrad
# TODO: This means we need one train function per slot. Do we?
x, y_s = T.ivector("x"), T.ivector("y_s")
dp = self.get_sym_relaxed_direct_prob(x, y_s, s)
cost = -T.log(dp)
relaxed_enc_params = []
if not self.no_hyp:
relaxed_enc_params.extend(self.hyp_model.get_params())
for i in range(self.num_slots):
if i == s:
continue
relaxed_enc_params.extend(self.wc_pref_models[i][s].get_params())
params = self.repr_params + relaxed_enc_params + self.rec_params
g_params = T.grad(cost, params)
# Updating the parameters only if the norm of the gradient is less than 100.
# Important: This check also takes care of any element in the gradients being nan. The conditional returns False even in that case.
updates=[ (p, ifelse(T.le(T.nlinalg.norm(g, None), T.constant(100.0, dtype='float64')), p - learning_rate * g, p)) for p, g in zip(params, g_params) ]
train_func = theano.function([x, y_s], cost, updates=updates)
return train_func
def get_relaxed_posterior_func(self, s):
# Works with NCE
x = T.ivector('x')
y = T.iscalar('y')
posterior_func = theano.function([x, y], self.get_sym_relaxed_posterior_num(x, y, s))
return posterior_func
def set_repr_params(self, repr_param_vals):
for i, param_val in enumerate(repr_param_vals):
self.repr_params[i].set_value(param_val)
def set_rec_params(self, rec_param_vals):
for i, param_val in enumerate(rec_param_vals):
self.rec_params[i].set_value(param_val)