Example #1
0
  def __init__(self, num_args, vocab_size, ont_size, hyp_hidden_size, wc_hidden_sizes, cc_hidden_sizes, word_dim=50, concept_dim=50, word_rep_param=False, hyp_model_type="weighted_prod", wc_pref_model_type="tanhlayer", cc_pref_model_type="tanhlayer", rec_model_type="gaussian", init_hyp_strengths=None, relaxed=False, no_hyp=False, wc_lr_wp_rank=10, cc_lr_wp_rank=10):
    print >>sys.stderr, "Initializing SPADE"
    print >>sys.stderr, "num_args: %d"%(num_args)
    print >>sys.stderr, "vocab_size: %d"%(vocab_size)
    print >>sys.stderr, "ont_size: %d"%(ont_size)
    print >>sys.stderr, "word_dim: %d"%(word_dim)
    print >>sys.stderr, "concept_dim: %d"%(concept_dim)
    print >>sys.stderr, "word_rep_param: %s"%(word_rep_param)
    if no_hyp:
      print >>sys.stderr, "Running without hypernymy links"
    else:
      print >>sys.stderr, "hyp_model: %s"%(hyp_model_type)
    print >>sys.stderr, "wc_pref_model: %s"%(wc_pref_model_type)
    if wc_pref_model_type == "lr_weighted_prod":
      print >>sys.stderr, "wc_lr_wp_rank: %d"%(wc_lr_wp_rank)
    if relaxed:
      print >>sys.stderr, "Running without inter-concept preferences"
    else:
      print >>sys.stderr, "cc_pref_model: %s"%(cc_pref_model_type)
      if cc_pref_model_type == "lr_weighted_prod":
        print >>sys.stderr, "cc_lr_wp_rank: %d"%(cc_lr_wp_rank)
    print >>sys.stderr, "rec_model: %s"%rec_model_type

    numpy_rng = numpy.random.RandomState(12345)
    self.theano_rng = RandomStreams(12345)
    self.ont_size = ont_size
    vocab_rep_range = 4 * numpy.sqrt(6. / (vocab_size + word_dim))
    init_vocab_rep = numpy.asarray(numpy_rng.uniform(low = -vocab_rep_range, high = vocab_rep_range, size=(vocab_size, word_dim)) )
    ont_rep_range = 4 * numpy.sqrt(6. / (ont_size + concept_dim))
    init_ont_rep = numpy.asarray(numpy_rng.uniform(low = -ont_rep_range, high = ont_rep_range, size=(ont_size, concept_dim)) )
    self.vocab_rep = theano.shared(value=init_vocab_rep, name='vocab_rep')
    self.ont_rep = theano.shared(value=init_ont_rep, name='ont_rep')
    self.repr_params = [self.vocab_rep] if word_rep_param else []
    self.repr_params.append(self.ont_rep)
    self.enc_params = []
    self.relaxed = relaxed
    self.no_hyp = no_hyp
    if not self.no_hyp:
      self.hyp_model = HypernymModel(hyp_model_type, hyp_hidden_size, self.vocab_rep, self.ont_rep)
      self.enc_params.extend(self.hyp_model.get_params())
    self.wc_pref_models = []
    self.cc_pref_models = []
    self.num_slots = num_args + 1 # +1 for the predicate
    self.num_args = num_args
    self.wc_pref_models = [{} for _ in range(self.num_slots)]
    for i in range(self.num_slots):
      for j in range(self.num_slots):
        if i == j:
          continue
        wc_pref_model = PreferenceModel('word_concept', wc_pref_model_type, wc_hidden_sizes[i], self.ont_rep, "wc_%d_%d"%(i, j),  self.vocab_rep, lr_wp_rank=wc_lr_wp_rank)
        self.wc_pref_models[i][j] = wc_pref_model
        self.enc_params.extend(wc_pref_model.get_params())
    if not self.relaxed:  
      for i in range(num_args):
        cc_pref_model = PreferenceModel('concept_concept', cc_pref_model_type, cc_hidden_sizes[i], self.ont_rep, "cc_%d"%i,  lr_wp_rank=cc_lr_wp_rank)
        self.cc_pref_models.append(cc_pref_model)
        self.enc_params.extend(cc_pref_model.get_params())
    self.rec_model = ReconstructionModel(self.ont_rep, self.vocab_rep, init_hyp_strengths=init_hyp_strengths, rec_model_type=rec_model_type)
    self.rec_params = self.rec_model.get_params()
    # Random y, sampled from uniform(|ont|^num_slots)
    self.y_r = T.cast(self.theano_rng.uniform(low=0, high=self.ont_size-1, size=(self.num_slots,)), 'int32')
    self.num_enc_ns = 1
    self.num_label_ns = 1
Example #2
0
class EventAE(object):
  def __init__(self, num_args, vocab_size, ont_size, hyp_hidden_size, wc_hidden_sizes, cc_hidden_sizes, word_dim=50, concept_dim=50, word_rep_param=False, hyp_model_type="weighted_prod", wc_pref_model_type="tanhlayer", cc_pref_model_type="tanhlayer", rec_model_type="gaussian", init_hyp_strengths=None, relaxed=False, no_hyp=False, wc_lr_wp_rank=10, cc_lr_wp_rank=10):
    print >>sys.stderr, "Initializing SPADE"
    print >>sys.stderr, "num_args: %d"%(num_args)
    print >>sys.stderr, "vocab_size: %d"%(vocab_size)
    print >>sys.stderr, "ont_size: %d"%(ont_size)
    print >>sys.stderr, "word_dim: %d"%(word_dim)
    print >>sys.stderr, "concept_dim: %d"%(concept_dim)
    print >>sys.stderr, "word_rep_param: %s"%(word_rep_param)
    if no_hyp:
      print >>sys.stderr, "Running without hypernymy links"
    else:
      print >>sys.stderr, "hyp_model: %s"%(hyp_model_type)
    print >>sys.stderr, "wc_pref_model: %s"%(wc_pref_model_type)
    if wc_pref_model_type == "lr_weighted_prod":
      print >>sys.stderr, "wc_lr_wp_rank: %d"%(wc_lr_wp_rank)
    if relaxed:
      print >>sys.stderr, "Running without inter-concept preferences"
    else:
      print >>sys.stderr, "cc_pref_model: %s"%(cc_pref_model_type)
      if cc_pref_model_type == "lr_weighted_prod":
        print >>sys.stderr, "cc_lr_wp_rank: %d"%(cc_lr_wp_rank)
    print >>sys.stderr, "rec_model: %s"%rec_model_type

    numpy_rng = numpy.random.RandomState(12345)
    self.theano_rng = RandomStreams(12345)
    self.ont_size = ont_size
    vocab_rep_range = 4 * numpy.sqrt(6. / (vocab_size + word_dim))
    init_vocab_rep = numpy.asarray(numpy_rng.uniform(low = -vocab_rep_range, high = vocab_rep_range, size=(vocab_size, word_dim)) )
    ont_rep_range = 4 * numpy.sqrt(6. / (ont_size + concept_dim))
    init_ont_rep = numpy.asarray(numpy_rng.uniform(low = -ont_rep_range, high = ont_rep_range, size=(ont_size, concept_dim)) )
    self.vocab_rep = theano.shared(value=init_vocab_rep, name='vocab_rep')
    self.ont_rep = theano.shared(value=init_ont_rep, name='ont_rep')
    self.repr_params = [self.vocab_rep] if word_rep_param else []
    self.repr_params.append(self.ont_rep)
    self.enc_params = []
    self.relaxed = relaxed
    self.no_hyp = no_hyp
    if not self.no_hyp:
      self.hyp_model = HypernymModel(hyp_model_type, hyp_hidden_size, self.vocab_rep, self.ont_rep)
      self.enc_params.extend(self.hyp_model.get_params())
    self.wc_pref_models = []
    self.cc_pref_models = []
    self.num_slots = num_args + 1 # +1 for the predicate
    self.num_args = num_args
    self.wc_pref_models = [{} for _ in range(self.num_slots)]
    for i in range(self.num_slots):
      for j in range(self.num_slots):
        if i == j:
          continue
        wc_pref_model = PreferenceModel('word_concept', wc_pref_model_type, wc_hidden_sizes[i], self.ont_rep, "wc_%d_%d"%(i, j),  self.vocab_rep, lr_wp_rank=wc_lr_wp_rank)
        self.wc_pref_models[i][j] = wc_pref_model
        self.enc_params.extend(wc_pref_model.get_params())
    if not self.relaxed:  
      for i in range(num_args):
        cc_pref_model = PreferenceModel('concept_concept', cc_pref_model_type, cc_hidden_sizes[i], self.ont_rep, "cc_%d"%i,  lr_wp_rank=cc_lr_wp_rank)
        self.cc_pref_models.append(cc_pref_model)
        self.enc_params.extend(cc_pref_model.get_params())
    self.rec_model = ReconstructionModel(self.ont_rep, self.vocab_rep, init_hyp_strengths=init_hyp_strengths, rec_model_type=rec_model_type)
    self.rec_params = self.rec_model.get_params()
    # Random y, sampled from uniform(|ont|^num_slots)
    self.y_r = T.cast(self.theano_rng.uniform(low=0, high=self.ont_size-1, size=(self.num_slots,)), 'int32')
    self.num_enc_ns = 1
    self.num_label_ns = 1

  ### Direct prob functions ###
  def get_sym_encoder_energy(self, x, y):
    # Works with NCE
    hsum = T.constant(0)
    if not self.no_hyp:
      for i in range(self.num_slots):
        hsum += self.hyp_model.get_symb_score(x[i], y[i])
    p_w_c_sum = T.constant(0)
    for i in range(self.num_slots):
      for j in range(self.num_slots):
        if i == j:
          continue
        p_w_c_sum += self.wc_pref_models[i][j].get_symb_score(x[i], y[j])
    p_c_c_sum = T.constant(0)
    for i in range(self.num_args):
      p_c_c_sum += self.cc_pref_models[i].get_symb_score(y[0], y[i + 1])
    return hsum + p_w_c_sum + p_c_c_sum

  def get_sym_encoder_partition(self, x, y_s):
    partial_sums, _ = theano.scan(fn=lambda y, interm_sum, x_0: interm_sum + T.exp(self.get_sym_encoder_energy(x_0, y)), outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=x)
    encoder_partition = partial_sums[-1]
    return encoder_partition

  def get_sym_rec_prob(self, x, y):
    # Works with NCE
    init_prob = T.constant(1.0, dtype='float64')
    partial_prods, _ = theano.scan(fn = lambda x_i, y_i, interm_prod: interm_prod * self.rec_model.get_sym_rec_prob(x_i, y_i), outputs_info=init_prob, sequences=[x, y])
    rec_prob = partial_prods[-1]
    return rec_prob
  
  def get_sym_posterior_num(self, x, y):
    # Needed for NCE
    enc_energy = self.get_sym_encoder_energy(x, y)
    rec_prob = self.get_sym_rec_prob(x, y)
    return T.exp(enc_energy) * rec_prob
      
  def get_sym_posterior_partition(self, x, y_s):
    partial_sums, _ = theano.scan(fn=lambda y, interm_sum, x_0: interm_sum + self.get_sym_posterior_num(x_0, y), outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=x)
    posterior_partition = partial_sums[-1]
    return posterior_partition
  
  def get_sym_direct_prob(self, x, y_s):
    def get_post_num_sum(y_0, interm_sum, x_0):
      posterior_num = self.get_sym_posterior_num(x_0, y_0)
      return interm_sum + posterior_num
    res, _ = theano.scan(fn=get_post_num_sum, outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=[x])
    direct_prob = res[-1] / self.get_sym_encoder_partition(x, y_s)
    return direct_prob

  # Following function is useless.
  def get_sym_posterior(self, x, y, y_s):
    return self.get_sym_posterior_num(x, y) / self.get_sym_posterior_partition(x, y_s)

  ### Complete expectation functions ###
  def get_sym_complete_expectation(self, x, y_s):
    encoder_partition = self.get_sym_encoder_partition(x, y_s)
    posterior_partition = self.get_sym_posterior_partition(x, y_s)
    def prod_fun(y_0, interm_sum, x_0): 
      post_num = self.get_sym_posterior_num(x_0, y_0)
      fixed_post_num = ifelse(T.le(post_num, SMALL_NUM), T.constant(0.0, dtype='float64'), post_num)
      return interm_sum + ifelse(T.le(fixed_post_num, SMALL_NUM), T.constant(0.0, dtype='float64'), fixed_post_num * T.log(fixed_post_num))
    partial_sums, _ = theano.scan(fn=prod_fun, outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=x)
    data_term = ifelse(T.eq(posterior_partition, T.constant(0.0, dtype='float64')), T.constant(0.0, dtype='float64'), partial_sums[-1] / posterior_partition)
    #data_term = partial_sums[-1]
    complete_expectation = data_term - T.log(encoder_partition)
    #complete_expectation = data_term
    return complete_expectation

  ### NCE functions  ###
  def get_sym_rand_y(self, y_s):
    # NCE function
    # Sample randomly from y|x
    rand_ind = T.cast(self.theano_rng.uniform(low=0, high=y_s.shape[0]-1, size=(1,)), 'int32')
    sample = y_s[rand_ind[0]]
    return sample
    
  def get_sym_nc_encoder_prob(self, x, y, y_s, num_noise_samples=None):
    # NCE function
    if num_noise_samples is None:
      num_noise_samples = self.num_enc_ns
    enc_energy = T.exp(self.get_sym_encoder_energy(x, y))
    ns_prob = num_noise_samples * ((1. / self.ont_size) ** self.num_slots)
    true_prob = enc_energy / (enc_energy + ns_prob)
    noise_prob = T.constant(1.0, dtype='float64')
    for _ in range(num_noise_samples):
      # Noise distribution is not conditioned on x. So we sample directly from ont, not from y_s
      ns_enc_energy = T.exp(self.get_sym_encoder_energy(x, self.y_r))
      #ns_enc_energy = T.exp(self.get_sym_encoder_energy(x, self.get_sym_rand_y(y_s)))
      noise_prob *= ns_prob / (ns_enc_energy + ns_prob)
    return true_prob * noise_prob

  def get_sym_nc_posterior(self, x, y, y_s, num_noise_samples=None):
    # NCE function
    # p(\hat{x}, y | x)
    if num_noise_samples is None:
      num_noise_samples = self.num_enc_ns
    return self.get_sym_nc_encoder_prob(x, y, y_s, num_noise_samples=num_noise_samples) * self.get_sym_rec_prob(x, y)

  def get_sym_nc_direct_prob(self, x, y_s):
    # NCE function
    def get_prob(y_0, interm_sum, x_0, Y):
      posterior = self.get_sym_nc_posterior(x_0, y_0, Y)
      return interm_sum + posterior
    res, _ = theano.scan(fn=get_prob, outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=[x, y_s])
    direct_prob = res[-1]
    return direct_prob

  def get_sym_nc_label_prob(self, x, y, y_s, num_noise_samples=None):
    # NCE function
    # p(y | x, \hat{x})
    if num_noise_samples is None:
      num_noise_samples = self.num_label_ns
    true_posterior = self.get_sym_nc_posterior(x, y)
    #TODO: Can make this more efficient
    noise_posterior = self.get_sym_nc_posterior(x, self.get_sym_rand_y(y_s), num_noise_samples=1)
    ns_prob = num_noise_samples * T.pow(1. / y_s.shape[0], self.num_slots)
    true_prob = true_posterior / (true_posterior + ns_prob)
    noise_prob = T.constant(1.0, dtype='float64')
    for _ in range(num_noise_samples):
      #noise_posterior = self.get_sym_nc_posterior(x, self.get_sym_rand_y(y_s))
      noise_prob *= ns_prob / (noise_posterior + ns_prob)
    return true_prob * noise_prob
  
  def get_sym_nc_complete_expectation(self, x, y_s):
    # NCE function
    def get_expectation(y_0, interm_sum, x_0, Y):
      label_prob = self.get_sym_nc_label_prob(x_0, y_0, Y)
      posterior = self.get_sym_nc_posterior(x_0, y_0)
      log_posterior = ifelse(T.le(posterior, SMALL_NUM), T.constant(LOG_SMALL_NUM, dtype='float64'), T.log(posterior))
      return interm_sum + (label_prob * log_posterior)
    res, _ = theano.scan(fn=get_expectation, outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=[x, y_s])
    complete_expectation = res[-1]
    return complete_expectation

  def get_train_func(self, learning_rate, nce=True, em=False):
    print >>sys.stderr, "Trainining type: EM = %s, NCE = %s"%(em, nce)
    # TODO: Implement AdaGrad
    x, y_s = T.ivector("x"), T.imatrix("y_s")
    if em:
      cost = -self.get_sym_nc_complete_expectation(x, y_s) if nce else -self.get_sym_complete_expectation(x, y_s)
    else:
      cost = -T.log(self.get_sym_nc_direct_prob(x, y_s)) if nce else -T.log(self.get_sym_direct_prob(x, y_s))
    params = self.repr_params + self.enc_params + self.rec_params
    g_params = T.grad(cost, params)
    # Updating the parameters only if the norm of the gradient is less than 100.
    # Important: This check also takes care of any element in the gradients being nan. The conditional returns False even in that case.
    updates=[ (p, ifelse(T.le(T.nlinalg.norm(g, None), T.constant(100.0, dtype='float64')), p - learning_rate * g, p)) for p, g in zip(params, g_params) ]
    train_func = theano.function([x, y_s], cost, updates=updates)
    return train_func

  def get_posterior_func(self):
    # Works with NCE
    x, y = T.ivectors('x', 'y')
    posterior_func = theano.function([x, y], self.get_sym_posterior_num(x, y))
    return posterior_func

  def get_rec_prob_func(self):
    # Works with NCE
    x, y = T.ivectors('x', 'y')
    rec_prob_func = theano.function([x, y], self.get_sym_rec_prob(x, y))
    return rec_prob_func
 
  ### Relaxed variant functions ### 
  def get_sym_relaxed_encoder_energy(self, x, y, s):
    h = self.hyp_model.get_symb_score(x[s], y) if not self.no_hyp else T.constant(0.0)
    p_sum = T.constant(0.0)
    # We need to sum up the preference scores of words in all slots except s with y
    for i in range(self.num_slots):
      if i == s:
        continue
      p_sum += self.wc_pref_models[i][s].get_symb_score(x[i], y)
    return h + p_sum

  def get_sym_relaxed_encoder_partition(self, x, y_s, s):
    partial_sums, _ = theano.scan(fn=lambda y, interm_sum, x_0: interm_sum + T.exp(self.get_sym_relaxed_encoder_energy(x_0, y, s)), outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=[x])
    encoder_partition = partial_sums[-1]
    return encoder_partition

  def get_sym_relaxed_posterior_num(self, x, y, s):
    # Needed for NCE
    enc_energy = self.get_sym_relaxed_encoder_energy(x, y, s)
    rec_prob = self.rec_model.get_sym_rec_prob(x[s], y)
    return T.exp(enc_energy) * rec_prob
      
  def get_sym_relaxed_posterior_partition(self, x, y_s, s):
    partial_sums, _ = theano.scan(fn=lambda y, interm_sum, x_0, s: interm_sum + self.get_sym_relaxed_posterior_num(x_0, y, s), outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=[x,s])
    posterior_partition = partial_sums[-1]
    return posterior_partition
  
  def get_sym_relaxed_direct_prob(self, x, y_s, s):
    def get_post_num_sum(y_0, interm_sum, x_0):
      posterior_num = self.get_sym_relaxed_posterior_num(x_0, y_0, s)
      return interm_sum + posterior_num
    res, _ = theano.scan(fn=get_post_num_sum, outputs_info=numpy.asarray(0.0, dtype='float64'), sequences=[y_s], non_sequences=[x])
    direct_prob = res[-1] / self.get_sym_relaxed_encoder_partition(x, y_s, s)
    return direct_prob

  def get_relaxed_train_func(self, learning_rate, s):
    # TODO: Implement AdaGrad
    # TODO: This means we need one train function per slot.  Do we?
    x, y_s = T.ivector("x"), T.ivector("y_s")
    dp = self.get_sym_relaxed_direct_prob(x, y_s, s)
    cost = -T.log(dp)
    relaxed_enc_params = []
    if not self.no_hyp:
      relaxed_enc_params.extend(self.hyp_model.get_params())
    for i in range(self.num_slots):
      if i == s:
        continue
      relaxed_enc_params.extend(self.wc_pref_models[i][s].get_params())
    params = self.repr_params + relaxed_enc_params + self.rec_params
    g_params = T.grad(cost, params)
    # Updating the parameters only if the norm of the gradient is less than 100.
    # Important: This check also takes care of any element in the gradients being nan. The conditional returns False even in that case.
    updates=[ (p, ifelse(T.le(T.nlinalg.norm(g, None), T.constant(100.0, dtype='float64')), p - learning_rate * g, p)) for p, g in zip(params, g_params) ]
    train_func = theano.function([x, y_s], cost, updates=updates)
    return train_func

  def get_relaxed_posterior_func(self, s):
    # Works with NCE
    x = T.ivector('x')
    y = T.iscalar('y')
    posterior_func = theano.function([x, y], self.get_sym_relaxed_posterior_num(x, y, s))
    return posterior_func

  def set_repr_params(self, repr_param_vals):
    for i, param_val in enumerate(repr_param_vals):
      self.repr_params[i].set_value(param_val)

  def set_rec_params(self, rec_param_vals):
    for i, param_val in enumerate(rec_param_vals):
      self.rec_params[i].set_value(param_val)