def __init__(self, vocab: Vocabulary, bow_embedder: TokenEmbedder, vae: VAE, apply_batchnorm_on_recon: bool = False, batchnorm_weight_learnable: bool = False, batchnorm_bias_learnable: bool = True, kl_weight_annealing: str = "constant", linear_scaling: float = 1000.0, sigmoid_weight_1: float = 0.25, sigmoid_weight_2: float = 15, reference_counts: str = None, reference_vocabulary: str = None, use_doc_info: str = False, use_background: str = False, background_data_path: str = None, update_background_freq: bool = False, track_topics: bool = True, track_npmi: bool = True, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab, regularizer) self.metrics = {'nkld': Average(), 'nll': Average(), 'perp': Average()} self.vocab = vocab self.vae = vae self.track_topics = track_topics self.track_npmi = track_npmi self.vocab_namespace = "persona_based" self._update_background_freq = update_background_freq vocab_size = self.vocab.get_vocab_size(self.vocab_namespace) self._use_doc_info = use_doc_info # bp() if use_doc_info: self.interpolation = torch.nn.Parameter(torch.zeros(2, requires_grad=True)) self._background_freq = self.initialize_bg_from_file(file_=background_data_path) if use_background else 0 print(self._background_freq) # bp() self._ref_counts = reference_counts if reference_vocabulary is not None: # Compute data necessary to compute NPMI every epoch logger.info("Loading reference vocabulary.") self._ref_vocab = read_json(cached_path(reference_vocabulary)) self._ref_vocab_index = dict(zip(self._ref_vocab, range(len(self._ref_vocab)))) logger.info("Loading reference count matrix.") self._ref_count_mat = load_sparse(cached_path(self._ref_counts)) logger.info("Computing word interaction matrix.") self._ref_doc_counts = (self._ref_count_mat > 0).astype(float) self._ref_interaction = self._ref_doc_counts.T.dot(self._ref_doc_counts) self._ref_doc_sum = np.array(self._ref_doc_counts.sum(0).tolist()[0]) logger.info("Generating npmi matrices.") (self._npmi_numerator, self._npmi_denominator) = self.generate_npmi_vals(self._ref_interaction, self._ref_doc_sum) self.n_docs = self._ref_count_mat.shape[0] self._bag_of_words_embedder = bow_embedder self._kl_weight_annealing = kl_weight_annealing self._linear_scaling = float(linear_scaling) self._sigmoid_weight_1 = float(sigmoid_weight_1) self._sigmoid_weight_2 = float(sigmoid_weight_2) if kl_weight_annealing == "linear": self._kld_weight = min(1.0, 1 / self._linear_scaling) elif kl_weight_annealing == "sigmoid": self._kld_weight = float(1 / (1 + np.exp(-self._sigmoid_weight_1 * (1 - self._sigmoid_weight_2)))) elif kl_weight_annealing == "constant": self._kld_weight = 1.0 else: raise ConfigurationError("anneal type {} not found".format(kl_weight_annealing)) # setup batchnorm self._apply_batchnorm_on_recon = apply_batchnorm_on_recon if apply_batchnorm_on_recon: self.bow_bn = create_trainable_BatchNorm1d(vocab_size, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) # Maintain these states for periodically printing topics and updating KLD self._metric_epoch_tracker = 0 self._kl_epoch_tracker = 0 self._cur_epoch = 0 self._cur_npmi = 0.0 self.batch_num = 0 initializer(self)
def __init__( self, vocab, encoder_topic: FeedForward, mean_projection_topic: FeedForward, log_variance_projection_topic: FeedForward, encoder_entity: FeedForward, mean_projection_entity: FeedForward, log_variance_projection_entity: FeedForward, encoder_entity_approx: FeedForward, mean_projection_entity_approx: FeedForward, log_variance_projection_entity_approx: FeedForward, decoder_topic: FeedForward, # decode topic input to persona hidden decoder_mean_projection_topic: FeedForward, decoder_log_variance_projection_topic: FeedForward, decoder_persona: FeedForward, prior: Dict = { "type": "normal", "mu": 0, "var": 1 }, apply_batchnorm_on_normal: bool = False, apply_batchnorm_on_decoder: bool = False, batchnorm_weight_learnable: bool = False, batchnorm_bias_learnable: bool = True, stochastic_beta: bool = False, z_dropout: float = 0.2) -> None: super(BasicLVAE, self).__init__(vocab) self.encoder_topic = encoder_topic self.mean_topic = mean_projection_topic self.log_var_topic = log_variance_projection_topic self.encoder_entity = encoder_entity self.mean_entity = mean_projection_entity self.log_var_entity = log_variance_projection_entity self.encoder_entity_approx = encoder_entity_approx self.mean_entity_approx = mean_projection_entity_approx self.log_var_entity_approx = log_variance_projection_entity_approx self.decoder_mean_topic = decoder_mean_projection_topic self.decoder_log_var_topic = decoder_log_variance_projection_topic self._decoder_topic = torch.nn.Linear(decoder_topic.get_input_dim(), decoder_topic.get_output_dim(), bias=False) self._decoder_persona = torch.nn.Linear( decoder_persona.get_input_dim(), decoder_persona.get_output_dim(), bias=False) self._z_dropout = torch.nn.Dropout(z_dropout) self.num_topic = encoder_topic.get_output_dim() self.num_persona = encoder_entity.get_output_dim() self.prior = prior self.p_params = None # self.p_mu, self.p_sigma, self.p_log_var = None, None, None self.initialize_prior(prior) # If specified, established batchnorm for both mean and log variance. self._apply_batchnorm_on_normal = apply_batchnorm_on_normal self.mean_bn_entity, self.log_var_bn_entity = None, None self.mean_bn_topic, self.log_var_bn_topic = None, None self.decoder_mean_bn_topic, self.decoder_log_var_bn_topic = None, None self.mean_bn_entity_approx, self.log_var_bn_entity_approx = None, None if apply_batchnorm_on_normal: self.mean_bn_topic = create_trainable_BatchNorm1d( self.num_topic, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.log_var_bn_topic = create_trainable_BatchNorm1d( self.num_topic, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.decoder_mean_bn_topic = create_trainable_BatchNorm1d( self.num_persona, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.decoder_log_var_bn_topic = create_trainable_BatchNorm1d( self.num_persona, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) # If specified, established batchnorm for reconstruction matrix, applying batch norm across vocabulary self._apply_batchnorm_on_decoder = apply_batchnorm_on_decoder if apply_batchnorm_on_decoder: self.decoder_bn_topic = create_trainable_BatchNorm1d( decoder_topic.get_output_dim(), weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.decoder_bn_persona = create_trainable_BatchNorm1d( decoder_persona.get_output_dim(), weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) # If specified, constrain each topic to be a distribution over vocabulary self._stochastic_beta = stochastic_beta
def __init__(self, vocab, extracter: Seq2VecEncoder, encoder: FeedForward, mean_projection: FeedForward, log_variance_projection: FeedForward, decoder: FeedForward, prior: Dict = {"type": "normal", "mu": 0, "var": 1}, apply_batchnorm_on_normal: bool = False, apply_batchnorm_on_decoder: bool = False, batchnorm_weight_learnable: bool = False, batchnorm_bias_learnable: bool = True, stochastic_beta: bool = False, z_dropout: float = 0.2) -> None: super(CNNVAE, self).__init__(vocab) self.extracter = extracter self.encoder = encoder self.mean_projection = mean_projection self.log_variance_projection = log_variance_projection self._decoder = torch.nn.Linear(decoder.get_input_dim(), decoder.get_output_dim(), bias=False) self._z_dropout = torch.nn.Dropout(z_dropout) self.latent_dim = mean_projection.get_output_dim() self.prior = prior if prior['type'] == "normal": if 'mu' not in prior or 'var' not in prior: raise Exception("MU, VAR undefined for normal") p_mu = torch.zeros(1, self.latent_dim).fill_(prior['mu']) p_var = torch.zeros(1, self.latent_dim).fill_(prior['var']) p_log_var = p_var.log() elif prior['type'] == "laplace-approx": a = torch.zeros(1, self.latent_dim).fill_(prior['alpha']) p_mu = a.log() - torch.mean(a.log(), 1) p_var = 1.0 / a * (1 - 2.0 / self.latent_dim) + 1.0 / self.latent_dim * torch.mean(1 / a) p_log_var = p_var.log() else: raise Exception("Invalid/Undefined prior!") # parameters of prior distribution are not trainable self.register_buffer("p_mu", p_mu) self.register_buffer("p_log_var", p_log_var) # If specified, established batchnorm for both mean and log variance. self._apply_batchnorm_on_normal = apply_batchnorm_on_normal if apply_batchnorm_on_normal: self.mean_bn = create_trainable_BatchNorm1d(self.latent_dim, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.log_var_bn = create_trainable_BatchNorm1d(self.latent_dim, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) # If specified, established batchnorm for reconstruction matrix, applying batch norm across vocabulary self._apply_batchnorm_on_decoder = apply_batchnorm_on_decoder if apply_batchnorm_on_decoder: self.decoder_bn = create_trainable_BatchNorm1d(decoder.get_output_dim(), weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) # If specified, constrain each topic to be a distribution over vocabulary self._stochastic_beta = stochastic_beta
def __init__(self, vocab, extracter: Seq2VecEncoder, encoder_d1: FeedForward, encoder_d2: FeedForward, mean_projection_d1: FeedForward, log_variance_projection_d1: FeedForward, mean_projection_d2: FeedForward, log_variance_projection_d2: FeedForward, encoder_t1: FeedForward, mean_projection_t1: FeedForward, log_variance_projection_t1: FeedForward, decoder1: FeedForward, decoder2: FeedForward, mean_projection_dec2: FeedForward, log_variance_projection_dec2: FeedForward, prior: Dict = {"type": "normal", "mu": 0, "var": 1}, apply_batchnorm_on_normal: bool = False, apply_batchnorm_on_decoder: bool = False, batchnorm_weight_learnable: bool = False, batchnorm_bias_learnable: bool = True, stochastic_beta: bool = False, z_dropout: float = 0.2) -> None: super(LadderVAE, self).__init__(vocab) self.extracter = extracter # bp() self.encoder_d1 = encoder_d1 self.mean_projection_d1 = mean_projection_d1 self.log_variance_projection_d1 = log_variance_projection_d1 self.encoder_d2 = encoder_d2 self.mean_projection_d2 = mean_projection_d2 self.log_variance_projection_d2 = log_variance_projection_d2 self.encoder_t1 = encoder_t1 self.mean_projection_t1 = mean_projection_t1 self.log_variance_projection_t1 = log_variance_projection_t1 self._decoder1 = torch.nn.Linear(decoder1.get_input_dim(), decoder1.get_output_dim(), bias=False) self._decoder2 = torch.nn.Linear(decoder2.get_input_dim(), decoder2.get_output_dim(), bias=False) self.mean_projection_dec2 = mean_projection_dec2 self.log_variance_projection_dec2 = log_variance_projection_dec2 self._z_dropout = torch.nn.Dropout(z_dropout) self.num_persona = mean_projection_d1.get_output_dim() self.num_topic = mean_projection_d2.get_output_dim() self.prior = prior self.p_params = None # self.p_mu, self.p_sigma, self.p_log_var = None, None, None self.initialize_prior(prior) # If specified, established batchnorm for both mean and log variance. self._apply_batchnorm_on_normal = apply_batchnorm_on_normal self.mean_bn_d1, self.log_var_bn_d1 = None, None self.mean_bn_d2, self.log_var_bn_d2 = None, None self.mean_bn_t1, self.log_var_bn_t1 = None, None if apply_batchnorm_on_normal: self.mean_bn_d1 = create_trainable_BatchNorm1d(self.num_persona, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.log_var_bn_d1 = create_trainable_BatchNorm1d(self.num_persona, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.mean_bn_d2 = create_trainable_BatchNorm1d(self.num_topic, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.log_var_bn_d2 = create_trainable_BatchNorm1d(self.num_topic, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.mean_bn_t1 = create_trainable_BatchNorm1d(self.num_persona, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.log_var_bn_t1 = create_trainable_BatchNorm1d(self.num_persona, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.mean_bn_dec2 = create_trainable_BatchNorm1d(self.num_persona, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.log_var_bn_dec2 = create_trainable_BatchNorm1d(self.num_persona, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) # If specified, established batchnorm for reconstruction matrix, applying batch norm across vocabulary self._apply_batchnorm_on_decoder = apply_batchnorm_on_decoder if apply_batchnorm_on_decoder: self.decoder_bn1 = create_trainable_BatchNorm1d(decoder1.get_output_dim(), weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.decoder_bn2 = create_trainable_BatchNorm1d(decoder2.get_output_dim(), weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) # If specified, constrain each topic to be a distribution over vocabulary self._stochastic_beta = stochastic_beta
def __init__( self, vocab, encoder_entity: FeedForward, encoder_entity_global: FeedForward, decoder_type: FeedForward, # (d_dim -> P) mean_projection_type: FeedForward, log_var_projection_type: FeedForward, decoder_topic: FeedForward, # (K -> V) decoder_persona: FeedForward, # (P -> K) prior: Dict = { "type": "normal", "mu": 0, "var": 1 }, pooling_layer: str = "max", apply_batchnorm_on_normal: bool = False, apply_batchnorm_on_decoder: bool = False, batchnorm_weight_learnable: bool = False, batchnorm_bias_learnable: bool = True, stochastic_weight: bool = False, z_dropout: float = 0.2) -> None: super(Bamman, self).__init__(vocab) self.encoder_entity = encoder_entity self.encoder_entity_global = encoder_entity_global self.mean_projection_type = mean_projection_type self.log_var_projection_type = log_var_projection_type self._decoder_type = torch.nn.Linear(decoder_type.get_input_dim(), decoder_type.get_output_dim(), bias=False) self._decoder_topic = torch.nn.Linear(decoder_topic.get_input_dim(), decoder_topic.get_output_dim(), bias=False) self._decoder_persona = torch.nn.Linear( decoder_persona.get_input_dim(), decoder_persona.get_output_dim(), bias=False) self._z_dropout = torch.nn.Dropout(z_dropout) self.num_type = decoder_type.get_input_dim() self.num_topic = decoder_persona.get_output_dim() self.num_persona = decoder_persona.get_input_dim() self.prior = prior if pooling_layer not in ["max", "sum", "mean"]: raise Exception("Undefined pooling function") self.pooling_func = pooling_layer self.pooling_layer = getattr(torch, pooling_layer) self.p_params = None # self.p_mu, self.p_sigma, self.p_log_var = None, None, None self.initialize_prior(prior) # If specified, established batchnorm for both mean and log variance. self._apply_batchnorm_on_normal = apply_batchnorm_on_normal self.mean_bn_type, self.log_var_bn_type = None, None if apply_batchnorm_on_normal: self.mean_bn_type = create_trainable_BatchNorm1d( self.num_type, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.log_var_bn_type = create_trainable_BatchNorm1d( self.num_type, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) # If specified, established batchnorm for reconstruction matrix, applying batch norm across vocabulary self._apply_batchnorm_on_decoder = apply_batchnorm_on_decoder if apply_batchnorm_on_decoder: self.decoder_bn_type = create_trainable_BatchNorm1d( decoder_type.get_output_dim(), weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.decoder_bn_topic = create_trainable_BatchNorm1d( decoder_topic.get_output_dim(), weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.decoder_bn_persona = create_trainable_BatchNorm1d( decoder_persona.get_output_dim(), weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) # If specified, constrain each topic to be a distribution over vocabulary self._stochastic_weight = stochastic_weight
def __init__( self, vocab, encoder: FeedForward, mean_projection: FeedForward, log_variance_projection: FeedForward, decoder: FeedForward, # prior: Dict = {"type": "normal", "mu": 0, "var": 1}, # apply_batchnorm_on_normal: bool = False, # apply_batchnorm_on_decoder: bool = False, # batchnorm_weight_learnable: bool = False, # batchnorm_bias_learnable: bool = True, # stochastic_beta: bool = False, apply_batchnorm: bool = False, z_dropout: float = 0.2) -> None: super(LogisticNormal, self).__init__(vocab) self.encoder = encoder self.mean_projection = mean_projection self.log_variance_projection = log_variance_projection self._decoder = torch.nn.Linear(decoder.get_input_dim(), decoder.get_output_dim(), bias=False) self._z_dropout = torch.nn.Dropout(z_dropout) self.latent_dim = mean_projection.get_output_dim() # self.prior = prior # if prior['type'] == "normal": # if 'mu' not in prior or 'var' not in prior: # raise Exception("MU, VAR undefined for normal") # p_mu = torch.zeros(1, self.latent_dim).fill_(prior['mu']) # p_var = torch.zeros(1, self.latent_dim).fill_(prior['var']) # p_log_var = p_var.log() # elif prior['type'] == "laplace-approx": # a = torch.zeros(1, self.latent_dim).fill_(prior['alpha']) # p_mu = a.log() - torch.mean(a.log(), 1) # p_var = 1.0 / a * (1 - 2.0 / self.latent_dim) + 1.0 / self.latent_dim * torch.mean(1 / a) # p_log_var = p_var.log() # else: # raise Exception("Invalid/Undefined prior!") # parameters of prior distribution are not trainable # self.register_buffer("p_mu", p_mu) # self.register_buffer("p_log_var", p_log_var) # If specified, established batchnorm for both mean and log variance. self._apply_batchnorm = apply_batchnorm if apply_batchnorm: self.mean_bn = create_trainable_BatchNorm1d( self.latent_dim, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) self.log_var_bn = create_trainable_BatchNorm1d( self.latent_dim, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True)