def __init__(self, vocab: Vocabulary, bow_embedder: TokenEmbedder, vae: VAE, apply_batchnorm_on_recon: bool = False, batchnorm_weight_learnable: bool = False, batchnorm_bias_learnable: bool = True, kl_weight_annealing: str = "constant", linear_scaling: float = 1000.0, sigmoid_weight_1: float = 0.25, sigmoid_weight_2: float = 15, reference_counts: str = None, reference_vocabulary: str = None, use_doc_info: str = False, use_background: str = False, background_data_path: str = None, update_background_freq: bool = False, track_topics: bool = True, track_npmi: bool = True, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab, regularizer) self.metrics = {'nkld': Average(), 'nll': Average(), 'perp': Average()} self.vocab = vocab self.vae = vae self.track_topics = track_topics self.track_npmi = track_npmi self.vocab_namespace = "persona_based" self._update_background_freq = update_background_freq vocab_size = self.vocab.get_vocab_size(self.vocab_namespace) self._use_doc_info = use_doc_info # bp() if use_doc_info: self.interpolation = torch.nn.Parameter(torch.zeros(2, requires_grad=True)) self._background_freq = self.initialize_bg_from_file(file_=background_data_path) if use_background else 0 print(self._background_freq) # bp() self._ref_counts = reference_counts if reference_vocabulary is not None: # Compute data necessary to compute NPMI every epoch logger.info("Loading reference vocabulary.") self._ref_vocab = read_json(cached_path(reference_vocabulary)) self._ref_vocab_index = dict(zip(self._ref_vocab, range(len(self._ref_vocab)))) logger.info("Loading reference count matrix.") self._ref_count_mat = load_sparse(cached_path(self._ref_counts)) logger.info("Computing word interaction matrix.") self._ref_doc_counts = (self._ref_count_mat > 0).astype(float) self._ref_interaction = self._ref_doc_counts.T.dot(self._ref_doc_counts) self._ref_doc_sum = np.array(self._ref_doc_counts.sum(0).tolist()[0]) logger.info("Generating npmi matrices.") (self._npmi_numerator, self._npmi_denominator) = self.generate_npmi_vals(self._ref_interaction, self._ref_doc_sum) self.n_docs = self._ref_count_mat.shape[0] self._bag_of_words_embedder = bow_embedder self._kl_weight_annealing = kl_weight_annealing self._linear_scaling = float(linear_scaling) self._sigmoid_weight_1 = float(sigmoid_weight_1) self._sigmoid_weight_2 = float(sigmoid_weight_2) if kl_weight_annealing == "linear": self._kld_weight = min(1.0, 1 / self._linear_scaling) elif kl_weight_annealing == "sigmoid": self._kld_weight = float(1 / (1 + np.exp(-self._sigmoid_weight_1 * (1 - self._sigmoid_weight_2)))) elif kl_weight_annealing == "constant": self._kld_weight = 1.0 else: raise ConfigurationError("anneal type {} not found".format(kl_weight_annealing)) # setup batchnorm self._apply_batchnorm_on_recon = apply_batchnorm_on_recon if apply_batchnorm_on_recon: self.bow_bn = create_trainable_BatchNorm1d(vocab_size, weight_learnable=batchnorm_weight_learnable, bias_learnable=batchnorm_bias_learnable, eps=0.001, momentum=0.001, affine=True) # Maintain these states for periodically printing topics and updating KLD self._metric_epoch_tracker = 0 self._kl_epoch_tracker = 0 self._cur_epoch = 0 self._cur_npmi = 0.0 self.batch_num = 0 initializer(self)
def __init__(self, vocab: Vocabulary, bow_embedder: TokenEmbedder, vae: VAE, doc_kl_weight_annealing: str = "constant", doc_linear_scaling: float = 1000.0, doc_sigmoid_weight_1: float = 0.25, doc_sigmoid_weight_2: float = 15, doc_saturation_period: int = 2, doc_period: int = 10, entity_kl_weight_annealing: str = "constant", entity_linear_scaling: float = 1000.0, entity_sigmoid_weight_1: float = 0.25, entity_sigmoid_weight_2: float = 15, entity_saturation_period: int = 2, entity_period: int = 10, reference_counts: str = None, reference_vocabulary: str = None, background_data_path: str = None, update_background_freq: bool = False, track_topics: bool = True, track_npmi: bool = True, visual_topic: bool = True, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super().__init__(vocab, regularizer) self.metrics = { 'nkld': Average(), 'd_nkld': Average(), 'e_nkld': Average(), 'nll': Average() } self.vocab = vocab self.vae = vae self.track_topics = track_topics self.track_npmi = track_npmi self.visual_topic = visual_topic self.vocab_namespace = "entity_based" self._update_background_freq = update_background_freq # bp() self._background_freq = self.initialize_bg_from_file( file_=background_data_path) # bp() self._ref_counts = reference_counts self._npmi_updated = False import pickle # if dev_path is not None: # self.dev_set = pickle.load(open(dev_path, "rb")) if reference_vocabulary is not None: # Compute data necessary to compute NPMI every epoch logger.info("Loading reference vocabulary.") self._ref_vocab = read_json(cached_path(reference_vocabulary)) self._ref_vocab_index = dict( zip(self._ref_vocab, range(len(self._ref_vocab)))) logger.info("Loading reference count matrix.") self._ref_count_mat = load_sparse(cached_path(self._ref_counts)) logger.info("Computing word interaction matrix.") self._ref_doc_counts = (self._ref_count_mat > 0).astype(float) self._ref_interaction = (self._ref_doc_counts).T.dot( self._ref_doc_counts) self._ref_doc_sum = np.array( self._ref_doc_counts.sum(0).tolist()[0]) logger.info("Generating npmi matrices.") (self._npmi_numerator, self._npmi_denominator) = self.generate_npmi_vals( self._ref_interaction, self._ref_doc_sum) self.n_docs = self._ref_count_mat.shape[0] vocab_size = self.vocab.get_vocab_size(self.vocab_namespace) self._bag_of_words_embedder = bow_embedder self._doc_kl_weight_annealing = doc_kl_weight_annealing self._doc_linear_scaling = float(doc_linear_scaling) self._doc_sigmoid_weight_1 = float(doc_sigmoid_weight_1) self._doc_sigmoid_weight_2 = float(doc_sigmoid_weight_2) if doc_kl_weight_annealing == "linear": self._doc_kld_weight = min(1, 1 / self._doc_linear_scaling) elif doc_kl_weight_annealing == "sigmoid": self._doc_kld_weight = float( 1 / (1 + np.exp(-self._doc_sigmoid_weight_1 * (1 - self._doc_sigmoid_weight_2)))) elif doc_kl_weight_annealing == "constant": self._doc_kld_weight = 1.0 elif doc_kl_weight_annealing == "cyclic-linear": self._doc_period = doc_period self._doc_saturation_period = doc_saturation_period self._doc_cyclic_kl_anneal_tracker = 0 self._doc_kld_weight = 1 / self._doc_period else: raise ConfigurationError("anneal type(doc) {} not found".format( doc_kl_weight_annealing)) self._entity_kl_weight_annealing = entity_kl_weight_annealing self._entity_linear_scaling = float(entity_linear_scaling) self._entity_sigmoid_weight_1 = float(entity_sigmoid_weight_1) self._entity_sigmoid_weight_2 = float(entity_sigmoid_weight_2) if entity_kl_weight_annealing == "linear": self._entity_kld_weight = min(1, 1 / self._entity_linear_scaling) elif entity_kl_weight_annealing == "sigmoid": self._entity_kld_weight = float( 1 / (1 + np.exp(-self._entity_sigmoid_weight_1 * (1 - self._entity_sigmoid_weight_2)))) elif entity_kl_weight_annealing == "constant": self._entity_kld_weight = 1.0 elif entity_kl_weight_annealing == "cyclic-linear": self._entity_period = entity_period self._entity_saturation_period = entity_saturation_period self._entity_cyclic_kl_anneal_tracker = 0 self._entity_kld_weight = 1 / self._entity_period else: raise ConfigurationError("anneal type(entity) {} not found".format( entity_kl_weight_annealing)) # setup batchnorm self.doc_bow_bn = torch.nn.BatchNorm1d(vocab_size, eps=0.001, momentum=0.001, affine=True) self.doc_bow_bn.weight.data.copy_( torch.ones(vocab_size, dtype=torch.float64)) self.doc_bow_bn.weight.requires_grad = False # self.entity_bow_bn = torch.nn.BatchNorm1d(vocab_size, eps=0.001, momentum=0.001, affine=True) # self.entity_bow_bn.weight.data.copy_(torch.ones(vocab_size, dtype=torch.float64)) # self.entity_bow_bn.weight.requires_grad = False # Maintain these states for periodically printing topics and updating KLD self._metric_epoch_tracker = 0 self._kl_epoch_tracker = 0 self._cur_epoch = 0 self._cur_entity_npmi = 0.0 self._cur_doc_npmi = 0.0 self.batch_num = 0 initializer(self)