def initialize_with_copying_topics(
dataset: Dataset,
model_number: int,
num_topics: int,
phi: pd.DataFrame,
num_topics_to_copy: int = None,
topic_indices_to_copy: List[int] = None) -> pd.DataFrame:
random = np.random.RandomState(seed=model_number)
if num_topics_to_copy is None and topic_indices_to_copy is None:
raise ValueError(
'Either `num_topics_to_copy` or `topic_indices_to_copy` should be specified!'
)
elif topic_indices_to_copy is None:
topic_indices_to_copy = list(range(len(phi.columns)))
elif num_topics_to_copy is None:
num_topics_to_copy = len(topic_indices_to_copy)
elif num_topics_to_copy != len(topic_indices_to_copy):
raise ValueError(
'If both `num_topics_to_copy` and `topic_indices_to_copy` are specified,'
' they shouldn\'t contradict each other!')
else:
assert False
topics_to_copy = random.choice(topic_indices_to_copy,
size=num_topics_to_copy,
replace=False)
artm_model_template = _get_artm_model_template(dataset, num_topics)
init_phi_utils._copy_phi(artm_model_template, phi.iloc[:, topics_to_copy])
model_template = TopicModel(artm_model=artm_model_template)
return model_template.get_phi()
def _get_phi_template(dataset: Dataset, num_topics: int) -> pd.DataFrame:
artm_model = _get_artm_model_template(dataset, num_topics)
model = TopicModel(artm_model=artm_model)
phi_template = model.get_phi()
del model
del artm_model
return phi_template
def _fit_model_with_scores(topic_model: TopicModel,
dataset: Dataset,
scores: List[BaseScore] = None,
num_fit_iterations: int = 1):
if scores is not None:
for score in scores:
score._attach(topic_model)
topic_model._fit(dataset.get_batch_vectorizer(),
num_iterations=num_fit_iterations)
def call(self, model: TopicModel):
phi = model.get_phi(class_ids=[self.modality])
V, T = phi.shape
D = self.num_docs
# TODO: consider the case of having vector of taus instead
hyperparams = len(model.regularizers)
# than2012 (https://link.springer.com/content/pdf/10.1007/978-3-642-33460-3_37.pdf)
# argues that number of free parameters in LDA and sparse models (such as PLSA)
# should should be calculated differently
if self.consider_sparsity:
N_p = phi.astype(bool).sum().sum() + hyperparams
else:
N_p = (V - 1) * T + hyperparams
ll = get_log_likelihood(model._model, self.modality)
if self.mode == "MDL":
return 0.5 * N_p * np.log(T * D) - ll
if self.mode == "AIC":
return 2 * N_p - 2 * ll
if self.mode == "BIC":
return N_p * np.log(D) - 2 * ll
raise ValueError(
f"Unsupported score type {self.mode}; Supported ones are: AIC/BIC/MDL"
)
def _load_phi(self, num_topics: int, subsample_number: int) -> pd.DataFrame:
topic_model = TopicModel.load(
self._folder_path_model(
num_topics, subsample_number=subsample_number
)
)
return topic_model.get_phi()
def call(self, model: TopicModel):
theta = model.get_theta(dataset=self._dataset)
phi = model.get_phi(class_ids=self.modalities)
c_m1 = np.linalg.svd(phi, compute_uv=False)
c_m2 = self.document_lengths.dot(theta.T)
c_m2 += 0.0001 # we need this to prevent components equal to zero
if len(c_m1) != phi.shape[1]:
warnings.warn(
f'Phi has {phi.shape[1]} topics'
f' but its SVD resulted in a vector of size {len(c_m1)}!'
f' To work correctly, SpectralDivergenceScore expects to get a vector'
f' of exactly {phi.shape[1]} singular values.')
return 1.0
# we do not need to normalize these vectors
return _symmetric_kl(c_m1, c_m2)
def call(self, model: TopicModel):
phi = model.get_phi(class_ids=self.modalities)
relevant_words = self._select_topwords(phi)
loglift = self._compute_lift(phi, relevant_words)
if self.topic_names is not None:
topic_names = self.topic_names
else:
topic_names = model.topic_names
total_loglift = loglift[topic_names]
return float(total_loglift.mean())
def _get_topic_model(dataset: Dataset,
phi: pd.DataFrame = None,
num_topics: int = None,
seed: int = None,
scores: List[BaseScore] = None,
num_safe_fit_iterations: int = 3,
num_processors: int = 3,
cache_theta: bool = False) -> TopicModel:
dictionary = dataset.get_dictionary()
if num_topics is not None and phi is not None:
assert num_topics >= phi.shape[1]
elif num_topics is None and phi is not None:
num_topics = phi.shape[1]
elif num_topics is None and phi is None:
raise ValueError()
topic_names = [f'topic_{i}' for i in range(num_topics)]
if seed is None:
artm_model = artm.ARTM(topic_names=topic_names)
else:
artm_model = artm.ARTM(topic_names=topic_names, seed=seed)
artm_model.num_processors = num_processors
artm_model.initialize(dictionary)
if phi is None:
pass
elif num_safe_fit_iterations is not None and num_safe_fit_iterations > 0:
init_phi_utils._safe_copy_phi(artm_model, phi, dataset,
num_safe_fit_iterations)
else:
init_phi_utils._copy_phi(artm_model, phi)
topic_model = TopicModel(artm_model=artm_model,
model_id='0',
cache_theta=cache_theta,
theta_columns_naming='title')
if scores is not None:
for score in scores:
score._attach(topic_model)
return topic_model
def _get_matrices(model: TopicModel) -> Tuple[np.array, np.array]:
pwt = model.get_phi().values
nwt = model._model.get_phi(model_name=model._model.model_nwt).values
return pwt, nwt
def test_optimize_for_model(self, keep_in_memory, model_family):
# Thetaless currently fails
# see https://github.com/machine-intelligence-laboratory/TopicNet/issues/79
artm_score_name = 'perplexity_score'
artm_score = PerplexityScore(
name=artm_score_name,
class_ids=[self.main_modality, self.other_modality])
custom_score_name = 'diversity_score'
custom_score = DiversityScore(custom_score_name,
class_ids=self.main_modality)
self.text_collection._set_dataset_kwargs(keep_in_memory=keep_in_memory)
min_num_topics = 1
max_num_topics = 2
num_topics_interval = 1
num_fit_iterations = 3
num_search_points = len(
list(range(min_num_topics, max_num_topics + 1,
num_topics_interval)))
num_restarts = 3
experiment_name = model_family.value
experiment_folder = self.working_folder_path
optimizer = OptimizeScoresMethod(
scores=[artm_score, custom_score],
model_family=model_family,
min_num_topics=min_num_topics,
max_num_topics=max_num_topics,
num_topics_interval=num_topics_interval,
num_fit_iterations=num_fit_iterations,
num_restarts=num_restarts,
one_model_num_processors=1,
separate_thread=False,
experiment_name=experiment_name,
experiment_directory=experiment_folder,
)
optimizer.search_for_optimum(text_collection=self.text_collection)
restart_folder_names = os.listdir(experiment_folder)
assert len(restart_folder_names) == num_restarts
for restart_folder_name in restart_folder_names:
assert restart_folder_name.startswith(experiment_name)
restart_folder_path = os.path.join(experiment_folder,
restart_folder_name)
model_folder_names = os.listdir(restart_folder_path)
assert len(model_folder_names) == num_search_points
for model_folder_name in model_folder_names:
topic_model = TopicModel.load(
os.path.join(restart_folder_path, model_folder_name))
assert artm_score_name in topic_model.scores
assert custom_score_name in topic_model.scores
assert len(
topic_model.scores[artm_score_name]) == num_fit_iterations
assert len(topic_model.scores[custom_score_name]) == 1
assert all(
isinstance(v, Number)
for v in topic_model.scores[artm_score_name])
assert all(
isinstance(v, Number)
for v in topic_model.scores[custom_score_name])
def call(self, model: TopicModel):
theta = model.get_theta(dataset=self._dataset)
T = theta.shape[0]
return _compute_kl(T, theta, self.document_lengths)
def _train_models(
self,
text_collection: VowpalWabbitTextCollection,
min_df_rate: float,
max_df_rate: float,
num_topics_to_skip: List[int] = None) -> None:
modalities_to_use = list(text_collection._modalities.keys())
main_modality = text_collection._main_modality
numbers_of_topics = list(range(
self._min_num_topics,
self._max_num_topics + 1,
self._num_topics_interval))
if num_topics_to_skip is not None:
numbers_of_topics = [
n for n in numbers_of_topics if n not in num_topics_to_skip
]
num_topics_for_message = ', '.join(
[str(n) for n in numbers_of_topics[:10]]
)
print(f'\n Folder for models saving: "{self._models_folder_path}"')
print(
f'Training models for {len(numbers_of_topics)}'
f' numbers of topics: {num_topics_for_message}...'
)
for num_topics in tqdm(
numbers_of_topics,
total=len(numbers_of_topics),
file=sys.stdout):
os.makedirs(
self._folder_path_num_topics(num_topics)
)
subsample_data_paths = self._get_dataset_subsample_file_paths()
for subsample_number, data_path in tqdm(
enumerate(subsample_data_paths),
total=len(subsample_data_paths),
file=sys.stdout):
dataset = Dataset(data_path=data_path)
dictionary = dataset.get_dictionary()
dictionary.filter(
min_df_rate=min_df_rate,
max_df_rate=max_df_rate,
)
artm_model = init_model_from_family(
family=self._model_family,
dataset=dataset,
modalities_to_use=modalities_to_use,
main_modality=main_modality,
num_topics=num_topics,
seed=self._model_seed,
num_processors=self._model_num_processors,
model_params=self._model_params,
)
topic_model = TopicModel(artm_model)
topic_model._fit(
dataset_trainable=dataset.get_batch_vectorizer(),
num_iterations=self._num_fit_iterations,
)
model_save_path = self._folder_path_model(num_topics, subsample_number)
topic_model.save(
model_save_path=model_save_path,
phi=True,
theta=False,
)
def compute(self,
model: TopicModel,
topics: List[str] = None,
documents: List[str] = None) -> Dict[str, float]:
phi = model.get_phi()
if topics is not None:
pass
if self._topics is not None:
topics = self._topics
else:
topics = list(phi.columns)
if self._modalities is not None:
# As self._modalities is list, here always will be df with multiIndex
subphi = model.get_phi().loc[self._modalities, topics]
else:
subphi = model.get_phi().loc[:, topics]
vocabulary_size = subphi.shape[0]
topic_coherences = dict()
if self._active_topic_threshold is None:
pass
else:
# TODO: can't do without transform here, cache theta didn't help
theta = model._model.transform(
self._dataset.get_batch_vectorizer())
subtheta_values = theta.loc[topics, :].values
max_probs = np.max(subtheta_values, axis=1)
active_topic_indices = np.where(
max_probs > self._active_topic_threshold)[0]
topics = [
t for i, t in enumerate(topics) if i in active_topic_indices
]
for topic in topics:
topic_column = subphi.loc[:, topic]
if not self._kernel:
tokens = topic_column\
.sort_values(ascending=False)[:self._num_top_tokens]\
.index\
.get_level_values(1)\
.to_list()
else:
# if self._num_top_tokens is None — also Ok
tokens = topic_column[topic_column > 1.0 / vocabulary_size][:self._num_top_tokens]\
.index\
.get_level_values(1)\
.to_list()
current_cooc_values = list()
for token_a, token_b in combinations(tokens, 2):
if (token_a, token_b) in self._cooc_values:
current_cooc_values.append(self._cooc_values[(token_a,
token_b)])
elif (token_b, token_a) in self._cooc_values:
current_cooc_values.append(self._cooc_values[(token_b,
token_a)])
else:
_logger.warning(
f'Cooc pair "{token_a}, {token_b}" not found in the provided data!'
f' Using zero 0 for this pair as cooc value')
current_cooc_values.append(0)
if len(current_cooc_values) > 0:
topic_coherences[topic] = float(np.mean(current_cooc_values))
else:
# TODO: warn?
topic_coherences[topic] = 0.0
return topic_coherences
def _get_matrices(self, model: TopicModel) -> Tuple[np.array, np.array]:
pwt = model.get_phi(class_ids=self._class_ids).values
nwt = model._model.get_phi(model_name=model._model.model_nwt).values
return pwt, nwt