def save_model_topics(
self,
name: str,
model: TopicModel,
topic_scores: List[Dict[str, float]] = None,
phi: pd.DataFrame = None,
dataset: Dataset = None,
theta: bool = False) -> None:
if phi is None:
phi = model.get_phi()
with open(os.path.join(self._path, f'{name}__phi.bin'), 'wb') as f:
f.write(dill.dumps(phi))
if not theta:
pass
else:
try:
theta = model.get_theta(dataset=dataset)
except ValueError:
pass
else:
with open(os.path.join(self._path, f'{name}__theta.bin'), 'wb') as f:
f.write(dill.dumps(theta))
if topic_scores is None:
topic_scores = dict()
with open(os.path.join(self._path, f'{name}__topic_scores.bin'), 'wb') as f:
f.write(dill.dumps(topic_scores))
def call(self, model: TopicModel):
theta = model.get_theta(dataset=self._dataset)
theta.columns = range(len(theta.columns))
objects_clusters = theta.values.argmax(axis=0)
# TODO: or return some numeric?
if len(set(objects_clusters)) == 1:
_Logger.warning(
'Only one unique cluster! Returning None as score value')
return float('nan')
return calinski_harabasz_score(theta.T.values, objects_clusters)
def call(self, model: TopicModel):
theta = model.get_theta(dataset=self._dataset)
theta.columns = range(len(theta.columns))
objects_clusters = theta.values.argmax(axis=0)
# TODO: or return some numeric?
if len(set(objects_clusters)) == 1:
_Logger.warning(
'Only one unique cluster! Returning None as score value')
return float('nan')
return _silhouette_score_by_sampling(
theta.T.values,
objects_clusters,
sample_size=self.sample_size,
batches_number=self.batches_number)
def call(self, model: TopicModel):
theta = model.get_theta(dataset=self._dataset)
phi = model.get_phi(class_ids=self.modalities)
c_m1 = np.linalg.svd(phi, compute_uv=False)
c_m2 = self.document_lengths.dot(theta.T)
c_m2 += 0.0001 # we need this to prevent components equal to zero
if len(c_m1) != phi.shape[1]:
warnings.warn(
f'Phi has {phi.shape[1]} topics'
f' but its SVD resulted in a vector of size {len(c_m1)}!'
f' To work correctly, SpectralDivergenceScore expects to get a vector'
f' of exactly {phi.shape[1]} singular values.')
return 1.0
# we do not need to normalize these vectors
return _symmetric_kl(c_m1, c_m2)
def call(self, model: TopicModel):
theta = model.get_theta(dataset=self._dataset)
T = theta.shape[0]
return _compute_kl(T, theta, self.document_lengths)