def cluster(ethnicity_model_path,
distance_model_path,
n_jobs,
signature_block=None):
"""Train the clustering model and process the output.
Args:
ethnicity_model_path (str): Full path where ethnicity model is saved.
distance_model_path (str): Full path where distance model is saved.
n_jobs (int): Number of processes to use.
signature_block (str): Signature block indicating which block should be
clustered. If set to None, clustering will run on all blocks.
"""
LOGGER.info("Pulling signatures for block '%s' from ES", signature_block)
signatures = get_signatures(signature_block=signature_block)
input_clusters = get_input_clusters(signatures)
LOGGER.debug(
"Got %s signature_blocks and %s input_clusters",
len(signatures),
len(input_clusters),
)
distance_estimator = DistanceEstimator.get(ethnicity_model_path,
distance_model_path)
clusterer = Clusterer(distance_estimator)
clusterer.load_data(signatures, input_clusters)
LOGGER.info("Starting clustering")
clusterer.fit(n_jobs=n_jobs)
return process_clustering_output(clusterer)
def cluster(
ethnicity_model_path,
distance_model_path,
n_jobs,
signature_block=None,
):
"""Train the clustering model and process the output.
Args:
ethnicity_model_path (str): Full path where ethnicity model is saved.
distance_model_path (str): Full path where distance model is saved.
n_jobs (int): Number of processes to use.
signature_block (str): Signature block indicating which block should be
clustered. If set to None, clustering will run on all blocks.
"""
start_time = datetime.now()
LOGGER.info("Preparing test dataset...")
signatures = get_signatures(signature_block=signature_block)
input_clusters = get_input_clusters(signatures)
LOGGER.info(
"Input data",
signature_block=signature_block,
signatures_count=len(signatures),
curated_signatures_count=len(
[sig for sig in signatures if sig.get("is_curated_author_id")]),
input_clusters_count=len(input_clusters),
input_clusters=input_clusters,
)
load_data_time = datetime.now()
distance_estimator = DistanceEstimator.get(ethnicity_model_path,
distance_model_path)
clusterer = Clusterer(distance_estimator)
clusterer.load_data(signatures, input_clusters)
prepare_clusterer_time = datetime.now()
LOGGER.info("Clustering", signature_block=signature_block)
clusterer.fit(n_jobs=n_jobs)
fit_time = datetime.now()
for phonetic_block, cluster in clusterer.clusterer.clusterers_.items():
LOGGER.info(
"Clustering stats",
load_data_runtime=str(load_data_time - start_time),
prepare_clusterer_runtime=str(prepare_clusterer_time -
load_data_time),
clustering_runtime=str(fit_time - prepare_clusterer_time),
total_runtime=str(fit_time - start_time),
threshold=getattr(cluster, "best_threshold_",
clusterer.clusterer.base_estimator.threshold),
signature_block=phonetic_block,
B3_f_score=cluster.supervised_scoring(clusterer.y, cluster.labels_)
if hasattr(cluster, "supervised_scoring") else None,
)
return process_clustering_output(clusterer)
def cluster_with_evaluation(
ethnicity_model_path,
distance_model_path,
n_jobs,
test_signatures_uuids=None,
):
"""Train the clustering model and process the output.
Args:
ethnicity_model_path (str): Full path where ethnicity model is saved.
distance_model_path (str): Full path where distance model is saved.
n_jobs (int): Number of processes to use.
signature_block (str): Signature block indicating which block should be
clustered. If set to None, clustering will run on all blocks.
test_signatures_uuids (set): Signature uuids which will be used
for model validation.
"""
start_time = datetime.now()
signature_blocks = get_curated_signature_blocks()
labels_train, labels_test, y_train, y_test = (
np.array([]),
np.array([]),
np.array([]),
np.array([]),
)
statistics_names = ("precision", "recall", "f1")
for clustered_blocks, block in enumerate(signature_blocks, 1):
LOGGER.info("Clustering a new block",
current=clustered_blocks,
total=(len(signature_blocks) + 1))
test_signatures = []
test_authors_ids = []
signatures = get_signatures(signature_block=block, only_curated=True)
input_clusters_with_all_labels = get_input_clusters(signatures)
for signature in signatures:
if signature.signature_uuid in test_signatures_uuids:
test_authors_ids.append(signature.author_id)
signature.author_id = None
test_signatures.append(signature.signature_uuid)
input_clusters = get_input_clusters(signatures)
test_labels = []
for cluster in input_clusters:
for signature in cluster["signature_uuids"]:
if signature in test_signatures:
test_labels.append(cluster["cluster_id"])
LOGGER.info(
"Input data",
signature_block=block,
signatures_count=len(signatures),
input_clusters_count=len(input_clusters),
input_clusters=input_clusters,
)
load_data_time = datetime.now()
distance_estimator = DistanceEstimator.get(ethnicity_model_path,
distance_model_path)
clusterer = Clusterer(distance_estimator)
clusterer.load_data(signatures, input_clusters)
prepare_clusterer_time = datetime.now()
LOGGER.info("Clustering", signature_block=block)
clusterer.fit(n_jobs=n_jobs)
fit_time = datetime.now()
for phonetic_block, cluster in clusterer.clusterer.clusterers_.items():
LOGGER.info(
"Clustering stats",
load_data_runtime=str(load_data_time - start_time),
prepare_clusterer_runtime=str(prepare_clusterer_time -
load_data_time),
clustering_runtime=str(fit_time - prepare_clusterer_time),
total_runtime=str(fit_time - start_time),
threshold=getattr(
cluster,
"best_threshold_",
clusterer.clusterer.base_estimator.threshold,
),
signature_block=phonetic_block,
)
(
labels_train_per_block,
y_train_per_block,
labels_test_per_block,
y_test_per_block,
) = clusterer.prepare_test_data(test_signatures_uuids, test_labels)
(
B3_statistics_all_per_block,
B3_statistics_training_per_block,
B3_statistics_test_per_block,
) = clusterer.score(
labels_train_per_block,
y_train_per_block,
labels_test_per_block,
y_test_per_block,
)
nb_of_clusters_per_author = clusterer.nb_of_clusters_predicted_for_author(
input_clusters_with_all_labels, test_authors_ids)
LOGGER.info(
"Clustering results for block {}".format(block),
train_dataset_size=y_train_per_block.size,
test_dataset_size=y_test_per_block.size,
true_number_of_clusters=np.unique(clusterer.y).size,
predicted_number_of_clusters=np.unique(
clusterer.clusterer.labels_).size,
B3_precision_recall_f_score_all=dict(
zip(statistics_names, B3_statistics_all_per_block)),
B3_precision_recall_f_score_training=dict(
zip(statistics_names, B3_statistics_training_per_block))
if B3_statistics_training_per_block else None,
B3_precision_recall_f_score_test=dict(
zip(statistics_names, B3_statistics_test_per_block))
if B3_statistics_test_per_block else None,
nb_of_clusters_per_author=nb_of_clusters_per_author)
labels_train = np.concatenate(
(labels_train, labels_train_per_block))
y_train = np.concatenate((y_train, y_train_per_block))
labels_test = np.concatenate((labels_test, labels_test_per_block))
y_test = np.concatenate((y_test, y_test_per_block))
B3_statistics_training = b3_precision_recall_fscore(y_train, labels_train)
B3_statistics_test = b3_precision_recall_fscore(y_test, labels_test)
B3_statistics_all = b3_precision_recall_fscore(
np.append(y_train, y_test), np.append(labels_train, labels_test))
LOGGER.info(
"Clustering results for all the blocks",
B3_precision_recall_f_score_all=B3_statistics_all,
B3_statistics_training=B3_statistics_training,
B3_statistics_test=B3_statistics_test,
)