def test_benchmark():
find_icon_benchmark = modules.benchmark_pipeline.BenchmarkPipeline()
correctness, avg_time_secs, avg_memory_mibs = find_icon_benchmark.evaluate(
icon_finder_object=icon_finder_shape_context.IconFinderShapeContext())
assert avg_memory_mibs <= 1000
assert avg_time_secs <= 60
assert correctness.accuracy >= 0
assert correctness.precision >= 0
assert correctness.recall >= 0
def test_multi_instance():
find_icon_multi_instance = modules.benchmark_pipeline.BenchmarkPipeline(
tfrecord_path="datasets/small_multi_instance_v2.tfrecord")
# test responsiveness to different desired levels of confidence (from 0 to 1)
correctness, _, _ = find_icon_multi_instance.evaluate(
icon_finder_object=icon_finder_shape_context.IconFinderShapeContext(
clusterer=clustering_algorithms.DBSCANClusterer(),
desired_confidence=0.9),
multi_instance_icon=True)
assert correctness.precision >= 0.7
find_icon_multi_instance = modules.benchmark_pipeline.BenchmarkPipeline(
tfrecord_path="datasets/small_multi_instance_v2.tfrecord")
correctness, _, _ = find_icon_multi_instance.evaluate(
icon_finder_object=icon_finder_shape_context.IconFinderShapeContext(
clusterer=clustering_algorithms.DBSCANClusterer(),
desired_confidence=0.1),
multi_instance_icon=True)
assert correctness.recall >= 0.8
def test_single_instance_benchmark():
find_icon_single_instance = modules.benchmark_pipeline.BenchmarkPipeline(
tfrecord_path="datasets/small_single_instance_v2.tfrecord")
correctness, avg_time_secs, avg_memory_mibs = find_icon_single_instance.evaluate(
icon_finder_object=icon_finder_shape_context.IconFinderShapeContext(
clusterer=clustering_algorithms.DBSCANClusterer()))
# current results to prevent any regressions due to algorithm changes
assert avg_memory_mibs <= 1000
assert avg_time_secs <= 5
assert correctness.accuracy >= 0.8
assert correctness.precision >= 0.8
assert correctness.recall >= 0.8
def dbscan_clustering_optimizer(eps_values: List[float],
min_samples: List[int], tfrecord_path: str,
multi_instance_icon: bool):
"""Plots recall given different DBSCAN clustering hyperparameters.
User can check the plots to visually check what the overall trend is to
determine what the next set of hyperpameters to try next (ie, what direction
the recall is moving). We make a plot of min samples and recall for each
epsilon value fixed, and we also make a final plot of epsilon and recall
using the min sample value that maximizes recall.
Arguments:
eps_values: List of dbscan epsilon hyperparameters to try out.
min_samples: List of dbscan min_sample hyperparameters to try out.
tfrecord_path: The path to the dataset to run this experiment on.
multi_instance_icon: Whether the dataset is single-instance
or multi-instance.
"""
recall_eps = []
for eps in eps_values:
recall_min_samples = []
for samples in min_samples:
icon_finder = icon_finder_shape_context.IconFinderShapeContext(
clusterer=clustering_algorithms.DBSCANClusterer(
eps=eps, min_samples=samples))
benchmark = benchmark_pipeline.BenchmarkPipeline(
tfrecord_path=tfrecord_path)
correctness, _, _ = benchmark.evaluate(
multi_instance_icon=multi_instance_icon,
icon_finder_object=icon_finder)
recall_min_samples.append(correctness.recall)
recall_eps.append(np.max(np.array(min_samples)))
analysis_util.generate_scatterplot(
x=min_samples,
y=recall_min_samples,
title="Effect of min samples on recall (Eps = %d)" % eps,
xlabel="Min samples",
ylabel="Recall",
output_path="min-samples-%d.png" % eps,
connect_points=False)
analysis_util.generate_scatterplot(
x=eps_values,
y=recall_eps,
title="Effect of eps on recall (Min sample = best of " +
" ".join(map(str, min_samples)),
xlabel="Epsilon Value",
ylabel="Recall",
output_path="best-epsilon-recall.png",
connect_points=False)
"""Configuration file for setting default arguments. This is where default configurations should be set and updated. """ from modules import icon_finder_shape_context FIND_ICON_OBJECT = icon_finder_shape_context.IconFinderShapeContext() IOU_THRESHOLD = 0.6 TFRECORD_PATH = "datasets/small_single_instance_v2.tfrecord" OUTPUT_PATH = ""