def setUp(self):
super(FrequencyEstimatorEvaluationAnalyzerTest, self).setUp()
self.sketch_estimator_config_list = [
configs.SketchEstimatorConfig(
name=evaluation_configs.construct_sketch_estimator_config_name(
sketch_name='exact_multi_set',
sketch_config='10000',
estimator_name='lossless'),
sketch_factory=exact_set.ExactMultiSet.get_sketch_factory(),
estimator=exact_set.LosslessEstimator(),
max_frequency=2,
),
]
self.evaluation_config = configs.EvaluationConfig(
name='frequency_end_to_end_test',
num_runs=1,
scenario_config_list=[
configs.ScenarioConfig(
name='homogeneous',
set_generator_factory=(
frequency_set_generator.HomogeneousMultiSetGenerator.
get_generator_factory_with_num_and_size(
universe_size=100,
num_sets=3,
set_size=50,
freq_rates=[5] * 3,
freq_cap=8,
)))
])
self.run_name = 'test_run'
def setUp(self):
super(ReportGeneratorTest, self).setUp()
exact_set_lossless = simulator.SketchEstimatorConfig(
name='exact_set-infty-infty-lossless',
sketch_factory=exact_set.ExactSet.get_sketch_factory(),
estimator=exact_set.LosslessEstimator(),
sketch_noiser=None,
estimate_noiser=None)
exact_set_less_one = simulator.SketchEstimatorConfig(
name='exact_set-infty-infty-less_one',
sketch_factory=exact_set.ExactSet.get_sketch_factory(),
estimator=exact_set.LessOneEstimator(),
sketch_noiser=exact_set.AddRandomElementsNoiser(
num_random_elements=0, random_state=np.random.RandomState()),
estimate_noiser=None)
self.sketch_estimator_config_list = (exact_set_lossless,
exact_set_less_one)
self.evaluation_config = configs.EvaluationConfig(
name='test_evaluation',
num_runs=2,
scenario_config_list=[
configs.ScenarioConfig(
name='ind1',
set_generator_factory=(
set_generator.IndependentSetGenerator.
get_generator_factory_with_num_and_size(
universe_size=10, num_sets=5, set_size=1))),
configs.ScenarioConfig(
name='ind2',
set_generator_factory=(
set_generator.IndependentSetGenerator.
get_generator_factory_with_num_and_size(
universe_size=10, num_sets=5, set_size=1))),
])
self.evaluation_run_name = 'test_run'
def _run_evaluation_and_simulation(out_dir):
self.evaluator = evaluator.Evaluator(
evaluation_config=self.evaluation_config,
sketch_estimator_config_list=self.sketch_estimator_config_list,
run_name=self.evaluation_run_name,
out_dir=out_dir)
self.evaluator()
self.analyzer = analyzer.CardinalityEstimatorEvaluationAnalyzer(
out_dir=out_dir,
evaluation_directory=out_dir,
evaluation_run_name=self.evaluation_run_name,
evaluation_name=self.evaluation_config.name,
estimable_criteria_list=[(0.05, 0.95), (1.01, 0.9)])
self.analyzer()
self.run_evaluation_and_simulation = _run_evaluation_and_simulation
def setUp(self):
super(AnalyzerTest, self).setUp()
exact_set_lossless = configs.SketchEstimatorConfig(
name='exact_set_lossless',
sketch_factory=exact_set.ExactSet.get_sketch_factory(),
estimator=exact_set.LosslessEstimator(),
noiser=None)
exact_set_less_one = configs.SketchEstimatorConfig(
name='exact_set_less_one',
sketch_factory=exact_set.ExactSet.get_sketch_factory(),
estimator=exact_set.LessOneEstimator(),
noiser=exact_set.AddRandomElementsNoiser(
num_random_elements=0, random_state=np.random.RandomState()))
self.sketch_estimator_config_list = (exact_set_lossless,
exact_set_less_one)
self.evaluation_config = configs.EvaluationConfig(
name='test_evaluation',
num_runs=2,
scenario_config_list=[
configs.ScenarioConfig(
name='ind1',
set_generator_factory=(
set_generator.IndependentSetGenerator.
get_generator_factory(universe_size=10,
num_sets=5,
set_size=1))),
configs.ScenarioConfig(
name='ind2',
set_generator_factory=(
set_generator.IndependentSetGenerator.
get_generator_factory(universe_size=10,
num_sets=5,
set_size=1))),
])
self.run_name = 'test_run'
def _get_test_evaluator(out_dir):
return evaluator.Evaluator(
evaluation_config=self.evaluation_config,
sketch_estimator_config_list=self.sketch_estimator_config_list,
run_name=self.run_name,
out_dir=out_dir)
self.get_test_evaluator = _get_test_evaluator
def _get_test_analyzer(out_dir, evaluation_dir):
return analyzer.CardinalityEstimatorEvaluationAnalyzer(
out_dir=out_dir,
evaluation_directory=evaluation_dir,
evaluation_run_name=self.run_name,
evaluation_name=self.evaluation_config.name)
self.get_test_analyzer = _get_test_analyzer
def setUp(self):
super(EvaluatorTest, self).setUp()
exact_set_lossless = simulator.SketchEstimatorConfig(
name='exact_set_lossless',
sketch_factory=exact_set.ExactSet.get_sketch_factory(),
estimator=exact_set.LosslessEstimator())
exact_set_less_one = simulator.SketchEstimatorConfig(
name='exact_set_less_one',
sketch_factory=exact_set.ExactSet.get_sketch_factory(),
estimator=exact_set.LessOneEstimator(),
sketch_noiser=exact_set.AddRandomElementsNoiser(
num_random_elements=0, random_state=np.random.RandomState()))
self.sketch_estimator_config_list = (exact_set_lossless,
exact_set_less_one)
self.evaluation_config = configs.EvaluationConfig(
name='test_evaluation',
num_runs=2,
scenario_config_list=[
configs.ScenarioConfig(
name='ind1',
set_generator_factory=(
set_generator.IndependentSetGenerator.
get_generator_factory_with_num_and_size(
universe_size=10, num_sets=2, set_size=5))),
configs.ScenarioConfig(
name='ind2',
set_generator_factory=(
set_generator.IndependentSetGenerator.
get_generator_factory_with_num_and_size(
universe_size=10, num_sets=2, set_size=5))),
])
self.run_name = 'test_run'
def _get_test_evaluator(out_dir, overwrite=False):
return evaluator.Evaluator(
evaluation_config=self.evaluation_config,
sketch_estimator_config_list=self.sketch_estimator_config_list,
run_name=self.run_name,
out_dir=out_dir,
overwrite=overwrite)
self.get_test_evaluator = _get_test_evaluator