def test_configure_once_per_benchmark(self):
suite = ClusterSuite(name, cluster, systems, [], benchmarks)
suite.execute()
# test if each system was only configured the necessary amount of times
for system in suite.systems:
self.assertEquals(system.configured, 1)
for benchmark in suite.benchmarks:
for system in benchmark.systems:
self.assertEquals(system.configured, 1)
def test_save(self):
suite = ClusterSuite(name, cluster, systems, [], benchmarks)
# run suite
suite.execute()
#check corresponding db entries
suite_id = suite.id
suite_uid = suite.uid
for b in suite.benchmarks:
data = (suite_uid, suite_id, b.id)
with results.DB() as db:
c = db.cursor()
c.execute("""
SELECT * FROM results
WHERE suite_uid = ? and suite_id = ? and
bench_id = ?
""", data)
self.assertEquals(c.fetchall().__len__(), b.times)
id = "Grep",
systems = [flink],
experiment = Grep(),
times = 3
),
# KMeans
# PageRank
Benchmark(
id = "ALS",
systems = [flink_als],
experiment = ALS(),
times = 3
),
Benchmark(
id = "Avro",
systems = [flink],
experiment = Avro(generators['avro']),
times = 3
)
]
suite = ClusterSuite("NightlySuite", cluster, systems, generators.values(), benchmarks)
suite.execute(retry_setup=0,
shutdown_on_failure=True,
email_results=True)
Benchmark(
id = "Grep",
systems = [flink],
experiment = Grep(),
times = 2
),
Benchmark(
id = "WordCount-new",
systems = [flink],
experiment = WordCount(),
times = 2
),
]
generators = [
Generator(
id = "TextGenerator",
systems = [flink],
experiment = generators.Text(
size_gb = 1,
dop = compute_engine_config['num_workers']
)
)
]
suite = ClusterSuite("DefaultSuite", cluster, systems, generators, benchmarks)
suite.execute()
def test_failed_experiment(self):
suite = ClusterSuite(name, cluster, systems, [], failing_benchmarks)
suite.execute()
def test_failed_shutdown(self):
suite = ClusterSuite(name, failing_running_cluster, systems, [], benchmarks)
try:
suite.execute()
except core.lib.ClusterSetupException:
pass
def test_email_plot(self):
suite = ClusterSuite(name, cluster, systems, [], benchmarks)
suite.execute(email_results=True)
def test_gen_plot(self):
suite = ClusterSuite(name, cluster, systems, [], benchmarks)
suite.execute()
results.gen_plot(name)
custom_flink = Flink(custom_flink_config)
benchmarks = [
# Normal benchmark
Benchmark(id="WordCount1000",
systems=[flink],
experiment=WordCountFromJar({'num_lines': 1000}),
times=5),
# Custom Flink version benchmark
Benchmark(id="WordCount1000-custom",
systems=[custom_flink],
experiment=WordCountFromJar({'num_lines': 1000}),
times=3),
Benchmark(id="Grep", systems=[flink], experiment=Grep(), times=2),
Benchmark(id="WordCount-new",
systems=[flink],
experiment=WordCount(),
times=2),
]
generators = [
Generator(id="TextGenerator",
systems=[flink],
experiment=generators.Text(
size_gb=1, dop=compute_engine_config['num_workers']))
]
suite = ClusterSuite("DefaultSuite", cluster, systems, generators, benchmarks)
suite.execute()
"taskmanager.memory.directAllocation: true"
}]
flink_custom = Flink(flink_config_custom)
systems = [hadoop, flink]
benchmarks = [
Benchmark(id="WordCount-heap",
systems=[flink],
experiment=WordCount(),
times=1),
Benchmark(id="WordCount-offheap",
systems=[flink_custom],
experiment=WordCount(),
times=1),
]
generators = [
Generator(
id="TextGenerator",
systems=[flink],
experiment=generators.Text(
size_gb=150, # 512 gB of data
dop=compute_engine_config['num_workers'] *
compute_engine_config['num_cores']))
]
suite = ClusterSuite("DefaultSuite", cluster, systems, generators, benchmarks)
suite.execute(retry_setup=0, shutdown_on_failure=True, email_results=True)
