def run_sequenced_benchmark(
api,
s3,
title_prefix,
name_prefix,
function,
pauses=None,
measurement_count=SEQUENCED_BENCHMARKS_DEFAULT_COUNT,
compute_stack_analysis_jobs_durations=False):
"""Start benchmarks by calling selected function sequentially."""
pauses = pauses or [10]
print("pauses: {p}".format(p=pauses))
print("measurement_count: {c}".format(c=measurement_count))
stack_analysis_jobs_durations = None
# for the stack analysis we are able to compute statistic for each job
if compute_stack_analysis_jobs_durations:
stack_analysis_jobs_durations = {}
stack_analysis_jobs_durations_min_times = {}
stack_analysis_jobs_durations_max_times = {}
stack_analysis_jobs_durations_avg_times = {}
for job_name in STACK_ANALYSIS_JOB_NAMES:
stack_analysis_jobs_durations[job_name] = []
stack_analysis_jobs_durations_min_times[job_name] = []
stack_analysis_jobs_durations_max_times[job_name] = []
stack_analysis_jobs_durations_avg_times[job_name] = []
measurements = []
min_times = []
max_times = []
avg_times = []
for pause in pauses:
if len(pauses) > 1:
title = "{t}, {s} seconds between calls".format(t=title_prefix,
s=pause)
name = "{n}_{s}_pause_time".format(n=name_prefix, s=pause)
else:
title = "{t}".format(t=title_prefix)
name = "{n}".format(n=name_prefix)
print(" " + title)
values, debug = function(api, s3, measurement_count, pause)
deltas = [value["delta"] for value in values]
graph.generate_wait_times_graph(title, name, deltas)
print("Breathe (statistic graph)...")
time.sleep(BREATHE_PAUSE)
min_times.append(min(deltas))
max_times.append(max(deltas))
avg_times.append(sum(deltas) / len(deltas))
measurements.extend(deltas)
if compute_stack_analysis_jobs_durations:
for job_name in STACK_ANALYSIS_JOB_NAMES:
durations = job_durations(job_name, debug)
# all durations for specific jobs need to be stored here
stack_analysis_jobs_durations[job_name].extend(durations)
# compute statistic
cnt = len(durations)
stack_analysis_jobs_durations_min_times[job_name].append(
min(durations))
stack_analysis_jobs_durations_max_times[job_name].append(
max(durations))
stack_analysis_jobs_durations_avg_times[job_name].append(
sum(durations) / cnt)
print(min_times)
print(max_times)
print(avg_times)
if compute_stack_analysis_jobs_durations:
print_job_durations(stack_analysis_jobs_durations,
stack_analysis_jobs_durations_min_times,
stack_analysis_jobs_durations_max_times,
stack_analysis_jobs_durations_avg_times)
title = "{t}: min. max. and avg times".format(t=title_prefix)
min_max_avg_name = "{n}_min_max_avg_times".format(n=name_prefix)
graph.generate_timing_statistic_graph(title, min_max_avg_name, pauses,
min_times, max_times, avg_times)
export_sequenced_benchmark_into_csv(name, measurements,
compute_stack_analysis_jobs_durations,
stack_analysis_jobs_durations)
def run_api_concurrent_benchmark(core_api, function_to_call, name_prefix):
"""Call given API endpoint concurrently."""
measurement_count = 1
min_thread_count = 1
max_thread_count = 2
pauses = [2.0, 1.5, 1.0, 0.5, 0] # 2, 1, 0.5, 0]
pauses = [
0.0,
]
summary_min_times = []
summary_max_times = []
summary_avg_times = []
for thread_count in range(min_thread_count, 1 + max_thread_count):
min_times = []
max_times = []
avg_times = []
for pause in pauses:
threads = []
q = queue.Queue()
for thread_id in range(0, thread_count):
t = threading.Thread(target=function_to_call,
args=(core_api, measurement_count, pause,
q, thread_id))
t.start()
threads.append(t)
wait_for_all_threads(threads)
values = sum([q.get() for t in threads], [])
title = "core API endpoint, {t} concurrent threads, {s} seconds between calls".format(
t=thread_count, s=pause)
name = "{p}_concurrent_{t}_threads_{s}_pause_time".format(
p=name_prefix, t=thread_count, s=pause)
graph.generate_wait_times_graph(title, name, values)
min_times.append(min(values))
max_times.append(max(values))
avg_times.append(sum(values) / len(values))
print("Breathe...")
time.sleep(BREATHE_PAUSE)
print(min_times)
print(max_times)
print(avg_times)
summary_min_times.append(min(values))
summary_max_times.append(max(values))
summary_avg_times.append(sum(values) / len(values))
generate_statistic_graph(name_prefix, thread_count, pauses, min_times,
max_times, avg_times)
print("Breathe (statistic graph)...")
time.sleep(BREATHE_PAUSE)
print(summary_min_times)
print(summary_max_times)
print(summary_avg_times)
t = range(min_thread_count, 1 + thread_count)
graph.generate_timing_threads_statistic_graph(
"Duration for concurrent API calls", "{p}_{t}".format(p=name_prefix,
t=thread_count),
t, summary_min_times, summary_max_times, summary_avg_times)
def run_analysis_concurrent_benchmark(api,
s3,
message,
name_prefix,
function_to_call,
thread_counts=None):
"""Universal function to call any callback function in more threads and collect results."""
thread_counts = thread_counts or [1, 2, 3, 4]
print(message + " concurrent benchmark")
measurement_count = 1
summary_min_times = []
summary_max_times = []
summary_avg_times = []
for thread_count in thread_counts:
print("Concurrent threads: {c}".format(c=thread_count))
min_times = []
max_times = []
avg_times = []
threads = []
q = queue.Queue()
for thread_id in range(0, thread_count):
t = threading.Thread(
target=lambda api, s3, measurement_count,
pause_time, q, thread_id: function_to_call(
api, s3, measurement_count, pause_time, q, thread_id),
args=(api, s3, measurement_count, 0, q, thread_id))
t.start()
threads.append(t)
print("---------------------------------")
print("Waiting for all threads to finish")
wait_for_all_threads(threads)
print("Done")
values = [q.get()[0][0]["delta"] for t in threads]
print("values")
print(len(values))
print(values)
print("----")
title = "{n}, {t} concurrent threads".format(n=message, t=thread_count)
name = "{n}_{t}_threads".format(n=name_prefix, t=thread_count)
graph.generate_wait_times_graph(title, name, values)
min_times.append(min(values))
max_times.append(max(values))
avg_times.append(sum(values) / len(values))
print("min_times:", min_times)
print("max_times:", max_times)
print("avg_times:", avg_times)
summary_min_times.append(min(values))
summary_max_times.append(max(values))
summary_avg_times.append(sum(values) / len(values))
generate_statistic_graph(name, thread_count, ["min/avg/max"],
min_times, max_times, avg_times)
print("Breathe (statistic graph)...")
time.sleep(BREATHE_PAUSE)
print(summary_min_times)
print(summary_max_times)
print(summary_avg_times)
t = thread_counts
graph.generate_timing_threads_statistic_graph("Duration for " + message,
"{p}".format(p=name_prefix),
t, summary_min_times,
summary_max_times,
summary_avg_times)
with open(name_prefix + ".csv", "w") as csvfile:
csv_writer = csv.writer(csvfile)
for i in range(0, len(thread_counts)):
csv_writer.writerow([
i, thread_counts[i], summary_min_times[i],
summary_max_times[i], summary_avg_times[i]
])
def run_component_analysis_concurrent_calls_benchmark(jobs_api, s3):
"""Call component analysis in more threads and collect results."""
print("Component analysis concurrent benchmark")
measurement_count = 1
min_thread_count = 1
max_thread_count = 100
summary_min_times = []
summary_max_times = []
summary_avg_times = []
for thread_count in range(min_thread_count, 1 + max_thread_count):
min_times = []
max_times = []
avg_times = []
threads = []
q = queue.Queue()
for thread_id in range(0, thread_count):
t = threading.Thread(
target=lambda api, s3, measurement_count, pause_time, q,
thread_id: benchmarks.component_analysis_thread(
api, s3, measurement_count, pause_time, q, thread_id),
args=(jobs_api, s3, measurement_count, 10, q, thread_id))
t.start()
threads.append(t)
print("---------------------------------")
print("Waiting for all threads to finish")
wait_for_all_threads(threads)
print("Done")
values = sum([q.get() for t in threads], [])
print("values")
print(len(values))
print(values)
print("----")
title = "Component analysis, {t} concurrent threads".format(
t=thread_count)
name = "jobs_flow_scheduling_{t}_threads".format(t=thread_count)
graph.generate_wait_times_graph(title, name, values)
min_times.append(min(values))
max_times.append(max(values))
avg_times.append(sum(values) / len(values))
print("min_times:", min_times)
print("max_times:", max_times)
print("avg_times:", avg_times)
summary_min_times.append(min(values))
summary_max_times.append(max(values))
summary_avg_times.append(sum(values) / len(values))
generate_statistic_graph("component_analysis", thread_count, [10],
min_times, max_times, avg_times)
print("Breathe (statistic graph)...")
time.sleep(BREATHE_PAUSE)
print(summary_min_times)
print(summary_max_times)
print(summary_avg_times)
t = range(min_thread_count, 1 + thread_count)
graph.generate_timing_threads_statistic_graph(
"Duration for concurrent analysis",
"durations_{i}".format(i=thread_count), t, summary_min_times,
summary_max_times, summary_avg_times)
def run_sequenced_benchmark(api,
s3,
title_prefix,
name_prefix,
function,
pauses=None,
measurement_count=10,
compute_stack_analysis_jobs_durations=False):
"""Start benchmarks by calling selected function sequentially."""
pauses = pauses or [10]
print("pauses:")
print(pauses)
# for the stack analysis we are able to compute statistic for each job
if compute_stack_analysis_jobs_durations:
stack_analysis_jobs_durations_min_times = {}
stack_analysis_jobs_durations_max_times = {}
stack_analysis_jobs_durations_avg_times = {}
for job_name in STACK_ANALYSIS_JOB_NAMES:
stack_analysis_jobs_durations_min_times[job_name] = []
stack_analysis_jobs_durations_max_times[job_name] = []
stack_analysis_jobs_durations_avg_times[job_name] = []
measurements = []
min_times = []
max_times = []
avg_times = []
for pause in pauses:
if len(pauses) > 1:
title = "{t}, {s} seconds between calls".format(t=title_prefix,
s=pause)
name = "{n}_{s}_pause_time".format(n=name_prefix, s=pause)
else:
title = "{t}".format(t=title_prefix)
name = "{n}".format(n=name_prefix)
print(" " + title)
values, debug = function(api, s3, measurement_count, pause)
graph.generate_wait_times_graph(title, name, values)
print("Breathe (statistic graph)...")
time.sleep(20)
min_times.append(min(values))
max_times.append(max(values))
avg_times.append(sum(values) / len(values))
measurements.extend(values)
if compute_stack_analysis_jobs_durations:
for job_name in STACK_ANALYSIS_JOB_NAMES:
durations = job_durations(job_name, debug)
cnt = len(durations)
stack_analysis_jobs_durations_min_times[job_name].append(
min(durations))
stack_analysis_jobs_durations_max_times[job_name].append(
max(durations))
stack_analysis_jobs_durations_avg_times[job_name].append(
sum(durations) / cnt)
print(min_times)
print(max_times)
print(avg_times)
if compute_stack_analysis_jobs_durations:
print("stack analysis jobs")
for job_name in STACK_ANALYSIS_JOB_NAMES:
print(job_name)
print(stack_analysis_jobs_durations_min_times[job_name])
print(stack_analysis_jobs_durations_max_times[job_name])
print(stack_analysis_jobs_durations_avg_times[job_name])
title = "{t}: min. max. and avg times".format(t=title_prefix)
min_max_avg_name = "{n}_min_max_avg_times".format(n=name_prefix)
graph.generate_timing_statistic_graph(title, min_max_avg_name, pauses,
min_times, max_times, avg_times)
with open(name + ".csv", "w") as csvfile:
csv_writer = csv.writer(csvfile)
for m in measurements:
csv_writer.writerow([m])
def run_api_concurrent_benchmark(core_api, function_to_call, name_prefix):
"""Call given API endpoint concurrently."""
measurement_count = 1
min_thread_count = 1
max_thread_count = 2
pauses = [2.0, 1.5, 1.0, 0.5, 0] # 2, 1, 0.5, 0]
pauses = [0.0, ]
summary_min_times = []
summary_max_times = []
summary_avg_times = []
for thread_count in range(min_thread_count, 1 + max_thread_count):
min_times = []
max_times = []
avg_times = []
for pause in pauses:
threads = []
q = queue.Queue()
for thread_id in range(0, thread_count):
t = threading.Thread(target=function_to_call,
args=(core_api, measurement_count, pause, q, thread_id))
t.start()
threads.append(t)
wait_for_all_threads(threads)
values = sum([q.get() for t in threads], [])
title = "core API endpoint, {t} concurrent threads, {s} seconds between calls".format(
t=thread_count, s=pause)
name = "{p}_concurrent_{t}_threads_{s}_pause_time".format(p=name_prefix,
t=thread_count,
s=pause)
graph.generate_wait_times_graph(title, name, values)
min_times.append(min(values))
max_times.append(max(values))
avg_times.append(sum(values) / len(values))
print("Breathe...")
time.sleep(BREATHE_PAUSE)
print(min_times)
print(max_times)
print(avg_times)
summary_min_times.append(min(values))
summary_max_times.append(max(values))
summary_avg_times.append(sum(values) / len(values))
generate_statistic_graph(name_prefix, thread_count, pauses,
min_times, max_times, avg_times)
print("Breathe (statistic graph)...")
time.sleep(BREATHE_PAUSE)
print(summary_min_times)
print(summary_max_times)
print(summary_avg_times)
t = range(min_thread_count, 1 + thread_count)
graph.generate_timing_threads_statistic_graph("Duration for concurrent API calls",
"{p}_{t}".format(p=name_prefix, t=thread_count),
t,
summary_min_times,
summary_max_times,
summary_avg_times)
def run_sequenced_benchmark(api, s3, title_prefix, name_prefix, function,
pauses=None, measurement_count=SEQUENCED_BENCHMARKS_DEFAULT_COUNT,
compute_stack_analysis_jobs_durations=False):
"""Start benchmarks by calling selected function sequentially."""
pauses = pauses or [10]
print("pauses: {p}".format(p=pauses))
print("measurement_count: {c}".format(c=measurement_count))
stack_analysis_jobs_durations = None
# for the stack analysis we are able to compute statistic for each job
if compute_stack_analysis_jobs_durations:
stack_analysis_jobs_durations = {}
stack_analysis_jobs_durations_min_times = {}
stack_analysis_jobs_durations_max_times = {}
stack_analysis_jobs_durations_avg_times = {}
for job_name in STACK_ANALYSIS_JOB_NAMES:
stack_analysis_jobs_durations[job_name] = []
stack_analysis_jobs_durations_min_times[job_name] = []
stack_analysis_jobs_durations_max_times[job_name] = []
stack_analysis_jobs_durations_avg_times[job_name] = []
measurements = []
min_times = []
max_times = []
avg_times = []
for pause in pauses:
if len(pauses) > 1:
title = "{t}, {s} seconds between calls".format(t=title_prefix, s=pause)
name = "{n}_{s}_pause_time".format(n=name_prefix, s=pause)
else:
title = "{t}".format(t=title_prefix)
name = "{n}".format(n=name_prefix)
print(" " + title)
values, debug = function(api, s3, measurement_count, pause)
deltas = [value["delta"] for value in values]
graph.generate_wait_times_graph(title, name, deltas)
print("Breathe (statistic graph)...")
time.sleep(BREATHE_PAUSE)
min_times.append(min(deltas))
max_times.append(max(deltas))
avg_times.append(sum(deltas) / len(deltas))
measurements.extend(deltas)
if compute_stack_analysis_jobs_durations:
for job_name in STACK_ANALYSIS_JOB_NAMES:
durations = job_durations(job_name, debug)
# all durations for specific jobs need to be stored here
stack_analysis_jobs_durations[job_name].extend(durations)
# compute statistic
cnt = len(durations)
stack_analysis_jobs_durations_min_times[job_name].append(min(durations))
stack_analysis_jobs_durations_max_times[job_name].append(max(durations))
stack_analysis_jobs_durations_avg_times[job_name].append(sum(durations) / cnt)
print(min_times)
print(max_times)
print(avg_times)
if compute_stack_analysis_jobs_durations:
print_job_durations(stack_analysis_jobs_durations, stack_analysis_jobs_durations_min_times,
stack_analysis_jobs_durations_max_times,
stack_analysis_jobs_durations_avg_times)
title = "{t}: min. max. and avg times".format(t=title_prefix)
min_max_avg_name = "{n}_min_max_avg_times".format(n=name_prefix)
graph.generate_timing_statistic_graph(title, min_max_avg_name,
pauses, min_times, max_times, avg_times)
export_sequenced_benchmark_into_csv(name, measurements,
compute_stack_analysis_jobs_durations,
stack_analysis_jobs_durations)
def run_component_analysis_concurrent_calls_benchmark(jobs_api, s3):
"""Call component analysis in more threads and collect results."""
print("Component analysis concurrent benchmark")
measurement_count = 1
min_thread_count = 1
max_thread_count = 100
summary_min_times = []
summary_max_times = []
summary_avg_times = []
for thread_count in range(min_thread_count, 1 + max_thread_count):
min_times = []
max_times = []
avg_times = []
threads = []
q = queue.Queue()
for thread_id in range(0, thread_count):
t = threading.Thread(target=lambda api, s3, measurement_count, pause_time, q,
thread_id:
benchmarks.component_analysis_thread(api, s3,
measurement_count,
pause_time, q, thread_id),
args=(jobs_api, s3, measurement_count, 10, q, thread_id))
t.start()
threads.append(t)
print("---------------------------------")
print("Waiting for all threads to finish")
wait_for_all_threads(threads)
print("Done")
values = sum([q.get() for t in threads], [])
print("values")
print(len(values))
print(values)
print("----")
title = "Component analysis, {t} concurrent threads".format(
t=thread_count)
name = "jobs_flow_scheduling_{t}_threads".format(t=thread_count)
graph.generate_wait_times_graph(title, name, values)
min_times.append(min(values))
max_times.append(max(values))
avg_times.append(sum(values) / len(values))
print("min_times:", min_times)
print("max_times:", max_times)
print("avg_times:", avg_times)
summary_min_times.append(min(values))
summary_max_times.append(max(values))
summary_avg_times.append(sum(values) / len(values))
generate_statistic_graph("component_analysis", thread_count, [10],
min_times, max_times, avg_times)
print("Breathe (statistic graph)...")
time.sleep(BREATHE_PAUSE)
print(summary_min_times)
print(summary_max_times)
print(summary_avg_times)
t = range(min_thread_count, 1 + thread_count)
graph.generate_timing_threads_statistic_graph("Duration for concurrent analysis",
"durations_{i}".format(i=thread_count), t,
summary_min_times,
summary_max_times,
summary_avg_times)
def run_analysis_concurrent_benchmark(api, s3, message, name_prefix, function_to_call,
thread_counts=None):
"""Universal function to call any callback function in more threads and collect results."""
thread_counts = thread_counts or [1, 2, 3, 4]
print(message + " concurrent benchmark")
measurement_count = 1
summary_min_times = []
summary_max_times = []
summary_avg_times = []
for thread_count in thread_counts:
print("Concurrent threads: {c}".format(c=thread_count))
min_times = []
max_times = []
avg_times = []
threads = []
q = queue.Queue()
for thread_id in range(0, thread_count):
t = threading.Thread(target=lambda api, s3, measurement_count, pause_time, q,
thread_id:
function_to_call(api, s3, measurement_count,
pause_time, q, thread_id),
args=(api, s3, measurement_count, 0, q, thread_id))
t.start()
threads.append(t)
print("---------------------------------")
print("Waiting for all threads to finish")
wait_for_all_threads(threads)
print("Done")
queue_size = q.qsize()
check_number_of_results(queue_size, thread_count)
# read all really stored results from the queue
values = [q.get()[0][0]["delta"] for i in range(queue_size)]
print("values")
print("count: {cnt}".format(cnt=len(values)))
print(values)
print("----")
title = "{n}, {t} concurrent threads".format(n=message,
t=thread_count)
name = "{n}_{t}_threads".format(n=name_prefix, t=thread_count)
graph.generate_wait_times_graph(title, name, values)
min_times.append(min(values))
max_times.append(max(values))
avg_times.append(sum(values) / len(values))
print("min_times:", min_times)
print("max_times:", max_times)
print("avg_times:", avg_times)
summary_min_times.append(min(values))
summary_max_times.append(max(values))
summary_avg_times.append(sum(values) / len(values))
generate_statistic_graph(name, thread_count, ["min/avg/max"],
min_times, max_times, avg_times)
print("Breathe (statistic graph)...")
time.sleep(BREATHE_PAUSE)
print(summary_min_times)
print(summary_max_times)
print(summary_avg_times)
t = thread_counts
graph.generate_timing_threads_statistic_graph("Duration for " + message,
"{p}".format(p=name_prefix),
t,
summary_min_times,
summary_max_times,
summary_avg_times)
with open(name_prefix + ".csv", "w") as csvfile:
csv_writer = csv.writer(csvfile)
for i in range(0, len(thread_counts)):
csv_writer.writerow([i, thread_counts[i],
summary_min_times[i], summary_max_times[i], summary_avg_times[i]])