def run_sequenced_benchmark( api, s3, title_prefix, name_prefix, function, pauses=None, measurement_count=SEQUENCED_BENCHMARKS_DEFAULT_COUNT, compute_stack_analysis_jobs_durations=False): """Start benchmarks by calling selected function sequentially.""" pauses = pauses or [10] print("pauses: {p}".format(p=pauses)) print("measurement_count: {c}".format(c=measurement_count)) stack_analysis_jobs_durations = None # for the stack analysis we are able to compute statistic for each job if compute_stack_analysis_jobs_durations: stack_analysis_jobs_durations = {} stack_analysis_jobs_durations_min_times = {} stack_analysis_jobs_durations_max_times = {} stack_analysis_jobs_durations_avg_times = {} for job_name in STACK_ANALYSIS_JOB_NAMES: stack_analysis_jobs_durations[job_name] = [] stack_analysis_jobs_durations_min_times[job_name] = [] stack_analysis_jobs_durations_max_times[job_name] = [] stack_analysis_jobs_durations_avg_times[job_name] = [] measurements = [] min_times = [] max_times = [] avg_times = [] for pause in pauses: if len(pauses) > 1: title = "{t}, {s} seconds between calls".format(t=title_prefix, s=pause) name = "{n}_{s}_pause_time".format(n=name_prefix, s=pause) else: title = "{t}".format(t=title_prefix) name = "{n}".format(n=name_prefix) print(" " + title) values, debug = function(api, s3, measurement_count, pause) deltas = [value["delta"] for value in values] graph.generate_wait_times_graph(title, name, deltas) print("Breathe (statistic graph)...") time.sleep(BREATHE_PAUSE) min_times.append(min(deltas)) max_times.append(max(deltas)) avg_times.append(sum(deltas) / len(deltas)) measurements.extend(deltas) if compute_stack_analysis_jobs_durations: for job_name in STACK_ANALYSIS_JOB_NAMES: durations = job_durations(job_name, debug) # all durations for specific jobs need to be stored here stack_analysis_jobs_durations[job_name].extend(durations) # compute statistic cnt = len(durations) stack_analysis_jobs_durations_min_times[job_name].append( min(durations)) stack_analysis_jobs_durations_max_times[job_name].append( max(durations)) stack_analysis_jobs_durations_avg_times[job_name].append( sum(durations) / cnt) print(min_times) print(max_times) print(avg_times) if compute_stack_analysis_jobs_durations: print_job_durations(stack_analysis_jobs_durations, stack_analysis_jobs_durations_min_times, stack_analysis_jobs_durations_max_times, stack_analysis_jobs_durations_avg_times) title = "{t}: min. max. and avg times".format(t=title_prefix) min_max_avg_name = "{n}_min_max_avg_times".format(n=name_prefix) graph.generate_timing_statistic_graph(title, min_max_avg_name, pauses, min_times, max_times, avg_times) export_sequenced_benchmark_into_csv(name, measurements, compute_stack_analysis_jobs_durations, stack_analysis_jobs_durations)
def run_api_concurrent_benchmark(core_api, function_to_call, name_prefix): """Call given API endpoint concurrently.""" measurement_count = 1 min_thread_count = 1 max_thread_count = 2 pauses = [2.0, 1.5, 1.0, 0.5, 0] # 2, 1, 0.5, 0] pauses = [ 0.0, ] summary_min_times = [] summary_max_times = [] summary_avg_times = [] for thread_count in range(min_thread_count, 1 + max_thread_count): min_times = [] max_times = [] avg_times = [] for pause in pauses: threads = [] q = queue.Queue() for thread_id in range(0, thread_count): t = threading.Thread(target=function_to_call, args=(core_api, measurement_count, pause, q, thread_id)) t.start() threads.append(t) wait_for_all_threads(threads) values = sum([q.get() for t in threads], []) title = "core API endpoint, {t} concurrent threads, {s} seconds between calls".format( t=thread_count, s=pause) name = "{p}_concurrent_{t}_threads_{s}_pause_time".format( p=name_prefix, t=thread_count, s=pause) graph.generate_wait_times_graph(title, name, values) min_times.append(min(values)) max_times.append(max(values)) avg_times.append(sum(values) / len(values)) print("Breathe...") time.sleep(BREATHE_PAUSE) print(min_times) print(max_times) print(avg_times) summary_min_times.append(min(values)) summary_max_times.append(max(values)) summary_avg_times.append(sum(values) / len(values)) generate_statistic_graph(name_prefix, thread_count, pauses, min_times, max_times, avg_times) print("Breathe (statistic graph)...") time.sleep(BREATHE_PAUSE) print(summary_min_times) print(summary_max_times) print(summary_avg_times) t = range(min_thread_count, 1 + thread_count) graph.generate_timing_threads_statistic_graph( "Duration for concurrent API calls", "{p}_{t}".format(p=name_prefix, t=thread_count), t, summary_min_times, summary_max_times, summary_avg_times)
def run_analysis_concurrent_benchmark(api, s3, message, name_prefix, function_to_call, thread_counts=None): """Universal function to call any callback function in more threads and collect results.""" thread_counts = thread_counts or [1, 2, 3, 4] print(message + " concurrent benchmark") measurement_count = 1 summary_min_times = [] summary_max_times = [] summary_avg_times = [] for thread_count in thread_counts: print("Concurrent threads: {c}".format(c=thread_count)) min_times = [] max_times = [] avg_times = [] threads = [] q = queue.Queue() for thread_id in range(0, thread_count): t = threading.Thread( target=lambda api, s3, measurement_count, pause_time, q, thread_id: function_to_call( api, s3, measurement_count, pause_time, q, thread_id), args=(api, s3, measurement_count, 0, q, thread_id)) t.start() threads.append(t) print("---------------------------------") print("Waiting for all threads to finish") wait_for_all_threads(threads) print("Done") values = [q.get()[0][0]["delta"] for t in threads] print("values") print(len(values)) print(values) print("----") title = "{n}, {t} concurrent threads".format(n=message, t=thread_count) name = "{n}_{t}_threads".format(n=name_prefix, t=thread_count) graph.generate_wait_times_graph(title, name, values) min_times.append(min(values)) max_times.append(max(values)) avg_times.append(sum(values) / len(values)) print("min_times:", min_times) print("max_times:", max_times) print("avg_times:", avg_times) summary_min_times.append(min(values)) summary_max_times.append(max(values)) summary_avg_times.append(sum(values) / len(values)) generate_statistic_graph(name, thread_count, ["min/avg/max"], min_times, max_times, avg_times) print("Breathe (statistic graph)...") time.sleep(BREATHE_PAUSE) print(summary_min_times) print(summary_max_times) print(summary_avg_times) t = thread_counts graph.generate_timing_threads_statistic_graph("Duration for " + message, "{p}".format(p=name_prefix), t, summary_min_times, summary_max_times, summary_avg_times) with open(name_prefix + ".csv", "w") as csvfile: csv_writer = csv.writer(csvfile) for i in range(0, len(thread_counts)): csv_writer.writerow([ i, thread_counts[i], summary_min_times[i], summary_max_times[i], summary_avg_times[i] ])
def run_component_analysis_concurrent_calls_benchmark(jobs_api, s3): """Call component analysis in more threads and collect results.""" print("Component analysis concurrent benchmark") measurement_count = 1 min_thread_count = 1 max_thread_count = 100 summary_min_times = [] summary_max_times = [] summary_avg_times = [] for thread_count in range(min_thread_count, 1 + max_thread_count): min_times = [] max_times = [] avg_times = [] threads = [] q = queue.Queue() for thread_id in range(0, thread_count): t = threading.Thread( target=lambda api, s3, measurement_count, pause_time, q, thread_id: benchmarks.component_analysis_thread( api, s3, measurement_count, pause_time, q, thread_id), args=(jobs_api, s3, measurement_count, 10, q, thread_id)) t.start() threads.append(t) print("---------------------------------") print("Waiting for all threads to finish") wait_for_all_threads(threads) print("Done") values = sum([q.get() for t in threads], []) print("values") print(len(values)) print(values) print("----") title = "Component analysis, {t} concurrent threads".format( t=thread_count) name = "jobs_flow_scheduling_{t}_threads".format(t=thread_count) graph.generate_wait_times_graph(title, name, values) min_times.append(min(values)) max_times.append(max(values)) avg_times.append(sum(values) / len(values)) print("min_times:", min_times) print("max_times:", max_times) print("avg_times:", avg_times) summary_min_times.append(min(values)) summary_max_times.append(max(values)) summary_avg_times.append(sum(values) / len(values)) generate_statistic_graph("component_analysis", thread_count, [10], min_times, max_times, avg_times) print("Breathe (statistic graph)...") time.sleep(BREATHE_PAUSE) print(summary_min_times) print(summary_max_times) print(summary_avg_times) t = range(min_thread_count, 1 + thread_count) graph.generate_timing_threads_statistic_graph( "Duration for concurrent analysis", "durations_{i}".format(i=thread_count), t, summary_min_times, summary_max_times, summary_avg_times)
def run_sequenced_benchmark(api, s3, title_prefix, name_prefix, function, pauses=None, measurement_count=10, compute_stack_analysis_jobs_durations=False): """Start benchmarks by calling selected function sequentially.""" pauses = pauses or [10] print("pauses:") print(pauses) # for the stack analysis we are able to compute statistic for each job if compute_stack_analysis_jobs_durations: stack_analysis_jobs_durations_min_times = {} stack_analysis_jobs_durations_max_times = {} stack_analysis_jobs_durations_avg_times = {} for job_name in STACK_ANALYSIS_JOB_NAMES: stack_analysis_jobs_durations_min_times[job_name] = [] stack_analysis_jobs_durations_max_times[job_name] = [] stack_analysis_jobs_durations_avg_times[job_name] = [] measurements = [] min_times = [] max_times = [] avg_times = [] for pause in pauses: if len(pauses) > 1: title = "{t}, {s} seconds between calls".format(t=title_prefix, s=pause) name = "{n}_{s}_pause_time".format(n=name_prefix, s=pause) else: title = "{t}".format(t=title_prefix) name = "{n}".format(n=name_prefix) print(" " + title) values, debug = function(api, s3, measurement_count, pause) graph.generate_wait_times_graph(title, name, values) print("Breathe (statistic graph)...") time.sleep(20) min_times.append(min(values)) max_times.append(max(values)) avg_times.append(sum(values) / len(values)) measurements.extend(values) if compute_stack_analysis_jobs_durations: for job_name in STACK_ANALYSIS_JOB_NAMES: durations = job_durations(job_name, debug) cnt = len(durations) stack_analysis_jobs_durations_min_times[job_name].append( min(durations)) stack_analysis_jobs_durations_max_times[job_name].append( max(durations)) stack_analysis_jobs_durations_avg_times[job_name].append( sum(durations) / cnt) print(min_times) print(max_times) print(avg_times) if compute_stack_analysis_jobs_durations: print("stack analysis jobs") for job_name in STACK_ANALYSIS_JOB_NAMES: print(job_name) print(stack_analysis_jobs_durations_min_times[job_name]) print(stack_analysis_jobs_durations_max_times[job_name]) print(stack_analysis_jobs_durations_avg_times[job_name]) title = "{t}: min. max. and avg times".format(t=title_prefix) min_max_avg_name = "{n}_min_max_avg_times".format(n=name_prefix) graph.generate_timing_statistic_graph(title, min_max_avg_name, pauses, min_times, max_times, avg_times) with open(name + ".csv", "w") as csvfile: csv_writer = csv.writer(csvfile) for m in measurements: csv_writer.writerow([m])
def run_api_concurrent_benchmark(core_api, function_to_call, name_prefix): """Call given API endpoint concurrently.""" measurement_count = 1 min_thread_count = 1 max_thread_count = 2 pauses = [2.0, 1.5, 1.0, 0.5, 0] # 2, 1, 0.5, 0] pauses = [0.0, ] summary_min_times = [] summary_max_times = [] summary_avg_times = [] for thread_count in range(min_thread_count, 1 + max_thread_count): min_times = [] max_times = [] avg_times = [] for pause in pauses: threads = [] q = queue.Queue() for thread_id in range(0, thread_count): t = threading.Thread(target=function_to_call, args=(core_api, measurement_count, pause, q, thread_id)) t.start() threads.append(t) wait_for_all_threads(threads) values = sum([q.get() for t in threads], []) title = "core API endpoint, {t} concurrent threads, {s} seconds between calls".format( t=thread_count, s=pause) name = "{p}_concurrent_{t}_threads_{s}_pause_time".format(p=name_prefix, t=thread_count, s=pause) graph.generate_wait_times_graph(title, name, values) min_times.append(min(values)) max_times.append(max(values)) avg_times.append(sum(values) / len(values)) print("Breathe...") time.sleep(BREATHE_PAUSE) print(min_times) print(max_times) print(avg_times) summary_min_times.append(min(values)) summary_max_times.append(max(values)) summary_avg_times.append(sum(values) / len(values)) generate_statistic_graph(name_prefix, thread_count, pauses, min_times, max_times, avg_times) print("Breathe (statistic graph)...") time.sleep(BREATHE_PAUSE) print(summary_min_times) print(summary_max_times) print(summary_avg_times) t = range(min_thread_count, 1 + thread_count) graph.generate_timing_threads_statistic_graph("Duration for concurrent API calls", "{p}_{t}".format(p=name_prefix, t=thread_count), t, summary_min_times, summary_max_times, summary_avg_times)
def run_sequenced_benchmark(api, s3, title_prefix, name_prefix, function, pauses=None, measurement_count=SEQUENCED_BENCHMARKS_DEFAULT_COUNT, compute_stack_analysis_jobs_durations=False): """Start benchmarks by calling selected function sequentially.""" pauses = pauses or [10] print("pauses: {p}".format(p=pauses)) print("measurement_count: {c}".format(c=measurement_count)) stack_analysis_jobs_durations = None # for the stack analysis we are able to compute statistic for each job if compute_stack_analysis_jobs_durations: stack_analysis_jobs_durations = {} stack_analysis_jobs_durations_min_times = {} stack_analysis_jobs_durations_max_times = {} stack_analysis_jobs_durations_avg_times = {} for job_name in STACK_ANALYSIS_JOB_NAMES: stack_analysis_jobs_durations[job_name] = [] stack_analysis_jobs_durations_min_times[job_name] = [] stack_analysis_jobs_durations_max_times[job_name] = [] stack_analysis_jobs_durations_avg_times[job_name] = [] measurements = [] min_times = [] max_times = [] avg_times = [] for pause in pauses: if len(pauses) > 1: title = "{t}, {s} seconds between calls".format(t=title_prefix, s=pause) name = "{n}_{s}_pause_time".format(n=name_prefix, s=pause) else: title = "{t}".format(t=title_prefix) name = "{n}".format(n=name_prefix) print(" " + title) values, debug = function(api, s3, measurement_count, pause) deltas = [value["delta"] for value in values] graph.generate_wait_times_graph(title, name, deltas) print("Breathe (statistic graph)...") time.sleep(BREATHE_PAUSE) min_times.append(min(deltas)) max_times.append(max(deltas)) avg_times.append(sum(deltas) / len(deltas)) measurements.extend(deltas) if compute_stack_analysis_jobs_durations: for job_name in STACK_ANALYSIS_JOB_NAMES: durations = job_durations(job_name, debug) # all durations for specific jobs need to be stored here stack_analysis_jobs_durations[job_name].extend(durations) # compute statistic cnt = len(durations) stack_analysis_jobs_durations_min_times[job_name].append(min(durations)) stack_analysis_jobs_durations_max_times[job_name].append(max(durations)) stack_analysis_jobs_durations_avg_times[job_name].append(sum(durations) / cnt) print(min_times) print(max_times) print(avg_times) if compute_stack_analysis_jobs_durations: print_job_durations(stack_analysis_jobs_durations, stack_analysis_jobs_durations_min_times, stack_analysis_jobs_durations_max_times, stack_analysis_jobs_durations_avg_times) title = "{t}: min. max. and avg times".format(t=title_prefix) min_max_avg_name = "{n}_min_max_avg_times".format(n=name_prefix) graph.generate_timing_statistic_graph(title, min_max_avg_name, pauses, min_times, max_times, avg_times) export_sequenced_benchmark_into_csv(name, measurements, compute_stack_analysis_jobs_durations, stack_analysis_jobs_durations)
def run_component_analysis_concurrent_calls_benchmark(jobs_api, s3): """Call component analysis in more threads and collect results.""" print("Component analysis concurrent benchmark") measurement_count = 1 min_thread_count = 1 max_thread_count = 100 summary_min_times = [] summary_max_times = [] summary_avg_times = [] for thread_count in range(min_thread_count, 1 + max_thread_count): min_times = [] max_times = [] avg_times = [] threads = [] q = queue.Queue() for thread_id in range(0, thread_count): t = threading.Thread(target=lambda api, s3, measurement_count, pause_time, q, thread_id: benchmarks.component_analysis_thread(api, s3, measurement_count, pause_time, q, thread_id), args=(jobs_api, s3, measurement_count, 10, q, thread_id)) t.start() threads.append(t) print("---------------------------------") print("Waiting for all threads to finish") wait_for_all_threads(threads) print("Done") values = sum([q.get() for t in threads], []) print("values") print(len(values)) print(values) print("----") title = "Component analysis, {t} concurrent threads".format( t=thread_count) name = "jobs_flow_scheduling_{t}_threads".format(t=thread_count) graph.generate_wait_times_graph(title, name, values) min_times.append(min(values)) max_times.append(max(values)) avg_times.append(sum(values) / len(values)) print("min_times:", min_times) print("max_times:", max_times) print("avg_times:", avg_times) summary_min_times.append(min(values)) summary_max_times.append(max(values)) summary_avg_times.append(sum(values) / len(values)) generate_statistic_graph("component_analysis", thread_count, [10], min_times, max_times, avg_times) print("Breathe (statistic graph)...") time.sleep(BREATHE_PAUSE) print(summary_min_times) print(summary_max_times) print(summary_avg_times) t = range(min_thread_count, 1 + thread_count) graph.generate_timing_threads_statistic_graph("Duration for concurrent analysis", "durations_{i}".format(i=thread_count), t, summary_min_times, summary_max_times, summary_avg_times)
def run_analysis_concurrent_benchmark(api, s3, message, name_prefix, function_to_call, thread_counts=None): """Universal function to call any callback function in more threads and collect results.""" thread_counts = thread_counts or [1, 2, 3, 4] print(message + " concurrent benchmark") measurement_count = 1 summary_min_times = [] summary_max_times = [] summary_avg_times = [] for thread_count in thread_counts: print("Concurrent threads: {c}".format(c=thread_count)) min_times = [] max_times = [] avg_times = [] threads = [] q = queue.Queue() for thread_id in range(0, thread_count): t = threading.Thread(target=lambda api, s3, measurement_count, pause_time, q, thread_id: function_to_call(api, s3, measurement_count, pause_time, q, thread_id), args=(api, s3, measurement_count, 0, q, thread_id)) t.start() threads.append(t) print("---------------------------------") print("Waiting for all threads to finish") wait_for_all_threads(threads) print("Done") queue_size = q.qsize() check_number_of_results(queue_size, thread_count) # read all really stored results from the queue values = [q.get()[0][0]["delta"] for i in range(queue_size)] print("values") print("count: {cnt}".format(cnt=len(values))) print(values) print("----") title = "{n}, {t} concurrent threads".format(n=message, t=thread_count) name = "{n}_{t}_threads".format(n=name_prefix, t=thread_count) graph.generate_wait_times_graph(title, name, values) min_times.append(min(values)) max_times.append(max(values)) avg_times.append(sum(values) / len(values)) print("min_times:", min_times) print("max_times:", max_times) print("avg_times:", avg_times) summary_min_times.append(min(values)) summary_max_times.append(max(values)) summary_avg_times.append(sum(values) / len(values)) generate_statistic_graph(name, thread_count, ["min/avg/max"], min_times, max_times, avg_times) print("Breathe (statistic graph)...") time.sleep(BREATHE_PAUSE) print(summary_min_times) print(summary_max_times) print(summary_avg_times) t = thread_counts graph.generate_timing_threads_statistic_graph("Duration for " + message, "{p}".format(p=name_prefix), t, summary_min_times, summary_max_times, summary_avg_times) with open(name_prefix + ".csv", "w") as csvfile: csv_writer = csv.writer(csvfile) for i in range(0, len(thread_counts)): csv_writer.writerow([i, thread_counts[i], summary_min_times[i], summary_max_times[i], summary_avg_times[i]])