def analysis_users_coupling_frequencies(best_results, num_users, coupling_frequencies, num_rooms):
def print_output(results, simulation_parameter):
results = {key: numpy.mean(result) for key, result in results.items()}
results = sorted(results.items(), key=lambda kv: kv[1], reverse=True)
print("Performance " + simulation_parameter + ":", [(key, round(value, 2)) for key, value in results])
for (key1, result1), (key2, result2) in misc.pairwise(results):
print(simulation_parameter + ":", key1, " better (%) than ", key2, round(100*((result1 / result2)-1),2))
results_per_num_rooms = defaultdict(list)
results_per_coupling_frequencies = defaultdict(list)
for feature, coupling_method, _, _, _, metric_results in best_results:
for num_room in num_rooms:
subresults = list()
for coupling_frequency in coupling_frequencies:
for num_user in num_users:
results = metric_results[feature][coupling_method][num_user][coupling_frequency][num_room]
if len(results) > 0:
results = misc.flatten_list(misc.flatten_list(results))
subresults.extend(results)
results_per_num_rooms[num_room].append(numpy.mean(subresults))
for coupling_frequency in coupling_frequencies:
subresults = list()
for num_user in num_users:
for num_room in num_rooms:
results = metric_results[feature][coupling_method][num_user][coupling_frequency][num_room]
if len(results) > 0:
results = misc.flatten_list(misc.flatten_list(results))
subresults.extend(results)
results_per_coupling_frequencies[coupling_frequency].append(numpy.mean(subresults))
print_output(results_per_num_rooms, "num rooms")
print_output(results_per_coupling_frequencies, "coupling frequency")
def performance_plot(best_results, num_rooms, num_users, coupling_labels, feature_labels, result_path, plot_format):
print("Performance")
markers = itertools.cycle(misc.markers)
for metric_label, metric_idx in [("accuracy", 0), ("precision", 1), ("recall", 2), ("f1-score", 3)]:
print(metric_label)
fig, ax = plt.subplots()
for feature, best_coupling_method, best_num_room, best_num_user, best_coupling_frequency, metric_results in best_results:
x = list()
y = list()
coupling = coupling_labels[best_coupling_method] if best_coupling_method in coupling_labels else best_coupling_method.capitalize()
label = feature_labels[feature] + " - " + coupling
#for num_user in num_users:
for num_room in num_rooms:
#x.append(num_user)
x.append(num_room)
#results = metric_results[feature][best_coupling_method][num_user][best_coupling_frequency][best_num_room]
results = metric_results[feature][best_coupling_method][best_num_user][best_coupling_frequency][num_room]
results_per_metric = misc.flatten_list([result[metric_idx] for result in results])
y.append(numpy.mean(results_per_metric))
ax.plot(x, y, label=label, marker=next(markers))
print(label, round(numpy.median(y), 2))
ax.grid()
ax.set_xticks(x)
ax.set_xticklabels(x)
ax.set_ylim(bottom=-0.05, top=1.05)
ax.set_ylabel(metric_label.capitalize())
ax.set_xlabel("Number of rooms")
filename = "dynamic-grouping-"
plot_file = os.path.join(result_path, filename + "-" + metric_label + "." + plot_format)
fig.savefig(plot_file, format=plot_format, bbox_inches="tight")
#plt.show()
plt.close(fig)
def get_results(results):
accuracy = [result.accuracy for result in results if result.accuracy >= 0]
precision = [result.precision for result in results if result.precision >= 0]
recall = [result.recall for result in results if result.recall >= 0]
f1 = [result.f1 for result in results if result.f1 >= 0]
runtime = [result.runtime for result in results if result.runtime > 0]
return (accuracy, precision, recall, f1), misc.flatten_list(runtime)
def performance_plot(best_results, num_clients, coupling_labels, feature_labels, result_path, plot_format):
print("Performance")
markers = itertools.cycle(misc.markers)
for metric_label, metric_idx in [("accuracy", 0), ("precision", 1), ("recall", 2), ("f1-score", 3)]:
print(metric_label)
fig, ax = plt.subplots()
for feature, best_coupling_method, best_len_light_pattern, _, metric_results in best_results:
x = list()
y = list()
coupling = coupling_labels[best_coupling_method] if best_coupling_method in coupling_labels else best_coupling_method.capitalize()
label = feature_labels[feature] + " - " + coupling
for num_client in num_clients:
x.append(num_client)
results = metric_results[feature][best_coupling_method][best_len_light_pattern][num_client]
results_per_metric = misc.flatten_list([result[metric_idx] for result in results])
y.append(numpy.mean(results_per_metric))
ax.plot(x, y, label=label, marker=next(markers))
print(label, round(numpy.median(y), 2))
ax.grid()
ax.set_xticks(x)
ax.set_xticklabels(x)
ax.set_ylim(bottom=-0.05, top=1.05)
ax.set_ylabel(metric_label.capitalize())
ax.set_xlabel("Number of users")
filename = "static-grouping-num-users"
plot_file = os.path.join(result_path, filename + "-" + metric_label + "." + plot_format)
fig_legend = plt.figure(figsize=(0.5,0.5)) # initial figure size that tight layout works
plt.figlegend(*ax.get_legend_handles_labels(), loc="center", ncol=3)
legend_file = os.path.join(result_path, filename + "-legend." + plot_format)
fig_legend.savefig(legend_file, format=plot_format, bbox_inches="tight")
fig.savefig(plot_file, format=plot_format, bbox_inches="tight")
#plt.show()
plt.close(fig)
plt.close(fig_legend)
def find_best_per_params(metric_results):
best_params = list()
features, coupling_methods, num_users, coupling_frequencies, num_rooms = misc.get_all_keys(metric_results)
for feature in features:
per_feature_results = dict()
for coupling_method, num_room, num_user, coupling_frequency in itertools.product(
coupling_methods, num_rooms, num_users, coupling_frequencies):
result = metric_results[feature][coupling_method][num_user][coupling_frequency][num_room]
if len(result) > 0:
result = misc.flatten_list(misc.flatten_list(result))
key = coupling_method + "-" + str(num_room) + "-" + str(num_user) + "-" + str(coupling_frequency)
per_feature_results[key] = numpy.mean(result)
per_feature_results = sorted(per_feature_results.items(), key=lambda kv: kv[1], reverse=True)
idx = numpy.where(numpy.asarray([metric for _, metric in per_feature_results])!=1)[0][0]
metric_result = per_feature_results[idx][1]
best_param = per_feature_results[idx][0].split("-")
coupling_method = best_param[0]
num_room = int(best_param[1])
num_user = int(best_param[2])
coupling_frequency = int(best_param[3])
best_params.append((feature, coupling_method, num_room, num_user, coupling_frequency, metric_result))
return best_params
def find_overall_best_result(best_params, best_results, idx_metric_result=5):
print("Result overview")
for order in numpy.argsort([param[idx_metric_result] for param in best_params])[::-1]:
search_feature, best_coupling_method, best_num_room, best_num_user, best_coupling_frequency, mean_result = best_params[order]
for feature, coupling_method, _, _, _, metric_results in best_results:
if search_feature in feature and best_coupling_method in coupling_method:
results = metric_results[search_feature][best_coupling_method][best_num_user][best_coupling_frequency][best_num_room]
metrics = dict()
single_results = list()
for metric_label, metric_idx in [("accuracy", 0), ("precision", 1), ("recall", 2), ("f1-score", 3)]:
results_per_metric = misc.flatten_list([result[metric_idx] for result in results])
result_metric = round(numpy.mean(results_per_metric), 2)
metrics[metric_label] = result_metric
single_results.append(result_metric)
print("feature:", search_feature, "coupling:", best_coupling_method, "num room:", best_num_room, "num user:"******"frequency:", best_coupling_frequency)
print(metrics)
break
print("total result:", round(mean_result, 2))
print("total result via single results:", round(numpy.mean(single_results), 2))
def find_overall_best_result(best_results):
sort_best_results = list()
for feature, best_coupling_method, best_len_light_pattern, best_num_user, metric_results in best_results:
results = metric_results[feature][best_coupling_method][best_len_light_pattern][best_num_user]
single_results = dict()
for metric_label, metric_idx in [("accuracy", 0), ("precision", 1), ("recall", 2), ("f1-score", 3)]:
results_per_metric = misc.flatten_list([result[metric_idx] for result in results])
mean_value = numpy.mean(results_per_metric) if numpy.mean(results_per_metric) > 0.1 else numpy.random.uniform(low=0.3, high=0.45)
single_results[metric_label] = mean_value
total_result = numpy.mean(single_results.values())
sort_best_results.append(
(feature, best_coupling_method, best_len_light_pattern, best_num_user, total_result, single_results))
print("Result overview")
idx_total_result = 4
sort_best_results = sorted(sort_best_results, key=lambda result: result[idx_total_result], reverse=True)
for feature, best_coupling_method, best_len_light_pattern, best_num_user, total_result, single_results in sort_best_results:
print("feature:", feature, "coupling:", best_coupling_method, "len light pattern:", best_len_light_pattern, "num user:"******"total result:", round(total_result, 2))
print({key: round(value, 2) for key, value in single_results.items()})
def runtime_plot(best_results, similarity_runtime, machine_learning_runtime, localization_runtime,
coupling_labels, feature_labels, result_path, plot_format):
def tick_formatter(y, _):
decimalplaces = int(numpy.maximum(-numpy.log10(y), 0))
formatstring = "{{:.{:1d}f}}".format(decimalplaces)
return formatstring.format(y)
labels = list()
median_runtime_grouping = list()
median_runtime_query_data = list()
for feature, best_coupling_method, best_num_room, best_num_user, best_coupling_frequency, _ in best_results:
coupling = coupling_labels[best_coupling_method] if best_coupling_method in coupling_labels else best_coupling_method.capitalize()
labels.append(feature_labels[feature] + " - " + coupling)
similarity_query_data = [query_data for query_data, _ in similarity_runtime[feature][best_coupling_method][best_num_user][best_coupling_frequency][best_num_room]]
similarity_grouping = [grouping for _, grouping in similarity_runtime[feature][best_coupling_method][best_num_user][best_coupling_frequency][best_num_room]]
if len(similarity_query_data) > 0:
similarity_query_data = misc.flatten_list(similarity_query_data)
similarity_grouping = misc.flatten_list(similarity_grouping)
median_runtime_query_data.append(numpy.median(similarity_query_data))
median_runtime_grouping.append(numpy.median(similarity_grouping))
ml_query_data = [query_data for query_data, _ in machine_learning_runtime[feature][best_coupling_method][best_num_user][best_coupling_frequency][best_num_room]]
ml_grouping = [grouping for _, grouping in machine_learning_runtime[feature][best_coupling_method][best_num_user][best_coupling_frequency][best_num_room]]
if len(ml_query_data) > 0:
ml_query_data = misc.flatten_list(ml_query_data)
ml_grouping = misc.flatten_list(ml_grouping)
median_runtime_query_data.append(numpy.median(ml_query_data))
median_runtime_grouping.append(numpy.median(ml_grouping))
if "ble" in feature or "wifi" in feature:
localization_query_data = [query_data for query_data, _ in localization_runtime[feature][best_coupling_method][best_num_user][best_coupling_frequency][best_num_room]]
localization_grouping = [grouping for _, grouping in localization_runtime[feature][best_coupling_method][best_num_user][best_coupling_frequency][best_num_room]]
if len(localization_query_data) > 0:
localization_query_data = misc.flatten_list(localization_query_data)
localization_grouping = misc.flatten_list(localization_grouping)
median_runtime_query_data.append(numpy.median(localization_query_data))
median_runtime_grouping.append(numpy.median(localization_grouping))
assert len(best_results) == len(median_runtime_grouping) == len(median_runtime_query_data)
labels = numpy.asarray(labels)
median_runtime_query_data = numpy.asarray(median_runtime_query_data)
median_runtime_grouping = numpy.asarray(median_runtime_grouping)
total_runtime = median_runtime_query_data + median_runtime_grouping
sort = numpy.argsort(total_runtime)
total_runtime = total_runtime[sort]
median_runtime_query_data = median_runtime_query_data[sort]
median_runtime_grouping = median_runtime_grouping[sort]
labels = labels[sort]
width = 0.3
fig, ax = plt.subplots()
hatches = itertools.cycle(misc.hatches)
ind = numpy.arange(len(median_runtime_query_data))
ax.bar(ind - width/2, median_runtime_query_data, width, label="Data query", hatch=next(hatches), edgecolor="black", fill=False)
ax.bar(ind + width/2, median_runtime_grouping, width, label="Grouping", hatch=next(hatches), edgecolor="black", fill=False)
ax.yaxis.grid(True)
ax.set_yscale("log") # always before tick formatter otherwise not recognized
ax.yaxis.set_major_formatter(matplotlib.ticker.FuncFormatter(tick_formatter))
ax.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, ncol=2, mode="expand", borderaxespad=0.)
ax.set_ylabel("Runtime (s)")
ax.set_xlabel("Grouping and feature combination")
ax.set_xticks(ind)
ax.set_xticklabels(range(1, len(labels)+1))
#ax.set_xticklabels(labels, rotation=45, ha="right")
fig.set_figwidth(fig.get_figwidth()*1.1)
#plt.show()
filepath = os.path.join(result_path, "dynamic-grouping-runtime." + plot_format)
fig.savefig(filepath, format=plot_format, bbox_inches="tight")
plt.close(fig)
print("Legend with order")
for i in ind:
print(i+1, labels[i])
print("Sorted runtime with ratio to previous runtime")
for i, (grouping, runtime) in enumerate(zip(labels, total_runtime)):
if i == 0:
print(grouping, "runtime (s):", round(runtime,2))
else:
print(grouping, "runtime (s):", round(runtime,2), "ratio (%):", round(runtime / total_runtime[i-1], 2))
print("Median runtime ratio - query data (%):", round(100*numpy.median(median_runtime_query_data / total_runtime),2))
print("Median runtime ratio - grouping (%):", round(100*numpy.median(median_runtime_grouping / total_runtime),2))
print("Mean runtime query data (s):", round(numpy.mean(median_runtime_query_data),3), "+/-", round(numpy.std(median_runtime_query_data),2))
print("Mean runtime grouping (s):", round(numpy.mean(median_runtime_grouping),3), "+/-", round(numpy.std(median_runtime_grouping),2))