def aggregate() -> str:
"""Plots IID and Non-IID dataset performance comparision
Expects FLAGS.group_name to be set
Returns:
str: Absolut path to saved plot
"""
group_name = FLAGS.group_name
dname = storage.create_output_subdir(group_name)
fname = storage.fname_with_default_dir("plot_final_task_accuracies.png", dname)
(data, xticks_args) = _prepare_aggregation_data(group_name)
assert len(data) == 2, "Expecting a list of two curves"
fpath = plot(
data,
title="", # TODO
xlabel="IID / Non-IID",
ylabel="Accuracy",
fname=fname,
ylim_max=1.0,
xlim_max=12,
xticks_args=xticks_args,
legend_loc="upper right",
)
logger.info("Data plotted and saved in file", filepath=fpath)
return fpath
def aggregate() -> str:
"""Plots task accuracies for all federated tasks in a group
Expects FLAGS.group_name to be set
Returns:
str: Absolut path to saved plot
"""
group_name = FLAGS.group_name
dname = storage.create_output_subdir(group_name)
fname = storage.fname_with_default_dir("plot_task_accuracies.png", dname)
data = _prepare_aggregation_data(group_name)
# Take highest length of values list as xlim_max
xlim_max = max([len(values) for _, values, _ in data]) + 1
fpath = plot(
data,
title="", # TODO
ylabel="Accuracy",
fname=fname,
ylim_max=1.0,
xlim_max=xlim_max,
)
logger.info("Data plotted and saved in file", filepath=fpath)
return fpath
def participant_history() -> List[str]:
"""Plot participant selection histories for group name flag.
For each task result in the group name flag extract the task metrics (number of
participants, task label, hist metrics), transform them into heatmap data as participant
indices x training rounds and plot/save them as participant selection history.
Returns:
~typing.List[str]: File paths for saved plots.
"""
group_name: str = FLAGS.group_name
dir_name: str = create_output_subdir(dname=group_name)
file_pre_name: str = fname_with_default_dir(
fname="plot_participant_history_{}.png", dname=dir_name)
file_paths: List[str] = list()
# Getting history metrics data from results.json
hist_metrics_group: List[Tuple[int, str,
List[List[Metrics]]]] = get_hist_metrics(
group_name=group_name)
# Creates heatmap data for each task metric in group metrics
matrices: List[Tuple[str,
ndarray]] = list(map(heatmap_data,
hist_metrics_group))
for task_matrix in matrices:
label: str = task_matrix[0]
matrix: ndarray = task_matrix[1]
file_path: str = plot_history_data(
matrix=matrix,
title="Participant Selection History",
file_name=file_pre_name.format(label),
save=True,
show=False,
)
file_paths.append(file_path)
logger.info("Task data plotted and saved in file", filepath=file_paths)
return file_paths
def aggregate() -> str:
"""Plots learning rate for federated tasks in a group
Expects FLAGS.group_name to be set
Returns:
str: Absolut path to saved plot
"""
group_name = FLAGS.group_name
dname = storage.create_output_subdir(group_name)
fname = storage.fname_with_default_dir("plot_learning_rates.png", dname)
data = _prepare_aggregation_data(group_name)
ylim_max: float = 0
xlim_max = 0
for _, lrs, ylabel in data:
if ylabel is not None:
xlim_max = max(ylabel + [xlim_max])
for lr in lrs:
ylim_max = max(lr, ylim_max)
ylim_max *= 1.1
xlim_max += 1
assert data, "Expecting a list with at least one item"
fpath = plot(
data,
title="Optimizer learning rates for federated training tasks",
xlabel="round",
ylabel="learning rate",
fname=fname,
ylim_max=ylim_max,
xlim_max=xlim_max,
legend_loc="upper right",
)
logger.info("Data plotted and saved in file", filepath=fpath)
return fpath
def _plot_fashion_mnist_dist():
dists = fashion_mnist_100p()
xs = np.arange(100)
plt.figure()
legend = []
for b, dist in dists:
legend.append(str(b))
plt.plot(xs, np.array(dist), "o", markersize=1.0)
plt.legend(legend, loc="upper left")
plt.xlabel("Partition ID")
plt.ylabel("Examples")
dname = storage.create_output_subdir("partition_volume_distributions")
fname = storage.fname_with_default_dir("plot-part-vol.png", dname)
plt.savefig(fname=fname, format=FORMAT)
# FIXME: Matplotlib is currently using agg, which is a non-GUI
# backend, so cannot show the figure.
# plt.show()
return fname
def plot(
data: List[PlotValues],
title: Optional[str] = None,
xlabel: str = "Epoch",
ylabel: str = None,
fname: Optional[str] = None,
save: bool = True,
show: bool = False,
ylim_max: float = 1.0,
xlim_max: float = 40.0,
xticks_args: Optional[Tuple[List[int], List[str]]] = None,
legend_loc: str = "lower right",
vline: bool = False,
) -> str:
"""Wrapper for plt.plot so have uniform plots in the project
Args:
data (List[Tuple[name, values, Optional[indices]]]): List of tuples where each
represents a line in the plot with tuple
beeing (name, values, Optional[indices])
save (bool): Indicates if plot should be saved to disk. Defaults to True
Returns:
str: For save=True returns absolut path to saved file otherwise None
"""
assert fname is not None
# if fname is an absolute path use fname directly otherwise assume
# fname is filename and prepend output_dir
fname_abspath = storage.fname_with_default_dir(fname, FLAGS.output_dir)
plt.figure()
plt.ylim(0.0, ylim_max)
plt.xlim(0.0, xlim_max)
if xticks_args is not None:
xticks_locations, xticks_labels = xticks_args
# if any label has length > 3 rotate labels by 90 degrees
rot = 90 if any([len(l) > 3 for l in xticks_labels]) else 0
plt.xticks(xticks_locations, xticks_labels, rotation=rot)
if title is not None:
plt.title(title)
plt.xlabel(xlabel)
plt.ylabel(ylabel)
if vline:
plt.axvline(x=50.0)
data.sort(key=lambda c: c[0])
legend = []
for name, values, indices in data:
legend.append(name)
if indices is None:
# x values are optional and default to range(len(values))
plt.plot(values)
else:
assert len(values) == len(indices)
plt.plot(indices, values)
plt.legend(legend, loc=legend_loc)
# https://matplotlib.org/users/tight_layout_guide.html
plt.tight_layout()
if save:
plt.savefig(fname=fname_abspath, format=FORMAT)
if show:
plt.show()
plt.close()
return fname_abspath
def plot_history_data(
matrix: ndarray,
file_name: str,
title: Optional[str] = None,
xlabel: str = "Training rounds",
ylabel: str = "Participants",
save: bool = True,
show: bool = False,
) -> str:
"""Plot participant selection history.
Plots or saves the participant selection history of an input matrix
as 2D regular raster with training rounds vs participants.
Args:
matrix (~numpy.ndarray): Image data for a 2D regular raster.
file_name (str): File name for storable plot.
title (~typing.Optional[str]): Title of the plot.
xlabel (str): Label for x-axis.
ylabel (str): Label for y-axis.
save (bool): If the plot should be stored as png.
show (bool): If the plot should be shown.
Returns:
str: File name of the plot as absolute path.
"""
file_name_abspath: str = storage.fname_with_default_dir(
fname=file_name, dname=FLAGS.output_dir)
# Creating figure with subplot
_, ax = plt.subplots(figsize=(11, 9))
# more information about colormaps: https://matplotlib.org/3.1.1/tutorials/colors/colormaps.html
color_map_name: str = "YlGn"
color_map: ListedColormap = prepare_colormap(name=color_map_name)
im = ax.imshow(matrix,
cmap=color_map,
interpolation="nearest",
aspect="auto")
ax.figure.colorbar(im, ax=ax)
# maxima for x and y axis
x_max: int = matrix.shape[1]
y_max: int = matrix.shape[0]
# Major ticks
ax.set_xticks(np.arange(0, x_max, 1))
ax.set_yticks(np.arange(0, y_max, 5))
# Labels for major ticks
ax.set_xticklabels(np.arange(1, x_max, 1))
ax.set_yticklabels(np.arange(0, y_max, 5))
# Display only each 2nd x-axis tick label
for x_tick_label in ax.xaxis.get_ticklabels()[::2]:
x_tick_label.set_visible(False)
# Minor ticks
ax.set_xticks(np.arange(-0.5, x_max, 1), minor=True)
ax.set_yticks(np.arange(-0.5, y_max, 1), minor=True)
# turn off gridline visibility for x and y axis
ax.tick_params(axis="both", which="minor", length=0)
# Gridlines based on minor ticks
ax.grid(which="minor", color="w", linestyle="-", linewidth=2)
ax.set_xlabel(xlabel, fontsize=14)
ax.set_ylabel(ylabel, fontsize=14)
# Set plot title if present
if title is not None:
ax.set_title(title, fontsize=14)
plt.tight_layout()
# Saving and showing plot
if save:
plt.savefig(fname=file_name_abspath, format=FORMAT)
if show:
plt.show(block=True)
plt.close()
return file_name_abspath