def print_metrics(metrics, weights, rounds):
"""Prints weighted averages of the given metrics.
Args:
metrics: dict with client ids as keys. Each entry is a dict
with the metrics of that client.
weights: dict with client ids as keys. Each entry is the weight
for that client.
"""
ordered_weights = [weights[c] for c in sorted(weights)]
metric_names = metrics_writer.get_metrics_names(metrics)
print('{} '.format(rounds), end='')
for metric in metric_names:
ordered_metric = [metrics[c][metric] for c in sorted(metrics)]
#print('%s: %g, 10th percentile: %g, 90th percentile %g' \
# % (metric,
# np.average(ordered_metric, weights=ordered_weights),
# np.percentile(ordered_metric, 10),
# np.percentile(ordered_metric, 90)))
print('{:.4f} {:.4f} {:.4f} '.format(\
np.average(ordered_metric, weights=ordered_weights),
np.percentile(ordered_metric, 10),
np.percentile(ordered_metric, 90)), end='')
print(' ')
def print_metrics(metrics, weights, prefix='', ct_round=""):
"""Prints weighted averages of the given metrics.
Args:
metrics: dict with client ids as keys. Each entry is a dict
with the metrics of that client.
weights: dict with client ids as keys. Each entry is the weight
for that client.
"""
ordered_weights = [weights[c] for c in sorted(weights)]
metric_names = metrics_writer.get_metrics_names(metrics)
to_ret = {}
for metric in metric_names:
ordered_metric = [metrics[c][metric] for c in sorted(metrics)]
print("RRound {} ".format(ct_round), end="")
print('%s: %g, 10th percentile: %g, 50th percentile: %g, 90th percentile %g' \
% (prefix + metric,
np.average(ordered_metric, weights=ordered_weights),
np.percentile(ordered_metric, 10),
np.percentile(ordered_metric, 50),
np.percentile(ordered_metric, 90)))
to_ret[metric] = np.average(ordered_metric, weights=ordered_weights)
return to_ret
def print_metrics(metrics, weights, prefix=''):
"""Prints weighted averages of the given metrics.
Args:
metrics: dict with client ids as keys. Each entry is a dict
with the metrics of that client.
weights: dict with client ids as keys. Each entry is the weight
for that client.
"""
client_ids = [c for c in sorted(metrics.keys())]
ordered_weights = [weights[c] for c in client_ids]
metric_names = metrics_writer.get_metrics_names(metrics)
to_ret = None
L = Logger()
logger = L.get_logger()
for metric in metric_names:
ordered_metric = [metrics[c][metric] for c in client_ids]
logger.info(
'{}: {}, 10th percentile: {}, 50th percentile: {}, 90th percentile {}'
.format(
prefix + metric,
np.average(ordered_metric, weights=ordered_weights, axis=0),
np.percentile(ordered_metric, 10, axis=0),
np.percentile(ordered_metric, 50, axis=0),
np.percentile(ordered_metric, 90, axis=0)))
def print_metrics(metrics, weights, prefix=''):
"""Prints weighted averages of the given metrics.
Args:
metrics: dict with client ids as keys. Each entry is a dict
with the metrics of that client.
weights: dict with client ids as keys. Each entry is the weight
for that client.
"""
ordered_weights = [weights[c] for c in sorted(weights)]
metric_names = metrics_writer.get_metrics_names(metrics)
# metric_names = metric_names[0:2]
to_ret = None
f = open("acc.txt", 'a')
for metric in metric_names:
ordered_metric = [metrics[c][metric] for c in sorted(metrics)]
f.write('{}: {}, 10th percentile: {}, 50th percentile: {}, 90th percentile {}'.format
(prefix + metric,
np.average(ordered_metric, weights=ordered_weights, axis=0),
np.percentile(ordered_metric, 10, axis=0),
np.percentile(ordered_metric, 50, axis=0),
np.percentile(ordered_metric, 90, axis=0)))
f.write("\n")
print('%s: %g, 10th percentile: %g, 50th percentile: %g, 90th percentile %g' \
% (prefix + metric,
np.average(ordered_metric, weights=ordered_weights),
np.percentile(ordered_metric, 10),
np.percentile(ordered_metric, 50),
np.percentile(ordered_metric, 90)))
def print_metrics(metrics, weights, prefix='', print_conf_matrix=False):
"""Prints weighted averages of the given metrics.
Args:
metrics: dict with client ids as keys. Each entry is a dict
with the metrics of that client.
weights: dict with client ids as keys. Each entry is the weight
for that client.
"""
ordered_weights = [weights[c] for c in sorted(weights)]
metric_names = metrics_writer.get_metrics_names(metrics)
to_ret = None
for metric in metric_names:
if metric == 'conf_matrix':
continue
ordered_metric = [metrics[c][metric] for c in sorted(metrics)]
print('%s: %g, 10th percentile: %g, 50th percentile: %g, 90th percentile %g' \
% (prefix + metric,
np.average(ordered_metric, weights=ordered_weights),
np.percentile(ordered_metric, 10),
np.percentile(ordered_metric, 50),
np.percentile(ordered_metric, 90)))
# print confusion matrix
if print_conf_matrix:
if 'conf_matrix' in metric_names:
full_conf_matrix = sum(
[metrics[c]['conf_matrix'] for c in sorted(metrics)])
print('Misclassification percentage: %.2f%%' %
(full_conf_matrix[FLIP_FROM_CLASS, FLIP_TO_CLASS] /
np.sum(full_conf_matrix[FLIP_FROM_CLASS]) * 100))
np.savetxt('conf_matrix.txt', full_conf_matrix, fmt='%4u')
def print_metrics(metrics, weights):
ordered_weights = [weights[c] for c in sorted(weights)]
metric_names = metrics_writer.get_metrics_names(metrics)
for metric in metric_names:
ordered_metric = [metrics[c][metric] for c in sorted(metrics)]
print('%s: %g, 10th percentile: %g, 90th percentile %g' \
% (metric,
np.average(ordered_metric, weights=ordered_weights),
np.percentile(ordered_metric, 10),
np.percentile(ordered_metric, 90)))
def print_metrics(metrics, weights, prefix=''):
"""Prints weighted averages of the given metrics.
Args:
metrics: dict with client ids as keys. Each entry is a dict
with the metrics of that client.
weights: dict with client ids as keys. Each entry is the weight
for that client.
"""
ordered_weights = [weights[c] for c in sorted(weights)]
metric_names = metrics_writer.get_metrics_names(metrics)
to_ret = None
for metric in metric_names:
ordered_metric = [metrics[c][metric] for c in sorted(metrics)]
print('%s: %g' \
% (prefix + metric,
np.average(ordered_metric, weights=None)))
def print_metrics(metrics, weights, prefix=''):
"""Prints weighted averages of the given metrics.
"""
ordered_weights = [weights[c] for c in sorted(weights)]
metric_names = metrics_writer.get_metrics_names(metrics)
micro_acc = metric_names[0]
miacc_metric = [metrics[c][micro_acc] for c in sorted(metrics)]
to_ret = np.average(miacc_metric, weights=ordered_weights)
for metric in metric_names[:2]:
ordered_metric = [metrics[c][metric] for c in sorted(metrics)]
print('%s: %g, 10th percentile: %g, 50th percentile: %g, 90th percentile %g' \
% (prefix + metric,
np.average(ordered_metric, weights=ordered_weights),
np.percentile(ordered_metric, 10),
np.percentile(ordered_metric, 50),
np.percentile(ordered_metric, 90)))
return to_ret
def print_metrics(metrics, weights, prefix="", log_fp=None):
"""Prints weighted averages of the given metrics.
Args:
metrics: dict with client ids as keys. Each entry is a dict
with the metrics of that client.
weights: dict with client ids as keys. Each entry is the weight
for that client.
"""
ordered_weights = [weights[c] for c in sorted(weights)]
metric_names = metrics_writer.get_metrics_names(metrics)
for metric in metric_names:
ordered_metric = [metrics[c][metric] for c in sorted(metrics)]
print('%s: %g, 10th percentile: %g, 50th percentile: %g, 90th percentile %g' \
% (prefix + metric,
np.average(ordered_metric, weights=ordered_weights),
np.percentile(ordered_metric, 10),
np.percentile(ordered_metric, 50),
np.percentile(ordered_metric, 90)),
file=log_fp, flush=True)
def save_metrics(metrics, weights, round, prefix=''):
"""Saves weighted averages of the given metrics.
Args:
metrics: dict with client ids as keys. Each entry is a dict
with the metrics of that client.
weights: dict with client ids as keys. Each entry is the weight
for that client.
"""
ordered_weights = [weights[c] for c in sorted(weights)]
metric_names = metrics_writer.get_metrics_names(metrics)
dirpath = os.path.abspath(f'./my_metrics/{DATETIME}')
os.makedirs(dirpath, exist_ok=dirpath)
for metric in metric_names:
filename = dirpath + f'/{prefix}-{metric}.csv'
if not os.path.exists(filename):
open(filename, 'w+').write(f'{metric},round\n')
with open(filename, 'a+') as outfile:
ordered_metric = [metrics[c][metric] for c in sorted(metrics)]
val = np.average(ordered_metric, weights=ordered_weights)
outfile.write(f'{val},{round}\n')
def print_metrics(metrics, weights):
"""Prints weighted averages of the given metrics.
Args:
metrics: dict with client ids as keys. Each entry is a dict
with the metrics of that client.
weights: dict with client ids as keys. Each entry is the weight
for that client.
"""
ordered_weights = [weights[c] for c in sorted(weights)]
metric_names = metrics_writer.get_metrics_names(metrics)
for metric in metric_names:
ordered_metric = [metrics[c][metric] for c in sorted(metrics)]
acc = np.average(ordered_metric, weights=ordered_weights),
print('%s: %g, 10th percentile: %g, 90th percentile %g' \
% (metric,
np.average(ordered_metric, weights=ordered_weights),
np.percentile(ordered_metric, 10),
np.percentile(ordered_metric, 90)))
#STOP when the global accuracy is achieved
if (acc[0] >= GLOB_ACC):
break
