def print_metrics(metrics, weights, rounds): """Prints weighted averages of the given metrics. Args: metrics: dict with client ids as keys. Each entry is a dict with the metrics of that client. weights: dict with client ids as keys. Each entry is the weight for that client. """ ordered_weights = [weights[c] for c in sorted(weights)] metric_names = metrics_writer.get_metrics_names(metrics) print('{} '.format(rounds), end='') for metric in metric_names: ordered_metric = [metrics[c][metric] for c in sorted(metrics)] #print('%s: %g, 10th percentile: %g, 90th percentile %g' \ # % (metric, # np.average(ordered_metric, weights=ordered_weights), # np.percentile(ordered_metric, 10), # np.percentile(ordered_metric, 90))) print('{:.4f} {:.4f} {:.4f} '.format(\ np.average(ordered_metric, weights=ordered_weights), np.percentile(ordered_metric, 10), np.percentile(ordered_metric, 90)), end='') print(' ')
def print_metrics(metrics, weights, prefix='', ct_round=""): """Prints weighted averages of the given metrics. Args: metrics: dict with client ids as keys. Each entry is a dict with the metrics of that client. weights: dict with client ids as keys. Each entry is the weight for that client. """ ordered_weights = [weights[c] for c in sorted(weights)] metric_names = metrics_writer.get_metrics_names(metrics) to_ret = {} for metric in metric_names: ordered_metric = [metrics[c][metric] for c in sorted(metrics)] print("RRound {} ".format(ct_round), end="") print('%s: %g, 10th percentile: %g, 50th percentile: %g, 90th percentile %g' \ % (prefix + metric, np.average(ordered_metric, weights=ordered_weights), np.percentile(ordered_metric, 10), np.percentile(ordered_metric, 50), np.percentile(ordered_metric, 90))) to_ret[metric] = np.average(ordered_metric, weights=ordered_weights) return to_ret
def print_metrics(metrics, weights, prefix=''): """Prints weighted averages of the given metrics. Args: metrics: dict with client ids as keys. Each entry is a dict with the metrics of that client. weights: dict with client ids as keys. Each entry is the weight for that client. """ client_ids = [c for c in sorted(metrics.keys())] ordered_weights = [weights[c] for c in client_ids] metric_names = metrics_writer.get_metrics_names(metrics) to_ret = None L = Logger() logger = L.get_logger() for metric in metric_names: ordered_metric = [metrics[c][metric] for c in client_ids] logger.info( '{}: {}, 10th percentile: {}, 50th percentile: {}, 90th percentile {}' .format( prefix + metric, np.average(ordered_metric, weights=ordered_weights, axis=0), np.percentile(ordered_metric, 10, axis=0), np.percentile(ordered_metric, 50, axis=0), np.percentile(ordered_metric, 90, axis=0)))
def print_metrics(metrics, weights, prefix=''): """Prints weighted averages of the given metrics. Args: metrics: dict with client ids as keys. Each entry is a dict with the metrics of that client. weights: dict with client ids as keys. Each entry is the weight for that client. """ ordered_weights = [weights[c] for c in sorted(weights)] metric_names = metrics_writer.get_metrics_names(metrics) # metric_names = metric_names[0:2] to_ret = None f = open("acc.txt", 'a') for metric in metric_names: ordered_metric = [metrics[c][metric] for c in sorted(metrics)] f.write('{}: {}, 10th percentile: {}, 50th percentile: {}, 90th percentile {}'.format (prefix + metric, np.average(ordered_metric, weights=ordered_weights, axis=0), np.percentile(ordered_metric, 10, axis=0), np.percentile(ordered_metric, 50, axis=0), np.percentile(ordered_metric, 90, axis=0))) f.write("\n") print('%s: %g, 10th percentile: %g, 50th percentile: %g, 90th percentile %g' \ % (prefix + metric, np.average(ordered_metric, weights=ordered_weights), np.percentile(ordered_metric, 10), np.percentile(ordered_metric, 50), np.percentile(ordered_metric, 90)))
def print_metrics(metrics, weights, prefix='', print_conf_matrix=False): """Prints weighted averages of the given metrics. Args: metrics: dict with client ids as keys. Each entry is a dict with the metrics of that client. weights: dict with client ids as keys. Each entry is the weight for that client. """ ordered_weights = [weights[c] for c in sorted(weights)] metric_names = metrics_writer.get_metrics_names(metrics) to_ret = None for metric in metric_names: if metric == 'conf_matrix': continue ordered_metric = [metrics[c][metric] for c in sorted(metrics)] print('%s: %g, 10th percentile: %g, 50th percentile: %g, 90th percentile %g' \ % (prefix + metric, np.average(ordered_metric, weights=ordered_weights), np.percentile(ordered_metric, 10), np.percentile(ordered_metric, 50), np.percentile(ordered_metric, 90))) # print confusion matrix if print_conf_matrix: if 'conf_matrix' in metric_names: full_conf_matrix = sum( [metrics[c]['conf_matrix'] for c in sorted(metrics)]) print('Misclassification percentage: %.2f%%' % (full_conf_matrix[FLIP_FROM_CLASS, FLIP_TO_CLASS] / np.sum(full_conf_matrix[FLIP_FROM_CLASS]) * 100)) np.savetxt('conf_matrix.txt', full_conf_matrix, fmt='%4u')
def print_metrics(metrics, weights): ordered_weights = [weights[c] for c in sorted(weights)] metric_names = metrics_writer.get_metrics_names(metrics) for metric in metric_names: ordered_metric = [metrics[c][metric] for c in sorted(metrics)] print('%s: %g, 10th percentile: %g, 90th percentile %g' \ % (metric, np.average(ordered_metric, weights=ordered_weights), np.percentile(ordered_metric, 10), np.percentile(ordered_metric, 90)))
def print_metrics(metrics, weights, prefix=''): """Prints weighted averages of the given metrics. Args: metrics: dict with client ids as keys. Each entry is a dict with the metrics of that client. weights: dict with client ids as keys. Each entry is the weight for that client. """ ordered_weights = [weights[c] for c in sorted(weights)] metric_names = metrics_writer.get_metrics_names(metrics) to_ret = None for metric in metric_names: ordered_metric = [metrics[c][metric] for c in sorted(metrics)] print('%s: %g' \ % (prefix + metric, np.average(ordered_metric, weights=None)))
def print_metrics(metrics, weights, prefix=''): """Prints weighted averages of the given metrics. """ ordered_weights = [weights[c] for c in sorted(weights)] metric_names = metrics_writer.get_metrics_names(metrics) micro_acc = metric_names[0] miacc_metric = [metrics[c][micro_acc] for c in sorted(metrics)] to_ret = np.average(miacc_metric, weights=ordered_weights) for metric in metric_names[:2]: ordered_metric = [metrics[c][metric] for c in sorted(metrics)] print('%s: %g, 10th percentile: %g, 50th percentile: %g, 90th percentile %g' \ % (prefix + metric, np.average(ordered_metric, weights=ordered_weights), np.percentile(ordered_metric, 10), np.percentile(ordered_metric, 50), np.percentile(ordered_metric, 90))) return to_ret
def print_metrics(metrics, weights, prefix="", log_fp=None): """Prints weighted averages of the given metrics. Args: metrics: dict with client ids as keys. Each entry is a dict with the metrics of that client. weights: dict with client ids as keys. Each entry is the weight for that client. """ ordered_weights = [weights[c] for c in sorted(weights)] metric_names = metrics_writer.get_metrics_names(metrics) for metric in metric_names: ordered_metric = [metrics[c][metric] for c in sorted(metrics)] print('%s: %g, 10th percentile: %g, 50th percentile: %g, 90th percentile %g' \ % (prefix + metric, np.average(ordered_metric, weights=ordered_weights), np.percentile(ordered_metric, 10), np.percentile(ordered_metric, 50), np.percentile(ordered_metric, 90)), file=log_fp, flush=True)
def save_metrics(metrics, weights, round, prefix=''): """Saves weighted averages of the given metrics. Args: metrics: dict with client ids as keys. Each entry is a dict with the metrics of that client. weights: dict with client ids as keys. Each entry is the weight for that client. """ ordered_weights = [weights[c] for c in sorted(weights)] metric_names = metrics_writer.get_metrics_names(metrics) dirpath = os.path.abspath(f'./my_metrics/{DATETIME}') os.makedirs(dirpath, exist_ok=dirpath) for metric in metric_names: filename = dirpath + f'/{prefix}-{metric}.csv' if not os.path.exists(filename): open(filename, 'w+').write(f'{metric},round\n') with open(filename, 'a+') as outfile: ordered_metric = [metrics[c][metric] for c in sorted(metrics)] val = np.average(ordered_metric, weights=ordered_weights) outfile.write(f'{val},{round}\n')
def print_metrics(metrics, weights): """Prints weighted averages of the given metrics. Args: metrics: dict with client ids as keys. Each entry is a dict with the metrics of that client. weights: dict with client ids as keys. Each entry is the weight for that client. """ ordered_weights = [weights[c] for c in sorted(weights)] metric_names = metrics_writer.get_metrics_names(metrics) for metric in metric_names: ordered_metric = [metrics[c][metric] for c in sorted(metrics)] acc = np.average(ordered_metric, weights=ordered_weights), print('%s: %g, 10th percentile: %g, 90th percentile %g' \ % (metric, np.average(ordered_metric, weights=ordered_weights), np.percentile(ordered_metric, 10), np.percentile(ordered_metric, 90))) #STOP when the global accuracy is achieved if (acc[0] >= GLOB_ACC): break