def save_splits(dataset_name: str, dataset: FederatedDataset, local_generator_dir: str):
"""Saves federated dataset to filesystem
Args:
dataset_name (str)
dataset (FederatedDataset)
local_generator_dir (str)
"""
fname_ndarray_tuple = dataset_to_fname_ndarray_tuple_list(dataset)
dataset_dir = get_dataset_dir(
dataset_name=dataset_name, local_generator_dir=local_generator_dir
)
logging.info("Storing dataset in {}".format(dataset_dir))
split_hashes: Dict[str, List[Optional[str]]] = {}
for fname, ndarr in fname_ndarray_tuple:
sha1cs = save(fname=fname, data=ndarr, storage_dir=dataset_dir)
storage_key = fname[2:-4]
if storage_key not in split_hashes:
split_hashes[storage_key] = [None, None]
split_hashes[storage_key][0 if "x_" in fname else 1] = sha1cs
hash_file = os.path.join(dataset_dir, f"../../{dataset_name}.json")
storage.write_json(split_hashes, hash_file)
logging.info("{} generated and stored\n".format(dataset_name))
def bench_cl_ul(
name: str, xy_train, xy_val, xy_test, epochs: int, use_coordinator: bool
):
start = time.time()
if use_coordinator:
hist, _, _, _, loss, acc = run.federated_training(
"blog_cnn", [xy_train], xy_val, xy_test, R=epochs, E=1, C=0, B=B
)
else:
hist, loss, acc = run.unitary_training(
"blog_cnn", xy_train, xy_val, xy_test, E=epochs, B=B
)
end = time.time()
# Write results JSON
results = {
"name": name,
"start": start,
"end": end,
"duration": end - start,
"E": epochs,
"B": B,
"unitary_learning": {"loss": float(loss), "acc": float(acc), "hist": hist},
}
storage.write_json(results, fname=name + "-results.json")
def benchmark_evolutionary_avg():
fn_name = benchmark_evolutionary_avg.__name__
logger.info("Starting {}".format(fn_name))
# Load dataset
xy_parts, xy_val, xy_test = load_splits("fashion-mnist-100p-noniid-03cpp")
# Run Federated Learning with evolutionary aggregation
evaluator = Evaluator(orig_cnn_compiled(), xy_val) # FIXME refactor
aggregator = EvoAgg(evaluator)
_, _, _, _, loss_a, acc_a = run.federated_training(
"blog_cnn",
xy_parts,
xy_val,
xy_test,
R=DEFAULT_R,
E=DEFAULT_E,
C=DEFAULT_C,
B=DEFAULT_B,
aggregator=aggregator,
)
# Run Federated Learning with weighted average aggregation
_, _, _, _, loss_b, acc_b = run.federated_training(
"blog_cnn",
xy_parts,
xy_val,
xy_test,
R=DEFAULT_R,
E=DEFAULT_E,
C=DEFAULT_C,
B=DEFAULT_B,
)
# Write results JSON
results = {}
results["loss_a"] = float(loss_a)
results["acc_a"] = float(acc_a)
results["loss_b"] = float(loss_b)
results["acc_b"] = float(acc_b)
# TODO add histories
storage.write_json(results, fname="EA-WA-results.json")
def _run_benchmark(benchmark_name: str):
logger.info("Building Docker image for benchmark",
benchmark_name=benchmark_name)
logger.info("Starting benchmark", benchmark_name=benchmark_name)
benchmark = benchmarks[benchmark_name]
group_name = FLAGS.group_name or f"{strftime('%Y%m%dT%H%M')}_{benchmark_name}"
task_names = {task.name for task in benchmark.tasks}
assert len(task_names) == len(benchmark.tasks), "Duplicate task names"
should_push = benchmark.runner == "ec2"
docker_image_name = docker.build(should_push=should_push)
# TODO Initiate tasks in parallel
for task in benchmark.tasks:
model_name = task.model_name
dataset_name = task.dataset_name
_run_task(
docker_image_name=docker_image_name,
group_name=group_name,
task_name=task.name,
task_label=task.label,
model=model_name,
dataset=dataset_name,
R=task.R,
E=task.E,
C=task.C,
B=task.B,
partition_id=task.partition_id,
instance_cores=task.instance_cores,
timeout=task.timeout,
runner=benchmark.runner,
)
with TemporaryDirectory() as tmpdir:
fname = os.path.join(tmpdir, "config.json")
data = {"aggregation_name": benchmark.aggregation_name}
storage.write_json(data, fname)
results.push(group_name=group_name, task_name="", output_dir=tmpdir)
def main(_):
# Set exit callback
if FLAGS.push_results:
atexit.register(after_main,
group_name=FLAGS.group_name,
task_name=FLAGS.task_name)
# Load data
xy_train_partitions, xy_val, xy_test = load_splits(FLAGS.dataset)
# Execute training
start = time.time()
partition_id = FLAGS.partition_id
num_participants = 1 # For unitary training
hist_opt_configs = None # For unitary training
hist_metrics = None # For unitary training
if partition_id is not None: # Use only a single partition if required (unitary)
hist, loss, acc = run.unitary_training(
model_name=FLAGS.model,
xy_train=xy_train_partitions[partition_id],
xy_val=xy_val,
xy_test=xy_test,
E=FLAGS.E,
B=FLAGS.B,
)
else:
hist, _, hist_opt_configs, hist_metrics, loss, acc = run.federated_training(
model_name=FLAGS.model,
xy_train_partitions=xy_train_partitions,
xy_val=xy_val,
xy_test=xy_test,
R=FLAGS.R,
E=FLAGS.E,
C=FLAGS.C,
B=FLAGS.B,
)
num_participants = len(xy_train_partitions)
end = time.time()
# Write results
res = {
"group_name": FLAGS.group_name,
"task_name": FLAGS.task_name,
"task_label": FLAGS.task_label,
"dataset": FLAGS.dataset,
"model": FLAGS.model,
"R": FLAGS.R,
"E": FLAGS.E,
"C": FLAGS.C,
"B": FLAGS.B,
"partition_id": partition_id,
"start": start,
"end": end,
"duration": end - start,
"loss": float(loss),
"acc": float(acc),
"hist": hist,
"hist_opt_configs": hist_opt_configs,
"hist_metrics": hist_metrics,
"num_participants": num_participants,
}
storage.write_json(res, fname="results.json")