def prepare_noniid_dataset(data, pivot_label: str, data_path: str, pickle_path: str, nb_cluster: int, double_check: bool =False):
# The TSNE representation is independent of the number of devices.
tsne_file = "{0}-tsne".format(data_path)
if not file_exist("{0}.pkl".format(tsne_file)):
# Running TNSE to obtain a 2D representation of data
logging.debug("The TSNE representation ({0}) doesn't exist."
.format(tsne_file))
embedded_data = tsne(data)
pickle_saver(embedded_data, tsne_file)
tsne_cluster_file = "{0}/tsne-cluster".format(pickle_path)
if not file_exist("{0}.pkl".format(tsne_cluster_file)):
# Finding clusters in the TNSE.
logging.debug("Finding non-iid clusters in the TNSE represesentation: {0}.pkl".format(tsne_file))
embedded_data = pickle_loader("{0}".format(tsne_file))
logging.debug("Saving found clusters : {0}.pkl".format(tsne_cluster_file))
predicted_cluster = find_cluster(embedded_data, nb_cluster)
pickle_saver(predicted_cluster, "{0}".format(tsne_cluster_file))
predicted_cluster = pickle_loader("{0}".format(tsne_cluster_file))
# With the found clusters, splitting data.
X, Y = clustering_data(data, predicted_cluster, pivot_label, nb_cluster)
if double_check:
logging.debug("Checking data cluserization, wait until completion before seeing the plots.")
# Checking that splitting data by cluster is valid.
check_data_clusterisation(X, Y, nb_cluster)
return X, Y
def run_one_scenario(cost_models,
list_algos,
logs_file: str,
experiments_settings: str,
batch_size: int = 1,
stochastic: bool = True,
nb_epoch: int = 250,
step_size=None,
compression: CompressionModel = None,
use_averaging: bool = False,
fraction_sampled_workers: int = 1,
modify_run=None) -> None:
pickle_file = "{0}/descent-{1}".format(logs_file, experiments_settings)
if modify_run is None:
if file_exist(pickle_file + ".pkl"):
remove_file(pickle_file + ".pkl")
algos = list_algos
else:
algos = [list_algos[i] for i in modify_run]
for type_params in tqdm(algos):
multiple_sg_descent = multiple_run_descent(
type_params,
cost_models=cost_models,
compression_model=compression,
use_averaging=use_averaging,
stochastic=stochastic,
nb_epoch=nb_epoch,
step_formula=step_size,
batch_size=batch_size,
logs_file=logs_file,
fraction_sampled_workers=fraction_sampled_workers)
if logs_file:
logs = open("{0}/logs.txt".format(logs_file), "a+")
logs.write(
"{0} size of the multiple SG descent: {1:.2e} bits\n".format(
type_params.name(), asizeof.asizeof(multiple_sg_descent)))
logs.close()
if file_exist(pickle_file + ".pkl"):
res = pickle_loader(pickle_file)
res.add_descent(multiple_sg_descent,
type_params.name(),
deep_learning_run=False)
else:
res = ResultsOfSeveralDescents(len(cost_models))
res.add_descent(multiple_sg_descent,
type_params.name(),
deep_learning_run=False)
pickle_saver(res, pickle_file)
del res
del multiple_sg_descent
if (epoch+1) % 20 == 0:
curr_lr /= 3
update_lr(optimizer, curr_lr)
# Test the model
model.eval()
with torch.no_grad():
correct = 0
total = 0
for images, labels in test_loader:
images = images.to(device)
labels = labels.to(device)
outputs = model(images)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
print('Accuracy of the model on the test images: {} %'.format(100 * correct / total))
# Save the model checkpoint
torch.save(model.state_dict(), 'resnet.ckpt')
res = pickle_loader("run_cifar10")
plt.plot(list(range(num_epochs)), run.train_losses)
plt.savefig('{0}.eps'.format("cifar10_train_losses"), format='eps')
plt.plot(list(range(num_epochs)), run.test_losses)
plt.savefig('{0}.eps'.format("cifar10_test_losses"), format='eps')
plt.plot(list(range(num_epochs)), run.test_accuracies)
plt.savefig('{0}.eps'.format("cifar10_accuracy"), format='eps')
def run_experiments_in_deeplearning(dataset: str):
with open("log.txt", 'a') as f:
print("==== NEW RUN ====", file=f)
fraction_sampled_workers = 1
batch_size = 128
nb_devices = 20
algos = "mcm-vs-existing"
iid = "non-iid"
data_path, pickle_path, algos_pickle_path, picture_path = create_path_and_folders(
nb_devices, dataset, iid, algos, fraction_sampled_workers)
default_up_compression = SQuantization(quantization_levels[dataset],
norm=norm_quantization[dataset])
default_down_compression = SQuantization(quantization_levels[dataset],
norm=norm_quantization[dataset])
exp_name = "{0}_m{1}_lr{2}_sup{3}_sdwn{4}_b{5}_wd{6}".format(
models[dataset].__name__, momentums[dataset],
optimal_steps_size[dataset], default_up_compression.level,
default_down_compression.level, batch_size, weight_decay[dataset])
all_descent = {}
for type_params in [VanillaSGD(), Diana(), Artemis(), MCM()]:
print(type_params)
torch.cuda.empty_cache()
params = type_params.define(
cost_models=None,
n_dimensions=None,
nb_epoch=200,
nb_devices=nb_devices,
batch_size=batch_size,
fraction_sampled_workers=1,
up_compression_model=default_up_compression,
down_compression_model=default_down_compression)
params = cast_to_DL(params, dataset, models[dataset],
optimal_steps_size[dataset], weight_decay[dataset])
params.log_file = "log.txt"
params.momentum = momentums[dataset]
params.print()
with open(params.log_file, 'a') as f:
print(type_params, file=f)
print("Optimal step size: ", params.optimal_step_size, file=f)
multiple_sg_descent = run_tuned_exp(params)
all_descent[type_params.name()] = multiple_sg_descent
res = ResultsOfSeveralDescents(all_descent, nb_devices)
# res.add_descent(multiple_sg_descent, type_params.name())
pickle_saver(res, "{0}/{1}".format(algos_pickle_path, exp_name))
res = pickle_loader("{0}/{1}".format(algos_pickle_path, exp_name))
# Plotting without averaging
plot_error_dist(res.get_loss(np.array(0), in_log=True),
res.names,
res.nb_devices_for_the_run,
batch_size=batch_size,
all_error=res.get_std(np.array(0), in_log=True),
x_legend="Number of passes on data",
ylegends="train_loss",
picture_name="{0}/{1}_train_losses".format(
picture_path, exp_name))
plot_error_dist(res.get_loss(np.array(0), in_log=True),
res.names,
res.nb_devices_for_the_run,
batch_size=batch_size,
x_points=res.X_number_of_bits,
ylegends="train_loss",
all_error=res.get_std(np.array(0), in_log=True),
x_legend="Communicated bits",
picture_name="{0}/{1}_train_losses_bits".format(
picture_path, exp_name))
plot_error_dist(res.get_test_accuracies(),
res.names,
res.nb_devices_for_the_run,
ylegends="accuracy",
all_error=res.get_test_accuracies_std(),
x_legend="Number of passes on data",
batch_size=batch_size,
picture_name="{0}/{1}_test_accuracies".format(
picture_path, exp_name))
plot_error_dist(res.get_test_losses(in_log=True),
res.names,
res.nb_devices_for_the_run,
batch_size=batch_size,
all_error=res.get_test_losses_std(in_log=True),
x_legend="Number of passes on data",
ylegends="test_loss",
picture_name="{0}/{1}_test_losses".format(
picture_path, exp_name))
def run_experiments(nb_devices: int, stochastic: bool, dataset: str, iid: str, algos: str, use_averaging: bool = False,
scenario: str = None, fraction_sampled_workers: int = 0.5, plot_only: bool = True):
print("Running with following parameters: {0}".format(["{0} -> {1}".format(k, v) for (k, v)
in zip(locals().keys(), locals().values())]))
assert dataset in ["quantum", "superconduct", 'synth_logistic', 'synth_linear_noised', 'synth_linear_nonoised'], \
"The available dataset are ['quantum', 'superconduct', 'synth_linear_noised', 'synth_linear_nonoised']."
assert iid in ['iid', 'non-iid'], "The iid option are ['iid', 'non-iid']."
assert scenario in [None, "compression", "step"], "The possible scenario are [None, 'compression', 'step']."
data_path, pickle_path, algos_pickle_path, picture_path = create_path_and_folders(nb_devices, dataset, iid, algos, fraction_sampled_workers)
list_algos = choose_algo(algos, stochastic, fraction_sampled_workers)
nb_devices = nb_devices
nb_epoch = 100 if stochastic else 400
iid_data = True if iid == 'iid' else False
# Select the correct dataset
if dataset == "quantum":
X, Y, dim_notebook = prepare_quantum(nb_devices, data_path=data_path, pickle_path=pickle_path, iid=iid_data)
batch_size = 400
model = LogisticModel
nb_epoch = 500 if stochastic else 400
elif dataset == "superconduct":
X, Y, dim_notebook = prepare_superconduct(nb_devices, data_path= data_path, pickle_path=pickle_path, iid=iid_data)
batch_size = 50
model = RMSEModel
nb_epoch = 500 if stochastic else 400
elif dataset == 'synth_logistic':
dim_notebook = 2
batch_size = 1
model = LogisticModel
if not file_exist("{0}/data.pkl".format(pickle_path)):
w = torch.FloatTensor([10, 10]).to(dtype=torch.float64)
X, Y = build_data_logistic(w, n_dimensions=2, n_devices=nb_devices, with_seed=False)
pickle_saver((X, Y), pickle_path + "/data")
else:
X, Y = pickle_loader(pickle_path + "/data")
elif dataset == 'synth_linear_noised':
dim_notebook = 20
if not file_exist("{0}/data.pkl".format(pickle_path)):
w_true = generate_param(dim_notebook-1)
X, Y = build_data_linear(w_true, n_dimensions=dim_notebook-1,
n_devices=nb_devices, with_seed=False, without_noise=False)
X = add_bias_term(X)
pickle_saver((X, Y), pickle_path + "/data")
else:
X, Y = pickle_loader(pickle_path + "/data")
model = RMSEModel
batch_size = 1
elif dataset == 'synth_linear_nonoised':
dim_notebook = 20
if not file_exist("{0}/data.pkl".format(pickle_path)):
w_true = generate_param(dim_notebook-1)
X, Y = build_data_linear(w_true, n_dimensions=dim_notebook-1,
n_devices=nb_devices, with_seed=False, without_noise=True)
X = add_bias_term(X)
pickle_saver((X, Y), pickle_path + "/data")
else:
X, Y = pickle_loader(pickle_path + "/data")
model = RMSEModel
batch_size = 1
compression_by_default = SQuantization(1, dim_notebook)
values_compression = [SQuantization(0, dim_notebook),
SQuantization(16, dim_notebook),
SQuantization(8, dim_notebook),
SQuantization(4, dim_notebook),
SQuantization(3, dim_notebook),
SQuantization(2, dim_notebook),
SQuantization(1, dim_notebook)
]
label_compression = ["SGD"] + [str(value.omega_c)[:4] for value in values_compression[1:]]
# Rebalancing cluster: the biggest one must not be more than 10times bigger than the smallest one.
X_r, Y_r = rebalancing_clusters(X, Y)
# Creating cost models which will be used to computed cost/loss, gradients, L ...
cost_models = build_several_cost_model(model, X_r, Y_r, nb_devices)
if not file_exist("{0}/obj_min.pkl".format(pickle_path)):
obj_min_by_N_descent = SGD_Descent(Parameters(n_dimensions=dim_notebook,
nb_devices=nb_devices,
nb_epoch=40000,
momentum=0.,
verbose=True,
cost_models=cost_models,
stochastic=False
))
obj_min_by_N_descent.run(cost_models)
obj_min = obj_min_by_N_descent.losses[-1]
pickle_saver(obj_min, "{0}/obj_min".format(pickle_path))
# Choice of step size
if stochastic and batch_size == 1:
step_size = iid_step_size
else:
step_size = batch_step_size
if 'synth' in dataset and stochastic:
step_size = deacreasing_step_size
stochasticity = 'sto' if stochastic else "full"
if not plot_only:
if scenario == "compression":
run_for_different_scenarios(cost_models, list_algos[1:], values_compression, label_compression,
filename=algos_pickle_path, batch_size=batch_size, stochastic=stochastic,
step_formula=step_size, scenario=scenario)
elif scenario == "step":
run_for_different_scenarios(cost_models, list_algos, step_formula, label_step_formula,
filename=algos_pickle_path, batch_size=batch_size, stochastic=stochastic,
scenario=scenario, compression=compression_by_default)
else:
run_one_scenario(cost_models=cost_models, list_algos=list_algos, filename=algos_pickle_path,
batch_size=batch_size, stochastic=stochastic, nb_epoch=nb_epoch, step_size=step_size,
use_averaging=use_averaging, compression=compression_by_default,
fraction_sampled_workers=fraction_sampled_workers)
obj_min = pickle_loader("{0}/obj_min".format(pickle_path))
if stochastic:
experiments_settings = "{0}-b{1}".format(stochasticity, batch_size)
else:
experiments_settings = stochasticity
if scenario is None:
res = pickle_loader("{0}/descent-{1}".format(algos_pickle_path, experiments_settings))
# Plotting without averaging
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook,
all_error=res.get_std(obj_min), x_legend="Number of passes on data\n({0})".format(iid),
picture_name="{0}/it-noavg-{1}".format(picture_path, experiments_settings))
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook,
x_points=res.X_number_of_bits, x_legend="Communicated bits ({0})".format(iid),
all_error=res.get_std(obj_min), picture_name="{0}/bits-noavg-{1}"
.format(picture_path, experiments_settings))
# Plotting with averaging
if use_averaging:
plot_error_dist(res.get_loss(obj_min, averaged=True), res.names, res.nb_devices,
dim_notebook, all_error=res.get_std(obj_min, averaged=True),
x_legend="Number of passes on data\n(Avg, {0})".format(iid),
picture_name="{0}/it-avg-{1}"
.format(picture_path, experiments_settings))
plot_error_dist(res.get_loss(obj_min, averaged=True), res.names, res.nb_devices, dim_notebook,
x_points=res.X_number_of_bits, all_error=res.get_std(obj_min, averaged=True),
x_legend="Communicated bits (Avg, {0})".format(iid),
picture_name="{0}/bits-avg-{1}"
.format(picture_path, experiments_settings))
if scenario == "step":
res = pickle_loader("{0}/{1}-{2}".format(algos_pickle_path, scenario, experiments_settings))
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook,
batch_size=batch_size,
all_error=res.get_std(obj_min),
x_legend="Step size ({0}, {1})".format(iid, str(compression_by_default.omega_c)[:4]),
one_on_two_points=True, xlabels=label_step_formula,
picture_name="{0}/{1}-{2}".format(picture_path, scenario, experiments_settings))
res = pickle_loader("{0}/{1}-optimal-{2}".format(algos_pickle_path, scenario, experiments_settings))
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook,
all_error=res.get_std(obj_min), batch_size=batch_size,
x_legend="(non-iid)", ylim=True,
picture_name="{0}/{1}-optimal-it-{2}".format(picture_path, scenario, experiments_settings))
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook,
x_points=res.X_number_of_bits, batch_size=batch_size,
x_legend="Communicated bits\n(non-iid)", all_error=res.get_std(obj_min), ylim=True,
picture_name="{0}/{1}-optimal-bits-{2}".format(picture_path, scenario, experiments_settings))
if scenario == "compression":
res = pickle_loader("{0}/{1}-{2}".format(algos_pickle_path, scenario, experiments_settings))
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook, batch_size=batch_size,
all_error=res.get_std(obj_min), x_legend="$\omega_c$ ({0})".format(iid),
one_on_two_points=True, xlabels=label_compression,
picture_name="{0}/{1}-{2}".format(picture_path, scenario, experiments_settings))
res = pickle_loader("{0}/{1}-optimal-{2}".format(algos_pickle_path, scenario, experiments_settings))
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook,
all_error=res.get_std(obj_min), batch_size=batch_size,
x_legend="(non-iid)", ylim=True,
picture_name="{0}/{1}-optimal-it-{2}".format(picture_path, scenario, experiments_settings))
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook,
x_points=res.X_number_of_bits, batch_size=batch_size,
x_legend="Communicated bits\n(non-iid)", all_error=res.get_std(obj_min), ylim=True,
picture_name="{0}/{1}-optimal-bits-{2}".format(picture_path, scenario, experiments_settings))
def run_experiments_in_deeplearning(dataset: str,
plot_only: bool = False) -> None:
"""Runs and plots experiments for a given dataset using an appropriate neural network.
:param dataset: Name of the dataset
:param plot_only: True if the goal is not to rerun all experiments but only to regenerate figures.
"""
fraction_sampled_workers = 1
batch_size = batch_sizes[dataset]
nb_devices = 20
algos = sys.argv[2]
iid = sys.argv[3]
stochastic = True
create_folder_if_not_existing(algos)
log_file = algos + "/log_" + dataset + "_" + iid + ".txt"
with open(log_file, 'a') as f:
print("==== NEW RUN ====", file=f)
with open(log_file, 'a') as f:
print(
"stochastic -> {0}, iid -> {1}, batch_size -> {2}, norm -> {3}, s -> {4}, momentum -> {5}, model -> {6}"
.format(stochastic, iid, batch_size, norm_quantization[dataset],
quantization_levels[dataset], momentums[dataset],
models[dataset].__name__),
file=f)
data_path, pickle_path, algos_pickle_path, picture_path = create_path_and_folders(
nb_devices, dataset, iid, algos, fraction_sampled_workers)
default_up_compression = SQuantization(quantization_levels[dataset],
norm=norm_quantization[dataset])
default_down_compression = SQuantization(quantization_levels[dataset],
norm=norm_quantization[dataset])
loaders = create_loaders(dataset, iid, nb_devices, batch_size, stochastic)
_, train_loader_workers_full, _ = loaders
dim = next(iter(train_loader_workers_full[0]))[0].shape[1]
if optimal_steps_size[dataset] is None:
L = compute_L(train_loader_workers_full)
optimal_steps_size[dataset] = 1 / L
print("Step size:", optimal_steps_size[dataset])
exp_name = name_of_the_experiments(dataset, stochastic)
pickle_file = "{0}/{1}".format(algos_pickle_path, exp_name)
list_algos = choose_algo(algos, stochastic, fraction_sampled_workers)
if not plot_only:
if file_exist(pickle_file + ".pkl"):
remove_file(pickle_file + ".pkl")
for type_params in list_algos:
print(type_params)
torch.cuda.empty_cache()
params = type_params.define(
cost_models=None,
n_dimensions=dim,
nb_epoch=300,
nb_devices=nb_devices,
stochastic=stochastic,
batch_size=batch_size,
fraction_sampled_workers=fraction_sampled_workers,
up_compression_model=default_up_compression,
down_compression_model=default_down_compression)
params = cast_to_DL(params, dataset, models[dataset],
optimal_steps_size[dataset],
weight_decay[dataset], iid)
params.log_file = log_file
params.momentum = momentums[dataset]
params.criterion = criterion[dataset]
params.print()
with open(params.log_file, 'a') as f:
print(type_params, file=f)
print("Optimal step size: ", params.optimal_step_size, file=f)
multiple_descent = AverageOfSeveralIdenticalRun()
seed_everything(seed=42)
start = time.time()
for i in range(NB_RUN):
print('Run {:3d}/{:3d}:'.format(i + 1, NB_RUN))
fixed_params = copy.deepcopy(params)
try:
training = Train(loaders, fixed_params)
multiple_descent.append_from_DL(training.run_training())
except ValueError as err:
print(err)
continue
with open(log_file, 'a') as f:
print("Time of the run: {:.2f}s".format(time.time() - start),
file=f)
with open(params.log_file, 'a') as f:
print("{0} size of the multiple SG descent: {1:.2e} bits\n".
format(type_params.name(),
asizeof.asizeof(multiple_descent)),
file=f)
if file_exist(pickle_file + ".pkl"):
res = pickle_loader(pickle_file)
res.add_descent(multiple_descent,
type_params.name(),
deep_learning_run=True)
else:
res = ResultsOfSeveralDescents(nb_devices)
res.add_descent(multiple_descent,
type_params.name(),
deep_learning_run=True)
pickle_saver(res, pickle_file)
# obj_min_cvx = pickle_loader("{0}/obj_min".format(pickle_path))
obj_min = 0 #pickle_loader("{0}/obj_min".format(pickle_path))
res = pickle_loader(pickle_file)
# obj_min = min(res.get_loss(np.array(0), in_log=False)[0])
# print("Obj min in convex:", obj_min_cvx)
print("Obj min in dl:", obj_min)
# Plotting
plot_error_dist(res.get_loss(np.array(obj_min)),
res.names,
all_error=res.get_std(np.array(obj_min)),
x_legend="Number of passes on data",
ylegends="train_loss",
picture_name="{0}/{1}_train_losses".format(
picture_path, exp_name))
plot_error_dist(res.get_loss(np.array(obj_min)),
res.names,
x_points=res.X_number_of_bits,
ylegends="train_loss",
all_error=res.get_std(np.array(obj_min)),
x_legend="Communicated bits",
picture_name="{0}/{1}_train_losses_bits".format(
picture_path, exp_name))
plot_error_dist(res.get_test_accuracies(),
res.names,
ylegends="accuracy",
all_error=res.get_test_accuracies_std(),
x_legend="Number of passes on data",
picture_name="{0}/{1}_test_accuracies".format(
picture_path, exp_name))
plot_error_dist(res.get_test_losses(in_log=True),
res.names,
all_error=res.get_test_losses_std(in_log=True),
x_legend="Number of passes on data",
ylegends="test_loss",
picture_name="{0}/{1}_test_losses".format(
picture_path, exp_name))