def setUpClass(cls):
""" get_some_resource() is slow, to avoid calling it for each test use setUpClass()
and store the result as class variable
"""
super(PerformancesTest, cls).setUpClass()
### RMSE ###
# Creating cost models which will be used to computed cost/loss, gradients, L ...
cls.linear_cost_models = build_several_cost_model(RMSEModel, linear_X, linear_Y, nb_devices)
# Defining parameters for the performances test.
cls.linear_params = Parameters(n_dimensions=dim_test + 1,
nb_devices=nb_devices,
up_compression_model=SQuantization(1, dim_test + 1),
step_formula=deacreasing_step_size,
nb_epoch=nb_epoch,
use_averaging=False,
cost_models=cls.linear_cost_models,
stochastic=True)
obj_min_by_N_descent = SGD_Descent(Parameters(n_dimensions=dim_test + 1,
nb_devices=nb_devices,
nb_epoch=200,
momentum=0.,
verbose=True,
cost_models=cls.linear_cost_models,
stochastic=False,
bidirectional=False
))
obj_min_by_N_descent.run(cls.linear_cost_models)
cls.linear_obj = obj_min_by_N_descent.train_losses[-1]
# For LOGISTIC:
cls.logistic_cost_models = build_several_cost_model(LogisticModel, logistic_X, logistic_Y, nb_devices)
# Defining parameters for the performances test.
cls.logistic_params = Parameters(n_dimensions=2,
nb_devices=nb_devices,
up_compression_model=SQuantization(1, 3),
step_formula=deacreasing_step_size,
nb_epoch=nb_epoch,
use_averaging=False,
cost_models=cls.logistic_cost_models,
stochastic=True)
obj_min_by_N_descent = SGD_Descent(Parameters(n_dimensions=2,
nb_devices=nb_devices,
nb_epoch=200,
momentum=0.,
verbose=True,
cost_models=cls.logistic_cost_models,
stochastic=False,
bidirectional=False
))
obj_min_by_N_descent.run(cls.logistic_cost_models)
cls.logistic_obj = obj_min_by_N_descent.train_losses[-1]
def setUpClass(cls):
""" get_some_resource() is slow, to avoid calling it for each test use setUpClass()
and store the result as class variable
"""
super(TestRandomizedAlgo, cls).setUpClass()
cls.cost_models = build_several_cost_model(RMSEModel, X, Y, number_of_device)
cls.params = RandMCM().define(n_dimensions=dim,
nb_devices=number_of_device,
up_compression_model=SQuantization(1, dim),
down_compression_model=SQuantization(1, dim),
nb_epoch=1,
cost_models=cls.cost_models,
step_formula=constant_step_size)
cls.params.down_learning_rate = 1 / cls.params.down_compression_model.omega_c
cls.params.up_learning_rate = 1
cls.workers = [Worker(i, cls.params, LocalArtemisUpdate) for i in range(number_of_device)]
def run_experiments(nb_devices: int,
stochastic: bool,
dataset: str,
iid: str,
algos: str,
use_averaging: bool = False,
scenario: str = None,
fraction_sampled_workers: int = 1,
plot_only: bool = False,
modify_run=None):
print("Running with following parameters: {0}".format([
"{0} -> {1}".format(k, v) for (k, v) in zip(locals().keys(),
locals().values())
]))
assert dataset in ["quantum", "superconduct", "mushroom", "phishing", "a9a", "abalone", "covtype", 'synth_logistic',
'madelon', 'gisette', 'w8a', 'synth_linear_noised', 'synth_linear_nonoised'], \
"The available dataset are ['quantum', 'superconduct', 'synth_linear_noised', 'synth_linear_nonoised']."
assert iid in ['iid', 'non-iid'], "The iid option are ['iid', 'non-iid']."
assert scenario in [
None, "compression", "step", "alpha"
], "The possible scenario are [None, 'compression', 'step', 'alpha']."
# tracemalloc.start()
# hp = hpy()
data_path, pickle_path, algos_pickle_path, picture_path = create_path_and_folders(
nb_devices, dataset, iid, algos, fraction_sampled_workers)
list_algos = choose_algo(algos, stochastic, fraction_sampled_workers)
nb_devices = nb_devices
nb_epoch = 600 if stochastic else 400
iid_data = True if iid == 'iid' else False
batch_size = batch_sizes[dataset]
model = models[dataset]
# Select the correct dataset
if dataset == "a9a":
X, Y, dim_notebook = prepare_a9a(nb_devices,
data_path=data_path,
pickle_path=pickle_path,
iid=iid_data)
if dataset == "abalone":
X, Y, dim_notebook = prepare_abalone(nb_devices,
data_path=data_path,
pickle_path=pickle_path,
iid=iid_data)
if dataset == "covtype":
X, Y, dim_notebook = prepare_covtype(nb_devices,
data_path=data_path,
pickle_path=pickle_path,
iid=iid_data)
if dataset == "gisette":
X, Y, dim_notebook = prepare_gisette(nb_devices,
data_path=data_path,
pickle_path=pickle_path,
iid=iid_data)
if dataset == "madelon":
X, Y, dim_notebook = prepare_madelon(nb_devices,
data_path=data_path,
pickle_path=pickle_path,
iid=iid_data)
if dataset == "mushroom":
X, Y, dim_notebook = prepare_mushroom(nb_devices,
data_path=data_path,
pickle_path=pickle_path,
iid=iid_data)
if dataset == "quantum":
X, Y, dim_notebook = prepare_quantum(nb_devices,
data_path=data_path,
pickle_path=pickle_path,
iid=iid_data)
if dataset == "phishing":
X, Y, dim_notebook = prepare_phishing(nb_devices,
data_path=data_path,
pickle_path=pickle_path,
iid=iid_data)
elif dataset == "superconduct":
X, Y, dim_notebook = prepare_superconduct(nb_devices,
data_path=data_path,
pickle_path=pickle_path,
iid=iid_data)
if dataset == "w8a":
X, Y, dim_notebook = prepare_w8a(nb_devices,
data_path=data_path,
pickle_path=pickle_path,
iid=iid_data)
elif dataset == 'synth_logistic':
nb_epoch = 100 if stochastic else 400
dim_notebook = 2
if not file_exist("{0}/data.pkl".format(pickle_path)):
w = torch.FloatTensor([10, 10]).to(dtype=torch.float64)
X, Y = build_data_logistic(w,
n_dimensions=2,
n_devices=nb_devices,
with_seed=False)
pickle_saver((X, Y), pickle_path + "/data")
else:
X, Y = pickle_loader(pickle_path + "/data")
elif dataset == 'synth_linear_noised':
nb_epoch = 100 if stochastic else 400
dim_notebook = 20
if not file_exist("{0}/data.pkl".format(pickle_path)):
w_true = generate_param(dim_notebook - 1)
X, Y = build_data_linear(w_true,
n_dimensions=dim_notebook - 1,
n_devices=nb_devices,
with_seed=False,
without_noise=False)
X = add_bias_term(X)
pickle_saver((X, Y), pickle_path + "/data")
else:
X, Y = pickle_loader(pickle_path + "/data")
elif dataset == 'synth_linear_nonoised':
nb_epoch = 100 if stochastic else 400
dim_notebook = 20
if not file_exist("{0}/data.pkl".format(pickle_path)):
w_true = generate_param(dim_notebook - 1)
X, Y = build_data_linear(w_true,
n_dimensions=dim_notebook - 1,
n_devices=nb_devices,
with_seed=False,
without_noise=True)
X = add_bias_term(X)
pickle_saver((X, Y), pickle_path + "/data")
else:
X, Y = pickle_loader(pickle_path + "/data")
default_level_of_quantization = 1 if fraction_sampled_workers == 1 else 2
compression_by_default = SQuantization(1, dim_notebook, norm=2)
values_compression = [
SQuantization(0, dim_notebook, norm=2),
SQuantization(16, dim_notebook, norm=2),
SQuantization(8, dim_notebook, norm=2),
SQuantization(4, dim_notebook, norm=2),
SQuantization(3, dim_notebook, norm=2),
SQuantization(2, dim_notebook, norm=2),
SQuantization(1, dim_notebook, norm=2)
]
label_compression = ["SGD"] + [
str(value.omega_c)[:4] for value in values_compression[1:]
]
values_alpha = [
SQuantization(1, dim_notebook, norm=2, constant=0.25),
SQuantization(1, dim_notebook, norm=2, constant=0.5),
SQuantization(1, dim_notebook, norm=2, constant=1),
SQuantization(1, dim_notebook, norm=2, constant=2),
SQuantization(1, dim_notebook, norm=2, constant=4),
SQuantization(1, dim_notebook, norm=2, constant=8)
]
label_alpha = [str(value.constant) for value in values_alpha]
# Creating cost models which will be used to computed cost/loss, gradients, L ...
cost_models = build_several_cost_model(model, X, Y, nb_devices)
# hp.setrelheap()
if not file_exist("{0}/obj_min.pkl".format(pickle_path)) or not file_exist(
"{0}/grads_min.pkl".format(pickle_path)):
obj_min_by_N_descent = SGD_Descent(
Parameters(n_dimensions=dim_notebook,
nb_devices=nb_devices,
nb_epoch=40000,
momentum=0.,
verbose=True,
cost_models=cost_models,
stochastic=False), None)
obj_min_by_N_descent.run(cost_models)
obj_min = obj_min_by_N_descent.train_losses[-1]
pickle_saver(obj_min, "{0}/obj_min".format(pickle_path))
grads_min = [
worker.local_update.g_i for worker in obj_min_by_N_descent.workers
]
pickle_saver(grads_min, "{0}/grads_min".format(pickle_path))
# Choice of step size
if stochastic and batch_size == 1:
step_size = iid_step_size
else:
step_size = batch_step_size
if 'synth' in dataset and stochastic:
step_size = deacreasing_step_size
stochasticity = 'sto' if stochastic else "full"
if stochastic:
experiments_settings = "{0}-{1}-b{2}".format(
compression_by_default.get_name(), stochasticity, batch_size)
else:
experiments_settings = "{0}-{1}".format(
compression_by_default.get_name(), stochasticity)
if not plot_only:
if scenario == "compression":
run_for_different_scenarios(
cost_models,
list_algos[1:],
values_compression,
label_compression,
experiments_settings=experiments_settings,
logs_file=algos_pickle_path,
batch_size=batch_size,
stochastic=stochastic,
step_formula=step_size,
scenario=scenario)
elif scenario == "step":
run_for_different_scenarios(
cost_models,
list_algos,
step_formula,
label_step_formula,
experiments_settings=experiments_settings,
logs_file=algos_pickle_path,
batch_size=batch_size,
stochastic=stochastic,
scenario=scenario,
compression=compression_by_default)
elif scenario == "alpha":
run_for_different_scenarios(
cost_models, [Artemis(), MCM(), RandMCM()],
values_alpha,
label_alpha,
experiments_settings=experiments_settings,
logs_file=algos_pickle_path,
batch_size=batch_size,
stochastic=stochastic,
scenario=scenario,
compression=compression_by_default)
else:
run_one_scenario(cost_models=cost_models,
list_algos=list_algos,
logs_file=algos_pickle_path,
batch_size=batch_size,
experiments_settings=experiments_settings,
stochastic=stochastic,
nb_epoch=nb_epoch,
step_size=step_size,
use_averaging=use_averaging,
compression=compression_by_default,
fraction_sampled_workers=fraction_sampled_workers,
modify_run=modify_run)
# snapshot = tracemalloc.take_snapshot()
# top_stats = snapshot.statistics('lineno')
#
# print("[ Top 10 ]")
# for stat in top_stats[:10]:
# print(stat)
#
# h = hp.heap()
# print(h)
obj_min = pickle_loader("{0}/obj_min".format(pickle_path))
print("Obj min:", obj_min)
if scenario is None:
res = pickle_loader("{0}/descent-{1}".format(algos_pickle_path,
experiments_settings))
# Plotting without averaging
plot_error_dist(res.get_loss(obj_min),
res.names,
all_error=res.get_std(obj_min),
x_legend="Number of passes on data",
picture_name="{0}/it-noavg-{1}".format(
picture_path, experiments_settings))
plot_error_dist(res.get_loss(obj_min),
res.names,
x_points=res.X_number_of_bits,
x_legend="Communicated bits",
all_error=res.get_std(obj_min),
picture_name="{0}/bits-noavg-{1}".format(
picture_path, experiments_settings))
# Plotting with averaging
if use_averaging:
plot_error_dist(res.get_loss(obj_min, averaged=True),
res.names,
all_error=res.get_std(obj_min, averaged=True),
x_legend="Number of passes on data (Avg)",
picture_name="{0}/it-avg-{1}".format(
picture_path, experiments_settings))
plot_error_dist(res.get_loss(obj_min, averaged=True),
res.names,
x_points=res.X_number_of_bits,
all_error=res.get_std(obj_min, averaged=True),
x_legend="Communicated bits (Avg)",
picture_name="{0}/bits-avg-{1}".format(
picture_path, experiments_settings))
if scenario == "step":
res = pickle_loader("{0}/{1}-{2}".format(algos_pickle_path, scenario,
experiments_settings))
plot_error_dist(res.get_loss(obj_min),
res.names,
all_error=res.get_std(obj_min),
x_legend="Step size ({0}, {1})".format(
iid,
str(compression_by_default.omega_c)[:4]),
one_on_two_points=True,
xlabels=label_step_formula,
picture_name="{0}/{1}-{2}".format(
picture_path, scenario, experiments_settings))
# res = pickle_loader("{0}/{1}-optimal-{2}".format(algos_pickle_path, scenario, experiments_settings))
#
# plot_error_dist(res.get_loss(obj_min), res.names, all_error=res.get_std(obj_min),
# x_legend="(non-iid)", ylim=True,
# picture_name="{0}/{1}-optimal-it-{2}".format(picture_path, scenario, experiments_settings))
# plot_error_dist(res.get_loss(obj_min), res.names, x_points=res.X_number_of_bits,
# x_legend="Communicated bits", all_error=res.get_std(obj_min), ylim=True,
# picture_name="{0}/{1}-optimal-bits-{2}".format(picture_path, scenario, experiments_settings))
if scenario in ["compression", "alpha"]:
create_folder_if_not_existing("{0}/{1}".format(picture_path, scenario))
res = pickle_loader("{0}/{1}-{2}".format(algos_pickle_path, scenario,
experiments_settings))
plot_error_dist(res.get_loss(obj_min),
res.names,
all_error=res.get_std(obj_min),
x_legend=[
"$\omega_c$",
"$\\alpha_{dwn}^{-1} \\times (\omega_c + 1)^{-1}$"
][scenario == "alpha"],
one_on_two_points=True,
xlabels=[label_compression,
label_alpha][scenario == "alpha"],
picture_name="{0}/{1}/{2}".format(
picture_path, scenario, experiments_settings))
res_by_algo = pickle_loader("{0}/{1}-descent_by_algo-{2}".format(
algos_pickle_path, scenario, experiments_settings))
for key in res_by_algo.keys():
res = res_by_algo[key]
plot_error_dist(res.get_loss(obj_min),
res.names,
all_error=res.get_std(obj_min),
x_legend="Number of passes on data",
ylim=1,
picture_name="{0}/{1}/{2}-it-noavg-{3}".format(
picture_path, scenario, key,
experiments_settings))
def run_experiments(nb_devices: int, stochastic: bool, dataset: str, iid: str, algos: str, use_averaging: bool = False,
scenario: str = None, fraction_sampled_workers: int = 0.5, plot_only: bool = True):
print("Running with following parameters: {0}".format(["{0} -> {1}".format(k, v) for (k, v)
in zip(locals().keys(), locals().values())]))
assert dataset in ["quantum", "superconduct", 'synth_logistic', 'synth_linear_noised', 'synth_linear_nonoised'], \
"The available dataset are ['quantum', 'superconduct', 'synth_linear_noised', 'synth_linear_nonoised']."
assert iid in ['iid', 'non-iid'], "The iid option are ['iid', 'non-iid']."
assert scenario in [None, "compression", "step"], "The possible scenario are [None, 'compression', 'step']."
data_path, pickle_path, algos_pickle_path, picture_path = create_path_and_folders(nb_devices, dataset, iid, algos, fraction_sampled_workers)
list_algos = choose_algo(algos, stochastic, fraction_sampled_workers)
nb_devices = nb_devices
nb_epoch = 100 if stochastic else 400
iid_data = True if iid == 'iid' else False
# Select the correct dataset
if dataset == "quantum":
X, Y, dim_notebook = prepare_quantum(nb_devices, data_path=data_path, pickle_path=pickle_path, iid=iid_data)
batch_size = 400
model = LogisticModel
nb_epoch = 500 if stochastic else 400
elif dataset == "superconduct":
X, Y, dim_notebook = prepare_superconduct(nb_devices, data_path= data_path, pickle_path=pickle_path, iid=iid_data)
batch_size = 50
model = RMSEModel
nb_epoch = 500 if stochastic else 400
elif dataset == 'synth_logistic':
dim_notebook = 2
batch_size = 1
model = LogisticModel
if not file_exist("{0}/data.pkl".format(pickle_path)):
w = torch.FloatTensor([10, 10]).to(dtype=torch.float64)
X, Y = build_data_logistic(w, n_dimensions=2, n_devices=nb_devices, with_seed=False)
pickle_saver((X, Y), pickle_path + "/data")
else:
X, Y = pickle_loader(pickle_path + "/data")
elif dataset == 'synth_linear_noised':
dim_notebook = 20
if not file_exist("{0}/data.pkl".format(pickle_path)):
w_true = generate_param(dim_notebook-1)
X, Y = build_data_linear(w_true, n_dimensions=dim_notebook-1,
n_devices=nb_devices, with_seed=False, without_noise=False)
X = add_bias_term(X)
pickle_saver((X, Y), pickle_path + "/data")
else:
X, Y = pickle_loader(pickle_path + "/data")
model = RMSEModel
batch_size = 1
elif dataset == 'synth_linear_nonoised':
dim_notebook = 20
if not file_exist("{0}/data.pkl".format(pickle_path)):
w_true = generate_param(dim_notebook-1)
X, Y = build_data_linear(w_true, n_dimensions=dim_notebook-1,
n_devices=nb_devices, with_seed=False, without_noise=True)
X = add_bias_term(X)
pickle_saver((X, Y), pickle_path + "/data")
else:
X, Y = pickle_loader(pickle_path + "/data")
model = RMSEModel
batch_size = 1
compression_by_default = SQuantization(1, dim_notebook)
values_compression = [SQuantization(0, dim_notebook),
SQuantization(16, dim_notebook),
SQuantization(8, dim_notebook),
SQuantization(4, dim_notebook),
SQuantization(3, dim_notebook),
SQuantization(2, dim_notebook),
SQuantization(1, dim_notebook)
]
label_compression = ["SGD"] + [str(value.omega_c)[:4] for value in values_compression[1:]]
# Rebalancing cluster: the biggest one must not be more than 10times bigger than the smallest one.
X_r, Y_r = rebalancing_clusters(X, Y)
# Creating cost models which will be used to computed cost/loss, gradients, L ...
cost_models = build_several_cost_model(model, X_r, Y_r, nb_devices)
if not file_exist("{0}/obj_min.pkl".format(pickle_path)):
obj_min_by_N_descent = SGD_Descent(Parameters(n_dimensions=dim_notebook,
nb_devices=nb_devices,
nb_epoch=40000,
momentum=0.,
verbose=True,
cost_models=cost_models,
stochastic=False
))
obj_min_by_N_descent.run(cost_models)
obj_min = obj_min_by_N_descent.losses[-1]
pickle_saver(obj_min, "{0}/obj_min".format(pickle_path))
# Choice of step size
if stochastic and batch_size == 1:
step_size = iid_step_size
else:
step_size = batch_step_size
if 'synth' in dataset and stochastic:
step_size = deacreasing_step_size
stochasticity = 'sto' if stochastic else "full"
if not plot_only:
if scenario == "compression":
run_for_different_scenarios(cost_models, list_algos[1:], values_compression, label_compression,
filename=algos_pickle_path, batch_size=batch_size, stochastic=stochastic,
step_formula=step_size, scenario=scenario)
elif scenario == "step":
run_for_different_scenarios(cost_models, list_algos, step_formula, label_step_formula,
filename=algos_pickle_path, batch_size=batch_size, stochastic=stochastic,
scenario=scenario, compression=compression_by_default)
else:
run_one_scenario(cost_models=cost_models, list_algos=list_algos, filename=algos_pickle_path,
batch_size=batch_size, stochastic=stochastic, nb_epoch=nb_epoch, step_size=step_size,
use_averaging=use_averaging, compression=compression_by_default,
fraction_sampled_workers=fraction_sampled_workers)
obj_min = pickle_loader("{0}/obj_min".format(pickle_path))
if stochastic:
experiments_settings = "{0}-b{1}".format(stochasticity, batch_size)
else:
experiments_settings = stochasticity
if scenario is None:
res = pickle_loader("{0}/descent-{1}".format(algos_pickle_path, experiments_settings))
# Plotting without averaging
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook,
all_error=res.get_std(obj_min), x_legend="Number of passes on data\n({0})".format(iid),
picture_name="{0}/it-noavg-{1}".format(picture_path, experiments_settings))
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook,
x_points=res.X_number_of_bits, x_legend="Communicated bits ({0})".format(iid),
all_error=res.get_std(obj_min), picture_name="{0}/bits-noavg-{1}"
.format(picture_path, experiments_settings))
# Plotting with averaging
if use_averaging:
plot_error_dist(res.get_loss(obj_min, averaged=True), res.names, res.nb_devices,
dim_notebook, all_error=res.get_std(obj_min, averaged=True),
x_legend="Number of passes on data\n(Avg, {0})".format(iid),
picture_name="{0}/it-avg-{1}"
.format(picture_path, experiments_settings))
plot_error_dist(res.get_loss(obj_min, averaged=True), res.names, res.nb_devices, dim_notebook,
x_points=res.X_number_of_bits, all_error=res.get_std(obj_min, averaged=True),
x_legend="Communicated bits (Avg, {0})".format(iid),
picture_name="{0}/bits-avg-{1}"
.format(picture_path, experiments_settings))
if scenario == "step":
res = pickle_loader("{0}/{1}-{2}".format(algos_pickle_path, scenario, experiments_settings))
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook,
batch_size=batch_size,
all_error=res.get_std(obj_min),
x_legend="Step size ({0}, {1})".format(iid, str(compression_by_default.omega_c)[:4]),
one_on_two_points=True, xlabels=label_step_formula,
picture_name="{0}/{1}-{2}".format(picture_path, scenario, experiments_settings))
res = pickle_loader("{0}/{1}-optimal-{2}".format(algos_pickle_path, scenario, experiments_settings))
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook,
all_error=res.get_std(obj_min), batch_size=batch_size,
x_legend="(non-iid)", ylim=True,
picture_name="{0}/{1}-optimal-it-{2}".format(picture_path, scenario, experiments_settings))
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook,
x_points=res.X_number_of_bits, batch_size=batch_size,
x_legend="Communicated bits\n(non-iid)", all_error=res.get_std(obj_min), ylim=True,
picture_name="{0}/{1}-optimal-bits-{2}".format(picture_path, scenario, experiments_settings))
if scenario == "compression":
res = pickle_loader("{0}/{1}-{2}".format(algos_pickle_path, scenario, experiments_settings))
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook, batch_size=batch_size,
all_error=res.get_std(obj_min), x_legend="$\omega_c$ ({0})".format(iid),
one_on_two_points=True, xlabels=label_compression,
picture_name="{0}/{1}-{2}".format(picture_path, scenario, experiments_settings))
res = pickle_loader("{0}/{1}-optimal-{2}".format(algos_pickle_path, scenario, experiments_settings))
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook,
all_error=res.get_std(obj_min), batch_size=batch_size,
x_legend="(non-iid)", ylim=True,
picture_name="{0}/{1}-optimal-it-{2}".format(picture_path, scenario, experiments_settings))
plot_error_dist(res.get_loss(obj_min), res.names, res.nb_devices, dim_notebook,
x_points=res.X_number_of_bits, batch_size=batch_size,
x_legend="Communicated bits\n(non-iid)", all_error=res.get_std(obj_min), ylim=True,
picture_name="{0}/{1}-optimal-bits-{2}".format(picture_path, scenario, experiments_settings))
# How to use Artemis implementation.
dim_notebook = 20
nb_devices = 20
nb_devices_for_the_run = nb_devices
if __name__ == '__main__':
### Following takes around 5 minutes. ###
# 1) Generating data.
w_true = generate_param(dim_notebook)
X, Y = build_data_linear(w_true, n_dimensions=dim_notebook,
n_devices=nb_devices, with_seed=False, without_noise=False)
# 2) Creating cost models which will be used to computed cost/loss, gradients, L ...
cost_models = build_several_cost_model(RMSEModel, X, Y, nb_devices)
# 3) Computing objective function.
obj_min_descent = SGD_Descent(Parameters(n_dimensions=dim_notebook,
nb_devices=nb_devices_for_the_run,
nb_epoch=600,
momentum=0.,
verbose=True,
cost_models=cost_models,
stochastic=False,
bidirectional=False
))
obj_min_descent.run(cost_models)
obj_min = obj_min_descent.train_losses[-1]
# 4) Running descent for two algorithms: Diana and Artemis