示例#1
0
def cast_to_DL(parameters: Parameters, dataset: str, model, optimal_step_size: int, weight_decay: int, iid: str) \
        -> DLParameters:
    """Casts a Parameters class in a DLParameters class which is the specific format for deep learning."""
    parameters.__class__ = DLParameters
    parameters.initialize_DL_params(dataset, model, optimal_step_size,
                                    weight_decay, iid)
    return parameters
示例#2
0
    def setUpClass(cls):
        """ get_some_resource() is slow, to avoid calling it for each test use setUpClass()
            and store the result as class variable
        """
        super(PerformancesTest, cls).setUpClass()

        ### RMSE ###

        # Creating cost models which will be used to computed cost/loss, gradients, L ...
        cls.linear_cost_models = build_several_cost_model(RMSEModel, linear_X, linear_Y, nb_devices)

        # Defining parameters for the performances test.
        cls.linear_params = Parameters(n_dimensions=dim_test + 1,
                                       nb_devices=nb_devices,
                                       up_compression_model=SQuantization(1, dim_test + 1),
                                       step_formula=deacreasing_step_size,
                                       nb_epoch=nb_epoch,
                                       use_averaging=False,
                                       cost_models=cls.linear_cost_models,
                                       stochastic=True)

        obj_min_by_N_descent = SGD_Descent(Parameters(n_dimensions=dim_test + 1,
                                                      nb_devices=nb_devices,
                                                      nb_epoch=200,
                                                      momentum=0.,
                                                      verbose=True,
                                                      cost_models=cls.linear_cost_models,
                                                      stochastic=False,
                                                      bidirectional=False
                                                      ))
        obj_min_by_N_descent.run(cls.linear_cost_models)
        cls.linear_obj = obj_min_by_N_descent.train_losses[-1]

        # For LOGISTIC:
        
        cls.logistic_cost_models = build_several_cost_model(LogisticModel, logistic_X, logistic_Y, nb_devices)
        
        # Defining parameters for the performances test.
        cls.logistic_params = Parameters(n_dimensions=2,
                                         nb_devices=nb_devices,
                                         up_compression_model=SQuantization(1, 3),
                                         step_formula=deacreasing_step_size,
                                         nb_epoch=nb_epoch,
                                         use_averaging=False,
                                         cost_models=cls.logistic_cost_models,
                                         stochastic=True)

        obj_min_by_N_descent = SGD_Descent(Parameters(n_dimensions=2,
                                                      nb_devices=nb_devices,
                                                      nb_epoch=200,
                                                      momentum=0.,
                                                      verbose=True,
                                                      cost_models=cls.logistic_cost_models,
                                                      stochastic=False,
                                                      bidirectional=False
                                                      ))
        obj_min_by_N_descent.run(cls.logistic_cost_models)
        cls.logistic_obj = obj_min_by_N_descent.train_losses[-1]
示例#3
0
 def define(self,
            cost_models,
            n_dimensions: int,
            nb_devices: int,
            up_compression_model: CompressionModel,
            down_compression_model: CompressionModel,
            step_formula=None,
            nb_epoch: int = NB_EPOCH,
            fraction_sampled_workers: int = 1.,
            use_averaging=False,
            stochastic=True,
            streaming=False,
            batch_size=1) -> Parameters:
     return Parameters(n_dimensions=n_dimensions,
                       nb_devices=nb_devices,
                       nb_epoch=nb_epoch,
                       fraction_sampled_workers=fraction_sampled_workers,
                       step_formula=step_formula,
                       up_compression_model=SQuantization(0),
                       down_compression_model=SQuantization(0),
                       stochastic=stochastic,
                       streaming=streaming,
                       batch_size=batch_size,
                       cost_models=cost_models,
                       use_averaging=use_averaging,
                       use_up_memory=False)
示例#4
0
def compute_number_of_bits(type_params: Parameters, nb_epoch: int, compress_model: bool):
    """Computing the theoretical number of bits used by an algorithm (with Elias encoding)."""
    # Initialization, the first element needs to be removed at the end.
    number_of_bits = np.array([0 for i in range(nb_epoch)])
    if isinstance(type_params, DLParameters):
        model = type_params.model()
        for p in model.parameters():
            d = p.numel()
            nb_bits = compute_number_of_bits_by_layer(type_params, d, nb_epoch, compress_model)
            number_of_bits = number_of_bits + nb_bits
        return number_of_bits
    else:
        d = type_params.n_dimensions
        return compute_number_of_bits_by_layer(type_params, d, nb_epoch, compress_model)
示例#5
0
 def define(self, n_dimensions: int, nb_devices: int, quantization_param: int = 0, step_formula=None,
            momentum: float = 0, nb_epoch: int = NB_EPOCH, use_averaging=False,
            model: ACostModel = RMSEModel(), stochastic=True):
     return Parameters(n_dimensions=n_dimensions,
                       nb_devices=nb_devices,
                       nb_epoch=nb_epoch,
                       step_formula=step_formula,
                       quantization_param=0,
                       momentum=momentum,
                       verbose=False,
                       stochastic=stochastic,
                       bidirectional=False,
                       cost_model=model,
                       use_averaging=use_averaging
                       )
示例#6
0
 def define(self, n_dimensions: int, nb_devices: int, quantization_param: int, step_formula=None,
            momentum: float = 0, nb_epoch: int = NB_EPOCH, use_averaging=False, model: ACostModel = RMSEModel(),
            stochastic=True):
     return Parameters(n_dimensions=n_dimensions,
                       nb_devices=nb_devices,
                       nb_epoch=nb_epoch,
                       step_formula=step_formula,
                       quantization_param=1,
                       learning_rate=0,
                       momentum=momentum,
                       verbose=False,
                       stochastic=stochastic,
                       cost_model=model,
                       use_averaging=use_averaging,
                       bidirectional=True,
                       double_use_memory=False,
                       compress_gradients=True
                       )
示例#7
0
def cast_to_DL(parameters: Parameters, dataset: str, model, optimal_step_size: int, weight_decay: int) -> DLParameters:
    parameters.__class__ = DLParameters
    parameters.initialize_DL_params(dataset, model, optimal_step_size, weight_decay)
    return parameters
示例#8
0
    ### Following takes around 5 minutes. ###

    # 1) Generating data.
    w_true = generate_param(dim_notebook)
    X, Y = build_data_linear(w_true, n_dimensions=dim_notebook,
                             n_devices=nb_devices, with_seed=False, without_noise=False)

    # 2) Creating cost models which will be used to computed cost/loss, gradients, L ...
    cost_models = build_several_cost_model(RMSEModel, X, Y, nb_devices)

    # 3) Computing objective function.
    obj_min_descent = SGD_Descent(Parameters(n_dimensions=dim_notebook,
                                             nb_devices=nb_devices_for_the_run,
                                             nb_epoch=600,
                                             momentum=0.,
                                             verbose=True,
                                             cost_models=cost_models,
                                             stochastic=False,
                                             bidirectional=False
                                             ))
    obj_min_descent.run(cost_models)
    obj_min = obj_min_descent.train_losses[-1]

    # 4) Running descent for two algorithms: Diana and Artemis
    all_descent = {}
    myX = X[:nb_devices_for_the_run]
    myY = Y[:nb_devices_for_the_run]
    X_number_of_bits = []
    for type_params in tqdm([Diana(), VanillaSGD()]):
        multiple_sg_descent = multiple_run_descent(type_params, cost_models=cost_models, nb_epoch=10, compression_model=SQuantization(1, dim_notebook))
        all_descent[type_params.name()] = multiple_sg_descent
    def setUpClass(cls):
        """ get_some_resource() is slow, to avoid calling it for each test use setUpClass()
            and store the result as class variable
        """
        super(PerformancesTest, cls).setUpClass()

        ### RMSE ###

        # Defining parameters for the performances test.
        cls.linear_params = Parameters(n_dimensions=dim_test + 1,
                                       nb_devices=nb_devices,
                                       quantization_param=1,
                                       step_formula=None,
                                       nb_epoch=nb_epoch,
                                       use_averaging=False,
                                       cost_model=RMSEModel(),
                                       stochastic=True)

        obj_min_by_N_descent = FL_VanillaSGD(
            Parameters(
                n_dimensions=dim_test + 1,
                nb_devices=nb_devices,
                nb_epoch=200,
                momentum=0.,
                quantization_param=0,
                verbose=True,
                cost_model=RMSEModel(),
                stochastic=False,
                bidirectional=False,
            ))
        obj_min_by_N_descent.set_data(linear_X[:nb_devices],
                                      linear_Y[:nb_devices])
        obj_min_by_N_descent.run()
        cls.linear_obj = obj_min_by_N_descent.losses[-1]

        # For LOGISTIC:

        # Defining parameters for the performances test.
        cls.logistic_params = Parameters(n_dimensions=2,
                                         nb_devices=nb_devices,
                                         quantization_param=1,
                                         step_formula=None,
                                         nb_epoch=nb_epoch,
                                         use_averaging=False,
                                         cost_model=LogisticModel(),
                                         stochastic=True)

        obj_min_by_N_descent = FL_VanillaSGD(
            Parameters(
                n_dimensions=2,
                nb_devices=nb_devices,
                nb_epoch=200,
                momentum=0.,
                quantization_param=0,
                verbose=True,
                cost_model=LogisticModel(),
                stochastic=False,
                bidirectional=False,
            ))
        obj_min_by_N_descent.set_data(logistic_X[:nb_devices],
                                      logistic_Y[:nb_devices])
        obj_min_by_N_descent.run()
        cls.logistic_obj = obj_min_by_N_descent.losses[-1]
示例#10
0
    X, Y = build_data_linear(w_true,
                             n_dimensions=dim_notebook - 1,
                             n_devices=nb_devices,
                             with_seed=False,
                             without_noise=False)
    X = add_bias_term(X)  # Add a column of ones.

    # 2) Creating cost models which will be used to computed cost/loss, gradients, L ...
    cost_models = build_several_cost_model(RMSEModel, X, Y, nb_devices)

    # 3) Computing objective function.
    obj_min_descent = SGD_Descent(
        Parameters(n_dimensions=dim_notebook,
                   nb_devices=nb_devices,
                   nb_epoch=4000,
                   momentum=0.,
                   verbose=True,
                   cost_models=cost_models,
                   stochastic=False), None)
    obj_min_descent.run(cost_models)
    obj_min = obj_min_descent.train_losses[-1]

    # 4) Defining settings of the run.
    compression = SQuantization(level=1, dim=dim_notebook, norm=2)
    step_size = deacreasing_step_size

    # 4) Running descent for two algorithms: Vanilla SGD, Diana and Artemis.
    all_descent = {}
    for type_params in tqdm([VanillaSGD(), Diana(), Artemis()]):
        multiple_sg_descent = multiple_run_descent(
            type_params,