Ejemplo n.º 1
0
def mini_smac(learn_delta):
    sample_num_m = s_mid
    sample_num_l = s_min
    if not learn_delta:
        sample_num_m = s_min

    start_time = time.time()
    config_space = create_configspace()
    types, bounds = get_types(config_space)
    num_hp = len(bounds)
    surrogate = RandomForestWithInstances(types=types, bounds=bounds)
    acquisition_func = EI(model=surrogate)
    acq_optimizer = RandomSampling(acquisition_func, config_space, n_samples=max(500, 50 * num_hp))
    X = []
    y = []
    y_delta = []
    c = []
    inc_y = 1.

    # Initial design.
    for _ in range(num_init):
        init_configs = sample_configurations(config_space, num_init)
        for config in init_configs:
            perf_t, _ = objective_function((config.get_dictionary(), sample_num_m))
            X.append(config)
            y.append(perf_t)
            if perf_t < inc_y:
                inc_y = perf_t
            c.append([time.time()-start_time, inc_y])
            if learn_delta:
                perf_l, _ = objective_function((config.get_dictionary(), sample_num_l))
                y_delta.append(perf_t - perf_l)
            else:
                y_delta.append(perf_t)

    # BO iterations.
    for _ in range(num_iter - num_init):
        # Update the surrogate model.
        surrogate.train(convert_configurations_to_array(X), np.array(y, dtype=np.float64))

        # Use EI acq to choose next config.
        incumbent = dict()
        best_index = np.argmin(y)
        incumbent['obj'] = y[best_index]
        incumbent['config'] = X[best_index]
        acquisition_func.update(model=surrogate, eta=incumbent)
        next_config = acq_optimizer.maximize(batch_size=1)[0]
        perf_t, _ = objective_function((next_config.get_dictionary(), sample_num_m))
        X.append(next_config)
        y.append(perf_t)
        if perf_t < inc_y:
            inc_y = perf_t
        c.append([time.time() - start_time, inc_y])
        if learn_delta:
            perf_l, _ = objective_function((config.get_dictionary(), sample_num_l))
            y_delta.append(perf_t - perf_l)
        else:
            y_delta.append(perf_t)

    return [convert_configurations_to_array(X), np.array(y_delta, dtype=np.float64)]
Ejemplo n.º 2
0
    def choose_next(self, num_config):
        if len(self.target_y[self.iterate_r[-1]]) == 0:
            return sample_configurations(self.config_space, num_config)

        conf_cnt = 0
        next_configs = []
        total_cnt = 0

        incumbent = dict()
        max_r = self.iterate_r[-1]
        best_index = np.argmin(self.target_y[max_r])
        incumbent['config'] = self.target_x[max_r][best_index]
        approximate_obj = self.weighted_surrogate.predict(
            convert_configurations_to_array([incumbent['config']]))[0]
        incumbent['obj'] = approximate_obj
        self.weighted_acquisition_func.update(model=self.weighted_surrogate,
                                              eta=incumbent)

        while conf_cnt < num_config and total_cnt < 2 * num_config:
            rand_config = self.weighted_acq_optimizer.maximize(batch_size=1)[0]
            if rand_config not in next_configs:
                next_configs.append(rand_config)
                conf_cnt += 1
            total_cnt += 1
        if conf_cnt < num_config:
            next_configs = expand_configurations(next_configs,
                                                 self.config_space, num_config)
        return next_configs
Ejemplo n.º 3
0
    def update_weight_vector(self):
        rho = self.rho
        max_r = self.iterate_r[-1]
        incumbent_configs = self.target_x[max_r]
        test_x = convert_configurations_to_array(incumbent_configs)
        test_y = minmax_normalization(self.target_y[max_r])

        r_list = self.weighted_surrogate.surrogate_r
        cur_confidence = self.weighted_surrogate.surrogate_weight
        curr_list = [cur_confidence[r] for r in r_list[:-1]]

        # calculate correlation coefficient.
        corrcoef_list = []
        for i, r in enumerate(r_list):
            mean, _ = self.weighted_surrogate.surrogate_container[r].predict(
                test_x)
            tmp_y = np.reshape(mean, -1)
            # corrcoef = np.corrcoef(np.vstack((test_y, tmp_y)))[0][1]/2 + 0.5
            corrcoef = max(0, np.corrcoef(np.vstack((test_y, tmp_y)))[0][1])
            corrcoef_list.append(corrcoef)
        corrcoef_list = np.array(corrcoef_list)
        self.logger.info('scale method %d, before normalization: %s' %
                         (self.scale_method, str(corrcoef_list)))

        if self.scale_method <= 4:
            corrcoef_list = corrcoef_list**self.scale_method / sum(
                corrcoef_list**self.scale_method)
        elif self.scale_method == 6:
            corrcoef_list = corrcoef_list**2 / sum(corrcoef_list**2)
        elif self.scale_method == 7:
            ref = corrcoef_list**2
            follow_list = (corrcoef_list != max(corrcoef_list)) * ref
            corrcoef_list = follow_list / sum(follow_list) * 0.5 + (
                corrcoef_list == max(corrcoef_list)) * 0.5

        self.logger.info('after normalization: %s' % str(corrcoef_list))

        if sum(np.isnan(corrcoef_list)) == len(corrcoef_list):
            corrcoef_list = [item for item in curr_list]
            self.logger.info('escape nan, current update', corrcoef_list)

        assert len(cur_confidence) == len(corrcoef_list)
        self.logger.info('cur rho %.3f; conf vector/update vector: %s/%s' %
                         (self.rho, str(cur_confidence), str(corrcoef_list)))

        updated_weights = list()
        for i, r in enumerate(r_list):
            if self.scale_method == 7:
                self.weighted_surrogate.surrogate_weight[r] = corrcoef_list[i]
            else:
                self.weighted_surrogate.surrogate_weight[
                    r] = corrcoef_list[i] * (1 - rho) + rho * cur_confidence[r]
            updated_weights.append(self.weighted_surrogate.surrogate_weight[r])
            print('update surrogate weight:', r,
                  self.weighted_surrogate.surrogate_weight[r])
            self.logger.info('update surrogate weight:%d-%.4f' %
                             (r, self.weighted_surrogate.surrogate_weight[r]))
        self.hist_weights.append(updated_weights)
        np.save('data/tmp_weights_%s.npy' % self.method_name,
                np.asarray(self.hist_weights))
Ejemplo n.º 4
0
    def choose_next_weighted(self, num_config):
        if len(self.target_y[self.iterate_r[-1]]) == 0:
            return sample_configurations(self.config_space, num_config)

        conf_cnt = 0
        next_configs = []
        total_cnt = 0

        while conf_cnt < num_config and total_cnt < 2 * num_config:
            # in Bayesian optimization, eliminate epsilon sampling.
            incumbent = dict()
            # TODO: problem-->use the best in maximal resource.
            # TODO: smac's optmization algorithm.
            max_r = self.iterate_r[-1]
            best_index = np.argmin(self.target_y[max_r])
            incumbent['config'] = self.target_x[max_r][best_index]
            approximate_obj = self.weighted_surrogate.predict(
                convert_configurations_to_array([incumbent['config']]))[0]
            incumbent['obj'] = approximate_obj

            self.weighted_acquisition_func.update(
                model=self.weighted_surrogate, eta=incumbent)
            rand_config = self.weighted_acq_optimizer.maximize(batch_size=1)[0]

            if rand_config not in next_configs:
                next_configs.append(rand_config)
                conf_cnt += 1
            total_cnt += 1

        if conf_cnt < num_config:
            next_configs = expand_configurations(next_configs,
                                                 self.config_space, num_config)
        return next_configs
Ejemplo n.º 5
0
    def choose_next(self, num_config):
        if len(self.incumbent_obj) < 2 * self.num_config:
            return sample_configurations(self.config_space, num_config)

        # print('choose next starts!')
        self.logger.info('train feature is: %s' % str(self.incumbent_configs[-5:]))
        self.logger.info('train target is: %s' % str(self.incumbent_obj))

        self.surrogate.train(convert_configurations_to_array(self.incumbent_configs),
                             np.array(self.incumbent_obj, dtype=np.float64))

        conf_cnt = 0
        total_cnt = 0
        next_configs = []
        while conf_cnt < num_config and total_cnt < 5 * num_config:
            incumbent = dict()
            best_index = np.argmin(self.incumbent_obj)
            incumbent['obj'] = self.incumbent_obj[best_index]
            incumbent['config'] = self.incumbent_configs[best_index]

            self.acquisition_func.update(model=self.surrogate, eta=incumbent)
            rand_config = self.acq_optimizer.maximize(batch_size=1)[0]
            if rand_config not in next_configs:
                next_configs.append(rand_config)
                conf_cnt += 1
            total_cnt += 1
        if conf_cnt < num_config:
            next_configs = expand_configurations(next_configs, self.config_space, num_config)
        return next_configs
Ejemplo n.º 6
0
    def choose_next(self, num_config):
        self.logger.info('LCNet: model-based choosing.')
        if len(self.incumbent_obj) <= 0:
            return sample_configurations(self.config_space, num_config)
        self.logger.info('start to training LCNet, training data shape: %s' % str(self.lc_training_x.shape))
        self.lcnet_model.train(self.lc_training_x, self.lc_training_y)

        next_configs = []
        random_configs = sample_configurations(self.config_space, 50 * self.num_config)
        random_configs_data = convert_configurations_to_array(random_configs)
        x_test = None
        for i in range(random_configs_data.shape[0]):
            x = np.concatenate((random_configs_data[i, None, :], np.array([[1.0]])), axis=1)
            if x_test is None:
                x_test = x
            else:
                x_test = np.concatenate((x_test, x), 0)
        m, v = self.lcnet_model.predict(x_test)
        sorted_configs = [random_configs[i] for i in np.argsort(-m)]
        print(sorted_configs[:5])
        number_flag = False
        for config in sorted_configs:
            if config not in next_configs:
                next_configs.append(config)
            if len(next_configs) == num_config:
                number_flag = True
                break
        if not number_flag:
            next_configs = expand_configurations(next_configs, self.config_space, num_config)
            self.logger.warning('MBHB: add random configuration here.' + '=' * 50)
        return next_configs
Ejemplo n.º 7
0
    def choose_next(self, num_config, r, mode):
        # different types of mode.
        if mode == 'Hybrid':
            mode = 'Backward' if self.iterate_id % 2 == 0 else 'Forward'

        if mode == 'Forward':
            if r != self.R:
                r *= self.eta
        elif mode == 'Backward':
            if r != 1:
                r /= self.eta
            else:
                r = self.R

        # TODO: in different types, this condition may not needed any more.
        n_exp = len(self.target_y[r])
        if n_exp < 2 * self.num_config:
            return sample_configurations(self.config_space, num_config)

        self.logger.info('train feature is: %s' % str(self.target_x[r]))
        self.logger.info('train target is: %s' % str(self.target_y[r]))

        self.surrogate.train(convert_configurations_to_array(self.target_x[r]),
                             np.array(self.target_y[r], dtype=np.float64))

        conf_cnt = 0
        next_configs = []
        total_cnt = 0
        # TODO: acceleration, maximize a batch of candidates.
        while conf_cnt < num_config and total_cnt < 5 * num_config:
            rand_config = None
            if random.uniform(0, 1) < self.init_tradeoff:
                rand_config = self.config_space.sample_configuration(1)
            else:
                # print('use surrogate to produce candidate.')
                incumbent = dict()
                incumbent['obj'] = np.min(self.target_y[r])
                incumbent['config'] = self.target_x[r][np.argmin(
                    self.target_y[r])]

                self.acquisition_func.update(model=self.surrogate,
                                             eta=incumbent)
                rand_config = self.acq_optimizer.maximize(batch_size=1)[0]
            if rand_config not in next_configs:
                next_configs.append(rand_config)
                conf_cnt += 1
            total_cnt += 1

        if conf_cnt < num_config:
            next_configs = expand_configurations(next_configs,
                                                 self.config_space, num_config)

        return next_configs
Ejemplo n.º 8
0
    def choose_next(self, num_config):
        if len(self.incumbent_obj) < 3:
            return sample_configurations(self.config_space, num_config)

        self.logger.info('BO Training - X: %s' %
                         str(self.incumbent_configs[-5:]))
        self.logger.info('BO Training - Y: %s' % str(self.incumbent_obj))
        self.surrogate.train(
            convert_configurations_to_array(self.incumbent_configs),
            np.array(self.incumbent_obj, dtype=np.float64))

        conf_cnt = 0
        total_cnt = 0
        _next_configs = []
        while conf_cnt < num_config and total_cnt < 5 * num_config:
            incumbent = dict()
            best_index = np.argmin(self.incumbent_obj)
            incumbent['obj'] = self.incumbent_obj[best_index]
            incumbent['config'] = self.incumbent_configs[best_index]

            self.acquisition_func.update(model=self.surrogate, eta=incumbent)
            rand_config = self.acq_optimizer.maximize(batch_size=1)[0]
            if rand_config not in _next_configs:
                _next_configs.append(rand_config)
                conf_cnt += 1
            total_cnt += 1
        if conf_cnt < num_config:
            _next_configs = expand_configurations(_next_configs,
                                                  self.config_space,
                                                  num_config)

        next_configs = []

        # Epsilon greedy
        for config in _next_configs:
            if random.random() < self.p:
                next_configs.append(
                    sample_configurations(self.config_space, 1)[0])
            else:
                next_configs.append(config)

        return next_configs
Ejemplo n.º 9
0
    def stop_early(self, T):
        configs_data = convert_configurations_to_array(T)
        x_test = None
        for i in range(configs_data.shape[0]):
            x = np.concatenate((configs_data[i, None, :], np.array([[1.0]])), axis=1)
            if x_test is None:
                x_test = x
            else:
                x_test = np.concatenate((x_test, x), 0)

        m, v = self.lcnet_model.predict(x_test)
        best_accuracy = 1 - self.global_incumbent
        s = np.sqrt(v)
        less_p = norm.cdf((best_accuracy - m) / s)
        self.logger.info('early stop prob: %s' % str(less_p))
        early_stop_flag = (less_p >= self.es_rho)

        self.logger.info('early stop vector: %s' % str(early_stop_flag))

        for i, flag in enumerate(early_stop_flag):
            if flag:
                self.incumbent_configs.append(T[i])
                self.incumbent_obj.append(m[i])
        return early_stop_flag
Ejemplo n.º 10
0
    def choose_next(self, num_config):
        if len(self.incumbent_obj) < 3:
            return sample_configurations(self.config_space, num_config)

        self.logger.info('Train feature is: %s' %
                         str(self.incumbent_configs[:5]))
        self.logger.info('Train target is: %s' % str(self.incumbent_obj))
        self.surrogate.train(
            convert_configurations_to_array(self.incumbent_configs),
            np.array(self.incumbent_obj, dtype=np.float64))

        config_cnt = 0
        total_sample_cnt = 0
        config_candidates = []
        while config_cnt < num_config and total_sample_cnt < 3 * num_config:
            if random.random() < self.p:
                rand_config = self.config_space.sample_configuration(1)
            else:
                # print('use surrogate to produce candidate.')
                incumbent = dict()
                best_index = np.argmin(self.incumbent_obj)
                incumbent['obj'] = self.incumbent_obj[best_index]
                incumbent['config'] = self.incumbent_configs[best_index]

                self.acquisition_func.update(model=self.surrogate,
                                             eta=incumbent)
                rand_config = self.acq_optimizer.maximize(batch_size=1)[0]
            if rand_config not in config_candidates:
                config_candidates.append(rand_config)
                config_cnt += 1
            total_sample_cnt += 1
        if config_cnt < num_config:
            config_candidates = expand_configurations(config_candidates,
                                                      self.config_space,
                                                      num_config)
        return config_candidates
Ejemplo n.º 11
0
    def update_weight(self):
        max_r = self.iterate_r[-1]
        r_list = self.iterate_r

        incumbent_configs = self.target_x[max_r]
        test_x = convert_configurations_to_array(incumbent_configs)
        test_y = minmax_normalization(self.target_y[max_r])

        predictions = []
        for i, r in enumerate(r_list[:-1]):
            mean, _ = self.weighted_surrogate.surrogate_container[r].predict(
                test_x)
            predictions.append(mean.flatten().tolist())
        predictions.append(
            self.obtain_cv_prediction(test_x, np.array(test_y,
                                                       dtype=np.float64)))
        solution, status = self.solve_optpro(
            np.mat(predictions).T,
            np.mat(test_y).T)
        if status:
            solution[solution < 1e-3] = 0.
            self.logger.info('New weight: %s' % str(solution))
            for i, r in enumerate(r_list):
                self.weighted_surrogate.surrogate_weight[r] = solution[i]
Ejemplo n.º 12
0
    def iterate(self, skip_last=0):

        for s in reversed(range(self.s_max + 1)):

            if self.update_enable and self.weight_update_id > self.s_max:
                self.update_weight_vector()
            self.weight_update_id += 1

            # initial number of configurations
            n = int(ceil(self.B / self.R / (s + 1) * self.eta**s))
            # initial number of iterations per config
            r = int(self.R * self.eta**(-s))

            # choose a batch of configurations in different mechanisms.
            start_time = time.time()
            if self.info_type != 'Weighted':
                T = self.choose_next(n, r, self.info_type)
            else:
                T = self.choose_next_weighted(n)
            time_elapsed = time.time() - start_time
            self.logger.info("choosing next configurations took %.2f sec." %
                             time_elapsed)

            extra_info = None
            last_run_num = None

            for i in range((s + 1) - int(skip_last)):  # changed from s + 1

                # Run each of the n configs for <iterations>
                # and keep best (n_configs / eta) configurations

                n_configs = n * self.eta**(-i)
                n_iterations = r * self.eta**(i)

                n_iter = n_iterations
                if last_run_num is not None and not self.restart_needed:
                    n_iter -= last_run_num
                last_run_num = n_iterations

                self.logger.info(
                    "XFHB-%s: %d configurations x %d iterations each" %
                    (self.info_type, int(n_configs), int(n_iterations)))

                ret_val, early_stops = self.run_in_parallel(
                    T, n_iter, extra_info)
                val_losses = [item['loss'] for item in ret_val]
                ref_list = [item['ref_id'] for item in ret_val]

                self.target_x[int(n_iterations)].extend(T)
                self.target_y[int(n_iterations)].extend(val_losses)

                if int(n_iterations) == self.R:
                    self.incumbent_configs.extend(T)
                    self.incumbent_obj.extend(val_losses)
                # select a number of best configurations for the next loop
                # filter out early stops, if any
                indices = np.argsort(val_losses)
                if len(T) == sum(early_stops):
                    break
                if len(T) >= self.eta:
                    T = [T[i] for i in indices if not early_stops[i]]
                    extra_info = [
                        ref_list[i] for i in indices if not early_stops[i]
                    ]
                    reduced_num = int(n_configs / self.eta)
                    T = T[0:reduced_num]
                    extra_info = extra_info[0:reduced_num]
                else:
                    T = [T[indices[0]]]
                    extra_info = [ref_list[indices[0]]]
                incumbent_loss = val_losses[indices[0]]
                self.add_stage_history(
                    self.stage_id, min(self.global_incumbent, incumbent_loss))
                self.stage_id += 1
            self.remove_immediate_model()

            if self.info_type == 'Weighted':
                for item in self.iterate_r[self.iterate_r.index(r):]:
                    # objective value normalization: min-max linear normalization
                    normalized_y = minmax_normalization(self.target_y[item])
                    self.weighted_surrogate.train(
                        convert_configurations_to_array(self.target_x[item]),
                        np.array(normalized_y, dtype=np.float64),
                        r=item)
        # TODO: trade off value: decay (bayesian optimization did? do we need trade off e&e again?)
        self.init_tradeoff *= self.tradeoff_dec_rate
Ejemplo n.º 13
0
    def iterate(self, skip_last=0):

        for s in reversed(range(self.s_max + 1)):

            # initial number of configurations
            n = int(ceil(self.B / self.R / (s + 1) * self.eta ** s))

            # initial number of iterations per config
            r = self.R * self.eta ** (-s)

            # n random configurations
            T = self.choose_next(n)
            extra_info = None
            last_run_num = None

            lc_info = dict()
            lc_conf_mapping = dict()
            # assume no same configuration in the same batch: T
            for item in T:
                conf_id = get_configuration_id(item.get_dictionary())
                sha = hashlib.sha1(conf_id.encode('utf8'))
                conf_id = sha.hexdigest()
                lc_conf_mapping[conf_id] = item

            for i in range((s + 1) - int(skip_last)):

                # Run each of the n configs for <iterations>
                # and keep best (n_configs / eta) configurations

                n_configs = n * self.eta ** (-i)
                n_iterations = r * self.eta ** (i)

                n_iter = n_iterations
                if last_run_num is not None and not self.restart_needed:
                    n_iter -= last_run_num
                last_run_num = n_iterations

                self.logger.info("MBHB: %d configurations x %d iterations each" % (int(n_configs), int(n_iterations)))

                ret_val, early_stops = self.run_in_parallel(T, n_iter, extra_info)

                val_losses = [item['loss'] for item in ret_val]
                ref_list = [item['ref_id'] for item in ret_val]

                for item in ret_val:
                    conf_id = item['ref_id']
                    if not self.restart_needed:
                        if conf_id not in lc_info:
                            lc_info[conf_id] = []
                        lc_info[conf_id].extend(item['lc_info'])
                    else:
                        lc_info[conf_id] = item['lc_info']

                if int(n_iterations) == self.R:

                    self.incumbent_configs.extend(T)
                    self.incumbent_obj.extend(val_losses)
                # select a number of best configurations for the next loop
                # filter out early stops, if any
                indices = np.argsort(val_losses)
                if len(T) == sum(early_stops):
                    break
                if len(T) >= self.eta:
                    T = [T[i] for i in indices if not early_stops[i]]
                    extra_info = [ref_list[i] for i in indices if not early_stops[i]]
                    reduced_num = int(n_configs / self.eta)
                    T = T[0:reduced_num]
                    extra_info = extra_info[0:reduced_num]
                else:
                    T = [T[indices[0]]]
                    extra_info = [ref_list[indices[0]]]
                incumbent_loss = val_losses[indices[0]]
                self.add_stage_history(self.stage_id, min(self.global_incumbent, incumbent_loss))
                self.stage_id += 1
            self.remove_immediate_model()

            # keep learning curve data
            for item, config in lc_conf_mapping.items():
                lc_data = lc_info[item]
                if len(lc_data) > 0:
                    n_epochs = len(lc_data)
                    # self.logger.info('insert one learning curve data into dataset.')
                    t_idx = np.arange(1, n_epochs + 1) / n_epochs
                    conf_data = convert_configurations_to_array([config])
                    x = np.repeat(conf_data, t_idx.shape[0], axis=0)
                    x = np.concatenate((x, t_idx[:, None]), axis=1)
                    y = np.array(lc_data)
                    if self.lc_training_x is None:
                        self.lc_training_x, self.lc_training_y = x, y
                    else:
                        self.lc_training_x = np.concatenate((self.lc_training_x, x), 0)
                        self.lc_training_y = np.concatenate((self.lc_training_y, y), 0)
Ejemplo n.º 14
0
    def update_weight(self):
        max_r = self.iterate_r[-1]
        incumbent_configs = self.target_x[max_r]
        test_x = convert_configurations_to_array(incumbent_configs)
        test_y = np.array(self.target_y[max_r], dtype=np.float64)

        r_list = self.weighted_surrogate.surrogate_r
        K = len(r_list)

        if len(test_y) >= 3:
            # Get previous weights
            if self.weight_method in [
                    'rank_loss_softmax', 'rank_loss_single', 'rank_loss_p_norm'
            ]:
                preserving_order_p = list()
                preserving_order_nums = list()
                for i, r in enumerate(r_list):
                    fold_num = 5
                    if i != K - 1:
                        mean, var = self.weighted_surrogate.surrogate_container[
                            r].predict(test_x)
                        tmp_y = np.reshape(mean, -1)
                        preorder_num, pair_num = MFSE.calculate_preserving_order_num(
                            tmp_y, test_y)
                        preserving_order_p.append(preorder_num / pair_num)
                        preserving_order_nums.append(preorder_num)
                    else:
                        if len(test_y) < 2 * fold_num:
                            preserving_order_p.append(0)
                        else:
                            # 5-fold cross validation.
                            kfold = KFold(n_splits=fold_num)
                            cv_pred = np.array([0] * len(test_y))
                            for train_idx, valid_idx in kfold.split(test_x):
                                train_configs, train_y = test_x[
                                    train_idx], test_y[train_idx]
                                valid_configs, valid_y = test_x[
                                    valid_idx], test_y[valid_idx]
                                types, bounds = get_types(self.config_space)
                                _surrogate = RandomForestWithInstances(
                                    types=types, bounds=bounds)
                                _surrogate.train(train_configs, train_y)
                                pred, _ = _surrogate.predict(valid_configs)
                                cv_pred[valid_idx] = pred.reshape(-1)
                            preorder_num, pair_num = MFSE.calculate_preserving_order_num(
                                cv_pred, test_y)
                            preserving_order_p.append(preorder_num / pair_num)
                            preserving_order_nums.append(preorder_num)

                if self.weight_method == 'rank_loss_softmax':
                    order_weight = np.array(np.sqrt(preserving_order_nums))
                    trans_order_weight = order_weight - np.max(order_weight)
                    # Softmax mapping.
                    new_weights = np.exp(trans_order_weight) / sum(
                        np.exp(trans_order_weight))
                elif self.weight_method == 'rank_loss_p_norm':
                    trans_order_weight = np.array(preserving_order_p)
                    power_sum = np.sum(
                        np.power(trans_order_weight, self.power_num))
                    new_weights = np.power(trans_order_weight,
                                           self.power_num) / power_sum
                else:
                    _idx = np.argmax(np.array(preserving_order_nums))
                    new_weights = [0.] * K
                    new_weights[_idx] = 1.
            elif self.weight_method == 'rank_loss_prob':
                # For basic surrogate i=1:K-1.
                mean_list, var_list = list(), list()
                for i, r in enumerate(r_list[:-1]):
                    mean, var = self.weighted_surrogate.surrogate_container[
                        r].predict(test_x)
                    mean_list.append(np.reshape(mean, -1))
                    var_list.append(np.reshape(var, -1))
                sample_num = 100
                min_probability_array = [0] * K
                for _ in range(sample_num):
                    order_preseving_nums = list()

                    # For basic surrogate i=1:K-1.
                    for idx in range(K - 1):
                        sampled_y = np.random.normal(mean_list[idx],
                                                     var_list[idx])
                        _num, _ = MFSE.calculate_preserving_order_num(
                            sampled_y, test_y)
                        order_preseving_nums.append(_num)

                    fold_num = 5
                    # For basic surrogate i=K. cv
                    if len(test_y) < 2 * fold_num:
                        order_preseving_nums.append(0)
                    else:
                        # 5-fold cross validation.
                        kfold = KFold(n_splits=fold_num)
                        cv_pred = np.array([0] * len(test_y))
                        for train_idx, valid_idx in kfold.split(test_x):
                            train_configs, train_y = test_x[train_idx], test_y[
                                train_idx]
                            valid_configs, valid_y = test_x[valid_idx], test_y[
                                valid_idx]
                            types, bounds = get_types(self.config_space)
                            _surrogate = RandomForestWithInstances(
                                types=types, bounds=bounds)
                            _surrogate.train(train_configs, train_y)
                            _pred, _var = _surrogate.predict(valid_configs)
                            sampled_pred = np.random.normal(
                                _pred.reshape(-1), _var.reshape(-1))
                            cv_pred[valid_idx] = sampled_pred
                        _num, _ = MFSE.calculate_preserving_order_num(
                            cv_pred, test_y)
                        order_preseving_nums.append(_num)
                    max_id = np.argmax(order_preseving_nums)
                    min_probability_array[max_id] += 1
                new_weights = np.array(min_probability_array) / sample_num

            elif self.weight_method == 'opt_based':
                mean_list, var_list = list(), list()
                for i, r in enumerate(r_list):
                    if i != K - 1:
                        mean, var = self.weighted_surrogate.surrogate_container[
                            r].predict(test_x)
                        tmp_y = np.reshape(mean, -1)
                        tmp_var = np.reshape(var, -1)
                        mean_list.append(tmp_y)
                        var_list.append(tmp_var)
                    else:
                        if len(test_y) < 8:
                            mean_list.append(np.array([0] * len(test_y)))
                            var_list.append(np.array([0] * len(test_y)))
                        else:
                            # 5-fold cross validation.
                            kfold = KFold(n_splits=5)
                            cv_pred = np.array([0] * len(test_y))
                            cv_var = np.array([0] * len(test_y))
                            for train_idx, valid_idx in kfold.split(test_x):
                                train_configs, train_y = test_x[
                                    train_idx], test_y[train_idx]
                                valid_configs, valid_y = test_x[
                                    valid_idx], test_y[valid_idx]
                                types, bounds = get_types(self.config_space)
                                _surrogate = RandomForestWithInstances(
                                    types=types, bounds=bounds)
                                _surrogate.train(train_configs, train_y)
                                pred, var = _surrogate.predict(valid_configs)
                                cv_pred[valid_idx] = pred.reshape(-1)
                                cv_var[valid_idx] = var.reshape(-1)
                            mean_list.append(cv_pred)
                            var_list.append(cv_var)
                means = np.array(mean_list)
                vars = np.array(var_list) + 1e-8

                def min_func(x):
                    x = np.reshape(np.array(x), (1, len(x)))
                    ensemble_vars = 1 / (x @ (1 / vars))
                    ensemble_means = x @ (means / vars) * ensemble_vars
                    ensemble_means = np.reshape(ensemble_means, -1)
                    self.logger.info("Loss:" + str(x))
                    return MFSE.calculate_ranking_loss(ensemble_means, test_y)

                constraints = [{
                    'type': 'eq',
                    'fun': lambda x: np.sum(x) - 1
                }, {
                    'type': 'ineq',
                    'fun': lambda x: x - 0
                }, {
                    'type': 'ineq',
                    'fun': lambda x: 1 - x
                }]
                res = minimize(min_func,
                               np.array([1e-8] * K),
                               constraints=constraints)
                new_weights = res.x
            else:
                raise ValueError('Invalid weight method: %s!' %
                                 self.weight_method)
        else:
            old_weights = list()
            for i, r in enumerate(r_list):
                _weight = self.weighted_surrogate.surrogate_weight[r]
                old_weights.append(_weight)
            new_weights = old_weights.copy()

        self.logger.info(
            '[%s] %d-th Updating weights: %s' %
            (self.weight_method, self.weight_changed_cnt, str(new_weights)))

        # Assign the weight to each basic surrogate.
        for i, r in enumerate(r_list):
            self.weighted_surrogate.surrogate_weight[r] = new_weights[i]
        self.weight_changed_cnt += 1
        # Save the weight data.
        self.hist_weights.append(new_weights)
        np.save(
            'data/%s_weights_%s.npy' % (self.method_name, self.method_name),
            np.asarray(self.hist_weights))
Ejemplo n.º 15
0
    def iterate(self, skip_last=0):

        for s in reversed(range(self.s_max + 1)):

            if self.update_enable and self.weight_update_id > self.s_max:
                self.update_weight()
            self.weight_update_id += 1

            # Set initial number of configurations
            n = int(ceil(self.B / self.R / (s + 1) * self.eta**s))
            # initial number of iterations per config
            r = int(self.R * self.eta**(-s))

            # Choose a batch of configurations in different mechanisms.
            start_time = time.time()
            T = self.choose_next_batch(n)
            time_elapsed = time.time() - start_time
            self.logger.info(
                "[%s] Choosing next configurations took %.2f sec." %
                (self.method_name, time_elapsed))

            extra_info = None
            last_run_num = None

            for i in range((s + 1) - int(skip_last)):  # changed from s + 1

                # Run each of the n configs for <iterations>
                # and keep best (n_configs / eta) configurations

                n_configs = n * self.eta**(-i)
                n_iterations = r * self.eta**(i)

                n_iter = n_iterations
                if last_run_num is not None and not self.restart_needed:
                    n_iter -= last_run_num
                last_run_num = n_iterations

                self.logger.info(
                    "MFSE: %d configurations x %d iterations each" %
                    (int(n_configs), int(n_iterations)))

                ret_val, early_stops = self.run_in_parallel(
                    T, n_iter, extra_info)
                val_losses = [item['loss'] for item in ret_val]
                ref_list = [item['ref_id'] for item in ret_val]

                self.target_x[int(n_iterations)].extend(T)
                self.target_y[int(n_iterations)].extend(val_losses)

                if int(n_iterations) == self.R:
                    self.incumbent_configs.extend(T)
                    self.incumbent_perfs.extend(val_losses)
                    # Update history container.
                    for _config, _perf in zip(T, val_losses):
                        self.history_container.add(_config, _perf)

                # Select a number of best configurations for the next loop.
                # Filter out early stops, if any.
                indices = np.argsort(val_losses)
                if len(T) == sum(early_stops):
                    break
                if len(T) >= self.eta:
                    T = [T[i] for i in indices if not early_stops[i]]
                    extra_info = [
                        ref_list[i] for i in indices if not early_stops[i]
                    ]
                    reduced_num = int(n_configs / self.eta)
                    T = T[0:reduced_num]
                    extra_info = extra_info[0:reduced_num]
                else:
                    T = [T[indices[0]]]
                    extra_info = [ref_list[indices[0]]]
                incumbent_loss = val_losses[indices[0]]
                self.add_stage_history(
                    self.stage_id, min(self.global_incumbent, incumbent_loss))
                self.stage_id += 1
            self.remove_immediate_model()

            for item in self.iterate_r[self.iterate_r.index(r):]:
                # NORMALIZE Objective value: normalization
                normalized_y = std_normalization(self.target_y[item])
                self.weighted_surrogate.train(convert_configurations_to_array(
                    self.target_x[item]),
                                              np.array(normalized_y,
                                                       dtype=np.float64),
                                              r=item)
Ejemplo n.º 16
0
    def iterate(self):
        for _ in range(self.inner_iteration_n):
            T = self.choose_next(self.num_workers)

            extra_info = None
            lc_info = dict()
            lc_conf_mapping = dict()

            # assume no same configuration in the same batch: T
            for item in T:
                conf_id = get_configuration_id(item.get_dictionary())
                sha = hashlib.sha1(conf_id.encode('utf8'))
                conf_id = sha.hexdigest()
                lc_conf_mapping[conf_id] = item

            total_iter_num = self.R // self.early_stop_gap
            for iter_num in range(1, 1 + total_iter_num):
                self.logger.info('start iteration gap %d' % iter_num)
                ret_val, early_stops = self.run_in_parallel(T, self.early_stop_gap, extra_info)
                val_losses = [item['loss'] for item in ret_val]
                ref_list = [item['ref_id'] for item in ret_val]
                for item in ret_val:
                    conf_id = item['ref_id']
                    if not self.restart_needed:
                        if conf_id not in lc_info:
                            lc_info[conf_id] = []
                        lc_info[conf_id].extend(item['lc_info'])
                    else:
                        lc_info[conf_id] = item['lc_info']

                if iter_num == total_iter_num:
                    self.incumbent_configs.extend(T)
                    self.incumbent_obj.extend(val_losses)

                T = [config for i, config in enumerate(T) if not early_stops[i]]
                extra_info = [ref for i, ref in enumerate(ref_list) if not early_stops[i]]
                if len(T) == 0:
                    break

                if len(self.incumbent_obj) >= 2 * self.num_config and iter_num != total_iter_num:
                    # learning curve based early stop strategy.
                    ref_list = extra_info
                    early_stops = self.stop_early(T)
                    T = [config for i, config in enumerate(T) if not early_stops[i]]
                    extra_info = [ref for i, ref in enumerate(ref_list) if not early_stops[i]]
                if len(T) == 0:
                    break

            # keep learning curve data
            for item, config in lc_conf_mapping.items():
                lc_data = lc_info[item]
                if len(lc_data) > 0:
                    n_epochs = len(lc_data)
                    # self.logger.info('insert one learning curve data into dataset.')
                    t_idx = np.arange(1, n_epochs + 1) / n_epochs
                    conf_data = convert_configurations_to_array([config])
                    x = np.repeat(conf_data, t_idx.shape[0], axis=0)
                    x = np.concatenate((x, t_idx[:, None]), axis=1)
                    y = np.array(lc_data)
                    if self.lc_training_x is None:
                        self.lc_training_x, self.lc_training_y = x, y
                    else:
                        self.lc_training_x = np.concatenate((self.lc_training_x, x), 0)
                        self.lc_training_y = np.concatenate((self.lc_training_y, y), 0)
            self.logger.info('training data shape: %s' % str(self.lc_training_x.shape))
            if len(self.incumbent_obj) >= 2 * self.num_config and len(self.incumbent_obj) % self.num_config == 0:
                self.lcnet_model.train(self.lc_training_x, self.lc_training_y)

            self.add_stage_history(self.stage_id, self.global_incumbent)
            self.stage_id += 1
            self.remove_immediate_model()
Ejemplo n.º 17
0
    def iterate(self, skip_last=0):

        for s in reversed(range(self.s_max + 1)):
            # Initial number of configurations
            n = int(ceil(self.B / self.R / (s + 1) * self.eta**s))
            # Initial number of iterations per config
            r = int(self.R * self.eta**(-s))

            # Choose a batch of configurations in different mechanisms.
            start_time = time.time()
            T = self.choose_next(n)
            time_elapsed = time.time() - start_time
            self.logger.info("Choosing next configurations took %.2f sec." %
                             time_elapsed)

            extra_info = None
            last_run_num = None

            for i in range((s + 1) - int(skip_last)):  # Changed from s + 1

                # Run each of the n configs for <iterations> and keep best (n_configs / eta) configurations
                n_configs = n * self.eta**(-i)
                n_iterations = r * self.eta**(i)

                n_iter = n_iterations
                if last_run_num is not None and not self.restart_needed:
                    n_iter -= last_run_num
                last_run_num = n_iterations

                self.logger.info(
                    "HOIST: %d configurations WITH %d units of resource" %
                    (int(n_configs), int(n_iterations)))

                ret_val, early_stops = self.run_in_parallel(
                    T, n_iter, extra_info)
                val_losses = [item['loss'] for item in ret_val]
                ref_list = [item['ref_id'] for item in ret_val]

                self.target_x[int(n_iterations)].extend(T)
                self.target_y[int(n_iterations)].extend(val_losses)

                if int(n_iterations) == self.R:
                    self.incumbent_configs.extend(T)
                    self.incumbent_obj.extend(val_losses)

                # Select a number of well-performed configurations for the next loop.
                indices = np.argsort(val_losses)
                if len(T) == sum(early_stops):
                    break
                if len(T) >= self.eta:
                    T = [T[i] for i in indices if not early_stops[i]]
                    extra_info = [
                        ref_list[i] for i in indices if not early_stops[i]
                    ]
                    reduced_num = int(n_configs / self.eta)
                    T = T[0:reduced_num]
                    extra_info = extra_info[0:reduced_num]
                else:
                    T = [T[indices[0]]]
                    extra_info = [ref_list[indices[0]]]
                incumbent_loss = val_losses[indices[0]]
                self.add_stage_history(
                    self.stage_id, min(self.global_incumbent, incumbent_loss))
                self.stage_id += 1
            self.remove_immediate_model()

            # Augment the intermediate evaluation data.
            for item in self.iterate_r[self.iterate_r.index(r):]:
                # objective value normalization: min-max linear normalization
                normalized_y = minmax_normalization(self.target_y[item])
                self.weighted_surrogate.train(convert_configurations_to_array(
                    self.target_x[item]),
                                              np.array(normalized_y,
                                                       dtype=np.float64),
                                              r=item)
            # Update the parameter in the ensemble model.
            if len(self.target_y[self.iterate_r[-1]]) >= 2:
                self.update_weight()
Ejemplo n.º 18
0
def tse(run_id, train_base_models=True):
    start_time = time.time()

    from concurrent.futures import ProcessPoolExecutor
    pool = ProcessPoolExecutor(max_workers=args.worker)
    X, y = [], []
    c = []
    inc = 1.
    X_l, y_l = [], []

    weight = np.array([1/K]*(K+1))
    config_evaluated = []
    config_space = create_configspace()
    # Initialize config L.
    config_L = sample_configurations(config_space, num_L_init)

    if train_base_models:
        func_configs = list()
        for iter_t in range(K):
            print('Build mid fidelity model', iter_t)
            func_configs.append(True)
        func_configs.append(False)
        training_data = run_parallel_async(pool, mini_smac, func_configs)
        with open('data/xgb/base_tse_data_%d.pkl' % run_id, 'wb') as f:
            pickle.dump(training_data, f)
    else:
        with open('data/xgb/base_tse_data_%d.pkl' % 10, 'rb') as f:
            training_data = pickle.load(f)
        print('Load training data for M evaluations!')

    # Create base models.
    base_models = list()
    config_space = create_configspace()
    types, bounds = get_types(config_space)
    for iter_t in range(K+1):
        config_x, config_y = training_data[iter_t]
        model = RandomForestWithInstances(types=types, bounds=bounds)
        model.train(config_x, config_y)
        base_models.append(model)
    low_fidelity_model = base_models[K]
    X_l.extend(training_data[K][0].tolist())
    y_l.extend(training_data[K][1].tolist())
    print('Base model building finished!')

    # The framework of TSE.
    for iter_t in range(iter_H):
        print('Iteration in TSE', iter_t)
        # Sample a batch of configurations according to tse model.
        configs = sample_configurations(config_space, iter_L * 10)
        config_arrays = convert_configurations_to_array(configs)
        perfs, _ = low_fidelity_model.predict(config_arrays)
        perfs = perfs[:, 0]
        if len(y) > 3:
            preds = []
            for i in range(K):
                m, _ = base_models[i].predict(config_arrays)
                preds.append(m[:, 0].tolist())
            preds = np.array(preds).T
            preds = np.mat(np.hstack((preds, np.ones((len(configs), 1)))))
            # Add the delta.
            delta = preds*np.mat(weight.reshape(-1, 1))
            perfs += delta.getA()[:, 0]
        configs_candidate = []
        indexes = np.argsort(perfs)[:iter_L]
        for index in indexes:
            configs_candidate.append(configs[index])

        # Evaluate the low-fidelity configurations.
        print('='*10 + 'Evaluating the low-fidelity configurations')
        config_params = []
        for config in configs_candidate:
            config_params.append((config.get_dictionary(), s_min))

        result_perf = run_parallel_async(pool, objective_function, config_params)

        for index, item in enumerate(result_perf):
            X_l.append(configs_candidate[index].get_array().tolist())
            y_l.append(item[0])

        print(np.array(X_l).shape, np.array(y_l, dtype=np.float64).shape)
        # Update f_L.
        print('=' * 10 + 'Retrain the f_L')
        low_fidelity_model.train(np.array(X_l), np.array(y_l, dtype=np.float64))
        config_L.extend(configs_candidate)

        configs_input = []
        for config in config_L:
            if config not in config_evaluated:
                configs_input.append(config)

        # Choose the next configuration.
        config_arrays = convert_configurations_to_array(configs_input)
        perfs, _ = low_fidelity_model.predict(config_arrays)
        perfs = perfs[:, 0]
        if len(y) > 3:
            preds = []
            for i in range(K):
                m, _ = base_models[i].predict(config_arrays)
                preds.append(m[:, 0].tolist())
            preds = np.array(preds).T
            preds = np.mat(np.hstack((preds, np.ones((len(configs_input), 1)))))
            # Add the delta.
            delta = preds * np.mat(weight.reshape(-1, 1))
            perfs += delta.getA()[:, 0]
        next_config = configs_input[np.argmin(perfs)]

        # Evaluate this config with a high-fidelity setting.
        print('=' * 10 + 'Evaluate the high-fidelity configuration')
        perf, _ = objective_function((next_config.get_dictionary(), s_max))
        X.append(next_config)
        y.append(perf)
        if perf < inc:
            inc = perf
        c.append([time.time()-start_time, inc])
        print('Current inc', inc)

        if len(y) < 3:
            continue
        # Learn the weight in TSE.
        Z = []
        for i in range(K):
            m, v = base_models[i].predict(convert_configurations_to_array(X))
            Z.append(m[:, 0].tolist())
        Z = np.mat(np.hstack((np.array(Z).T, np.ones((len(y), 1)))))
        f = np.mat(np.array(y).reshape((-1, 1)))
        # Compute the weight.
        try:
            ZtZ_inv = np.linalg.inv(Z.T * Z)
            weight = (ZtZ_inv * Z.T * f)[:, 0]
            print('The weight updated is', weight)
        except np.linalg.LinAlgError as err:
            if 'Singular matrix' in str(err):
                print('Singular matrix encountered, and do not update the weight!')
            else:
                raise ValueError('Unexpected error!')

        # Save the result.
        np.save('data/xgb/tse_%d.npy' % run_id, np.array(c))
        plt.plot(np.array(c)[:, 0], np.array(c)[:, 1])
        plt.xlabel('time_elapsed (s)')
        plt.ylabel('validation error')
        plt.savefig("data/xgb/tse_%d.png" % run_id)
        if time.time() - start_time > 21600:
            raise ValueError('Runtime budget meets!')

    pool.shutdown(wait=True)