示例#1
0
    def split(self):

        # this is leaf node
        if self.left is None and self.right is None:

            if self.is_splitting():
                # print("It's splitting")
                # instantiate the splitter
                self.region_splitter = RegionSplitter(self.training_data, self.training_label)

                # don't split if the split quality is low
                if self.region_splitter.split_quality < self.split_quality_thres:
                    self.split_lock_count = self.split_lock_count_thres
                    return

                # instantiate the left and right expert
                child_config = self.config.copy()
                child_config['split_thres'] = self.split_thres*self.split_thres_growth_rate
                child_config['split_quality_decay'] = self.split_quality_decay*(2-self.split_quality_decay)

                self.right = Expert(id=(self.expert_id + (1 << self.expert_level)), level=self.expert_level+1,
                                    **child_config)
                self.left = Expert(id=self.expert_id,  level=self.expert_level+1,
                                   **child_config)

                # split the data to the correct region
                for i in range(len(self.training_data)):
                    if self.region_splitter.classify(self.training_data[i]):
                        self.right.training_data.append(self.training_data[i])
                        self.right.training_label.append(self.training_label[i])
                    else:
                        self.left.training_data.append(self.training_data[i])
                        self.left.training_label.append(self.training_label[i])

                # if either of them is empty (which shouldn't happen)
                if len(self.left.training_data) <= 1 or len(self.right.training_data) <= 1:
                    # do not split
                    self.right = None
                    self.left = None
                    self.region_splitter = None
                    # print("split cancelled")
                    self.split_lock_count = self.split_lock_count_thres
                    return

                # transferring "knowledge" to child nodes
                self.right.train()
                self.right.mean_error = self.mean_error
                self.right.rewards_history = copy(self.rewards_history)
                self.right.prediction_model = copy(self.predict_model)
                self.right.kga.errors = copy(self.kga.errors)
                self.right.training_count = 0
                self.right.action_count = self.action_count
                self.right.split_quality_thres = self.region_splitter.split_quality * self.split_quality_decay

                self.left.train()
                self.left.mean_error = self.mean_error
                self.left.rewards_history = copy(self.rewards_history)
                self.left.prediction_model = copy(self.predict_model)
                self.left.kga.errors = copy(self.kga.errors)
                self.left.training_count = 0
                self.left.action_count = self.action_count
                self.left.split_quality_thres = self.region_splitter.split_quality * self.split_quality_decay

                # clear the training data at the parent node so they don't get modified accidentally
                self.training_data.clear()
                self.training_label.clear()
                # clear everything as they are not needed any more
                self.mean_error = None
                self.predict_model = None
                self.kga = None
                self.rewards_history = None

        # Cases when only one of the child is NONE
        elif self.left is None and self.right is None:
            raise(Exception, "Expert's Tree structure is corrupted! One child branch is missing")

        else:
            # delegate to child nodes
            self.right.split()
            self.left.split()
示例#2
0
class Expert():

    def __init__(self, id=0, level=0, **config_kwargs):

        # default expert configuration
        self.config = defaultdict(int)
        self.config['reward_smoothing'] = 1
        self.config['split_thres'] = 40
        self.config['split_thres_growth_rate'] = 1.0
        self.config['split_lock_count_thres'] = 1
        self.config['split_quality_thres_0'] = 0.0
        self.config['split_quality_decay'] = 1.0
        self.config['mean_err_thres'] = 0.0
        self.config['mean_err_0'] = 1.0
        self.config['action_value_0'] = 0.0
        self.config['reward_smoothing'] = 1
        self.config['learning_rate'] = 0.25
        self.config['kga_delta'] = 10
        self.config['kga_tau'] = 30
        self.config['max_training_data_num'] = 500
        self.config['prediction_model'] = linear_model.LinearRegression()

        # custom configurations
        for param, value in config_kwargs.items():
            self.config[param] = value

        # expert id
        self.expert_id = id
        self.expert_level = level

        # child expert
        self.left = None
        self.right = None

        # region splitter
        self.region_splitter = None

        # memory
        self.training_data = deque(maxlen=int(max(self.config['split_thres'], self.config['max_training_data_num'])))
        self.training_label = deque(maxlen=int(max(self.config['split_thres'], self.config['max_training_data_num'])))

        # prediction model
        self.predict_model = self.config['prediction_model']

        # error is max at first
        self.mean_error = self.config['mean_err_0']

        # expected reward - action-value
        self.learning_rate = self.config['learning_rate']
        self.action_value = self.config['action_value_0']

        # knowledge gain assessor
        self.kga = KGA(self.mean_error, delta=self.config['kga_delta'], tau=self.config['kga_tau'])

        # historical reward history
        self.rewards_history = [self.action_value]
        self.rewards_smoothing = self.config['reward_smoothing']

        # number of re-training
        self.training_count = 0

        # number of times that action associated with this region has been selected
        self.action_count = 0

        # the splitting thresholds
        self.split_thres = self.config['split_thres']
        self.split_thres_growth_rate = self.config['split_thres_growth_rate']
        self.split_quality_thres = self.config['split_quality_thres_0']
        self.split_quality_decay = self.config['split_quality_decay']
        self.mean_error_thres = self.config['mean_err_thres']
        self.split_lock_count = 0
        self.split_lock_count_thres = self.config['split_lock_count_thres']


    def append(self, SM, S1, S1_predicted=None):

        if not isinstance(SM, tuple):
            raise(TypeError, "SM must be a tuple")
        if not isinstance(S1, tuple):
            raise(TypeError, "S1 must be a tuple")
        if S1_predicted is not None and not isinstance(S1_predicted, tuple):
            raise(TypeError, "S1_predicted must be a tuple")

        self.training_count += 1
        self.action_count += 1
        if self.left is None and self.right is None:
            self.training_data.append(SM)
            self.training_label.append(S1)

            # update prediction model
            self.train()

            # update the KGA
            if S1_predicted is not None:
                self.kga.append_error(S1, S1_predicted)
                self.mean_error = self.kga.calc_mean_error()  # used to determine if splitting is necessary
                self.rewards_history.append(self.kga.calc_reward())
                self.rewards_history = self.rewards_history[-self.rewards_smoothing:]
                self.update_action_value()

            # split if necessary
            self.split()

            return self.expert_id

        # Cases when only one of the child is NONE
        elif self.left is None or self.right is None:
            raise(Exception, "Expert's Tree structure is corrupted! One child branch is missing")

        # delegate to child nodes
        elif self.region_splitter.classify(SM):
            return self.right.append(SM, S1, S1_predicted)
        else:
            return self.left.append(SM, S1, S1_predicted)

    def train(self):
        # number of samples needs to be at least the number of features
        num_sample = len(self.training_data)

        # no data
        if num_sample < 1:
            return

        # not enough features
        num_fea = len(self.training_data[0])
        if num_sample < num_fea:
            return

        try:
            self.predict_model.fit(self.training_data, self.training_label)
            # print(self.predict_model.coef_)
        except ValueError:
            pass

    def predict(self, S, M):

        if not isinstance(S, tuple):
            raise TypeError("S must be a tuple")
        if not isinstance(M, tuple):
            raise TypeError("M must be a tuple")

        # this is leaf node
        if self.left is None and self.right is None:
            try:
                S1 = self.predict_model.predict(S+M)
            except AttributeError:
                S1 = S

            if isinstance(S1, np.ndarray) and (len(S1) != len(S)):
                S1 = tuple(S1[0])
            else:
                S1 = tuple(S1)

            return S1

        # Cases when only one of the child is NONE
        elif self.left is None or self.right is None:
            raise(Exception, "Expert's Tree structure is corrupted! One child branch is missing")

        # delegate to child nodes
        if self.region_splitter.classify(S+M):
            return self.right.predict(S,M)
        else:
            return self.left.predict(S,M)

    def is_splitting(self):

        try:
            if self.split_lock_count > 0:
                self.split_lock_count -= 1
                return False
        except AttributeError:
            pass
        if len(self.training_data) > self.split_thres and \
            (self.mean_error > self.mean_error_thres):
            return True
        return False

    def split(self):

        # this is leaf node
        if self.left is None and self.right is None:

            if self.is_splitting():
                # print("It's splitting")
                # instantiate the splitter
                self.region_splitter = RegionSplitter(self.training_data, self.training_label)

                # don't split if the split quality is low
                if self.region_splitter.split_quality < self.split_quality_thres:
                    self.split_lock_count = self.split_lock_count_thres
                    return

                # instantiate the left and right expert
                child_config = self.config.copy()
                child_config['split_thres'] = self.split_thres*self.split_thres_growth_rate
                child_config['split_quality_decay'] = self.split_quality_decay*(2-self.split_quality_decay)

                self.right = Expert(id=(self.expert_id + (1 << self.expert_level)), level=self.expert_level+1,
                                    **child_config)
                self.left = Expert(id=self.expert_id,  level=self.expert_level+1,
                                   **child_config)

                # split the data to the correct region
                for i in range(len(self.training_data)):
                    if self.region_splitter.classify(self.training_data[i]):
                        self.right.training_data.append(self.training_data[i])
                        self.right.training_label.append(self.training_label[i])
                    else:
                        self.left.training_data.append(self.training_data[i])
                        self.left.training_label.append(self.training_label[i])

                # if either of them is empty (which shouldn't happen)
                if len(self.left.training_data) <= 1 or len(self.right.training_data) <= 1:
                    # do not split
                    self.right = None
                    self.left = None
                    self.region_splitter = None
                    # print("split cancelled")
                    self.split_lock_count = self.split_lock_count_thres
                    return

                # transferring "knowledge" to child nodes
                self.right.train()
                self.right.mean_error = self.mean_error
                self.right.rewards_history = copy(self.rewards_history)
                self.right.prediction_model = copy(self.predict_model)
                self.right.kga.errors = copy(self.kga.errors)
                self.right.training_count = 0
                self.right.action_count = self.action_count
                self.right.split_quality_thres = self.region_splitter.split_quality * self.split_quality_decay

                self.left.train()
                self.left.mean_error = self.mean_error
                self.left.rewards_history = copy(self.rewards_history)
                self.left.prediction_model = copy(self.predict_model)
                self.left.kga.errors = copy(self.kga.errors)
                self.left.training_count = 0
                self.left.action_count = self.action_count
                self.left.split_quality_thres = self.region_splitter.split_quality * self.split_quality_decay

                # clear the training data at the parent node so they don't get modified accidentally
                self.training_data.clear()
                self.training_label.clear()
                # clear everything as they are not needed any more
                self.mean_error = None
                self.predict_model = None
                self.kga = None
                self.rewards_history = None

        # Cases when only one of the child is NONE
        elif self.left is None and self.right is None:
            raise(Exception, "Expert's Tree structure is corrupted! One child branch is missing")

        else:
            # delegate to child nodes
            self.right.split()
            self.left.split()

    def update_action_value(self):
        #self.action_value *= 0.95
        #self.action_value += self.learning_rate*(math.fsum(self.rewards_history[-self.rewards_smoothing:])/len(self.rewards_history[-self.rewards_smoothing:]))
        self.action_value = math.fsum(self.rewards_history[-self.rewards_smoothing:])/len(self.rewards_history[-self.rewards_smoothing:])
        return self.action_value

    def get_largest_action_value(self):

        # this is leaf node
        if self.left is None and self.right is None:

            if len(self.training_data) == 0:
                raise (Exception, "This node has no training data!")

            return self.action_value

        # Cases when only one of the child is NONE
        elif self.left is None and self.right is None:
            raise (Exception, "Expert's Tree structure is corrupted! One child branch is missing")

        else:
            return max(self.left.get_largest_action_value(), self.right.get_largest_action_value())

    def get_num_experts(self):

        if self.left is None and self.right is None:
            return 1
        else:
            return self.left.get_num_experts() + self.right.get_num_experts()

    def evaluate_action(self, S1, M1):

        if not isinstance(S1, tuple):
            raise TypeError("S1 must be a tuple (current value = %s)" % str(S1) )
        if not isinstance(M1, tuple):
            raise TypeError("M1 must be a tuple (current value = %s)" % str(S1))

        # this is leaf node
        if self.left is None and self.right is None:
            #print("Mean Error", self.mean_error)
            return self.action_value

        # Cases when only one of the child is NONE
        elif self.left is None and self.right is None:
            raise(Exception, "Expert's Tree structure is corrupted! One child branch is missing")

        else:

            if self.region_splitter.classify(S1+M1):
                return self.right.evaluate_action(S1, M1)
            else:
                return self.left.evaluate_action(S1, M1)

    def print(self, level=0):

        # this is leaf node
        if self.left is None and self.right is None:
            mean_error_string = '%.*f' % (2, self.mean_error)
            print(len(self.training_data), "#", str(self.training_count), "(err =", mean_error_string, ";ER =", self.rewards_history[-1], ") --", self.training_data)
            #print(len(self.training_data), "#", str(self.expert_id), "(err =", mean_error_string, ";ER =", self.rewards_history[-1], ") --", self.training_data)

        else:
            print(" L ** ", end="")
            self.left.print(level+1)
            print((" ")*len(" L ** ")*level, "R ** ", end="")
            self.right.print(level+1)

    def save_expert_info(self, info, include_exemplars=False):

        # this is leaf node
        if self.left is None and self.right is None:

            try:
                info['expert_ids'].append(self.expert_id)
            except AttributeError:
                info['expert_ids'] = [self.expert_id]

            info['mean_errors'][self.expert_id] = self.mean_error
            info['action_values'][self.expert_id] = self.action_value
            info['action_counts'][self.expert_id] = self.action_count
            info['latest_rewards'][self.expert_id] = self.rewards_history[-1]

            if include_exemplars:
                info['exemplars'][self.expert_id] = [copy(self.training_data), copy(self.training_label)]

            info['prediction_model'][self.expert_id] = self.predict_model
        else:
            self.left.save_expert_info(info, include_exemplars=include_exemplars)
            self.right.save_expert_info(info, include_exemplars=include_exemplars)