def __configure(self): if hasattr(self.base_estimator, "reset"): self.base_estimator.reset() self.actual_n_estimators = self.n_estimators self.ensemble = [ cp.deepcopy(self.base_estimator) for _ in range(self.actual_n_estimators) ] self.adwin_ensemble = [ ADWIN(self.delta) for _ in range(self.actual_n_estimators) ] self._random_state = check_random_state(self.random_state) self.n_detected_changes = 0 self.classes = None self.init_matrix_codes = True
def __configure(self, base_estimator): base_estimator.reset() self.base_estimator = base_estimator self.n_estimators = self._init_n_estimators self.adwin_ensemble = [] for i in range(self.n_estimators): self.adwin_ensemble.append(ADWIN()) self.ensemble = [cp.deepcopy(base_estimator) for _ in range(self.n_estimators)] self.random_state = check_random_state(self._init_random_state) self.lam_fn = np.zeros(self.n_estimators) self.lam_fp = np.zeros(self.n_estimators) self.lam_sum = np.zeros(self.n_estimators) self.werr = np.zeros(self.n_estimators) self.lam_sw = np.zeros(self.n_estimators) self.epsilon = np.zeros(self.n_estimators)
def __configure(self): if hasattr(self.base_estimator, "reset"): self.base_estimator.reset() self.actual_n_estimators = self._init_n_estimators self.adwin_ensemble = [] for i in range(self.actual_n_estimators): self.adwin_ensemble.append(ADWIN()) self.ensemble = [ cp.deepcopy(self.base_estimator) for _ in range(self.actual_n_estimators) ] self._random_state = check_random_state(self.random_state) self.lam_sc = np.zeros(self.actual_n_estimators) self.lam_sw = np.zeros(self.actual_n_estimators) self.epsilon = np.zeros(self.actual_n_estimators)
def __init__( self, base_ensemble, drift_detector=ADWIN(), ensemble_size=10, min_lambda=1, max_lambda=6, ): self.ensemble = DESDDEnsemble( base_ensemble, ensemble_size=ensemble_size, min_lambda=min_lambda, max_lambda=max_lambda, ) self.drift_detector = drift_detector self.max_ensemble_size = ensemble_size self.min_lambda = min_lambda self.max_lambda = max_lambda self.dcs_method = DESDDSel() self.classes = None
def __init__(self, initial_class_observations, random_state=None): super().__init__(initial_class_observations) self._estimation_error_weight = ADWIN() self.error_change = False self._random_state = check_random_state(random_state)
def __init__(self, initial_class_observations): super().__init__(initial_class_observations) self._estimation_error_weight = ADWIN() self.error_change = False self._randomSeed = 1 self._classifier_random = check_random_state(self._randomSeed)
def learn_from_instance(self, X, y, weight, hat, parent, parent_branch): true_class = y class_prediction = 0 leaf = self.filter_instance_to_leaf(X, parent, parent_branch) if leaf.node is not None: class_prediction = get_max_value_key(leaf.node.get_class_votes(X, hat)) bl_correct = (true_class == class_prediction) if self._estimation_error_weight is None: self._estimation_error_weight = ADWIN() old_error = self.get_error_estimation() # Add element to ADWIN add = 0.0 if (bl_correct is True) else 1.0 self._estimation_error_weight.add_element(add) # Detect change with ADWIN self.error_change = self._estimation_error_weight.detected_change() if self.error_change is True and old_error > self.get_error_estimation(): self.error_change = False # Check condition to build a new alternate tree if self.error_change is True: self._alternate_tree = hat._new_learning_node() hat.alternate_trees_cnt += 1 # Condition to replace alternate tree elif self._alternate_tree is not None and self._alternate_tree.is_null_error() is False: if self.get_error_width() > error_width_threshold \ and self._alternate_tree.get_error_width() > error_width_threshold: old_error_rate = self.get_error_estimation() alt_error_rate = self._alternate_tree.get_error_estimation() fDelta = .05 fN = 1.0 / self._alternate_tree.get_error_width() + 1.0 / (self.get_error_width()) bound = math.sqrt(2.0 * old_error_rate * (1.0 - old_error_rate) * math.log(2.0 / fDelta) * fN) # To check, bound never less than (old_error_rate - alt_error_rate) if bound < (old_error_rate - alt_error_rate): hat._active_leaf_node_cnt -= self.number_leaves() hat._active_leaf_node_cnt += self._alternate_tree.number_leaves() self.kill_tree_children(hat) if parent is not None: parent.set_child(parent_branch, self._alternate_tree) else: # Switch tree root hat._tree_root = hat._tree_root.alternateTree hat.switch_alternate_trees_cnt += 1 elif bound < alt_error_rate - old_error_rate: if isinstance(self._alternate_tree, TS_HAT.ActiveLearningNode): self._alternate_tree = None elif isinstance(self._alternate_tree, TS_HAT.InactiveLearningNode): self._alternate_tree = None else: self._alternate_tree.kill_tree_children(hat) hat.pruned_alternate_trees_cnt += 1 # hat.pruned_alternate_trees_cnt to check # Learn_From_Instance alternate Tree and Child nodes if self._alternate_tree is not None: self._alternate_tree.learn_from_instance(X, y, weight, hat, parent, parent_branch) child_branch = self.instance_child_index(X) child = self.get_child(child_branch) if child is not None: child.learn_from_instance(X, y, weight, hat, parent, parent_branch)
def __init__(self, n_estimators=10, max_features='auto', disable_weighted_vote=False, lambda_value=6, performance_metric='acc', drift_detection_method: BaseDriftDetector = ADWIN(0.001), warning_detection_method: BaseDriftDetector = ADWIN(0.01), max_byte_size=33554432, memory_estimate_period=2000000, grace_period=50, split_criterion='info_gain', split_confidence=0.01, tie_threshold=0.05, binary_split=False, stop_mem_management=False, remove_poor_atts=False, no_preprune=False, leaf_prediction='nba', nb_threshold=0, nominal_attributes=None, random_state=None): """AdaptiveRandomForestClassifier class constructor.""" super().__init__() self.n_estimators = n_estimators self.max_features = max_features self.disable_weighted_vote = disable_weighted_vote self.lambda_value = lambda_value if isinstance(drift_detection_method, BaseDriftDetector): self.drift_detection_method = drift_detection_method else: self.drift_detection_method = None if isinstance(warning_detection_method, BaseDriftDetector): self.warning_detection_method = warning_detection_method else: self.warning_detection_method = None self.instances_seen = 0 self.classes = None self._train_weight_seen_by_model = 0.0 self.ensemble = None self.random_state = random_state self._random_state = check_random_state( self.random_state) # Actual random_state object if performance_metric in ['acc', 'kappa']: self.performance_metric = performance_metric else: raise ValueError( 'Invalid performance metric: {}'.format(performance_metric)) # ARH Hoeffding Tree configuration self.max_byte_size = max_byte_size self.memory_estimate_period = memory_estimate_period self.grace_period = grace_period self.split_criterion = split_criterion self.split_confidence = split_confidence self.tie_threshold = tie_threshold self.binary_split = binary_split self.stop_mem_management = stop_mem_management self.remove_poor_atts = remove_poor_atts self.no_preprune = no_preprune self.leaf_prediction = leaf_prediction self.nb_threshold = nb_threshold self.nominal_attributes = nominal_attributes
class AdaSplitNode(SplitNode, AdaNode): """ Node that splits the data in a Hoeffding Adaptive Tree. Parameters ---------- split_test: skmultiflow.split_test.InstanceConditionalTest Split test. class_observations: dict (class_value, weight) or None Class observations """ def __init__(self, split_test, class_observations): super().__init__(split_test, class_observations) self._estimation_error_weight = ADWIN() self._alternate_tree = None self.error_change = False self._random_seed = 1 self._classifier_random = check_random_state(self._random_seed) # Override AdaNode def number_leaves(self): num_of_leaves = 0 for child in self._children: if child is not None: num_of_leaves += child.number_leaves() return num_of_leaves # Override AdaNode def get_error_estimation(self): return self._estimation_error_weight.estimation # Override AdaNode def get_error_width(self): w = 0.0 if self.is_null_error() is False: w = self._estimation_error_weight.width return w # Override AdaNode def is_null_error(self): return self._estimation_error_weight is None # Override AdaNode def learn_from_instance(self, X, y, weight, hat, parent, parent_branch): true_class = y class_prediction = 0 leaf = self.filter_instance_to_leaf(X, parent, parent_branch) if leaf.node is not None: class_prediction = get_max_value_key( leaf.node.get_class_votes(X, hat)) bl_correct = (true_class == class_prediction) if self._estimation_error_weight is None: self._estimation_error_weight = ADWIN() old_error = self.get_error_estimation() # Add element to ADWIN add = 0.0 if (bl_correct is True) else 1.0 self._estimation_error_weight.add_element(add) # Detect change with ADWIN self.error_change = self._estimation_error_weight.detected_change() if self.error_change is True and old_error > self.get_error_estimation( ): self.error_change = False # Check condition to build a new alternate tree if self.error_change is True: self._alternate_tree = hat._new_learning_node() hat.alternate_trees_cnt += 1 # Condition to replace alternate tree elif self._alternate_tree is not None and self._alternate_tree.is_null_error( ) is False: if self.get_error_width() > ERROR_WIDTH_THRESHOLD \ and self._alternate_tree.get_error_width() > ERROR_WIDTH_THRESHOLD: old_error_rate = self.get_error_estimation() alt_error_rate = self._alternate_tree.get_error_estimation() fDelta = .05 fN = 1.0 / self._alternate_tree.get_error_width() + 1.0 / ( self.get_error_width()) bound = math.sqrt(2.0 * old_error_rate * (1.0 - old_error_rate) * math.log(2.0 / fDelta) * fN) # To check, bound never less than (old_error_rate - alt_error_rate) if bound < (old_error_rate - alt_error_rate): hat._active_leaf_node_cnt -= self.number_leaves() hat._active_leaf_node_cnt += self._alternate_tree.number_leaves( ) self.kill_tree_children(hat) if parent is not None: parent.set_child(parent_branch, self._alternate_tree) else: # Switch tree root hat._tree_root = hat._tree_root.alternateTree hat.switch_alternate_trees_cnt += 1 elif bound < alt_error_rate - old_error_rate: if isinstance(self._alternate_tree, ActiveLearningNode): self._alternate_tree = None elif isinstance(self._alternate_tree, InactiveLearningNode): self._alternate_tree = None else: self._alternate_tree.kill_tree_children(hat) hat.pruned_alternate_trees_cnt += 1 # hat.pruned_alternate_trees_cnt to check # Learn_From_Instance alternate Tree and Child nodes if self._alternate_tree is not None: self._alternate_tree.learn_from_instance(X, y, weight, hat, parent, parent_branch) child_branch = self.instance_child_index(X) child = self.get_child(child_branch) if child is not None: child.learn_from_instance(X, y, weight, hat, self, child_branch) # Instance contains a categorical value previously unseen by the split # node elif isinstance(self.get_split_test(), NominalAttributeMultiwayTest) and \ self.get_split_test().branch_for_instance(X) < 0: # Creates a new learning node to encompass the new observed feature # value leaf_node = hat._new_learning_node() branch_id = self.get_split_test().add_new_branch( X[self.get_split_test().get_atts_test_depends_on()[0]]) self.set_child(branch_id, leaf_node) hat._active_leaf_node_cnt += 1 leaf_node.learn_from_instance(X, y, weight, hat, parent, parent_branch) # Override AdaNode def kill_tree_children(self, hat): for child in self._children: if child is not None: # Delete alternate tree if it exists if isinstance(child, SplitNode) and child._alternate_tree is not None: child._alternate_tree.kill_tree_children(hat) self._pruned_alternate_trees += 1 # Recursive delete of SplitNodes if isinstance(child, SplitNode): child.kill_tree_children(hat) if isinstance(child, ActiveLearningNode): child = None hat._active_leaf_node_cnt -= 1 elif isinstance(child, InactiveLearningNode): child = None hat._inactive_leaf_node_cnt -= 1 # override AdaNode def filter_instance_to_leaves(self, X, y, weight, parent, parent_branch, update_splitter_counts=False, found_nodes=None): if found_nodes is None: found_nodes = [] if update_splitter_counts: try: self._observed_class_distribution[ y] += weight # Dictionary (class_value, weight) except KeyError: self._observed_class_distribution[y] = weight child_index = self.instance_child_index(X) if child_index >= 0: child = self.get_child(child_index) if child is not None: child.filter_instance_to_leaves(X, y, weight, parent, parent_branch, update_splitter_counts, found_nodes) else: found_nodes.append(FoundNode(None, self, child_index)) if self._alternate_tree is not None: self._alternate_tree.filter_instance_to_leaves( X, y, weight, self, -999, update_splitter_counts, found_nodes)
def __init__( self, # Forest parameters n_estimators: int = 10, max_features='auto', aggregation_method: str = 'median', weighted_vote_strategy: str = None, lambda_value: int = 6, drift_detection_method: BaseDriftDetector = ADWIN(0.001), warning_detection_method: BaseDriftDetector = ADWIN(0.01), drift_detection_criteria: str = 'mse', # Tree parameters max_byte_size: int = 1048576000, memory_estimate_period: int = 2000000, grace_period: int = 50, split_confidence: float = 0.01, tie_threshold: float = 0.05, binary_split: bool = False, stop_mem_management: bool = False, remove_poor_atts: bool = False, no_preprune: bool = False, leaf_prediction: str = 'perceptron', nominal_attributes: list = None, learning_ratio_perceptron: float = 0.1, learning_ratio_decay: float = 0.001, learning_ratio_const: bool = True, random_state=None): super().__init__( n_estimators=n_estimators, max_features=max_features, lambda_value=lambda_value, drift_detection_method=drift_detection_method, warning_detection_method=warning_detection_method, # Tree parameters max_byte_size=max_byte_size, memory_estimate_period=memory_estimate_period, grace_period=grace_period, split_confidence=split_confidence, tie_threshold=tie_threshold, binary_split=binary_split, stop_mem_management=stop_mem_management, remove_poor_atts=remove_poor_atts, no_preprune=no_preprune, leaf_prediction=leaf_prediction, nominal_attributes=nominal_attributes, random_state=random_state) self.learning_ratio_perceptron = learning_ratio_perceptron self.learning_ratio_decay = learning_ratio_decay self.learning_ratio_const = learning_ratio_const if weighted_vote_strategy in [self._MSE, self._MAE, None]: self.weighted_vote_strategy = weighted_vote_strategy else: raise ValueError('Invalid weighted vote strategy: {}'.format( weighted_vote_strategy)) if aggregation_method in [self._MEAN, self._MEDIAN]: self.aggregation_method = aggregation_method else: raise ValueError( 'Invalid aggregation method: {}'.format(aggregation_method)) if drift_detection_criteria in [ self._MSE, self._MAE, self._PREDICTIONS ]: self.drift_detection_criteria = drift_detection_criteria else: raise ValueError('Invalid drift detection criteria: {}'.format( drift_detection_criteria))
class DeepNNPytorch(BaseSKMObject, ClassifierMixin): def __init__( self, class_labels=['0', '1'], # {'up':0,'down':1} use_cpu=True, process_as_a_batch=False, use_threads=False, background_training_after=4): # configuration variables (which has the same name as init parameters) self.class_labels = class_labels self.use_threads = use_threads self.background_training_after = background_training_after super().__init__() # status variables self.class_to_label = {} self.foreground_nets = [] # type: List[ANN] self.background_nets = [] # type: List[ANN] self.drift_detection_method = None self.warning_detection_method = None self.detected_warnings = 0 self.samples_seen = 0 self.last_detected_drift_around = 0 self.background_learner_threads = [] self.background_train_results = None self.foreground_train_results = None self.init_status_values() def init_status_values(self): # init status variables self.class_to_label = {} for i in range(len(self.class_labels)): self.class_to_label.update({i: self.class_labels[i]}) for i in range(len(foreground_net_config)): self.foreground_nets.append( ANN(learning_rate=foreground_net_config[i]['l_rate'], optimizer_type=foreground_net_config[i]['optimizer_type'], class_labels=self.class_labels)) for i in range(len(background_net_config)): self.background_nets.append( ANN(learning_rate=foreground_net_config[i]['l_rate'], optimizer_type=background_net_config[i]['optimizer_type'], class_labels=self.class_labels)) self.drift_detection_method = ADWIN(delta=1e-3, direction=ADWIN.DETECT_DOWN) self.warning_detection_method = ADWIN(delta=1e-8, direction=ADWIN.DETECT_DOWN) self.detected_warnings = 0 self.samples_seen = 0 self.last_detected_drift_around = 0 self.background_learner_threads = [] self.background_train_results = None self.foreground_train_results = None print(self) def partial_fit(self, X, y, classes=None, sample_weight=None): r, c = get_dimensions(X) self.samples_seen += r # if self.samples_seen % 2 == 0: if len(self.background_learner_threads) == 0: if self.samples_seen % self.background_training_after == 0: self.background_train_results = { 'probas': [None] * len(self.background_nets), 'y_hats': [None] * len(self.background_nets), 'avg_loss_since_last_detected_drift_by_parent': [0] * len(self.background_nets) } for i in range(len(self.background_nets)): self.background_learner_threads.append( threading.Thread(target=net_train, args=( self.background_nets[i], X, r, c, y, self.background_train_results, i, self.last_detected_drift_around, ))) for i in range(len(self.background_nets)): self.background_learner_threads[i].start() else: # there are live background learner threads # wait for self.background_training_after instances to join them if self.samples_seen % self.background_training_after == self.background_training_after - 1: # TODO: CPython does not support multi threading: https://docs.python.org/3/library/threading.html # we still may be fine as long as we don't compile the module using CPython. # Multiprocessing is an alternative: # https://docs.python.org/3/library/multiprocessing.html#module-multiprocessing for i in range(len(self.background_nets)): self.background_learner_threads[i].join() self.background_learner_threads = [] if self.foreground_train_results is not None: min_back = np.argmin(self.background_train_results[ 'avg_loss_since_last_detected_drift_by_parent'], axis=0) max_fore = np.argmax(self.foreground_train_results[ 'avg_loss_since_last_detected_drift_by_parent'], axis=0) # min_back < max_fore if self.background_train_results['avg_loss_since_last_detected_drift_by_parent'][min_back] \ < self.foreground_train_results['avg_loss_since_last_detected_drift_by_parent'][max_fore]: tmp_net = self.foreground_nets[max_fore] self.foreground_nets[max_fore] = self.background_nets[ min_back] self.background_nets[min_back] = tmp_net self.foreground_train_results = { 'probas': [None] * len(self.foreground_nets), 'y_hats': [None] * len(self.foreground_nets), 'avg_loss_since_last_detected_drift_by_parent': [0] * len(self.foreground_nets) } if self.use_threads: t = [] for i in range(len(self.foreground_nets)): t.append( threading.Thread(target=net_train, args=( self.foreground_nets[i], X, r, c, y, self.foreground_train_results, i, self.last_detected_drift_around, ))) for i in range(len(self.foreground_nets)): t[i].start() for i in range(len(self.foreground_nets)): t[i].join() else: for i in range(len(self.foreground_nets)): net_train(self.foreground_nets[i], X, r, c, y, self.foreground_train_results, i, self.last_detected_drift_around) if self.drift_detection_method is not None: # get predicted class and compare with actual class label predicted_label = vectorized_map_class_to_label( np.argmax( np.sum(self.foreground_train_results['probas'], axis=0) / len(self.foreground_nets), axis=1), class_to_label_map=self.class_to_label) # TODO: we may have to have a special case for batch processing predicted_matches_actual = predicted_label == y self.drift_detection_method.add_element( 1 if predicted_matches_actual else 0) if self.warning_detection_method is not None: self.warning_detection_method.add_element( 1 if predicted_matches_actual else 0) # pass the difference to the detector # predicted_matches_actual = torch.abs(y-output).detach().numpy()[0] # self.drift_detection_method.add_element(predicted_matches_actual) # Check if the was a warning if self.warning_detection_method is not None: if self.warning_detection_method.detected_change(): self.detected_warnings += 1 else: # warning detector is None, hence drift detector has warning detection capability. if self.drift_detection_method.detected_warning_zone(): self.detected_warnings += 1 # 3 is the threshold level # Check if the was a change if self.detected_warnings > 3 and self.drift_detection_method.detected_change( ): print('Drift detected by {} around {} th sample.'.format( self.drift_detection_method, self.samples_seen)) self.detected_warnings = 0 self.last_detected_drift_around = self.samples_seen # Find the the worst learner from the foreground and replace it with the background return self def predict(self, X): y_proba = self.predict_proba(X) pred_sum_per_class = np.sum(y_proba, axis=0) pred_avgsum_per_class = np.divide(pred_sum_per_class, len(self.foreground_nets)) y_pred = np.argmax(pred_avgsum_per_class, axis=0) return vectorized_map_class_to_label( np.asarray([y_pred]), class_to_label_map=self.class_to_label) def predict_proba(self, X): r, c = get_dimensions(X) probas = np.zeros([len(self.foreground_nets), len(self.class_labels)]) # if self.use_threads: # t = [] # for i in range(len(self.nets)): # t.append(threading.Thread(target=net_predict_proba, args=(self.nets[i], X, r, c, probas, i,))) # # for i in range(len(self.nets)): # t[i].start() # # for i in range(len(self.nets)): # t[i].join() # else: for i in range(len(self.foreground_nets)): net_predict_proba(self.foreground_nets[i], X, r, c, probas, i) return np.asarray(probas) def reset(self): # configuration variables (which has the same name as init parameters) should be copied by the caller function for i in range(len(self.foreground_nets)): self.foreground_nets[i].reset() return self def __str__(self): return str(self.__class__) + ": " + str(self.__dict__) def stream_ended(self): print('\nNetwork configuration:\n' '{}\n' '=======================================\n' 'Foreground Nets\n'.format(self)) print( 'optimizer_type,learning_rate,accumulated_loss,accumulated_loss_since_last_detected_drift_by_parent' ) for i in range(len(self.foreground_nets)): print('{},{},{},{}'.format( self.foreground_nets[i].optimizer_type, self.foreground_nets[i].learning_rate, self.foreground_nets[i].accumulated_loss / self.foreground_nets[i].samples_seen, self.foreground_nets[i]. accumulated_loss_since_last_detected_drift_by_parent / self.foreground_nets[i]. samples_seen_after_last_detected_drift_by_parent)) print('\n' 'Background Nets\n'.format(self)) for i in range(len(self.background_nets)): print('{},{},{},{}'.format( self.background_nets[i].optimizer_type, self.background_nets[i].learning_rate, self.background_nets[i].accumulated_loss / self.background_nets[i].samples_seen, self.background_nets[i]. accumulated_loss_since_last_detected_drift_by_parent / self.background_nets[i]. samples_seen_after_last_detected_drift_by_parent)) print('\n')
def EvaluateModels(stream, run, n_trees, n_samples_max, n_samples_meas, metaModel = None): # For adaptive experiments stream[0].prepare_for_use() # Evaluate model (with adaptation or not) arf = AdaptiveRandomForest(n_estimators = n_trees, lambda_value=6, grace_period=10, split_confidence=0.1, tie_threshold=0.005, warning_detection_method= ADWIN(delta=0.01), drift_detection_method=ADWIN(delta=0.001)) modelsList = [arf] modelsNames = ['ARF'] try : dirpath = os.getcwd() new_dir_path = dirpath+'/ExperimentTuningTrees' os.mkdir(new_dir_path) except FileExistsError : pass if metaModel == None : fileName = new_dir_path+'\\'+stream[1]+'_'+str(n_trees)+'Trees_Adapt_Drift_ARDClassic_Run'+str(run)+'.csv' else : fileName = new_dir_path+'\\'+stream[1]+'_AdativeSetting_Trees_Adapt_Drift_ARDClassic_Run'+str(run)+'.csv' evaluator = EvaluatePrequentialAndAdaptTreesARF(metrics=['accuracy','kappa','running_time','ram_hours'], show_plot=False, n_wait=n_samples_meas, pretrain_size=200, max_samples=n_samples_max, output_file = fileName, metaKB=metaModel) # Run evaluation model, acc = evaluator.evaluate(stream=stream[0], model=modelsList, model_names=modelsNames)
def __init__(self, initial_stats=None, random_state=None): super().__init__(initial_stats) self._adwin = ADWIN() self.error_change = False self._random_state = check_random_state(random_state)
class AdaLearningNode(ActiveLearningNodeNBA, AdaNode): """ Learning node for Hoeffding Adaptive Tree. Uses Adaptive Naive Bayes models. Parameters ---------- initial_stats: dict (class_value, weight) or None Initial class observations """ def __init__(self, initial_stats=None, random_state=None): super().__init__(initial_stats) self._adwin = ADWIN() self.error_change = False self._random_state = check_random_state(random_state) @property def n_leaves(self): return 1 @property def error_estimation(self): return self._adwin.estimation @property def error_width(self): return self._adwin.width def error_is_null(self): return self._adwin is None def kill_tree_children(self, hat): pass def learn_one(self, X, y, weight, tree, parent, parent_branch): true_class = y if tree.bootstrap_sampling: # Perform bootstrap-sampling k = self._random_state.poisson(1.0) if k > 0: weight = weight * k class_prediction = get_max_value_key(self.predict_one(X, tree=tree)) is_correct = (true_class == class_prediction) if self._adwin is None: self._adwin = ADWIN() old_error = self.error_estimation # Add element to ADWIN self._adwin.add_element(0.0 if is_correct else 1.0) # Detect change with Adwin self.error_change = self._adwin.detected_change() if self.error_change and old_error > self.error_estimation: self.error_change = False # Update statistics super().learn_one(X, y, weight=weight, tree=tree) weight_seen = self.total_weight if weight_seen - self.last_split_attempt_at >= tree.grace_period: tree._attempt_to_split(self, parent, parent_branch) self.last_split_attempt_at = weight_seen # Override LearningNodeNBAdaptive def predict_one(self, X, *, tree=None): prediction_option = tree.leaf_prediction # MC if prediction_option == tree._MAJORITY_CLASS: dist = self.stats # NB elif prediction_option == tree._NAIVE_BAYES: dist = do_naive_bayes_prediction(X, self.stats, self.attribute_observers) # NBAdaptive (default) else: dist = super().predict_one(X, tree=tree) dist_sum = sum(dist.values()) # sum all values in dictionary normalization_factor = dist_sum * self.error_estimation * self.error_estimation if normalization_factor > 0.0: dist = normalize_values_in_dict(dist, normalization_factor, inplace=False) return dist # Override AdaNode, New for option votes def filter_instance_to_leaves(self, X, y, weight, parent, parent_branch, update_splitter_counts, found_nodes=None): if found_nodes is None: found_nodes = [] found_nodes.append(FoundNode(self, parent, parent_branch))
def learn_one(self, X, y, weight, tree, parent, parent_branch): true_class = y class_prediction = 0 leaf = self.filter_instance_to_leaf(X, parent, parent_branch) if leaf.node is not None: class_prediction = get_max_value_key( leaf.node.predict_one(X, tree=tree)) is_correct = (true_class == class_prediction) if self._adwin is None: self._adwin = ADWIN() old_error = self.error_estimation # Add element to ADWIN add = 0.0 if is_correct else 1.0 self._adwin.add_element(add) # Detect change with ADWIN self.error_change = self._adwin.detected_change() if self.error_change and old_error > self.error_estimation: self.error_change = False # Check condition to build a new alternate tree if self.error_change: self._alternate_tree = tree._new_learning_node() tree.alternate_trees_cnt += 1 # Condition to replace alternate tree elif self._alternate_tree is not None and not self._alternate_tree.error_is_null( ): if self.error_width > tree._ERROR_WIDTH_THRESHOLD \ and self._alternate_tree.error_width > tree._ERROR_WIDTH_THRESHOLD: old_error_rate = self.error_estimation alt_error_rate = self._alternate_tree.error_estimation fDelta = .05 fN = 1.0 / self._alternate_tree.error_width + 1.0 / self.error_width bound = math.sqrt(2.0 * old_error_rate * (1.0 - old_error_rate) * math.log(2.0 / fDelta) * fN) # To check, bound never less than (old_error_rate - alt_error_rate) if bound < (old_error_rate - alt_error_rate): tree._active_leaf_node_cnt -= self.n_leaves tree._active_leaf_node_cnt += self._alternate_tree.n_leaves self.kill_tree_children(tree) if parent is not None: parent.set_child(parent_branch, self._alternate_tree) else: # Switch tree root tree._tree_root = tree._tree_root._alternate_tree tree.switch_alternate_trees_cnt += 1 elif bound < alt_error_rate - old_error_rate: if isinstance(self._alternate_tree, SplitNode): self._alternate_tree.kill_tree_children(tree) else: self._alternate_tree = None tree.pruned_alternate_trees_cnt += 1 # hat.pruned_alternate_trees_cnt to check # Learn one sample in alternate tree and child nodes if self._alternate_tree is not None: self._alternate_tree.learn_one(X, y, weight, tree, parent, parent_branch) child_branch = self.instance_child_index(X) child = self.get_child(child_branch) if child is not None: child.learn_one(X, y, weight, tree, parent=self, parent_branch=child_branch) # Instance contains a categorical value previously unseen by the split # node elif isinstance(self.split_test, NominalAttributeMultiwayTest) and \ self.split_test.branch_for_instance(X) < 0: # Creates a new learning node to encompass the new observed feature # value leaf_node = tree._new_learning_node() branch_id = self.split_test.add_new_branch( X[self.split_test.get_atts_test_depends_on()[0]]) self.set_child(branch_id, leaf_node) tree._active_leaf_node_cnt += 1 leaf_node.learn_one(X, y, weight, tree, parent, parent_branch)
class ADWINChangeDetector(BaseDriftDetector): """ Drift detection method based in ADWIN. Parameters ---------- delta : float (default=0.002) The delta parameter for the ADWIN algorithm. Notes ----- ADWIN [1]_ (ADaptive WINdowing) is an adaptive sliding window algorithm for detecting change, and keeping updated statistics about a data stream. ADWIN allows algorithms not adapted for drifting data, to be resistant to this phenomenon. The general idea is to keep statistics from a window of variable size while detecting concept drift. The algorithm will decide the size of the window by cutting the statistics' window at different points and analysing the average of some statistic over these two windows. If the absolute value of the difference between the two averages surpasses a pre-defined threshold, change is detected at that point and all data before that time is discarded. References ---------- .. [1] Bifet, Albert, and Ricard Gavalda. "Learning from time-changing data with adaptive windowing." In Proceedings of the 2007 SIAM international conference on data mining, pp. 443-448. Society for Industrial and Applied Mathematics, 2007. Examples -------- >>> # Imports >>> import numpy as np >>> from skmultiflow.drift_detection import ADWINChangeDetector >>> adwin_change_detector = ADWINChangeDetector() >>> # Simulating a data stream as a normal distribution of 1's and 0's >>> data_stream = np.random.randint(2, size=2000) >>> # Changing the data concept from index 999 to 2000 >>> for i in range(999, 2000): ... data_stream[i] = np.random.randint(4, high=8) >>> # Adding stream elements to ADWIN and verifying if drift occurred >>> for i in range(2000): ... adwin_change_detector.add_element(data_stream[i]) ... if adwin_change_detector.detected_change(): ... print('Change detected in data: ' + str(data_stream[i]) + ' - at index: ' + str(i)) """ def __init__(self, delta=.002): super().__init__() self.adwin = ADWIN(delta=delta) super().reset() def add_element(self, input_value): err_estim = self.adwin.estimation self.adwin.add_element(input_value) res_input = self.adwin.detected_change() self.in_concept_change = False self.in_warning_zone = False if self.adwin.detected_warning_zone(): self.in_warning_zone = True if res_input: if self.adwin.estimation > err_estim: self.in_concept_change = True self.in_warning_zone = False self.estimation = self.adwin.estimation
def __init__(self, delta=.002): super().__init__() self.adwin = ADWIN(delta=delta) super().reset()
class KNNADWINClassifier(KNNClassifier): """ K-Nearest Neighbors classifier with ADWIN change detector. This Classifier is an improvement from the regular KNNClassifier, as it is resistant to concept drift. It utilises the ADWIN change detector to decide which samples to keep and which ones to forget, and by doing so it regulates the sample window size. To know more about the ADWIN change detector, please see :class:`skmultiflow.drift_detection.ADWIN` It uses the regular KNNClassifier as a base class, with the major difference that this class keeps a variable size window, instead of a fixed size one and also it updates the adwin algorithm at each partial_fit call. Parameters ---------- n_neighbors: int (default=5) The number of nearest neighbors to search for. max_window_size: int (default=1000) The maximum size of the window storing the last viewed samples. leaf_size: int (default=30) The maximum number of samples that can be stored in one leaf node, which determines from which point the algorithm will switch for a brute-force approach. The bigger this number the faster the tree construction time, but the slower the query time will be. metric: string or sklearn.DistanceMetric object sklearn.KDTree parameter. The distance metric to use for the KDTree. Default=’euclidean’. KNNClassifier.valid_metrics() gives a list of the metrics which are valid for KDTree. Notes ----- This estimator is not optimal for a mixture of categorical and numerical features. This implementation treats all features from a given stream as numerical. Examples -------- >>> # Imports >>> from skmultiflow.lazy import KNNADWINClassifier >>> from skmultiflow.data import ConceptDriftStream >>> # Setting up the stream >>> stream = ConceptDriftStream(position=2500, width=100, random_state=1) >>> # Setting up the KNNAdwin classifier >>> knn_adwin = KNNADWINClassifier(n_neighbors=8, leaf_size=40, max_window_size=1000) >>> # Keep track of sample count and correct prediction count >>> n_samples = 0 >>> corrects = 0 >>> while n_samples < 5000: ... X, y = stream.next_sample() ... pred = knn_adwin.predict(X) ... if y[0] == pred[0]: ... corrects += 1 ... knn_adwin = knn_adwin.partial_fit(X, y) ... n_samples += 1 >>> >>> # Displaying the results >>> print('KNNClassifier usage example') >>> print(str(n_samples) + ' samples analyzed.') 5000 samples analyzed. >>> print("KNNADWINClassifier's performance: " + str(corrects/n_samples)) KNNAdwin's performance: 0.5714 """ def __init__(self, n_neighbors=5, max_window_size=1000, leaf_size=30, metric='euclidean'): super().__init__(n_neighbors=n_neighbors, max_window_size=max_window_size, leaf_size=leaf_size, metric=metric) self.adwin = ADWIN() def reset(self): """ Reset the estimator. Resets the ADWIN Drift detector as well as the KNN model. Returns ------- KNNADWINClassifier self """ self.adwin = ADWIN() return super().reset() def partial_fit(self, X, y, classes=None, sample_weight=None): """ Partially (incrementally) fit the model. Parameters ---------- X: Numpy.ndarray of shape (n_samples, n_features) The data upon which the algorithm will create its model. y: Array-like An array-like containing the classification targets for all samples in X. classes: numpy.ndarray, optional (default=None) Array with all possible/known classes. sample_weight: Not used. Returns ------- KNNADWINClassifier self Notes ----- Partially fits the model by updating the window with new samples while also updating the ADWIN algorithm. IF ADWIN detects a change, the window is split in such a wat that samples from the previous concept are dropped. """ r, c = get_dimensions(X) if classes is not None: self.classes = list(set().union(self.classes, classes)) for i in range(r): self.data_window.add_sample(X[i], y[i]) if self.data_window.size >= self.n_neighbors: correctly_classifies = 1 if self.predict(np.asarray( [X[i]])) == y[i] else 0 self.adwin.add_element(correctly_classifies) else: self.adwin.add_element(0) if self.data_window.size >= self.n_neighbors: if self.adwin.detected_change(): if self.adwin.width < self.data_window.size: for i in range(self.data_window.size, self.adwin.width, -1): self.data_window.delete_oldest_sample() return self
adwin_param = [0.002, 0.005, 0.01] ddm_param = [3, 5, 7] ks_param1 = [100, 150, 200] ks_param2 = [30, 50, 100] ph_param1 = [25, 50, 75] ph_param2 = [0.005, 0.01, 0.02] knn = KNNClassifier() stream = driftStreams[0] for i in range(0, 3): trainX, trainY = stream.next_sample(2000) knn.partial_fit(trainX, trainY) adwin = ADWIN(delta=adwin_param[i]) ddm = DDM(out_control_level=ddm_param[i]) kswin1 = KSWIN(window_size=ks_param1[i]) # kswin2 = KSWIN(stat_size=ks_param2[i]) ph1 = PageHinkley(threshold=ph_param1[i]) ph2 = PageHinkley(delta=ph_param2[i]) adwin_results = [] ddm_results = [] kswin1_results = [] kswin2_results = [] ph1_results = [] ph2_results = [] n_samples = 0 corrects = 0
def __init__( self, base_estimator=HoeffdingTreeClassifier(grace_period=50, split_confidence=0.01), n_estimators: int = 100, subspace_mode: str = "percentage", subspace_size: int = 60, training_method: str = "randompatches", lam: float = 6.0, drift_detection_method: BaseDriftDetector = ADWIN(delta=1e-5), warning_detection_method: BaseDriftDetector = ADWIN(delta=1e-4), disable_weighted_vote: bool = False, disable_drift_detection: bool = False, disable_background_learner: bool = False, nominal_attributes=None, random_state=None): self.base_estimator = base_estimator # Not restricted to a specific base estimator. self.n_estimators = n_estimators if subspace_mode not in { self._FEATURES_SQRT, self._FEATURES_SQRT_INV, self._FEATURES_PERCENT, self._FEATURES_M }: raise ValueError( "Invalid subspace_mode: {}.\n" "Valid options are: {}".format( subspace_mode, { self._FEATURES_M, self._FEATURES_SQRT, self._FEATURES_SQRT_INV, self._FEATURES_PERCENT })) self.subspace_mode = subspace_mode self.subspace_size = subspace_size if training_method not in { self._TRAIN_RESAMPLING, self._TRAIN_RANDOM_PATCHES, self._TRAIN_RANDOM_SUBSPACES }: raise ValueError( "Invalid training_method: {}.\n" "Valid options are: {}".format( training_method, { self._TRAIN_RANDOM_PATCHES, self._TRAIN_RANDOM_SUBSPACES, self._TRAIN_RESAMPLING })) self.training_method = training_method self.lam = lam self.drift_detection_method = drift_detection_method self.warning_detection_method = warning_detection_method self.disable_weighted_vote = disable_weighted_vote self.disable_drift_detection = disable_drift_detection self.disable_background_learner = disable_background_learner # Single option (accuracy) for drift detection criteria. Could be extended in the future. self.drift_detection_criteria = 'accuracy' self.nominal_attributes = nominal_attributes if nominal_attributes else [] self.random_state = random_state # self._random_state is the actual object used internally self._random_state = check_random_state(self.random_state) self.ensemble = None self._n_samples_seen = 0 self._subspaces = None self._base_performance_evaluator = ClassificationMeasurements() self._base_learner_class = StreamingRandomPatchesBaseLearner
def learn_from_instance(self, X, y, weight, hat, parent, parent_branch): true_class = y class_prediction = 0 leaf = self.filter_instance_to_leaf(X, parent, parent_branch) if leaf.node is not None: class_prediction = get_max_value_key( leaf.node.get_class_votes(X, hat)) bl_correct = (true_class == class_prediction) if self._estimation_error_weight is None: self._estimation_error_weight = ADWIN() old_error = self.get_error_estimation() # Add element to ADWIN add = 0.0 if (bl_correct is True) else 1.0 self._estimation_error_weight.add_element(add) # Detect change with ADWIN self.error_change = self._estimation_error_weight.detected_change() if self.error_change is True and old_error > self.get_error_estimation( ): self.error_change = False # Check condition to build a new alternate tree if self.error_change is True: self._alternate_tree = hat._new_learning_node() hat.alternate_trees_cnt += 1 # Condition to replace alternate tree elif self._alternate_tree is not None and self._alternate_tree.is_null_error( ) is False: if self.get_error_width() > ERROR_WIDTH_THRESHOLD \ and self._alternate_tree.get_error_width() > ERROR_WIDTH_THRESHOLD: old_error_rate = self.get_error_estimation() alt_error_rate = self._alternate_tree.get_error_estimation() fDelta = .05 fN = 1.0 / self._alternate_tree.get_error_width() + 1.0 / ( self.get_error_width()) bound = math.sqrt(2.0 * old_error_rate * (1.0 - old_error_rate) * math.log(2.0 / fDelta) * fN) # To check, bound never less than (old_error_rate - alt_error_rate) if bound < (old_error_rate - alt_error_rate): hat._active_leaf_node_cnt -= self.number_leaves() hat._active_leaf_node_cnt += self._alternate_tree.number_leaves( ) self.kill_tree_children(hat) if parent is not None: parent.set_child(parent_branch, self._alternate_tree) else: # Switch tree root hat._tree_root = hat._tree_root.alternateTree hat.switch_alternate_trees_cnt += 1 elif bound < alt_error_rate - old_error_rate: if isinstance(self._alternate_tree, ActiveLearningNode): self._alternate_tree = None elif isinstance(self._alternate_tree, InactiveLearningNode): self._alternate_tree = None else: self._alternate_tree.kill_tree_children(hat) hat.pruned_alternate_trees_cnt += 1 # hat.pruned_alternate_trees_cnt to check # Learn_From_Instance alternate Tree and Child nodes if self._alternate_tree is not None: self._alternate_tree.learn_from_instance(X, y, weight, hat, parent, parent_branch) child_branch = self.instance_child_index(X) child = self.get_child(child_branch) if child is not None: child.learn_from_instance(X, y, weight, hat, self, child_branch) # Instance contains a categorical value previously unseen by the split # node elif isinstance(self.get_split_test(), NominalAttributeMultiwayTest) and \ self.get_split_test().branch_for_instance(X) < 0: # Creates a new learning node to encompass the new observed feature # value leaf_node = hat._new_learning_node() branch_id = self.get_split_test().add_new_branch( X[self.get_split_test().get_atts_test_depends_on()[0]]) self.set_child(branch_id, leaf_node) hat._active_leaf_node_cnt += 1 leaf_node.learn_from_instance(X, y, weight, hat, parent, parent_branch)
async def predict(self, X: PredictionData) -> bool: detector = self.detectors.get(X.identifier, ADWIN(self.delta)) self.detectors[X.identifier] = detector detector.add_element(X.data["sensor"]) return detector.detected_change()
class AdaSplitNode(SplitNode, NewNode): def __init__(self, split_test, class_observations): super().__init__(split_test, class_observations) self._estimation_error_weight = ADWIN() self._alternate_tree = None self.error_change = False self._random_seed = 1 self._classifier_random = check_random_state(self._random_seed) # Override NewNode def number_leaves(self): num_of_leaves = 0 for child in self._children: if child is not None: num_of_leaves += child.number_leaves() return num_of_leaves # Override NewNode def get_error_estimation(self): return self._estimation_error_weight.estimation # Override NewNode def get_error_width(self): w = 0.0 if self.is_null_error() is False: w = self._estimation_error_weight.width return w # Override NewNode def is_null_error(self): return self._estimation_error_weight is None # Override NewNode def learn_from_instance(self, X, y, weight, hat, parent, parent_branch): true_class = y class_prediction = 0 leaf = self.filter_instance_to_leaf(X, parent, parent_branch) if leaf.node is not None: class_prediction = get_max_value_key(leaf.node.get_class_votes(X, hat)) bl_correct = (true_class == class_prediction) if self._estimation_error_weight is None: self._estimation_error_weight = ADWIN() old_error = self.get_error_estimation() # Add element to ADWIN add = 0.0 if (bl_correct is True) else 1.0 self._estimation_error_weight.add_element(add) # Detect change with ADWIN self.error_change = self._estimation_error_weight.detected_change() if self.error_change is True and old_error > self.get_error_estimation(): self.error_change = False # Check condition to build a new alternate tree if self.error_change is True: self._alternate_tree = hat._new_learning_node() hat.alternate_trees_cnt += 1 # Condition to replace alternate tree elif self._alternate_tree is not None and self._alternate_tree.is_null_error() is False: if self.get_error_width() > error_width_threshold \ and self._alternate_tree.get_error_width() > error_width_threshold: old_error_rate = self.get_error_estimation() alt_error_rate = self._alternate_tree.get_error_estimation() fDelta = .05 fN = 1.0 / self._alternate_tree.get_error_width() + 1.0 / (self.get_error_width()) bound = math.sqrt(2.0 * old_error_rate * (1.0 - old_error_rate) * math.log(2.0 / fDelta) * fN) # To check, bound never less than (old_error_rate - alt_error_rate) if bound < (old_error_rate - alt_error_rate): hat._active_leaf_node_cnt -= self.number_leaves() hat._active_leaf_node_cnt += self._alternate_tree.number_leaves() self.kill_tree_children(hat) if parent is not None: parent.set_child(parent_branch, self._alternate_tree) else: # Switch tree root hat._tree_root = hat._tree_root.alternateTree hat.switch_alternate_trees_cnt += 1 elif bound < alt_error_rate - old_error_rate: if isinstance(self._alternate_tree, TS_HAT.ActiveLearningNode): self._alternate_tree = None elif isinstance(self._alternate_tree, TS_HAT.InactiveLearningNode): self._alternate_tree = None else: self._alternate_tree.kill_tree_children(hat) hat.pruned_alternate_trees_cnt += 1 # hat.pruned_alternate_trees_cnt to check # Learn_From_Instance alternate Tree and Child nodes if self._alternate_tree is not None: self._alternate_tree.learn_from_instance(X, y, weight, hat, parent, parent_branch) child_branch = self.instance_child_index(X) child = self.get_child(child_branch) if child is not None: child.learn_from_instance(X, y, weight, hat, parent, parent_branch) # Override NewNode def kill_tree_children(self, hat): for child in self._children: if child is not None: # Delete alternate tree if it exists if isinstance(child, TS_HAT.AdaSplitNode) and child._alternate_tree is not None: child._alternate_tree.kill_tree_children(hat) self._pruned_alternate_trees += 1 # Recursive delete of SplitNodes if isinstance(child, TS_HAT.AdaSplitNode): child.kill_tree_children(hat) if isinstance(child, TS_HAT.ActiveLearningNode): child = None hat._active_leaf_node_cnt -= 1 elif isinstance(child, TS_HAT.InactiveLearningNode): child = None hat._inactive_leaf_node_cnt -= 1 # override NewNode def filter_instance_to_leaves(self, X, y, weight, parent, parent_branch, update_splitter_counts=False, found_nodes=None): if found_nodes is None: found_nodes = [] if update_splitter_counts: try: self._observed_class_distribution[y] += weight # Dictionary (class_value, weight) except KeyError: self._observed_class_distribution[y] = weight child_index = self.instance_child_index(X) if child_index >= 0: child = self.get_child(child_index) if child is not None: child.filter_instance_to_leaves(X, y, weight, parent, parent_branch, update_splitter_counts, found_nodes) else: found_nodes.append(HoeffdingTree.FoundNode(None, self, child_index)) if self._alternate_tree is not None: self._alternate_tree.filter_instance_to_leaves(X, y, weight, self, -999, update_splitter_counts, found_nodes)
def partial_fit(self, X, y, classes=None, weight=None): """ partial_fit Partially fits the model, based on the X and y matrix. Since it's an ensemble learner, if X and y matrix of more than one sample are passed, the algorithm will partial fit the model one sample at a time. Each sample is trained by each classifier a total of K times, where K is drawn by a Poisson(l) distribution. l is updated after every example using :math:`lambda_{sc}` if th estimator correctly classifies the example or :math:`lambda_{sw}` in the other case. Parameters ---------- X: Numpy.ndarray of shape (n_samples, n_features) Features matrix used for partially updating the model. y: Array-like An array-like of all the class labels for the samples in X. classes: list List of all existing classes. This is an optional parameter, except for the first partial_fit call, when it becomes obligatory. weight: Array-like Instance weight. If not provided, uniform weights are assumed. Raises ------ ValueError: A ValueError is raised if the 'classes' parameter is not passed in the first partial_fit call, or if they are passed in further calls but differ from the initial classes list passed.. """ if self.classes is None: if classes is None: raise ValueError( "The first partial_fit call should pass all the classes.") else: self.classes = classes if self.classes is not None and classes is not None: if set(self.classes) == set(classes): pass else: raise ValueError( "The classes passed to the partial_fit function differ from those passed earlier." ) self.__adjust_ensemble_size() r, _ = get_dimensions(X) for j in range(r): change_detected = False for i in range(self.n_estimators): a = (i + 1) / self.n_estimators if y[j] == 1: lam = a * self.sampling_rate else: lam = a k = self.random_state.poisson(lam) if k > 0: for b in range(k): self.ensemble[i].partial_fit([X[j]], [y[j]], classes, weight) if self.drift_detection: try: pred = self.ensemble[i].predict(X) error_estimation = self.adwin_ensemble[i].estimation for j in range(r): if pred[j] is not None: if pred[j] == y[j]: self.adwin_ensemble[i].add_element(1) else: self.adwin_ensemble[i].add_element(0) if self.adwin_ensemble[i].detected_change(): if self.adwin_ensemble[ i].estimation > error_estimation: change_detected = True except ValueError: change_detected = False pass if change_detected and self.drift_detection: max_threshold = 0.0 i_max = -1 for i in range(self.n_estimators): if max_threshold < self.adwin_ensemble[i].estimation: max_threshold = self.adwin_ensemble[i].estimation i_max = i if i_max != -1: self.ensemble[i_max].reset() self.adwin_ensemble[i_max] = ADWIN()
class AdaLearningNode(LearningNodeNBAdaptive, NewNode): def __init__(self, initial_class_observations): super().__init__(initial_class_observations) self._estimation_error_weight = ADWIN() self.error_change = False self._randomSeed = 1 self._classifier_random = check_random_state(self._randomSeed) # Override NewNode def number_leaves(self): return 1 # Override NewNode def get_error_estimation(self): return self._estimation_error_weight.estimation # Override NewNode def get_error_width(self): return self._estimation_error_weight.width # Override NewNode def is_null_error(self): return self._estimation_error_weight is None def kill_tree_children(self, hat): pass # Override NewNode def learn_from_instance(self, X, y, weight, hat, parent, parent_branch): true_class = y # k = self._classifier_random.poisson(1.0) # if k > 0: # weight = weight * k tmp = self.get_class_votes(X, hat) class_prediction = get_max_value_key(tmp) bl_correct = (true_class == class_prediction) if self._estimation_error_weight is None: self._estimation_error_weight = ADWIN() old_error = self.get_error_estimation() # Add element to Adwin add = 0.0 if (bl_correct is True) else 1.0 self._estimation_error_weight.add_element(add) # Detect change with Adwin self.error_change = self._estimation_error_weight.detected_change() if self.error_change is True and old_error > self.get_error_estimation(): self.error_change = False # Update statistics super().learn_from_instance(X, y, weight, hat) # call ActiveLearningNode weight_seen = self.get_weight_seen() if weight_seen - self.get_weight_seen_at_last_split_evaluation() >= hat.grace_period: hat._attempt_to_split(self, parent, parent_branch) self.set_weight_seen_at_last_split_evaluation(weight_seen) # Override LearningNodeNBAdaptive def get_class_votes(self, X, ht): # dist = {} prediction_option = ht.leaf_prediction # MC if prediction_option == MAJORITY_CLASS: dist = self.get_observed_class_distribution() # NB elif prediction_option == NAIVE_BAYES: dist = do_naive_bayes_prediction(X, self._observed_class_distribution, self._attribute_observers) # NBAdaptive else: if self._mc_correct_weight > self._nb_correct_weight: dist = self.get_observed_class_distribution() else: dist = do_naive_bayes_prediction(X, self._observed_class_distribution, self._attribute_observers) dist_sum = sum(dist.values()) # sum all values in dictionary normalization_factor = dist_sum * self.get_error_estimation() * self.get_error_estimation() if normalization_factor > 0.0: normalize_values_in_dict(dist, normalization_factor) return dist # Override NewNode, New for option votes def filter_instance_to_leaves(self, X, y, weight, parent, parent_branch, update_splitter_counts, found_nodes=None): if found_nodes is None: found_nodes = [] found_nodes.append(HoeffdingTree.FoundNode(self, parent, parent_branch))
def __partial_fit(self, X, y): if self.init_matrix_codes: self.matrix_codes = np.zeros( (self.actual_n_estimators, len(self.classes)), dtype=int) for i in range(self.actual_n_estimators): n_zeros = 0 n_ones = 0 while (n_ones - n_zeros) * ( n_ones - n_zeros) > self.actual_n_estimators % 2: n_zeros = 0 n_ones = 0 for j in range(len(self.classes)): if (j == 1) and (len(self.classes) == 2): result = 1 - self.matrix_codes[i][0] else: result = self._random_state.randint(2) self.matrix_codes[i][j] = result if result == 1: n_ones += 1 else: n_zeros += 1 self.init_matrix_codes = False change_detected = False X_cp, y_cp = cp.deepcopy(X), cp.deepcopy(y) for i in range(self.actual_n_estimators): k = 0.0 if self.leverage_algorithm == self.LEVERAGE_ALGORITHMS[0]: k = self._random_state.poisson(self.w) elif self.leverage_algorithm == self.LEVERAGE_ALGORITHMS[1]: error = self.adwin_ensemble[i].estimation pred = self.ensemble[i].predict(np.asarray([X])) if pred is None: k = 1.0 elif pred[0] != y: k = 1.0 elif self._random_state.rand() < (error / (1.0 - error)): k = 1.0 else: k = 0.0 elif self.leverage_algorithm == self.LEVERAGE_ALGORITHMS[2]: w = 1.0 k = 0.0 if (self._random_state.randint(2) == 1) else w elif self.leverage_algorithm == self.LEVERAGE_ALGORITHMS[3]: w = 1.0 k = 1.0 + self._random_state.poisson(w) elif self.leverage_algorithm == self.LEVERAGE_ALGORITHMS[4]: w = 1.0 k = self._random_state.poisson(1) k = w if k > 0 else 0 if k > 0: if self.enable_code_matrix: y_cp = self.matrix_codes[i][int(y_cp)] for l in range(int(k)): self.ensemble[i].partial_fit(np.asarray([X_cp]), np.asarray([y_cp]), self.classes) try: pred = self.ensemble[i].predict(np.asarray([X])) if pred is not None: add = 1 if (pred[0] == y_cp) else 0 error = self.adwin_ensemble[i].estimation self.adwin_ensemble[i].add_element(add) if self.adwin_ensemble[i].detected_change(): if self.adwin_ensemble[i].estimation > error: change_detected = True except ValueError: change_detected = False if change_detected: self.n_detected_changes += 1 max_threshold = 0.0 i_max = -1 for i in range(self.actual_n_estimators): if max_threshold < self.adwin_ensemble[i].estimation: max_threshold = self.adwin_ensemble[i].estimation i_max = i if i_max != -1: self.ensemble[i_max].reset() self.adwin_ensemble[i_max] = ADWIN(self.delta) return self
class AdaLearningNode(LearningNodeNBAdaptive, AdaNode): """ Learning node for Hoeffding Adaptive Tree that uses Adaptive Naive Bayes models. Parameters ---------- initial_class_observations: dict (class_value, weight) or None Initial class observations """ def __init__(self, initial_class_observations, random_state=None): super().__init__(initial_class_observations) self._estimation_error_weight = ADWIN() self.error_change = False self._random_state = check_random_state(random_state) # Override AdaNode def number_leaves(self): return 1 # Override AdaNode def get_error_estimation(self): return self._estimation_error_weight.estimation # Override AdaNode def get_error_width(self): return self._estimation_error_weight.width # Override AdaNode def is_null_error(self): return self._estimation_error_weight is None def kill_tree_children(self, hat): pass # Override AdaNode def learn_from_instance(self, X, y, weight, hat, parent, parent_branch): true_class = y if hat.bootstrap_sampling: # Perform bootstrap-sampling k = self._random_state.poisson(1.0) if k > 0: weight = weight * k class_prediction = get_max_value_key(self.get_class_votes(X, hat)) bl_correct = (true_class == class_prediction) if self._estimation_error_weight is None: self._estimation_error_weight = ADWIN() old_error = self.get_error_estimation() # Add element to Adwin add = 0.0 if bl_correct else 1.0 self._estimation_error_weight.add_element(add) # Detect change with Adwin self.error_change = self._estimation_error_weight.detected_change() if self.error_change and old_error > self.get_error_estimation(): self.error_change = False # Update statistics super().learn_from_instance(X, y, weight, hat) # call ActiveLearningNode weight_seen = self.get_weight_seen() if weight_seen - self.get_weight_seen_at_last_split_evaluation() >= hat.grace_period: hat._attempt_to_split(self, parent, parent_branch) self.set_weight_seen_at_last_split_evaluation(weight_seen) # Override LearningNodeNBAdaptive def get_class_votes(self, X, ht): # dist = {} prediction_option = ht.leaf_prediction # MC if prediction_option == ht._MAJORITY_CLASS: dist = self.get_observed_class_distribution() # NB elif prediction_option == ht._NAIVE_BAYES: dist = do_naive_bayes_prediction(X, self._observed_class_distribution, self._attribute_observers) # NBAdaptive (default) else: if self._mc_correct_weight > self._nb_correct_weight: dist = self.get_observed_class_distribution() else: dist = do_naive_bayes_prediction(X, self._observed_class_distribution, self._attribute_observers) dist_sum = sum(dist.values()) # sum all values in dictionary normalization_factor = dist_sum * self.get_error_estimation() * self.get_error_estimation() if normalization_factor > 0.0: dist = normalize_values_in_dict(dist, normalization_factor, inplace=False) return dist # Override AdaNode, New for option votes def filter_instance_to_leaves(self, X, y, weight, parent, parent_branch, update_splitter_counts, found_nodes=None): if found_nodes is None: found_nodes = [] found_nodes.append(FoundNode(self, parent, parent_branch))
def __partial_fit(self, X, y): if self.init_matrix_codes and self.enable_code_matrix: self.__init_output_codes() change_detected = False for i in range(self.actual_n_estimators): # leveraging_bag - Leveraging Bagging if self.leverage_algorithm == self._LEVERAGE_ALGORITHMS[0]: k = self._random_state.poisson(self.w) # leveraging_bag_me - Missclassification Error elif self.leverage_algorithm == self._LEVERAGE_ALGORITHMS[1]: error = self.adwin_ensemble[i].estimation pred = self.ensemble[i].predict(np.asarray([X])) if pred is None: k = 1.0 elif pred[0] != y: k = 1.0 elif (error != 1.0 and self._random_state.rand() < (error / (1.0 - error))): k = 1.0 else: k = 0.0 # leveraging_bag_half - Resampling without replacement for # half of the instances elif self.leverage_algorithm == self._LEVERAGE_ALGORITHMS[2]: w = 1.0 k = 0.0 if (self._random_state.randint(2) == 1) else w # leveraging_bag_wt - Without taking out all instances elif self.leverage_algorithm == self._LEVERAGE_ALGORITHMS[3]: w = 1.0 k = 1.0 + self._random_state.poisson(w) # leveraging_subag - Resampling without replacement elif self.leverage_algorithm == self._LEVERAGE_ALGORITHMS[4]: w = 1.0 k = self._random_state.poisson(1) k = w if k > 0 else 0 else: raise RuntimeError("Invalid option for leverage_algorithm: '{}'\n" "Valid options are: {}".format(self.leverage_algorithm, self._LEVERAGE_ALGORITHMS)) y_coded = cp.deepcopy(y) if k > 0: classes = self.classes if self.enable_code_matrix: y_coded = self.matrix_codes[i][int(y)] classes = [0, 1] for _ in range(int(k)): self.ensemble[i].partial_fit(X=np.asarray([X]), y=np.asarray([y_coded]), classes=classes) pred = self.ensemble[i].predict(np.asarray([X])) if pred is not None: add = 0 if (pred[0] == y_coded) else 1 error = self.adwin_ensemble[i].estimation self.adwin_ensemble[i].add_element(add) if self.adwin_ensemble[i].detected_change(): if self.adwin_ensemble[i].estimation > error: change_detected = True if change_detected: self.n_detected_changes += 1 max_threshold = 0.0 i_max = -1 for i in range(self.actual_n_estimators): if max_threshold < self.adwin_ensemble[i].estimation: max_threshold = self.adwin_ensemble[i].estimation i_max = i if i_max != -1: self.ensemble[i_max].reset() self.adwin_ensemble[i_max] = ADWIN(self.delta) return self
def partial_fit(self, X, y, classes=None, weight=None): """ Partially fits the model, based on the X and y matrix. Since it's an ensemble learner, if X and y matrix of more than one sample are passed, the algorithm will partial fit the model one sample at a time. Each sample is trained by each classifier a total of K times, where K is drawn by a Poisson(l) distribution. l is updated after every example using :math:`lambda_{sc}` if th estimator correctly classifies the example or :math:`lambda_{sw}` in the other case. Parameters ---------- X : numpy.ndarray of shape (n_samples, n_features) The features to train the model. y: numpy.ndarray of shape (n_samples) An array-like with the class labels of all samples in X. classes: numpy.ndarray, optional (default=None) Array with all possible/known class labels. This is an optional parameter, except for the first partial_fit call where it is compulsory. sample_weight: Array-like Instance weight. If not provided, uniform weights are assumed. Usage varies depending on the base estimator. Raises ------ ValueError: A ValueError is raised if the 'classes' parameter is not passed in the first partial_fit call, or if they are passed in further calls but differ from the initial classes list passed. Returns ------- self """ if self.classes is None: if classes is None: raise ValueError("The first partial_fit call should pass all the classes.") else: self.classes = classes if self.classes is not None and classes is not None: if set(self.classes) == set(classes): pass else: raise ValueError("The classes passed to the partial_fit function differ from those passed earlier.") self.__adjust_ensemble_size() r, _ = get_dimensions(X) for j in range(r): change_detected = False lam = 1 for i in range(self.actual_n_estimators): if y[j] == 1: self.lam_pos[i] += lam self.n_pos += 1 else: self.lam_neg[i] += lam self.n_neg += 1 lam_rus = 1 if self.algorithm == 1: if y[j] == 1: if self.n_neg != 0: lam_rus = lam * ((self.lam_pos[i] + self.lam_neg[i]) / (self.lam_pos[i] + self.lam_neg[i] * (self.sampling_rate * (self.n_pos / self.n_neg))) * (((self.sampling_rate + 1) * self.n_pos) / (self.n_pos + self.n_neg))) else: if self.n_pos != 0: lam_rus = lam * ((self.lam_pos[i] + self.lam_neg[i]) / (self.lam_pos[i] + self.lam_neg[i] * (self.n_neg / (self.n_pos * self.sampling_rate))) * (((self.sampling_rate + 1) * self.n_pos) / (self.n_pos + self.n_neg))) elif self.algorithm == 2: if y[j] == 1: lam_rus = ((lam * self.n_pos) / (self.n_pos + self.n_neg)) / \ (self.lam_pos[i] / (self.lam_pos[i] + self.lam_neg[i])) else: lam_rus = ((lam * self.sampling_rate * self.n_pos) / (self.n_pos + self.n_neg)) / \ (self.lam_neg[i] / (self.lam_pos[i] + self.lam_neg[i])) elif self.algorithm == 3: if y[j] == 1: lam_rus = lam else: lam_rus = lam / self.sampling_rate k = self._random_state.poisson(lam_rus) if k > 0: for b in range(k): self.ensemble[i].partial_fit([X[j]], [y[j]], classes, weight) if self.ensemble[i].predict([X[j]])[0] == y[j]: self.lam_sc[i] += lam self.epsilon[i] = (self.lam_sw[i]) / (self.lam_sc[i] + self.lam_sw[i]) if self.epsilon[i] != 1: lam = lam / (2 * (1 - self.epsilon[i])) else: self.lam_sw[i] += lam self.epsilon[i] = (self.lam_sw[i]) / (self.lam_sc[i] + self.lam_sw[i]) if self.epsilon[i] != 0: lam = lam / (2 * self.epsilon[i]) if self.drift_detection: try: pred = self.ensemble[i].predict(X) error_estimation = self.adwin_ensemble[i].estimation for k in range(r): if pred[k] is not None: self.adwin_ensemble[i].add_element(int(pred[k] == y[k])) if self.adwin_ensemble[i].detected_change(): if self.adwin_ensemble[i].estimation > error_estimation: change_detected = True except ValueError: change_detected = False pass if change_detected and self.drift_detection: max_threshold = 0.0 i_max = -1 for i in range(self.actual_n_estimators): if max_threshold < self.adwin_ensemble[i].estimation: max_threshold = self.adwin_ensemble[i].estimation i_max = i if i_max != -1: self.ensemble[i_max].reset() self.adwin_ensemble[i_max] = ADWIN() return self
today = date.today() result_dir='result/multiflow/'+today.strftime("%Y-%m-%d")+'/' if not os.path.exists(result_dir): os.makedirs(result_dir) file_name = "CMGMM-"+test_dataset+".log" DETECTOR=args.detector#"" nama_model = "CMGMM" if (prune_comp): nama_model = nama_model+"+ " else: nama_model = nama_model+" " if DETECTOR == "ADWIN": print ("adwin") nama_model = nama_model+DETECTOR detector = ADWIN() elif DETECTOR == "DDM": print ("DDM") nama_model = nama_model+DETECTOR detector = DDM() elif DETECTOR == "EDDM": print ("EDDM") nama_model = nama_model+DETECTOR detector = EDDM() elif DETECTOR == "HDDM_A": print ("HDDM_A") nama_model = nama_model+DETECTOR detector = HDDM_A() elif DETECTOR == "HDDM_W": print ("HDDM_W") nama_model = nama_model+DETECTOR