class Goowe(StreamModel):
#class Goowe(BaseEstimator):
""" GOOWE (Geometrically Optimum Online Weighted Ensemble), as it is
described in Bonab and Can (2017). Common notation in the code is
as follows:
K for maximum number of classifiers in the ensemble.
N for data instances.
A, d as they are, in the aforementioned paper.
Parameters
----------
n_max_components: int
Ensemble size limit. Maximum number of component classifiers.
chunk_size: int
The amount of instances necessary for ensemble to learn concepts from.
At each chunk_size many instances, some training is done.
window_size: int
Size of sliding window, which keeps record of the last k instances
that are encountered in the data stream.
"""
def __init__(self,
n_max_components: int = 10,
chunk_size: int = 500,
window_size: int = 100,
logging=True):
super().__init__()
self._num_of_max_classifiers = n_max_components
self._chunk_size = chunk_size
self._Logging = logging
self._num_of_current_classifiers = 0
self._num_of_processed_instances = 0
self._classifiers = np.empty((self._num_of_max_classifiers),
dtype=object)
self._weights = np.zeros((self._num_of_max_classifiers, ))
# What to save from current Data Chunk --> will be used for
# adjusting weights, pruning purposes and so on.
# Individual predictions of components, overall prediction of ensemble,
# and ground truth info.
self._chunk_comp_preds = FastBuffer(max_size=chunk_size)
self._chunk_ensm_preds = FastBuffer(max_size=chunk_size)
# chunk_data has instances in the chunk and their ground truth.
# To be initialized after receiving n_features, n_targets
self._chunk_data = None
# self._chunk_truths = FastBuffer(max_size=chunk_size)
# some external stuff that is about the data we are dealing with
# but useful for recording predictions
self._num_classes = None
self._target_values = None # Required to correctly train HTs
self._record = False # Boolean for keeping records to files
# TODO: Implement Sliding Window Continuous Evaluator.
# What to save at Sliding Window (last n instances) --> will be
# used for continuous evaluation.
# self._sliding_window_ensemble_preds =FastBuffer(max_size=window_size)
# self._sliding_window_truths = FastBuffer(max_size=window_size)
def prepare_post_analysis_req(self,
num_features,
num_targets,
num_classes,
target_values,
record=False):
# Need to get the dataset information but we do not want to
# take it as an argument to the classifier itself, nor we do want to
# ask it at each data instance. Hence we take dataset info from user
# explicitly to create _chunk_data entries.
#chunk_size = self._chunk_size
self._chunk_data = InstanceWindow(n_features=num_features,
n_targets=num_targets,
max_size=self._chunk_size)
#self._chunk_data = chunk_data
# num_targets shows how many columns you want to predict in the data.
# num classes is eqv to possible number of values that that column
# can have.
self._num_classes = num_classes
self._target_values = target_values
self._record = record
if (self._record):
# Create files that keeps record of:
# - weights at each chunk
# - individual component results for every instance
# - ground truths for every instance.
self._f_comp_preds = open("component_predictions.csv", "w+")
self._f_truths = open("ground_truths.csv", "w+")
self._f_weights = open("weights.csv", "w+")
self._f_comp_preds.write(str(self._chunk_size) + '\n')
self._f_comp_preds.close()
self._f_truths.close()
self._f_weights.close()
return
def _get_components_predictions_for_instance(self, inst):
""" For a given data instance, takes predictions of
individual components from the ensemble as a matrix.
Parameters
----------
inst: data instance for which votes of components are delivered.
Returns
----------
numpy.array
A 2-d numpy array where each row corresponds to predictions of
each classifier.
"""
preds = np.zeros((self._num_of_current_classifiers, self._num_classes))
# print(np.shape(preds))
for k in range(len(preds)):
kth_comp_pred = self._classifiers[k].predict_proba(inst)
# print(kth_comp_pred[0])
# print(preds)
# print("Component {}'s Prediction: {}".format(k, kth_comp_pred))
preds[k, :] = kth_comp_pred[0]
if (self._Logging):
print('Component Predictions:')
print(preds)
return preds
def _adjust_weights(self):
""" Weight adustment by solving linear least squares, as it is
described in Bonab and Can (2017).
"""
# Prepare variables for Weight Adjustment
# print('number of current classifiers: {}'.format(self._num_of_current_classifiers))
A = np.zeros(shape=(self._num_of_current_classifiers,
self._num_of_current_classifiers))
d = np.zeros(shape=(self._num_of_current_classifiers, ))
# Go over all the data chunk, calculate values of (S_i x S_j) for A.
# (S_i x O) for d.
y_all = self._chunk_data.get_targets_matrix().astype(int)
# print(y_all)
for i in range(len(y_all)):
class_index = y_all[i]
comp_preds = self._chunk_comp_preds.get_next_element()
#print("{} components predictions:".format(i))
#print(comp_preds)
A = A + comp_preds.dot(comp_preds.T)
d = d + comp_preds[0][class_index]
# A and d are filled. Now, the linear system Aw=d to be solved
# to get our desired weights. w is of size K.
# print("Solving Aw=d")
# print(A)
# print(d)
w = np.linalg.lstsq(A, d, rcond=None)[0]
# _weights has maximum size but what we found can be
# smaller. Therefore, need to put the values of w to global weights
if (self._num_of_current_classifiers < self._num_of_max_classifiers):
for i in range(len(w)):
self._weights[i] = w[i]
else: # If full size, there is no problem.
self._weights = w
# print("After solving Aw=d weights:")
# print(self._weights)
return
def _normalize_weights(self):
""" Normalizes the weights of the ensemble to (0, 1) range.
Performs (x_i - min(x)) / (max(x) - min(x)) on the nonzero elements
of the weight vector.
"""
min = np.amin(self._weights[:self._num_of_current_classifiers])
max = np.amax(self._weights[:self._num_of_current_classifiers])
if (min == max): # all weights are the same
for i in range(self._num_of_current_classifiers):
self._weights[i] = 1. / self._num_of_current_classifiers
else:
for i in range(self._num_of_current_classifiers):
self._weights[i] = (self._weights[i] - min) / (max - min)
return
def _normalize_weights_softmax(self):
""" Normalizes the weights of the ensemble to (0, 1) range.
Performs (x_i - min(x)) / (max(x) - min(x)) on the nonzero elements
of the weight vector.
"""
cur_weights = self._weights[:self._num_of_current_classifiers]
self._weights[:self._num_of_current_classifiers] = np.exp(
cur_weights) / sum(np.exp(cur_weights))
return
def _process_chunk(self):
""" A subroutine that runs at the end of each chunk, allowing
the components to be trained and ensemble weights to be adjusted.
Until the first _process_chunk call, the ensemble is not yet ready.
At first call, the first component is learned.
At the rest of the calls, new components are formed, and the older ones
are trained by the given chunk.
If the ensemble size is reached, then the lowest weighted component is
removed from the ensemble.
"""
new_clf = HoeffdingTree() # with default parameters for now
new_clf.reset()
# Save records of previous chunk
if (self._record and self._num_of_current_classifiers > 0):
self._record_truths_this_chunk()
self._record_comp_preds_this_chunk()
self._record_weights_this_chunk()
# Case 1: No classifier in the ensemble yet, first chunk:
if (self._num_of_current_classifiers == 0):
self._classifiers[0] = new_clf
self._weights[0] = 1.0 # weight is 1 for the first clf
self._num_of_current_classifiers += 1
else:
# First, adjust the weights of the old component classifiers
# according to what happened in this chunk.
self._adjust_weights()
# Case 2: There are classifiers in the ensemble but
# the ensemble size is still not capped.
if (self._num_of_current_classifiers <
self._num_of_max_classifiers):
# Put the new classifier to ensemble with the weight of 1
self._classifiers[self._num_of_current_classifiers] = new_clf
self._weights[self._num_of_current_classifiers] = float(1.0)
self._num_of_current_classifiers += 1
# Case 3: Ensemble size is capped. Need to replace the component
# with lowest weight.
else:
assert (self._num_of_current_classifiers ==
self._num_of_max_classifiers), "Ensemble not full."
index_of_lowest_weight = np.argmin(self._weights)
self._classifiers[index_of_lowest_weight] = new_clf
self._weights[index_of_lowest_weight] = 1.0
# Normalizing weigths to simplify numbers
self._normalize_weights_softmax() # maybe useful. we'll see.
if (self._Logging):
print("After normalization weights: ")
print(self._weights)
# Ensemble maintenance is done. Now train all classifiers
# in the ensemble from the current chunk.
# Can be parallelized.
data_features = self._chunk_data.get_attributes_matrix()
data_truths = self._chunk_data.get_targets_matrix()
data_truths = data_truths.astype(int).flatten()
if (self._Logging):
print("Starting training the components with the current chunk...")
for k in range(self._num_of_current_classifiers):
print("Training classifier {}".format(k))
self._classifiers[k].partial_fit(data_features,
data_truths,
classes=self._target_values)
print(
"Training the components with the current chunk completed...")
else:
for k in range(self._num_of_current_classifiers):
self._classifiers[k].partial_fit(data_features,
data_truths,
classes=self._target_values)
return
def _record_truths_this_chunk(self):
f = open("ground_truths.csv", "ab")
data_truths = self._chunk_data.get_targets_matrix()
data_truths = data_truths.astype(int).flatten()
# Default behaviour is to store list of lists for savetxt.
# Hence, to prevent newline after each element of list, we surround
# the truth array with one more set of bracketts.
np.savetxt(f, [data_truths], delimiter=",", fmt='%d')
f.close()
return
def _record_comp_preds_this_chunk(self):
f = open("component_predictions.csv", "a+")
np.savetxt(f, [self._num_of_current_classifiers], fmt='%d')
comp_preds = np.array(self._chunk_comp_preds.get_queue())
for i in range(len(comp_preds)):
np.savetxt(f, comp_preds[i], delimiter=',', fmt='%1.5f')
f.close()
return
def _record_weights_this_chunk(self):
f = open("weights.csv", "a+")
np.savetxt(f, [self._num_of_current_classifiers], fmt='%d')
weights = self._weights
np.savetxt(f, [weights], delimiter=',', fmt='%1.5f')
f.close()
return
# --------------------------------------------------
# Overridden methods from the parent (StreamModel)
# --------------------------------------------------
def fit(self, X, y, classes=None, weight=None):
raise NotImplementedError("For now, only the stream version "
"is implemented. Use partial_fit()")
def partial_fit(self, X, y, classes=None, weight=None):
# This method should work with individual instances, as well as bunch
# of instances, since there can be pre-training for warm start.
# If an individual instance is inputted, then just save X and y to
# train from them later.
if (len(X) == 1):
# Save X and y to train classifiers later
# y is required to be 1x1, and hence the square bracketts.
y_i = np.array([y])
# print(type(X))
# print(type(y_i))
# print(X)
# print(y_i)
self._chunk_data.add_element(X, y_i)
# If still filling the chunk, then just add the instance to the
# current data chunk, wait for it to be filled.
self._num_of_processed_instances += 1
# If at the end of a chunk, start training components
# and adjusting weights using information in this chunk.
if (self._num_of_processed_instances % self._chunk_size == 0):
print("Instance {}".format(self._num_of_processed_instances))
self._process_chunk()
elif (len(X) > 1):
# Input is a chunk. Add them individually.
for i in range(len(X)):
X_i = np.array([X[i]])
y_i = np.array([[y[i]]])
# print(X_i)
# print(y_i)
self._chunk_data.add_element(X_i, y_i)
self._num_of_processed_instances += 1
# If at the end of a chunk, start training components
# and adjusting weights using information in this chunk.
if (self._num_of_processed_instances % self._chunk_size == 0):
print("Instance {}".format(
self._num_of_processed_instances))
self._process_chunk()
else:
print("Something wrong with the data...")
print("len(X) is: {}".format(len(X)))
return
def predict(self, X):
""" For a given data instance, yields the prediction values.
Parameters
----------
X: numpy.ndarray of shape (n_samples, n_features)
Samples for which we want to predict the labels.
Returns
-------
numpy.array
Predicted labels for all instances in X.
"""
predictions = []
if (len(X) == 1):
predictions.append(np.argmax(self.predict_proba(X)))
elif (len(X) > 1):
# Add many predictions
for i in range(len(X)):
relevance_scores = self.predict_proba(X[i])
predictions.append(np.argmax(relevance_scores))
# print(np.argmax(relevance_scores))
if (self._Logging):
print('Ensemble Prediction:')
print(np.array(predictions))
return np.array(predictions) #, one_hot
def predict_proba(self, X):
""" For a given data instance, takes WEIGHTED combination
of components to get relevance scores for each class.
Parameters
----------
X: data instance for which weighted combination is delivered.
Returns
----------
numpy.array
A vector with number_of_classes elements where each element
represents class score of corresponding class for this instance.
"""
weights = np.array(self._weights)
# get only the useful weights
weights = weights[:self._num_of_current_classifiers]
components_preds = self._get_components_predictions_for_instance(X)
#print('*****************************')
#print(components_preds)
#print('*****************************')
# Save individual component predictions and ensemble prediction
# for later analysis.
self._chunk_comp_preds.add_element([components_preds])
#print(weights)
#print(components_preds)
#print(self.get_classifiers())
weighted_ensemble_vote = np.dot(weights, components_preds)
# print("Weighted Ensemble vote: {}".format(weighted_ensemble_vote))
self._chunk_ensm_preds.add_element(weighted_ensemble_vote)
return weighted_ensemble_vote
def reset(self):
pass
def score(self, X, y):
pass
def get_info(self):
return 'The Ensemble GOOWE (Bonab and Can, 2017) with' + \
' - n_max_components: ' + str(self._num_of_max_classifiers) + \
' - num_of_current_components: ' + str(self._num_of_current_classifiers) + \
' - chunk_size: ' + str(self._chunk_size) + \
' - num_dimensions_in_label_space(num_classes): ' + str(self._num_classes) + \
' - recording: ' + str(self._record)
def get_class_type(self):
pass
# Some getters and setters..
def get_number_of_current_classifiers(self):
return self._num_of_current_classifiers
def get_number_of_max_classifiers(self):
return self._num_of_max_classifiers
# Helper methods for GooweMS
def get_classifiers(self):
return self._classifiers
def set_classifiers(self, classifiers):
self._classifiers = classifiers
def get_weights(self):
return self._weights
class KNNAdwin(KNN):
""" K-Nearest Neighbors Classifier with ADWIN Change detector
This Classifier is an improvement from the regular KNN classifier,
as it is resistant to concept drift. It utilises the ADWIN change
detector to decide which samples to keep and which ones to forget,
and by doing so it regulates the sample window size.
To know more about the ADWIN change detector, please visit
skmultiflow.classification.core.drift_detection.adwin
It uses the regular KNN Classifier as a base class, with the
major difference that this class keeps a variable size window,
instead of a fixed size one and also it updates the adwin algorithm
at each partial_fit call.
Parameters
----------
n_neighbors: int
The number of nearest neighbors to search for.
max_window_size: int
The maximum size of the window storing the last viewed samples.
leaf_size: int
The maximum number of samples that can be stored in one leaf node,
which determines from which point the algorithm will switch for a
brute-force approach. The bigger this number the faster the tree
construction time, but the slower the query time will be.
categorical_list: An array-like
Each entry is the index of a categorical feature. May be requested
further filtering.
Raises
------
NotImplementedError: A few of the functions described here are not
implemented since they have no application in this context.
ValueError: A ValueError is raised if the predict function is called
before at least k samples have been analyzed by the algorithm.
Examples
--------
>>> # Imports
>>> from skmultiflow.lazy.knn_adwin import KNNAdwin
>>> from skmultiflow.data.file_stream import FileStream
>>> # Setting up the stream
>>> stream = FileStream('skmultiflow/data/datasets/covtype.csv')
>>> stream.prepare_for_use()
>>> # Setting up the KNNAdwin classifier
>>> knn_adwin = KNNAdwin(n_neighbors=8, leaf_size=40, max_window_size=2000)
>>> # Pre training the classifier with 200 samples
>>> X, y = stream.next_sample(200)
>>> knn_adwin = knn_adwin.partial_fit(X, y)
>>> # Keeping track of sample count and correct prediction count
>>> n_samples = 0
>>> corrects = 0
>>> while n_samples < 5000:
... X, y = stream.next_sample()
... pred = knn_adwin.predict(X)
... if y[0] == pred[0]:
... corrects += 1
... knn_adwin = knn_adwin.partial_fit(X, y)
... n_samples += 1
>>>
>>> # Displaying the results
>>> print('KNN usage example')
>>> print(str(n_samples) + ' samples analyzed.')
5000 samples analyzed.
>>> print("KNNAdwin's performance: " + str(corrects/n_samples))
KNNAdwin's performance: 0.7798
"""
def __init__(self, n_neighbors=5, max_window_size=sys.maxsize, leaf_size=30, categorical_list=None):
super().__init__(n_neighbors=n_neighbors, max_window_size=max_window_size, leaf_size=leaf_size,
categorical_list=categorical_list)
self.adwin = ADWIN()
self.window = None
def reset(self):
""" reset
Resets the adwin algorithm as well as the base model
kept by the KNN base class.
Returns
-------
KNNAdwin
self
"""
self.adwin = ADWIN()
return super().reset()
def fit(self, X, y, classes=None, weights=None):
self.partial_fit(X, y, classes, weights)
return self
def partial_fit(self, X, y, classes=None, weight=None):
""" partial_fit
Partially fits the model. This is done by updating the window
with new samples while also updating the adwin algorithm. Then
we verify if a change was detected, and if so, the window is
correctly split at the drift moment.
Parameters
----------
X: Numpy.ndarray of shape (n_samples, n_features)
The data upon which the algorithm will create its model.
y: Array-like
An array-like containing the classification targets for all
samples in X.
classes: Not used.
weight: Not used.
Returns
-------
KNNAdwin
self
"""
r, c = get_dimensions(X)
if self.window is None:
self.window = InstanceWindow(max_size=self.max_window_size)
for i in range(r):
self.window.add_element(np.asarray([X[i]]), np.asarray([[y[i]]]))
if self.window.n_samples >= self.n_neighbors:
add = 1 if self.predict(np.asarray([X[i]])) == y[i] else 0
self.adwin.add_element(add)
else:
self.adwin.add_element(0)
if self.window.n_samples >= self.n_neighbors:
changed = self.adwin.detected_change()
if changed:
if self.adwin.width < self.window.n_samples:
for i in range(self.window.n_samples, self.adwin.width, -1):
self.window.delete_element()
return self
def get_info(self):
info = '{}:'.format(type(self).__name__)
info += ' - n_neighbors: {}'.format(self.n_neighbors)
info += ' - max_window_size: {}'.format(self.max_window_size)
info += ' - leaf_size: {}'.format(self.leaf_size)
return info
class TimeSeriesRegressor(BaseSKMObject, RegressorMixin):
def __init__(self, estimator: RegressorMixin, max_window_size=100):
super().__init__()
if not isinstance(estimator, RegressorMixin):
raise ValueError(
"estimator must be a Regressor, "
"Call TimeSeriesRegressor with an instance of RegressorMixin")
self.max_window_size = max_window_size
self.estimator = estimator
self.window = InstanceWindow(max_size=max_window_size, dtype=float)
self.first_fit = True
def partial_fit(self, X, y=None, sample_weight=None):
""" Partially fits the model on the samples X and corresponding targets y.
Parameters
----------
X: Numpy.ndarray of shape (n_samples, n_features)
The data upon which the algorithm will create its model.
If y is not provided the value X[t+1] in X will be used as target For X[t]
y: numpy.ndarray, optional
An array-like containing the targets for all samples in X.
y must have the shape as X
sample_weight: Not used.
Returns
-------
TimeSeriesRegressor
self
Notes
-----
For the TimeSeries Classifier, fitting the model is the
equivalent of inserting the newer samples in the observed window,
and if the max_window_size is reached, removing older results and then using
the the max_window_size past X values to predict future X values by feeding
them as features to the provided model.
To store the viewed samples we use a InstanceWindow object. For this
class' documentation please visit skmultiflow.core.utils.data_structures
"""
if len(X.shape) == 1:
X = X.reshape(-1, 1)
if len(y.shape) == 1:
y = y.reshape(-1, 1)
r = X.shape[0]
if y is not None:
r_t = y.shape[0]
if r != r_t:
raise ValueError(
"Batch size of X is different from the number of attributes in y "
"Batch size of must be the same for X and y")
if self.first_fit:
if r <= self.max_window_size:
raise ValueError(
"Number of elments of First call to partial_fit less than max_window_size "
"Call partial_fit with more than {} elements".format(
self.max_window_size))
for i in range(r):
if y is not None:
self.window.add_element(np.asarray([X[i]]), np.asarray([y[i]]))
elif i > 0:
self.window.add_element(np.asarray([X[i - 1]]),
np.asarray([X[i]]))
if self.max_window_size == self.window.n_samples:
self.estimator.partial_fit(
self.window.get_attributes_matrix().reshape((1, -1)),
self.window.get_targets_matrix()[-1].reshape((1, -1)),
sample_weight=sample_weight)
self.first_fit = False
return self
def reset(self):
self.window.reset()
self.estimator.reset()
return self
def clone_window(self):
window = InstanceWindow(
n_features=self.window.n_attributes,
n_targets=self.window.n_targets,
categorical_list=self.window.categorical_attributes,
max_size=self.window.max_size)
window._buffer = np.array(self.window._buffer)
window._n_samples = self.window._n_samples
return window
def predict(self, X):
""" Predicts the next value For all values in X.
The estimator consider X[0] as the value conming exactly after the last partially fit value.
Parameters
----------
X: Numpy.ndarray of shape (n_samples, n_features)
All the samples we want to predict the label for.
Returns
-------
list
A list containing the predicted values for all instances in X.
"""
if len(X.shape) == 1:
X = X.reshape(-1, 1)
r = X.shape[0]
window = self.clone_window()
predictions = []
for i in range(r):
window.add_element(np.asarray([X[i]]), np.asarray([X[i]]))
if self.max_window_size == self.window.n_samples:
pred = self.estimator.predict(
window.get_attributes_matrix().reshape((1, -1)))
if (len(pred.flatten()) == 1):
pred = pred[0]
predictions.append(pred[0])
return np.array(predictions)
def predict_proba(self, X):
"""
Method not implemented for this Estimator
"""
raise NotImplementedError
def forcast(self, X, n_steps):
""" Predicts the next n_steps values coming after all values in X.
The estimator consider X[0] as the value conming exactly after the last partially fit value.
Parameters
----------
X: Numpy.ndarray of shape (n_samples, n_features)
All the samples we want to predict the next value for.
n_steps:
The number of values to Forcast
Returns
-------
list
A list containing the predicted n_steps to come after values in X.
"""
if len(X.shape) == 1:
X = X.reshape(-1, 1)
r = X.shape[0]
window = self.clone_window()
for i in range(r):
window.add_element(np.asarray([X[i]]), np.asarray([X[i]]))
forecasts = []
for i in range(n_steps):
next_element = self.estimator.predict(
window.get_attributes_matrix().reshape((1, -1)))
window.add_element(next_element.reshape((1, -1)),
next_element.reshape((1, -1)))
if (len(next_element.flatten()) == 1):
next_element = next_element[0]
forecasts.append(next_element[0])
return np.asarray(forecasts)
class KNNClassifier(BaseSKMObject, ClassifierMixin):
""" K-Nearest Neighbors classifier.
This is a non-parametric classification method. The output of this
algorithm are the n_neighbors closest training examples to the query sample
X.
It works by keeping track of a fixed number of training samples, in
our case it keeps track of the last max_window_size training samples.
Then, whenever a query request is executed, the algorithm will search
its stored samples and find the closest ones using a selected distance
metric.
To store the samples, while reducing search times, we use a structure
called KD Tree (a K Dimensional Tree, for n_neighbors dimensional problems).
Although we do have our own KDTree implementation, which accepts
custom metrics, we recommend using the standard scikit-learn KDTree,
that even though doesn't accept custom metrics, is optimized and will
function faster.
Parameters
----------
n_neighbors: int (default=5)
The number of nearest neighbors to search for.
max_window_size: int (default=1000)
The maximum size of the window storing the last viewed samples.
leaf_size: int (default=30)
The maximum number of samples that can be stored in one leaf node,
which determines from which point the algorithm will switch for a
brute-force approach. The bigger this number the faster the tree
construction time, but the slower the query time will be.
nominal_attributes: numpy.ndarray (optional, default=None)
List of Nominal attributes. If empty, then assume that all attributes are numerical.
Raises
------
NotImplementedError: A few of the functions described here are not
implemented since they have no application in this context.
ValueError: A ValueError is raised if the predict function is called
before at least n_neighbors samples have been analyzed by the algorithm.
Notes
-----
For a KDTree functionality explanation, please see our KDTree
documentation, under skmultiflow.lazy.neighbors.kdtree.
This classifier is not optimal for a mixture of categorical and
numerical features.
If you wish to use our KDTree implementation please refer to this class'
function __predict_proba
Examples
--------
>>> # Imports
>>> from skmultiflow.lazy import KNNClassifier
>>> from skmultiflow.data import SEAGenerator
>>> # Setting up the stream
>>> stream = SEAGenerator(random_state=1, noise_percentage=.1)
>>> stream.prepare_for_use()
>>> # Pre training the classifier with 200 samples
>>> X, y = stream.next_sample(200)
>>> knn = KNNClassifier(n_neighbors=8, max_window_size=2000, leaf_size=40)
>>> knn.partial_fit(X, y)
>>> # Preparing the processing of 5000 samples and correct prediction count
>>> n_samples = 0
>>> corrects = 0
>>> while n_samples < 5000:
... X, y = stream.next_sample()
... my_pred = knn.predict(X)
... if y[0] == my_pred[0]:
... corrects += 1
... knn = knn.partial_fit(X, y)
... n_samples += 1
>>>
>>> # Displaying results
>>> print('KNNClassifier usage example')
>>> print('{} samples analyzed.'.format(n_samples))
5000 samples analyzed.
>>> print("KNNClassifier's performance: {}".format(corrects/n_samples))
KNN's performance: 0.8788
"""
def __init__(self,
n_neighbors=5,
max_window_size=1000,
leaf_size=30,
nominal_attributes=None):
super().__init__()
self.n_neighbors = n_neighbors
self.max_window_size = max_window_size
self.c = 0
self.window = InstanceWindow(max_size=max_window_size, dtype=float)
self.first_fit = True
self.classes = []
self.leaf_size = leaf_size
self.nominal_attributes = nominal_attributes
if self.nominal_attributes is None:
self._nominal_attributes = []
def partial_fit(self, X, y, classes=None, sample_weight=None):
""" Partially fits the model on the samples X and corresponding targets y.
Parameters
----------
X: Numpy.ndarray of shape (n_samples, n_features)
The data upon which the algorithm will create its model.
y: Array-like
An array-like containing the classification targets for all
samples in X.
classes: numpy.ndarray, optional (default=None)
Array with all possible/known classes.
sample_weight: Not used.
Returns
-------
KNNClassifier
self
Notes
-----
For the K-Nearest Neighbors Classifier, fitting the model is the
equivalent of inserting the newer samples in the observed window,
and if the size_limit is reached, removing older results. To store
the viewed samples we use a InstanceWindow object. For this class'
documentation please visit skmultiflow.core.utils.data_structures
"""
r, c = get_dimensions(X)
if classes is not None:
self.classes = list(set().union(self.classes, classes))
for i in range(r):
self.window.add_element(np.asarray([X[i]]), np.asarray([[y[i]]]))
return self
def reset(self):
self.window.reset()
return self
def predict(self, X):
""" predict
Predicts the label of the X sample, by searching the KDTree for
the n_neighbors-Nearest Neighbors.
Parameters
----------
X: Numpy.ndarray of shape (n_samples, n_features)
All the samples we want to predict the label for.
Returns
-------
list
A list containing the predicted labels for all instances in X.
"""
r, c = get_dimensions(X)
proba = self.predict_proba(X)
predictions = []
for i in range(r):
predictions.append(np.argmax(proba[i]))
return np.array(predictions)
def predict_proba(self, X):
""" predict_proba
Calculates the probability of each sample in X belonging to each
of the labels, based on the knn algorithm.
Parameters
----------
X: Numpy.ndarray of shape (n_samples, n_features)
Raises
------
ValueError: If there is an attempt to call this function before,
at least, n_neighbors samples have been analyzed by the learner, a ValueError
is raised.
Returns
-------
numpy.ndarray
An array of shape (n_samples, n_features), in which each outer entry is
associated with the X entry of the same index. And where the list in
index [i] contains len(self.target_value) elements, each of which represents
the probability that the i-th sample of X belongs to a certain label.
"""
if self.window is None or self.window.n_samples < self.n_neighbors:
raise ValueError(
"KNNClassifier must be (partially) fitted on n_neighbors samples before doing any prediction."
)
proba = []
r, c = get_dimensions(X)
self.classes = list(set().union(
self.classes, np.unique(self.window.get_targets_matrix())))
new_dist, new_ind = self.__predict_proba(X)
for i in range(r):
votes = [0.0 for _ in range(int(max(self.classes) + 1))]
for index in new_ind[i]:
votes[int(
self.window.get_targets_matrix()[index])] += 1. / len(
new_ind[i])
proba.append(votes)
return np.array(proba)
def __predict_proba(self, X):
""" __predict_proba
Private implementation of the predict_proba method.
Parameters
----------
X: Numpy.ndarray of shape (n_samples, n_features)
Returns
-------
tuple list
One list with the k-nearest neighbor's distances and another
one with their indexes.
"""
# To use our own KDTree implementation please replace it as follows
# tree = KDTree(self.window.get_attributes_matrix(), metric='euclidean',
# nominal_attributes=self._nominal_attributes, return_distance=True)
tree = sk.KDTree(self.window.get_attributes_matrix(),
self.leaf_size,
metric='euclidean')
dist, ind = tree.query(np.asarray(X), k=self.n_neighbors)
return dist, ind
class KNNClassifier(BaseSKMObject, ClassifierMixin):
""" k-Nearest Neighbors classifier.
This non-parametric classification method keeps a data window with the last max_window_size
training samples. The predicted class-label for a given query sample is obtained in two steps:
first, find the closest n_neighbors to the query sample in the data window. Second, aggregate
the class-labels of the n_neighbors to define the predicted class for the query sample.
Parameters
----------
n_neighbors: int (default=5)
The number of nearest neighbors to search for.
max_window_size: int (default=1000)
The maximum size of the window storing the last viewed samples.
leaf_size: int (default=30)
The maximum number of samples that can be stored in one leaf node,
which determines from which point the algorithm will switch for a
brute-force approach. The bigger this number the faster the tree
construction time, but the slower the query time will be.
nominal_attributes: numpy.ndarray (optional, default=None)
List of Nominal attributes. If empty, then assume that all attributes are numerical.
Notes
-----
This classifier is not optimal for a mixture of categorical and numerical features.
Examples
--------
>>> # Imports
>>> from skmultiflow.lazy import KNNClassifier
>>> from skmultiflow.data import SEAGenerator
>>> # Setting up the stream
>>> stream = SEAGenerator(random_state=1, noise_percentage=.1)
>>> # Pre training the classifier with 200 samples
>>> X, y = stream.next_sample(200)
>>> knn = KNNClassifier(n_neighbors=8, max_window_size=2000, leaf_size=40)
>>> knn.partial_fit(X, y)
>>> # Preparing the processing of 5000 samples and correct prediction count
>>> n_samples = 0
>>> corrects = 0
>>> while n_samples < 5000:
... X, y = stream.next_sample()
... my_pred = knn.predict(X)
... if y[0] == my_pred[0]:
... corrects += 1
... knn = knn.partial_fit(X, y)
... n_samples += 1
>>>
>>> # Displaying results
>>> print('KNNClassifier usage example')
>>> print('{} samples analyzed.'.format(n_samples))
5000 samples analyzed.
>>> print("KNNClassifier's performance: {}".format(corrects/n_samples))
KNN's performance: 0.8788
"""
def __init__(self,
n_neighbors=5,
max_window_size=1000,
leaf_size=30,
nominal_attributes=None):
super().__init__()
self.n_neighbors = n_neighbors
self.max_window_size = max_window_size
self.c = 0
self.window = InstanceWindow(max_size=max_window_size, dtype=float)
self.first_fit = True
self.classes = []
self.leaf_size = leaf_size
self.nominal_attributes = nominal_attributes
if self.nominal_attributes is None:
self._nominal_attributes = []
def partial_fit(self, X, y, classes=None, sample_weight=None):
""" Partially fits the model on the samples X and corresponding targets y.
Parameters
----------
X: Numpy.ndarray of shape (n_samples, n_features)
The data upon which the algorithm will create its model.
y: Array-like
An array-like containing the classification targets for all
samples in X.
classes: numpy.ndarray, optional (default=None)
Array with all possible/known classes.
sample_weight: Not used.
Returns
-------
KNNClassifier
self
Notes
-----
For the K-Nearest Neighbors Classifier, fitting the model is the
equivalent of inserting the newer samples in the observed window,
and if the size_limit is reached, removing older results. To store
the viewed samples we use a InstanceWindow object. For this class'
documentation please visit skmultiflow.core.utils.data_structures
"""
r, c = get_dimensions(X)
if classes is not None:
self.classes = list(set().union(self.classes, classes))
for i in range(r):
self.window.add_element(np.asarray([X[i]]), np.asarray([[y[i]]]))
return self
def reset(self):
self.window.reset()
return self
def predict(self, X):
""" Predicts the class label of the X sample.
Parameters
----------
X: Numpy.ndarray of shape (n_samples, n_features)
All the samples we want to predict the label for.
Returns
-------
list
A list containing the predicted labels for all instances in X.
"""
r, c = get_dimensions(X)
proba = self.predict_proba(X)
predictions = []
for i in range(r):
predictions.append(np.argmax(proba[i]))
return np.array(predictions)
def predict_proba(self, X):
""" Estimates the probability of each sample in X belonging to each of the class-labels.
Parameters
----------
X: Numpy.ndarray of shape (n_samples, n_features)
Returns
-------
numpy.ndarray
An array of shape (n_samples, n_features), in which each outer entry is
associated with the X entry of the same index. And where the list in
index [i] contains len(self.target_value) elements, each of which represents
the probability that the i-th sample of X belongs to a certain label.
"""
r, c = get_dimensions(X)
if self.window is None or self.window.n_samples < self.n_neighbors:
# The model is empty, defaulting to zero
return np.zeros(shape=(r, 1))
proba = []
self.classes = list(set().union(
self.classes, np.unique(self.window.get_targets_matrix())))
new_dist, new_ind = self.__predict_proba(X)
for i in range(r):
votes = [0.0 for _ in range(int(max(self.classes) + 1))]
for index in new_ind[i]:
votes[int(
self.window.get_targets_matrix()[index])] += 1. / len(
new_ind[i])
proba.append(votes)
return np.asarray(proba)
def __predict_proba(self, X):
""" __predict_proba
Private implementation of the predict_proba method.
Parameters
----------
X: Numpy.ndarray of shape (n_samples, n_features)
Returns
-------
tuple list
One list with the k-nearest neighbor's distances and another
one with their indexes.
"""
# To use our own KDTree implementation please replace it as follows
# tree = KDTree(self.window.get_attributes_matrix(), metric='euclidean',
# nominal_attributes=self._nominal_attributes, return_distance=True)
tree = sk.KDTree(self.window.get_attributes_matrix(),
self.leaf_size,
metric='euclidean')
dist, ind = tree.query(np.asarray(X), k=self.n_neighbors)
return dist, ind