def test_invalid_get_labels(self):
loader = StubLoader()
schema = Schema({'v': Schema.LABEL})
ds = Dataset(loader, schema, static=False)
# get_labels returns generator; as generator will be evaluated
# when actually iterating over it, pass it to list().
self.assertRaises(RuntimeError, list, ds.get_labels())
def test_invalid_get_labels(self):
loader = StubLoader()
schema = Schema({'v': Schema.LABEL})
ds = Dataset(loader, schema, static=False)
# get_labels returns generator; as generator will be evaluated
# when actually iterating over it, pass it to list().
self.assertRaises(RuntimeError, list, ds.get_labels())
def test_simple(self):
loader = StubLoader()
schema = Schema({'v': Schema.LABEL})
ds = Dataset(loader, schema)
for (idx, (label, d)) in ds:
self.assertEqual(unicode_t(idx + 1), label)
self.assertEqual(0, len(d.string_values))
self.assertEqual(0, len(d.num_values))
self.assertEqual(0, len(d.binary_values))
self.assertEqual(['1', '2', '3'], list(ds.get_labels()))
def test_simple(self):
loader = StubLoader()
schema = Schema({'v': Schema.LABEL})
ds = Dataset(loader, schema)
for (idx, (label, d)) in ds:
self.assertEqual(unicode_t(idx+1), label)
self.assertEqual(0, len(d.string_values))
self.assertEqual(0, len(d.num_values))
self.assertEqual(0, len(d.binary_values))
self.assertEqual(['1','2','3'], list(ds.get_labels()))
def test_from_data(self):
# load from array format
ds = Dataset.from_data(
[[10, 20, 30], [20, 10, 50], [40, 10, 30]], # data
[0, 1, 0], # labels
['k1', 'k2', 'k3'], # feature_names
['pos', 'neg'], # label_names
)
expected_labels = ['pos', 'neg', 'pos']
expected_k1s = [10, 20, 40]
actual_labels = []
actual_k1s = []
for (idx, (label, d)) in ds:
actual_labels.append(label)
actual_k1s.append(dict(d.num_values)['k1'])
self.assertEqual(expected_labels, actual_labels)
self.assertEqual(expected_k1s, actual_k1s)
# load from scipy.sparse format
ds = Dataset.from_data(
self._create_matrix(), # data
[0, 1, 0], # labels
['k1', 'k2', 'k3'], # feature_names
['pos', 'neg'], # label_names
)
expected_labels = ['pos', 'neg', 'pos']
expected_k1s = [1, None, 4]
expected_k3s = [2, 3, 6]
actual_labels = []
actual_k1s = []
actual_k3s = []
for (idx, (label, d)) in ds:
actual_labels.append(label)
actual_k1s.append(dict(d.num_values).get('k1', None))
actual_k3s.append(dict(d.num_values).get('k3', None))
self.assertEqual(expected_labels, actual_labels)
self.assertEqual(expected_k1s, actual_k1s)
self.assertEqual(expected_k3s, actual_k3s)
def test_from_array(self):
ds = Dataset.from_array(
[ [10,20,30], [20,10,50], [40,10,30] ], # data
[ 0, 1, 0 ], # labels
['k1', 'k2', 'k3'], # feature_names
['pos', 'neg'], # label_names
)
expected_labels = ['pos', 'neg', 'pos']
expected_k1s = [10, 20, 40]
actual_labels = []
actual_k1s = []
for (idx, (label, d)) in ds:
actual_labels.append(label)
actual_k1s.append(dict(d.num_values)['k1'])
self.assertEqual(expected_labels, actual_labels)
self.assertEqual(expected_k1s, actual_k1s)
def test_from_array_without_label(self):
ds = Dataset.from_array(
[[10, 20, 30], [20, 10, 50], [40, 10, 30]], # data
None, # labels
['k1', 'k2', 'k3'], # feature_names
['pos', 'neg'], # label_names
)
expected_labels = [None, None, None]
expected_k1s = [10, 20, 40]
actual_labels = []
actual_k1s = []
for (idx, (label, d)) in ds:
actual_labels.append(label)
actual_k1s.append(dict(d.num_values)['k1'])
self.assertEqual(expected_labels, actual_labels)
self.assertEqual(expected_k1s, actual_k1s)
def test_from_matrix(self):
ds = Dataset.from_matrix(
self._create_matrix(), # data
[0, 1, 0], # labels
['k1', 'k2', 'k3'], # feature_names
['pos', 'neg'], # label_names
)
expected_labels = ['pos', 'neg', 'pos']
expected_k1s = [1, None, 4]
expected_k3s = [2, 3, 6]
actual_labels = []
actual_k1s = []
actual_k3s = []
for (idx, (label, d)) in ds:
actual_labels.append(label)
actual_k1s.append(dict(d.num_values).get('k1', None))
actual_k3s.append(dict(d.num_values).get('k3', None))
self.assertEqual(expected_labels, actual_labels)
self.assertEqual(expected_k1s, actual_k1s)
self.assertEqual(expected_k3s, actual_k3s)
def test_from_matrix(self):
ds = Dataset.from_matrix(
self._create_matrix(), # data
[ 0, 1, 0 ], # labels
[ 'k1', 'k2', 'k3'], # feature_names
[ 'pos', 'neg'], # label_names
)
expected_labels = ['pos', 'neg', 'pos']
expected_k1s = [1,None,4]
expected_k3s = [2,3,6]
actual_labels = []
actual_k1s = []
actual_k3s = []
for (idx, (label, d)) in ds:
actual_labels.append(label)
actual_k1s.append(dict(d.num_values).get('k1', None))
actual_k3s.append(dict(d.num_values).get('k3', None))
self.assertEqual(expected_labels, actual_labels)
self.assertEqual(expected_k1s, actual_k1s)
self.assertEqual(expected_k3s, actual_k3s)
le.fit(labels)
c = le.transform(y)
# scale dataset with (mean, variance) = (0, 1)
scaler = StandardScaler()
X = scaler.fit_transform(X)
# calculate the domain
X_min = X.min(axis=0)
#X_min = np.ones(X.shape[1])
X_max = X.max(axis=0)
X0, X1 = np.meshgrid(np.linspace(X_min[0], X_max[0], meshsize),
np.linspace(X_min[1], X_max[1], meshsize))
# make training dataset
dataset = Dataset.from_array(X, y)
# make mesh dataset to plot decision surface
contourf_dataset = Dataset.from_array(np.c_[X0.ravel(), X1.ravel()])
# setup and run jubatus
config = Config(method=method,
parameter={'regularization_weight': regularization_weight})
classifier = Classifier.run(config, port=port)
# construct classifier prediction models and dump model weights
for i, _ in enumerate(classifier.train(dataset)):
model_name = 'decision_surface_{}'.format(i)
classifier.save(name=model_name)
# prepare figure
fig, ax = plt.subplots()
def test_get_labels(self):
loader = StubLoader()
schema = Schema({'v': Schema.LABEL})
ds = Dataset(loader, schema)
self.assertEqual(['1', '2', '3'], list(ds.get_labels()))
def test_predict(self):
loader = StubLoader()
dataset = Dataset(loader) # predict
self.assertEqual(['v', 1.0], dataset[0][1].num_values[0])
from jubakit.classifier import Classifier, Schema, Dataset, Config
from jubakit.loader.csv import CSVLoader
# Load the shogun dataset.
train_loader = CSVLoader('shogun.train.csv')
test_loader = CSVLoader('shogun.test.csv')
# Define a Schema that defines types for each columns of the CSV file.
schema = Schema({
'family_name': Schema.LABEL,
'first_name': Schema.STRING,
})
# Create a Dataset.
train_dataset = Dataset(train_loader, schema).shuffle()
test_dataset = Dataset(test_loader, schema)
# Create a Classifier Service.
cfg = Config(
method = 'PA',
converter = {
'string_rules': [{'key': 'first_name', 'type': 'unigram', 'sample_weight': 'bin', 'global_weight': 'bin'}]
}
)
classifier = Classifier.run(cfg)
# Train the classifier.
for _ in classifier.train(train_dataset): pass
# Classify using the classifier.
for (idx, label, result) in classifier.classify(test_dataset):
from sklearn.datasets import load_svmlight_files
import sklearn.metrics
import jubakit
from jubakit.classifier import Classifier, Dataset, Config
# Load LIBSVM files.
# Note that these example files are not included in this repository.
# You can fetch them from: https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass.html#news20
print("Loading LIBSVM files...")
(train_X, train_y, test_X,
test_y) = load_svmlight_files(['news20', 'news20.t'])
# Create a Train Dataset.
print("Creating train dataset...")
train_ds = Dataset.from_matrix(train_X, train_y)
# Create a Test Dataset
print("Creating test dataset...")
test_ds = Dataset.from_matrix(test_X, test_y)
# Create a Classifier Service
classifier = Classifier.run(Config())
# Train the classifier.
print("Training...")
for (idx, _) in classifier.train(train_ds):
if idx % 1000 == 0:
print("Training... ({0} %)".format(100 * idx / len(train_ds)))
# Test the classifier.
from sklearn.datasets import load_svmlight_files
import sklearn.metrics
import jubakit
from jubakit.classifier import Classifier, Dataset, Config
# Load LIBSVM files.
# Note that these example files are not included in this repository.
# You can fetch them from: https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass.html#news20
print("Loading LIBSVM files...")
(train_X, train_y, test_X, test_y) = load_svmlight_files(['news20', 'news20.t'])
# Create a Train Dataset.
print("Creating train dataset...")
train_ds = Dataset.from_matrix(train_X, train_y)
# Create a Test Dataset
print("Creating test dataset...")
test_ds = Dataset.from_matrix(test_X, test_y)
# Create a Classifier Service
classifier = Classifier.run(Config())
# Train the classifier.
print("Training...")
for (idx, _) in classifier.train(train_ds):
if idx % 1000 == 0:
print("Training... ({0} %)".format(100 * idx / len(train_ds)))
# Test the classifier.
# switch StratifiedKFold API
sklearn_version = int(sklearn.__version__.split('.')[1])
if sklearn_version < 18:
from sklearn.cross_validation import StratifiedKFold
else:
from sklearn.model_selection import StratifiedKFold
# Load built-in `iris` dataset from scikit-learn.
iris = sklearn.datasets.load_iris()
# Convert it into jubakit Dataset.
#dataset = Dataset.from_array(iris.data, iris.target)
# ... or, optionally you can assign feature/label names to improve human-readbility.
dataset = Dataset.from_array(iris.data, iris.target, iris.feature_names, iris.target_names)
# Shuffle the dataset, as the dataset is sorted by label.
dataset = dataset.shuffle()
# Create a Classifier Service.
# Classifier process starts using a default configuration.
classifier = Classifier.run(Config())
# Prepare arrays to keep true/predicted labels to display a report later.
true_labels = []
predicted_labels = []
# Run stratified K-fold validation.
labels = list(dataset.get_labels())
if sklearn_version < 18:
===================================================
In this example we show classification using Digits dataset.
"""
import sklearn.datasets
import sklearn.metrics
import jubakit
from jubakit.classifier import Classifier, Dataset, Config
# Load the digits dataset.
digits = sklearn.datasets.load_digits()
# Create a Dataset.
dataset = Dataset.from_array(digits.data, digits.target)
n_samples = len(dataset)
n_train_samples = int(n_samples / 2)
# Create a Classifier Service
cfg = Config(method='AROW', parameter={'regularization_weight': 0.1})
classifier = Classifier.run(cfg)
print("Started Service: {0}".format(classifier))
# Train the classifier using the first half of the dataset.
train_ds = dataset[:n_train_samples]
print("Training...: {0}".format(train_ds))
for _ in classifier.train(train_ds):
pass
n_redundant=2,
n_repeated=0,
n_classes=2,
n_clusters_per_class=2,
weights=None,
flip_y=0.01,
class_sep=1.0,
hypercube=True,
shift=0.0,
scale=1.0,
shuffle=True,
random_state=0, # fixed seed
)
# Convert arrays into jubakit Dataset.
dataset = Dataset.from_array(X, y)
# Try finding the best classifier parameter.
param2metrics = {}
for method in ['AROW', 'NHERD', 'CW']:
for rw in [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0]:
print('Running ({0} / regularization_weight = {1})...'.format(
method, rw))
# Create a config data structure.
jubatus_config = Config(method=method,
parameter={'regularization_weight': rw})
# It is equivalent to:
#jubatus_config = Config.default()
#jubatus_config['method'] = method
def test_get_labels(self):
loader = StubLoader()
schema = Schema({'v': Schema.LABEL})
ds = Dataset(loader, schema)
self.assertEqual(['1', '2', '3'], list(ds.get_labels()))
from jubakit.classifier import Classifier, Schema, Dataset, Config
from jubakit.loader.csv import CSVLoader
# Load the shogun dataset.
train_loader = CSVLoader('shogun.train.csv')
test_loader = CSVLoader('shogun.test.csv')
# Define a Schema that defines types for each columns of the CSV file.
schema = Schema({
'family_name': Schema.LABEL,
'first_name': Schema.STRING,
})
# Create a Dataset.
train_dataset = Dataset(train_loader, schema).shuffle()
test_dataset = Dataset(test_loader, schema)
# Create a Classifier Service.
cfg = Config(method='PA',
converter={
'string_rules': [{
'key': 'first_name',
'type': 'unigram',
'sample_weight': 'bin',
'global_weight': 'bin'
}]
})
classifier = Classifier.run(cfg)
# Train the classifier.
===================================================
In this example we show classification using Digits dataset.
"""
import sklearn.datasets
import sklearn.metrics
import jubakit
from jubakit.classifier import Classifier, Dataset, Config
# Load the digits dataset.
digits = sklearn.datasets.load_digits()
# Create a Dataset.
dataset = Dataset.from_array(digits.data, digits.target)
n_samples = len(dataset)
n_train_samples = int(n_samples / 2)
# Create a Classifier Service
cfg = Config(method='AROW', parameter={'regularization_weight': 0.1})
classifier = Classifier.run(cfg)
print("Started Service: {0}".format(classifier))
# Train the classifier using the first half of the dataset.
train_ds = dataset[:n_train_samples]
print("Training...: {0}".format(train_ds))
for _ in classifier.train(train_ds): pass
# Test the classifier using the last half of the dataset.
# what's going on in jubakit.
jubakit.logger.setup_logger(jubakit.logger.INFO)
# Load a CSV file.
loader = CSVLoader('iris.csv')
# Define a Schema that defines types for each columns of the CSV file.
schema = Schema({
'Species': Schema.LABEL,
}, Schema.NUMBER)
# Display Schema
print('Schema: {0}'.format(schema))
# Create a Dataset.
dataset = Dataset(loader, schema).shuffle()
n_samples = len(dataset)
n_train_samples = int(n_samples / 2)
# Create a Classifier configuration.
cfg = Config()
# Bulk train-test the classifier.
result = Classifier.train_and_classify(
cfg,
dataset[:n_train_samples],
dataset[n_train_samples:],
sklearn.metrics.classification_report
)
print('---- Classification Report -----------------------------------')
n_redundant=2,
n_repeated=0,
n_classes=2,
n_clusters_per_class=2,
weights=None,
flip_y=0.01,
class_sep=1.0,
hypercube=True,
shift=0.0,
scale=1.0,
shuffle=True,
random_state=0, # fixed seed
)
# Convert arrays into jubakit Dataset.
dataset = Dataset.from_array(X, y)
# Try finding the best classifier parameter.
param2metrics = {}
for method in ['AROW', 'NHERD', 'CW']:
for rw in [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0]:
print('Running ({0} / regularization_weight = {1})...'.format(method, rw))
# Create a config data structure.
jubatus_config = Config(method=method, parameter={'regularization_weight': rw})
# It is equivalent to:
#jubatus_config = Config.default()
#jubatus_config['method'] = method
#jubatus_config['parameter']['regularization_weight'] = rw
from jubakit.classifier import Classifier, Dataset, Config
# switch StratifiedKFold API
sklearn_version = int(sklearn.__version__.split('.')[1])
if sklearn_version < 18:
from sklearn.cross_validation import StratifiedKFold
else:
from sklearn.model_selection import StratifiedKFold
# Load built-in `iris` dataset from scikit-learn.
iris = sklearn.datasets.load_iris()
# Convert it into jubakit Dataset.
#dataset = Dataset.from_array(iris.data, iris.target)
# ... or, optionally you can assign feature/label names to improve human-readbility.
dataset = Dataset.from_array(iris.data, iris.target, iris.feature_names,
iris.target_names)
# Shuffle the dataset, as the dataset is sorted by label.
dataset = dataset.shuffle()
# Create a Classifier Service.
# Classifier process starts using a default configuration.
classifier = Classifier.run(Config())
# Prepare arrays to keep true/predicted labels to display a report later.
true_labels = []
predicted_labels = []
# Run stratified K-fold validation.
labels = list(dataset.get_labels())
if sklearn_version < 18:
'.user.lang': Schema.STRING,
'.user.description': Schema.STRING,
}, Schema.IGNORE)
# Create a Classifier Service.
classifier = Classifier.run(Config())
# Number of tweets used for training.
n_train = 1000
print('---- Train: {0} tweets -------------------------------------'.format(
n_train))
# Train the classifier using tweets from Twitter stream.
trained_labels = set()
dataset = Dataset(get_loader(), schema)
for (idx, label) in classifier.train(dataset):
if idx == n_train: break
trained_labels.add(label)
text_summary = dataset.get(idx)['.text'].replace('\n', '')
print('Train[{0}]: language {1} >> {2}'.format(idx, label, text_summary))
print('Languages Trained: {0}'.format(str(trained_labels)))
print('---- Prediction (Ctrl-C to stop) -------------------------------------')
try:
# Classify tweets using the classifier.
(y_true, y_pred) = ([], [])
dataset = Dataset(get_loader(), schema)