def jubatus_config(params): """ convert hyperopt config to jubatus config """ if params['classifier_type'] == 'LinearClassifier': config = Config(method=params['linear_method'], parameter={ 'regularization_weight': params['regularization_weight'] }) elif params['classifier_type'] == 'NearestNeighbor': config = Config(method='NN', parameter={ 'method': params['nn_method'], 'nearest_neighbor_num': int(params['nearest_neighbor_num']), 'local_sensitivity': params['local_sensitivity'], 'parameter': { 'hash_num': int(params['hash_num']) } }) else: raise NotImplementedError() return config
def test_method_param(self): self.assertTrue('parameter' not in Config(method='PA')) self.assertTrue('regularization_weight' in Config( method='PA1')['parameter']) self.assertTrue('nearest_neighbor_num' in Config( method='NN')['parameter']) self.assertTrue('nearest_neighbor_num' in Config( method='cosine')['parameter']) self.assertTrue('nearest_neighbor_num' in Config( method='euclidean')['parameter'])
'Species': Schema.LABEL, 'Sepal.Length': Schema.NUMBER, 'Sepal.Width': Schema.NUMBER, 'Petal.Length': Schema.NUMBER, 'Petal.Width': Schema.NUMBER, }) # Create a Dataset, which is an abstract representation of a set of data # that can be fed to Services like Classifier. `shuffle()` returns a new # Dataset whose order of data is shuffled. Note that datasets are immutable # objects. dataset = Dataset(loader, schema).shuffle() # Create a Classifier Service. # Classifier process starts using a default configuration. cfg = Config.default() classifier = Classifier.run(cfg) # You can also connect to an existing service instead. #classifier = Classifier('127.0.0.1', 9199) # Train the classifier with every data in the dataset. for (idx, label) in classifier.train(dataset): # You can peek the datum being trained. print("Train: {0}".format(dataset[idx])) # Save the trained model file. print("Saving model file...") classifier.save('example_snapshot') # Classify using the same dataset.
# Define a Schema that defines types for each columns of the CSV file. schema = Schema({ 'family_name': Schema.LABEL, 'first_name': Schema.STRING, }) # Create a Dataset. train_dataset = Dataset(train_loader, schema).shuffle() test_dataset = Dataset(test_loader, schema) # Create a Classifier Service. cfg = Config(method='PA', converter={ 'string_rules': [{ 'key': 'first_name', 'type': 'unigram', 'sample_weight': 'bin', 'global_weight': 'bin' }] }) classifier = Classifier.run(cfg) # Train the classifier. for _ in classifier.train(train_dataset): pass # Classify using the classifier. for (idx, label, result) in classifier.classify(test_dataset): true_family_name = label pred_family_name = result[0][0] first_name = test_dataset.get(idx)['first_name']
def setUp(self): self._service = Classifier.run(Config())
import sklearn.datasets import sklearn.metrics import jubakit from jubakit.classifier import Classifier, Dataset, Config # Load the digits dataset. digits = sklearn.datasets.load_digits() # Create a Dataset. dataset = Dataset.from_array(digits.data, digits.target) n_samples = len(dataset) n_train_samples = int(n_samples / 2) # Create a Classifier Service cfg = Config(method='AROW', parameter={'regularization_weight': 0.1}) classifier = Classifier.run(cfg) print("Started Service: {0}".format(classifier)) # Train the classifier using the first half of the dataset. train_ds = dataset[:n_train_samples] print("Training...: {0}".format(train_ds)) for _ in classifier.train(train_ds): pass # Test the classifier using the last half of the dataset. test_ds = dataset[n_train_samples:] print("Testing...: {0}".format(test_ds)) y_true = [] y_pred = []
def test_default(self): config = Config.default() self.assertEqual('AROW', config['method'])
X = scaler.fit_transform(X) # calculate the domain X_min = X.min(axis=0) #X_min = np.ones(X.shape[1]) X_max = X.max(axis=0) X0, X1 = np.meshgrid(np.linspace(X_min[0], X_max[0], meshsize), np.linspace(X_min[1], X_max[1], meshsize)) # make training dataset dataset = Dataset.from_array(X, y) # make mesh dataset to plot decision surface contourf_dataset = Dataset.from_array(np.c_[X0.ravel(), X1.ravel()]) # setup and run jubatus config = Config(method=method, parameter={'regularization_weight': regularization_weight}) classifier = Classifier.run(config, port=port) # construct classifier prediction models and dump model weights for i, _ in enumerate(classifier.train(dataset)): model_name = 'decision_surface_{}'.format(i) classifier.save(name=model_name) # prepare figure fig, ax = plt.subplots() def draw_decision_surface(i): midx = int(i / 2) sidx = int(i / 2) + (i % 2) # load jubatus prediction model
def test_simple(self): config = Config() self.assertEqual('AROW', config['method'])
def test_default(self): config = Config.default() self.assertEqual('AROW', config['method'])
def test_methods(self): config = Config() self.assertTrue(isinstance(config.methods(), list))
shuffle=True, random_state=0, # fixed seed ) # Convert arrays into jubakit Dataset. dataset = Dataset.from_array(X, y) # Try finding the best classifier parameter. param2metrics = {} for method in ['AROW', 'NHERD', 'CW']: for rw in [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0]: print('Running ({0} / regularization_weight = {1})...'.format( method, rw)) # Create a config data structure. jubatus_config = Config(method=method, parameter={'regularization_weight': rw}) # It is equivalent to: #jubatus_config = Config.default() #jubatus_config['method'] = method #jubatus_config['parameter']['regularization_weight'] = rw # Launch Jubatus server using the specified configuration. classifier = Classifier.run(jubatus_config) # Train with the dataset. for _ in classifier.train(dataset): pass # Classify with the same dataset. y_true = []
def test_embedded(self): classifier = Classifier.run(Config(), embedded=True)
# Load a CSV file. loader = CSVLoader('iris.csv') # Define a Schema that defines types for each columns of the CSV file. schema = Schema({ 'Species': Schema.LABEL, }, Schema.NUMBER) # Display Schema print('Schema: {0}'.format(schema)) # Create a Dataset. dataset = Dataset(loader, schema).shuffle() n_samples = len(dataset) n_train_samples = int(n_samples / 2) # Create a Classifier configuration. cfg = Config() # Bulk train-test the classifier. result = Classifier.train_and_classify( cfg, dataset[:n_train_samples], dataset[n_train_samples:], sklearn.metrics.classification_report ) print('---- Classification Report -----------------------------------') print(result)
def test_methods(self): config = Config() self.assertTrue(isinstance(config.methods(), list))
# Load built-in `iris` dataset from scikit-learn. iris = sklearn.datasets.load_iris() # Convert it into jubakit Dataset. #dataset = Dataset.from_array(iris.data, iris.target) # ... or, optionally you can assign feature/label names to improve human-readbility. dataset = Dataset.from_array(iris.data, iris.target, iris.feature_names, iris.target_names) # Shuffle the dataset, as the dataset is sorted by label. dataset = dataset.shuffle() # Create a Classifier Service. # Classifier process starts using a default configuration. classifier = Classifier.run(Config()) # Prepare arrays to keep true/predicted labels to display a report later. true_labels = [] predicted_labels = [] # Run stratified K-fold validation. labels = list(dataset.get_labels()) if sklearn_version < 18: train_test_indices = StratifiedKFold(labels, n_folds=10) else: skf = StratifiedKFold(n_splits=10) train_test_indices = skf.split(labels, labels) for train_idx, test_idx in train_test_indices: # Clear the classifier (call `clear` RPC).
def setUp(self): self._service = Classifier.run(Config()) self._sh = self._service._shell()