def test_fsstore(self): with tempfile.TemporaryDirectory() as tmpdir: storage = FSStore(tmpdir) model = FakeModel('val') model_pickled = pickle.dumps(model) storage.write(model_pickled, 'for_testing.model') assert os.path.isfile(os.path.join( tmpdir, 'for_testing.model')) == storage.exists( 'for_testing.model') == True with storage.open("for_testing_compressed.model", "wb") as f: joblib.dump(model, f, compress=True) assert storage.exists("for_testing_compressed.model") with open_sesame( os.path.join(tmpdir, "for_testing_compressed.model"), "rb") as f: model_loaded = joblib.load(f) assert model.val == model_loaded.val model_loaded = storage.load('for_testing.model') model_loaded = pickle.loads(model_loaded) assert model_loaded.val == 'val' storage.delete('for_testing.model') assert os.path.isfile(os.path.join( tmpdir, 'for_testing.model')) == storage.exists( 'for_testing.model') == False
def test_fsstore(self): with tempfile.TemporaryDirectory() as tmpdir: storage = FSStore(tmpdir) model = pickle.dumps(FakeModel('val')) storage.write(model, 'for_testing.model') assert os.path.isfile(os.path.join( tmpdir, 'for_testing.model')) == storage.exists( 'for_testing.model') == True model_loaded = storage.load('for_testing.model') model_loaded = pickle.loads(model_loaded) assert model_loaded.val == 'val' storage.delete('for_testing.model') assert os.path.isfile(os.path.join( tmpdir, 'for_testing.model')) == storage.exists( 'for_testing.model') == False
'max_depth': [20, 50], 'max_features': ['log2'], 'min_samples_split': [10, 20] }, 'sklearn.neural_network.MLPClassifier': { 'hidden_layer_sizes': [100, 200, 300, 500, 1000], 'activation': ['identity', 'logistic', 'tanh', 'relu'], 'solver': ['lbfgs', 'sgd', 'adam'] }, 'sklearn.svm.SVC': { 'C': [0.1, 1, 10, 100, 1000], 'kernel': ['linear', 'poly', 'sigmoid', 'rbf', 'precomputed'], 'shrinking': [True, False], 'decision_function_shape': ['ovo', 'ovr'] } } trainer = OccupationClassifierTrainer( matrix=matrix, k_folds=3, grid_config=grid_config, storage=FSStore('tmp/soc_classifiers'), n_jobs = num_of_worker ) trainer.train() fs = FSStore(os.path.join('soc_classifiers', trainer.train_time)) fs.write(train_bytes, "train.data") fs.write(test_bytes, "test_data")