def test_fsstore(self):
with tempfile.TemporaryDirectory() as tmpdir:
storage = FSStore(tmpdir)
model = FakeModel('val')
model_pickled = pickle.dumps(model)
storage.write(model_pickled, 'for_testing.model')
assert os.path.isfile(os.path.join(
tmpdir, 'for_testing.model')) == storage.exists(
'for_testing.model') == True
with storage.open("for_testing_compressed.model", "wb") as f:
joblib.dump(model, f, compress=True)
assert storage.exists("for_testing_compressed.model")
with open_sesame(
os.path.join(tmpdir, "for_testing_compressed.model"),
"rb") as f:
model_loaded = joblib.load(f)
assert model.val == model_loaded.val
model_loaded = storage.load('for_testing.model')
model_loaded = pickle.loads(model_loaded)
assert model_loaded.val == 'val'
storage.delete('for_testing.model')
assert os.path.isfile(os.path.join(
tmpdir, 'for_testing.model')) == storage.exists(
'for_testing.model') == False
def test_fsstore(self):
with tempfile.TemporaryDirectory() as tmpdir:
storage = FSStore(tmpdir)
model = pickle.dumps(FakeModel('val'))
storage.write(model, 'for_testing.model')
assert os.path.isfile(os.path.join(
tmpdir, 'for_testing.model')) == storage.exists(
'for_testing.model') == True
model_loaded = storage.load('for_testing.model')
model_loaded = pickle.loads(model_loaded)
assert model_loaded.val == 'val'
storage.delete('for_testing.model')
assert os.path.isfile(os.path.join(
tmpdir, 'for_testing.model')) == storage.exists(
'for_testing.model') == False
'max_depth': [20, 50],
'max_features': ['log2'],
'min_samples_split': [10, 20]
},
'sklearn.neural_network.MLPClassifier': {
'hidden_layer_sizes': [100, 200, 300, 500, 1000],
'activation': ['identity', 'logistic', 'tanh', 'relu'],
'solver': ['lbfgs', 'sgd', 'adam']
},
'sklearn.svm.SVC': {
'C': [0.1, 1, 10, 100, 1000],
'kernel': ['linear', 'poly', 'sigmoid', 'rbf', 'precomputed'],
'shrinking': [True, False],
'decision_function_shape': ['ovo', 'ovr']
}
}
trainer = OccupationClassifierTrainer(
matrix=matrix,
k_folds=3,
grid_config=grid_config,
storage=FSStore('tmp/soc_classifiers'),
n_jobs = num_of_worker
)
trainer.train()
fs = FSStore(os.path.join('soc_classifiers', trainer.train_time))
fs.write(train_bytes, "train.data")
fs.write(test_bytes, "test_data")
