def test_fsstore(self):
with tempfile.TemporaryDirectory() as tmpdir:
storage = FSStore(tmpdir)
model = FakeModel('val')
model_pickled = pickle.dumps(model)
storage.write(model_pickled, 'for_testing.model')
assert os.path.isfile(os.path.join(
tmpdir, 'for_testing.model')) == storage.exists(
'for_testing.model') == True
with storage.open("for_testing_compressed.model", "wb") as f:
joblib.dump(model, f, compress=True)
assert storage.exists("for_testing_compressed.model")
with open_sesame(
os.path.join(tmpdir, "for_testing_compressed.model"),
"rb") as f:
model_loaded = joblib.load(f)
assert model.val == model_loaded.val
model_loaded = storage.load('for_testing.model')
model_loaded = pickle.loads(model_loaded)
assert model_loaded.val == 'val'
storage.delete('for_testing.model')
assert os.path.isfile(os.path.join(
tmpdir, 'for_testing.model')) == storage.exists(
'for_testing.model') == False
def test_s3store(self):
import boto3
client = boto3.client('s3')
client.create_bucket(Bucket='fake-open-skills',
ACL='public-read-write')
s3 = s3fs.S3FileSystem()
storage = S3Store(path=f"s3://fake-open-skills/model_cache")
assert not s3.exists(storage.path) == True
model = FakeModel('val')
model_pickled = pickle.dumps(model)
storage.write(model_pickled, 'for_testing.model')
assert storage.exists("for_testing.model")
with storage.open("for_testing_compressed.model", "wb") as f:
joblib.dump(model, f, compress=True)
assert storage.exists("for_testing_compressed.model")
with open_sesame(
"s3://fake-open-skills/model_cache/for_testing_compressed.model",
"rb") as f:
model_loaded = joblib.load(f)
assert model.val == model_loaded.val
model_loaded = storage.load('for_testing.model')
model_loaded = pickle.loads(model_loaded)
assert model_loaded.val == 'val'
fake_lookup = {'1': 1, '2': 2, '3': 3}
fake_lookup_bytes = json.dumps(fake_lookup).encode()
storage.write(fake_lookup_bytes, 'for_testing.json')
assert storage.exists("for_testing.json")
fake_lookup_loaded = json.loads(
storage.load('for_testing.json').decode())
assert fake_lookup == fake_lookup_loaded
storage.delete('for_testing.model')
assert not storage.exists("for_testing.model")
def _save(self, cls_cv, path_to_save):
with open_sesame(path_to_save, 'wb') as f:
joblib.dump(cls_cv, f, compress=True)
def _save(self, cls_cv, path_to_save):
with open_sesame(path_to_save, 'wb') as f:
dill.dump(cls_cv, f)