def test_read_no_model_with_given_version(self, File):
with patch('palladium.persistence.os.path.exists') as exists:
exists.return_value = False
f = File('/models/model-{version}')
with pytest.raises(LookupError) as exc:
f.read(1)
assert exc.value.args[0] == 'No such version: 1'
def test_read_no_model_with_given_version(self, File):
with patch('palladium.persistence.os.path.exists') as exists:
exists.return_value = False
f = File('/models/model-{version}')
with pytest.raises(LookupError) as exc:
f.read(1)
assert exc.value.args[0] == 'No such version: 1'
def test_upgrade_from_version(self, File):
from palladium import __version__
persister = File('model-{version}')
with patch.object(File, 'upgrade_steps') as upgrade_steps:
persister.upgrade(from_version='0.34')
upgrade_steps.run.assert_called_with(persister, '0.34', __version__)
assert persister.list_properties()['db-version'] == __version__
def test_upgrade_from_version(self, File):
from palladium import __version__
persister = File('model-{version}')
with patch.object(File, 'upgrade_steps') as upgrade_steps:
persister.upgrade(from_version='0.34')
upgrade_steps.run.assert_called_with(persister, '0.34', __version__)
assert persister.list_properties()['db-version'] == __version__
def test_read_no_model(self, File):
with patch('palladium.persistence.File.list_models') as lm,\
patch('palladium.persistence.File.list_properties') as lp:
lp.return_value = {}
lm.return_value = []
f = File('/models/model-{version}')
with pytest.raises(LookupError) as exc:
f.read()
assert exc.value.args[0] == 'No active model available'
def test_read_no_model(self, File):
with patch('palladium.persistence.File.list_models') as lm,\
patch('palladium.persistence.File.list_properties') as lp:
lp.return_value = {}
lm.return_value = []
f = File('/models/model-{version}')
with pytest.raises(LookupError) as exc:
f.read()
assert exc.value.args[0] == 'No active model available'
def test_upgrade_no_args(self, File):
from palladium import __version__
persister = File('model-{version}')
with patch.object(File, '_read_md',
return_value={'properties': {'db-version': '0.33'}}):
with patch.object(File, 'upgrade_steps') as upgrade_steps:
persister.upgrade()
upgrade_steps.run.assert_called_with(persister, '0.33', __version__)
assert persister.list_properties()['db-version'] == __version__
def test_upgrade_with_legacy_md(self, File):
from palladium import __version__
persister = File('model-{version}')
legacy_md = [{'some': 'model'}]
with patch.object(File, '_read_md',
side_effect=[legacy_md, {'properties': {}}]):
with patch.object(File, 'upgrade_steps') as upgrade_steps:
persister.upgrade()
upgrade_steps.run.assert_called_with(persister, '0.0', __version__)
assert persister.list_properties()['db-version'] == __version__
def test_read_no_active_model(self, File):
with patch('palladium.persistence.File.list_models') as lm,\
patch('palladium.persistence.File.list_properties') as lp:
lp.return_value = {}
lm.return_value = [{'version': 99}]
filename = '/models/model-{version}'
f = File(filename)
with pytest.raises(LookupError) as exc:
f.read()
assert exc.value.args[0] == 'No active model available: {}'.format(
filename)
def persister(self, tmpdir):
from palladium.persistence import File
model1 = Dummy()
annotate(model1, {
'attachments/myatt.txt': 'aGV5',
'attachments/my2ndatt.txt': 'aG8='
})
model2 = Dummy()
annotate(model2, {'attachments/myatt.txt': 'aG8='})
persister = File(str(tmpdir) + '/model-{version}')
persister.write(model1)
persister.write(model2)
return persister
def test_upgrade_no_args(self, File):
from palladium import __version__
persister = File('model-{version}')
with patch.object(File,
'_read_md',
return_value={'properties': {
'db-version': '0.33'
}}):
with patch.object(File, 'upgrade_steps') as upgrade_steps:
persister.upgrade()
upgrade_steps.run.assert_called_with(persister, '0.33', __version__)
assert persister.list_properties()['db-version'] == __version__
def test_upgrade_with_legacy_md(self, File):
from palladium import __version__
persister = File('model-{version}')
legacy_md = [{'some': 'model'}]
with patch.object(File,
'_read_md',
side_effect=[legacy_md, {
'properties': {}
}]):
with patch.object(File, 'upgrade_steps') as upgrade_steps:
persister.upgrade()
upgrade_steps.run.assert_called_with(persister, '0.0', __version__)
assert persister.list_properties()['db-version'] == __version__
def test_upgrade_1_0(self, File):
with patch('builtins.open') as open,\
patch('palladium.persistence.os.path.exists') as exists,\
patch('palladium.persistence.json.load') as load,\
patch('palladium.persistence.json.dump') as dump:
exists.return_value = True
load.side_effect = [
[{
'version': '1'
}, {
'version': '2'
}],
{
'properties': {}
},
]
File('model-{version}').upgrade(from_version="0.0",
to_version="1.0")
exists.assert_called_with('model-metadata.json')
open_rv = open.return_value.__enter__.return_value
load.assert_called_with(open_rv)
new_md = {
'models': [{
'version': '1'
}, {
'version': '2'
}],
'properties': {
'active-model': '2',
},
}
dump.assert_called_with(new_md, open_rv, indent=4)
def test_list_with_metadata(self, File):
with patch('palladium.persistence.os.path.exists') as exists,\
patch('builtins.open') as open,\
patch('palladium.persistence.json.load') as load:
exists.return_value = True
assert File('model-{version}').list() == load.return_value
exists.assert_called_with('model-metadata.json')
open.assert_called_with('model-metadata.json', 'r')
def test_list_properties_no_metadata(self, File):
from palladium import __version__
with patch('palladium.persistence.os.path.exists') as exists:
exists.return_value = False
assert File('model-{version}').list_properties() == {
'db-version': __version__,
}
exists.assert_called_with('model-metadata.json')
def test_read_md(self, File):
with patch('builtins.open') as open,\
patch('palladium.persistence.os.path.exists') as exists,\
patch('palladium.persistence.json.load') as load:
exists.return_value = True
result = File('model-{version}')._read_md()
exists.assert_called_with('model-metadata.json')
open.assert_called_with('model-metadata.json', 'r')
load.assert_called_with(open.return_value.__enter__.return_value, )
assert result == load.return_value
def test_read_with_version(self, File):
with patch('palladium.persistence.File.list') as list,\
patch('palladium.persistence.gzip.open') as open,\
patch('palladium.persistence.pickle.load') as load:
list.return_value = [{'version': 99}]
open.return_value = MagicMock()
result = File('/models/model-{version}').read(432)
open.assert_called_with('/models/model-432.pkl.gz', 'rb')
assert result == load.return_value
load.assert_called_with(open.return_value.__enter__.return_value)
def test_read_md_no_file(self, File):
from palladium import __version__
with patch('palladium.persistence.os.path.exists') as exists:
exists.return_value = False
assert File('model-{version}')._read_md() == {
'models': [],
'properties': {
'db-version': __version__
},
}
def test_write_md(self, File):
with patch('builtins.open') as open,\
patch('palladium.persistence.json.dump') as dump:
md = [{'hello': 'world'}]
File('model-{version}')._write_md(md)
open.assert_called_with('model-metadata.json', 'w')
dump.assert_called_with(
md,
open.return_value.__enter__.return_value,
indent=4,
)
def test_list_properties_with_metadata(self, File):
with patch('palladium.persistence.File._read_md') as read_md:
read_md.return_value = {
'properties': {
'active-model': '33',
'hello': 'world',
},
}
assert File('model-{version}').list_properties() == {
'active-model': '33',
'hello': 'world',
}
def test_upgrade_1_0_no_metadata(self, File):
with patch('builtins.open') as open,\
patch('palladium.persistence.os.path.exists') as exists,\
patch('palladium.persistence.json.dump') as dump:
exists.return_value = False
File('model-{version}').upgrade(from_version="0.0",
to_version="1.0")
exists.assert_called_with('model-metadata.json')
open_rv = open.return_value.__enter__.return_value
new_md = {'models': [], 'properties': {}}
dump.assert_called_with(new_md, open_rv, indent=4)
def test_read_with_version(self, File):
with patch('palladium.persistence.File.list_models') as lm,\
patch('palladium.persistence.os.path.exists') as exists,\
patch('palladium.persistence.open') as open,\
patch('palladium.persistence.gzip.open') as gzopen,\
patch('palladium.persistence.pickle.load') as load:
lm.return_value = [{'version': 99}]
exists.side_effect = lambda fn: fn == '/models/model-432.pkl.gz'
open.return_value = MagicMock()
result = File('/models/model-{version}').read(432)
open.assert_called_with('/models/model-432.pkl.gz', 'rb')
assert result == load.return_value
load.assert_called_with(gzopen.return_value.__enter__.return_value)
def test_read(self, File):
with patch('palladium.persistence.File.list_models') as lm,\
patch('palladium.persistence.File.list_properties') as lp,\
patch('palladium.persistence.os.path.exists') as exists,\
patch('palladium.persistence.gzip.open') as open,\
patch('palladium.persistence.pickle.load') as load:
lm.return_value = [{'version': 99}]
lp.return_value = {'active-model': '99'}
exists.return_value = True
open.return_value = MagicMock()
result = File('/models/model-{version}').read()
open.assert_called_with('/models/model-99.pkl.gz', 'rb')
assert result == load.return_value
load.assert_called_with(open.return_value.__enter__.return_value)
def test_write_no_model_files(self, File):
with patch('palladium.persistence.File.list_models') as lm,\
patch('palladium.persistence.File._update_md') as update_md,\
patch('palladium.persistence.gzip.open') as open,\
patch('palladium.persistence.pickle.dump') as dump:
lm.return_value = []
open.return_value = MagicMock()
model = MagicMock()
result = File('/models/model-{version}').write(model)
open.assert_called_with('/models/model-1.pkl.gz', 'wb')
dump.assert_called_with(
model,
open.return_value.__enter__.return_value,
)
update_md.assert_called_with({'models': [model.__metadata__]})
assert result == 1
def test_activate_bad_version(self, File):
with patch('palladium.persistence.File._read_md') as read_md,\
patch('palladium.persistence.File._update_md') as update_md:
read_md.return_value = {
'models': [{
'version': 1
}, {
'version': 2
}],
'properties': {
'active-model': '2'
},
}
with pytest.raises(LookupError) as exc:
File('model-{version}').activate(3)
assert exc.value.args[0] == 'No such version: 3'
def test_update_md(self, File):
with patch('palladium.persistence.File._read_md') as read_md,\
patch('builtins.open') as open:
read_md.return_value = {
'hello': 'world',
'models': [1],
'properties': {},
}
File('model-{version}')._update_md_orig({'models': [2]})
open.assert_called_with('model-metadata.json', 'wb')
fh = open.return_value.__enter__.return_value
json_written = json.loads(fh.write.call_args[0][0].decode('utf-8'))
assert json_written == {
'hello': 'world',
'models': [2],
'properties': {},
}
def test_read_activated_model_missing(self, File):
with patch('palladium.persistence.File._read_md') as read_md,\
patch('palladium.persistence.File._update_md') as update_md:
read_md.return_value = {
'models': [{
'version': 1
}, {
'version': 3
}],
'properties': {
'active-model': '2'
},
}
with pytest.raises(LookupError) as exc:
File('model-{version}').read()
assert (exc.value.args[0] ==
'Activated model not available. Maybe it was deleted.')
def test_activate(self, File):
with patch('palladium.persistence.File._read_md') as read_md,\
patch('palladium.persistence.File._update_md') as update_md:
read_md.return_value = {
'models': [{
'version': 1
}, {
'version': 2
}],
'properties': {
'active-model': '2'
},
}
File('model-{version}').activate(1)
update_md.assert_called_with({
'properties': {
'active-model': '1'
},
})
def test_update_md(self, File):
with patch('palladium.persistence.File._read_md') as read_md,\
patch('palladium.persistence.json.dump') as dump,\
patch('builtins.open') as open:
read_md.return_value = {
'hello': 'world',
'models': [1],
'properties': {},
}
File('model-{version}')._update_md_orig({'models': [2]})
open.assert_called_with('model-metadata.json', 'w')
dump.assert_called_with(
{
'hello': 'world',
'models': [2],
'properties': {}
},
open.return_value.__enter__.return_value,
indent=4,
)
def test_update_metadata(self, File):
model = MagicMock(__metadata__={
'existing': 'entry',
'version': 'overwritten'
})
with patch('palladium.persistence.File.list_models') as lm,\
patch('palladium.persistence.File._update_md') as update_md,\
patch('palladium.persistence.gzip.open'),\
patch('palladium.persistence.pickle.dump'):
lm.return_value = [{'version': 99}]
File('/models/model-{version}').write(model)
assert model.__metadata__ == {
'existing': 'entry',
'version': 100,
}
update_md.assert_called_with(
{'models': [{
'version': 99
}, model.__metadata__]})
def test_write_with_model_files(self, File):
with patch('palladium.persistence.File.list') as list,\
patch('palladium.persistence.File._write_md') as write_md,\
patch('palladium.persistence.gzip.open') as open,\
patch('palladium.persistence.pickle.dump') as dump:
list.return_value = [{'version': 99}]
open.return_value = MagicMock()
model = MagicMock()
result = File('/models/model-{version}').write(model)
open.assert_called_with('/models/model-100.pkl.gz', 'wb')
dump.assert_called_with(
model,
open.return_value.__enter__.return_value,
)
write_md.assert_called_with([
{
'version': 99
},
model.__metadata__,
])
assert result == 100
def test_delete(self, File):
with patch('palladium.persistence.File._read_md') as read_md,\
patch('palladium.persistence.File._update_md') as update_md,\
patch('palladium.persistence.os') as os:
read_md.return_value = {
'models': [{
'version': 1
}, {
'version': 2
}],
'properties': {
'active-model': '2'
},
}
File('model-{version}').delete(1)
update_md.assert_called_with({
'models': [{
'version': 2
}],
})
os.remove.assert_called_with('model-1.pkl.gz')
def test_init_path_without_version(self, File):
with pytest.raises(ValueError):
File('path_without')
def test_list_models_no_metadata(self, File):
with patch('palladium.persistence.os.path.exists') as exists:
exists.return_value = False
assert File('model-{version}').list_models() == []
exists.assert_called_with('model-metadata.json')
def test_list_models_with_metadata(self, File):
with patch('palladium.persistence.File._read_md') as read_md:
read_md.return_value = {'models': [{'version': 99}]}
assert File('model-{version}').list_models() == [{'version': 99}]
def test_read_no_model_with_given_version(self, File):
with patch('palladium.persistence.File.list') as list:
list.return_value = []
f = File('/models/model-{version}')
with pytest.raises(IOError):
f.read(1)
