def test_replace_list_in_list(self, backend, schema_subnode,
valid_subnode_data):
schema_lstnode = schema.List(schema_subnode)
data_node = backend.module.List(schema.List(schema_lstnode),
parent=None,
new_params=backend.new_params)
data_node.replace([[valid_subnode_data, valid_subnode_data]])
assert compare_values(data_node[0][0].value, valid_subnode_data)
assert compare_values(data_node[0][1].value, valid_subnode_data)
def test_init_from_storage(self):
data_storage = {'item_0': np.array([True]),
'item_1': np.array([False])}
data_node = npz.List(schema.List(schema.Bool()), parent=None,
data_storage=data_storage)
assert data_node[0].value is True
assert data_node[1].value is False
class TestFileStorage:
@pytest.fixture(params=('hdf5', 'mat', 'npz'))
def backend(self, request, tmpdir):
backend = backend_data(
module=importlib.import_module('dsch.backends.' + request.param),
storage_path=str(tmpdir.join('test_frontend.' + request.param)))
return backend
def test_file_exists(self, backend):
schema_node = schema.Bool()
storage_obj = backend.module.Storage(storage_path=backend.storage_path,
schema_node=schema_node)
storage_obj.save()
del storage_obj
with pytest.raises(FileExistsError):
# Give schema_node --> request file creation
storage_obj = backend.module.Storage(
storage_path=backend.storage_path, schema_node=schema_node)
def test_file_not_found(self, backend):
# Omit schema node --> request file loading
with pytest.raises(FileNotFoundError):
storage_obj = backend.module.Storage(
storage_path=backend.storage_path)
@pytest.mark.parametrize('schema_node', (
schema.Bool(),
schema.Compilation({
'spam': schema.Bool(),
'eggs': schema.Bool()
}),
schema.List(schema.Bool()),
))
def test_incomplete_data(self, backend, schema_node):
storage_obj = backend.module.Storage(storage_path=backend.storage_path,
schema_node=schema_node)
storage_obj.save()
del storage_obj
storage_obj = backend.module.Storage(storage_path=backend.storage_path)
assert storage_obj.schema_node
def test_missing_optional_data(self, backend):
schema_node = schema.Compilation({
'ham':
schema.Compilation({
'spam': schema.Bool(),
'eggs': schema.Bool()
}, )
})
storage_obj = backend.module.Storage(storage_path=backend.storage_path,
schema_node=schema_node)
storage_obj.data.ham.spam.value = True
storage_obj.save()
del storage_obj
storage_obj = backend.module.Storage(storage_path=backend.storage_path)
assert hasattr(storage_obj.data.ham, 'eggs')
def test_validate_fail_max_length(self):
subnode = schema.Bool()
node = schema.List(subnode, max_length=3)
with pytest.raises(ValidationError) as err:
node.validate([1, 2, 3, 4])
assert err.value.message == 'Maximum list length exceeded.'
assert err.value.expected == 3
assert err.value.got == 4
def test_validate_fail_min_length(self):
subnode = schema.Bool()
node = schema.List(subnode, min_length=3)
with pytest.raises(ValidationError) as err:
node.validate([1, 2])
assert err.value.message == 'Minimum list length undercut.'
assert err.value.expected == 3
assert err.value.got == 2
def test_save(self):
data_node = mat.List(schema.List(schema.Bool()), parent=None)
data_node.append(True)
data_node.append(False)
data_storage = data_node.save()
assert len(data_storage) == 2
assert data_storage[0] == np.array([True])
assert data_storage[1] == np.array([False])
def test_replace_list_in_compilation(self, backend, schema_subnode,
valid_subnode_data):
schema_node = schema.Compilation({'spam': schema.List(schema_subnode)})
data_node = backend.module.Compilation(schema_node,
parent=None,
new_params=backend.new_params)
data_node.replace({'spam': [valid_subnode_data, valid_subnode_data]})
assert compare_values(data_node.spam[0].value, valid_subnode_data)
assert compare_values(data_node.spam[1].value, valid_subnode_data)
def test_init_from_storage(self):
data_storage = np.zeros((2, ), dtype=np.object)
data_storage[0] = np.array([True])
data_storage[1] = np.array([False])
data_node = mat.List(schema.List(schema.Bool()),
parent=None,
data_storage=data_storage)
assert data_node[0].value is True
assert data_node[1].value is False
def test_init_from_storage(self, hdf5file):
test_list = hdf5file.create_group('test_list')
test_list.create_dataset('item_0', data=True)
test_list.create_dataset('item_1', data=False)
schema_node = schema.List(schema.Bool())
data_node = hdf5.List(schema_node, parent=None, data_storage=test_list)
assert data_node[0].value is True
assert data_node[1].value is False
def test_init_new(self, hdf5file):
schema_node = schema.List(schema.Bool())
hdf5.List(schema_node,
parent=None,
new_params={
'name': 'test_list',
'parent': hdf5file
})
assert 'test_list' in hdf5file
assert isinstance(hdf5file['test_list'], h5py.Group)
def test_load_from(self, data_node, foreign_backend, schema_subnode,
valid_subnode_data):
data_node_foreign = foreign_backend.module.List(
schema.List(schema_subnode),
parent=None,
new_params=foreign_backend.new_params)
data_node_foreign.append(valid_subnode_data)
data_node_foreign.append(valid_subnode_data)
data_node.load_from(data_node_foreign)
assert len(data_node) == 2
assert compare_values(data_node[0].value, valid_subnode_data)
assert compare_values(data_node[1].value, valid_subnode_data)
def test_validation_error_chain(backend):
schema_node = schema.Compilation({
'spam':
schema.List(
schema.Compilation(
{'eggs': schema.List(schema.String(max_length=3))}))
})
data_node = backend.module.Compilation(schema_node,
parent=None,
new_params=backend.new_params)
data_node.spam.append({'eggs': ['abc', 'def']})
data_node.spam.append({'eggs': ['abc', 'def', 'ghij']})
with pytest.raises(exceptions.SubnodeValidationError) as err:
data_node.validate()
assert err.value.node_path() == 'spam[1].eggs[2]'
assert err.value.__cause__.node_path() == '[1].eggs[2]'
assert err.value.__cause__.__cause__.node_path() == 'eggs[2]'
assert err.value.__cause__.__cause__.__cause__.node_path() == '[2]'
assert str(
err.value).startswith('Node "spam[1].eggs[2]" failed validation:')
assert str(err.value).endswith(str(err.value.original_cause()))
def test_load_list(self, tmpdir):
schema_node = schema.List(schema.Bool())
schema_data = json.dumps(schema_node.to_dict(), sort_keys=True)
storage_path = str(tmpdir.join('test_load_list.npz'))
test_data = {'data.item_0': True, 'data.item_1': False,
'_schema': schema_data}
np.savez(storage_path, **test_data)
npz_file = npz.Storage(storage_path=storage_path)
assert hasattr(npz_file, 'data')
assert isinstance(npz_file.data, npz.List)
assert npz_file.data[0].value is True
assert npz_file.data[1].value is False
def test_replace_compilation_in_list(self, backend, schema_subnode,
valid_subnode_data):
schema_compnode = schema.Compilation({'spam': schema_subnode})
data_node = backend.module.List(schema.List(schema_compnode),
parent=None,
new_params=backend.new_params)
data_node.replace([{
'spam': valid_subnode_data
}, {
'spam': valid_subnode_data
}])
assert compare_values(data_node[0].spam.value, valid_subnode_data)
assert compare_values(data_node[1].spam.value, valid_subnode_data)
def test_load_from_incompatible(self, backend, foreign_backend):
source_node_class = getattr(foreign_backend.module, 'List')
source_node = source_node_class(schema.List(self.schema_node),
parent=None,
new_params=foreign_backend.new_params)
source_node.append(self.valid_data)
dest_node_class = getattr(backend.module, self.class_name)
dest_node = dest_node_class(self.schema_node,
parent=None,
new_params=backend.new_params)
with pytest.raises(exceptions.IncompatibleNodesError):
dest_node.load_from(source_node)
def test_to_dict(self):
subnode = schema.Bool()
node = schema.List(subnode, max_length=3, min_length=1)
node_dict = node.to_dict()
assert 'node_type' in node_dict
assert node_dict['node_type'] == 'List'
assert 'config' in node_dict
assert 'subnode' in node_dict['config']
assert node_dict['config']['subnode'] == subnode.to_dict()
assert 'max_length' in node_dict['config']
assert node_dict['config']['max_length'] == 3
assert 'min_length' in node_dict['config']
assert node_dict['config']['min_length'] == 1
def test_load_list(self, tmpdir):
schema_node = schema.List(schema.Bool())
schema_data = json.dumps(schema_node.to_dict(), sort_keys=True)
storage_path = str(tmpdir.join('test_load_list.mat'))
data = np.zeros((2, ), dtype=np.object)
data[0] = np.array([True])
data[1] = np.array([False])
test_data = {'data': data, 'schema': schema_data}
sio.savemat(storage_path, test_data)
mat_file = mat.Storage(storage_path=storage_path)
assert hasattr(mat_file, 'data')
assert isinstance(mat_file.data, mat.List)
assert mat_file.data[0].value is True
assert mat_file.data[1].value is False
def test_save_list(self, tmpdir):
schema_node = schema.List(schema.Bool())
storage_path = str(tmpdir.join('test_save_list.mat'))
mat_file = mat.Storage(storage_path=storage_path,
schema_node=schema_node)
mat_file.data.replace([True, False])
mat_file.save()
file_ = sio.loadmat(storage_path, squeeze_me=True)
assert 'schema' in file_
assert file_['schema'] == json.dumps(schema_node.to_dict(),
sort_keys=True)
assert 'data' in file_
assert len(file_['data']) == 2
assert file_['data'][0]
assert not file_['data'][1]
def test_load_list(self, tmpdir):
schema_node = schema.List(schema.Bool())
schema_data = json.dumps(schema_node.to_dict(), sort_keys=True)
file_name = str(tmpdir.join('test_load_list.hdf5'))
raw_file = h5py.File(file_name, 'x')
raw_file.attrs['dsch_schema'] = schema_data
data = raw_file.create_group('dsch_data')
data.create_dataset('item_0', data=True)
data.create_dataset('item_1', data=False)
raw_file.flush()
del raw_file
hdf5_file = hdf5.Storage(storage_path=file_name)
assert hasattr(hdf5_file, 'data')
assert isinstance(hdf5_file.data, hdf5.List)
assert hdf5_file.data[0].value is True
assert hdf5_file.data[1].value is False
def test_save_list(self, tmpdir):
schema_node = schema.List(schema.Bool())
storage_path = str(tmpdir.join('test_save_list.npz'))
npz_file = npz.Storage(storage_path=storage_path,
schema_node=schema_node)
npz_file.data.replace([True, False])
npz_file.save()
with np.load(storage_path) as file_:
assert '_schema' in file_
assert file_['_schema'][()] == json.dumps(schema_node.to_dict(),
sort_keys=True)
assert 'data.item_0' in file_
assert file_['data.item_0'].dtype == 'bool'
assert file_['data.item_0'][0]
assert 'data.item_1' in file_
assert file_['data.item_1'].dtype == 'bool'
assert not file_['data.item_1'][0]
def test_save_list(self, tmpdir):
schema_node = schema.List(schema.Bool())
file_name = str(tmpdir.join('test_save_list.h5'))
hdf5_file = hdf5.Storage(storage_path=file_name,
schema_node=schema_node)
hdf5_file.data.replace([True, False])
hdf5_file.save()
file_ = h5py.File(file_name, 'r')
assert 'dsch_schema' in file_.attrs
assert file_.attrs['dsch_schema'] == json.dumps(schema_node.to_dict(),
sort_keys=True)
assert 'dsch_data' in file_
assert 'item_0' in file_['dsch_data']
assert file_['dsch_data']['item_0'].dtype == 'bool'
assert file_['dsch_data']['item_0'][()]
assert 'item_1' in file_['dsch_data']
assert file_['dsch_data']['item_1'].dtype == 'bool'
assert not file_['dsch_data']['item_1'][()]
import datetime
import json
import numpy as np
import pytest
from dsch import schema
from dsch.exceptions import ValidationError
@pytest.mark.parametrize(
'node', (schema.Array(dtype='int32'), schema.Bool(), schema.Bytes(),
schema.Compilation({'spam': schema.Bool()}), schema.Date(),
schema.DateTime(), schema.List(schema.Bool()),
schema.Scalar(dtype='int32'), schema.String(), schema.Time()))
class TestGenericSchemaNode:
def test_to_json(self, node):
json_str = node.to_json()
node_dict = json.loads(json_str)
assert isinstance(node_dict, dict)
def test_hash(self, node):
hash = node.hash()
assert len(hash) == 64
@pytest.mark.parametrize('node1, node2', (
(schema.Array(dtype='float'), schema.Array(dtype='float')),
(schema.Bool(), schema.Bool()),
(schema.Bytes(), schema.Bytes()),
(schema.Compilation({'spam': schema.Bool()
def test_init_defaults(self):
node = schema.List(schema.Bool())
assert node.max_length is None
assert node.min_length is None
def test_validate(self):
subnode = schema.Bool()
node = schema.List(subnode, max_length=3, min_length=1)
node.validate([23, 42])
def data_node(self, backend, schema_subnode):
data_node = backend.module.List(schema.List(schema_subnode),
parent=None,
new_params=backend.new_params)
return data_node
def test_init(self):
node = schema.List(schema.Bool(), max_length=3, min_length=1)
assert isinstance(node.subnode, schema.Bool)
assert node.max_length == 3
assert node.min_length == 1
