def test_input_uncertainties_no_distributions(sensitivity_dps):
mm = MappedMatrix(
packages=sensitivity_dps,
matrix="matrix",
use_vectors=True,
use_arrays=True,
use_distributions=False,
)
magic = 191664963
expected = np.array(
[
(99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
(99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
(99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
(98, -8.5, 1.118034, np.NaN, np.NaN, np.NaN, False),
(98, 9.5, 1.118034, np.NaN, np.NaN, np.NaN, False),
(0, 1, np.NaN, np.NaN, np.NaN, np.NaN, False),
(0, 2, np.NaN, np.NaN, np.NaN, np.NaN, False),
(0, -1, np.NaN, np.NaN, np.NaN, np.NaN, False),
(0, -3, np.NaN, np.NaN, np.NaN, np.NaN, False),
(99, 1, np.NaN, np.NaN, np.NaN, np.NaN, False),
(99, 2, np.NaN, np.NaN, np.NaN, np.NaN, False),
(99, 3, np.NaN, np.NaN, np.NaN, np.NaN, False),
],
dtype=bwp.UNCERTAINTY_DTYPE,
)
ua = mm.input_uncertainties()
for field, _ in bwp.UNCERTAINTY_DTYPE:
assert np.allclose(ua[field], expected[field], equal_nan=True)
def test_input_data_vector(sensitivity_dps):
indices = [
191664963,
1662057957,
1405681631,
]
mm = MappedMatrix(
packages=sensitivity_dps,
matrix="matrix",
use_vectors=True,
use_arrays=True,
use_distributions=False,
)
for index in indices:
expected = np.array([
index % 100,
index % 100,
index % 100,
-(index % 4 + 7),
(index % 4 + 8),
1,
2,
-1,
-3,
1,
2,
3,
])
assert np.allclose(mm.input_data_vector(), expected)
next(mm)
def test_input_uncertainties_limit_samples(sensitivity_dps):
mm = MappedMatrix(
packages=sensitivity_dps,
matrix="matrix",
use_vectors=True,
use_arrays=True,
use_distributions=True,
)
magic = 191664963
expected = np.array(
[
(99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
(99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
(99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
(98, -8, 0.8164966, np.NaN, np.NaN, np.NaN, False),
(98, 9, 0.8164966, np.NaN, np.NaN, np.NaN, False),
(4, 1, np.NaN, np.NaN, 0.5, 1.5, False),
(4, 2, np.NaN, np.NaN, 1.5, 2.5, False),
(0, 1, np.NaN, np.NaN, np.NaN, np.NaN, True),
(4, 3, np.NaN, np.NaN, 2.5, 3.5, True),
(99, 1, np.NaN, np.NaN, np.NaN, np.NaN, False),
(99, 2, np.NaN, np.NaN, np.NaN, np.NaN, False),
(99, 3, np.NaN, np.NaN, np.NaN, np.NaN, False),
],
dtype=bwp.UNCERTAINTY_DTYPE,
)
ua = mm.input_uncertainties(number_samples=3)
for field, _ in bwp.UNCERTAINTY_DTYPE:
assert np.allclose(ua[field], expected[field], equal_nan=True)
def test_custom_filter():
mm = MappedMatrix(
packages=[diagonal()],
matrix="foo",
use_arrays=False,
use_distributions=False,
)
assert mm.matrix.shape == (4, 2)
mm = MappedMatrix(
packages=[diagonal()],
matrix="foo",
use_arrays=False,
use_distributions=False,
custom_filter=lambda x: x["col"] == 1,
)
assert mm.matrix.shape == (3, 1)
assert mm.matrix.sum() == 1 - 2.3 + 25
with pytest.raises(EmptyArray):
mm = MappedMatrix(
packages=[diagonal()],
matrix="foo",
use_arrays=False,
use_distributions=False,
custom_filter=lambda x: x["col"] == 2,
)
def test_input_row_col_indices(sensitivity_dps):
mm = MappedMatrix(
packages=sensitivity_dps,
matrix="matrix",
use_vectors=True,
use_arrays=True,
use_distributions=False,
)
expected = np.array(
[
(0, 0),
(1, 2),
(2, 1),
(0, 0),
(1, 1),
(2, 0),
(2, 1),
(2, 1),
(2, 2),
(0, 2),
(2, 0),
(2, 2),
],
dtype=bwp.INDICES_DTYPE,
)
assert np.allclose(mm.input_row_col_indices()["row"], expected["row"])
assert np.allclose(mm.input_row_col_indices()["col"], expected["col"])
def test_matrix_building_multiple_dps_no_distributions_yes_arrays_sum_both(
sensitivity_dps, ):
indices = [
191664963,
1662057957,
1405681631,
]
sensitivity_dps[0].metadata["sum_inter_duplicates"] = True
sensitivity_dps[1].metadata["sum_inter_duplicates"] = True
mm = MappedMatrix(
packages=sensitivity_dps,
matrix="matrix",
use_vectors=True,
use_arrays=True,
use_distributions=False,
)
for index in indices:
expected = np.array([
[index % 100 - (index % 4 + 7), 0, 1],
[0, (index % 4 + 8), index % 100],
[1 + 2, 2 + index % 100 - 1, 3 - 3],
])
assert np.allclose(mm.matrix.toarray(), expected)
next(mm)
def test_all_empty_after_custom_filter():
s = bwp.create_datapackage()
s.add_persistent_vector(
matrix="foo",
data_array=np.arange(2),
indices_array=np.array([(0, 0), (1, 0)], dtype=bwp.INDICES_DTYPE),
)
with pytest.raises(EmptyArray):
MappedMatrix(packages=[s],
matrix="foo",
custom_filter=lambda x: x['col'] > 0)
assert MappedMatrix(packages=[s],
matrix="foo",
custom_filter=lambda x: x['col'] > 0,
empty_ok=True)
def test_input_provenance(sensitivity_dps):
mm = MappedMatrix(
packages=sensitivity_dps,
matrix="matrix",
use_vectors=True,
use_arrays=True,
use_distributions=False,
)
expected = [
(sensitivity_dps[0], "a", (0, 3)),
(sensitivity_dps[0], "b", (3, 5)),
(sensitivity_dps[1], "c", (5, 9)),
(sensitivity_dps[1], "d", (9, 12)),
]
assert mm.input_provenance() == expected
def test_indices_matrix_with_aggregation():
dp = create_datapackage(sum_intra_duplicates=True)
data_array = np.array([101, 111, 112, 113])
indices_array = np.array([(1, 4), (2, 5), (3, 6), (3, 6)], dtype=INDICES_DTYPE)
dp.add_persistent_vector(
matrix="matrix-b",
data_array=data_array,
name="x-third",
indices_array=indices_array,
)
expected = np.array([0, 1, 2])
mm = MappedMatrix(packages=[dp], matrix="matrix-b")
for group in mm.groups:
assert np.allclose(group.row_matrix, expected)
assert np.allclose(group.col_matrix, expected)
expected = np.array([0, 1])
mm = MappedMatrix(
packages=[dp], matrix="matrix-b", custom_filter=lambda x: x["row"] < 3
)
for group in mm.groups:
assert np.allclose(group.row_matrix, expected)
assert np.allclose(group.col_matrix, expected)
rexpected = np.array([0, 1])
cexpected = np.array([1, 2])
am = ArrayMapper(array=np.array([2, 3]))
mm = MappedMatrix(packages=[dp], matrix="matrix-b", row_mapper=am)
for group in mm.groups:
assert np.allclose(group.row_matrix, rexpected)
assert np.allclose(group.col_matrix, cexpected)
rexpected = np.array([0])
cexpected = np.array([1])
mm = MappedMatrix(
packages=[dp],
matrix="matrix-b",
row_mapper=am,
custom_filter=lambda x: x["row"] < 3,
)
for group in mm.groups:
assert np.allclose(group.row_matrix, rexpected)
assert np.allclose(group.col_matrix, cexpected)
def test_transpose_matrix():
mm = MappedMatrix(
packages=[diagonal()],
matrix="foo",
use_arrays=False,
use_distributions=False,
transpose=True,
)
assert mm.matrix.shape == (2, 4)
assert np.allclose(mm.matrix.data, [4, 1, -2.3, 25])
def test_group_filtering():
mm = MappedMatrix(
packages=[basic_mm()],
matrix="foo",
use_arrays=False,
use_distributions=False,
)
assert len(mm.groups) == 2
assert mm.groups[0].label == "vector"
assert mm.groups[1].label == "vector2"
def test_input_index_vector(sensitivity_dps):
dp = bwp.create_datapackage(combinatorial=True)
indices_array = np.array([(10, 10), (11, 11)], dtype=bwp.INDICES_DTYPE)
data_array = np.array([[7, 8], [8, 9]]).T
flip_array = np.array([1, 0], dtype=bool)
dp.add_persistent_array(
matrix="matrix",
data_array=data_array,
name="g",
indices_array=indices_array,
flip_array=flip_array,
)
dp.add_persistent_array(
matrix="matrix",
data_array=data_array,
name="h",
indices_array=indices_array,
flip_array=flip_array,
)
mm = MappedMatrix(
packages=list(sensitivity_dps) + [dp],
matrix="matrix",
use_vectors=True,
use_arrays=True,
use_distributions=False,
)
expected = [
(191664963, 191664963, 0, 0),
(1662057957, 1662057957, 0, 1),
(1405681631, 1405681631, 1, 0),
(942484272, 942484272, 1, 1),
]
for row in expected:
print(mm.input_indexer_vector())
assert np.allclose(mm.input_indexer_vector(), row)
try:
next(mm)
except StopIteration:
pass
def test_data_original():
dp = load_datapackage(ZipFS(dirpath / "a-first.zip")).filter_by_attribute(
"group", "x-third"
)
mm = MappedMatrix(packages=[dp], matrix="matrix-b")
expected = np.array([101, 111, 112])
for group in mm.groups:
assert np.allclose(group.data_original, expected)
mm = MappedMatrix(
packages=[dp], matrix="matrix-b", custom_filter=lambda x: x["row"] < 3
)
for group in mm.groups:
assert np.allclose(group.data_original, expected)
am = ArrayMapper(array=np.array([2, 3]))
mm = MappedMatrix(packages=[dp], matrix="matrix-b", row_mapper=am)
for group in mm.groups:
assert np.allclose(group.data_original, expected)
def test_mappers():
given = np.array([4, 2, 1, 0])
expected = np.array([2, 1, -1, 0])
mm = MappedMatrix(
packages=[basic_mm()],
matrix="foo",
use_arrays=False,
use_distributions=False,
)
result = mm.row_mapper.map_array(given)
assert np.allclose(result, expected)
def test_diagonal_matrix():
mm = MappedMatrix(
packages=[diagonal()],
matrix="foo",
use_arrays=False,
use_distributions=False,
)
assert mm.matrix.shape == (4, 2)
assert np.allclose(mm.matrix.data, [1, -2.3, 4, 25])
mm = MappedMatrix(
packages=[diagonal()],
matrix="foo",
use_arrays=False,
use_distributions=False,
diagonal=True,
)
assert mm.matrix.shape == (4, 4)
for x, y in zip(range(4), [1, -2.3, 4, 25]):
assert mm.matrix[x, x] == y
assert np.allclose(mm.matrix.data, [1, -2.3, 4, 25])
def test_indexer_override():
s = bwp.create_datapackage(sequential=True)
s.add_persistent_array(
matrix="foo",
data_array=np.arange(12).reshape(3, 4),
indices_array=np.array([(0, 0), (1, 1), (0, 1)],
dtype=bwp.INDICES_DTYPE),
)
mm = MappedMatrix(
packages=[s],
matrix="foo",
use_arrays=True,
use_distributions=False,
)
assert np.allclose(mm.matrix.toarray(), [[0, 8], [0, 4]])
next(mm)
assert np.allclose(mm.matrix.toarray(), [[1, 9], [0, 5]])
next(mm)
assert np.allclose(mm.matrix.toarray(), [[2, 10], [0, 6]])
class MyIndexer:
index = 2
def __next__(self):
pass
mm = MappedMatrix(
packages=[s],
matrix="foo",
use_arrays=True,
use_distributions=False,
indexer_override=MyIndexer(),
)
assert np.allclose(mm.matrix.toarray(), [[2, 10], [0, 6]])
next(mm)
assert np.allclose(mm.matrix.toarray(), [[2, 10], [0, 6]])
next(mm)
assert np.allclose(mm.matrix.toarray(), [[2, 10], [0, 6]])
def test_indices_mapped():
dp = load_datapackage(ZipFS(dirpath / "a-first.zip")).filter_by_attribute(
"group", "x-third"
)
expected = np.array([0, 1, 2])
mm = MappedMatrix(packages=[dp], matrix="matrix-b")
for group in mm.groups:
assert np.allclose(group.row_mapped, expected)
assert np.allclose(group.col_mapped, expected)
expected = np.array([0, 1, -1])
mm = MappedMatrix(
packages=[dp], matrix="matrix-b", custom_filter=lambda x: x["row"] < 3
)
for group in mm.groups:
assert np.allclose(group.row_mapped, expected)
assert np.allclose(group.col_mapped, expected)
rexpected = np.array([-1, 0, 1])
cexpected = np.array([0, 1, 2])
am = ArrayMapper(array=np.array([2, 3]))
mm = MappedMatrix(packages=[dp], matrix="matrix-b", row_mapper=am)
for group in mm.groups:
assert np.allclose(group.row_mapped, rexpected)
assert np.allclose(group.col_mapped, cexpected)
rexpected = np.array([-1, 0, 1])
cexpected = np.array([0, 1, -1])
mm = MappedMatrix(
packages=[dp],
matrix="matrix-b",
row_mapper=am,
custom_filter=lambda x: x["row"] < 3,
)
for group in mm.groups:
assert np.allclose(group.row_mapped, rexpected)
assert np.allclose(group.col_mapped, cexpected)
def test_matrix_building_multiple_dps_no_distributions_no_arrays_replacement_summing_intra(
sensitivity_dps, ):
mm = MappedMatrix(
packages=sensitivity_dps,
matrix="matrix",
use_vectors=True,
use_arrays=False,
use_distributions=False,
)
expected = np.array([
[0, 0, 1],
[2, 1, 3],
])
assert np.allclose(mm.matrix.toarray(), expected)
def test_flip_masked():
dp = load_datapackage(ZipFS(dirpath / "a-first.zip")).filter_by_attribute(
"group", "x-third"
)
expected = np.array([True])
am = ArrayMapper(array=np.array([2, 3]))
mm = MappedMatrix(
packages=[dp],
matrix="matrix-b",
row_mapper=am,
custom_filter=lambda x: x["row"] < 3,
)
for group in mm.groups:
assert np.allclose(group.flip, expected)
def test_matrix_building_multiple_dps_no_distributions_no_arrays_replacement(
sensitivity_dps, ):
sensitivity_dps[1].metadata["sum_intra_duplicates"] = False
mm = MappedMatrix(
packages=sensitivity_dps,
matrix="matrix",
use_vectors=True,
use_arrays=False,
use_distributions=False,
)
expected = np.array([
[0, 0, 1],
[2, -1, 3],
])
assert np.allclose(mm.matrix.toarray(), expected)
def test_matrix_building_multiple_dps_no_distributions_no_arrays_summing_inter_within_package(
sensitivity_dps, ):
sensitivity_dps[1].metadata["sum_inter_duplicates"] = True
mm = MappedMatrix(
packages=sensitivity_dps,
matrix="matrix",
use_vectors=True,
use_arrays=False,
use_distributions=False,
)
expected = np.array([
[0, 0, 1],
[3, 1, 0],
])
assert np.allclose(mm.matrix.toarray(), expected)
def test_ordering():
dps = [
load_datapackage(ZipFS(dirpath / "b-second.zip")),
load_datapackage(ZipFS(dirpath / "a-first.zip")),
]
for dp in dps:
dp.rehydrate_interface("w-fourth", Interface())
print(list(dp.groups))
mm = MappedMatrix(packages=dps, matrix="matrix-a")
assert [grp.label for grp in mm.groups] == [
"y-second",
"w-fourth",
"y-second",
"w-fourth",
]
def test_one_empty_after_custom_filter():
s = bwp.create_datapackage()
s.add_persistent_vector(
matrix="foo",
data_array=np.arange(2),
indices_array=np.array([(0, 0), (1, 0)], dtype=bwp.INDICES_DTYPE),
)
s.add_persistent_vector(
matrix="foo",
data_array=np.arange(10, 12),
indices_array=np.array([(0, 1), (1, 1)], dtype=bwp.INDICES_DTYPE),
)
mm = MappedMatrix(packages=[s],
matrix="foo",
custom_filter=lambda x: x['col'] > 0)
assert mm.matrix.sum() == 21
def test_matrix_building_multiple_dps_yes_distributions_no_arrays_sum_intra_only(
sensitivity_dps, ):
mm = MappedMatrix(
packages=sensitivity_dps,
matrix="matrix",
use_vectors=True,
use_arrays=False,
use_distributions=True,
)
sample = [0.87454012, 2.45071431, 1.0, 3.23199394]
expected = np.array([
[0, 0, 1],
[2, sample[1] - sample[2], 3],
])
assert np.allclose(mm.matrix.toarray(), expected)
def test_matrix_building_multiple_dps_yes_distributions_no_arrays_sum_both_within_package(
sensitivity_dps, ):
sensitivity_dps[1].metadata["sum_inter_duplicates"] = True
mm = MappedMatrix(
packages=sensitivity_dps,
matrix="matrix",
use_vectors=True,
use_arrays=False,
use_distributions=True,
)
sample = [0.87454012, 2.45071431, 1.0, 3.23199394]
expected = np.array([
[0, 0, 1],
[2 + sample[0], sample[1] - sample[2], 3 - sample[3]],
])
assert np.allclose(mm.matrix.toarray(), expected)
def test_all_empty_after_mapping():
s = bwp.create_datapackage()
s.add_persistent_vector(
matrix="foo",
data_array=np.arange(2),
indices_array=np.array([(0, 0), (1, 0)], dtype=bwp.INDICES_DTYPE),
)
am = ArrayMapper(array=np.array([2, 3]))
mm = MappedMatrix(
packages=[s],
matrix="foo",
row_mapper=am,
)
# doesn't error out because matrix has shape from array mapper
assert not mm.matrix.sum()
assert mm.matrix.shape == (2, 1)
assert not mm.matrix.tocoo().data.sum()
def test_one_empty_after_mapping():
s = bwp.create_datapackage()
s.add_persistent_vector(
matrix="foo",
data_array=np.arange(2),
indices_array=np.array([(0, 0), (1, 0)], dtype=bwp.INDICES_DTYPE),
)
s.add_persistent_vector(
matrix="foo",
data_array=np.arange(10, 12),
indices_array=np.array([(10, 1), (11, 1)], dtype=bwp.INDICES_DTYPE),
)
am = ArrayMapper(array=np.array([10, 11]))
mm = MappedMatrix(
packages=[s],
matrix="foo",
row_mapper=am,
)
assert mm.matrix.sum() == 21
def test_matrix_building_multiple_dps_no_distributions_yes_arrays_sum_intra_only(
sensitivity_dps, ):
indices = [
191664963,
1662057957,
1405681631,
]
mm = MappedMatrix(
packages=sensitivity_dps,
matrix="matrix",
use_vectors=True,
use_arrays=True,
use_distributions=False,
)
for index in indices:
expected = np.array([
[-(index % 4 + 7), 0, 1],
[0, (index % 4 + 8), index % 100],
[2, 2 - 1, 3],
])
assert np.allclose(mm.matrix.toarray(), expected)
next(mm)
def test_no_packages_empty_ok():
mm = MappedMatrix(packages=[], matrix="foo", empty_ok=True)
assert mm.matrix.shape == (0, 0)
def test_no_useful_packages_empty_ok():
mm = MappedMatrix(packages=[diagonal()], matrix="bar", empty_ok=True)
assert mm.matrix.shape == (0, 0)
