def test_bounds(self):
"""
Tests to make sure get index is has compatible behavior as the regular numpy array
"""
original = np.arange(5**4).reshape(tuple([5] * 4))
global_offset = (100, 200, 300, 400)
no_offset_array = GlobalOffsetArray(original)
offset_array = GlobalOffsetArray(original, global_offset=global_offset)
slices_tests = [
# pre indexed sub slicing
(slice(-5, 2), slice(0, 5), slice(0, 5), slice(0, 5)),
# post indexed sub slicing
(slice(3, 8), slice(0, 5), slice(0, 5), slice(0, 5)),
# pre and post indexed mixed sub slicing
(slice(-5, 2), slice(3, 8), slice(4, 7), slice(2, 8)),
# pre to post sub slicing
(slice(-2, 8), slice(0, 5), slice(0, 5), slice(0, 5)),
# completely outside bounding box
(slice(5, 6), slice(4, 5), slice(4, 7), slice(7, 8)),
# single addressed index
(2, slice(2, 5), slice(6, 8), slice(0, 2)),
# single addressed index out of bounds
(8, slice(2, 5), slice(6, 8), slice(0, 2)),
# two addressed index but one is out of bounds
(2, 9, slice(0, 3), slice(0, 2)),
]
offset_slices_tests = [
tuple(
slice(s.start + o, s.stop + o) if isinstance(s, slice) else s +
o for s, o in zip(slices, global_offset))
for slices in slices_tests
]
for slices, offset_slices in zip(slices_tests, offset_slices_tests):
expected_exception = None
expected_value = None
try:
expected_value = original[no_wrap_slices(slices)]
except Exception as e:
expected_exception = e
if expected_exception is not None:
with pytest.raises(type(expected_exception)):
no_offset_array[slices]
with pytest.raises(type(expected_exception)):
offset_array[offset_slices]
else:
assert np.array_equal(expected_value, no_offset_array[slices])
assert np.array_equal(expected_value,
offset_array[offset_slices])
def test_bad_offset(self):
"""
Make sure error is thrown when trying to access out of bounds
"""
original = np.arange(5**4).reshape(tuple([5] * 4))
global_offset = (100, 200, 300, 400)
with pytest.raises(ValueError):
GlobalOffsetArray(original, global_offset=global_offset + (32, ))
with pytest.raises(ValueError):
GlobalOffsetArray(original, global_offset=global_offset[1:])
def test_pickle(self):
dimensions = (4, 3, 2, 1)
data = np.arange(0, np.product(dimensions)).reshape(dimensions)
global_offset = (3, 2, 1, 0)
global_offset_data = GlobalOffsetArray(data,
global_offset=global_offset)
bounds = global_offset_data.bounds()
pickled = pickle.dumps(global_offset_data)
unpickled = pickle.loads(pickled)
assert global_offset_data.data is not unpickled
assert global_offset == unpickled.global_offset
assert np.array_equal(global_offset_data.data, unpickled.data)
assert bounds == unpickled.bounds()
def test_autofill_dimensions(self):
dimensions = (4, 3, 2, 1)
data = np.arange(0, np.product(dimensions)).reshape(dimensions)
global_offset_data = GlobalOffsetArray(data,
global_offset=(3, 2, 1, 0))
assert np.array_equal(global_offset_data[5], data[2])
assert np.array_equal(global_offset_data[5, 3], data[2, 1])
def test_broadcast(self):
larger = GlobalOffsetArray(
np.array(tuple(np.ones((3, 3)) * i for i in range(1, 4))))
smaller = np.ones((3, 3)) * 2
result = larger * smaller
assert np.all(result[0] == 2)
assert np.all(result[1] == 4)
assert np.all(result[2] == 6)
result = smaller * larger
assert np.all(result[0] == 2)
assert np.all(result[1] == 4)
assert np.all(result[2] == 6)
larger *= smaller
assert np.all(larger[0] == 2)
assert np.all(larger[1] == 4)
assert np.all(larger[2] == 6)
with pytest.raises(ValueError):
smaller *= larger
def test_overlap_slices_3d(self):
bounds = (slice(0, 7), slice(0, 7), slice(0, 7))
chunk_shape = (3, 3, 3)
overlap = (1, 1, 1)
block = Block(bounds=bounds, chunk_shape=chunk_shape, overlap=overlap)
assert block.num_chunks == (3, 3, 3)
fake_data = GlobalOffsetArray(np.zeros(block.shape),
global_offset=(0, 0, 0))
for chunk in block.chunk_iterator((1, 0, 1)):
for edge_slice in block.overlap_slices(chunk):
fake_data[edge_slice] += 1
fake_data[block.core_slices(chunk)] += 1
assert fake_data.sum() == np.product(fake_data.shape)
def test_slice_fill_missing_dimensions(self):
for test_array in TEST_ARRAYS[:]:
sub_slices = tuple(
slice(0, dim // 2) for dim in test_array.shape[1:])
offset_array = 0 + GlobalOffsetArray(test_array)
sub_offset_array = 0 + offset_array[sub_slices]
assert sub_offset_array.global_offset == offset_array.global_offset
def test_get_no_offset(self):
"""
Make sure all arrays are properly equivalent when no offset is given
"""
for test_array in TEST_ARRAYS:
offset_array = GlobalOffsetArray(test_array)
assert offset_array.global_offset == tuple([0] * test_array.ndim)
assert np.array_equal(test_array, offset_array)
def test_get_offset_origin(self):
"""
Make sure all arrays are properly equivalent when offset of the origin is given
"""
for test_array in TEST_ARRAYS:
offset_array = GlobalOffsetArray(
test_array,
global_offset=tuple(0 for _ in range(0, test_array.ndim)))
assert offset_array.global_offset == tuple([0] * test_array.ndim)
assert np.array_equal(test_array, offset_array)
self.recurse_compare(test_array, offset_array, test_array.shape)
def test_slice_same_dimensions(self):
for test_array in TEST_ARRAYS:
sub_slices = tuple(slice(0, dim // 2) for dim in test_array.shape)
offset_array = GlobalOffsetArray(test_array)
sub_test_array = test_array[sub_slices]
sub_offset_array = offset_array[sub_slices]
assert sub_offset_array.shape == sub_test_array.shape
assert len(sub_offset_array.global_offset) == len(
sub_test_array.shape)
def generate_data(self, ndim, length):
"""
Generate test data
"""
original = np.arange(1,
length**ndim + 1).reshape(tuple([length] * ndim))
copy = original.copy()
global_offset = tuple(dimension * 100
for dimension in range(1, ndim + 1))
offset_array = GlobalOffsetArray(original, global_offset=global_offset)
return (copy, offset_array)
def test_no_wrap(self):
"""
Make sure index ranges before start does not perform wrap around
"""
def test_recurse_slices(expected, actual, index, shape, slices):
if index == len(shape):
normalized_slices = no_wrap_slices(slices)
assert np.array_equal(expected[normalized_slices], actual[slices]), \
'Incorrect value found requested slices: %s, normalized slices: %s, actual: %s, expected: %s' % (
slices, normalized_slices, actual[slices], expected[normalized_slices])
else:
dim = shape[index]
for start in range(-dim, dim):
for stop in range(start, dim * 2):
test_recurse_slices(expected, actual, index + 1, shape,
(slices + (slice(start, stop), )))
for test_array in TEST_ARRAYS:
offset_array = GlobalOffsetArray(test_array)
test_recurse_slices(test_array, offset_array, 0,
offset_array.shape, ())
def generate_replacement(self, ndim, length, global_offset):
"""
"""
# Test with regulard ndarray are properly set into the offset_array
replacement_length = floor(length / 2)
replacement = np.arange(1, replacement_length**ndim + 1).reshape(
tuple([replacement_length] * ndim))
# replace global offset array with new replaced value
replacement_slice = ()
offset_replace_slice = ()
replacement_offset = ()
for offset in global_offset:
replacement_slice += (slice(replacement_length,
replacement_length * 2), )
offset_replace_slice += (slice(replacement_length + offset,
replacement_length * 2 + offset), )
replacement_offset += (replacement_length + offset, )
replacement = GlobalOffsetArray(replacement,
global_offset=replacement_offset)
return (replacement_slice, offset_replace_slice, replacement)
def test_get_with_offset(self):
"""
Make sure all global_offset_arrays are equivalent when given offset the proper offset indices.
"""
for test_array in TEST_ARRAYS:
# set offset at each dimension to the dimension index + 1
offset = tuple(
[index + 1 for index in range(0, len(test_array.shape))])
shape = test_array.shape
offset_array = GlobalOffsetArray(test_array, global_offset=offset)
assert offset_array.global_offset == offset
test_slices = tuple(
slice(0, shape[dimension])
for dimension in range(0, test_array.ndim))
# same as test_slices but at offset
offset_slices = tuple(
slice(test_slice.start + offset[dimension], test_slice.stop +
offset[dimension])
for dimension, test_slice in enumerate(test_slices))
sliced_offset_array = offset_array[offset_slices]
assert np.array_equal(test_array[test_slices], sliced_offset_array)
self.recurse_compare(test_array[test_slices], sliced_offset_array,
test_array.shape)
def test_subarray(self):
"""
Make sure subarrays of contain the correct adjusted global_offset and a copy is returned
"""
def to_offsets(slice_or_indices):
return tuple(
slice(s.start + o, s.stop + o) if isinstance(s, slice) else s +
o for s, o in zip(original_index, global_offset))
original = np.arange(5**4).reshape(tuple([5] * 4))
global_offset = (100, 200, 300, 400)
offset_array = GlobalOffsetArray(original, global_offset=global_offset)
# test slice with only slices
original_index = (slice(0, 2), slice(2, 5), slice(3, 5), slice(0, 3))
offset_index = to_offsets(original_index)
sub_array = offset_array[offset_index]
assert np.array_equal(sub_array, original[original_index])
assert sub_array.global_offset == (100, 202, 303, 400)
assert np.array_equal(sub_array[offset_index],
offset_array[offset_index])
# ensure that returned sub_array is actually a view
sub_array[sub_array.global_offset] = 1337
assert offset_array[sub_array.global_offset] == 1337
# test slice with some slices some fixed
original_index = (slice(0, 2), 3, slice(3, 5), slice(0, 3))
offset_index = to_offsets(original_index)
sub_array = offset_array[offset_index]
assert np.array_equal(original[original_index], sub_array)
assert sub_array.global_offset == (100, 303, 400)
assert np.array_equal(
sub_array[tuple(s for s in offset_index if isinstance(s, slice))],
offset_array[offset_index])
def get(i):
return GlobalOffsetArray(np.zeros((length, ) * dim),
global_offset=global_offsets[i])
def test_standard_operators(self):
"""
Tests that standard operators will only work when:
* global offset and size are same for both operands
* one operand is fully encapsulated by another (returns a copy of the larger with the smaller added)
All other cases should throw errors
"""
ndim = 5
length = 4
all_slices = (slice(None, None), ) * ndim
# test all operators
for op in STANDARD_OPERATORS:
# test operation commutativity
for forward in [True, False]:
(original,
fixed_operand_template) = self.generate_data(ndim, length)
# test different sizes
for shape in [
fixed_operand_template.shape,
tuple(s * 2 for s in fixed_operand_template.shape),
tuple(s // 2 + 1 for s in fixed_operand_template.shape)
]:
# test different overlap offsets
for offset_of_offset in [-2, 0, 2, 100]:
fixed_operand = fixed_operand_template.copy()
# offset array used as other operand for operator
offsetted_operand = GlobalOffsetArray(
np.arange(np.prod(shape),
dtype=fixed_operand.dtype).reshape(shape)
* 10 + 1,
global_offset=tuple(
offset_of_offset + offset
for offset in fixed_operand.global_offset))
expected_error = expected_result = None
actual_error = actual_result = None
fixed_in_offsetted = fixed_operand.is_contained_within(
offsetted_operand)
offsetted_in_fixed = offsetted_operand.is_contained_within(
fixed_operand)
# Determine the slices when one is fully encapsulated by the other
fixed_slices = tuple(
slice(offset_of_offset, offset_of_offset + s)
for s in offsetted_operand.shape)
offsetted_slices = tuple(
slice(-1 *
offset_of_offset, -1 * offset_of_offset + s)
for s in fixed_operand.shape)
if fixed_in_offsetted and not offsetted_in_fixed:
sub_slices = offsetted_slices
expected_result = offsetted_operand.view(
np.ndarray).copy()
elif not fixed_in_offsetted and offsetted_in_fixed:
sub_slices = fixed_slices
expected_result = original.view(np.ndarray).copy()
else:
# either exactly the same or separate slices
fixed_slices = offsetted_slices = all_slices
sub_slices = all_slices
# swap operands around on forward flag
if forward:
left_expected = original.copy()
left_actual = fixed_operand
left_slices = fixed_slices
right_expected = offsetted_operand.view(np.ndarray)
right_actual = offsetted_operand
right_slices = offsetted_slices
else:
left_expected = offsetted_operand.view(
np.ndarray).copy()
left_actual = offsetted_operand
left_slices = offsetted_slices
right_expected = original.view(np.ndarray)
right_actual = fixed_operand
right_slices = fixed_slices
left_slices = no_wrap_slices(left_slices)
right_slices = no_wrap_slices(right_slices)
sub_slices = no_wrap_slices(sub_slices)
# Condition to autofail on partial overlaps
if not fixed_in_offsetted and not offsetted_in_fixed:
with pytest.raises(ValueError):
op(left_actual, right_actual)
continue
try:
interim_result = op(left_expected[left_slices],
right_expected[right_slices])
except Exception as e:
expected_error = e
if expected_result is not None:
expected_result = expected_result.astype(
interim_result.dtype)
expected_result[sub_slices] = interim_result
else:
expected_result = interim_result
try:
actual_result = op(left_actual, right_actual)
except Exception as e:
actual_error = e
assert expected_error is None == actual_error is None, \
'Expected error: (%s) Actual error: (%s)' % (expected_error, actual_error) # noqa E711
# ensure global_offset is preserved
assert hasattr(actual_result, 'global_offset')
assert tuple(offset_of_offset + o
for o in fixed_operand.global_offset
) == actual_result.global_offset
# ensure actual results match that of a regular ndarray
assert np.array_equal(expected_result, actual_result)
# ensure the results behave like ndarray to return a copy of the array instead of a view by
# testing that the original arrays were not modified
actual_result[actual_result.global_offset] = 1337
assert np.all(fixed_operand != 1337)
def test_in_place_operators(self):
"""
Tests that in-place operators will only work when:
* global offset and size are same for both operands
* one operand is fully encapsulated by another (returns a copy of the larger with the smaller added)
All other cases should throw errors
"""
ndim = 5
length = 4
# test all operators
for op in IN_PLACE_OPERATORS:
# test operation commutativity
for forward in [True, False]:
(original,
fixed_operand_template) = self.generate_data(ndim, length)
# in place only works with floats because we can't mix int and float into the same ndarray
if op == operator.itruediv:
original = original.astype(np.float64)
fixed_operand_template = fixed_operand_template.astype(
np.float64)
# test different sizes
for shape in [
fixed_operand_template.shape,
tuple(s * 2 for s in fixed_operand_template.shape),
tuple(s // 2 + 1 for s in fixed_operand_template.shape)
]:
# test different overlap offsets
for offset_of_offset in [-2, 0, 2, 100]:
fixed_operand = fixed_operand_template.copy()
# offset array used as other operand for operator
offsetted_operand = GlobalOffsetArray(
np.arange(np.prod(shape),
dtype=fixed_operand.dtype).reshape(shape)
* 10 + 1,
global_offset=tuple(
offset_of_offset + offset
for offset in fixed_operand.global_offset))
expected_error = expected_result = None
actual_error = actual_result = None
fixed_in_offsetted = fixed_operand.is_contained_within(
offsetted_operand)
offsetted_in_fixed = offsetted_operand.is_contained_within(
fixed_operand)
fixed_slices = tuple(
slice(offset_of_offset, offset_of_offset + s)
for s in offsetted_operand.shape)
offsetted_slices = tuple(
slice(-1 *
offset_of_offset, -1 * offset_of_offset + s)
for s in fixed_operand.shape)
# swap operands around on forward flag
if forward:
left_expected = original.copy()
left_actual = fixed_operand
left_slices = fixed_slices
right_expected = offsetted_operand.view(np.ndarray)
right_actual = offsetted_operand
right_slices = offsetted_slices
sub_slices = fixed_slices
else:
left_expected = offsetted_operand.view(
np.ndarray).copy()
left_actual = offsetted_operand
left_slices = offsetted_slices
right_expected = original.view(np.ndarray)
right_actual = fixed_operand
right_slices = fixed_slices
sub_slices = offsetted_slices
expected_result = left_expected
left_slices = no_wrap_slices(left_slices)
right_slices = no_wrap_slices(right_slices)
sub_slices = no_wrap_slices(sub_slices)
# Condition to autofail on disjoint
if not fixed_in_offsetted and not offsetted_in_fixed and shape[
0] > length * 2:
with pytest.raises(ValueError):
op(left_actual, right_actual)
continue
interim_result = op(left_expected[left_slices],
right_expected[right_slices])
expected_result = expected_result.astype(
interim_result.dtype)
expected_result[sub_slices] = interim_result
try:
actual_result = op(left_actual, right_actual)
except Exception as e:
actual_error = e
assert expected_error is None == actual_error is None, \
'Expected error: (%s) Actual error: (%s)' % (expected_error, actual_error) # noqa E711
# ensure global_offset is preserved
assert hasattr(actual_result, 'global_offset')
assert left_actual.global_offset == actual_result.global_offset
# ensure actual results match that of a regular ndarray
assert np.array_equal(expected_result, actual_result)
# ensure that in place modifies the actual result
actual_result[actual_result.global_offset] = 1337
assert 1337 == left_actual[actual_result.global_offset]
def test_aggregate_function(self):
for test_array in TEST_ARRAYS:
offset_array = GlobalOffsetArray(test_array)
assert type(offset_array.sum()) == type(test_array.sum()) #noqa