def test_dont_fuse_fancy_indexing_in_getarray_nofancy():
dsk = {'a': (getitem, (getarray_nofancy, 'x', (slice(10, 20, None), slice(100, 200, None))),
([1, 3], slice(50, 60, None)))}
assert optimize_slices(dsk) == dsk
dsk = {'a': (getitem, (getarray_nofancy, 'x', [1, 2, 3]), 0)}
assert optimize_slices(dsk) == dsk
def test_dont_fuse_fancy_indexing_in_getter_nofancy():
dsk = {'a': (getitem, (getter_nofancy, 'x', (slice(10, 20, None), slice(100, 200, None))),
([1, 3], slice(50, 60, None)))}
assert optimize_slices(dsk) == dsk
dsk = {'a': (getitem, (getter_nofancy, 'x', [1, 2, 3]), 0)}
assert optimize_slices(dsk) == dsk
def test_dont_fuse_fancy_indexing_in_getter_nofancy(getitem, getter_nofancy):
dsk = {
"a": (
getitem,
(getter_nofancy, "x", (slice(10, 20, None), slice(100, 200, None))),
([1, 3], slice(50, 60, None)),
)
}
_assert_getter_dsk_eq(optimize_slices(dsk), dsk)
dsk = {"a": (getitem, (getter_nofancy, "x", [1, 2, 3]), 0)}
_assert_getter_dsk_eq(optimize_slices(dsk), dsk)
def test_fuse_getitem():
pairs = [
((getarray, (getarray, 'x', slice(1000, 2000)), slice(15, 20)),
(getarray, 'x', slice(1015, 1020))),
((getitem, (getarray, 'x', (slice(1000, 2000), slice(100, 200))),
(slice(15, 20), slice(50, 60))), (getarray, 'x', (slice(1015, 1020),
slice(150, 160)))),
((getarray, (getarray, 'x', slice(1000,
2000)), 10), (getarray, 'x', 1010)),
((getitem, (getarray, 'x', (slice(1000, 2000), 10)),
(slice(15, 20), )), (getarray, 'x', (slice(1015, 1020), 10))),
((getarray, (getarray, 'x', (10, slice(1000, 2000))),
(slice(15, 20), )), (getarray, 'x', (10, slice(1015, 1020)))),
((getarray, (getarray, 'x', (slice(1000, 2000), slice(100, 200))),
(slice(None, None), slice(50,
60))), (getarray, 'x', (slice(1000, 2000),
slice(150, 160)))),
((getarray, (getarray, 'x', (None, slice(None, None))),
(slice(None, None), 5)), (getarray, 'x', (None, 5))),
((getarray, (getarray, 'x', (slice(1000, 2000), slice(10, 20))),
(slice(5, 10), )), (getarray, 'x', (slice(1005, 1010), slice(10,
20)))),
((getitem, (getitem, 'x', (slice(1000, 2000), )),
(slice(5, 10), slice(10, 20))), (getitem, 'x', (slice(1005, 1010),
slice(10, 20))))
]
for inp, expected in pairs:
result = optimize_slices({'y': inp})
assert result == {'y': expected}
def test_dont_remove_no_op_slices_for_getter_or_getter_nofancy(
which, getitem, getter, getter_nofancy
):
# Test that no-op slices are *not* removed when using getter or
# getter_nofancy. This ensures that `get` calls are always made in all
# tasks created by `from_array`, even after optimization
# Pytest doesn't make it easy to parameterize over parameterized fixtures
if which == "getter":
get = getter
else:
get = getter_nofancy
null = slice(0, None)
opts = [
(
(get, "x", null, False, False),
(get, "x", null, False, False),
),
(
(getitem, (get, "x", null, False, False), null),
(get, "x", null, False, False),
),
(
(getitem, (get, "x", (null, null), False, False), ()),
(get, "x", (null, null), False, False),
),
]
for orig, final in opts:
_assert_getter_dsk_eq(optimize_slices({"a": orig}), {"a": final})
def test_hard_fuse_slice_cases():
dsk = {
"x": (getter, (getter, "x", (None, slice(None,
None))), (slice(None,
None), 5))
}
assert optimize_slices(dsk) == {"x": (getter, "x", (None, 5))}
def test_hard_fuse_slice_cases():
dsk = {
'x':
(getarray, (getarray, 'x', (None, slice(None,
None))), (slice(None,
None), 5))
}
assert optimize_slices(dsk) == {'x': (getarray, 'x', (None, 5))}
def test_optimize_slicing():
dsk = {'a': (range, 10),
'b': (getarray, 'a', (slice(None, None, None),)),
'c': (getarray, 'b', (slice(None, None, None),)),
'd': (getarray, 'c', (slice(0, 5, None),)),
'e': (getarray, 'd', (slice(None, None, None),))}
expected = {'e': (getarray, (range, 10), (slice(0, 5, None),))}
result = optimize_slices(fuse(dsk, [])[0])
assert result == expected
# protect output keys
expected = {'c': (getarray, (range, 10), (slice(0, None, None),)),
'd': (getarray, 'c', (slice(0, 5, None),)),
'e': (getarray, 'd', (slice(None, None, None),))}
result = optimize_slices(fuse(dsk, ['c', 'd', 'e'])[0])
assert result == expected
def test_fuse_getitem():
pairs = [((getter, (getter, 'x', slice(1000, 2000)), slice(15, 20)),
(getter, 'x', slice(1015, 1020))),
((getitem, (getter, 'x', (slice(1000, 2000), slice(100, 200))),
(slice(15, 20), slice(50, 60))),
(getter, 'x', (slice(1015, 1020), slice(150, 160)))),
((getitem, (getter_nofancy, 'x', (slice(1000, 2000), slice(100, 200))),
(slice(15, 20), slice(50, 60))),
(getter_nofancy, 'x', (slice(1015, 1020), slice(150, 160)))),
((getter, (getter, 'x', slice(1000, 2000)), 10),
(getter, 'x', 1010)),
((getitem, (getter, 'x', (slice(1000, 2000), 10)),
(slice(15, 20),)),
(getter, 'x', (slice(1015, 1020), 10))),
((getitem, (getter_nofancy, 'x', (slice(1000, 2000), 10)),
(slice(15, 20),)),
(getter_nofancy, 'x', (slice(1015, 1020), 10))),
((getter, (getter, 'x', (10, slice(1000, 2000))),
(slice(15, 20), )),
(getter, 'x', (10, slice(1015, 1020)))),
((getter, (getter, 'x', (slice(1000, 2000), slice(100, 200))),
(slice(None, None), slice(50, 60))),
(getter, 'x', (slice(1000, 2000), slice(150, 160)))),
((getter, (getter, 'x', (None, slice(None, None))),
(slice(None, None), 5)),
(getter, 'x', (None, 5))),
((getter, (getter, 'x', (slice(1000, 2000), slice(10, 20))),
(slice(5, 10),)),
(getter, 'x', (slice(1005, 1010), slice(10, 20)))),
((getitem, (getitem, 'x', (slice(1000, 2000),)),
(slice(5, 10), slice(10, 20))),
(getitem, 'x', (slice(1005, 1010), slice(10, 20)))),
((getter, (getter, 'x', slice(1000, 2000), False, False), slice(15, 20)),
(getter, 'x', slice(1015, 1020))),
((getter, (getter, 'x', slice(1000, 2000)), slice(15, 20), False, False),
(getter, 'x', slice(1015, 1020))),
((getter, (getter_nofancy, 'x', slice(1000, 2000), False, False),
slice(15, 20), False, False),
(getter_nofancy, 'x', slice(1015, 1020), False, False)),
]
for inp, expected in pairs:
result = optimize_slices({'y': inp})
assert result == {'y': expected}
def test_remove_no_op_slices_for_getitem(getitem):
null = slice(0, None)
opts = [
((getitem, "x", null, False, False), "x"),
((getitem, (getitem, "x", null, False, False), null), "x"),
((getitem, (getitem, "x", (null, null), False, False), ()), "x"),
]
for orig, final in opts:
_assert_getter_dsk_eq(optimize_slices({"a": orig}), {"a": final})
def test_optimize_slicing():
dsk = {'a': (range, 10),
'b': (getter, 'a', (slice(None, None, None),)),
'c': (getter, 'b', (slice(None, None, None),)),
'd': (getter, 'c', (slice(0, 5, None),)),
'e': (getter, 'd', (slice(None, None, None),))}
expected = {'e': (getter, (range, 10), (slice(0, 5, None),))}
result = optimize_slices(fuse(dsk, [], rename_keys=False)[0])
assert result == expected
# protect output keys
expected = {'c': (getter, (range, 10), (slice(0, None, None),)),
'd': (getter, 'c', (slice(0, 5, None),)),
'e': (getter, 'd', (slice(None, None, None),))}
result = optimize_slices(fuse(dsk, ['c', 'd', 'e'], rename_keys=False)[0])
assert result == expected
def test_fuse_getitem_lock():
lock1 = SerializableLock()
lock2 = SerializableLock()
pairs = [
(
(getter, (getter, "x", slice(1000,
2000), True, lock1), slice(15, 20)),
(getter, "x", slice(1015, 1020), True, lock1),
),
(
(
getitem,
(getter, "x", (slice(1000, 2000), slice(100,
200)), True, lock1),
(slice(15, 20), slice(50, 60)),
),
(getter, "x", (slice(1015, 1020), slice(150, 160)), True, lock1),
),
(
(
getitem,
(
getter_nofancy,
"x",
(slice(1000, 2000), slice(100, 200)),
True,
lock1,
),
(slice(15, 20), slice(50, 60)),
),
(getter_nofancy, "x", (slice(1015,
1020), slice(150, 160)), True, lock1),
),
(
(
getter,
(getter, "x", slice(1000, 2000), True, lock1),
slice(15, 20),
True,
lock2,
),
(
getter,
(getter, "x", slice(1000, 2000), True, lock1),
slice(15, 20),
True,
lock2,
),
),
]
for inp, expected in pairs:
result = optimize_slices({"y": inp})
assert result == {"y": expected}
def test_remove_no_op_slices_if_get_is_not_getter_or_getter_nofancy(get, remove):
# Test that no-op slices are removed as long as get is not getter or
# getter_nofancy. This ensures that `get` calls are always made in all
# tasks created by `from_array`, even after optimization
null = slice(0,None)
opts = [((get, 'x', null, False, False),
'x' if remove else (get, 'x', null, False, False)),
((getitem, (get, 'x', null, False, False), null),
'x' if remove else (get, 'x', null, False, False)),
((getitem, (get, 'x', (null, null), False, False), ()),
'x' if remove else (get, 'x', (null, null), False, False))]
for orig, final in opts:
assert optimize_slices({'a': orig}) == {'a': final}
def test_remove_no_op_slices_if_get_is_not_getter_or_getter_nofancy(get, remove):
# Test that no-op slices are removed as long as get is not getter or
# getter_nofancy. This ensures that `get` calls are always made in all
# tasks created by `from_array`, even after optimization
null = slice(0,None)
opts = [((get, 'x', null, False, False),
'x' if remove else (get, 'x', null, False, False)),
((getitem, (get, 'x', null, False, False), null),
'x' if remove else (get, 'x', null, False, False)),
((getitem, (get, 'x', (null, null), False, False), ()),
'x' if remove else (get, 'x', (null, null), False, False))]
for orig, final in opts:
assert optimize_slices({'a': orig}) == {'a': final}
def test_optimize_slicing():
dsk = {
"a": (range, 10),
"b": (getter, "a", (slice(None, None, None), )),
"c": (getter, "b", (slice(None, None, None), )),
"d": (getter, "c", (slice(0, 5, None), )),
"e": (getter, "d", (slice(None, None, None), )),
}
expected = {"e": (getter, (range, 10), (slice(0, 5, None), ))}
result = optimize_slices(fuse(dsk, [], rename_keys=False)[0])
assert result == expected
# protect output keys
expected = {
"c": (getter, (range, 10), (slice(0, None, None), )),
"d": (getter, "c", (slice(0, 5, None), )),
"e": (getter, "d", (slice(None, None, None), )),
}
result = optimize_slices(fuse(dsk, ["c", "d", "e"], rename_keys=False)[0])
assert result == expected
def test_fuse_getitem_lock():
lock1 = SerializableLock()
lock2 = SerializableLock()
pairs = [((getter, (getter, 'x', slice(1000, 2000), True, lock1), slice(15, 20)),
(getter, 'x', slice(1015, 1020), True, lock1)),
((getitem, (getter, 'x', (slice(1000, 2000), slice(100, 200)), True, lock1),
(slice(15, 20), slice(50, 60))),
(getter, 'x', (slice(1015, 1020), slice(150, 160)), True, lock1)),
((getitem, (getter_nofancy, 'x', (slice(1000, 2000), slice(100, 200)), True, lock1),
(slice(15, 20), slice(50, 60))),
(getter_nofancy, 'x', (slice(1015, 1020), slice(150, 160)), True, lock1)),
((getter, (getter, 'x', slice(1000, 2000), True, lock1), slice(15, 20), True, lock2),
(getter, (getter, 'x', slice(1000, 2000), True, lock1), slice(15, 20), True, lock2))]
for inp, expected in pairs:
result = optimize_slices({'y': inp})
assert result == {'y': expected}
def test_fuse_getitem_lock():
lock1 = SerializableLock()
lock2 = SerializableLock()
pairs = [((getarray, (getarray, 'x', slice(1000, 2000), lock1),
slice(15, 20)), (getarray, 'x', slice(1015, 1020), lock1)),
((getitem, (getarray, 'x', (slice(1000, 2000), slice(100, 200)),
lock1), (slice(15, 20), slice(50, 60))),
(getarray, 'x', (slice(1015, 1020), slice(150, 160)), lock1)),
((getitem, (getarray_nofancy, 'x', (slice(1000,
2000), slice(100, 200)),
lock1), (slice(15, 20), slice(50, 60))),
(getarray_nofancy, 'x', (slice(1015, 1020), slice(150,
160)), lock1)),
((getarray, (getarray, 'x', slice(1000,
2000), lock1), slice(15, 20),
lock2), (getarray, (getarray, 'x', slice(1000, 2000), lock1),
slice(15, 20), lock2))]
for inp, expected in pairs:
result = optimize_slices({'y': inp})
assert result == {'y': expected}
def test_hard_fuse_slice_cases():
dsk = {'x': (getarray, (getarray, 'x', (None, slice(None, None))),
(slice(None, None), 5))}
assert optimize_slices(dsk) == {'x': (getarray, 'x', (None, 5))}
def test_fuse_getitem():
pairs = [
(
(getter, (getter, "x", slice(1000, 2000)), slice(15, 20)),
(getter, "x", slice(1015, 1020)),
),
(
(
getitem,
(getter, "x", (slice(1000, 2000), slice(100, 200))),
(slice(15, 20), slice(50, 60)),
),
(getter, "x", (slice(1015, 1020), slice(150, 160))),
),
(
(
getitem,
(getter_nofancy, "x", (slice(1000, 2000), slice(100, 200))),
(slice(15, 20), slice(50, 60)),
),
(getter_nofancy, "x", (slice(1015, 1020), slice(150, 160))),
),
((getter, (getter, "x", slice(1000, 2000)), 10), (getter, "x", 1010)),
(
(getitem, (getter, "x", (slice(1000, 2000), 10)), (slice(15,
20), )),
(getter, "x", (slice(1015, 1020), 10)),
),
(
(getitem, (getter_nofancy, "x", (slice(1000,
2000), 10)), (slice(15,
20), )),
(getter_nofancy, "x", (slice(1015, 1020), 10)),
),
(
(getter, (getter, "x", (10, slice(1000, 2000))), (slice(15,
20), )),
(getter, "x", (10, slice(1015, 1020))),
),
(
(
getter,
(getter, "x", (slice(1000, 2000), slice(100, 200))),
(slice(None, None), slice(50, 60)),
),
(getter, "x", (slice(1000, 2000), slice(150, 160))),
),
(
(getter, (getter, "x", (None, slice(None,
None))), (slice(None,
None), 5)),
(getter, "x", (None, 5)),
),
(
(
getter,
(getter, "x", (slice(1000, 2000), slice(10, 20))),
(slice(5, 10), ),
),
(getter, "x", (slice(1005, 1010), slice(10, 20))),
),
(
(
getitem,
(getitem, "x", (slice(1000, 2000), )),
(slice(5, 10), slice(10, 20)),
),
(getitem, "x", (slice(1005, 1010), slice(10, 20))),
),
(
(getter, (getter, "x", slice(1000,
2000), False, False), slice(15, 20)),
(getter, "x", slice(1015, 1020)),
),
(
(getter, (getter, "x", slice(1000,
2000)), slice(15, 20), False, False),
(getter, "x", slice(1015, 1020)),
),
(
(
getter,
(getter_nofancy, "x", slice(1000, 2000), False, False),
slice(15, 20),
False,
False,
),
(getter_nofancy, "x", slice(1015, 1020), False, False),
),
]
for inp, expected in pairs:
result = optimize_slices({"y": inp})
assert result == {"y": expected}
def test_hard_fuse_slice_cases(getter):
dsk = {
"x": (getter, (getter, "x", (None, slice(None, None))), (slice(None, None), 5))
}
_assert_getter_dsk_eq(optimize_slices(dsk), {"x": (getter, "x", (None, 5))})