def test_highlevel(): array = awkward1.virtual(lambda: [[1.1, 2.2, 3.3], [], [4.4, 5.5]]) assert isinstance(array.layout, awkward1.layout.VirtualArray) assert awkward1.to_list(array) == [[1.1, 2.2, 3.3], [], [4.4, 5.5]] counter = [0] def generate(): counter[0] += 1 return [[1.1, 2.2, 3.3], [], [4.4, 5.5]] array = awkward1.virtual(generate, length=3, form={ "class": "ListOffsetArray64", "offsets": "i64", "content": "float64" }) assert counter[0] == 0 assert len(array) == 3 assert counter[0] == 0 assert str(awkward1.type(array)) == "3 * var * float64" assert counter[0] == 0 assert awkward1.to_list(array[2]) == [4.4, 5.5] assert counter[0] == 1
def test_cache_chain(): cache1 = {} one = awkward1.virtual(lambda: [[1.1, 2.2, 3.3], [], [4.4, 5.5]], cache=cache1, length=3) two = awkward1.virtual(lambda: [100, 200, 300], cache=cache1, length=3) array1 = awkward1.zip({"x": one, "y": two}, depth_limit=1) assert len(cache1) == 0 cache2 = {} array2 = awkward1.with_cache(array1, cache2) assert awkward1.to_list(array2["x"]) == [[1.1, 2.2, 3.3], [], [4.4, 5.5]] assert len(cache1) == 0 assert len(cache2) == 1 array3 = awkward1.with_cache(array2, cache1, chain="first") assert awkward1.to_list(array3) == [{ "x": [1.1, 2.2, 3.3], "y": 100 }, { "x": [], "y": 200 }, { "x": [4.4, 5.5], "y": 300 }] assert len(cache1) == 1 assert len(cache2) == 1
def test_virtual_slice_numba(): numba = pytest.importorskip("numba") materialize_count = defaultdict(int) def gen(x, name): materialize_count[name] += 1 return x x1 = awkward1.Array([1, 2, 3, 4, 5]) x2 = awkward1.Array([1, 2, 3, 4, 5]) x = awkward1.zip({"x1": x1, "x2": x2}, with_name="xthing") xv = awkward1.virtual(lambda: gen(x, "x"), length=len(x), form=x.layout.form) y = x1 * 10. yv = awkward1.virtual(lambda: gen(y, "y"), length=len(y), form=y.layout.form) array = awkward1.zip({"x": xv, "y": yv}, with_name="Point", depth_limit=1) virtual = awkward1.virtual(lambda: gen(array, "array"), length=len(array), form=array.layout.form) @numba.njit def dostuff(array): x = 0 for item in array: x += item return x assert dostuff(virtual.x.x1) == 15 assert dict(materialize_count) == {"x": 3, "array": 3} materialize_count.clear() @numba.njit def dostuff(array): x = 0 for item in array: x += item.x.x1 return x assert dostuff(virtual[["x"]]) == 15 assert dict(materialize_count) == {"x": 1, "array": 2} materialize_count.clear() assert dostuff(virtual[::2]) == 9 assert dict(materialize_count) == {"x": 1, "array": 2} materialize_count.clear() assert dostuff(virtual[:3]) == 6 assert dict(materialize_count) == {"x": 1, "array": 1} materialize_count.clear() slicedvirtual = virtual[awkward1.Array([True, False, False, True, False])] assert dostuff(slicedvirtual) == 5 assert dict(materialize_count) == {"x": 1, "array": 2} materialize_count.clear()
def test_virtual_record(): materialize_count = defaultdict(int) def gen(x, name): materialize_count[name] += 1 return x x1 = awkward1.Array([1, 2, 3, 4, 5]) x2 = awkward1.Array([1, 2, 3, 4, 5]) x = awkward1.zip({"x1": x1, "x2": x2}, with_name="xthing") assert x.layout.purelist_parameter("__record__") == "xthing" xv = awkward1.virtual(lambda: gen(x, "x"), length=len(x), form=x.layout.form) assert xv.layout.purelist_parameter("__record__") == "xthing" y = x1 * 10. yv = awkward1.virtual(lambda: gen(y, "y"), length=len(y), form=y.layout.form) array = awkward1.zip({"x": xv, "y": yv}, with_name="Point", depth_limit=1) assert array.layout.purelist_parameter("__record__") == "Point" virtual = awkward1.virtual(lambda: gen(array, "array"), length=len(array), form=array.layout.form) assert virtual.layout.purelist_parameter("__record__") == "Point" assert len(materialize_count) == 0 slicedvirtual = virtual.x assert len(materialize_count) == 0 assert slicedvirtual.layout.purelist_parameter("__record__") == "xthing" assert len(materialize_count) == 0 slicedvirtual = virtual[["x", "y"]] assert len(materialize_count) == 0 assert slicedvirtual.layout.purelist_parameter("__record__") == None assert len(materialize_count) == 0 slicedvirtual = virtual[::2] assert len(materialize_count) == 0 assert slicedvirtual.layout.purelist_parameter("__record__") == "Point" assert len(materialize_count) == 0 slicedvirtual = virtual[:3] assert len(materialize_count) == 0 assert slicedvirtual.layout.purelist_parameter("__record__") == "Point" assert len(materialize_count) == 0 slicedvirtual = virtual[awkward1.Array([True, False, False, True, False])] assert len(materialize_count) == 0 assert slicedvirtual.layout.purelist_parameter("__record__") == "Point" assert len(materialize_count) == 0
def test_0230(): rec = awkward1.zip( { "x": awkward1.virtual(lambda: awkward1.Array([1, 2, 3, 4]), length=4), }, depth_limit=1) assert awkward1.to_list(rec.x[1:]) == [2, 3, 4] assert awkward1.to_list(rec.x[1:] * 2) == [4, 6, 8]
def test(): example = awkward1.Array([[{"x": 1, "y": []}, {"x": 2, "y": [1, 1]}], []]) cache = {} virtualarray = awkward1.virtual(lambda: example, form=example.layout.form, length=len(example)) assert len(cache) == 0 tmp1 = virtualarray["x"] assert json.loads(str(tmp1.layout.form)) == { "class": "VirtualArray", "form": { "class": "ListOffsetArray64", "offsets": "i64", "content": "int64" }, "has_length": True } assert len(cache) == 0 tmp2 = virtualarray["y"] assert json.loads(str(tmp2.layout.form)) == { "class": "VirtualArray", "form": { "class": "ListOffsetArray64", "offsets": "i64", "content": { "class": "ListOffsetArray64", "offsets": "i64", "content": "int64" } }, "has_length": True } assert len(cache) == 0 assert tmp1.tolist() == [[1, 2], []] assert tmp2.tolist() == [[[], [1, 1]], []]
def test_no_break_regular_broadcasting(): # because we redefined Form::has_virtual_form to agree with an interpretation of its name in English assert (numpy.array([[1, 2, 3], [4, 5, 6]]) + numpy.array([[10], [20]])).tolist() == [[11, 12, 13], [24, 25, 26]] assert (awkward1.Array(numpy.array([[1, 2, 3], [4, 5, 6]])) + awkward1.Array(numpy.array([[10], [20]]))).tolist() == [[ 11, 12, 13 ], [24, 25, 26]] with pytest.raises(ValueError): awkward1.Array([[1, 2, 3], [4, 5, 6]]) + awkward1.Array([[10], [20]]) left, right = awkward1.Array(numpy.array([[1, 2, 3], [4, 5, 6]])), awkward1.Array( numpy.array([[10], [20]])) assert (awkward1.virtual( lambda: left, form=left.layout.form, length=len(left)) + awkward1.virtual(lambda: right, form=right.layout.form, length=len(right))).tolist() == [[11, 12, 13], [24, 25, 26]] with pytest.raises(ValueError): left, right = awkward1.Array([[1, 2, 3], [4, 5, 6]]), awkward1.Array([[10], [20]]) awkward1.virtual( lambda: left, form=left.layout.form, length=len(left)) + awkward1.virtual( lambda: right, form=right.layout.form, length=len(right)) assert (numpy.array([[1, 2, 3], [4, 5, 6]]) + numpy.array([10])).tolist() == [[11, 12, 13], [14, 15, 16]] assert (awkward1.Array(numpy.array([[1, 2, 3], [4, 5, 6]])) + awkward1.Array(numpy.array([10]))).tolist() == [[11, 12, 13], [14, 15, 16]] assert (awkward1.Array([[1, 2, 3], [4, 5, 6]]) + awkward1.Array([10])).tolist() == [[11, 12, 13], [14, 15, 16]] left, right = awkward1.Array(numpy.array([[1, 2, 3], [4, 5, 6]])), awkward1.Array( numpy.array([10])) assert (awkward1.virtual( lambda: left, form=left.layout.form, length=len(left)) + awkward1.virtual(lambda: right, form=right.layout.form, length=len(right))).tolist() == [[11, 12, 13], [14, 15, 16]] left, right = awkward1.Array([[1, 2, 3], [4, 5, 6]]), awkward1.Array([10]) assert (awkward1.virtual( lambda: left, form=left.layout.form, length=len(left)) + awkward1.virtual(lambda: right, form=right.layout.form, length=len(right))).tolist() == [[11, 12, 13], [14, 15, 16]] assert (awkward1.Array([[1, 2, 3], [4, 5, 6]]) + awkward1.Array([10, 20])).tolist() == [[11, 12, 13], [24, 25, 26]] left, right = awkward1.Array([[1, 2, 3], [4, 5, 6]]), awkward1.Array([10, 20]) assert (awkward1.virtual( lambda: left, form=left.layout.form, length=len(left)) + awkward1.virtual(lambda: right, form=right.layout.form, length=len(right))).tolist() == [[11, 12, 13], [24, 25, 26]]
def test_virtual_slice_numba(): numba = pytest.importorskip("numba") materialize_count = defaultdict(int) def gen(x, name): materialize_count[name] += 1 return x x1 = awkward1.Array([1, 2, 3, 4, 5]) x2 = awkward1.Array([1, 2, 3, 4, 5]) x = awkward1.zip({"x1": x1, "x2": x2}, with_name="xthing") xv = awkward1.virtual(lambda: gen(x, "x"), length=len(x), form=x.layout.form) y = x1 * 10.0 yv = awkward1.virtual(lambda: gen(y, "y"), length=len(y), form=y.layout.form) array = awkward1.zip({"x": xv, "y": yv}, with_name="Point", depth_limit=1) virtual = awkward1.virtual(lambda: gen(array, "array"), length=len(array), form=awkward1.forms.Form.fromjson( json.dumps({ "class": "RecordArray", "contents": { "x": { "class": "VirtualArray", "form": json.loads(str(x.layout.form)), "has_length": True }, "y": { "class": "VirtualArray", "form": json.loads(str(y.layout.form)), "has_length": True } }, "parameters": { "__record__": "Point" } }))) @numba.njit def dostuff(array): x = 0 for item in array: x += item return x assert dostuff(virtual.x.x1) == 15 assert dict(materialize_count) == {"x": 1, "array": 1} materialize_count.clear() @numba.njit def dostuff(array): x = 0 for item in array: x += item.x.x1 return x assert dostuff(virtual[["x"]]) == 15 assert dict(materialize_count) == {"x": 1, "array": 2} materialize_count.clear() assert dostuff(virtual[::2]) == 9 assert dict(materialize_count) == {"x": 1, "array": 2} materialize_count.clear() assert dostuff(virtual[:3]) == 6 assert dict(materialize_count) == {"x": 1, "array": 1} materialize_count.clear() slicedvirtual = virtual[awkward1.Array([True, False, False, True, False])] assert dostuff(slicedvirtual) == 5 assert dict(materialize_count) == {"x": 1, "array": 2} materialize_count.clear()