def test_highlevel():
array = awkward1.virtual(lambda: [[1.1, 2.2, 3.3], [], [4.4, 5.5]])
assert isinstance(array.layout, awkward1.layout.VirtualArray)
assert awkward1.to_list(array) == [[1.1, 2.2, 3.3], [], [4.4, 5.5]]
counter = [0]
def generate():
counter[0] += 1
return [[1.1, 2.2, 3.3], [], [4.4, 5.5]]
array = awkward1.virtual(generate,
length=3,
form={
"class": "ListOffsetArray64",
"offsets": "i64",
"content": "float64"
})
assert counter[0] == 0
assert len(array) == 3
assert counter[0] == 0
assert str(awkward1.type(array)) == "3 * var * float64"
assert counter[0] == 0
assert awkward1.to_list(array[2]) == [4.4, 5.5]
assert counter[0] == 1
def test_cache_chain():
cache1 = {}
one = awkward1.virtual(lambda: [[1.1, 2.2, 3.3], [], [4.4, 5.5]],
cache=cache1,
length=3)
two = awkward1.virtual(lambda: [100, 200, 300], cache=cache1, length=3)
array1 = awkward1.zip({"x": one, "y": two}, depth_limit=1)
assert len(cache1) == 0
cache2 = {}
array2 = awkward1.with_cache(array1, cache2)
assert awkward1.to_list(array2["x"]) == [[1.1, 2.2, 3.3], [], [4.4, 5.5]]
assert len(cache1) == 0
assert len(cache2) == 1
array3 = awkward1.with_cache(array2, cache1, chain="first")
assert awkward1.to_list(array3) == [{
"x": [1.1, 2.2, 3.3],
"y": 100
}, {
"x": [],
"y": 200
}, {
"x": [4.4, 5.5],
"y": 300
}]
assert len(cache1) == 1
assert len(cache2) == 1
def test_virtual_slice_numba():
numba = pytest.importorskip("numba")
materialize_count = defaultdict(int)
def gen(x, name):
materialize_count[name] += 1
return x
x1 = awkward1.Array([1, 2, 3, 4, 5])
x2 = awkward1.Array([1, 2, 3, 4, 5])
x = awkward1.zip({"x1": x1, "x2": x2}, with_name="xthing")
xv = awkward1.virtual(lambda: gen(x, "x"),
length=len(x),
form=x.layout.form)
y = x1 * 10.
yv = awkward1.virtual(lambda: gen(y, "y"),
length=len(y),
form=y.layout.form)
array = awkward1.zip({"x": xv, "y": yv}, with_name="Point", depth_limit=1)
virtual = awkward1.virtual(lambda: gen(array, "array"),
length=len(array),
form=array.layout.form)
@numba.njit
def dostuff(array):
x = 0
for item in array:
x += item
return x
assert dostuff(virtual.x.x1) == 15
assert dict(materialize_count) == {"x": 3, "array": 3}
materialize_count.clear()
@numba.njit
def dostuff(array):
x = 0
for item in array:
x += item.x.x1
return x
assert dostuff(virtual[["x"]]) == 15
assert dict(materialize_count) == {"x": 1, "array": 2}
materialize_count.clear()
assert dostuff(virtual[::2]) == 9
assert dict(materialize_count) == {"x": 1, "array": 2}
materialize_count.clear()
assert dostuff(virtual[:3]) == 6
assert dict(materialize_count) == {"x": 1, "array": 1}
materialize_count.clear()
slicedvirtual = virtual[awkward1.Array([True, False, False, True, False])]
assert dostuff(slicedvirtual) == 5
assert dict(materialize_count) == {"x": 1, "array": 2}
materialize_count.clear()
def test_virtual_record():
materialize_count = defaultdict(int)
def gen(x, name):
materialize_count[name] += 1
return x
x1 = awkward1.Array([1, 2, 3, 4, 5])
x2 = awkward1.Array([1, 2, 3, 4, 5])
x = awkward1.zip({"x1": x1, "x2": x2}, with_name="xthing")
assert x.layout.purelist_parameter("__record__") == "xthing"
xv = awkward1.virtual(lambda: gen(x, "x"),
length=len(x),
form=x.layout.form)
assert xv.layout.purelist_parameter("__record__") == "xthing"
y = x1 * 10.
yv = awkward1.virtual(lambda: gen(y, "y"),
length=len(y),
form=y.layout.form)
array = awkward1.zip({"x": xv, "y": yv}, with_name="Point", depth_limit=1)
assert array.layout.purelist_parameter("__record__") == "Point"
virtual = awkward1.virtual(lambda: gen(array, "array"),
length=len(array),
form=array.layout.form)
assert virtual.layout.purelist_parameter("__record__") == "Point"
assert len(materialize_count) == 0
slicedvirtual = virtual.x
assert len(materialize_count) == 0
assert slicedvirtual.layout.purelist_parameter("__record__") == "xthing"
assert len(materialize_count) == 0
slicedvirtual = virtual[["x", "y"]]
assert len(materialize_count) == 0
assert slicedvirtual.layout.purelist_parameter("__record__") == None
assert len(materialize_count) == 0
slicedvirtual = virtual[::2]
assert len(materialize_count) == 0
assert slicedvirtual.layout.purelist_parameter("__record__") == "Point"
assert len(materialize_count) == 0
slicedvirtual = virtual[:3]
assert len(materialize_count) == 0
assert slicedvirtual.layout.purelist_parameter("__record__") == "Point"
assert len(materialize_count) == 0
slicedvirtual = virtual[awkward1.Array([True, False, False, True, False])]
assert len(materialize_count) == 0
assert slicedvirtual.layout.purelist_parameter("__record__") == "Point"
assert len(materialize_count) == 0
def test_0230():
rec = awkward1.zip(
{
"x": awkward1.virtual(lambda: awkward1.Array([1, 2, 3, 4]),
length=4),
},
depth_limit=1)
assert awkward1.to_list(rec.x[1:]) == [2, 3, 4]
assert awkward1.to_list(rec.x[1:] * 2) == [4, 6, 8]
def test():
example = awkward1.Array([[{"x": 1, "y": []}, {"x": 2, "y": [1, 1]}], []])
cache = {}
virtualarray = awkward1.virtual(lambda: example,
form=example.layout.form,
length=len(example))
assert len(cache) == 0
tmp1 = virtualarray["x"]
assert json.loads(str(tmp1.layout.form)) == {
"class": "VirtualArray",
"form": {
"class": "ListOffsetArray64",
"offsets": "i64",
"content": "int64"
},
"has_length": True
}
assert len(cache) == 0
tmp2 = virtualarray["y"]
assert json.loads(str(tmp2.layout.form)) == {
"class": "VirtualArray",
"form": {
"class": "ListOffsetArray64",
"offsets": "i64",
"content": {
"class": "ListOffsetArray64",
"offsets": "i64",
"content": "int64"
}
},
"has_length": True
}
assert len(cache) == 0
assert tmp1.tolist() == [[1, 2], []]
assert tmp2.tolist() == [[[], [1, 1]], []]
def test_no_break_regular_broadcasting():
# because we redefined Form::has_virtual_form to agree with an interpretation of its name in English
assert (numpy.array([[1, 2, 3], [4, 5, 6]]) +
numpy.array([[10], [20]])).tolist() == [[11, 12, 13], [24, 25, 26]]
assert (awkward1.Array(numpy.array([[1, 2, 3], [4, 5, 6]])) +
awkward1.Array(numpy.array([[10], [20]]))).tolist() == [[
11, 12, 13
], [24, 25, 26]]
with pytest.raises(ValueError):
awkward1.Array([[1, 2, 3], [4, 5, 6]]) + awkward1.Array([[10], [20]])
left, right = awkward1.Array(numpy.array([[1, 2, 3],
[4, 5, 6]])), awkward1.Array(
numpy.array([[10], [20]]))
assert (awkward1.virtual(
lambda: left, form=left.layout.form, length=len(left)) +
awkward1.virtual(lambda: right,
form=right.layout.form,
length=len(right))).tolist() == [[11, 12, 13],
[24, 25, 26]]
with pytest.raises(ValueError):
left, right = awkward1.Array([[1, 2, 3], [4, 5,
6]]), awkward1.Array([[10],
[20]])
awkward1.virtual(
lambda: left, form=left.layout.form,
length=len(left)) + awkward1.virtual(
lambda: right, form=right.layout.form, length=len(right))
assert (numpy.array([[1, 2, 3], [4, 5, 6]]) +
numpy.array([10])).tolist() == [[11, 12, 13], [14, 15, 16]]
assert (awkward1.Array(numpy.array([[1, 2, 3], [4, 5, 6]])) +
awkward1.Array(numpy.array([10]))).tolist() == [[11, 12, 13],
[14, 15, 16]]
assert (awkward1.Array([[1, 2, 3], [4, 5, 6]]) +
awkward1.Array([10])).tolist() == [[11, 12, 13], [14, 15, 16]]
left, right = awkward1.Array(numpy.array([[1, 2, 3],
[4, 5, 6]])), awkward1.Array(
numpy.array([10]))
assert (awkward1.virtual(
lambda: left, form=left.layout.form, length=len(left)) +
awkward1.virtual(lambda: right,
form=right.layout.form,
length=len(right))).tolist() == [[11, 12, 13],
[14, 15, 16]]
left, right = awkward1.Array([[1, 2, 3], [4, 5, 6]]), awkward1.Array([10])
assert (awkward1.virtual(
lambda: left, form=left.layout.form, length=len(left)) +
awkward1.virtual(lambda: right,
form=right.layout.form,
length=len(right))).tolist() == [[11, 12, 13],
[14, 15, 16]]
assert (awkward1.Array([[1, 2, 3], [4, 5, 6]]) +
awkward1.Array([10, 20])).tolist() == [[11, 12, 13], [24, 25, 26]]
left, right = awkward1.Array([[1, 2, 3], [4, 5,
6]]), awkward1.Array([10, 20])
assert (awkward1.virtual(
lambda: left, form=left.layout.form, length=len(left)) +
awkward1.virtual(lambda: right,
form=right.layout.form,
length=len(right))).tolist() == [[11, 12, 13],
[24, 25, 26]]
def test_virtual_slice_numba():
numba = pytest.importorskip("numba")
materialize_count = defaultdict(int)
def gen(x, name):
materialize_count[name] += 1
return x
x1 = awkward1.Array([1, 2, 3, 4, 5])
x2 = awkward1.Array([1, 2, 3, 4, 5])
x = awkward1.zip({"x1": x1, "x2": x2}, with_name="xthing")
xv = awkward1.virtual(lambda: gen(x, "x"),
length=len(x),
form=x.layout.form)
y = x1 * 10.0
yv = awkward1.virtual(lambda: gen(y, "y"),
length=len(y),
form=y.layout.form)
array = awkward1.zip({"x": xv, "y": yv}, with_name="Point", depth_limit=1)
virtual = awkward1.virtual(lambda: gen(array, "array"),
length=len(array),
form=awkward1.forms.Form.fromjson(
json.dumps({
"class": "RecordArray",
"contents": {
"x": {
"class":
"VirtualArray",
"form":
json.loads(str(x.layout.form)),
"has_length":
True
},
"y": {
"class":
"VirtualArray",
"form":
json.loads(str(y.layout.form)),
"has_length":
True
}
},
"parameters": {
"__record__": "Point"
}
})))
@numba.njit
def dostuff(array):
x = 0
for item in array:
x += item
return x
assert dostuff(virtual.x.x1) == 15
assert dict(materialize_count) == {"x": 1, "array": 1}
materialize_count.clear()
@numba.njit
def dostuff(array):
x = 0
for item in array:
x += item.x.x1
return x
assert dostuff(virtual[["x"]]) == 15
assert dict(materialize_count) == {"x": 1, "array": 2}
materialize_count.clear()
assert dostuff(virtual[::2]) == 9
assert dict(materialize_count) == {"x": 1, "array": 2}
materialize_count.clear()
assert dostuff(virtual[:3]) == 6
assert dict(materialize_count) == {"x": 1, "array": 1}
materialize_count.clear()
slicedvirtual = virtual[awkward1.Array([True, False, False, True, False])]
assert dostuff(slicedvirtual) == 5
assert dict(materialize_count) == {"x": 1, "array": 2}
materialize_count.clear()
